#include "xe_pt.h"
#include <linux/error-injection.h>
#include "regs/xe_gtt_defs.h"
#include "xe_bo.h"
#include "xe_cpu_bind.h"
#include "xe_device.h"
#include "xe_drm_client.h"
#include "xe_exec_queue.h"
#include "xe_gt.h"
#include "xe_gt_stats.h"
#include "xe_page_reclaim.h"
#include "xe_pat.h"
#include "xe_pt_types.h"
#include "xe_pt_walk.h"
#include "xe_res_cursor.h"
#include "xe_sched_job.h"
#include "xe_svm.h"
#include "xe_sync.h"
#include "xe_tlb_inval_job.h"
#include "xe_trace.h"
#include "xe_ttm_stolen_mgr.h"
#include "xe_userptr.h"
#include "xe_vm.h"
struct xe_pt_dir {
struct xe_pt pt;
struct xe_ptw *children[XE_PDES];
struct xe_ptw *staging[XE_PDES];
};
#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
#define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr))
#define xe_pt_addr(__xe_pt) ((__xe_pt)->addr)
#else
#define xe_pt_set_addr(__xe_pt, __addr)
#define xe_pt_addr(__xe_pt) 0ull
#endif
static const u64 xe_normal_pt_shifts[] = {12, 21, 30, 39, 48};
static const u64 xe_compact_pt_shifts[] = {16, 21, 30, 39, 48};
#define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1)
static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt)
{
return container_of(pt, struct xe_pt_dir, pt);
}
static struct xe_pt *
xe_pt_entry_staging(struct xe_pt_dir *pt_dir, unsigned int index)
{
return container_of(pt_dir->staging[index], struct xe_pt, base);
}
static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm,
unsigned int level)
{
struct xe_device *xe = tile_to_xe(tile);
u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
u8 id = tile->id;
if (!xe_vm_has_scratch(vm))
return 0;
if (level > MAX_HUGEPTE_LEVEL)
return vm->pt_ops->pde_encode_bo(vm->scratch_pt[id][level - 1]->bo,
0);
return vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) |
XE_PTE_NULL;
}
static void xe_pt_free(struct xe_pt *pt)
{
if (pt->level)
kfree(as_xe_pt_dir(pt));
else
kfree(pt);
}
struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile,
unsigned int level, struct drm_exec *exec)
{
struct xe_pt *pt;
struct xe_bo *bo;
u32 bo_flags;
int err;
if (level) {
struct xe_pt_dir *dir = kzalloc_obj(*dir);
pt = (dir) ? &dir->pt : NULL;
} else {
pt = kzalloc_obj(*pt);
}
if (!pt)
return ERR_PTR(-ENOMEM);
bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) |
XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE |
XE_BO_FLAG_NO_RESV_EVICT | XE_BO_FLAG_PAGETABLE;
if (vm->xef)
bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE | XE_BO_FLAG_FORCE_USER_VRAM;
pt->level = level;
drm_WARN_ON(&vm->xe->drm, IS_ERR_OR_NULL(exec));
bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K,
ttm_bo_type_kernel,
bo_flags, exec);
if (IS_ERR(bo)) {
err = PTR_ERR(bo);
goto err_kfree;
}
pt->bo = bo;
pt->base.children = level ? as_xe_pt_dir(pt)->children : NULL;
pt->base.staging = level ? as_xe_pt_dir(pt)->staging : NULL;
if (vm->xef)
xe_drm_client_add_bo(vm->xef->client, pt->bo);
xe_tile_assert(tile, level <= XE_VM_MAX_LEVEL);
return pt;
err_kfree:
xe_pt_free(pt);
return ERR_PTR(err);
}
ALLOW_ERROR_INJECTION(xe_pt_create, ERRNO);
void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm,
struct xe_pt *pt)
{
struct iosys_map *map = &pt->bo->vmap;
u64 empty;
int i;
if (!xe_vm_has_scratch(vm)) {
xe_map_memset(vm->xe, map, 0, 0, SZ_4K);
} else {
empty = __xe_pt_empty_pte(tile, vm, pt->level);
for (i = 0; i < XE_PDES; i++)
xe_pt_write(vm->xe, map, i, empty);
}
}
unsigned int xe_pt_shift(unsigned int level)
{
return XE_PTE_SHIFT + XE_PDE_SHIFT * level;
}
void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred)
{
bool added = false;
int i;
if (!pt)
return;
XE_WARN_ON(!list_empty(&pt->bo->ttm.base.gpuva.list));
xe_bo_unpin(pt->bo);
xe_bo_put_deferred(pt->bo, deferred, &added);
if (added)
xe_assert(pt->bo->vm->xe, !kref_read(&pt->bo->ttm.base.refcount));
if (pt->level > 0 && pt->num_live) {
struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
for (i = 0; i < XE_PDES; i++) {
if (xe_pt_entry_staging(pt_dir, i))
xe_pt_destroy(xe_pt_entry_staging(pt_dir, i), flags,
deferred);
}
}
xe_pt_free(pt);
}
void xe_pt_clear(struct xe_device *xe, struct xe_pt *pt)
{
struct xe_bo *bo = pt->bo;
xe_bo_assert_held(bo);
if (!iosys_map_is_null(&bo->vmap))
xe_map_memset(xe, &bo->vmap, 0, 0, SZ_4K);
}
struct xe_pt_update {
struct xe_vm_pgtable_update *update;
struct xe_pt *parent;
bool preexisting;
};
struct xe_pt_stage_bind_walk {
struct xe_pt_walk base;
struct xe_vm *vm;
struct xe_tile *tile;
u64 default_vram_pte;
u64 default_system_pte;
u64 dma_offset;
bool needs_64K;
bool clear_pt;
u32 target_leaf_level;
struct xe_vma *vma;
struct xe_res_cursor *curs;
u64 va_curs_start;
struct xe_walk_update {
struct xe_vm_pgtable_update *entries;
unsigned int num_used_entries;
struct xe_pt_update updates[XE_VM_MAX_LEVEL + 1];
} wupd;
u64 l0_end_addr;
u64 addr_64K;
bool found_64K;
};
static int
xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent,
pgoff_t offset, bool alloc_entries)
{
struct xe_pt_update *upd = &wupd->updates[parent->level];
struct xe_vm_pgtable_update *entry;
if (likely(upd->parent == parent))
return 0;
upd->parent = parent;
upd->preexisting = true;
if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1)
return -EINVAL;
entry = wupd->entries + wupd->num_used_entries++;
upd->update = entry;
entry->ofs = offset;
entry->pt_bo = parent->bo;
entry->pt = parent;
entry->flags = 0;
entry->qwords = 0;
entry->level = parent->level;
if (alloc_entries) {
entry->pt_entries = kmalloc_objs(*entry->pt_entries, XE_PDES);
if (!entry->pt_entries)
return -ENOMEM;
}
return 0;
}
static int
xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent,
pgoff_t offset, struct xe_pt *xe_child, u64 pte)
{
struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level];
struct xe_pt_update *child_upd = xe_child ?
&xe_walk->wupd.updates[xe_child->level] : NULL;
int ret;
ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true);
if (unlikely(ret))
return ret;
if (unlikely(child_upd)) {
child_upd->update = NULL;
child_upd->parent = xe_child;
child_upd->preexisting = false;
}
if (likely(!upd->preexisting)) {
struct iosys_map *map = &parent->bo->vmap;
if (unlikely(xe_child)) {
parent->base.children[offset] = &xe_child->base;
parent->base.staging[offset] = &xe_child->base;
}
xe_pt_write(xe_walk->vm->xe, map, offset, pte);
parent->num_live++;
} else {
unsigned int idx;
struct xe_vm_pgtable_update *entry = upd->update;
idx = offset - entry->ofs;
entry->pt_entries[idx].pt = xe_child;
entry->pt_entries[idx].pte = pte;
entry->qwords++;
}
return 0;
}
static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level,
struct xe_pt_stage_bind_walk *xe_walk)
{
struct xe_bo *bo = xe_vma_bo(xe_walk->vma);
u64 size, dma;
if (level > MAX_HUGEPTE_LEVEL)
return false;
if (!xe_pt_covers(addr, next, level, &xe_walk->base))
return false;
if (xe_vma_is_null(xe_walk->vma) || (bo && xe_bo_is_purged(bo)))
return true;
if (xe_walk->clear_pt)
return true;
if (next - xe_walk->va_curs_start > xe_walk->curs->size)
return false;
size = next - addr;
dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs);
return IS_ALIGNED(dma, size);
}
static bool
xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk)
{
struct xe_bo *bo = xe_vma_bo(xe_walk->vma);
struct xe_res_cursor curs = *xe_walk->curs;
if (!IS_ALIGNED(addr, SZ_64K))
return false;
if (next > xe_walk->l0_end_addr)
return false;
if (xe_vma_is_null(xe_walk->vma) || (bo && xe_bo_is_purged(bo)))
return true;
xe_res_next(&curs, addr - xe_walk->va_curs_start);
for (; addr < next; addr += SZ_64K) {
if (!IS_ALIGNED(xe_res_dma(&curs), SZ_64K) || curs.size < SZ_64K)
return false;
xe_res_next(&curs, SZ_64K);
}
return addr == next;
}
static bool
xe_pt_is_pte_ps64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk)
{
if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K)
return true;
xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk);
xe_walk->addr_64K = addr;
return xe_walk->found_64K;
}
static bool xe_pt_huge_leaf_allowed(u64 addr, u64 next, unsigned int level,
struct xe_pt_stage_bind_walk *xe_walk)
{
if (xe_walk->clear_pt)
return xe_pt_hugepte_possible(addr, next, level, xe_walk);
if (!xe_debug_page_size_supported(xe_walk->vm->xe))
return xe_pt_hugepte_possible(addr, next, level, xe_walk);
if (!xe_walk->target_leaf_level)
return xe_pt_hugepte_possible(addr, next, level, xe_walk);
if (level == xe_walk->target_leaf_level)
return xe_pt_hugepte_possible(addr, next, level, xe_walk);
return false;
}
static bool xe_pt_exact_leaf_required_but_invalid(u64 addr, u64 next,
unsigned int level,
struct xe_pt_stage_bind_walk *xe_walk)
{
struct xe_device *xe = xe_walk->vm->xe;
if (!xe_debug_page_size_mode_not_none(xe))
return false;
return !xe_walk->clear_pt &&
xe_walk->target_leaf_level &&
level == xe_walk->target_leaf_level &&
!xe_pt_hugepte_possible(addr, next, level, xe_walk);
}
static int
xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset,
unsigned int level, u64 addr, u64 next,
struct xe_ptw **child,
enum page_walk_action *action,
struct xe_pt_walk *walk)
{
struct xe_pt_stage_bind_walk *xe_walk =
container_of(walk, typeof(*xe_walk), base);
u16 pat_index = xe_walk->vma->attr.pat_index;
struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base);
struct xe_vm *vm = xe_walk->vm;
struct xe_pt *xe_child;
bool covers;
int ret = 0;
u64 pte;
if (xe_pt_exact_leaf_required_but_invalid(addr, next, level, xe_walk))
return -EINVAL;
if (level == 0 || xe_pt_huge_leaf_allowed(addr, next, level, xe_walk)) {
struct xe_res_cursor *curs = xe_walk->curs;
struct xe_bo *bo = xe_vma_bo(xe_walk->vma);
bool is_null_or_purged = xe_vma_is_null(xe_walk->vma) ||
(bo && xe_bo_is_purged(bo));
bool is_vram = is_null_or_purged ? false : xe_res_is_vram(curs);
XE_WARN_ON(xe_walk->va_curs_start != addr);
if (xe_walk->clear_pt) {
pte = 0;
} else {
pte = vm->pt_ops->pte_encode_vma(is_null_or_purged ? 0 :
xe_res_dma(curs) +
xe_walk->dma_offset,
xe_walk->vma,
pat_index, level);
if (!is_null_or_purged)
pte |= is_vram ? xe_walk->default_vram_pte :
xe_walk->default_system_pte;
if (level == 0 && !xe_parent->is_compact) {
if (xe_pt_is_pte_ps64K(addr, next, xe_walk)) {
xe_walk->vma->gpuva.flags |=
XE_VMA_PTE_64K;
pte |= XE_PTE_PS64;
} else if (XE_WARN_ON(xe_walk->needs_64K &&
is_vram)) {
return -EINVAL;
}
}
}
ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, NULL, pte);
if (unlikely(ret))
return ret;
if (!is_null_or_purged && !xe_walk->clear_pt)
xe_res_next(curs, next - addr);
xe_walk->va_curs_start = next;
xe_walk->vma->gpuva.flags |= (XE_VMA_PTE_4K << level);
*action = ACTION_CONTINUE;
return ret;
}
if (level == 1) {
walk->shifts = xe_normal_pt_shifts;
xe_walk->l0_end_addr = next;
}
covers = xe_pt_covers(addr, next, level, &xe_walk->base);
if (covers || !*child) {
u64 flags = 0;
xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1,
xe_vm_validation_exec(vm));
if (IS_ERR(xe_child))
return PTR_ERR(xe_child);
xe_pt_set_addr(xe_child,
round_down(addr, 1ull << walk->shifts[level]));
if (!covers)
xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child);
*child = &xe_child->base;
if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 &&
covers && xe_pt_scan_64K(addr, next, xe_walk)) {
walk->shifts = xe_compact_pt_shifts;
xe_walk->vma->gpuva.flags |= XE_VMA_PTE_COMPACT;
flags |= XE_PDE_64K;
xe_child->is_compact = true;
}
pte = vm->pt_ops->pde_encode_bo(xe_child->bo, 0) | flags;
ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, xe_child,
pte);
}
*action = ACTION_SUBTREE;
return ret;
}
static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = {
.pt_entry = xe_pt_stage_bind_entry,
};
static bool xe_atomic_for_vram(struct xe_vm *vm, struct xe_vma *vma)
{
if (vma->attr.atomic_access == DRM_XE_ATOMIC_CPU)
return false;
return true;
}
static bool xe_atomic_for_system(struct xe_vm *vm, struct xe_vma *vma)
{
struct xe_device *xe = vm->xe;
struct xe_bo *bo = xe_vma_bo(vma);
if (!xe->info.has_device_atomics_on_smem ||
vma->attr.atomic_access == DRM_XE_ATOMIC_CPU)
return false;
if (vma->attr.atomic_access == DRM_XE_ATOMIC_DEVICE)
return true;
return (!IS_DGFX(xe) || (!xe_vm_in_lr_mode(vm) ||
(bo && xe_bo_has_single_placement(bo))));
}
static u32 xe_pt_target_leaf_level_from_bo(struct xe_device *xe,
struct xe_vma *vma)
{
struct xe_bo *bo = xe_vma_bo(vma);
if (!xe_debug_page_size_mode_not_none(xe))
return 0;
if (!bo || !xe_bo_is_vram(bo) || !(bo->flags & XE_BO_FLAG_USER))
return 0;
if (bo->flags & XE_BO_FLAG_NEEDS_1G)
return 2;
if (bo->flags & XE_BO_FLAG_NEEDS_2M)
return 1;
return 0;
}
static int
xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma,
struct xe_svm_range *range,
struct xe_vm_pgtable_update *entries,
u32 *num_entries, bool clear_pt)
{
struct xe_device *xe = tile_to_xe(tile);
struct xe_bo *bo = xe_vma_bo(vma);
struct xe_res_cursor curs = {};
struct xe_vm *vm = xe_vma_vm(vma);
struct xe_pt_stage_bind_walk xe_walk = {
.base = {
.ops = &xe_pt_stage_bind_ops,
.shifts = xe_normal_pt_shifts,
.max_level = XE_PT_HIGHEST_LEVEL,
.staging = true,
},
.vm = vm,
.tile = tile,
.curs = &curs,
.va_curs_start = range ? xe_svm_range_start(range) :
xe_vma_start(vma),
.vma = vma,
.wupd.entries = entries,
.clear_pt = clear_pt,
};
struct xe_pt *pt = xe_vm_pt_root(vm, tile->id);
int ret;
bool is_purged = false;
if (bo && xe_bo_is_purged(bo)) {
is_purged = true;
if (!xe_vm_has_scratch(vm))
xe_walk.clear_pt = true;
}
if (range) {
xe_svm_notifier_lock(vm);
if (!xe_svm_range_pages_valid(range)) {
xe_svm_range_debug(range, "BIND PREPARE - RETRY");
xe_svm_notifier_unlock(vm);
return -EAGAIN;
}
if (xe_svm_range_has_dma_mapping(range)) {
const struct drm_pagemap_addr *addr;
bool contiguous;
addr = xe_svm_range_first_dma(range, &contiguous);
xe_res_first_dma(addr, 0, xe_svm_range_size(range),
contiguous, &curs);
xe_svm_range_debug(range, "BIND PREPARE - MIXED");
} else {
xe_assert(xe, false);
}
xe_svm_notifier_unlock(vm);
}
xe_walk.target_leaf_level = xe_pt_target_leaf_level_from_bo(xe, vma);
xe_walk.needs_64K = (vm->flags & XE_VM_FLAG_64K);
if (clear_pt) {
xe_assert(xe, !range);
curs.size = xe_vma_size(vma);
goto walk_pt;
}
if (vma->gpuva.flags & XE_VMA_ATOMIC_PTE_BIT) {
xe_walk.default_vram_pte = xe_atomic_for_vram(vm, vma) ? XE_USM_PPGTT_PTE_AE : 0;
xe_walk.default_system_pte = xe_atomic_for_system(vm, vma) ?
XE_USM_PPGTT_PTE_AE : 0;
}
xe_walk.default_vram_pte |= XE_PPGTT_PTE_DM;
xe_walk.dma_offset = (bo && !is_purged) ? vram_region_gpu_offset(bo->ttm.resource) : 0;
if (!range)
xe_bo_assert_held(bo);
if (!xe_vma_is_null(vma) && !range && !is_purged) {
if (xe_vma_is_userptr(vma)) {
const struct drm_pagemap_addr *addr;
bool contiguous;
addr = drm_gpusvm_pages_first_dma(&to_userptr_vma(vma)->userptr.pages,
&contiguous);
xe_res_first_dma(addr, 0, xe_vma_size(vma), contiguous,
&curs);
} else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo))
xe_res_first(bo->ttm.resource, xe_vma_bo_offset(vma),
xe_vma_size(vma), &curs);
else
xe_res_first_sg(xe_bo_sg(bo), xe_vma_bo_offset(vma),
xe_vma_size(vma), &curs);
} else if (!range) {
curs.size = xe_vma_size(vma);
}
walk_pt:
ret = xe_pt_walk_range(&pt->base, pt->level,
range ? xe_svm_range_start(range) : xe_vma_start(vma),
range ? xe_svm_range_end(range) : xe_vma_end(vma),
&xe_walk.base);
*num_entries = xe_walk.wupd.num_used_entries;
return ret;
}
static bool xe_pt_nonshared_offsets(u64 addr, u64 end, unsigned int level,
struct xe_pt_walk *walk,
enum page_walk_action *action,
pgoff_t *offset, pgoff_t *end_offset)
{
u64 size = 1ull << walk->shifts[level];
*offset = xe_pt_offset(addr, level, walk);
*end_offset = xe_pt_num_entries(addr, end, level, walk) + *offset;
if (!level)
return true;
*action = ACTION_CONTINUE;
if (!IS_ALIGNED(addr, size)) {
*action = ACTION_SUBTREE;
(*offset)++;
}
if (!IS_ALIGNED(end, size)) {
*action = ACTION_SUBTREE;
(*end_offset)--;
}
return *end_offset > *offset;
}
struct xe_pt_zap_ptes_walk {
struct xe_pt_walk base;
struct xe_tile *tile;
bool needs_invalidate;
};
static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset,
unsigned int level, u64 addr, u64 next,
struct xe_ptw **child,
enum page_walk_action *action,
struct xe_pt_walk *walk)
{
struct xe_pt_zap_ptes_walk *xe_walk =
container_of(walk, typeof(*xe_walk), base);
struct xe_pt *xe_child;
pgoff_t end_offset;
XE_WARN_ON(!level);
if (XE_WARN_ON(!*child))
return 0;
xe_child = container_of(*child, typeof(*xe_child), base);
if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset,
&end_offset)) {
xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap,
offset * sizeof(u64), 0,
(end_offset - offset) * sizeof(u64));
xe_walk->needs_invalidate = true;
}
return 0;
}
static bool pt_mirroring_disabled_for_tile(struct xe_vm *vm, u8 tile_id)
{
return !vm->xe->info.has_pt_mirror && tile_id;
}
static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = {
.pt_entry = xe_pt_zap_ptes_entry,
};
bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma)
{
struct xe_pt_zap_ptes_walk xe_walk = {
.base = {
.ops = &xe_pt_zap_ptes_ops,
.shifts = xe_normal_pt_shifts,
.max_level = XE_PT_HIGHEST_LEVEL,
},
.tile = tile,
};
struct xe_pt *pt = xe_vma_vm(vma)->pt_root[tile->id];
u8 pt_mask = (vma->tile_present & ~vma->tile_invalidated);
if (xe_vma_bo(vma))
xe_bo_assert_held(xe_vma_bo(vma));
else if (xe_vma_is_userptr(vma))
lockdep_assert_held(&xe_vma_vm(vma)->svm.gpusvm.notifier_lock);
if (!(pt_mask & BIT(tile->id)))
return false;
if (pt_mirroring_disabled_for_tile(xe_vma_vm(vma), tile->id))
return true;
(void)xe_pt_walk_shared(&pt->base, pt->level, xe_vma_start(vma),
xe_vma_end(vma), &xe_walk.base);
return xe_walk.needs_invalidate;
}
bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm,
struct xe_svm_range *range)
{
struct xe_pt_zap_ptes_walk xe_walk = {
.base = {
.ops = &xe_pt_zap_ptes_ops,
.shifts = xe_normal_pt_shifts,
.max_level = XE_PT_HIGHEST_LEVEL,
},
.tile = tile,
};
struct xe_pt *pt = vm->pt_root[tile->id];
u8 pt_mask = (range->tile_present & ~range->tile_invalidated);
lockdep_assert(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 0) ||
(lockdep_is_held_type(&vm->svm.gpusvm.notifier_lock, 1) &&
lockdep_is_held_type(&vm->lock, 0)));
if (!(pt_mask & BIT(tile->id)))
return false;
if (pt_mirroring_disabled_for_tile(vm, tile->id))
return true;
(void)xe_pt_walk_shared(&pt->base, pt->level, xe_svm_range_start(range),
xe_svm_range_end(range), &xe_walk.base);
return xe_walk.needs_invalidate;
}
static void
xe_vm_populate_pgtable(struct xe_tile *tile, struct iosys_map *map,
const struct xe_vm_pgtable_update *update)
{
struct xe_pt_entry *ptes = update->pt_entries;
u32 i;
xe_assert(tile_to_xe(tile), map);
xe_assert(tile_to_xe(tile), !iosys_map_is_null(map));
for (i = 0; i < update->qwords; i++)
xe_map_wr(tile_to_xe(tile), map, (update->ofs + i) *
sizeof(u64), u64, ptes[i].pte);
}
static void xe_pt_cancel_bind(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries)
{
u32 i, j;
for (i = 0; i < num_entries; i++) {
struct xe_pt *pt = entries[i].pt;
if (!pt)
continue;
if (pt->level) {
for (j = 0; j < entries[i].qwords; j++)
xe_pt_destroy(entries[i].pt_entries[j].pt,
xe_vma_vm(vma)->flags, NULL);
}
kfree(entries[i].pt_entries);
entries[i].pt_entries = NULL;
entries[i].qwords = 0;
}
}
#define XE_INVALID_VMA ((struct xe_vma *)(0xdeaddeadull))
static void xe_pt_commit_prepare_locks_assert(struct xe_vma *vma)
{
struct xe_vm *vm;
if (vma == XE_INVALID_VMA)
return;
vm = xe_vma_vm(vma);
lockdep_assert_held(&vm->lock);
if (!xe_vma_has_no_bo(vma))
dma_resv_assert_held(xe_vma_bo(vma)->ttm.base.resv);
xe_vm_assert_held(vm);
}
static void xe_pt_commit_locks_assert(struct xe_vma *vma)
{
struct xe_vm *vm;
if (vma == XE_INVALID_VMA)
return;
vm = xe_vma_vm(vma);
xe_pt_commit_prepare_locks_assert(vma);
if (xe_vma_is_userptr(vma))
xe_svm_assert_held_read_or_inject_write(vm);
}
static void xe_pt_commit(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries, struct llist_head *deferred)
{
u32 i, j;
xe_pt_commit_locks_assert(vma);
for (i = 0; i < num_entries; i++) {
struct xe_pt *pt = entries[i].pt;
struct xe_pt_dir *pt_dir;
if (!pt->level)
continue;
pt_dir = as_xe_pt_dir(pt);
for (j = 0; j < entries[i].qwords; j++) {
struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
int j_ = j + entries[i].ofs;
pt_dir->children[j_] = pt_dir->staging[j_];
xe_pt_destroy(oldpte, (vma == XE_INVALID_VMA) ? 0 :
xe_vma_vm(vma)->flags, deferred);
}
}
}
static void xe_pt_abort_bind(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries, bool rebind)
{
int i, j;
xe_pt_commit_prepare_locks_assert(vma);
for (i = num_entries - 1; i >= 0; --i) {
struct xe_pt *pt = entries[i].pt;
struct xe_pt_dir *pt_dir;
if (!rebind)
pt->num_live -= entries[i].qwords;
if (!pt->level)
continue;
pt_dir = as_xe_pt_dir(pt);
for (j = 0; j < entries[i].qwords; j++) {
u32 j_ = j + entries[i].ofs;
struct xe_pt *newpte = xe_pt_entry_staging(pt_dir, j_);
struct xe_pt *oldpte = entries[i].pt_entries[j].pt;
pt_dir->staging[j_] = oldpte ? &oldpte->base : 0;
xe_pt_destroy(newpte, xe_vma_vm(vma)->flags, NULL);
}
}
}
static void xe_pt_commit_prepare_bind(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries, bool rebind)
{
u32 i, j;
xe_pt_commit_prepare_locks_assert(vma);
for (i = 0; i < num_entries; i++) {
struct xe_pt *pt = entries[i].pt;
struct xe_pt_dir *pt_dir;
if (!rebind)
pt->num_live += entries[i].qwords;
if (!pt->level)
continue;
pt_dir = as_xe_pt_dir(pt);
for (j = 0; j < entries[i].qwords; j++) {
u32 j_ = j + entries[i].ofs;
struct xe_pt *newpte = entries[i].pt_entries[j].pt;
struct xe_pt *oldpte = NULL;
if (xe_pt_entry_staging(pt_dir, j_))
oldpte = xe_pt_entry_staging(pt_dir, j_);
pt_dir->staging[j_] = &newpte->base;
entries[i].pt_entries[j].pt = oldpte;
}
}
}
static void xe_pt_free_bind(struct xe_vm_pgtable_update *entries,
u32 num_entries)
{
u32 i;
for (i = 0; i < num_entries; i++)
kfree(entries[i].pt_entries);
}
static int
xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma,
struct xe_svm_range *range,
struct xe_vm_pgtable_update *entries,
u32 *num_entries, bool invalidate_on_bind)
{
int err;
*num_entries = 0;
err = xe_pt_stage_bind(tile, vma, range, entries, num_entries,
invalidate_on_bind);
if (!err)
xe_tile_assert(tile, *num_entries);
return err;
}
static void xe_vm_dbg_print_entries(struct xe_device *xe,
const struct xe_vm_pgtable_update *entries,
unsigned int num_entries, bool bind)
#if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM))
{
unsigned int i;
vm_dbg(&xe->drm, "%s: %u entries to update\n", bind ? "bind" : "unbind",
num_entries);
for (i = 0; i < num_entries; i++) {
const struct xe_vm_pgtable_update *entry = &entries[i];
struct xe_pt *xe_pt = entry->pt;
u64 page_size = 1ull << xe_pt_shift(xe_pt->level);
u64 end;
u64 start;
xe_assert(xe, !entry->pt->is_compact);
start = entry->ofs * page_size;
end = start + page_size * entry->qwords;
vm_dbg(&xe->drm,
"\t%u: Update level %u at (%u + %u) [%llx...%llx) f:%x\n",
i, xe_pt->level, entry->ofs, entry->qwords,
xe_pt_addr(xe_pt) + start, xe_pt_addr(xe_pt) + end, 0);
}
}
#else
{}
#endif
static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs)
{
int i;
for (i = 0; i < num_syncs; i++) {
struct dma_fence *fence = syncs[i].fence;
if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
&fence->flags))
return false;
}
return true;
}
static int job_test_add_deps(struct xe_sched_job *job,
struct dma_resv *resv,
enum dma_resv_usage usage)
{
if (!job) {
if (!dma_resv_test_signaled(resv, usage))
return -ETIME;
return 0;
}
return xe_sched_job_add_deps(job, resv, usage);
}
static int vma_add_deps(struct xe_vma *vma, struct xe_sched_job *job)
{
struct xe_bo *bo = xe_vma_bo(vma);
xe_bo_assert_held(bo);
if (bo && !bo->vm)
return job_test_add_deps(job, bo->ttm.base.resv,
DMA_RESV_USAGE_KERNEL);
return 0;
}
static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op,
struct xe_sched_job *job)
{
int err = 0;
switch (op->base.op) {
case DRM_GPUVA_OP_MAP:
if (!op->map.immediate && xe_vm_in_fault_mode(vm))
break;
err = vma_add_deps(op->map.vma, job);
break;
case DRM_GPUVA_OP_REMAP:
if (op->remap.prev)
err = vma_add_deps(op->remap.prev, job);
if (!err && op->remap.next)
err = vma_add_deps(op->remap.next, job);
break;
case DRM_GPUVA_OP_UNMAP:
break;
case DRM_GPUVA_OP_PREFETCH:
err = vma_add_deps(gpuva_to_vma(op->base.prefetch.va), job);
break;
case DRM_GPUVA_OP_DRIVER:
break;
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
return err;
}
static int xe_pt_vm_dependencies(struct xe_sched_job *job,
struct xe_tlb_inval_job **ijobs,
struct xe_vm *vm,
struct xe_vma_ops *vops,
struct xe_range_fence_tree *rftree)
{
struct xe_range_fence *rtfence;
struct dma_fence *fence;
struct xe_vma_op *op;
int err = 0, i;
xe_vm_assert_held(vm);
if (!job && !no_in_syncs(vops->syncs, vops->num_syncs))
return -ETIME;
if (!job && !xe_exec_queue_is_idle(vops->q, 0))
return -ETIME;
if (vops->flags & (XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP |
XE_VMA_OPS_FLAG_WAIT_VM_KERNEL)) {
enum dma_resv_usage usage = DMA_RESV_USAGE_KERNEL;
if (vops->flags & XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP)
usage = DMA_RESV_USAGE_BOOKKEEP;
err = job_test_add_deps(job, xe_vm_resv(vm), usage);
if (err)
return err;
}
rtfence = xe_range_fence_tree_first(rftree, vops->start, vops->last);
while (rtfence) {
fence = rtfence->fence;
if (!dma_fence_is_signaled(fence)) {
if (!job)
return -ETIME;
dma_fence_get(fence);
err = drm_sched_job_add_dependency(&job->drm, fence);
if (err)
return err;
}
rtfence = xe_range_fence_tree_next(rtfence, vops->start,
vops->last);
}
list_for_each_entry(op, &vops->list, link) {
err = op_add_deps(vm, op, job);
if (err)
return err;
}
for (i = 0; job && !err && i < vops->num_syncs; i++)
err = xe_sync_entry_add_deps(&vops->syncs[i], job);
if (job) {
for (i = 0; i < XE_CPU_BIND_INVAL_JOB_COUNT; ++i) {
if (!ijobs[i])
continue;
err = xe_tlb_inval_job_alloc_dep(ijobs[i]);
if (err)
return err;
}
}
return err;
}
static int xe_pt_pre_commit(struct xe_cpu_bind_pt_update *pt_update)
{
struct xe_vma_ops *vops = pt_update->vops;
struct xe_vm *vm = vops->vm;
struct xe_range_fence_tree *rftree = &vm->rftree;
return xe_pt_vm_dependencies(pt_update->job, pt_update->ijobs,
vm, vops, rftree);
}
#if IS_ENABLED(CONFIG_DRM_GPUSVM)
static void xe_pt_svm_userptr_notifier_lock(struct xe_vm *vm)
{
#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
down_write(&vm->svm.gpusvm.notifier_lock);
#else
xe_svm_notifier_lock(vm);
#endif
}
static void xe_pt_svm_userptr_notifier_unlock(struct xe_vm *vm)
{
#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
up_write(&vm->svm.gpusvm.notifier_lock);
#else
xe_svm_notifier_unlock(vm);
#endif
}
#else
static inline void xe_pt_svm_userptr_notifier_lock(struct xe_vm *vm) { }
static inline void xe_pt_svm_userptr_notifier_unlock(struct xe_vm *vm) { }
#endif
#if IS_ENABLED(CONFIG_DRM_GPUSVM)
#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT
static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
{
u32 divisor = uvma->userptr.divisor ? uvma->userptr.divisor : 2;
static u32 count;
if (count++ % divisor == divisor - 1) {
uvma->userptr.divisor = divisor << 1;
return true;
}
return false;
}
#else
static bool xe_pt_userptr_inject_eagain(struct xe_userptr_vma *uvma)
{
return false;
}
#endif
static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma)
{
struct xe_userptr_vma *uvma;
unsigned long notifier_seq;
xe_svm_assert_held_read_or_inject_write(vm);
if (!xe_vma_is_userptr(vma))
return 0;
uvma = to_userptr_vma(vma);
if (xe_pt_userptr_inject_eagain(uvma))
xe_vma_userptr_force_invalidate(uvma);
notifier_seq = uvma->userptr.pages.notifier_seq;
if (!mmu_interval_read_retry(&uvma->userptr.notifier,
notifier_seq))
return 0;
if (xe_vm_in_fault_mode(vm))
return -EAGAIN;
return 0;
}
static int op_check_svm_userptr(struct xe_vm *vm, struct xe_vma_op *op)
{
int err = 0;
xe_svm_assert_held_read_or_inject_write(vm);
switch (op->base.op) {
case DRM_GPUVA_OP_MAP:
if (!op->map.immediate && xe_vm_in_fault_mode(vm))
break;
err = vma_check_userptr(vm, op->map.vma);
break;
case DRM_GPUVA_OP_REMAP:
if (op->remap.prev && !op->remap.skip_prev)
err = vma_check_userptr(vm, op->remap.prev);
if (!err && op->remap.next && !op->remap.skip_next)
err = vma_check_userptr(vm, op->remap.next);
break;
case DRM_GPUVA_OP_UNMAP:
break;
case DRM_GPUVA_OP_PREFETCH:
if (xe_vma_is_cpu_addr_mirror(gpuva_to_vma(op->base.prefetch.va))) {
struct xe_svm_range *range = op->map_range.range;
unsigned long i;
xe_assert(vm->xe,
xe_vma_is_cpu_addr_mirror(gpuva_to_vma(op->base.prefetch.va)));
xa_for_each(&op->prefetch_range.range, i, range) {
xe_svm_range_debug(range, "PRE-COMMIT");
if (!xe_svm_range_pages_valid(range)) {
xe_svm_range_debug(range, "PRE-COMMIT - RETRY");
return -ENODATA;
}
}
} else {
err = vma_check_userptr(vm, gpuva_to_vma(op->base.prefetch.va));
}
break;
#if IS_ENABLED(CONFIG_DRM_XE_GPUSVM)
case DRM_GPUVA_OP_DRIVER:
if (op->subop == XE_VMA_SUBOP_MAP_RANGE) {
struct xe_svm_range *range = op->map_range.range;
xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(op->map_range.vma));
xe_svm_range_debug(range, "PRE-COMMIT");
if (!xe_svm_range_pages_valid(range)) {
xe_svm_range_debug(range, "PRE-COMMIT - RETRY");
return -EAGAIN;
}
}
break;
#endif
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
return err;
}
static int xe_pt_svm_userptr_pre_commit(struct xe_cpu_bind_pt_update *pt_update)
{
struct xe_vm *vm = pt_update->vops->vm;
struct xe_vma_ops *vops = pt_update->vops;
struct xe_vma_op *op;
int err;
err = xe_pt_pre_commit(pt_update);
if (err)
return err;
xe_pt_svm_userptr_notifier_lock(vm);
list_for_each_entry(op, &vops->list, link) {
err = op_check_svm_userptr(vm, op);
if (err) {
xe_pt_svm_userptr_notifier_unlock(vm);
break;
}
}
return err;
}
#endif
struct xe_pt_stage_unbind_walk {
struct xe_pt_walk base;
struct xe_tile *tile;
u64 modified_start;
u64 modified_end;
struct xe_page_reclaim_list *prl;
struct xe_walk_update wupd;
};
static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level,
const struct xe_pt *child,
enum page_walk_action *action,
struct xe_pt_walk *walk)
{
struct xe_pt_stage_unbind_walk *xe_walk =
container_of(walk, typeof(*xe_walk), base);
unsigned int shift = walk->shifts[level];
u64 size = 1ull << shift;
if (IS_ALIGNED(addr, size) && IS_ALIGNED(next, size) &&
((next - addr) >> shift) == child->num_live) {
u64 size = 1ull << walk->shifts[level + 1];
*action = ACTION_CONTINUE;
if (xe_walk->modified_start >= addr)
xe_walk->modified_start = round_down(addr, size);
if (xe_walk->modified_end <= next)
xe_walk->modified_end = round_up(next, size);
return true;
}
return false;
}
static int generate_reclaim_entry(struct xe_tile *tile,
struct xe_page_reclaim_list *prl,
u64 pte, struct xe_pt *xe_child)
{
struct xe_gt *gt = tile->primary_gt;
struct xe_guc_page_reclaim_entry *reclaim_entries = prl->entries;
bool is_2m = xe_child->level == 1 && (pte & XE_PDE_PS_2M);
bool is_64k = xe_child->level == 0 && ((pte & XE_PTE_PS64) || xe_child->is_compact);
u32 page_shift = is_2m ? ilog2(SZ_2M) : is_64k ? ilog2(SZ_64K) : ilog2(SZ_4K);
u64 phys_addr = pte & XE_PAGE_ADDR_MASK(page_shift);
u64 phys_page = phys_addr >> XE_PTE_SHIFT;
int num_entries = prl->num_entries;
u32 reclamation_size = page_shift - XE_PTE_SHIFT;
xe_tile_assert(tile, xe_child->level <= MAX_HUGEPTE_LEVEL);
xe_tile_assert(tile, reclaim_entries);
xe_tile_assert(tile, num_entries < XE_PAGE_RECLAIM_MAX_ENTRIES - 1);
if (!xe_page_reclaim_list_valid(prl))
return -EINVAL;
if (is_2m) {
xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_2M_ENTRY_COUNT, 1);
} else if (is_64k) {
xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_64K_ENTRY_COUNT, 1);
} else if (xe_child->level == 0) {
xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_4K_ENTRY_COUNT, 1);
} else {
xe_page_reclaim_list_abort(tile->primary_gt, prl,
"unsupported PTE level=%u pte=%#llx",
xe_child->level, pte);
return -EINVAL;
}
reclaim_entries[num_entries].qw =
FIELD_PREP(XE_PAGE_RECLAIM_VALID, 1) |
FIELD_PREP(XE_PAGE_RECLAIM_SIZE, reclamation_size) |
FIELD_PREP(XE_PAGE_RECLAIM_ADDR_LO, phys_page) |
FIELD_PREP(XE_PAGE_RECLAIM_ADDR_HI, phys_page >> 20);
prl->num_entries++;
vm_dbg(&tile_to_xe(tile)->drm,
"PRL add entry: level=%u pte=%#llx reclamation_size=%u prl_idx=%d\n",
xe_child->level, pte, reclamation_size, num_entries);
return 0;
}
static int add_pte_to_prl(struct xe_tile *tile, struct xe_page_reclaim_list *prl,
struct xe_pt *xe_child, u64 pte, u64 addr)
{
if (!pte) {
xe_page_reclaim_list_abort(tile->primary_gt, prl,
"found zero pte at addr=%#llx", addr);
return -EINVAL;
}
xe_tile_assert(tile, xe_child->level == 0 ||
(pte & (XE_PDE_PS_2M | XE_PDPE_PS_1G)));
if (prl->num_entries >= XE_PAGE_RECLAIM_MAX_ENTRIES - 1) {
xe_page_reclaim_list_abort(tile->primary_gt, prl,
"overflow while adding pte=%#llx", pte);
return -ENOSPC;
}
return generate_reclaim_entry(tile, prl, pte, xe_child);
}
static bool add_compact_pt_prl(struct xe_tile *tile, struct xe_page_reclaim_list *prl,
struct xe_device *xe, struct xe_pt *compact_pt, u64 addr)
{
struct iosys_map *map = &compact_pt->bo->vmap;
for (pgoff_t i = 0; i < SZ_2M / SZ_64K && xe_page_reclaim_list_valid(prl); i++) {
u64 pte = xe_map_rd(xe, map, i * sizeof(u64), u64);
if (add_pte_to_prl(tile, prl, compact_pt, pte, addr + i * SZ_64K))
break;
}
return xe_page_reclaim_list_valid(prl);
}
static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset,
unsigned int level, u64 addr, u64 next,
struct xe_ptw **child,
enum page_walk_action *action,
struct xe_pt_walk *walk)
{
struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
struct xe_pt_stage_unbind_walk *xe_walk =
container_of(walk, typeof(*xe_walk), base);
struct xe_page_reclaim_list *prl = xe_walk->prl;
struct xe_tile *tile = xe_walk->tile;
struct xe_device *xe = tile_to_xe(tile);
pgoff_t first = xe_pt_offset(addr, xe_child->level, walk);
bool killed;
XE_WARN_ON(!*child);
XE_WARN_ON(!level);
if (prl && xe_page_reclaim_list_valid(prl) &&
xe_child->level <= MAX_HUGEPTE_LEVEL) {
struct iosys_map *leaf_map = &xe_child->bo->vmap;
pgoff_t count = xe_pt_num_entries(addr, next, xe_child->level, walk);
for (pgoff_t i = 0; i < count; i++) {
u64 pte;
if (xe_child->base.children && xe_child->base.children[first + i]) {
u64 pt_size = 1ULL << walk->shifts[xe_child->level];
bool edge_pt = (i == 0 && !IS_ALIGNED(addr, pt_size)) ||
(i == count - 1 && !IS_ALIGNED(next, pt_size));
struct xe_pt *child_pt =
container_of(xe_child->base.children[first + i],
struct xe_pt, base);
XE_WARN_ON(child_pt->is_compact && edge_pt);
if (edge_pt)
continue;
if (child_pt->is_compact) {
if (!add_compact_pt_prl(tile, prl, xe, child_pt,
addr + (u64)i * pt_size))
break;
} else {
xe_page_reclaim_list_abort(tile->primary_gt,
prl,
"PT is skipped by walk at level=%u offset=%lu",
xe_child->level, first + i);
break;
}
continue;
}
pte = xe_map_rd(xe, leaf_map, (first + i) * sizeof(u64), u64);
if (add_pte_to_prl(tile, prl, xe_child, pte, addr))
break;
if (pte & XE_PTE_PS64)
i += 15;
}
}
killed = xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk);
if (prl && xe_page_reclaim_list_valid(prl) &&
xe_child->level > MAX_HUGEPTE_LEVEL && xe_child->num_live) {
bool covered = xe_pt_covers(addr, next, xe_child->level, &xe_walk->base);
if (killed || covered)
xe_page_reclaim_list_abort(tile->primary_gt, prl,
"kill at level=%u addr=%#llx next=%#llx num_live=%u",
level, addr, next, xe_child->num_live);
}
return 0;
}
static int
xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset,
unsigned int level, u64 addr, u64 next,
struct xe_ptw **child,
enum page_walk_action *action,
struct xe_pt_walk *walk)
{
struct xe_pt_stage_unbind_walk *xe_walk =
container_of(walk, typeof(*xe_walk), base);
struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base);
pgoff_t end_offset;
u64 size = 1ull << walk->shifts[--level];
int err;
if (!IS_ALIGNED(addr, size))
addr = xe_walk->modified_start;
if (!IS_ALIGNED(next, size))
next = xe_walk->modified_end;
if (parent != *child &&
xe_pt_check_kill(addr, next, level, xe_child, action, walk))
return 0;
if (!xe_pt_nonshared_offsets(addr, next, level, walk, action, &offset,
&end_offset))
return 0;
err = xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, true);
if (err)
return err;
xe_walk->wupd.updates[level].update->qwords = end_offset - offset;
return 0;
}
static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = {
.pt_entry = xe_pt_stage_unbind_entry,
.pt_post_descend = xe_pt_stage_unbind_post_descend,
};
static unsigned int xe_pt_stage_unbind(struct xe_tile *tile,
struct xe_vm *vm,
struct xe_vma *vma,
struct xe_svm_range *range,
struct xe_vm_pgtable_update *entries)
{
u64 start = range ? xe_svm_range_start(range) : xe_vma_start(vma);
u64 end = range ? xe_svm_range_end(range) : xe_vma_end(vma);
struct xe_vm_pgtable_update_op *pt_update_op =
container_of(entries, struct xe_vm_pgtable_update_op, entries[0]);
struct xe_pt_stage_unbind_walk xe_walk = {
.base = {
.ops = &xe_pt_stage_unbind_ops,
.shifts = xe_normal_pt_shifts,
.max_level = XE_PT_HIGHEST_LEVEL,
.staging = true,
},
.tile = tile,
.modified_start = start,
.modified_end = end,
.wupd.entries = entries,
.prl = pt_update_op->prl,
};
struct xe_pt *pt = xe_vm_pt_root(vm, tile->id);
(void)xe_pt_walk_shared(&pt->base, pt->level, start, end,
&xe_walk.base);
return xe_walk.wupd.num_used_entries;
}
static void
xe_pt_clear_pgtable_callback(struct xe_vm *vm, struct xe_tile *tile,
struct iosys_map *map,
const struct xe_vm_pgtable_update *update)
{
u64 empty = __xe_pt_empty_pte(tile, vm, update->level);
int i;
xe_assert(vm->xe, map);
xe_assert(vm->xe, !iosys_map_is_null(map));
if (map->is_iomem)
for (i = 0; i < update->qwords; ++i)
xe_map_wr(tile_to_xe(tile), map, (update->ofs + i) *
sizeof(u64), u64, empty);
else
memset64(map->vaddr + update->ofs * sizeof(u64), empty,
update->qwords);
}
static void xe_pt_abort_unbind(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries)
{
int i, j;
xe_pt_commit_prepare_locks_assert(vma);
for (i = num_entries - 1; i >= 0; --i) {
struct xe_vm_pgtable_update *entry = &entries[i];
struct xe_pt *pt = entry->pt;
struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt);
pt->num_live += entry->qwords;
if (!pt->level)
continue;
for (j = entry->ofs; j < entry->ofs + entry->qwords; j++)
pt_dir->staging[j] =
entries[i].pt_entries[j - entry->ofs].pt ?
&entries[i].pt_entries[j - entry->ofs].pt->base : NULL;
}
}
static void
xe_pt_commit_prepare_unbind(struct xe_vma *vma,
struct xe_vm_pgtable_update *entries,
u32 num_entries)
{
int i, j;
xe_pt_commit_prepare_locks_assert(vma);
for (i = 0; i < num_entries; ++i) {
struct xe_vm_pgtable_update *entry = &entries[i];
struct xe_pt *pt = entry->pt;
struct xe_pt_dir *pt_dir;
pt->num_live -= entry->qwords;
if (!pt->level)
continue;
pt_dir = as_xe_pt_dir(pt);
for (j = entry->ofs; j < entry->ofs + entry->qwords; j++) {
entry->pt_entries[j - entry->ofs].pt =
xe_pt_entry_staging(pt_dir, j);
pt_dir->staging[j] = NULL;
}
}
}
static struct xe_vm_pgtable_update_op *
to_pt_op(struct xe_vm_pgtable_update_ops *pt_update_ops, u32 op_idx)
{
return &pt_update_ops->pt_job_ops->ops[op_idx];
}
static u32
get_current_op(struct xe_vm_pgtable_update_ops *pt_update_ops)
{
if (!pt_update_ops->pt_job_ops)
return 0;
return pt_update_ops->pt_job_ops->current_op;
}
static struct xe_vm_pgtable_update_op *
to_current_pt_op(struct xe_vm_pgtable_update_ops *pt_update_ops)
{
return to_pt_op(pt_update_ops, get_current_op(pt_update_ops));
}
static void
incr_current_op(struct xe_vm_pgtable_update_ops *pt_update_ops)
{
++pt_update_ops->pt_job_ops->current_op;
}
static void
xe_pt_update_ops_rfence_interval(struct xe_vm_pgtable_update_ops *pt_update_ops,
u64 start, u64 end)
{
u64 last;
struct xe_vm_pgtable_update_op *pt_op =
to_current_pt_op(pt_update_ops);
int i, level = 0;
for (i = 0; i < pt_op->num_entries; i++) {
const struct xe_vm_pgtable_update *entry = &pt_op->entries[i];
if (entry->pt->level > level)
level = entry->pt->level;
}
start = ALIGN_DOWN(start, 0x1ull << xe_pt_shift(level));
last = ALIGN(end, 0x1ull << xe_pt_shift(level)) - 1;
if (start < pt_update_ops->start)
pt_update_ops->start = start;
if (last > pt_update_ops->last)
pt_update_ops->last = last;
}
static int vma_reserve_fences(struct xe_device *xe, struct xe_vma *vma)
{
int shift = xe_device_get_root_tile(xe)->media_gt ? 1 : 0;
if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm)
return dma_resv_reserve_fences(xe_vma_bo(vma)->ttm.base.resv,
xe->info.tile_count << shift);
return 0;
}
static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_vma *vma, bool invalidate_on_bind)
{
struct xe_vm_pgtable_update_op *pt_op =
to_current_pt_op(pt_update_ops);
int err;
xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
xe_bo_assert_held(xe_vma_bo(vma));
vm_dbg(&xe_vma_vm(vma)->xe->drm,
"Preparing bind, with range [%llx...%llx)\n",
xe_vma_start(vma), xe_vma_end(vma) - 1);
pt_op->vma = NULL;
pt_op->bind = true;
pt_op->rebind = BIT(tile->id) & vma->tile_present;
err = vma_reserve_fences(tile_to_xe(tile), vma);
if (err)
return err;
err = xe_pt_prepare_bind(tile, vma, NULL, pt_op->entries,
&pt_op->num_entries, invalidate_on_bind);
if (!err) {
xe_tile_assert(tile, pt_op->num_entries <=
ARRAY_SIZE(pt_op->entries));
xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
pt_op->num_entries, true);
xe_pt_update_ops_rfence_interval(pt_update_ops,
xe_vma_start(vma),
xe_vma_end(vma));
incr_current_op(pt_update_ops);
pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma);
if ((!pt_op->rebind && xe_vm_has_scratch(vm) &&
xe_vm_in_lr_mode(vm)))
pt_update_ops->needs_invalidation = true;
else if (pt_op->rebind && xe_vm_in_preempt_fence_mode(vm) &&
vm->xe->info.has_ctx_tlb_inval)
pt_update_ops->needs_invalidation = true;
else if (pt_op->rebind && !xe_vm_in_lr_mode(vm))
vm->tlb_flush_seqno++;
vma->tile_staged |= BIT(tile->id);
pt_op->vma = vma;
xe_pt_commit_prepare_bind(vma, pt_op->entries,
pt_op->num_entries, pt_op->rebind);
} else {
xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries);
}
return err;
}
static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_vma *vma, struct xe_svm_range *range)
{
struct xe_vm_pgtable_update_op *pt_op =
to_current_pt_op(pt_update_ops);
int err;
xe_tile_assert(tile, xe_vma_is_cpu_addr_mirror(vma));
vm_dbg(&xe_vma_vm(vma)->xe->drm,
"Preparing bind, with range [%lx...%lx)\n",
xe_svm_range_start(range), xe_svm_range_end(range) - 1);
pt_op->vma = NULL;
pt_op->bind = true;
pt_op->rebind = BIT(tile->id) & range->tile_present;
err = xe_pt_prepare_bind(tile, vma, range, pt_op->entries,
&pt_op->num_entries, false);
if (!err) {
xe_tile_assert(tile, pt_op->num_entries <=
ARRAY_SIZE(pt_op->entries));
xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
pt_op->num_entries, true);
xe_pt_update_ops_rfence_interval(pt_update_ops,
xe_svm_range_start(range),
xe_svm_range_end(range));
incr_current_op(pt_update_ops);
pt_update_ops->needs_svm_lock = true;
pt_op->vma = vma;
xe_pt_commit_prepare_bind(vma, pt_op->entries,
pt_op->num_entries, pt_op->rebind);
} else {
xe_pt_cancel_bind(vma, pt_op->entries, pt_op->num_entries);
}
return err;
}
static int unbind_op_prepare(struct xe_tile *tile,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_vma *vma)
{
struct xe_device *xe = tile_to_xe(tile);
struct xe_vm_pgtable_update_op *pt_op =
to_current_pt_op(pt_update_ops);
int err;
if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id)))
return 0;
xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma));
xe_bo_assert_held(xe_vma_bo(vma));
vm_dbg(&xe_vma_vm(vma)->xe->drm,
"Preparing unbind, with range [%llx...%llx)\n",
xe_vma_start(vma), xe_vma_end(vma) - 1);
pt_op->vma = vma;
pt_op->bind = false;
pt_op->rebind = false;
if (xe->info.has_page_reclaim_hw_assist &&
xe_page_reclaim_list_is_new(&pt_update_ops->prl))
xe_page_reclaim_list_alloc_entries(&pt_update_ops->prl);
pt_op->prl = (xe_page_reclaim_list_valid(&pt_update_ops->prl) &&
!xe_page_reclaim_skip(tile, vma)) ? &pt_update_ops->prl : NULL;
err = vma_reserve_fences(tile_to_xe(tile), vma);
if (err)
return err;
pt_op->num_entries = xe_pt_stage_unbind(tile, xe_vma_vm(vma),
vma, NULL, pt_op->entries);
xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
pt_op->num_entries, false);
xe_pt_update_ops_rfence_interval(pt_update_ops, xe_vma_start(vma),
xe_vma_end(vma));
incr_current_op(pt_update_ops);
pt_update_ops->needs_svm_lock |= xe_vma_is_userptr(vma);
pt_update_ops->needs_invalidation = true;
xe_pt_commit_prepare_unbind(vma, pt_op->entries, pt_op->num_entries);
return 0;
}
static bool
xe_pt_op_check_range_skip_invalidation(struct xe_vm_pgtable_update_op *pt_op,
struct xe_svm_range *range)
{
struct xe_vm_pgtable_update *update = pt_op->entries;
XE_WARN_ON(!pt_op->num_entries);
if (pt_op->num_entries > 1)
return false;
if (update->pt->level == 0)
return true;
if (update->pt->level == 1)
return xe_svm_range_size(range) >= SZ_2M;
return false;
}
static int unbind_range_prepare(struct xe_vm *vm,
struct xe_tile *tile,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_svm_range *range)
{
struct xe_vm_pgtable_update_op *pt_op =
to_current_pt_op(pt_update_ops);
if (!(range->tile_present & BIT(tile->id)))
return 0;
vm_dbg(&vm->xe->drm,
"Preparing unbind, with range [%lx...%lx)\n",
xe_svm_range_start(range), xe_svm_range_end(range) - 1);
pt_op->vma = XE_INVALID_VMA;
pt_op->bind = false;
pt_op->rebind = false;
pt_op->prl = NULL;
pt_op->num_entries = xe_pt_stage_unbind(tile, vm, NULL, range,
pt_op->entries);
xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries,
pt_op->num_entries, false);
xe_pt_update_ops_rfence_interval(pt_update_ops, xe_svm_range_start(range),
xe_svm_range_end(range));
incr_current_op(pt_update_ops);
pt_update_ops->needs_svm_lock = true;
pt_update_ops->needs_invalidation |= xe_vm_has_scratch(vm) ||
xe_vm_has_valid_gpu_mapping(tile, range->tile_present,
range->tile_invalidated) ||
!xe_pt_op_check_range_skip_invalidation(pt_op, range);
xe_pt_commit_prepare_unbind(XE_INVALID_VMA, pt_op->entries,
pt_op->num_entries);
return 0;
}
static int op_prepare(struct xe_vm *vm,
struct xe_tile *tile,
struct xe_vma_ops *vops,
struct xe_vm_pgtable_update_ops *pt_update_ops,
struct xe_vma_op *op)
{
int err = 0;
xe_vm_assert_held(vm);
switch (op->base.op) {
case DRM_GPUVA_OP_MAP:
if ((!op->map.immediate && xe_vm_in_fault_mode(vm) &&
!op->map.invalidate_on_bind) ||
(op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR))
break;
err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma,
op->map.invalidate_on_bind);
vops->flags |= XE_VMA_OPS_FLAG_WAIT_VM_KERNEL;
break;
case DRM_GPUVA_OP_REMAP:
{
struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va);
if (xe_vma_is_cpu_addr_mirror(old))
break;
err = unbind_op_prepare(tile, pt_update_ops, old);
if (!err && op->remap.prev && !op->remap.skip_prev) {
err = bind_op_prepare(vm, tile, pt_update_ops,
op->remap.prev, false);
vops->flags |= XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP;
}
if (!err && op->remap.next && !op->remap.skip_next) {
err = bind_op_prepare(vm, tile, pt_update_ops,
op->remap.next, false);
vops->flags |= XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP;
}
break;
}
case DRM_GPUVA_OP_UNMAP:
{
struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
if (xe_vma_is_cpu_addr_mirror(vma))
break;
err = unbind_op_prepare(tile, pt_update_ops, vma);
break;
}
case DRM_GPUVA_OP_PREFETCH:
{
struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
if (xe_vma_is_cpu_addr_mirror(vma)) {
struct xe_svm_range *range;
unsigned long i;
xa_for_each(&op->prefetch_range.range, i, range) {
err = bind_range_prepare(vm, tile, pt_update_ops,
vma, range);
if (err == -EAGAIN)
err = -ENODATA;
if (err)
return err;
}
} else {
err = bind_op_prepare(vm, tile, pt_update_ops, vma, false);
vops->flags |= XE_VMA_OPS_FLAG_WAIT_VM_KERNEL;
}
break;
}
case DRM_GPUVA_OP_DRIVER:
if (op->subop == XE_VMA_SUBOP_MAP_RANGE) {
xe_assert(vm->xe, xe_vma_is_cpu_addr_mirror(op->map_range.vma));
err = bind_range_prepare(vm, tile, pt_update_ops,
op->map_range.vma,
op->map_range.range);
} else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE) {
err = unbind_range_prepare(vm, tile, pt_update_ops,
op->unmap_range.range);
}
break;
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
return err;
}
static void
xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops)
{
pt_update_ops->pt_job_ops->current_op = 0;
pt_update_ops->start = ~0x0ull;
pt_update_ops->last = 0x0ull;
pt_update_ops->needs_svm_lock = false;
pt_update_ops->needs_invalidation = false;
xe_page_reclaim_list_init(&pt_update_ops->prl);
}
static int __xe_pt_update_ops_prepare(struct xe_tile *tile,
struct xe_vma_ops *vops)
{
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[tile->id];
struct xe_vma_op *op;
int shift = tile->media_gt ? 1 : 0;
int err;
lockdep_assert_held(&vops->vm->lock);
xe_vm_assert_held(vops->vm);
xe_pt_update_ops_init(pt_update_ops);
err = dma_resv_reserve_fences(xe_vm_resv(vops->vm),
tile_to_xe(tile)->info.tile_count << shift);
if (err)
return err;
list_for_each_entry(op, &vops->list, link) {
err = op_prepare(vops->vm, tile, vops, pt_update_ops, op);
if (err)
return err;
}
xe_tile_assert(tile, get_current_op(pt_update_ops) <=
pt_update_ops->num_ops);
if (pt_update_ops->start < vops->start)
vops->start = pt_update_ops->start;
if (pt_update_ops->last > vops->last)
vops->last = pt_update_ops->last;
if (pt_update_ops->needs_invalidation)
vops->flags |= XE_VMA_OPS_FLAG_NEEDS_INVALIDATION;
if (pt_update_ops->needs_svm_lock)
vops->flags |= XE_VMA_OPS_FLAG_NEEDS_SVM_LOCK;
#ifdef TEST_VM_OPS_ERROR
if (vops->inject_error &&
vops->vm->xe->vm_inject_error_position == FORCE_OP_ERROR_PREPARE)
return -ENOSPC;
#endif
return 0;
}
int xe_pt_update_ops_prepare(struct xe_device *xe, struct xe_vma_ops *vops)
{
struct xe_tile *tile;
int id, err;
for_each_tile(tile, xe, id) {
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[id];
if (!vops->pt_update_ops[id].num_ops)
continue;
if (pt_mirroring_disabled_for_tile(vops->vm, id)) {
struct xe_page_reclaim_list *prl = &pt_update_ops->prl;
*prl = vops->pt_update_ops[0].prl;
xe_page_reclaim_entries_get(prl->entries);
continue;
}
err = __xe_pt_update_ops_prepare(tile, vops);
if (err)
return err;
}
return 0;
}
ALLOW_ERROR_INJECTION(xe_pt_update_ops_prepare, ERRNO);
static void vma_add_fences(struct xe_vma *vma, struct dma_fence **fences,
int fence_count, enum dma_resv_usage usage)
{
int i;
if (xe_vma_has_no_bo(vma) || xe_vma_bo(vma)->vm)
return;
for (i = 0; i < fence_count; ++i)
if (fences[i])
dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv,
fences[i], usage);
}
static void bind_op_commit(struct xe_vm *vm, struct xe_vma *vma,
struct dma_fence **fences, int fence_count,
enum dma_resv_usage usage, u8 tile_mask,
bool invalidate_on_bind)
{
xe_assert(vm->xe, !xe_vma_is_cpu_addr_mirror(vma));
vma_add_fences(vma, fences, fence_count, usage);
WRITE_ONCE(vma->tile_present, vma->tile_present | tile_mask);
if (invalidate_on_bind)
WRITE_ONCE(vma->tile_invalidated,
vma->tile_invalidated | tile_mask);
else
WRITE_ONCE(vma->tile_invalidated,
vma->tile_invalidated & ~tile_mask);
vma->tile_staged &= ~tile_mask;
if (xe_vma_is_userptr(vma)) {
xe_svm_assert_held_read_or_inject_write(vm);
to_userptr_vma(vma)->userptr.initial_bind = true;
}
if (usage == DMA_RESV_USAGE_KERNEL &&
xe_vm_in_preempt_fence_mode(vm) &&
!current->mm)
xe_vm_queue_rebind_worker(vm);
}
static void unbind_op_commit(struct xe_vm *vm, struct xe_vma *vma,
struct dma_fence **fences, int fence_count,
enum dma_resv_usage usage, u8 tile_mask)
{
xe_assert(vm->xe, !xe_vma_is_cpu_addr_mirror(vma));
vma_add_fences(vma, fences, fence_count, usage);
vma->tile_present &= ~tile_mask;
if (!vma->tile_present) {
list_del_init(&vma->combined_links.rebind);
if (xe_vma_is_userptr(vma)) {
xe_svm_assert_held_read_or_inject_write(vm);
spin_lock(&vm->userptr.invalidated_lock);
list_del_init(&to_userptr_vma(vma)->userptr.invalidate_link);
spin_unlock(&vm->userptr.invalidated_lock);
}
}
}
static void range_present_and_invalidated_tile(struct xe_vm *vm,
struct xe_svm_range *range,
u8 tile_mask)
{
lockdep_assert_held(&vm->svm.gpusvm.notifier_lock);
WRITE_ONCE(range->tile_present, range->tile_present | tile_mask);
WRITE_ONCE(range->tile_invalidated, range->tile_invalidated & ~tile_mask);
}
static void op_commit(struct xe_vm *vm, struct xe_vma_op *op,
struct dma_fence **fences, int fence_count,
enum dma_resv_usage usage, u8 tile_mask)
{
xe_vm_assert_held(vm);
switch (op->base.op) {
case DRM_GPUVA_OP_MAP:
if ((!op->map.immediate && xe_vm_in_fault_mode(vm)) ||
(op->map.vma_flags & XE_VMA_SYSTEM_ALLOCATOR))
break;
bind_op_commit(vm, op->map.vma, fences, fence_count, usage,
tile_mask, op->map.invalidate_on_bind);
break;
case DRM_GPUVA_OP_REMAP:
{
struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va);
if (xe_vma_is_cpu_addr_mirror(old))
break;
unbind_op_commit(vm, old, fences, fence_count, usage,
tile_mask);
if (op->remap.prev && !op->remap.skip_prev)
bind_op_commit(vm, op->remap.prev, fences, fence_count,
usage, tile_mask, false);
if (op->remap.next && !op->remap.skip_next)
bind_op_commit(vm, op->remap.next, fences, fence_count,
usage, tile_mask, false);
break;
}
case DRM_GPUVA_OP_UNMAP:
{
struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va);
if (!xe_vma_is_cpu_addr_mirror(vma))
unbind_op_commit(vm, vma, fences, fence_count,
usage, tile_mask);
break;
}
case DRM_GPUVA_OP_PREFETCH:
{
struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va);
if (xe_vma_is_cpu_addr_mirror(vma)) {
struct xe_svm_range *range = NULL;
unsigned long i;
xa_for_each(&op->prefetch_range.range, i, range)
range_present_and_invalidated_tile(vm, range,
tile_mask);
} else {
bind_op_commit(vm, vma, fences, fence_count, usage,
tile_mask, false);
}
break;
}
case DRM_GPUVA_OP_DRIVER:
{
if (op->subop == XE_VMA_SUBOP_MAP_RANGE)
range_present_and_invalidated_tile(vm, op->map_range.range,
tile_mask);
else if (op->subop == XE_VMA_SUBOP_UNMAP_RANGE)
WRITE_ONCE(op->unmap_range.range->tile_present,
op->unmap_range.range->tile_present &
~tile_mask);
break;
}
default:
drm_warn(&vm->xe->drm, "NOT POSSIBLE");
}
}
static const struct xe_cpu_bind_pt_update_ops cpu_bind_ops = {
.populate = xe_vm_populate_pgtable,
.clear = xe_pt_clear_pgtable_callback,
.pre_commit = xe_pt_pre_commit,
};
#if IS_ENABLED(CONFIG_DRM_GPUSVM)
static const struct xe_cpu_bind_pt_update_ops svm_userptr_cpu_bind_ops = {
.populate = xe_vm_populate_pgtable,
.clear = xe_pt_clear_pgtable_callback,
.pre_commit = xe_pt_svm_userptr_pre_commit,
};
#else
static const struct xe_cpu_bind_pt_update_ops svm_userptr_cpu_bind_ops;
#endif
struct dma_fence *
xe_pt_update_ops_run(struct xe_device *xe, struct xe_vma_ops *vops)
{
struct xe_vm *vm = vops->vm;
struct xe_exec_queue *q = vops->q;
struct dma_fence *fence;
struct dma_fence *ifences[XE_CPU_BIND_INVAL_JOB_COUNT] = {};
struct xe_range_fence *rfence;
enum dma_resv_usage usage = DMA_RESV_USAGE_BOOKKEEP;
struct xe_vma_op *op;
struct xe_tile *tile;
int err = 0, total_ops = 0, i, j;
u8 tile_mask = 0;
bool needs_invalidation = vops->flags &
XE_VMA_OPS_FLAG_NEEDS_INVALIDATION;
bool needs_svm_lock = vops->flags &
XE_VMA_OPS_FLAG_NEEDS_SVM_LOCK;
struct xe_cpu_bind_pt_update update = {
.ops = needs_svm_lock ? &svm_userptr_cpu_bind_ops :
&cpu_bind_ops,
.vops = vops,
};
lockdep_assert_held(&vm->lock);
xe_vm_assert_held(vm);
for_each_tile(tile, xe, j) {
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[j];
total_ops += get_current_op(pt_update_ops);
}
if (!total_ops) {
xe_assert(xe, xe_vm_in_fault_mode(vm));
return dma_fence_get_stub();
}
#ifdef TEST_VM_OPS_ERROR
if (vops->inject_error &&
xe->vm_inject_error_position == FORCE_OP_ERROR_RUN)
return ERR_PTR(-ENOSPC);
#endif
if (needs_invalidation) {
for_each_tlb_inval(q, i) {
struct xe_dep_scheduler *dep_scheduler =
q->tlb_inval[i].dep_scheduler;
struct xe_tile *tile =
&xe->tiles[i / XE_MAX_GT_PER_TILE];
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[tile->id];
struct xe_page_reclaim_list *prl = &pt_update_ops->prl;
struct xe_tlb_inval_job *ijob;
struct xe_gt *gt = i % XE_MAX_GT_PER_TILE ?
tile->media_gt : tile->primary_gt;
ijob = xe_tlb_inval_job_create(q, >->tlb_inval,
dep_scheduler,
vm, vops->start,
vops->last, i);
if (IS_ERR(ijob)) {
err = PTR_ERR(ijob);
goto free_ijob;
}
update.ijobs[i] = ijob;
if (xe_page_reclaim_list_valid(prl)) {
xe_tlb_inval_job_add_page_reclaim(ijob, prl);
xe_page_reclaim_list_invalidate(prl);
}
}
}
rfence = kzalloc_obj(*rfence);
if (!rfence) {
err = -ENOMEM;
goto free_ijob;
}
fence = xe_cpu_bind_update_pgtables(xe->cpu_bind, &update);
if (IS_ERR(fence)) {
err = PTR_ERR(fence);
goto free_rfence;
}
for_each_tile(tile, xe, j) {
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[j];
if (pt_mirroring_disabled_for_tile(vm, j)) {
xe_tile_assert(tile, !get_current_op(pt_update_ops));
tile_mask |= BIT(tile->id);
continue;
}
for (i = 0; i < get_current_op(pt_update_ops); ++i) {
struct xe_vm_pgtable_update_op *pt_op =
to_pt_op(pt_update_ops, i);
xe_pt_commit(pt_op->vma, pt_op->entries,
pt_op->num_entries,
&pt_update_ops->pt_job_ops->deferred);
pt_op->vma = NULL;
tile_mask |= BIT(tile->id);
}
}
if (xe_range_fence_insert(&vm->rftree, rfence,
&xe_range_fence_kfree_ops,
vops->start, vops->last, fence))
dma_fence_wait(fence, false);
if (vops->flags & XE_VMA_OPS_FLAG_WAIT_VM_BOOKKEEP)
usage = DMA_RESV_USAGE_KERNEL;
if (!needs_invalidation) {
dma_resv_add_fence(xe_vm_resv(vm), fence, usage);
list_for_each_entry(op, &vops->list, link)
op_commit(vops->vm, op, &fence, 1, usage, tile_mask);
} else {
for (i = 0; i < XE_CPU_BIND_INVAL_JOB_COUNT; ++i) {
if (!update.ijobs[i])
continue;
ifences[i] = xe_tlb_inval_job_push(update.ijobs[i],
fence);
xe_assert(xe, !IS_ERR_OR_NULL(ifences[i]));
dma_resv_add_fence(xe_vm_resv(vm), ifences[i], usage);
}
list_for_each_entry(op, &vops->list, link)
op_commit(vops->vm, op, ifences,
XE_CPU_BIND_INVAL_JOB_COUNT, usage,
tile_mask);
}
if (needs_svm_lock)
xe_pt_svm_userptr_notifier_unlock(vm);
if (!(q->flags & EXEC_QUEUE_FLAG_MIGRATE))
xe_exec_queue_last_fence_set(q, vm, fence);
for (i = 0; i < XE_CPU_BIND_INVAL_JOB_COUNT; ++i) {
xe_tlb_inval_job_put(update.ijobs[i]);
dma_fence_put(ifences[i]);
}
return fence;
free_rfence:
kfree(rfence);
free_ijob:
for (i = 0; i < XE_CPU_BIND_INVAL_JOB_COUNT; ++i)
xe_tlb_inval_job_put(update.ijobs[i]);
return ERR_PTR(err);
}
ALLOW_ERROR_INJECTION(xe_pt_update_ops_run, ERRNO);
void xe_pt_update_ops_fini(struct xe_device *xe, struct xe_vma_ops *vops)
{
struct xe_tile *tile;
int id;
for_each_tile(tile, xe, id) {
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[id];
if (!pt_update_ops->num_ops)
continue;
xe_page_reclaim_entries_put(pt_update_ops->prl.entries);
}
}
void xe_pt_update_ops_abort(struct xe_device *xe, struct xe_vma_ops *vops)
{
struct xe_tile *tile;
int id;
lockdep_assert_held(&vops->vm->lock);
xe_vm_assert_held(vops->vm);
for_each_tile(tile, xe, id) {
struct xe_vm_pgtable_update_ops *pt_update_ops =
&vops->pt_update_ops[id];
int i;
if (pt_mirroring_disabled_for_tile(vops->vm, id))
continue;
for (i = pt_update_ops->num_ops - 1; i >= 0; --i) {
struct xe_vm_pgtable_update_op *pt_op =
to_pt_op(pt_update_ops, i);
if (!pt_op->vma || i >= get_current_op(pt_update_ops))
continue;
if (pt_op->bind)
xe_pt_abort_bind(pt_op->vma, pt_op->entries,
pt_op->num_entries,
pt_op->rebind);
else
xe_pt_abort_unbind(pt_op->vma, pt_op->entries,
pt_op->num_entries);
}
}
xe_pt_update_ops_fini(xe, vops);
}
struct xe_pt_job_ops *xe_pt_job_ops_alloc(u32 num_ops)
{
struct xe_pt_job_ops *pt_job_ops;
pt_job_ops = kmalloc_obj(*pt_job_ops);
if (!pt_job_ops)
return NULL;
pt_job_ops->ops = kvmalloc_objs(*pt_job_ops->ops, num_ops);
if (!pt_job_ops->ops) {
kfree(pt_job_ops);
return NULL;
}
pt_job_ops->current_op = 0;
kref_init(&pt_job_ops->refcount);
init_llist_head(&pt_job_ops->deferred);
return pt_job_ops;
}
struct xe_pt_job_ops *xe_pt_job_ops_get(struct xe_pt_job_ops *pt_job_ops)
{
if (pt_job_ops)
kref_get(&pt_job_ops->refcount);
return pt_job_ops;
}
static void xe_pt_update_ops_free(struct xe_vm_pgtable_update_op *pt_op,
u32 num_ops)
{
u32 i;
for (i = 0; i < num_ops; ++i, ++pt_op)
xe_pt_free_bind(pt_op->entries, pt_op->num_entries);
}
static void xe_pt_job_ops_destroy(struct kref *ref)
{
struct xe_pt_job_ops *pt_job_ops =
container_of(ref, struct xe_pt_job_ops, refcount);
struct llist_node *freed;
struct xe_bo *bo, *next;
xe_pt_update_ops_free(pt_job_ops->ops,
pt_job_ops->current_op);
freed = llist_del_all(&pt_job_ops->deferred);
if (freed) {
llist_for_each_entry_safe(bo, next, freed, freed) {
struct xe_bo_dev *bo_device =
&xe_bo_device(bo)->bo_device;
if (llist_add(&bo->freed, &bo_device->async_list))
schedule_work(&bo_device->async_free);
}
}
kvfree(pt_job_ops->ops);
kfree(pt_job_ops);
}
void xe_pt_job_ops_put(struct xe_pt_job_ops *pt_job_ops)
{
if (!pt_job_ops)
return;
kref_put(&pt_job_ops->refcount, xe_pt_job_ops_destroy);
}