#include <linux/circ_buf.h>
#include <drm/drm_exec.h>
#include <drm/drm_managed.h>
#include "xe_bo.h"
#include "xe_device.h"
#include "xe_gt_printk.h"
#include "xe_gt_types.h"
#include "xe_gt_stats.h"
#include "xe_log.h"
#include "xe_migrate.h"
#include "xe_pagefault.h"
#include "xe_pagefault_types.h"
#include "xe_pm.h"
#include "xe_svm.h"
#include "xe_trace_bo.h"
#include "xe_vm.h"
enum xe_pagefault_alloc_state {
XE_PAGEFAULT_ALLOC_STATE_FREE = 0,
XE_PAGEFAULT_ALLOC_STATE_QUEUED = 1,
XE_PAGEFAULT_ALLOC_STATE_CHAINED = 2,
XE_PAGEFAULT_ALLOC_STATE_ACTIVE = 3,
XE_PAGEFAULT_ALLOC_STATE_COUNT = 4,
};
static int xe_pagefault_entry_size(void)
{
return roundup_pow_of_two(sizeof(struct xe_pagefault));
}
static int xe_pagefault_begin(struct drm_exec *exec, struct xe_vma *vma,
struct xe_vram_region *vram, bool need_vram_move)
{
struct xe_bo *bo = xe_vma_bo(vma);
struct xe_vm *vm = xe_vma_vm(vma);
int err;
err = xe_vm_lock_vma(exec, vma);
if (err)
return err;
if (!bo)
return 0;
if (unlikely(xe_bo_madv_is_dontneed(bo) || xe_bo_is_purged(bo))) {
if (!xe_vm_has_scratch(vm))
return -EACCES;
return 0;
}
return need_vram_move ? xe_bo_migrate(bo, vram->placement, NULL, exec) :
xe_bo_validate(bo, vm, true, exec);
}
static int xe_pagefault_handle_vma(struct xe_gt *gt, struct xe_vma *vma,
struct xe_pagefault *pf, bool atomic)
{
struct xe_vm *vm = xe_vma_vm(vma);
struct xe_tile *tile = gt_to_tile(gt);
struct xe_validation_ctx ctx;
struct drm_exec exec;
struct dma_fence *fence;
int err = 0, needs_vram;
lockdep_assert_held(&vm->lock);
needs_vram = xe_vma_need_vram_for_atomic(vm->xe, vma, atomic);
if (needs_vram < 0) {
xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK);
return needs_vram;
}
if (needs_vram && xe_vma_is_userptr(vma)) {
xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_ATOMIC_USERPTR);
return -EACCES;
}
xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT, 1);
xe_gt_stats_incr(gt, XE_GT_STATS_ID_VMA_PAGEFAULT_KB,
xe_vma_size(vma) / SZ_1K);
trace_xe_vma_pagefault(vma);
guard(mutex)(&vma->fault_lock);
if (xe_vm_has_valid_gpu_mapping(tile, vma->tile_present,
vma->tile_invalidated) && !atomic) {
xe_pagefault_set_start_addr(pf, xe_vma_start(vma));
xe_pagefault_set_end_addr(pf, xe_vma_end(vma));
return 0;
}
do {
xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_NONE);
if (xe_vma_is_userptr(vma) &&
xe_vma_userptr_check_repin(to_userptr_vma(vma))) {
struct xe_userptr_vma *uvma = to_userptr_vma(vma);
err = xe_vma_userptr_pin_pages(uvma);
if (err) {
xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_USERPTR_PIN);
return err;
}
}
xe_validation_ctx_init(&ctx, &vm->xe->val, &exec,
(struct xe_val_flags) {});
drm_exec_until_all_locked(&exec) {
err = xe_pagefault_begin(&exec, vma, tile->mem.vram,
needs_vram == 1);
drm_exec_retry_on_contention(&exec);
xe_validation_retry_on_oom(&ctx, &err);
if (err) {
xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_VALIDATE);
break;
}
trace_xe_vma_pf_bind(vma);
xe_vm_set_validation_exec(vm, &exec);
fence = xe_vma_rebind(vm, vma, BIT(tile->id));
xe_vm_set_validation_exec(vm, NULL);
if (IS_ERR(fence)) {
err = PTR_ERR(fence);
xe_validation_retry_on_oom(&ctx, &err);
xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_REBIND);
break;
}
}
xe_validation_ctx_fini(&ctx);
} while (err == -EAGAIN);
if (!err) {
xe_pagefault_set_start_addr(pf, xe_vma_start(vma));
xe_pagefault_set_end_addr(pf, xe_vma_end(vma));
dma_fence_wait(fence, false);
dma_fence_put(fence);
}
return err;
}
static bool
xe_pagefault_access_is_atomic(enum xe_pagefault_access_type access_type)
{
return (access_type & XE_PAGEFAULT_ACCESS_TYPE_MASK) == XE_PAGEFAULT_ACCESS_TYPE_ATOMIC;
}
static struct xe_vm *xe_pagefault_asid_to_vm(struct xe_pagefault *pf, u32 asid)
{
struct xe_device *xe = gt_to_xe(pf->gt);
struct xe_vm *vm;
down_read(&xe->usm.lock);
vm = xa_load(&xe->usm.asid_to_vm, asid);
if (!vm) {
xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VM_NOT_FOUND);
vm = ERR_PTR(-EINVAL);
} else if (!xe_vm_in_fault_mode(vm)) {
xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE);
vm = ERR_PTR(-EINVAL);
} else {
xe_vm_get(vm);
}
up_read(&xe->usm.lock);
return vm;
}
static int xe_pagefault_service(struct xe_pagefault *pf)
{
struct xe_gt *gt = pf->gt;
struct xe_device *xe = gt_to_xe(gt);
struct xe_vm *vm;
struct xe_vma *vma = NULL;
int err;
bool atomic;
u32 asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id);
if (pf->consumer.fault_type_level == XE_PAGEFAULT_TYPE_LEVEL_NACK)
return -EFAULT;
vm = xe_pagefault_asid_to_vm(pf, asid);
if (IS_ERR(vm)) {
err = PTR_ERR(vm);
trace_xe_pagefault_fail(xe, NULL, NULL, asid,
xe_pagefault_addr(pf),
xe_pagefault_get_error(pf), err);
return err;
}
xe_migrate_ulls_enter(gt_to_tile(gt)->migrate);
down_read(&vm->lock);
if (xe_vm_is_closed(vm)) {
err = -ENOENT;
xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VM_CLOSED);
goto unlock_vm;
}
vma = xe_vm_find_vma_by_addr(vm, xe_pagefault_addr(pf));
if (!vma) {
err = -EINVAL;
xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_VMA_NOT_FOUND);
goto unlock_vm;
}
if (xe_vma_read_only(vma) &&
pf->consumer.access_type != XE_PAGEFAULT_ACCESS_TYPE_READ) {
err = -EPERM;
xe_pagefault_set_error(pf, XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION);
goto unlock_vm;
}
atomic = xe_pagefault_access_is_atomic(pf->consumer.access_type);
if (xe_vma_is_cpu_addr_mirror(vma))
err = xe_svm_handle_pagefault(vm, vma, pf, gt,
xe_pagefault_addr(pf), atomic);
else
err = xe_pagefault_handle_vma(gt, vma, pf, atomic);
unlock_vm:
if (err)
trace_xe_pagefault_fail(xe, vm, vma, asid,
xe_pagefault_addr(pf),
xe_pagefault_get_error(pf), err);
up_read(&vm->lock);
xe_vm_put(vm);
return err;
}
#define XE_PAGEFAULT_CACHE_START_INVALID U64_MAX
#define xe_pagefault_cache_start_invalidate(val) \
(val = XE_PAGEFAULT_CACHE_START_INVALID)
static void
xe_pagefault_cache_invalidate(struct xe_pagefault_queue *pf_queue,
struct xe_pagefault_work *pf_work)
{
lockdep_assert_held(&pf_queue->lock);
xe_pagefault_cache_start_invalidate(pf_work->cache.start);
}
static bool xe_pagefault_queue_full(struct xe_pagefault_queue *pf_queue)
{
lockdep_assert_held(&pf_queue->lock);
return CIRC_SPACE(pf_queue->head, pf_queue->tail,
pf_queue->size) <= xe_pagefault_entry_size();
}
static struct xe_pagefault *
xe_pagefault_queue_add(struct xe_pagefault_queue *pf_queue,
struct xe_pagefault *pf)
{
struct xe_device *xe = container_of(pf_queue, typeof(*xe),
usm.pf_queue);
struct xe_pagefault *lpf;
lockdep_assert_held(&pf_queue->lock);
do {
if (WARN_ON_ONCE(xe_pagefault_queue_full(pf_queue))) {
xe_log_err(xe, PAGEFAULT, -ENOSPC, "Queue full!\n");
return NULL;
}
lpf = (pf_queue->data + pf_queue->head);
pf_queue->head = (pf_queue->head + xe_pagefault_entry_size()) %
pf_queue->size;
} while (lpf->consumer.alloc_state != XE_PAGEFAULT_ALLOC_STATE_FREE);
xe_assert(xe, lpf != pf);
memcpy(lpf, pf, sizeof(*pf));
lpf->consumer.alloc_state = XE_PAGEFAULT_ALLOC_STATE_QUEUED;
return lpf;
}
static struct xe_pagefault *
xe_pagefault_queue_unchain_requeue(struct xe_pagefault_queue *pf_queue,
struct xe_pagefault *pf, struct xe_gt *gt)
{
struct xe_device *xe = container_of(pf_queue, typeof(*xe),
usm.pf_queue);
struct xe_pagefault *next = pf->consumer.next, *lpf;
lockdep_assert_held(&pf_queue->lock);
xe_assert(xe, pf->consumer.alloc_state ==
XE_PAGEFAULT_ALLOC_STATE_CHAINED);
xe_gt_stats_incr(gt, XE_GT_STATS_ID_CHAIN_MISMATCH_PAGEFAULT_COUNT, 1);
pf->consumer.alloc_state = XE_PAGEFAULT_ALLOC_STATE_FREE;
lpf = xe_pagefault_queue_add(pf_queue, pf);
if (lpf) {
lpf->consumer.next = NULL;
lpf->consumer.fault_type_level |= XE_PAGEFAULT_REQUEUE_MASK;
}
return next;
}
static bool xe_pagefault_match(struct xe_pagefault *pf, u64 start,
u64 end, u64 cache_asid)
{
struct xe_device *xe = gt_to_xe(pf->gt);
u64 page_addr = xe_pagefault_addr(pf);
u32 pf_asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id);
xe_assert(xe, pf->consumer.alloc_state !=
XE_PAGEFAULT_ALLOC_STATE_FREE);
return page_addr >= start && page_addr < end &&
pf_asid == cache_asid;
}
static bool xe_pagefault_try_chain(struct xe_pagefault_queue *pf_queue,
struct xe_pagefault *pf)
{
struct xe_device *xe = container_of(pf_queue, typeof(*xe),
usm.pf_queue);
struct xe_pagefault_work *pf_work;
bool requeue = FIELD_GET(XE_PAGEFAULT_REQUEUE_MASK,
pf->consumer.fault_type_level);
int i;
lockdep_assert_held(&pf_queue->lock);
xe_assert(xe, pf->consumer.alloc_state ==
XE_PAGEFAULT_ALLOC_STATE_QUEUED);
if (pf->consumer.next)
return false;
for (i = 0, pf_work = xe->usm.pf_workers;
i < xe->info.num_pf_work; ++i, ++pf_work) {
u64 start = pf_work->cache.start;
u64 end = requeue ? start + SZ_4K : pf_work->cache.end;
u32 asid = pf_work->cache.asid;
if (xe_pagefault_match(pf, start, end, asid)) {
xe_assert(xe, pf_work->cache.pf->consumer.alloc_state ==
XE_PAGEFAULT_ALLOC_STATE_ACTIVE);
if (pf->producer.private !=
pf_work->cache.pf->producer.private)
continue;
xe_gt_stats_incr(pf->gt,
XE_GT_STATS_ID_CHAIN_PAGEFAULT_COUNT,
1);
pf->consumer.alloc_state =
XE_PAGEFAULT_ALLOC_STATE_CHAINED;
pf->consumer.next = pf_work->cache.pf->consumer.next;
pf_work->cache.pf->consumer.next = pf;
return true;
}
}
return false;
}
static void xe_pagefault_queue_advance(struct xe_pagefault_queue *pf_queue)
{
lockdep_assert_held(&pf_queue->lock);
pf_queue->tail = (pf_queue->tail + xe_pagefault_entry_size()) %
pf_queue->size;
}
static struct xe_pagefault *
xe_pagefault_queue_tail_fault(struct xe_pagefault_queue *pf_queue)
{
lockdep_assert_held(&pf_queue->lock);
return pf_queue->data + pf_queue->tail;
}
static bool xe_pagefault_queue_empty(struct xe_pagefault_queue *pf_queue)
{
lockdep_assert_held(&pf_queue->lock);
return pf_queue->head == pf_queue->tail;
}
static bool xe_pagefault_queue_pop(struct xe_pagefault_queue *pf_queue,
struct xe_pagefault **pf, int id)
{
struct xe_device *xe = container_of(pf_queue, typeof(*xe),
usm.pf_queue);
struct xe_pagefault_work *pf_work, *__pf_work;
struct xe_pagefault *lpf;
size_t align = SZ_2M;
int i;
guard(spinlock_irq)(&pf_queue->lock);
for (*pf = NULL; !*pf;) {
if (xe_pagefault_queue_empty(pf_queue))
return false;
lpf = xe_pagefault_queue_tail_fault(pf_queue);
xe_pagefault_queue_advance(pf_queue);
if (lpf->consumer.alloc_state !=
XE_PAGEFAULT_ALLOC_STATE_QUEUED)
continue;
if (xe_pagefault_try_chain(pf_queue, lpf))
continue;
*pf = lpf;
}
pf_work = xe->usm.pf_workers + id;
if (FIELD_GET(XE_PAGEFAULT_REQUEUE_MASK,
lpf->consumer.fault_type_level))
align = SZ_4K;
pf_work->cache.start = ALIGN_DOWN(xe_pagefault_addr(lpf), align);
pf_work->cache.end = pf_work->cache.start + align;
pf_work->cache.asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, lpf->consumer.id);
pf_work->cache.pf = lpf;
lpf->consumer.alloc_state = XE_PAGEFAULT_ALLOC_STATE_ACTIVE;
for (i = 0, __pf_work = xe->usm.pf_workers;
i < xe->info.num_pf_work; ++i, ++__pf_work) {
u64 cache_start = __pf_work->cache.start;
if (__pf_work == pf_work)
continue;
if (cache_start != XE_PAGEFAULT_CACHE_START_INVALID) {
xe_gt_stats_incr(xe_root_mmio_gt(xe),
XE_GT_STATS_ID_PARALLEL_PAGEFAULT_COUNT,
1);
break;
}
}
while (1) {
if (xe_pagefault_queue_empty(pf_queue))
break;
lpf = xe_pagefault_queue_tail_fault(pf_queue);
if (lpf->consumer.alloc_state !=
XE_PAGEFAULT_ALLOC_STATE_QUEUED) {
xe_pagefault_queue_advance(pf_queue);
continue;
}
if (!xe_pagefault_try_chain(pf_queue, lpf))
break;
xe_pagefault_queue_advance(pf_queue);
}
return true;
}
static const char *xe_pagefault_error_to_str(enum xe_pagefault_error error)
{
switch (error) {
case XE_PAGEFAULT_ERROR_NONE:
return "NONE";
case XE_PAGEFAULT_ERROR_VM_NOT_FOUND:
return "VM_NOT_FOUND";
case XE_PAGEFAULT_ERROR_VM_NOT_IN_FAULT_MODE:
return "VM_NOT_IN_FAULT_MODE";
case XE_PAGEFAULT_ERROR_VM_CLOSED:
return "VM_CLOSED";
case XE_PAGEFAULT_ERROR_VMA_NOT_FOUND:
return "VMA_NOT_FOUND";
case XE_PAGEFAULT_ERROR_READ_ONLY_VIOLATION:
return "READ_ONLY_VIOLATION";
case XE_PAGEFAULT_ERROR_VMA_NEEDS_VRAM_CHECK:
return "VMA_NEEDS_VRAM_CHECK";
case XE_PAGEFAULT_ERROR_VMA_ATOMIC_USERPTR:
return "VMA_ATOMIC_USERPTR";
case XE_PAGEFAULT_ERROR_VMA_USERPTR_PIN:
return "VMA_USERPTR_PIN";
case XE_PAGEFAULT_ERROR_VMA_VALIDATE:
return "VMA_VALIDATE";
case XE_PAGEFAULT_ERROR_VMA_REBIND:
return "VMA_REBIND";
case XE_PAGEFAULT_ERROR_SVM_GARBAGE_COLLECTOR:
return "SVM_GARBAGE_COLLECTOR";
case XE_PAGEFAULT_ERROR_SVM_RANGE_NOT_FOUND:
return "SVM_RANGE_NOT_FOUND";
case XE_PAGEFAULT_ERROR_SVM_REBIND:
return "SVM_REBIND";
case XE_PAGEFAULT_ERROR_SVM_NEEDS_VRAM_CHECK:
return "SVM_NEEDS_VRAM_CHECK";
case XE_PAGEFAULT_ERROR_SVM_VMA_NOT_FOUND:
return "SVM_VMA_NOT_FOUND";
case XE_PAGEFAULT_ERROR_SVM_SERVICE_FAILED:
return "SVM_SERVICE_FAILED";
default:
return "UNKNOWN";
}
}
static void xe_pagefault_print(struct xe_pagefault *pf)
{
enum xe_pagefault_error err = xe_pagefault_get_error(pf);
if (WARN_ON_ONCE(!pf->producer.ops->print))
return;
pf->producer.ops->print(pf, xe_pagefault_error_to_str(err));
}
static void xe_pagefault_save_to_vm(struct xe_device *xe, struct xe_pagefault *pf)
{
struct xe_vm *vm;
down_read(&xe->usm.lock);
vm = xa_load(&xe->usm.asid_to_vm,
FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id));
if (vm)
xe_vm_get(vm);
else
vm = ERR_PTR(-EINVAL);
up_read(&xe->usm.lock);
if (IS_ERR(vm))
return;
xe_vm_add_fault_entry_pf(vm, pf);
xe_vm_put(vm);
}
static void xe_pagefault_queue_work(struct work_struct *w)
{
struct xe_pagefault_work *pf_work =
container_of(w, typeof(*pf_work), work);
struct xe_device *xe = pf_work->xe;
struct xe_pagefault_queue *pf_queue = &xe->usm.pf_queue;
struct xe_pagefault *pf;
ktime_t start = xe_gt_stats_ktime_get();
unsigned long threshold;
u64 cache_start = XE_PAGEFAULT_CACHE_START_INVALID, cache_end = 0;
u32 cache_asid = 0;
guard(xe_pm_runtime)(xe);
guard(xe_pm_runtime)(xe);
#define USM_QUEUE_MAX_RUNTIME_MS 20
threshold = jiffies + msecs_to_jiffies(USM_QUEUE_MAX_RUNTIME_MS);
while (xe_pagefault_queue_pop(pf_queue, &pf, pf_work->id)) {
const struct xe_pagefault_ops *ops = pf->producer.ops;
void *private = pf->producer.private;
struct xe_gt *gt = pf->gt;
u32 asid = FIELD_GET(XE_PAGEFAULT_ASID_MASK, pf->consumer.id);
int err = 0;
bool invalidated = false;
if (xe_pagefault_match(pf, cache_start, cache_end, cache_asid)) {
xe_gt_stats_incr(gt, XE_GT_STATS_ID_LAST_PAGEFAULT_COUNT, 1);
goto ack_fault;
}
err = xe_pagefault_service(pf);
if (err) {
if (!(pf->consumer.access_type & XE_PAGEFAULT_ACCESS_PREFETCH)) {
xe_pagefault_save_to_vm(gt_to_xe(gt), pf);
xe_pagefault_cache_start_invalidate(cache_start);
xe_pagefault_print(pf);
xe_log_err_info(pf->gt, PAGEFAULT, err, "Unsuccessful response\n");
} else {
xe_gt_stats_incr(pf->gt, XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT, 1);
xe_gt_dbg(pf->gt, "Prefetch Fault response: Unsuccessful %pe\n",
ERR_PTR(err));
}
} else {
cache_start = xe_pagefault_start_addr(pf);
cache_end = xe_pagefault_end_addr(pf);
cache_asid = asid;
}
ack_fault:
xe_assert(xe, pf->consumer.alloc_state ==
XE_PAGEFAULT_ALLOC_STATE_ACTIVE);
xe_assert(xe, pf == pf_work->cache.pf);
ops->ack_fault_begin(private);
while (pf) {
xe_assert(xe, pf->consumer.alloc_state ==
XE_PAGEFAULT_ALLOC_STATE_ACTIVE);
xe_assert(xe, ops == pf->producer.ops);
xe_assert(xe, gt == pf->gt);
ops->ack_fault(pf, err);
spin_lock_irq(&pf_queue->lock);
if (!invalidated) {
invalidated = true;
xe_pagefault_cache_invalidate(pf_queue,
pf_work);
}
pf->consumer.alloc_state = XE_PAGEFAULT_ALLOC_STATE_FREE;
pf = pf->consumer.next;
while (pf && !xe_pagefault_match(pf, cache_start,
cache_end, cache_asid))
pf = xe_pagefault_queue_unchain_requeue(pf_queue, pf, gt);
if (pf)
pf->consumer.alloc_state =
XE_PAGEFAULT_ALLOC_STATE_ACTIVE;
spin_unlock_irq(&pf_queue->lock);
}
ops->ack_fault_end(private);
if (time_after(jiffies, threshold)) {
queue_work(xe->usm.pagefault_wq, w);
break;
}
}
#undef USM_QUEUE_MAX_RUNTIME_MS
xe_gt_stats_incr(xe_root_mmio_gt(xe), XE_GT_STATS_ID_PAGEFAULT_US,
xe_gt_stats_ktime_us_delta(start));
}
static int xe_pagefault_queue_init(struct xe_device *xe,
struct xe_pagefault_queue *pf_queue)
{
struct xe_gt *gt;
int total_num_eus = 0;
u8 id;
for_each_gt(gt, xe, id) {
xe_dss_mask_t all_dss;
int num_dss, num_eus;
num_dss = bitmap_weighted_or(all_dss, gt->fuse_topo.g_dss_mask,
gt->fuse_topo.c_dss_mask, XE_MAX_DSS_FUSE_BITS);
num_eus = bitmap_weight(gt->fuse_topo.eu_mask_per_dss,
XE_MAX_EU_FUSE_BITS) * num_dss;
total_num_eus += num_eus;
}
xe_assert(xe, total_num_eus);
#define PF_MULTIPLIER 8
pf_queue->size = (total_num_eus + XE_NUM_HW_ENGINES) *
xe_pagefault_entry_size() * PF_MULTIPLIER;
pf_queue->size = roundup_pow_of_two(pf_queue->size);
#undef PF_MULTIPLIER
drm_dbg(&xe->drm, "xe_pagefault_entry_size=%d, total_num_eus=%d, pf_queue->size=%u",
xe_pagefault_entry_size(), total_num_eus, pf_queue->size);
pf_queue->xe = xe;
spin_lock_init(&pf_queue->lock);
pf_queue->data = drmm_kzalloc(&xe->drm, pf_queue->size, GFP_KERNEL);
if (!pf_queue->data)
return -ENOMEM;
return 0;
}
static void xe_pagefault_fini(void *arg)
{
struct xe_device *xe = arg;
destroy_workqueue(xe->usm.prefetch_wq);
destroy_workqueue(xe->usm.pagefault_wq);
}
int xe_pagefault_init(struct xe_device *xe)
{
int err, i;
if (!xe->info.has_usm)
return 0;
xe->usm.pagefault_wq = alloc_workqueue("xe_page_fault_work_queue",
WQ_UNBOUND | WQ_HIGHPRI,
xe->info.num_pf_work);
if (!xe->usm.pagefault_wq)
return -ENOMEM;
xe->usm.prefetch_wq = alloc_workqueue("xe_prefetch_work_queue",
WQ_UNBOUND,
xe->info.num_pf_work);
if (!xe->usm.prefetch_wq) {
err = -ENOMEM;
goto err_pagefault_wq;
}
err = xe_pagefault_queue_init(xe, &xe->usm.pf_queue);
if (err)
goto err_out;
for (i = 0; i < xe->info.num_pf_work; ++i) {
struct xe_pagefault_work *pf_work = xe->usm.pf_workers + i;
pf_work->xe = xe;
pf_work->id = i;
xe_pagefault_cache_start_invalidate(pf_work->cache.start);
INIT_WORK(&pf_work->work, xe_pagefault_queue_work);
}
return devm_add_action_or_reset(xe->drm.dev, xe_pagefault_fini, xe);
err_out:
destroy_workqueue(xe->usm.prefetch_wq);
err_pagefault_wq:
destroy_workqueue(xe->usm.pagefault_wq);
return err;
}
static void xe_pagefault_queue_reset(struct xe_device *xe, struct xe_gt *gt,
struct xe_pagefault_queue *pf_queue)
{
u32 i;
if (!pf_queue->data)
return;
guard(spinlock_irq)(&pf_queue->lock);
for (i = 0; i < pf_queue->size; i += xe_pagefault_entry_size()) {
struct xe_pagefault *pf = pf_queue->data + i;
bool active = pf->consumer.alloc_state ==
XE_PAGEFAULT_ALLOC_STATE_ACTIVE;
if (pf->gt != gt || active) {
if (active)
pf->consumer.next = NULL;
continue;
}
pf->consumer.alloc_state =
XE_PAGEFAULT_ALLOC_STATE_FREE;
pf->consumer.next = NULL;
}
}
void xe_pagefault_reset(struct xe_device *xe, struct xe_gt *gt)
{
xe_pagefault_queue_reset(xe, gt, &xe->usm.pf_queue);
}
static int xe_pagefault_work_index(struct xe_device *xe)
{
lockdep_assert_held(&xe->usm.pf_queue.lock);
return xe->usm.current_pf_work++ % xe->info.num_pf_work;
}
int xe_pagefault_handler(struct xe_device *xe, struct xe_pagefault *pf)
{
struct xe_pagefault_queue *pf_queue = &xe->usm.pf_queue;
struct xe_pagefault *lpf;
bool empty;
guard(spinlock_irqsave)(&pf_queue->lock);
empty = xe_pagefault_queue_empty(pf_queue);
lpf = xe_pagefault_queue_add(pf_queue, pf);
if (!lpf)
return -ENOSPC;
lpf->consumer.next = NULL;
if (xe_pagefault_try_chain(pf_queue, lpf)) {
xe_gt_stats_incr(pf->gt, XE_GT_STATS_ID_CHAIN_IRQ_PAGEFAULT_COUNT, 1);
if (empty) {
xe_gt_stats_incr(pf->gt,
XE_GT_STATS_ID_CHAIN_DRAIN_IRQ_PAGEFAULT_COUNT, 1);
xe_pagefault_queue_advance(pf_queue);
}
} else {
int work_index = xe_pagefault_work_index(xe);
queue_work(xe->usm.pagefault_wq,
&xe->usm.pf_workers[work_index].work);
}
return 0;
}
void xe_pagefault_print_info(struct xe_device *xe, struct drm_printer *p)
{
struct xe_pagefault_queue *pf_queue = &xe->usm.pf_queue;
struct xe_pagefault_work *pf_work;
static const char * const alloc_state_names[] = {
[XE_PAGEFAULT_ALLOC_STATE_FREE] = "free",
[XE_PAGEFAULT_ALLOC_STATE_QUEUED] = "queued",
[XE_PAGEFAULT_ALLOC_STATE_CHAINED] = "chained",
[XE_PAGEFAULT_ALLOC_STATE_ACTIVE] = "active",
};
u32 i, counts[XE_PAGEFAULT_ALLOC_STATE_COUNT] = {};
if (!pf_queue->data)
return;
guard(spinlock_irq)(&pf_queue->lock);
drm_printf(p, "pagefault size: %u\n", xe_pagefault_entry_size());
drm_printf(p, "pagefault queue size: %u\n", pf_queue->size);
drm_printf(p, "pagefault queue head: %u\n", pf_queue->head);
drm_printf(p, "pagefault queue tail: %u\n", pf_queue->tail);
for (i = 0; i < pf_queue->size; i += xe_pagefault_entry_size()) {
struct xe_pagefault *pf = pf_queue->data + i;
if (pf->consumer.alloc_state >=
XE_PAGEFAULT_ALLOC_STATE_COUNT) {
drm_printf(p, "pagefault[%u] corrupted alloc_state=%u\n",
i, pf->consumer.alloc_state);
continue;
}
counts[pf->consumer.alloc_state]++;
}
for (i = 0; i < XE_PAGEFAULT_ALLOC_STATE_COUNT; ++i)
drm_printf(p, "pagefault queue %s count: %u\n",
alloc_state_names[i], counts[i]);
for (i = 0, pf_work = xe->usm.pf_workers;
i < xe->info.num_pf_work; ++i, ++pf_work) {
if (pf_work->cache.start == XE_PAGEFAULT_CACHE_START_INVALID)
drm_printf(p, "pagefault work[%u] cache invalid\n", i);
else
drm_printf(p, "pagefault work[%u] cache valid\n", i);
}
}