#include <drm/drm_managed.h>
#include <linux/mutex.h>
#include "xe_cpu_bind.h"
#include "xe_device_types.h"
#include "xe_exec_queue.h"
#include "xe_pt.h"
#include "xe_sched_job.h"
#include "xe_trace_bo.h"
#include "xe_vm.h"
struct xe_cpu_bind {
struct xe_device *xe;
struct xe_exec_queue *q;
struct mutex job_mutex;
};
static bool is_cpu_bind_queue(struct xe_cpu_bind *cpu_bind,
struct xe_exec_queue *q)
{
return cpu_bind->q == q;
}
static void xe_cpu_bind_fini(void *arg)
{
struct xe_cpu_bind *cpu_bind = arg;
mutex_destroy(&cpu_bind->job_mutex);
xe_exec_queue_put(cpu_bind->q);
}
int xe_cpu_bind_init(struct xe_device *xe)
{
struct xe_cpu_bind *cpu_bind =
drmm_kzalloc(&xe->drm, sizeof(*cpu_bind), GFP_KERNEL);
struct xe_exec_queue *q;
if (!cpu_bind)
return -ENOMEM;
q = xe_exec_queue_create_bind(xe, xe_device_get_root_tile(xe), NULL,
EXEC_QUEUE_FLAG_KERNEL |
EXEC_QUEUE_FLAG_MIGRATE, 0);
if (IS_ERR(q))
return PTR_ERR(q);
cpu_bind->xe = xe;
cpu_bind->q = q;
xe->cpu_bind = cpu_bind;
mutex_init(&cpu_bind->job_mutex);
fs_reclaim_acquire(GFP_KERNEL);
might_lock(&cpu_bind->job_mutex);
fs_reclaim_release(GFP_KERNEL);
return devm_add_action_or_reset(cpu_bind->xe->drm.dev, xe_cpu_bind_fini,
cpu_bind);
}
struct xe_exec_queue *xe_cpu_bind_queue(struct xe_cpu_bind *cpu_bind)
{
return cpu_bind->q;
}
void
xe_cpu_bind_update_pgtables_execute(struct xe_vm *vm, struct xe_tile *tile,
const struct xe_cpu_bind_pt_update_ops *ops,
struct xe_vm_pgtable_update_op *pt_op,
u32 num_ops, bool force_clear)
{
u32 j, i;
for (j = 0; j < num_ops; ++j, ++pt_op) {
for (i = 0; i < pt_op->num_entries; i++) {
const struct xe_vm_pgtable_update *update =
&pt_op->entries[i];
xe_assert(vm->xe, !iosys_map_is_null(&update->pt_bo->vmap));
if (pt_op->bind && !force_clear)
ops->populate(tile, &update->pt_bo->vmap,
update);
else
ops->clear(vm, tile, &update->pt_bo->vmap,
update);
}
}
trace_xe_vm_cpu_bind(vm);
xe_device_wmb(vm->xe);
}
static struct dma_fence *
xe_cpu_bind_update_pgtables_no_job(struct xe_cpu_bind *cpu_bind,
struct xe_cpu_bind_pt_update *pt_update)
{
const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops;
struct xe_vm *vm = pt_update->vops->vm;
struct xe_tile *tile;
int err, id;
if (ops->pre_commit) {
pt_update->job = NULL;
err = ops->pre_commit(pt_update);
if (err)
return ERR_PTR(err);
}
for_each_tile(tile, vm->xe, id) {
struct xe_vm_pgtable_update_ops *pt_update_ops =
&pt_update->vops->pt_update_ops[tile->id];
if (!pt_update_ops->pt_job_ops)
continue;
xe_cpu_bind_update_pgtables_execute(vm, tile, ops,
pt_update_ops->pt_job_ops->ops,
pt_update_ops->pt_job_ops->current_op,
false);
}
return dma_fence_get_stub();
}
static struct dma_fence *
xe_cpu_bind_update_pgtables_job(struct xe_cpu_bind *cpu_bind,
struct xe_cpu_bind_pt_update *pt_update)
{
const struct xe_cpu_bind_pt_update_ops *ops = pt_update->ops;
struct xe_exec_queue *q = pt_update->vops->q;
struct xe_device *xe = cpu_bind->xe;
struct xe_sched_job *job;
struct dma_fence *fence;
struct xe_tile *tile;
int err, id;
bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
job = xe_sched_job_create(q, NULL);
if (IS_ERR(job))
return ERR_CAST(job);
xe_assert(xe, job->is_pt_job);
if (ops->pre_commit) {
pt_update->job = job;
err = ops->pre_commit(pt_update);
if (err)
goto err_job;
}
if (is_cpu_bind)
mutex_lock(&cpu_bind->job_mutex);
job->pt_update[0].vm = pt_update->vops->vm;
job->pt_update[0].ops = ops;
for_each_tile(tile, xe, id) {
struct xe_vm_pgtable_update_ops *pt_update_ops =
&pt_update->vops->pt_update_ops[tile->id];
job->pt_update[0].pt_job_ops[tile->id] =
xe_pt_job_ops_get(pt_update_ops->pt_job_ops);
}
xe_sched_job_arm(job);
fence = dma_fence_get(&job->drm.s_fence->finished);
xe_sched_job_push(job);
if (is_cpu_bind)
mutex_unlock(&cpu_bind->job_mutex);
return fence;
err_job:
xe_sched_job_put(job);
return ERR_PTR(err);
}
struct dma_fence *
xe_cpu_bind_update_pgtables(struct xe_cpu_bind *cpu_bind,
struct xe_cpu_bind_pt_update *pt_update)
{
struct dma_fence *fence;
fence = xe_cpu_bind_update_pgtables_no_job(cpu_bind, pt_update);
if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME)
return fence;
return xe_cpu_bind_update_pgtables_job(cpu_bind, pt_update);
}
void xe_cpu_bind_job_lock(struct xe_cpu_bind *cpu_bind,
struct xe_exec_queue *q)
{
bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
if (is_cpu_bind)
mutex_lock(&cpu_bind->job_mutex);
else
xe_vm_assert_held(q->user_vm);
}
void xe_cpu_bind_job_unlock(struct xe_cpu_bind *cpu_bind,
struct xe_exec_queue *q)
{
bool is_cpu_bind = is_cpu_bind_queue(cpu_bind, q);
if (is_cpu_bind)
mutex_unlock(&cpu_bind->job_mutex);
else
xe_vm_assert_held(q->user_vm);
}
#if IS_ENABLED(CONFIG_PROVE_LOCKING)
void xe_cpu_bind_job_lock_assert(struct xe_exec_queue *q)
{
struct xe_device *xe = gt_to_xe(q->gt);
struct xe_cpu_bind *cpu_bind = xe->cpu_bind;
xe_assert(xe, q == cpu_bind->q);
lockdep_assert_held(&cpu_bind->job_mutex);
}
#endif