#include <linux/export.h>
#include <linux/slab.h>
#include <linux/completion.h>
#include <drm/drm_print.h>
#include <drm/gpu_scheduler.h>
#include "sched_internal.h"
#include "gpu_scheduler_trace.h"
void drm_sched_entity_stats_release(struct kref *kref)
{
struct drm_sched_entity_stats *stats =
container_of(kref, typeof(*stats), kref);
kfree(stats);
}
static struct drm_sched_entity_stats *drm_sched_entity_stats_new(void)
{
struct drm_sched_entity_stats *stats;
stats = kzalloc_obj(*stats);
if (!stats)
return NULL;
kref_init(&stats->kref);
spin_lock_init(&stats->lock);
ewma_drm_sched_avgtime_init(&stats->avg_job_us);
return stats;
}
ktime_t drm_sched_entity_stats_job_add_gpu_time(struct drm_sched_job *job)
{
struct drm_sched_entity_stats *stats = job->entity_stats;
struct drm_sched_fence *s_fence = job->s_fence;
ktime_t start, end, duration;
start = dma_fence_timestamp(&s_fence->scheduled);
end = dma_fence_timestamp(&s_fence->finished);
duration = ktime_sub(end, start);
spin_lock(&stats->lock);
stats->runtime = ktime_add(stats->runtime, duration);
ewma_drm_sched_avgtime_add(&stats->avg_job_us, ktime_to_us(duration));
spin_unlock(&stats->lock);
return duration;
}
int drm_sched_entity_init(struct drm_sched_entity *entity,
enum drm_sched_priority priority,
struct drm_gpu_scheduler **sched_list,
unsigned int num_sched_list,
atomic_t *guilty)
{
if (!entity || !sched_list || !num_sched_list || !sched_list[0])
return -EINVAL;
memset(entity, 0, sizeof(struct drm_sched_entity));
entity->stats = drm_sched_entity_stats_new();
if (!entity->stats)
return -ENOMEM;
INIT_LIST_HEAD(&entity->list);
entity->guilty = guilty;
entity->last_user = current->group_leader;
entity->num_sched_list = num_sched_list;
entity->sched_list = num_sched_list > 1 ? sched_list : NULL;
RB_CLEAR_NODE(&entity->rb_tree_node);
if (!sched_list[0]->sched_rq) {
pr_warn("%s: called with uninitialized scheduler\n", __func__);
} else if (priority >= sched_list[0]->num_user_rqs) {
dev_err(sched_list[0]->dev, "entity with out-of-bounds priority:%u num_user_rqs:%u\n",
priority, sched_list[0]->num_user_rqs);
priority = max_t(s32,
(s32)sched_list[0]->num_user_rqs - 1,
(s32)DRM_SCHED_PRIORITY_KERNEL);
}
entity->priority = priority;
entity->rq_priority = drm_sched_policy == DRM_SCHED_POLICY_FAIR ?
DRM_SCHED_PRIORITY_KERNEL : priority;
if (sched_list[0]->sched_rq)
entity->rq = sched_list[0]->sched_rq[entity->rq_priority];
init_completion(&entity->entity_idle);
complete_all(&entity->entity_idle);
spin_lock_init(&entity->lock);
spsc_queue_init(&entity->job_queue);
atomic_set(&entity->fence_seq, 0);
entity->fence_context = dma_fence_context_alloc(2);
return 0;
}
EXPORT_SYMBOL(drm_sched_entity_init);
void drm_sched_entity_modify_sched(struct drm_sched_entity *entity,
struct drm_gpu_scheduler **sched_list,
unsigned int num_sched_list)
{
WARN_ON(!num_sched_list || !sched_list);
spin_lock(&entity->lock);
entity->sched_list = sched_list;
entity->num_sched_list = num_sched_list;
spin_unlock(&entity->lock);
}
EXPORT_SYMBOL(drm_sched_entity_modify_sched);
static bool drm_sched_entity_is_idle(struct drm_sched_entity *entity)
{
bool idle = false;
spin_lock(&entity->lock);
if (list_empty(&entity->list) ||
spsc_queue_count(&entity->job_queue) == 0 ||
entity->stopped)
idle = true;
spin_unlock(&entity->lock);
return idle;
}
int drm_sched_entity_error(struct drm_sched_entity *entity)
{
struct dma_fence *fence;
int r;
spin_lock(&entity->lock);
fence = entity->last_scheduled;
r = fence ? fence->error : 0;
spin_unlock(&entity->lock);
return r;
}
EXPORT_SYMBOL(drm_sched_entity_error);
static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
struct dma_fence_cb *cb);
static void drm_sched_entity_kill_jobs_work(struct work_struct *wrk)
{
struct drm_sched_job *job = container_of(wrk, typeof(*job), work);
struct dma_fence *f;
unsigned long index;
xa_for_each(&job->dependencies, index, f) {
struct drm_sched_fence *s_fence = to_drm_sched_fence(f);
if (s_fence && f == &s_fence->scheduled) {
f = dma_fence_get_rcu(&s_fence->finished);
dma_fence_put(&s_fence->scheduled);
}
xa_erase(&job->dependencies, index);
if (f && !dma_fence_add_callback(f, &job->finish_cb,
drm_sched_entity_kill_jobs_cb))
return;
dma_fence_put(f);
}
drm_sched_fence_scheduled(job->s_fence, NULL);
drm_sched_fence_finished(job->s_fence, -ESRCH);
WARN_ON(job->s_fence->parent);
job->sched->ops->free_job(job);
}
static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
struct dma_fence_cb *cb)
{
struct drm_sched_job *job = container_of(cb, struct drm_sched_job,
finish_cb);
dma_fence_put(f);
INIT_WORK(&job->work, drm_sched_entity_kill_jobs_work);
schedule_work(&job->work);
}
void drm_sched_entity_kill(struct drm_sched_entity *entity)
{
struct drm_sched_job *job;
struct dma_fence *prev;
spin_lock(&entity->lock);
entity->stopped = true;
drm_sched_rq_remove_entity(entity->rq, entity);
spin_unlock(&entity->lock);
wait_for_completion(&entity->entity_idle);
spin_lock(&entity->lock);
prev = dma_fence_get(entity->last_scheduled);
spin_unlock(&entity->lock);
while ((job = drm_sched_entity_queue_pop(entity))) {
struct drm_sched_fence *s_fence = job->s_fence;
dma_fence_get(&s_fence->finished);
if (!prev ||
dma_fence_add_callback(prev, &job->finish_cb,
drm_sched_entity_kill_jobs_cb)) {
dma_fence_put(prev);
drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb);
}
prev = &s_fence->finished;
}
dma_fence_put(prev);
}
EXPORT_SYMBOL(drm_sched_entity_kill);
long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout)
{
struct drm_gpu_scheduler *sched;
struct task_struct *last_user;
long ret = timeout;
sched = entity->rq->sched;
if (current->flags & PF_EXITING) {
if (timeout)
ret = wait_event_timeout(
sched->job_scheduled,
drm_sched_entity_is_idle(entity),
timeout);
} else {
wait_event_killable(sched->job_scheduled,
drm_sched_entity_is_idle(entity));
}
last_user = cmpxchg(&entity->last_user, current->group_leader, NULL);
if (last_user == current->group_leader &&
(current->flags & PF_EXITING) && (current->exit_code == SIGKILL))
drm_sched_entity_kill(entity);
return ret;
}
EXPORT_SYMBOL(drm_sched_entity_flush);
void drm_sched_entity_fini(struct drm_sched_entity *entity)
{
drm_sched_entity_kill(entity);
if (entity->dependency) {
dma_fence_remove_callback(entity->dependency, &entity->cb);
dma_fence_put(entity->dependency);
entity->dependency = NULL;
}
dma_fence_put(entity->last_scheduled);
drm_sched_entity_stats_put(entity->stats);
}
EXPORT_SYMBOL(drm_sched_entity_fini);
void drm_sched_entity_destroy(struct drm_sched_entity *entity)
{
drm_sched_entity_flush(entity, MAX_WAIT_SCHED_ENTITY_Q_EMPTY);
drm_sched_entity_fini(entity);
}
EXPORT_SYMBOL(drm_sched_entity_destroy);
static void drm_sched_entity_wakeup(struct dma_fence *f,
struct dma_fence_cb *cb)
{
struct drm_sched_entity *entity =
container_of(cb, struct drm_sched_entity, cb);
entity->dependency = NULL;
dma_fence_put(f);
drm_sched_wakeup(entity->rq->sched);
}
void drm_sched_entity_set_priority(struct drm_sched_entity *entity,
enum drm_sched_priority priority)
{
spin_lock(&entity->lock);
entity->priority = priority;
if (drm_sched_policy != DRM_SCHED_POLICY_FAIR)
entity->rq_priority = priority;
spin_unlock(&entity->lock);
}
EXPORT_SYMBOL(drm_sched_entity_set_priority);
static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity,
struct drm_sched_job *sched_job)
{
struct drm_gpu_scheduler *sched = entity->rq->sched;
struct dma_fence *fence = entity->dependency;
struct drm_sched_fence *s_fence;
if (fence->context == entity->fence_context ||
fence->context == entity->fence_context + 1) {
dma_fence_put(entity->dependency);
return false;
}
s_fence = to_drm_sched_fence(fence);
if (!fence->error && s_fence && s_fence->sched == sched &&
!test_bit(DRM_SCHED_FENCE_DONT_PIPELINE, &fence->flags)) {
fence = dma_fence_get(&s_fence->scheduled);
dma_fence_put(entity->dependency);
entity->dependency = fence;
}
if (trace_drm_sched_job_unschedulable_enabled() &&
!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &entity->dependency->flags))
trace_drm_sched_job_unschedulable(sched_job, entity->dependency);
if (!dma_fence_add_callback(entity->dependency, &entity->cb,
drm_sched_entity_wakeup))
return true;
dma_fence_put(entity->dependency);
return false;
}
static struct dma_fence *
drm_sched_job_dependency(struct drm_sched_job *job,
struct drm_sched_entity *entity)
{
struct dma_fence *f;
f = xa_load(&job->dependencies, job->last_dependency);
if (f) {
job->last_dependency++;
return dma_fence_get(f);
}
if (job->sched->ops->prepare_job)
return job->sched->ops->prepare_job(job, entity);
return NULL;
}
struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity)
{
struct dma_fence *prev_last_scheduled;
struct drm_sched_job *sched_job;
sched_job = drm_sched_entity_queue_peek(entity);
if (!sched_job)
return NULL;
while ((entity->dependency =
drm_sched_job_dependency(sched_job, entity))) {
if (drm_sched_entity_add_dependency_cb(entity, sched_job))
return NULL;
}
if (entity->guilty && atomic_read(entity->guilty))
dma_fence_set_error(&sched_job->s_fence->finished, -ECANCELED);
spin_lock(&entity->lock);
prev_last_scheduled = entity->last_scheduled;
entity->last_scheduled = dma_fence_get(&sched_job->s_fence->finished);
spsc_queue_pop(&entity->job_queue);
drm_sched_rq_pop_entity(entity);
spin_unlock(&entity->lock);
dma_fence_put(prev_last_scheduled);
sched_job->entity = NULL;
return sched_job;
}
void drm_sched_entity_select_rq(struct drm_sched_entity *entity)
{
struct dma_fence *fence;
struct drm_gpu_scheduler *sched;
struct drm_sched_rq *rq;
if (!entity->sched_list)
return;
if (spsc_queue_count(&entity->job_queue))
return;
spin_lock(&entity->lock);
fence = entity->last_scheduled;
if (fence && !dma_fence_is_signaled(fence)) {
spin_unlock(&entity->lock);
return;
}
sched = drm_sched_pick_best(entity->sched_list, entity->num_sched_list);
rq = sched ? sched->sched_rq[entity->rq_priority] : NULL;
if (rq != entity->rq) {
drm_sched_rq_remove_entity(entity->rq, entity);
entity->rq = rq;
}
if (entity->num_sched_list == 1)
entity->sched_list = NULL;
spin_unlock(&entity->lock);
}
void drm_sched_entity_push_job(struct drm_sched_job *sched_job)
{
struct drm_sched_entity *entity = sched_job->entity;
bool first;
ktime_t submit_ts;
trace_drm_sched_job_queue(sched_job, entity);
if (trace_drm_sched_job_add_dep_enabled()) {
struct dma_fence *entry;
unsigned long index;
xa_for_each(&sched_job->dependencies, index, entry)
trace_drm_sched_job_add_dep(sched_job, entry);
}
atomic_inc(entity->rq->sched->score);
WRITE_ONCE(entity->last_user, current->group_leader);
sched_job->submit_ts = submit_ts = ktime_get();
spin_lock(&entity->lock);
first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
if (first) {
struct drm_gpu_scheduler *sched;
sched = drm_sched_rq_add_entity(entity, submit_ts);
if (sched)
drm_sched_wakeup(sched);
}
spin_unlock(&entity->lock);
}
EXPORT_SYMBOL