diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index b57bafc42feb..672b5c57ed8e 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -59,6 +59,7 @@ static struct drm_sched_entity_stats *drm_sched_entity_stats_new(void) kref_init(&stats->kref); spin_lock_init(&stats->lock); + ewma_drm_sched_avgtime_init(&stats->avg_job_us); return stats; } @@ -68,19 +69,25 @@ static struct drm_sched_entity_stats *drm_sched_entity_stats_new(void) * @job: Scheduler job to account. * * Accounts the execution time of @job to its respective entity stats object. + * + * Return: Job's real duration in micro seconds. */ -void drm_sched_entity_stats_job_add_gpu_time(struct drm_sched_job *job) +ktime_t drm_sched_entity_stats_job_add_gpu_time(struct drm_sched_job *job) { struct drm_sched_entity_stats *stats = job->entity_stats; struct drm_sched_fence *s_fence = job->s_fence; - ktime_t start, end; + ktime_t start, end, duration; start = dma_fence_timestamp(&s_fence->scheduled); end = dma_fence_timestamp(&s_fence->finished); + duration = ktime_sub(end, start); spin_lock(&stats->lock); - stats->runtime = ktime_add(stats->runtime, ktime_sub(end, start)); + stats->runtime = ktime_add(stats->runtime, duration); + ewma_drm_sched_avgtime_add(&stats->avg_job_us, ktime_to_us(duration)); spin_unlock(&stats->lock); + + return duration; } /** diff --git a/drivers/gpu/drm/scheduler/sched_internal.h b/drivers/gpu/drm/scheduler/sched_internal.h index 64acd4b7a918..a901801fce85 100644 --- a/drivers/gpu/drm/scheduler/sched_internal.h +++ b/drivers/gpu/drm/scheduler/sched_internal.h @@ -14,6 +14,7 @@ * @runtime: time entity spent on the GPU. * @prev_runtime: previous @runtime used to get the runtime delta. * @vruntime: virtual runtime as accumulated by the fair algorithm. + * @avg_job_us: average job duration. * * Because jobs and entities have decoupled lifetimes, ie. we cannot access the * entity once the job has been de-queued, and we do need know how much GPU time @@ -26,6 +27,8 @@ struct drm_sched_entity_stats { ktime_t runtime; ktime_t prev_runtime; ktime_t vruntime; + + struct ewma_drm_sched_avgtime avg_job_us; }; /* Used to choose between FIFO and RR job-scheduling */ @@ -146,6 +149,6 @@ drm_sched_entity_stats_put(struct drm_sched_entity_stats *stats) kref_put(&stats->kref, drm_sched_entity_stats_release); } -void drm_sched_entity_stats_job_add_gpu_time(struct drm_sched_job *job); +ktime_t drm_sched_entity_stats_job_add_gpu_time(struct drm_sched_job *job); #endif diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index 0aca41b4e334..337db7b1e688 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -1004,7 +1004,12 @@ static void drm_sched_free_job_work(struct work_struct *w) struct drm_sched_job *job; while ((job = drm_sched_get_finished_job(sched))) { - drm_sched_entity_stats_job_add_gpu_time(job); + ktime_t duration = drm_sched_entity_stats_job_add_gpu_time(job); + + /* Serialized by the worker. */ + ewma_drm_sched_avgtime_add(&sched->avg_job_us, + ktime_to_us(duration)); + sched->ops->free_job(job); } @@ -1165,6 +1170,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, const struct drm_sched_init_ atomic_set(&sched->_score, 0); atomic64_set(&sched->job_id_count, 0); sched->pause_submit = false; + ewma_drm_sched_avgtime_init(&sched->avg_job_us); sched->ready = true; return 0; diff --git a/drivers/gpu/drm/scheduler/sched_rq.c b/drivers/gpu/drm/scheduler/sched_rq.c index 8fec2d0c5a43..0464d324d98d 100644 --- a/drivers/gpu/drm/scheduler/sched_rq.c +++ b/drivers/gpu/drm/scheduler/sched_rq.c @@ -165,13 +165,21 @@ drm_sched_entity_restore_vruntime(struct drm_sched_entity *entity, enum drm_sched_priority rq_prio) { struct drm_sched_entity_stats *stats = entity->stats; + struct drm_gpu_scheduler *sched = entity->rq->sched; enum drm_sched_priority prio = entity->priority; + unsigned long avg_us, sched_avg_us; ktime_t vruntime; BUILD_BUG_ON(DRM_SCHED_PRIORITY_NORMAL < DRM_SCHED_PRIORITY_HIGH); spin_lock(&stats->lock); vruntime = stats->vruntime; + avg_us = ewma_drm_sched_avgtime_read(&stats->avg_job_us); + /* + * Unlocked read of the scheduler average is fine since it is just + * heuristics and data type is a natural word size. + */ + sched_avg_us = ewma_drm_sched_avgtime_read(&sched->avg_job_us); /* * Special handling for entities which were picked from the top of the @@ -181,14 +189,24 @@ drm_sched_entity_restore_vruntime(struct drm_sched_entity *entity, if (prio > rq_prio) { /* * Lower priority should not overtake higher when re- - * joining at the top of the queue. + * joining at the top of the queue so push it back + * somewhere behind the "middle" of the run-queue, + * proportional to the scheduler and entity average job + * durations. */ - vruntime = ns_to_ktime(prio - rq_prio); + vruntime = us_to_ktime((1 + avg_us + sched_avg_us) << + vruntime_shift[prio]); } else if (prio < rq_prio) { /* * Higher priority can go first. */ vruntime = -ns_to_ktime(rq_prio - prio); + } else { + /* Favour entity with shorter jobs (interactivity). */ + if (avg_us <= sched_avg_us) + vruntime = -ns_to_ktime(1); + else + vruntime = ns_to_ktime(1); } } diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 69c2097664fd..790f7ecb6b85 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -25,11 +25,14 @@ #define _DRM_GPU_SCHEDULER_H_ #include +#include #include #include #include #include +DECLARE_EWMA(drm_sched_avgtime, 6, 4); + #define MAX_WAIT_SCHED_ENTITY_Q_EMPTY msecs_to_jiffies(1000) /** @@ -582,6 +585,7 @@ struct drm_sched_backend_ops { * @job_id_count: used to assign unique id to the each job. * @submit_wq: workqueue used to queue @work_run_job and @work_free_job * @timeout_wq: workqueue used to queue @work_tdr + * @avg_job_us: Average job duration. * @work_run_job: work which calls run_job op of each scheduler. * @work_free_job: work which calls free_job op of each scheduler. * @work_tdr: schedules a delayed call to @drm_sched_job_timedout after the @@ -613,6 +617,7 @@ struct drm_gpu_scheduler { atomic64_t job_id_count; struct workqueue_struct *submit_wq; struct workqueue_struct *timeout_wq; + struct ewma_drm_sched_avgtime avg_job_us; struct work_struct work_run_job; struct work_struct work_free_job; struct delayed_work work_tdr;