From 65781c78ad74e4260fbec92c0ecc05738044e177 Mon Sep 17 00:00:00 2001 From: Monk Liu Date: Thu, 11 May 2017 13:36:44 +0800 Subject: drm/amdgpu/SRIOV:implement guilty job TDR for(V2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1,TDR will kickout guilty job if it hang exceed the threshold of the given one from kernel paramter "job_hang_limit", that way a bad command stream will not infinitly cause GPU hang. by default this threshold is 1 so a job will be kicked out after it hang. 2,if a job timeout TDR routine will not reset all sched/ring, instead if will only reset on the givn one which is indicated by @job of amdgpu_sriov_gpu_reset, that way we don't need to reset and recover each sched/ring if we already know which job cause GPU hang. 3,unblock sriov_gpu_reset for AI family. V2: 1:put kickout guilty job after sched parked. 2:since parking scheduler prior to kickout already occupies a while, we can do last check on the in question job before doing hw_reset. TODO: 1:when a job is considered as guilty, we should mark some flag in its fence status flag, and let UMD side aware that this fence signaling is not due to job complete but job hang. 2:if gpu reset cause all video memory lost, we need introduce a new policy to implement TDR, like drop all jobs not yet signaled, and all IOCTL on this device will return ERROR DEVICE_LOST. this will be implemented later. Signed-off-by: Monk Liu Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 11 ++++++++++- drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 7 +++++++ 2 files changed, 17 insertions(+), 1 deletion(-) (limited to 'drivers/gpu/drm/amd/scheduler') diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index fea96a765cf1..38cea6fb25a8 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -409,9 +409,18 @@ void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched) &s_job->s_fence->cb)) { dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL; + atomic_dec(&sched->hw_rq_count); } } - atomic_set(&sched->hw_rq_count, 0); + spin_unlock(&sched->job_list_lock); +} + +void amd_sched_job_kickout(struct amd_sched_job *s_job) +{ + struct amd_gpu_scheduler *sched = s_job->sched; + + spin_lock(&sched->job_list_lock); + list_del_init(&s_job->node); spin_unlock(&sched->job_list_lock); } diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index 924d4a5899e1..f9d8f28efd16 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -81,6 +81,7 @@ struct amd_sched_job { struct list_head node; struct delayed_work work_tdr; uint64_t id; + atomic_t karma; }; extern const struct dma_fence_ops amd_sched_fence_ops_scheduled; @@ -96,6 +97,11 @@ static inline struct amd_sched_fence *to_amd_sched_fence(struct dma_fence *f) return NULL; } +static inline bool amd_sched_invalidate_job(struct amd_sched_job *s_job, int threshold) +{ + return (s_job && atomic_inc_return(&s_job->karma) > threshold); +} + /** * Define the backend operations called by the scheduler, * these functions should be implemented in driver side @@ -160,4 +166,5 @@ void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched); void amd_sched_job_recovery(struct amd_gpu_scheduler *sched); bool amd_sched_dependency_optimized(struct dma_fence* fence, struct amd_sched_entity *entity); +void amd_sched_job_kickout(struct amd_sched_job *s_job); #endif -- cgit v1.2.3