From c94943b0863ef3b8e88769f0805f715c8247b2bf Mon Sep 17 00:00:00 2001 From: "Jesse.zhang@amd.com" Date: Fri, 21 Feb 2025 10:26:52 +0800 Subject: drm/amdgpu: Update amdgpu_job_timedout to check if the ring is guilty This patch updates the `amdgpu_job_timedout` function to check if the ring is actually guilty of causing the timeout. If not, it skips error handling and fence completion. v2: move the is_guilty check down into the queue reset area (Alex) v3: need to call is_guilty before reset (Alex) v4: squash in is_guilty logic fixes (Alex) Signed-off-by: Alex Deucher Signed-off-by: Jesse Zhang Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_job.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 100f04475943..abfbbc6babe7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -130,29 +130,45 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) amdgpu_vm_put_task_info(ti); } - dma_fence_set_error(&s_job->s_fence->finished, -ETIME); - /* attempt a per ring reset */ if (amdgpu_gpu_recovery && ring->funcs->reset) { + bool is_guilty; + dev_err(adev->dev, "Starting %s ring reset\n", s_job->sched->name); /* stop the scheduler, but don't mess with the * bad job yet because if ring reset fails * we'll fall back to full GPU reset. */ drm_sched_wqueue_stop(&ring->sched); + + /* for engine resets, we need to reset the engine, + * but individual queues may be unaffected. + * check here to make sure the accounting is correct. + */ + if (ring->funcs->is_guilty) + is_guilty = ring->funcs->is_guilty(ring); + else + is_guilty = true; + + if (is_guilty) + dma_fence_set_error(&s_job->s_fence->finished, -ETIME); + r = amdgpu_ring_reset(ring, job->vmid); if (!r) { if (amdgpu_ring_sched_ready(ring)) drm_sched_stop(&ring->sched, s_job); - atomic_inc(&ring->adev->gpu_reset_counter); - amdgpu_fence_driver_force_completion(ring); + if (is_guilty) { + atomic_inc(&ring->adev->gpu_reset_counter); + amdgpu_fence_driver_force_completion(ring); + } if (amdgpu_ring_sched_ready(ring)) drm_sched_start(&ring->sched, 0); goto exit; } dev_err(adev->dev, "Ring %s reset failure\n", ring->sched.name); } + dma_fence_set_error(&s_job->s_fence->finished, -ETIME); if (amdgpu_device_should_recover_gpu(ring->adev)) { struct amdgpu_reset_context reset_context; -- cgit v1.2.3 From b7fd6528b5ad80eea66df6240f2399602d9fd388 Mon Sep 17 00:00:00 2001 From: André Almeida Date: Thu, 20 Feb 2025 13:27:49 -0300 Subject: drm/amdgpu: Log after a successful ring reset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a ring reset happens, the kernel log shows only "amdgpu: Starting ring reset", but when it finishes nothing appears in the log. Explicitly write in the log that the reset has finished correctly. Reviewed-by: Christian König Signed-off-by: André Almeida Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_job.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index abfbbc6babe7..c37bc683253a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -164,6 +164,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) } if (amdgpu_ring_sched_ready(ring)) drm_sched_start(&ring->sched, 0); + dev_err(adev->dev, "Ring %s reset succeeded\n", ring->sched.name); goto exit; } dev_err(adev->dev, "Ring %s reset failure\n", ring->sched.name); -- cgit v1.2.3 From 9c696cc57c1a6dab6da6b51f4b30a7d16e233cbc Mon Sep 17 00:00:00 2001 From: André Almeida Date: Wed, 26 Feb 2025 10:11:18 -0300 Subject: drm/amdgpu: Create a debug option to disable ring reset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prior to the addition of ring reset, the debug option `debug_disable_soft_recovery` could be used to force a full device reset. Now that we have ring reset, create a debug option to disable them in amdgpu, forcing the driver to go with the full device reset path again when both options are combined. This option is useful for testing and debugging purposes when one wants to test the full reset from userspace. Signed-off-by: André Almeida Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 5 +++-- 3 files changed, 10 insertions(+), 2 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_job.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 2b1990ea9639..2a9a41f4e748 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1192,6 +1192,7 @@ struct amdgpu_device { bool debug_use_vram_fw_buf; bool debug_enable_ras_aca; bool debug_exp_resets; + bool debug_disable_gpu_ring_reset; bool enforce_isolation[MAX_XCP]; /* Added this mutex for cleaner shader isolation between GFX and compute processes */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 1819166cb4cf..8c1aa9feda53 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -137,6 +137,7 @@ enum AMDGPU_DEBUG_MASK { AMDGPU_DEBUG_USE_VRAM_FW_BUF = BIT(3), AMDGPU_DEBUG_ENABLE_RAS_ACA = BIT(4), AMDGPU_DEBUG_ENABLE_EXP_RESETS = BIT(5), + AMDGPU_DEBUG_DISABLE_GPU_RING_RESET = BIT(6), }; unsigned int amdgpu_vram_limit = UINT_MAX; @@ -2223,6 +2224,11 @@ static void amdgpu_init_debug_options(struct amdgpu_device *adev) pr_info("debug: enable experimental reset features\n"); adev->debug_exp_resets = true; } + + if (amdgpu_debug_mask & AMDGPU_DEBUG_DISABLE_GPU_RING_RESET) { + pr_info("debug: ring reset disabled\n"); + adev->debug_disable_gpu_ring_reset = true; + } } static unsigned long amdgpu_fix_asic_type(struct pci_dev *pdev, unsigned long flags) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index c37bc683253a..5537c8bfd227 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -131,8 +131,9 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) } /* attempt a per ring reset */ - if (amdgpu_gpu_recovery && - ring->funcs->reset) { + if (unlikely(adev->debug_disable_gpu_ring_reset)) { + dev_err(adev->dev, "Ring reset disabled by debug mask\n"); + } else if (amdgpu_gpu_recovery && ring->funcs->reset) { bool is_guilty; dev_err(adev->dev, "Starting %s ring reset\n", s_job->sched->name); -- cgit v1.2.3