From fb796c308767606bbd6c2e1bf4fa9446ec68ce51 Mon Sep 17 00:00:00 2001 From: Shashank Sharma Date: Tue, 27 Aug 2024 15:44:43 +0530 Subject: drm/amdgpu: add gfx eviction fence helpers This patch adds basic eviction fence framework for the gfx buffers. The idea is to: - One eviction fence is created per gfx process, at kms_open. - This fence is attached to all the gem buffers created by this process. - This fence is detached to all the gem buffers at postclose_kms. This framework will be further used for usermode queues. V2: Addressed review comments from Christian - keep fence_ctx and fence_seq directly in fpriv - evcition_fence should be dynamically allocated - do not save eviction fence instance in BO, there could be many such fences attached to one BO - use dma_resv_replace_fence() in detach V3: Addressed review comments from Christian - eviction fence create and destroy functions should be called only once from fpriv create/destroy - use dma_fence_put() in eviction_fence_destroy V4: Addressed review comments from Christian: - create a separate ev_fence_mgr structure - cleanup fence init part - do not add a domain for fence owner KGD V5: Addressed review comments from Christian: - drop the dma_fence_is_signaled check - use a local variable to access evf_mgr->ev_fence under the spin_lock() multiple places - remove the vm->is_compute_ctx check to attach gfx eviction fence, in gem_object_open V6: Addressed review comments from Christian: - drop the return value from eviction_fence_signal - reserve_fence should be the first thing inside the attach_eviction_fence function, also keep the resv_add_fence inside the lock - remove the unwanted ev_fence check inside detach function - fix wrong variable check in eviction_fence_init function - return the error value of eviction_fence_init to the caller, dont keep it void. - fail gem_object_open if attaching of eviction_fence fails - detach the eviction fence only when amdgpu_vm_is_bo_always_valid is not true. V7: Addressed review comments from Christian: - Do not add a uq_mgr ptr in ev_fence, rather add evf_mgr V8: Move eviction fence enabling into separate patch for CI Cc: Christian Koenig Cc: Alex Deucher Reviewed-by: Christian Koenig Signed-off-by: Shashank Sharma Signed-off-by: Arvind Yadav Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 144 +++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c new file mode 100644 index 000000000000..056798e2b050 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright 2024 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include +#include "amdgpu.h" + +static const char * +amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence) +{ + return "amdgpu"; +} + +static const char * +amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f) +{ + struct amdgpu_eviction_fence *ef; + + ef = container_of(f, struct amdgpu_eviction_fence, base); + return ef->timeline_name; +} + +static const struct dma_fence_ops amdgpu_eviction_fence_ops = { + .use_64bit_seqno = true, + .get_driver_name = amdgpu_eviction_fence_get_driver_name, + .get_timeline_name = amdgpu_eviction_fence_get_timeline_name, +}; + +void amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr) +{ + spin_lock(&evf_mgr->ev_fence_lock); + dma_fence_signal(&evf_mgr->ev_fence->base); + spin_unlock(&evf_mgr->ev_fence_lock); +} + +struct amdgpu_eviction_fence * +amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr) +{ + struct amdgpu_eviction_fence *ev_fence; + + ev_fence = kzalloc(sizeof(*ev_fence), GFP_KERNEL); + if (!ev_fence) + return NULL; + + ev_fence->evf_mgr = evf_mgr; + get_task_comm(ev_fence->timeline_name, current); + spin_lock_init(&ev_fence->lock); + dma_fence_init(&ev_fence->base, &amdgpu_eviction_fence_ops, + &ev_fence->lock, evf_mgr->ev_fence_ctx, + atomic_inc_return(&evf_mgr->ev_fence_seq)); + return ev_fence; +} + +void amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr) +{ + struct amdgpu_eviction_fence *ev_fence; + + spin_lock(&evf_mgr->ev_fence_lock); + ev_fence = evf_mgr->ev_fence; + spin_unlock(&evf_mgr->ev_fence_lock); + + if (!ev_fence) + return; + + /* Last unref of ev_fence */ + dma_fence_put(&evf_mgr->ev_fence->base); +} + +int amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr, + struct amdgpu_bo *bo) +{ + struct dma_fence *ef; + struct amdgpu_eviction_fence *ev_fence; + struct dma_resv *resv = bo->tbo.base.resv; + int ret; + + if (!resv) + return 0; + + ret = dma_resv_reserve_fences(resv, 1); + if (ret) { + DRM_DEBUG_DRIVER("Failed to resv fence space\n"); + return ret; + } + + spin_lock(&evf_mgr->ev_fence_lock); + ev_fence = evf_mgr->ev_fence; + if (ev_fence) { + ef = dma_fence_get(&ev_fence->base); + dma_resv_add_fence(resv, ef, DMA_RESV_USAGE_BOOKKEEP); + } + spin_unlock(&evf_mgr->ev_fence_lock); + return 0; +} + +void amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr, + struct amdgpu_bo *bo) +{ + struct dma_fence *stub = dma_fence_get_stub(); + + dma_resv_replace_fences(bo->tbo.base.resv, evf_mgr->ev_fence_ctx, + stub, DMA_RESV_USAGE_BOOKKEEP); + dma_fence_put(stub); +} + +int amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr) +{ + struct amdgpu_eviction_fence *ev_fence; + + /* This needs to be done one time per open */ + atomic_set(&evf_mgr->ev_fence_seq, 0); + evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1); + spin_lock_init(&evf_mgr->ev_fence_lock); + + ev_fence = amdgpu_eviction_fence_create(evf_mgr); + if (!ev_fence) { + DRM_ERROR("Failed to craete eviction fence\n"); + return -ENOMEM; + } + + spin_lock(&evf_mgr->ev_fence_lock); + evf_mgr->ev_fence = ev_fence; + spin_unlock(&evf_mgr->ev_fence_lock); + return 0; +} -- cgit v1.2.3 From b0328087c179f47ea6558c3b91b4487c5e10deda Mon Sep 17 00:00:00 2001 From: Shashank Sharma Date: Wed, 20 Nov 2024 18:59:49 +0100 Subject: drm/amdgpu: suspend gfx userqueues This patch adds suspend support for gfx userqueues. It typically does the following: - adds an enable_signaling function for the eviction fence, so that it can trigger the userqueue suspend, - adds a delayed work to handle suspending of the eviction_fence - adds a suspend function to handle suspending of userqueues which suspends all the queues under this userq manager and signals the eviction fence, - adds a function to replace the old eviction fence with a new one and attach it to each of the objects, - adds reference of userq manager in the eviction fence container so that it can be used in the suspend function. V2: Addressed Christian's review comments: - schedule suspend work immediately V4: Addressed Christian's review comments: - wait for pending uq fences before starting suspend, added queue->last_fence for the same - accommodate ev_fence_mgr into existing code - some bug fixes and NULL checks V5: Addressed Christian's review comments (gitlab) - Wait for eviction fence to get signaled in destroy, don't signal it - Wait for eviction fence to get signaled in replace fence, don't signal it V6: Addressed Christian's review comments - Do not destroy the old eviction fence until we have it replaced - Change the sequence of fence replacement sub-tasks - reusing the ev_fence delayed work for userqueue suspend as well (Shashank). V7: Addressed Christian's review comments - give evf_mgr as argument (instead of fpriv) to replace_fence() - save ptr to evf_mgr in ev_fence (instead of uq_mgr) - modify suspend_all_queues logic to reflect error properly - remove the garbage drm_exec_lock section in wait_for_signal - grab the userqueue mutex before starting the wait for fence - remove the unrelated gobj check from signal_ioctl V8: Added race condition fixes Cc: Alex Deucher Cc: Christian Koenig Acked-by: Christian Koenig Signed-off-by: Shashank Sharma Signed-off-by: Arvind Yadav Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 127 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h | 4 + drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 37 ++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c | 110 ++++++++++++++++++ drivers/gpu/drm/amd/include/amdgpu_userqueue.h | 9 ++ 5 files changed, 276 insertions(+), 11 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index 056798e2b050..189afb872775 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -22,8 +22,12 @@ * */ #include +#include #include "amdgpu.h" +#define work_to_evf_mgr(w, name) container_of(w, struct amdgpu_eviction_fence_mgr, name) +#define evf_mgr_to_fpriv(e) container_of(e, struct amdgpu_fpriv, evf_mgr) + static const char * amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence) { @@ -39,10 +43,131 @@ amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f) return ef->timeline_name; } +int +amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, + struct drm_exec *exec) +{ + struct amdgpu_eviction_fence *old_ef, *new_ef; + struct drm_gem_object *obj; + unsigned long index; + int ret; + + /* + * Steps to replace eviction fence: + * * lock all objects in exec (caller) + * * create a new eviction fence + * * update new eviction fence in evf_mgr + * * attach the new eviction fence to BOs + * * release the old fence + * * unlock the objects (caller) + */ + new_ef = amdgpu_eviction_fence_create(evf_mgr); + if (!new_ef) { + DRM_ERROR("Failed to create new eviction fence\n"); + return -ENOMEM; + } + + /* Update the eviction fence now */ + spin_lock(&evf_mgr->ev_fence_lock); + old_ef = evf_mgr->ev_fence; + evf_mgr->ev_fence = new_ef; + spin_unlock(&evf_mgr->ev_fence_lock); + + /* Attach the new fence */ + drm_exec_for_each_locked_object(exec, index, obj) { + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + + if (!bo) + continue; + ret = amdgpu_eviction_fence_attach(evf_mgr, bo); + if (ret) { + DRM_ERROR("Failed to attch new eviction fence\n"); + goto free_err; + } + } + + /* Free old fence */ + dma_fence_put(&old_ef->base); + return 0; + +free_err: + kfree(new_ef); + return ret; +} + +static void +amdgpu_eviction_fence_suspend_worker(struct work_struct *work) +{ + struct amdgpu_eviction_fence_mgr *evf_mgr = work_to_evf_mgr(work, suspend_work.work); + struct amdgpu_fpriv *fpriv = evf_mgr_to_fpriv(evf_mgr); + struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; + struct amdgpu_vm *vm = &fpriv->vm; + struct amdgpu_bo_va *bo_va; + struct drm_exec exec; + bool userq_active = amdgpu_userqueue_active(uq_mgr); + int ret; + + + /* For userqueues, the fence replacement happens in resume path */ + if (userq_active) { + amdgpu_userqueue_suspend(uq_mgr); + return; + } + + /* Signal old eviction fence */ + amdgpu_eviction_fence_signal(evf_mgr); + + /* Prepare the objects to replace eviction fence */ + drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); + drm_exec_until_all_locked(&exec) { + ret = amdgpu_vm_lock_pd(vm, &exec, 2); + drm_exec_retry_on_contention(&exec); + if (unlikely(ret)) + goto unlock_drm; + + /* Lock the done list */ + list_for_each_entry(bo_va, &vm->done, base.vm_status) { + struct amdgpu_bo *bo = bo_va->base.bo; + + if (!bo) + continue; + + ret = drm_exec_lock_obj(&exec, &bo->tbo.base); + drm_exec_retry_on_contention(&exec); + if (unlikely(ret)) + goto unlock_drm; + } + } + + /* Replace old eviction fence with new one */ + ret = amdgpu_eviction_fence_replace_fence(&fpriv->evf_mgr, &exec); + if (ret) + DRM_ERROR("Failed to replace eviction fence\n"); + +unlock_drm: + drm_exec_fini(&exec); +} + +static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f) +{ + struct amdgpu_eviction_fence_mgr *evf_mgr; + struct amdgpu_eviction_fence *ev_fence; + + if (!f) + return true; + + ev_fence = to_ev_fence(f); + evf_mgr = ev_fence->evf_mgr; + + schedule_delayed_work(&evf_mgr->suspend_work, 0); + return true; +} + static const struct dma_fence_ops amdgpu_eviction_fence_ops = { .use_64bit_seqno = true, .get_driver_name = amdgpu_eviction_fence_get_driver_name, .get_timeline_name = amdgpu_eviction_fence_get_timeline_name, + .enable_signaling = amdgpu_eviction_fence_enable_signaling, }; void amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr) @@ -140,5 +265,7 @@ int amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr) spin_lock(&evf_mgr->ev_fence_lock); evf_mgr->ev_fence = ev_fence; spin_unlock(&evf_mgr->ev_fence_lock); + + INIT_DELAYED_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h index aba12a43f433..12f168ec3ef0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h @@ -37,6 +37,7 @@ struct amdgpu_eviction_fence_mgr { atomic_t ev_fence_seq; spinlock_t ev_fence_lock; struct amdgpu_eviction_fence *ev_fence; + struct delayed_work suspend_work; }; /* Eviction fence helper functions */ @@ -60,4 +61,7 @@ amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr); void amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr); +int +amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, + struct drm_exec *exec); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c index 6157a540c929..877cb17a14e9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c @@ -466,6 +466,16 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data, } } + /* Save the fence to wait for during suspend */ + mutex_lock(&userq_mgr->userq_mutex); + + /* Retrieve the user queue */ + queue = idr_find(&userq_mgr->userq_idr, args->queue_id); + if (!queue) { + r = -ENOENT; + mutex_unlock(&userq_mgr->userq_mutex); + } + drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, (num_read_bo_handles + num_write_bo_handles)); @@ -473,30 +483,35 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data, drm_exec_until_all_locked(&exec) { r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1); drm_exec_retry_on_contention(&exec); - if (r) + if (r) { + mutex_unlock(&userq_mgr->userq_mutex); goto exec_fini; + } r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1); drm_exec_retry_on_contention(&exec); - if (r) + if (r) { + mutex_unlock(&userq_mgr->userq_mutex); goto exec_fini; - } - - /*Retrieve the user queue */ - queue = idr_find(&userq_mgr->userq_idr, args->queue_id); - if (!queue) { - r = -ENOENT; - goto exec_fini; + } } r = amdgpu_userq_fence_read_wptr(queue, &wptr); - if (r) + if (r) { + mutex_unlock(&userq_mgr->userq_mutex); goto exec_fini; + } /* Create a new fence */ r = amdgpu_userq_fence_create(queue, wptr, &fence); - if (r) + if (r) { + mutex_unlock(&userq_mgr->userq_mutex); goto exec_fini; + } + + dma_fence_put(queue->last_fence); + queue->last_fence = dma_fence_get(fence); + mutex_unlock(&userq_mgr->userq_mutex); for (i = 0; i < num_read_bo_handles; i++) { if (!gobj_read || !gobj_read[i]->resv) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c index e946c6d1dd6b..43fff8869ac9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c @@ -60,6 +60,16 @@ amdgpu_userqueue_cleanup(struct amdgpu_userq_mgr *uq_mgr, { struct amdgpu_device *adev = uq_mgr->adev; const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type]; + struct dma_fence *f = queue->last_fence; + int ret; + + if (f && !dma_fence_is_signaled(f)) { + ret = dma_fence_wait_timeout(f, true, msecs_to_jiffies(100)); + if (ret <= 0) { + DRM_ERROR("Timed out waiting for fence f=%p\n", f); + return; + } + } uq_funcs->mqd_destroy(uq_mgr, queue); amdgpu_userq_fence_driver_free(queue); @@ -67,6 +77,22 @@ amdgpu_userqueue_cleanup(struct amdgpu_userq_mgr *uq_mgr, kfree(queue); } +int +amdgpu_userqueue_active(struct amdgpu_userq_mgr *uq_mgr) +{ + struct amdgpu_usermode_queue *queue; + int queue_id; + int ret = 0; + + mutex_lock(&uq_mgr->userq_mutex); + /* Resume all the queues for this process */ + idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) + ret += queue->queue_active; + + mutex_unlock(&uq_mgr->userq_mutex); + return ret; +} + #ifdef CONFIG_DRM_AMDGPU_NAVI3X_USERQ static struct amdgpu_usermode_queue * amdgpu_userqueue_find(struct amdgpu_userq_mgr *uq_mgr, int qid) @@ -202,6 +228,7 @@ amdgpu_userqueue_destroy(struct drm_file *filp, int queue_id) amdgpu_bo_unpin(queue->db_obj.obj); amdgpu_bo_unref(&queue->db_obj.obj); amdgpu_userqueue_cleanup(uq_mgr, queue, queue_id); + uq_mgr->num_userqs--; mutex_unlock(&uq_mgr->userq_mutex); return 0; } @@ -277,6 +304,7 @@ amdgpu_userqueue_create(struct drm_file *filp, union drm_amdgpu_userq *args) goto unlock; } args->out.queue_id = qid; + uq_mgr->num_userqs++; unlock: mutex_unlock(&uq_mgr->userq_mutex); @@ -317,11 +345,93 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, } #endif +static int +amdgpu_userqueue_suspend_all(struct amdgpu_userq_mgr *uq_mgr) +{ + struct amdgpu_device *adev = uq_mgr->adev; + const struct amdgpu_userq_funcs *userq_funcs; + struct amdgpu_usermode_queue *queue; + int queue_id; + int ret = 0; + + userq_funcs = adev->userq_funcs[AMDGPU_HW_IP_GFX]; + + /* Try to suspend all the queues in this process ctx */ + idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) + ret += userq_funcs->suspend(uq_mgr, queue); + + if (ret) + DRM_ERROR("Couldn't suspend all the queues\n"); + return ret; +} + +static int +amdgpu_userqueue_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr) +{ + struct amdgpu_usermode_queue *queue; + int queue_id, ret; + + idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) { + struct dma_fence *f = queue->last_fence; + + if (!f || dma_fence_is_signaled(f)) + continue; + ret = dma_fence_wait_timeout(f, true, msecs_to_jiffies(100)); + if (ret <= 0) { + DRM_ERROR("Timed out waiting for fence f=%p\n", f); + return -ETIMEDOUT; + } + } + + return 0; +} + +void +amdgpu_userqueue_suspend(struct amdgpu_userq_mgr *uq_mgr) +{ + int ret; + struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); + struct amdgpu_eviction_fence_mgr *evf_mgr = &fpriv->evf_mgr; + + mutex_lock(&uq_mgr->userq_mutex); + + /* Wait for any pending userqueue fence to signal */ + ret = amdgpu_userqueue_wait_for_signal(uq_mgr); + if (ret) { + DRM_ERROR("Not suspending userqueue, timeout waiting for work\n"); + goto unlock; + } + + ret = amdgpu_userqueue_suspend_all(uq_mgr); + if (ret) { + DRM_ERROR("Failed to evict userqueue\n"); + goto unlock; + } + + /* Signal current eviction fence */ + amdgpu_eviction_fence_signal(evf_mgr); + + /* Cleanup old eviction fence entry */ + amdgpu_eviction_fence_destroy(evf_mgr); + +unlock: + mutex_unlock(&uq_mgr->userq_mutex); +} + int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct amdgpu_device *adev) { + struct amdgpu_fpriv *fpriv; + mutex_init(&userq_mgr->userq_mutex); idr_init_base(&userq_mgr->userq_idr, 1); userq_mgr->adev = adev; + userq_mgr->num_userqs = 0; + + fpriv = uq_mgr_to_fpriv(userq_mgr); + if (!fpriv->evf_mgr.ev_fence) { + DRM_ERROR("Eviction fence not initialized yet\n"); + return -EINVAL; + } return 0; } diff --git a/drivers/gpu/drm/amd/include/amdgpu_userqueue.h b/drivers/gpu/drm/amd/include/amdgpu_userqueue.h index ede0178e0332..7781ee59653b 100644 --- a/drivers/gpu/drm/amd/include/amdgpu_userqueue.h +++ b/drivers/gpu/drm/amd/include/amdgpu_userqueue.h @@ -27,6 +27,9 @@ #define AMDGPU_MAX_USERQ_COUNT 512 +#define to_ev_fence(f) container_of(f, struct amdgpu_eviction_fence, base) +#define uq_mgr_to_fpriv(u) container_of(u, struct amdgpu_fpriv, userq_mgr) + struct amdgpu_mqd_prop; struct amdgpu_userq_obj { @@ -50,6 +53,7 @@ struct amdgpu_usermode_queue { struct amdgpu_userq_obj wptr_obj; struct xarray fence_drv_xa; struct amdgpu_userq_fence_driver *fence_drv; + struct dma_fence *last_fence; }; struct amdgpu_userq_funcs { @@ -69,6 +73,7 @@ struct amdgpu_userq_mgr { struct idr userq_idr; struct mutex userq_mutex; struct amdgpu_device *adev; + int num_userqs; }; int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); @@ -83,4 +88,8 @@ int amdgpu_userqueue_create_object(struct amdgpu_userq_mgr *uq_mgr, void amdgpu_userqueue_destroy_object(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_userq_obj *userq_obj); + +void amdgpu_userqueue_suspend(struct amdgpu_userq_mgr *uq_mgr); + +int amdgpu_userqueue_active(struct amdgpu_userq_mgr *uq_mgr); #endif -- cgit v1.2.3 From b8e6d3f68c3bd1ac54492e210ece87475e7f862b Mon Sep 17 00:00:00 2001 From: Shashank Sharma Date: Wed, 20 Nov 2024 18:04:33 +0100 Subject: drm/amdgpu: handle eviction fence race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eviction process can get into a race condition between the eviction fence suspend work (which replaces the old fence with new) and kms_close (which destroys the fence and doesn't expect a new one). This patch: - adds a flag to indicate that fd is closing, so fence replacement is not required (evf_mgr->fd_closing) - adds a flush_work() during the ev_fence_destroy routine V2: Addressed review comments from Christian: - Do not use mutex to sync - Use flush_work and wait for suspend_work to be done V3: Fixed state machine for queue->active, which adds into race between suspend/resume and queue ops Cc: Alex Deucher Cc: Christian König Reviewed-by: Christian König Signed-off-by: Shashank Sharma Signed-off-by: Arvind Yadav Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 7 +++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 4 +++- drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c | 3 ++- drivers/gpu/drm/amd/amdgpu/mes_v11_0_userqueue.c | 9 +++++---- 4 files changed, 17 insertions(+), 6 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index 189afb872775..c22767a75348 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -117,6 +117,10 @@ amdgpu_eviction_fence_suspend_worker(struct work_struct *work) /* Signal old eviction fence */ amdgpu_eviction_fence_signal(evf_mgr); + /* Do not replace eviction fence is fd is getting closed */ + if (evf_mgr->fd_closing) + return; + /* Prepare the objects to replace eviction fence */ drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); drm_exec_until_all_locked(&exec) { @@ -199,6 +203,9 @@ void amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr) { struct amdgpu_eviction_fence *ev_fence; + /* Wait for any pending work to execute */ + flush_delayed_work(&evf_mgr->suspend_work); + spin_lock(&evf_mgr->ev_fence_lock); ev_fence = evf_mgr->ev_fence; spin_unlock(&evf_mgr->ev_fence_lock); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 700442584f97..f8370da08038 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -1490,10 +1490,12 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, amdgpu_bo_unreserve(pd); } + fpriv->evf_mgr.fd_closing = true; + amdgpu_userq_mgr_fini(&fpriv->userq_mgr); amdgpu_eviction_fence_destroy(&fpriv->evf_mgr); + amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr); amdgpu_vm_fini(adev, &fpriv->vm); - amdgpu_userq_mgr_fini(&fpriv->userq_mgr); if (pasid) amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c index 57e502d014a1..cba51bdf2e2c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c @@ -614,9 +614,10 @@ void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr) cancel_delayed_work(&userq_mgr->resume_work); + mutex_lock(&userq_mgr->userq_mutex); idr_for_each_entry(&userq_mgr->userq_idr, queue, queue_id) amdgpu_userqueue_cleanup(userq_mgr, queue, queue_id); - idr_destroy(&userq_mgr->userq_idr); + mutex_unlock(&userq_mgr->userq_mutex); mutex_destroy(&userq_mgr->userq_mutex); } diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0_userqueue.c index ee0a757fcfa3..b1b7bc47d39f 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0_userqueue.c @@ -139,6 +139,7 @@ static int mes_v11_0_userq_map(struct amdgpu_userq_mgr *uq_mgr, return r; } + queue->queue_active = true; DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index); return 0; } @@ -160,6 +161,7 @@ static void mes_v11_0_userq_unmap(struct amdgpu_userq_mgr *uq_mgr, amdgpu_mes_unlock(&adev->mes); if (r) DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r); + queue->queue_active = false; } static int mes_v11_0_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr, @@ -331,7 +333,6 @@ static int mes_v11_0_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr, goto free_ctx; } - queue->queue_active = true; return 0; free_ctx: @@ -350,12 +351,12 @@ static void mes_v11_0_userq_mqd_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_queue *queue) { - mes_v11_0_userq_unmap(uq_mgr, queue); - amdgpu_bo_unref(&queue->wptr_obj.obj); + if (queue->queue_active) + mes_v11_0_userq_unmap(uq_mgr, queue); + amdgpu_userqueue_destroy_object(uq_mgr, &queue->fw_obj); kfree(queue->userq_prop); amdgpu_userqueue_destroy_object(uq_mgr, &queue->mqd); - queue->queue_active = false; } static int mes_v11_0_userq_suspend(struct amdgpu_userq_mgr *uq_mgr, -- cgit v1.2.3 From f2234816a31d0ec85ab63899762ec962ab682704 Mon Sep 17 00:00:00 2001 From: Amaranath Somalapuram Date: Wed, 27 Nov 2024 17:06:45 +0100 Subject: drm/amdgpu: fix IGT CI regression with eviction fence This patch fixes one of the regressions in eviction fence code with IGT tests. Reviewed-by: Shashank Sharma Signed-off-by: Amaranath Somalapuram Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index c22767a75348..f7fb1674278c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -136,6 +136,9 @@ amdgpu_eviction_fence_suspend_worker(struct work_struct *work) if (!bo) continue; + if (vm != bo_va->base.vm) + continue; + ret = drm_exec_lock_obj(&exec, &bo->tbo.base); drm_exec_retry_on_contention(&exec); if (unlikely(ret)) -- cgit v1.2.3 From a242a3e4b5be22ffd85fb768d8c96df79b018136 Mon Sep 17 00:00:00 2001 From: Shashank Sharma Date: Wed, 11 Dec 2024 12:09:00 +0100 Subject: drm/amdgpu: simplify eviction fence suspend/resume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The basic idea in this redesign is to add an eviction fence only in UQ resume path. When userqueue is not present, keep ev_fence as NULL Main changes are: - do not create the eviction fence during evf_mgr_init, keeping evf_mgr->ev_fence=NULL until UQ get active. - do not replace the ev_fence in evf_resume path, but replace it only in uq_resume path, so remove all the unnecessary code from ev_fence_resume. - add a new helper function (amdgpu_userqueue_ensure_ev_fence) which will do the following: - flush any pending uq_resume work, so that it could create an eviction_fence - if there is no pending uq_resume_work, add a uq_resume work and wait for it to execute so that we always have a valid ev_fence - call this helper function from two places, to ensure we have a valid ev_fence: - when a new uq is created - when a new uq completion fence is created v2: Worked on review comments by Christian. v3: Addressed few more review comments by Christian. v4: Move mutex lock outside of the amdgpu_userqueue_suspend() function (Christian). v5: squash in build fix (Alex) Cc: Alex Deucher Reviewed-by: Christian König Signed-off-by: Arvind Yadav Signed-off-by: Shashank Sharma Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 78 ++++------------------ drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h | 3 +- drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 24 +++---- drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c | 68 +++++++++++-------- drivers/gpu/drm/amd/include/amdgpu_userqueue.h | 7 +- 5 files changed, 69 insertions(+), 111 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index f7fb1674278c..8358dc6b68ad 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -87,7 +87,8 @@ amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, } /* Free old fence */ - dma_fence_put(&old_ef->base); + if (old_ef) + dma_fence_put(&old_ef->base); return 0; free_err: @@ -101,58 +102,17 @@ amdgpu_eviction_fence_suspend_worker(struct work_struct *work) struct amdgpu_eviction_fence_mgr *evf_mgr = work_to_evf_mgr(work, suspend_work.work); struct amdgpu_fpriv *fpriv = evf_mgr_to_fpriv(evf_mgr); struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; - struct amdgpu_vm *vm = &fpriv->vm; - struct amdgpu_bo_va *bo_va; - struct drm_exec exec; - bool userq_active = amdgpu_userqueue_active(uq_mgr); - int ret; - - - /* For userqueues, the fence replacement happens in resume path */ - if (userq_active) { - amdgpu_userqueue_suspend(uq_mgr); - return; - } - - /* Signal old eviction fence */ - amdgpu_eviction_fence_signal(evf_mgr); - - /* Do not replace eviction fence is fd is getting closed */ - if (evf_mgr->fd_closing) - return; - - /* Prepare the objects to replace eviction fence */ - drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); - drm_exec_until_all_locked(&exec) { - ret = amdgpu_vm_lock_pd(vm, &exec, 2); - drm_exec_retry_on_contention(&exec); - if (unlikely(ret)) - goto unlock_drm; - - /* Lock the done list */ - list_for_each_entry(bo_va, &vm->done, base.vm_status) { - struct amdgpu_bo *bo = bo_va->base.bo; - - if (!bo) - continue; - - if (vm != bo_va->base.vm) - continue; + struct amdgpu_eviction_fence *ev_fence; - ret = drm_exec_lock_obj(&exec, &bo->tbo.base); - drm_exec_retry_on_contention(&exec); - if (unlikely(ret)) - goto unlock_drm; - } - } + mutex_lock(&uq_mgr->userq_mutex); + ev_fence = evf_mgr->ev_fence; + if (!ev_fence) + goto unlock; - /* Replace old eviction fence with new one */ - ret = amdgpu_eviction_fence_replace_fence(&fpriv->evf_mgr, &exec); - if (ret) - DRM_ERROR("Failed to replace eviction fence\n"); + amdgpu_userqueue_suspend(uq_mgr, ev_fence); -unlock_drm: - drm_exec_fini(&exec); +unlock: + mutex_unlock(&uq_mgr->userq_mutex); } static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f) @@ -177,10 +137,11 @@ static const struct dma_fence_ops amdgpu_eviction_fence_ops = { .enable_signaling = amdgpu_eviction_fence_enable_signaling, }; -void amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr) +void amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr, + struct amdgpu_eviction_fence *ev_fence) { spin_lock(&evf_mgr->ev_fence_lock); - dma_fence_signal(&evf_mgr->ev_fence->base); + dma_fence_signal(&ev_fence->base); spin_unlock(&evf_mgr->ev_fence_lock); } @@ -244,6 +205,7 @@ int amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr, dma_resv_add_fence(resv, ef, DMA_RESV_USAGE_BOOKKEEP); } spin_unlock(&evf_mgr->ev_fence_lock); + return 0; } @@ -259,23 +221,11 @@ void amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr, int amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr) { - struct amdgpu_eviction_fence *ev_fence; - /* This needs to be done one time per open */ atomic_set(&evf_mgr->ev_fence_seq, 0); evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1); spin_lock_init(&evf_mgr->ev_fence_lock); - ev_fence = amdgpu_eviction_fence_create(evf_mgr); - if (!ev_fence) { - DRM_ERROR("Failed to craete eviction fence\n"); - return -ENOMEM; - } - - spin_lock(&evf_mgr->ev_fence_lock); - evf_mgr->ev_fence = ev_fence; - spin_unlock(&evf_mgr->ev_fence_lock); - INIT_DELAYED_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h index 787182bd1069..fcd867b7147d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.h @@ -60,7 +60,8 @@ int amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr); void -amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr); +amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr, + struct amdgpu_eviction_fence *ev_fence); int amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c index 877cb17a14e9..20c36dc97c2e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c @@ -466,14 +466,11 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data, } } - /* Save the fence to wait for during suspend */ - mutex_lock(&userq_mgr->userq_mutex); - /* Retrieve the user queue */ queue = idr_find(&userq_mgr->userq_idr, args->queue_id); if (!queue) { r = -ENOENT; - mutex_unlock(&userq_mgr->userq_mutex); + goto put_gobj_write; } drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, @@ -483,31 +480,26 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data, drm_exec_until_all_locked(&exec) { r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1); drm_exec_retry_on_contention(&exec); - if (r) { - mutex_unlock(&userq_mgr->userq_mutex); + if (r) goto exec_fini; - } r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1); drm_exec_retry_on_contention(&exec); - if (r) { - mutex_unlock(&userq_mgr->userq_mutex); + if (r) goto exec_fini; - } } r = amdgpu_userq_fence_read_wptr(queue, &wptr); - if (r) { - mutex_unlock(&userq_mgr->userq_mutex); + if (r) goto exec_fini; - } /* Create a new fence */ r = amdgpu_userq_fence_create(queue, wptr, &fence); - if (r) { - mutex_unlock(&userq_mgr->userq_mutex); + if (r) goto exec_fini; - } + + /* We are here means UQ is active, make sure the eviction fence is valid */ + amdgpu_userqueue_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); dma_fence_put(queue->last_fence); queue->last_fence = dma_fence_get(fence); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c index cba51bdf2e2c..c11fcdd604fc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c @@ -101,6 +101,31 @@ amdgpu_userqueue_find(struct amdgpu_userq_mgr *uq_mgr, int qid) return idr_find(&uq_mgr->userq_idr, qid); } +void +amdgpu_userqueue_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_eviction_fence_mgr *evf_mgr) +{ + struct amdgpu_eviction_fence *ev_fence; + +retry: + /* Flush any pending resume work to create ev_fence */ + flush_delayed_work(&uq_mgr->resume_work); + + mutex_lock(&uq_mgr->userq_mutex); + spin_lock(&evf_mgr->ev_fence_lock); + ev_fence = evf_mgr->ev_fence; + spin_unlock(&evf_mgr->ev_fence_lock); + if (!ev_fence || dma_fence_is_signaled(&ev_fence->base)) { + mutex_unlock(&uq_mgr->userq_mutex); + /* + * Looks like there was no pending resume work, + * add one now to create a valid eviction fence + */ + schedule_delayed_work(&uq_mgr->resume_work, 0); + goto retry; + } +} + int amdgpu_userqueue_create_object(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_userq_obj *userq_obj, int size) @@ -253,7 +278,14 @@ amdgpu_userqueue_create(struct drm_file *filp, union drm_amdgpu_userq *args) return -EINVAL; } - mutex_lock(&uq_mgr->userq_mutex); + /* + * There could be a situation that we are creating a new queue while + * the other queues under this UQ_mgr are suspended. So if there is any + * resume work pending, wait for it to get done. + * + * This will also make sure we have a valid eviction fence ready to be used. + */ + amdgpu_userqueue_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); uq_funcs = adev->userq_funcs[args->in.ip_type]; if (!uq_funcs) { @@ -308,14 +340,6 @@ amdgpu_userqueue_create(struct drm_file *filp, union drm_amdgpu_userq *args) unlock: mutex_unlock(&uq_mgr->userq_mutex); - if (!r) { - /* - * There could be a situation that we are creating a new queue while - * the other queues under this UQ_mgr are suspended. So if there is any - * resume work pending, wait for it to get done. - */ - flush_delayed_work(&uq_mgr->resume_work); - } return r; } @@ -551,58 +575,44 @@ amdgpu_userqueue_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr) } void -amdgpu_userqueue_suspend(struct amdgpu_userq_mgr *uq_mgr) +amdgpu_userqueue_suspend(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_eviction_fence *ev_fence) { int ret; struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); struct amdgpu_eviction_fence_mgr *evf_mgr = &fpriv->evf_mgr; - - mutex_lock(&uq_mgr->userq_mutex); - - /* Wait for any pending userqueue fence to signal */ + /* Wait for any pending userqueue fence work to finish */ ret = amdgpu_userqueue_wait_for_signal(uq_mgr); if (ret) { DRM_ERROR("Not suspending userqueue, timeout waiting for work\n"); - goto unlock; + return; } ret = amdgpu_userqueue_suspend_all(uq_mgr); if (ret) { DRM_ERROR("Failed to evict userqueue\n"); - goto unlock; + return; } /* Signal current eviction fence */ - amdgpu_eviction_fence_signal(evf_mgr); + amdgpu_eviction_fence_signal(evf_mgr, ev_fence); if (evf_mgr->fd_closing) { - mutex_unlock(&uq_mgr->userq_mutex); cancel_delayed_work(&uq_mgr->resume_work); return; } /* Schedule a resume work */ schedule_delayed_work(&uq_mgr->resume_work, 0); - -unlock: - mutex_unlock(&uq_mgr->userq_mutex); } int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct amdgpu_device *adev) { - struct amdgpu_fpriv *fpriv; - mutex_init(&userq_mgr->userq_mutex); idr_init_base(&userq_mgr->userq_idr, 1); userq_mgr->adev = adev; - fpriv = uq_mgr_to_fpriv(userq_mgr); - if (!fpriv->evf_mgr.ev_fence) { - DRM_ERROR("Eviction fence not initialized yet\n"); - return -EINVAL; - } - INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userqueue_resume_worker); return 0; } diff --git a/drivers/gpu/drm/amd/include/amdgpu_userqueue.h b/drivers/gpu/drm/amd/include/amdgpu_userqueue.h index 2bf28f3454cb..e7e8d79b689d 100644 --- a/drivers/gpu/drm/amd/include/amdgpu_userqueue.h +++ b/drivers/gpu/drm/amd/include/amdgpu_userqueue.h @@ -24,6 +24,7 @@ #ifndef AMDGPU_USERQUEUE_H_ #define AMDGPU_USERQUEUE_H_ +#include "amdgpu_eviction_fence.h" #define AMDGPU_MAX_USERQ_COUNT 512 @@ -90,7 +91,11 @@ int amdgpu_userqueue_create_object(struct amdgpu_userq_mgr *uq_mgr, void amdgpu_userqueue_destroy_object(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_userq_obj *userq_obj); -void amdgpu_userqueue_suspend(struct amdgpu_userq_mgr *uq_mgr); +void amdgpu_userqueue_suspend(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_eviction_fence *ev_fence); int amdgpu_userqueue_active(struct amdgpu_userq_mgr *uq_mgr); + +void amdgpu_userqueue_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr, + struct amdgpu_eviction_fence_mgr *evf_mgr); #endif -- cgit v1.2.3 From 31f7efcdca4d1caaa3a0babc33377e27e6f9b593 Mon Sep 17 00:00:00 2001 From: Shashank Sharma Date: Wed, 20 Nov 2024 18:45:33 +0100 Subject: drm/amdgpu: enable eviction fence This patch enables attachment and detachment of eviction fences. This is just a fork of eviction fence enabling code from the first patch of the series so that the CI testing can happen on fully fledged code. Cc: Alex Deucher Cc: Christian Koenig Reviewed-by: Christian Koenig Signed-off-by: Shashank Sharma Signed-off-by: Arvind Yadav Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 10 ++++++++++ 2 files changed, 12 insertions(+) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index 8358dc6b68ad..167951aee502 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -177,6 +177,8 @@ void amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr) if (!ev_fence) return; + dma_fence_wait(&ev_fence->base, false); + /* Last unref of ev_fence */ dma_fence_put(&evf_mgr->ev_fence->base); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 682b76cad359..6850a6954a71 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -294,6 +294,13 @@ static int amdgpu_gem_object_open(struct drm_gem_object *obj, else ++bo_va->ref_count; + /* attach gfx eviction fence */ + r = amdgpu_eviction_fence_attach(&fpriv->evf_mgr, abo); + if (r) { + DRM_DEBUG_DRIVER("Failed to attach eviction fence to BO\n"); + return r; + } + amdgpu_bo_unreserve(abo); /* Validate and add eviction fence to DMABuf imports with dynamic @@ -344,6 +351,9 @@ static void amdgpu_gem_object_close(struct drm_gem_object *obj, struct drm_exec exec; long r; + if (!amdgpu_vm_is_bo_always_valid(vm, bo)) + amdgpu_eviction_fence_detach(&fpriv->evf_mgr, bo); + drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); drm_exec_until_all_locked(&exec) { r = drm_exec_prepare_obj(&exec, &bo->tbo.base, 1); -- cgit v1.2.3 From 32bd8b3ea7071ca96d4adcd0817046e09c5df415 Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Tue, 18 Mar 2025 18:45:40 +0530 Subject: drm/amdgpu: Fix display freezing issue when resizing apps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The display is freezing because the amdgpu_userq_wait_ioctl() is waiting for a non-user queue fence(specifically, the PT update fence). RootCause: The resume_work is initiated by both amdgpu_userq_suspend and amdgpu_userqueue_ensure_ev_fence at same time. The amdgpu_userq_suspend signals a dma-fence and subsequently triggers the resume_work, which is intended to replace the existing fence by creating new dma-fence. However, following this, the amdgpu_userqueue_ensure_ev_fence schedules another resume_work that generates a new dma-fence, thereby replacing the one created by amdgpu_userq_suspend. Consequently, the original fence will never be signaled. Cc: Alex Deucher Cc: Christian König Cc: Shashank Sharma Cc: Sunil Khatri Reviewed-by: Christian König Reviewed-by: Shashank Sharma Signed-off-by: Arvind Yadav Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 3 +++ drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c | 3 +++ 2 files changed, 6 insertions(+) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index 167951aee502..0075469550b0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -52,6 +52,9 @@ amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, unsigned long index; int ret; + if (evf_mgr->ev_fence && + !dma_fence_is_signaled(&evf_mgr->ev_fence->base)) + return 0; /* * Steps to replace eviction fence: * * lock all objects in exec (caller) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c index a02614cbda36..beae931152a3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c @@ -555,8 +555,11 @@ unlock_all: static void amdgpu_userqueue_resume_worker(struct work_struct *work) { struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, resume_work.work); + struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); int ret; + flush_work(&fpriv->evf_mgr.suspend_work.work); + mutex_lock(&uq_mgr->userq_mutex); ret = amdgpu_userqueue_validate_bos(uq_mgr); -- cgit v1.2.3 From 4fdbe3a623b264d0f361a81a41baff16b4aa6bb1 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Wed, 16 Apr 2025 17:27:41 -0400 Subject: drm/amdgpu/userq: rename eviction helpers suspend/resume -> evict/restore Rename to avoid confusion with the system suspend and resume helpers. v2: update error messages Reviewed-by: Prike Liang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c | 20 ++++++++++---------- drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.h | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index 0075469550b0..02164bca51a7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -112,7 +112,7 @@ amdgpu_eviction_fence_suspend_worker(struct work_struct *work) if (!ev_fence) goto unlock; - amdgpu_userqueue_suspend(uq_mgr, ev_fence); + amdgpu_userqueue_evict(uq_mgr, ev_fence); unlock: mutex_unlock(&uq_mgr->userq_mutex); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c index b75b93e69c09..82741dcb2a45 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c @@ -528,7 +528,7 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, #endif static int -amdgpu_userqueue_resume_all(struct amdgpu_userq_mgr *uq_mgr) +amdgpu_userqueue_restore_all(struct amdgpu_userq_mgr *uq_mgr) { struct amdgpu_device *adev = uq_mgr->adev; struct amdgpu_usermode_queue *queue; @@ -659,7 +659,7 @@ unlock_all: return ret; } -static void amdgpu_userqueue_resume_worker(struct work_struct *work) +static void amdgpu_userqueue_restore_worker(struct work_struct *work) { struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, resume_work.work); struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); @@ -675,9 +675,9 @@ static void amdgpu_userqueue_resume_worker(struct work_struct *work) goto unlock; } - ret = amdgpu_userqueue_resume_all(uq_mgr); + ret = amdgpu_userqueue_restore_all(uq_mgr); if (ret) { - DRM_ERROR("Failed to resume all queues\n"); + DRM_ERROR("Failed to restore all queues\n"); goto unlock; } @@ -686,7 +686,7 @@ unlock: } static int -amdgpu_userqueue_suspend_all(struct amdgpu_userq_mgr *uq_mgr) +amdgpu_userqueue_evict_all(struct amdgpu_userq_mgr *uq_mgr) { struct amdgpu_device *adev = uq_mgr->adev; struct amdgpu_usermode_queue *queue; @@ -728,8 +728,8 @@ amdgpu_userqueue_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr) } void -amdgpu_userqueue_suspend(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_eviction_fence *ev_fence) +amdgpu_userqueue_evict(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_eviction_fence *ev_fence) { int ret; struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); @@ -738,11 +738,11 @@ amdgpu_userqueue_suspend(struct amdgpu_userq_mgr *uq_mgr, /* Wait for any pending userqueue fence work to finish */ ret = amdgpu_userqueue_wait_for_signal(uq_mgr); if (ret) { - DRM_ERROR("Not suspending userqueue, timeout waiting for work\n"); + DRM_ERROR("Not evicting userqueue, timeout waiting for work\n"); return; } - ret = amdgpu_userqueue_suspend_all(uq_mgr); + ret = amdgpu_userqueue_evict_all(uq_mgr); if (ret) { DRM_ERROR("Failed to evict userqueue\n"); return; @@ -770,7 +770,7 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct amdgpu_devi list_add(&userq_mgr->list, &adev->userq_mgr_list); mutex_unlock(&adev->userq_mutex); - INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userqueue_resume_worker); + INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userqueue_restore_worker); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.h index 8f392a0947a2..a9f0e46bcec0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.h @@ -108,8 +108,8 @@ int amdgpu_userqueue_create_object(struct amdgpu_userq_mgr *uq_mgr, void amdgpu_userqueue_destroy_object(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_userq_obj *userq_obj); -void amdgpu_userqueue_suspend(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_eviction_fence *ev_fence); +void amdgpu_userqueue_evict(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_eviction_fence *ev_fence); int amdgpu_userqueue_active(struct amdgpu_userq_mgr *uq_mgr); -- cgit v1.2.3 From 42a66677805d03df9e2600fab82d0cbe855500e1 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Wed, 16 Apr 2025 17:49:45 -0400 Subject: drm/amdgpu/userq: use consistent function naming s/userqueue/userq/ 1. remove the mix of amdgpu_userqueue and amdgpu_userq 2. to be consistent with other amdgpu_userq_fence.c 3. it's shorter Reviewed-by: Prike Liang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/Makefile | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c | 915 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h | 133 +++ drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c | 915 --------------------- drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.h | 133 --- drivers/gpu/drm/amd/amdgpu/mes_userqueue.c | 12 +- drivers/gpu/drm/amd/amdgpu/mes_userqueue.h | 2 +- 13 files changed, 1064 insertions(+), 1064 deletions(-) create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h delete mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c delete mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.h (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index 513c4d64f554..8595e05c691b 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -257,7 +257,7 @@ amdgpu-y += \ amdgpu-y += amdgpu_amdkfd.o # add gfx usermode queue -amdgpu-y += amdgpu_userqueue.o +amdgpu-y += amdgpu_userq.o ifneq ($(CONFIG_HSA_AMD),) AMDKFD_PATH := ../amdkfd diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index decf66c2a718..cc26cf1bd843 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -113,7 +113,7 @@ #include "amdgpu_xcp.h" #include "amdgpu_seq64.h" #include "amdgpu_reg_state.h" -#include "amdgpu_userqueue.h" +#include "amdgpu_userq.h" #include "amdgpu_eviction_fence.h" #if defined(CONFIG_DRM_AMD_ISP) #include "amdgpu_isp.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index e24b0c730baf..b9a1ef343c79 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -51,7 +51,7 @@ #include "amdgpu_reset.h" #include "amdgpu_sched.h" #include "amdgpu_xgmi.h" -#include "amdgpu_userqueue.h" +#include "amdgpu_userq.h" #include "amdgpu_userq_fence.h" #include "../amdxcp/amdgpu_xcp_drv.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index 02164bca51a7..faa3f59b20c5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -112,7 +112,7 @@ amdgpu_eviction_fence_suspend_worker(struct work_struct *work) if (!ev_fence) goto unlock; - amdgpu_userqueue_evict(uq_mgr, ev_fence); + amdgpu_userq_evict(uq_mgr, ev_fence); unlock: mutex_unlock(&uq_mgr->userq_mutex); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 151366ecc0af..8f992314c5a1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -45,7 +45,7 @@ #include "amdgpu_ras.h" #include "amdgpu_reset.h" #include "amd_pcie.h" -#include "amdgpu_userqueue.h" +#include "amdgpu_userq.h" void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev) { @@ -1009,7 +1009,7 @@ out: } } - dev_info->userq_ip_mask = amdgpu_userqueue_get_supported_ip_mask(adev); + dev_info->userq_ip_mask = amdgpu_userq_get_supported_ip_mask(adev); ret = copy_to_user(out, dev_info, min((size_t)size, sizeof(*dev_info))) ? -EFAULT : 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c new file mode 100644 index 000000000000..4be72bebcf34 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -0,0 +1,915 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright 2023 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include +#include + +#include "amdgpu.h" +#include "amdgpu_vm.h" +#include "amdgpu_userq.h" +#include "amdgpu_userq_fence.h" + +u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev) +{ + int i; + u32 userq_ip_mask = 0; + + for (i = 0; i < AMDGPU_HW_IP_NUM; i++) { + if (adev->userq_funcs[i]) + userq_ip_mask |= (1 << i); + } + + return userq_ip_mask; +} + +static int +amdgpu_userq_unmap_helper(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_usermode_queue *queue) +{ + struct amdgpu_device *adev = uq_mgr->adev; + const struct amdgpu_userq_funcs *userq_funcs = + adev->userq_funcs[queue->queue_type]; + int r = 0; + + if (queue->state == AMDGPU_USERQ_STATE_MAPPED) { + r = userq_funcs->unmap(uq_mgr, queue); + if (r) + queue->state = AMDGPU_USERQ_STATE_HUNG; + else + queue->state = AMDGPU_USERQ_STATE_UNMAPPED; + } + return r; +} + +static int +amdgpu_userq_map_helper(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_usermode_queue *queue) +{ + struct amdgpu_device *adev = uq_mgr->adev; + const struct amdgpu_userq_funcs *userq_funcs = + adev->userq_funcs[queue->queue_type]; + int r = 0; + + if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) { + r = userq_funcs->map(uq_mgr, queue); + if (r) { + queue->state = AMDGPU_USERQ_STATE_HUNG; + } else { + queue->state = AMDGPU_USERQ_STATE_MAPPED; + } + } + return r; +} + +static void +amdgpu_userq_wait_for_last_fence(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_usermode_queue *queue) +{ + struct amdgpu_device *adev = uq_mgr->adev; + struct dma_fence *f = queue->last_fence; + int ret; + + if (f && !dma_fence_is_signaled(f)) { + ret = dma_fence_wait_timeout(f, true, msecs_to_jiffies(100)); + if (ret <= 0) + dev_err(adev->dev, "Timed out waiting for fence f=%p\n", f); + } +} + +static void +amdgpu_userq_cleanup(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_usermode_queue *queue, + int queue_id) +{ + struct amdgpu_device *adev = uq_mgr->adev; + const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type]; + + uq_funcs->mqd_destroy(uq_mgr, queue); + amdgpu_userq_fence_driver_free(queue); + idr_remove(&uq_mgr->userq_idr, queue_id); + kfree(queue); +} + +int +amdgpu_userq_active(struct amdgpu_userq_mgr *uq_mgr) +{ + struct amdgpu_usermode_queue *queue; + int queue_id; + int ret = 0; + + mutex_lock(&uq_mgr->userq_mutex); + /* Resume all the queues for this process */ + idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) + ret += queue->state == AMDGPU_USERQ_STATE_MAPPED; + + mutex_unlock(&uq_mgr->userq_mutex); + return ret; +} + +#ifdef CONFIG_DRM_AMDGPU_NAVI3X_USERQ +static struct amdgpu_usermode_queue * +amdgpu_userq_find(struct amdgpu_userq_mgr *uq_mgr, int qid) +{ + return idr_find(&uq_mgr->userq_idr, qid); +} + +void +amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_eviction_fence_mgr *evf_mgr) +{ + struct amdgpu_eviction_fence *ev_fence; + +retry: + /* Flush any pending resume work to create ev_fence */ + flush_delayed_work(&uq_mgr->resume_work); + + mutex_lock(&uq_mgr->userq_mutex); + spin_lock(&evf_mgr->ev_fence_lock); + ev_fence = evf_mgr->ev_fence; + spin_unlock(&evf_mgr->ev_fence_lock); + if (!ev_fence || dma_fence_is_signaled(&ev_fence->base)) { + mutex_unlock(&uq_mgr->userq_mutex); + /* + * Looks like there was no pending resume work, + * add one now to create a valid eviction fence + */ + schedule_delayed_work(&uq_mgr->resume_work, 0); + goto retry; + } +} + +int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_userq_obj *userq_obj, + int size) +{ + struct amdgpu_device *adev = uq_mgr->adev; + struct amdgpu_bo_param bp; + int r; + + memset(&bp, 0, sizeof(bp)); + bp.byte_align = PAGE_SIZE; + bp.domain = AMDGPU_GEM_DOMAIN_GTT; + bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + bp.type = ttm_bo_type_kernel; + bp.size = size; + bp.resv = NULL; + bp.bo_ptr_size = sizeof(struct amdgpu_bo); + + r = amdgpu_bo_create(adev, &bp, &userq_obj->obj); + if (r) { + DRM_ERROR("Failed to allocate BO for userqueue (%d)", r); + return r; + } + + r = amdgpu_bo_reserve(userq_obj->obj, true); + if (r) { + DRM_ERROR("Failed to reserve BO to map (%d)", r); + goto free_obj; + } + + r = amdgpu_ttm_alloc_gart(&(userq_obj->obj)->tbo); + if (r) { + DRM_ERROR("Failed to alloc GART for userqueue object (%d)", r); + goto unresv; + } + + r = amdgpu_bo_kmap(userq_obj->obj, &userq_obj->cpu_ptr); + if (r) { + DRM_ERROR("Failed to map BO for userqueue (%d)", r); + goto unresv; + } + + userq_obj->gpu_addr = amdgpu_bo_gpu_offset(userq_obj->obj); + amdgpu_bo_unreserve(userq_obj->obj); + memset(userq_obj->cpu_ptr, 0, size); + return 0; + +unresv: + amdgpu_bo_unreserve(userq_obj->obj); + +free_obj: + amdgpu_bo_unref(&userq_obj->obj); + return r; +} + +void amdgpu_userq_destroy_object(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_userq_obj *userq_obj) +{ + amdgpu_bo_kunmap(userq_obj->obj); + amdgpu_bo_unref(&userq_obj->obj); +} + +uint64_t +amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_db_info *db_info, + struct drm_file *filp) +{ + uint64_t index; + struct drm_gem_object *gobj; + struct amdgpu_userq_obj *db_obj = db_info->db_obj; + int r, db_size; + + gobj = drm_gem_object_lookup(filp, db_info->doorbell_handle); + if (gobj == NULL) { + DRM_ERROR("Can't find GEM object for doorbell\n"); + return -EINVAL; + } + + db_obj->obj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); + drm_gem_object_put(gobj); + + /* Pin the BO before generating the index, unpin in queue destroy */ + r = amdgpu_bo_pin(db_obj->obj, AMDGPU_GEM_DOMAIN_DOORBELL); + if (r) { + DRM_ERROR("[Usermode queues] Failed to pin doorbell object\n"); + goto unref_bo; + } + + r = amdgpu_bo_reserve(db_obj->obj, true); + if (r) { + DRM_ERROR("[Usermode queues] Failed to pin doorbell object\n"); + goto unpin_bo; + } + + switch (db_info->queue_type) { + case AMDGPU_HW_IP_GFX: + case AMDGPU_HW_IP_COMPUTE: + case AMDGPU_HW_IP_DMA: + db_size = sizeof(u64); + break; + + case AMDGPU_HW_IP_VCN_ENC: + db_size = sizeof(u32); + db_info->doorbell_offset += AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1; + break; + + case AMDGPU_HW_IP_VPE: + db_size = sizeof(u32); + db_info->doorbell_offset += AMDGPU_NAVI10_DOORBELL64_VPE << 1; + break; + + default: + DRM_ERROR("[Usermode queues] IP %d not support\n", db_info->queue_type); + r = -EINVAL; + goto unpin_bo; + } + + index = amdgpu_doorbell_index_on_bar(uq_mgr->adev, db_obj->obj, + db_info->doorbell_offset, db_size); + DRM_DEBUG_DRIVER("[Usermode queues] doorbell index=%lld\n", index); + amdgpu_bo_unreserve(db_obj->obj); + return index; + +unpin_bo: + amdgpu_bo_unpin(db_obj->obj); + +unref_bo: + amdgpu_bo_unref(&db_obj->obj); + return r; +} + +static int +amdgpu_userq_destroy(struct drm_file *filp, int queue_id) +{ + struct amdgpu_fpriv *fpriv = filp->driver_priv; + struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; + struct amdgpu_device *adev = uq_mgr->adev; + struct amdgpu_usermode_queue *queue; + int r = 0; + + cancel_delayed_work(&uq_mgr->resume_work); + mutex_lock(&uq_mgr->userq_mutex); + + queue = amdgpu_userq_find(uq_mgr, queue_id); + if (!queue) { + DRM_DEBUG_DRIVER("Invalid queue id to destroy\n"); + mutex_unlock(&uq_mgr->userq_mutex); + return -EINVAL; + } + amdgpu_userq_wait_for_last_fence(uq_mgr, queue); + r = amdgpu_userq_unmap_helper(uq_mgr, queue); + amdgpu_bo_unpin(queue->db_obj.obj); + amdgpu_bo_unref(&queue->db_obj.obj); + amdgpu_userq_cleanup(uq_mgr, queue, queue_id); + mutex_unlock(&uq_mgr->userq_mutex); + + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); + + return r; +} + +static int amdgpu_userq_priority_permit(struct drm_file *filp, + int priority) +{ + if (priority < AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH) + return 0; + + if (capable(CAP_SYS_NICE)) + return 0; + + if (drm_is_current_master(filp)) + return 0; + + return -EACCES; +} + +static int +amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) +{ + struct amdgpu_fpriv *fpriv = filp->driver_priv; + struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; + struct amdgpu_device *adev = uq_mgr->adev; + const struct amdgpu_userq_funcs *uq_funcs; + struct amdgpu_usermode_queue *queue; + struct amdgpu_db_info db_info; + bool skip_map_queue; + uint64_t index; + int qid, r = 0; + int priority = + (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK) >> + AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT; + + /* Usermode queues are only supported for GFX IP as of now */ + if (args->in.ip_type != AMDGPU_HW_IP_GFX && + args->in.ip_type != AMDGPU_HW_IP_DMA && + args->in.ip_type != AMDGPU_HW_IP_COMPUTE) { + DRM_ERROR("Usermode queue doesn't support IP type %u\n", args->in.ip_type); + return -EINVAL; + } + + r = amdgpu_userq_priority_permit(filp, priority); + if (r) + return r; + + if ((args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE) && + (args->in.ip_type != AMDGPU_HW_IP_GFX) && + (args->in.ip_type != AMDGPU_HW_IP_COMPUTE) && + !amdgpu_is_tmz(adev)) { + drm_err(adev_to_drm(adev), "Secure only supported on GFX/Compute queues\n"); + return -EINVAL; + } + + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); + if (r < 0) { + dev_err(adev->dev, "pm_runtime_get_sync() failed for userqueue create\n"); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); + return r; + } + + /* + * There could be a situation that we are creating a new queue while + * the other queues under this UQ_mgr are suspended. So if there is any + * resume work pending, wait for it to get done. + * + * This will also make sure we have a valid eviction fence ready to be used. + */ + amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); + + uq_funcs = adev->userq_funcs[args->in.ip_type]; + if (!uq_funcs) { + DRM_ERROR("Usermode queue is not supported for this IP (%u)\n", args->in.ip_type); + r = -EINVAL; + goto unlock; + } + + queue = kzalloc(sizeof(struct amdgpu_usermode_queue), GFP_KERNEL); + if (!queue) { + DRM_ERROR("Failed to allocate memory for queue\n"); + r = -ENOMEM; + goto unlock; + } + queue->doorbell_handle = args->in.doorbell_handle; + queue->queue_type = args->in.ip_type; + queue->vm = &fpriv->vm; + queue->priority = priority; + + db_info.queue_type = queue->queue_type; + db_info.doorbell_handle = queue->doorbell_handle; + db_info.db_obj = &queue->db_obj; + db_info.doorbell_offset = args->in.doorbell_offset; + + /* Convert relative doorbell offset into absolute doorbell index */ + index = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp); + if (index == (uint64_t)-EINVAL) { + DRM_ERROR("Failed to get doorbell for queue\n"); + kfree(queue); + goto unlock; + } + + queue->doorbell_index = index; + xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC); + r = amdgpu_userq_fence_driver_alloc(adev, queue); + if (r) { + DRM_ERROR("Failed to alloc fence driver\n"); + goto unlock; + } + + r = uq_funcs->mqd_create(uq_mgr, &args->in, queue); + if (r) { + DRM_ERROR("Failed to create Queue\n"); + amdgpu_userq_fence_driver_free(queue); + kfree(queue); + goto unlock; + } + + + qid = idr_alloc(&uq_mgr->userq_idr, queue, 1, AMDGPU_MAX_USERQ_COUNT, GFP_KERNEL); + if (qid < 0) { + DRM_ERROR("Failed to allocate a queue id\n"); + amdgpu_userq_fence_driver_free(queue); + uq_funcs->mqd_destroy(uq_mgr, queue); + kfree(queue); + r = -ENOMEM; + goto unlock; + } + + /* don't map the queue if scheduling is halted */ + mutex_lock(&adev->userq_mutex); + if (adev->userq_halt_for_enforce_isolation && + ((queue->queue_type == AMDGPU_HW_IP_GFX) || + (queue->queue_type == AMDGPU_HW_IP_COMPUTE))) + skip_map_queue = true; + else + skip_map_queue = false; + if (!skip_map_queue) { + r = amdgpu_userq_map_helper(uq_mgr, queue); + if (r) { + mutex_unlock(&adev->userq_mutex); + DRM_ERROR("Failed to map Queue\n"); + idr_remove(&uq_mgr->userq_idr, qid); + amdgpu_userq_fence_driver_free(queue); + uq_funcs->mqd_destroy(uq_mgr, queue); + kfree(queue); + goto unlock; + } + } + mutex_unlock(&adev->userq_mutex); + + + args->out.queue_id = qid; + +unlock: + mutex_unlock(&uq_mgr->userq_mutex); + + return r; +} + +int amdgpu_userq_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + union drm_amdgpu_userq *args = data; + int r; + + switch (args->in.op) { + case AMDGPU_USERQ_OP_CREATE: + if (args->in.flags & ~(AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK | + AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE)) + return -EINVAL; + r = amdgpu_userq_create(filp, args); + if (r) + DRM_ERROR("Failed to create usermode queue\n"); + break; + + case AMDGPU_USERQ_OP_FREE: + if (args->in.ip_type || + args->in.doorbell_handle || + args->in.doorbell_offset || + args->in.flags || + args->in.queue_va || + args->in.queue_size || + args->in.rptr_va || + args->in.wptr_va || + args->in.wptr_va || + args->in.mqd || + args->in.mqd_size) + return -EINVAL; + r = amdgpu_userq_destroy(filp, args->in.queue_id); + if (r) + DRM_ERROR("Failed to destroy usermode queue\n"); + break; + + default: + DRM_DEBUG_DRIVER("Invalid user queue op specified: %d\n", args->in.op); + return -EINVAL; + } + + return r; +} +#else +int amdgpu_userq_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp) +{ + return -ENOTSUPP; +} +#endif + +static int +amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr) +{ + struct amdgpu_device *adev = uq_mgr->adev; + struct amdgpu_usermode_queue *queue; + int queue_id; + int ret = 0, r; + + /* Resume all the queues for this process */ + idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) { + r = amdgpu_userq_map_helper(uq_mgr, queue); + if (r) + ret = r; + } + + if (ret) + dev_err(adev->dev, "Failed to map all the queues\n"); + return ret; +} + +static int +amdgpu_userq_validate_vm_bo(void *_unused, struct amdgpu_bo *bo) +{ + struct ttm_operation_ctx ctx = { false, false }; + int ret; + + amdgpu_bo_placement_from_domain(bo, bo->allowed_domains); + + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + if (ret) + DRM_ERROR("Fail to validate\n"); + + return ret; +} + +static int +amdgpu_userq_validate_bos(struct amdgpu_userq_mgr *uq_mgr) +{ + struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); + struct amdgpu_vm *vm = &fpriv->vm; + struct amdgpu_device *adev = uq_mgr->adev; + struct amdgpu_bo_va *bo_va; + struct ww_acquire_ctx *ticket; + struct drm_exec exec; + struct amdgpu_bo *bo; + struct dma_resv *resv; + bool clear, unlock; + int ret = 0; + + drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); + drm_exec_until_all_locked(&exec) { + ret = amdgpu_vm_lock_pd(vm, &exec, 2); + drm_exec_retry_on_contention(&exec); + if (unlikely(ret)) { + DRM_ERROR("Failed to lock PD\n"); + goto unlock_all; + } + + /* Lock the done list */ + list_for_each_entry(bo_va, &vm->done, base.vm_status) { + bo = bo_va->base.bo; + if (!bo) + continue; + + ret = drm_exec_lock_obj(&exec, &bo->tbo.base); + drm_exec_retry_on_contention(&exec); + if (unlikely(ret)) + goto unlock_all; + } + } + + spin_lock(&vm->status_lock); + while (!list_empty(&vm->moved)) { + bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va, + base.vm_status); + spin_unlock(&vm->status_lock); + + /* Per VM BOs never need to bo cleared in the page tables */ + ret = amdgpu_vm_bo_update(adev, bo_va, false); + if (ret) + goto unlock_all; + spin_lock(&vm->status_lock); + } + + ticket = &exec.ticket; + while (!list_empty(&vm->invalidated)) { + bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, + base.vm_status); + resv = bo_va->base.bo->tbo.base.resv; + spin_unlock(&vm->status_lock); + + bo = bo_va->base.bo; + ret = amdgpu_userq_validate_vm_bo(NULL, bo); + if (ret) { + DRM_ERROR("Failed to validate BO\n"); + goto unlock_all; + } + + /* Try to reserve the BO to avoid clearing its ptes */ + if (!adev->debug_vm && dma_resv_trylock(resv)) { + clear = false; + unlock = true; + /* The caller is already holding the reservation lock */ + } else if (ticket && dma_resv_locking_ctx(resv) == ticket) { + clear = false; + unlock = false; + /* Somebody else is using the BO right now */ + } else { + clear = true; + unlock = false; + } + + ret = amdgpu_vm_bo_update(adev, bo_va, clear); + + if (unlock) + dma_resv_unlock(resv); + if (ret) + goto unlock_all; + + spin_lock(&vm->status_lock); + } + spin_unlock(&vm->status_lock); + + ret = amdgpu_eviction_fence_replace_fence(&fpriv->evf_mgr, &exec); + if (ret) + DRM_ERROR("Failed to replace eviction fence\n"); + +unlock_all: + drm_exec_fini(&exec); + return ret; +} + +static void amdgpu_userq_restore_worker(struct work_struct *work) +{ + struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, resume_work.work); + struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); + int ret; + + flush_work(&fpriv->evf_mgr.suspend_work.work); + + mutex_lock(&uq_mgr->userq_mutex); + + ret = amdgpu_userq_validate_bos(uq_mgr); + if (ret) { + DRM_ERROR("Failed to validate BOs to restore\n"); + goto unlock; + } + + ret = amdgpu_userq_restore_all(uq_mgr); + if (ret) { + DRM_ERROR("Failed to restore all queues\n"); + goto unlock; + } + +unlock: + mutex_unlock(&uq_mgr->userq_mutex); +} + +static int +amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr) +{ + struct amdgpu_device *adev = uq_mgr->adev; + struct amdgpu_usermode_queue *queue; + int queue_id; + int ret = 0, r; + + /* Try to unmap all the queues in this process ctx */ + idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) { + r = amdgpu_userq_unmap_helper(uq_mgr, queue); + if (r) + ret = r; + } + + if (ret) + dev_err(adev->dev, "Couldn't unmap all the queues\n"); + return ret; +} + +static int +amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr) +{ + struct amdgpu_usermode_queue *queue; + int queue_id, ret; + + idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) { + struct dma_fence *f = queue->last_fence; + + if (!f || dma_fence_is_signaled(f)) + continue; + ret = dma_fence_wait_timeout(f, true, msecs_to_jiffies(100)); + if (ret <= 0) { + DRM_ERROR("Timed out waiting for fence=%llu:%llu\n", + f->context, f->seqno); + return -ETIMEDOUT; + } + } + + return 0; +} + +void +amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_eviction_fence *ev_fence) +{ + int ret; + struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); + struct amdgpu_eviction_fence_mgr *evf_mgr = &fpriv->evf_mgr; + + /* Wait for any pending userqueue fence work to finish */ + ret = amdgpu_userq_wait_for_signal(uq_mgr); + if (ret) { + DRM_ERROR("Not evicting userqueue, timeout waiting for work\n"); + return; + } + + ret = amdgpu_userq_evict_all(uq_mgr); + if (ret) { + DRM_ERROR("Failed to evict userqueue\n"); + return; + } + + /* Signal current eviction fence */ + amdgpu_eviction_fence_signal(evf_mgr, ev_fence); + + if (evf_mgr->fd_closing) { + cancel_delayed_work(&uq_mgr->resume_work); + return; + } + + /* Schedule a resume work */ + schedule_delayed_work(&uq_mgr->resume_work, 0); +} + +int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct amdgpu_device *adev) +{ + mutex_init(&userq_mgr->userq_mutex); + idr_init_base(&userq_mgr->userq_idr, 1); + userq_mgr->adev = adev; + + mutex_lock(&adev->userq_mutex); + list_add(&userq_mgr->list, &adev->userq_mgr_list); + mutex_unlock(&adev->userq_mutex); + + INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker); + return 0; +} + +void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr) +{ + struct amdgpu_device *adev = userq_mgr->adev; + struct amdgpu_usermode_queue *queue; + struct amdgpu_userq_mgr *uqm, *tmp; + uint32_t queue_id; + + cancel_delayed_work(&userq_mgr->resume_work); + + mutex_lock(&userq_mgr->userq_mutex); + idr_for_each_entry(&userq_mgr->userq_idr, queue, queue_id) { + amdgpu_userq_wait_for_last_fence(userq_mgr, queue); + amdgpu_userq_unmap_helper(userq_mgr, queue); + amdgpu_userq_cleanup(userq_mgr, queue, queue_id); + } + mutex_lock(&adev->userq_mutex); + list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { + if (uqm == userq_mgr) { + list_del(&uqm->list); + break; + } + } + mutex_unlock(&adev->userq_mutex); + idr_destroy(&userq_mgr->userq_idr); + mutex_unlock(&userq_mgr->userq_mutex); + mutex_destroy(&userq_mgr->userq_mutex); +} + +int amdgpu_userq_suspend(struct amdgpu_device *adev) +{ + u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev); + struct amdgpu_usermode_queue *queue; + struct amdgpu_userq_mgr *uqm, *tmp; + int queue_id; + int ret = 0, r; + + if (!ip_mask) + return 0; + + mutex_lock(&adev->userq_mutex); + list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { + cancel_delayed_work_sync(&uqm->resume_work); + idr_for_each_entry(&uqm->userq_idr, queue, queue_id) { + r = amdgpu_userq_unmap_helper(uqm, queue); + if (r) + ret = r; + } + } + mutex_unlock(&adev->userq_mutex); + return ret; +} + +int amdgpu_userq_resume(struct amdgpu_device *adev) +{ + u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev); + struct amdgpu_usermode_queue *queue; + struct amdgpu_userq_mgr *uqm, *tmp; + int queue_id; + int ret = 0, r; + + if (!ip_mask) + return 0; + + mutex_lock(&adev->userq_mutex); + list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { + idr_for_each_entry(&uqm->userq_idr, queue, queue_id) { + r = amdgpu_userq_map_helper(uqm, queue); + if (r) + ret = r; + } + } + mutex_unlock(&adev->userq_mutex); + return ret; +} + +int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev, + u32 idx) +{ + u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev); + struct amdgpu_usermode_queue *queue; + struct amdgpu_userq_mgr *uqm, *tmp; + int queue_id; + int ret = 0, r; + + /* only need to stop gfx/compute */ + if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE)))) + return 0; + + mutex_lock(&adev->userq_mutex); + if (adev->userq_halt_for_enforce_isolation) + dev_warn(adev->dev, "userq scheduling already stopped!\n"); + adev->userq_halt_for_enforce_isolation = true; + list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { + cancel_delayed_work_sync(&uqm->resume_work); + idr_for_each_entry(&uqm->userq_idr, queue, queue_id) { + if (((queue->queue_type == AMDGPU_HW_IP_GFX) || + (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) && + (queue->xcp_id == idx)) { + r = amdgpu_userq_unmap_helper(uqm, queue); + if (r) + ret = r; + } + } + } + mutex_unlock(&adev->userq_mutex); + return ret; +} + +int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev, + u32 idx) +{ + u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev); + struct amdgpu_usermode_queue *queue; + struct amdgpu_userq_mgr *uqm, *tmp; + int queue_id; + int ret = 0, r; + + /* only need to stop gfx/compute */ + if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE)))) + return 0; + + mutex_lock(&adev->userq_mutex); + if (!adev->userq_halt_for_enforce_isolation) + dev_warn(adev->dev, "userq scheduling already started!\n"); + adev->userq_halt_for_enforce_isolation = false; + list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { + idr_for_each_entry(&uqm->userq_idr, queue, queue_id) { + if (((queue->queue_type == AMDGPU_HW_IP_GFX) || + (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) && + (queue->xcp_id == idx)) { + r = amdgpu_userq_map_helper(uqm, queue); + if (r) + ret = r; + } + } + } + mutex_unlock(&adev->userq_mutex); + return ret; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h new file mode 100644 index 000000000000..4d3eb651acf1 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright 2023 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef AMDGPU_USERQ_H_ +#define AMDGPU_USERQ_H_ +#include "amdgpu_eviction_fence.h" + +#define AMDGPU_MAX_USERQ_COUNT 512 + +#define to_ev_fence(f) container_of(f, struct amdgpu_eviction_fence, base) +#define uq_mgr_to_fpriv(u) container_of(u, struct amdgpu_fpriv, userq_mgr) +#define work_to_uq_mgr(w, name) container_of(w, struct amdgpu_userq_mgr, name) + +enum amdgpu_userq_state { + AMDGPU_USERQ_STATE_UNMAPPED = 0, + AMDGPU_USERQ_STATE_MAPPED, + AMDGPU_USERQ_STATE_PREEMPTED, + AMDGPU_USERQ_STATE_HUNG, +}; + +struct amdgpu_mqd_prop; + +struct amdgpu_userq_obj { + void *cpu_ptr; + uint64_t gpu_addr; + struct amdgpu_bo *obj; +}; + +struct amdgpu_usermode_queue { + int queue_type; + enum amdgpu_userq_state state; + uint64_t doorbell_handle; + uint64_t doorbell_index; + uint64_t flags; + struct amdgpu_mqd_prop *userq_prop; + struct amdgpu_userq_mgr *userq_mgr; + struct amdgpu_vm *vm; + struct amdgpu_userq_obj mqd; + struct amdgpu_userq_obj db_obj; + struct amdgpu_userq_obj fw_obj; + struct amdgpu_userq_obj wptr_obj; + struct xarray fence_drv_xa; + struct amdgpu_userq_fence_driver *fence_drv; + struct dma_fence *last_fence; + u32 xcp_id; + int priority; +}; + +struct amdgpu_userq_funcs { + int (*mqd_create)(struct amdgpu_userq_mgr *uq_mgr, + struct drm_amdgpu_userq_in *args, + struct amdgpu_usermode_queue *queue); + void (*mqd_destroy)(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_usermode_queue *uq); + int (*unmap)(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_usermode_queue *queue); + int (*map)(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_usermode_queue *queue); +}; + +/* Usermode queues for gfx */ +struct amdgpu_userq_mgr { + struct idr userq_idr; + struct mutex userq_mutex; + struct amdgpu_device *adev; + struct delayed_work resume_work; + struct list_head list; +}; + +struct amdgpu_db_info { + uint64_t doorbell_handle; + uint32_t queue_type; + uint32_t doorbell_offset; + struct amdgpu_userq_obj *db_obj; +}; + +int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); + +int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct amdgpu_device *adev); + +void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr); + +int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_userq_obj *userq_obj, + int size); + +void amdgpu_userq_destroy_object(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_userq_obj *userq_obj); + +void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_eviction_fence *ev_fence); + +int amdgpu_userq_active(struct amdgpu_userq_mgr *uq_mgr); + +void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr, + struct amdgpu_eviction_fence_mgr *evf_mgr); + +uint64_t amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr, + struct amdgpu_db_info *db_info, + struct drm_file *filp); + +u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev); + +int amdgpu_userq_suspend(struct amdgpu_device *adev); +int amdgpu_userq_resume(struct amdgpu_device *adev); + +int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev, + u32 idx); +int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev, + u32 idx); + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c index ca198360cfda..be068e8e37d1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c @@ -292,7 +292,7 @@ static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq, static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f) { - return "amdgpu_userqueue_fence"; + return "amdgpu_userq_fence"; } static const char *amdgpu_userq_fence_get_timeline_name(struct dma_fence *f) @@ -513,7 +513,7 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data, goto put_gobj_write; /* We are here means UQ is active, make sure the eviction fence is valid */ - amdgpu_userqueue_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); + amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); /* Create a new fence */ r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h index 2af4e0c15773..97a125ab8a78 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.h @@ -27,7 +27,7 @@ #include -#include "amdgpu_userqueue.h" +#include "amdgpu_userq.h" struct amdgpu_userq_fence { struct dma_fence base; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c deleted file mode 100644 index 82741dcb2a45..000000000000 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.c +++ /dev/null @@ -1,915 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include -#include -#include - -#include "amdgpu.h" -#include "amdgpu_vm.h" -#include "amdgpu_userqueue.h" -#include "amdgpu_userq_fence.h" - -u32 amdgpu_userqueue_get_supported_ip_mask(struct amdgpu_device *adev) -{ - int i; - u32 userq_ip_mask = 0; - - for (i = 0; i < AMDGPU_HW_IP_NUM; i++) { - if (adev->userq_funcs[i]) - userq_ip_mask |= (1 << i); - } - - return userq_ip_mask; -} - -static int -amdgpu_userqueue_unmap_helper(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_usermode_queue *queue) -{ - struct amdgpu_device *adev = uq_mgr->adev; - const struct amdgpu_userq_funcs *userq_funcs = - adev->userq_funcs[queue->queue_type]; - int r = 0; - - if (queue->state == AMDGPU_USERQ_STATE_MAPPED) { - r = userq_funcs->unmap(uq_mgr, queue); - if (r) - queue->state = AMDGPU_USERQ_STATE_HUNG; - else - queue->state = AMDGPU_USERQ_STATE_UNMAPPED; - } - return r; -} - -static int -amdgpu_userqueue_map_helper(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_usermode_queue *queue) -{ - struct amdgpu_device *adev = uq_mgr->adev; - const struct amdgpu_userq_funcs *userq_funcs = - adev->userq_funcs[queue->queue_type]; - int r = 0; - - if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) { - r = userq_funcs->map(uq_mgr, queue); - if (r) { - queue->state = AMDGPU_USERQ_STATE_HUNG; - } else { - queue->state = AMDGPU_USERQ_STATE_MAPPED; - } - } - return r; -} - -static void -amdgpu_userqueue_wait_for_last_fence(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_usermode_queue *queue) -{ - struct amdgpu_device *adev = uq_mgr->adev; - struct dma_fence *f = queue->last_fence; - int ret; - - if (f && !dma_fence_is_signaled(f)) { - ret = dma_fence_wait_timeout(f, true, msecs_to_jiffies(100)); - if (ret <= 0) - dev_err(adev->dev, "Timed out waiting for fence f=%p\n", f); - } -} - -static void -amdgpu_userqueue_cleanup(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_usermode_queue *queue, - int queue_id) -{ - struct amdgpu_device *adev = uq_mgr->adev; - const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type]; - - uq_funcs->mqd_destroy(uq_mgr, queue); - amdgpu_userq_fence_driver_free(queue); - idr_remove(&uq_mgr->userq_idr, queue_id); - kfree(queue); -} - -int -amdgpu_userqueue_active(struct amdgpu_userq_mgr *uq_mgr) -{ - struct amdgpu_usermode_queue *queue; - int queue_id; - int ret = 0; - - mutex_lock(&uq_mgr->userq_mutex); - /* Resume all the queues for this process */ - idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) - ret += queue->state == AMDGPU_USERQ_STATE_MAPPED; - - mutex_unlock(&uq_mgr->userq_mutex); - return ret; -} - -#ifdef CONFIG_DRM_AMDGPU_NAVI3X_USERQ -static struct amdgpu_usermode_queue * -amdgpu_userqueue_find(struct amdgpu_userq_mgr *uq_mgr, int qid) -{ - return idr_find(&uq_mgr->userq_idr, qid); -} - -void -amdgpu_userqueue_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_eviction_fence_mgr *evf_mgr) -{ - struct amdgpu_eviction_fence *ev_fence; - -retry: - /* Flush any pending resume work to create ev_fence */ - flush_delayed_work(&uq_mgr->resume_work); - - mutex_lock(&uq_mgr->userq_mutex); - spin_lock(&evf_mgr->ev_fence_lock); - ev_fence = evf_mgr->ev_fence; - spin_unlock(&evf_mgr->ev_fence_lock); - if (!ev_fence || dma_fence_is_signaled(&ev_fence->base)) { - mutex_unlock(&uq_mgr->userq_mutex); - /* - * Looks like there was no pending resume work, - * add one now to create a valid eviction fence - */ - schedule_delayed_work(&uq_mgr->resume_work, 0); - goto retry; - } -} - -int amdgpu_userqueue_create_object(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_userq_obj *userq_obj, - int size) -{ - struct amdgpu_device *adev = uq_mgr->adev; - struct amdgpu_bo_param bp; - int r; - - memset(&bp, 0, sizeof(bp)); - bp.byte_align = PAGE_SIZE; - bp.domain = AMDGPU_GEM_DOMAIN_GTT; - bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | - AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; - bp.type = ttm_bo_type_kernel; - bp.size = size; - bp.resv = NULL; - bp.bo_ptr_size = sizeof(struct amdgpu_bo); - - r = amdgpu_bo_create(adev, &bp, &userq_obj->obj); - if (r) { - DRM_ERROR("Failed to allocate BO for userqueue (%d)", r); - return r; - } - - r = amdgpu_bo_reserve(userq_obj->obj, true); - if (r) { - DRM_ERROR("Failed to reserve BO to map (%d)", r); - goto free_obj; - } - - r = amdgpu_ttm_alloc_gart(&(userq_obj->obj)->tbo); - if (r) { - DRM_ERROR("Failed to alloc GART for userqueue object (%d)", r); - goto unresv; - } - - r = amdgpu_bo_kmap(userq_obj->obj, &userq_obj->cpu_ptr); - if (r) { - DRM_ERROR("Failed to map BO for userqueue (%d)", r); - goto unresv; - } - - userq_obj->gpu_addr = amdgpu_bo_gpu_offset(userq_obj->obj); - amdgpu_bo_unreserve(userq_obj->obj); - memset(userq_obj->cpu_ptr, 0, size); - return 0; - -unresv: - amdgpu_bo_unreserve(userq_obj->obj); - -free_obj: - amdgpu_bo_unref(&userq_obj->obj); - return r; -} - -void amdgpu_userqueue_destroy_object(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_userq_obj *userq_obj) -{ - amdgpu_bo_kunmap(userq_obj->obj); - amdgpu_bo_unref(&userq_obj->obj); -} - -uint64_t -amdgpu_userqueue_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_db_info *db_info, - struct drm_file *filp) -{ - uint64_t index; - struct drm_gem_object *gobj; - struct amdgpu_userq_obj *db_obj = db_info->db_obj; - int r, db_size; - - gobj = drm_gem_object_lookup(filp, db_info->doorbell_handle); - if (gobj == NULL) { - DRM_ERROR("Can't find GEM object for doorbell\n"); - return -EINVAL; - } - - db_obj->obj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); - drm_gem_object_put(gobj); - - /* Pin the BO before generating the index, unpin in queue destroy */ - r = amdgpu_bo_pin(db_obj->obj, AMDGPU_GEM_DOMAIN_DOORBELL); - if (r) { - DRM_ERROR("[Usermode queues] Failed to pin doorbell object\n"); - goto unref_bo; - } - - r = amdgpu_bo_reserve(db_obj->obj, true); - if (r) { - DRM_ERROR("[Usermode queues] Failed to pin doorbell object\n"); - goto unpin_bo; - } - - switch (db_info->queue_type) { - case AMDGPU_HW_IP_GFX: - case AMDGPU_HW_IP_COMPUTE: - case AMDGPU_HW_IP_DMA: - db_size = sizeof(u64); - break; - - case AMDGPU_HW_IP_VCN_ENC: - db_size = sizeof(u32); - db_info->doorbell_offset += AMDGPU_NAVI10_DOORBELL64_VCN0_1 << 1; - break; - - case AMDGPU_HW_IP_VPE: - db_size = sizeof(u32); - db_info->doorbell_offset += AMDGPU_NAVI10_DOORBELL64_VPE << 1; - break; - - default: - DRM_ERROR("[Usermode queues] IP %d not support\n", db_info->queue_type); - r = -EINVAL; - goto unpin_bo; - } - - index = amdgpu_doorbell_index_on_bar(uq_mgr->adev, db_obj->obj, - db_info->doorbell_offset, db_size); - DRM_DEBUG_DRIVER("[Usermode queues] doorbell index=%lld\n", index); - amdgpu_bo_unreserve(db_obj->obj); - return index; - -unpin_bo: - amdgpu_bo_unpin(db_obj->obj); - -unref_bo: - amdgpu_bo_unref(&db_obj->obj); - return r; -} - -static int -amdgpu_userqueue_destroy(struct drm_file *filp, int queue_id) -{ - struct amdgpu_fpriv *fpriv = filp->driver_priv; - struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; - struct amdgpu_device *adev = uq_mgr->adev; - struct amdgpu_usermode_queue *queue; - int r = 0; - - cancel_delayed_work(&uq_mgr->resume_work); - mutex_lock(&uq_mgr->userq_mutex); - - queue = amdgpu_userqueue_find(uq_mgr, queue_id); - if (!queue) { - DRM_DEBUG_DRIVER("Invalid queue id to destroy\n"); - mutex_unlock(&uq_mgr->userq_mutex); - return -EINVAL; - } - amdgpu_userqueue_wait_for_last_fence(uq_mgr, queue); - r = amdgpu_userqueue_unmap_helper(uq_mgr, queue); - amdgpu_bo_unpin(queue->db_obj.obj); - amdgpu_bo_unref(&queue->db_obj.obj); - amdgpu_userqueue_cleanup(uq_mgr, queue, queue_id); - mutex_unlock(&uq_mgr->userq_mutex); - - pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); - pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); - - return r; -} - -static int amdgpu_userq_priority_permit(struct drm_file *filp, - int priority) -{ - if (priority < AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH) - return 0; - - if (capable(CAP_SYS_NICE)) - return 0; - - if (drm_is_current_master(filp)) - return 0; - - return -EACCES; -} - -static int -amdgpu_userqueue_create(struct drm_file *filp, union drm_amdgpu_userq *args) -{ - struct amdgpu_fpriv *fpriv = filp->driver_priv; - struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; - struct amdgpu_device *adev = uq_mgr->adev; - const struct amdgpu_userq_funcs *uq_funcs; - struct amdgpu_usermode_queue *queue; - struct amdgpu_db_info db_info; - bool skip_map_queue; - uint64_t index; - int qid, r = 0; - int priority = - (args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK) >> - AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT; - - /* Usermode queues are only supported for GFX IP as of now */ - if (args->in.ip_type != AMDGPU_HW_IP_GFX && - args->in.ip_type != AMDGPU_HW_IP_DMA && - args->in.ip_type != AMDGPU_HW_IP_COMPUTE) { - DRM_ERROR("Usermode queue doesn't support IP type %u\n", args->in.ip_type); - return -EINVAL; - } - - r = amdgpu_userq_priority_permit(filp, priority); - if (r) - return r; - - if ((args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE) && - (args->in.ip_type != AMDGPU_HW_IP_GFX) && - (args->in.ip_type != AMDGPU_HW_IP_COMPUTE) && - !amdgpu_is_tmz(adev)) { - drm_err(adev_to_drm(adev), "Secure only supported on GFX/Compute queues\n"); - return -EINVAL; - } - - r = pm_runtime_get_sync(adev_to_drm(adev)->dev); - if (r < 0) { - dev_err(adev->dev, "pm_runtime_get_sync() failed for userqueue create\n"); - pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); - return r; - } - - /* - * There could be a situation that we are creating a new queue while - * the other queues under this UQ_mgr are suspended. So if there is any - * resume work pending, wait for it to get done. - * - * This will also make sure we have a valid eviction fence ready to be used. - */ - amdgpu_userqueue_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr); - - uq_funcs = adev->userq_funcs[args->in.ip_type]; - if (!uq_funcs) { - DRM_ERROR("Usermode queue is not supported for this IP (%u)\n", args->in.ip_type); - r = -EINVAL; - goto unlock; - } - - queue = kzalloc(sizeof(struct amdgpu_usermode_queue), GFP_KERNEL); - if (!queue) { - DRM_ERROR("Failed to allocate memory for queue\n"); - r = -ENOMEM; - goto unlock; - } - queue->doorbell_handle = args->in.doorbell_handle; - queue->queue_type = args->in.ip_type; - queue->vm = &fpriv->vm; - queue->priority = priority; - - db_info.queue_type = queue->queue_type; - db_info.doorbell_handle = queue->doorbell_handle; - db_info.db_obj = &queue->db_obj; - db_info.doorbell_offset = args->in.doorbell_offset; - - /* Convert relative doorbell offset into absolute doorbell index */ - index = amdgpu_userqueue_get_doorbell_index(uq_mgr, &db_info, filp); - if (index == (uint64_t)-EINVAL) { - DRM_ERROR("Failed to get doorbell for queue\n"); - kfree(queue); - goto unlock; - } - - queue->doorbell_index = index; - xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC); - r = amdgpu_userq_fence_driver_alloc(adev, queue); - if (r) { - DRM_ERROR("Failed to alloc fence driver\n"); - goto unlock; - } - - r = uq_funcs->mqd_create(uq_mgr, &args->in, queue); - if (r) { - DRM_ERROR("Failed to create Queue\n"); - amdgpu_userq_fence_driver_free(queue); - kfree(queue); - goto unlock; - } - - - qid = idr_alloc(&uq_mgr->userq_idr, queue, 1, AMDGPU_MAX_USERQ_COUNT, GFP_KERNEL); - if (qid < 0) { - DRM_ERROR("Failed to allocate a queue id\n"); - amdgpu_userq_fence_driver_free(queue); - uq_funcs->mqd_destroy(uq_mgr, queue); - kfree(queue); - r = -ENOMEM; - goto unlock; - } - - /* don't map the queue if scheduling is halted */ - mutex_lock(&adev->userq_mutex); - if (adev->userq_halt_for_enforce_isolation && - ((queue->queue_type == AMDGPU_HW_IP_GFX) || - (queue->queue_type == AMDGPU_HW_IP_COMPUTE))) - skip_map_queue = true; - else - skip_map_queue = false; - if (!skip_map_queue) { - r = amdgpu_userqueue_map_helper(uq_mgr, queue); - if (r) { - mutex_unlock(&adev->userq_mutex); - DRM_ERROR("Failed to map Queue\n"); - idr_remove(&uq_mgr->userq_idr, qid); - amdgpu_userq_fence_driver_free(queue); - uq_funcs->mqd_destroy(uq_mgr, queue); - kfree(queue); - goto unlock; - } - } - mutex_unlock(&adev->userq_mutex); - - - args->out.queue_id = qid; - -unlock: - mutex_unlock(&uq_mgr->userq_mutex); - - return r; -} - -int amdgpu_userq_ioctl(struct drm_device *dev, void *data, - struct drm_file *filp) -{ - union drm_amdgpu_userq *args = data; - int r; - - switch (args->in.op) { - case AMDGPU_USERQ_OP_CREATE: - if (args->in.flags & ~(AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK | - AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE)) - return -EINVAL; - r = amdgpu_userqueue_create(filp, args); - if (r) - DRM_ERROR("Failed to create usermode queue\n"); - break; - - case AMDGPU_USERQ_OP_FREE: - if (args->in.ip_type || - args->in.doorbell_handle || - args->in.doorbell_offset || - args->in.flags || - args->in.queue_va || - args->in.queue_size || - args->in.rptr_va || - args->in.wptr_va || - args->in.wptr_va || - args->in.mqd || - args->in.mqd_size) - return -EINVAL; - r = amdgpu_userqueue_destroy(filp, args->in.queue_id); - if (r) - DRM_ERROR("Failed to destroy usermode queue\n"); - break; - - default: - DRM_DEBUG_DRIVER("Invalid user queue op specified: %d\n", args->in.op); - return -EINVAL; - } - - return r; -} -#else -int amdgpu_userq_ioctl(struct drm_device *dev, void *data, - struct drm_file *filp) -{ - return -ENOTSUPP; -} -#endif - -static int -amdgpu_userqueue_restore_all(struct amdgpu_userq_mgr *uq_mgr) -{ - struct amdgpu_device *adev = uq_mgr->adev; - struct amdgpu_usermode_queue *queue; - int queue_id; - int ret = 0, r; - - /* Resume all the queues for this process */ - idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) { - r = amdgpu_userqueue_map_helper(uq_mgr, queue); - if (r) - ret = r; - } - - if (ret) - dev_err(adev->dev, "Failed to map all the queues\n"); - return ret; -} - -static int -amdgpu_userqueue_validate_vm_bo(void *_unused, struct amdgpu_bo *bo) -{ - struct ttm_operation_ctx ctx = { false, false }; - int ret; - - amdgpu_bo_placement_from_domain(bo, bo->allowed_domains); - - ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); - if (ret) - DRM_ERROR("Fail to validate\n"); - - return ret; -} - -static int -amdgpu_userqueue_validate_bos(struct amdgpu_userq_mgr *uq_mgr) -{ - struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); - struct amdgpu_vm *vm = &fpriv->vm; - struct amdgpu_device *adev = uq_mgr->adev; - struct amdgpu_bo_va *bo_va; - struct ww_acquire_ctx *ticket; - struct drm_exec exec; - struct amdgpu_bo *bo; - struct dma_resv *resv; - bool clear, unlock; - int ret = 0; - - drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); - drm_exec_until_all_locked(&exec) { - ret = amdgpu_vm_lock_pd(vm, &exec, 2); - drm_exec_retry_on_contention(&exec); - if (unlikely(ret)) { - DRM_ERROR("Failed to lock PD\n"); - goto unlock_all; - } - - /* Lock the done list */ - list_for_each_entry(bo_va, &vm->done, base.vm_status) { - bo = bo_va->base.bo; - if (!bo) - continue; - - ret = drm_exec_lock_obj(&exec, &bo->tbo.base); - drm_exec_retry_on_contention(&exec); - if (unlikely(ret)) - goto unlock_all; - } - } - - spin_lock(&vm->status_lock); - while (!list_empty(&vm->moved)) { - bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va, - base.vm_status); - spin_unlock(&vm->status_lock); - - /* Per VM BOs never need to bo cleared in the page tables */ - ret = amdgpu_vm_bo_update(adev, bo_va, false); - if (ret) - goto unlock_all; - spin_lock(&vm->status_lock); - } - - ticket = &exec.ticket; - while (!list_empty(&vm->invalidated)) { - bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, - base.vm_status); - resv = bo_va->base.bo->tbo.base.resv; - spin_unlock(&vm->status_lock); - - bo = bo_va->base.bo; - ret = amdgpu_userqueue_validate_vm_bo(NULL, bo); - if (ret) { - DRM_ERROR("Failed to validate BO\n"); - goto unlock_all; - } - - /* Try to reserve the BO to avoid clearing its ptes */ - if (!adev->debug_vm && dma_resv_trylock(resv)) { - clear = false; - unlock = true; - /* The caller is already holding the reservation lock */ - } else if (ticket && dma_resv_locking_ctx(resv) == ticket) { - clear = false; - unlock = false; - /* Somebody else is using the BO right now */ - } else { - clear = true; - unlock = false; - } - - ret = amdgpu_vm_bo_update(adev, bo_va, clear); - - if (unlock) - dma_resv_unlock(resv); - if (ret) - goto unlock_all; - - spin_lock(&vm->status_lock); - } - spin_unlock(&vm->status_lock); - - ret = amdgpu_eviction_fence_replace_fence(&fpriv->evf_mgr, &exec); - if (ret) - DRM_ERROR("Failed to replace eviction fence\n"); - -unlock_all: - drm_exec_fini(&exec); - return ret; -} - -static void amdgpu_userqueue_restore_worker(struct work_struct *work) -{ - struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, resume_work.work); - struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); - int ret; - - flush_work(&fpriv->evf_mgr.suspend_work.work); - - mutex_lock(&uq_mgr->userq_mutex); - - ret = amdgpu_userqueue_validate_bos(uq_mgr); - if (ret) { - DRM_ERROR("Failed to validate BOs to restore\n"); - goto unlock; - } - - ret = amdgpu_userqueue_restore_all(uq_mgr); - if (ret) { - DRM_ERROR("Failed to restore all queues\n"); - goto unlock; - } - -unlock: - mutex_unlock(&uq_mgr->userq_mutex); -} - -static int -amdgpu_userqueue_evict_all(struct amdgpu_userq_mgr *uq_mgr) -{ - struct amdgpu_device *adev = uq_mgr->adev; - struct amdgpu_usermode_queue *queue; - int queue_id; - int ret = 0, r; - - /* Try to unmap all the queues in this process ctx */ - idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) { - r = amdgpu_userqueue_unmap_helper(uq_mgr, queue); - if (r) - ret = r; - } - - if (ret) - dev_err(adev->dev, "Couldn't unmap all the queues\n"); - return ret; -} - -static int -amdgpu_userqueue_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr) -{ - struct amdgpu_usermode_queue *queue; - int queue_id, ret; - - idr_for_each_entry(&uq_mgr->userq_idr, queue, queue_id) { - struct dma_fence *f = queue->last_fence; - - if (!f || dma_fence_is_signaled(f)) - continue; - ret = dma_fence_wait_timeout(f, true, msecs_to_jiffies(100)); - if (ret <= 0) { - DRM_ERROR("Timed out waiting for fence=%llu:%llu\n", - f->context, f->seqno); - return -ETIMEDOUT; - } - } - - return 0; -} - -void -amdgpu_userqueue_evict(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_eviction_fence *ev_fence) -{ - int ret; - struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); - struct amdgpu_eviction_fence_mgr *evf_mgr = &fpriv->evf_mgr; - - /* Wait for any pending userqueue fence work to finish */ - ret = amdgpu_userqueue_wait_for_signal(uq_mgr); - if (ret) { - DRM_ERROR("Not evicting userqueue, timeout waiting for work\n"); - return; - } - - ret = amdgpu_userqueue_evict_all(uq_mgr); - if (ret) { - DRM_ERROR("Failed to evict userqueue\n"); - return; - } - - /* Signal current eviction fence */ - amdgpu_eviction_fence_signal(evf_mgr, ev_fence); - - if (evf_mgr->fd_closing) { - cancel_delayed_work(&uq_mgr->resume_work); - return; - } - - /* Schedule a resume work */ - schedule_delayed_work(&uq_mgr->resume_work, 0); -} - -int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct amdgpu_device *adev) -{ - mutex_init(&userq_mgr->userq_mutex); - idr_init_base(&userq_mgr->userq_idr, 1); - userq_mgr->adev = adev; - - mutex_lock(&adev->userq_mutex); - list_add(&userq_mgr->list, &adev->userq_mgr_list); - mutex_unlock(&adev->userq_mutex); - - INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userqueue_restore_worker); - return 0; -} - -void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr) -{ - struct amdgpu_device *adev = userq_mgr->adev; - struct amdgpu_usermode_queue *queue; - struct amdgpu_userq_mgr *uqm, *tmp; - uint32_t queue_id; - - cancel_delayed_work(&userq_mgr->resume_work); - - mutex_lock(&userq_mgr->userq_mutex); - idr_for_each_entry(&userq_mgr->userq_idr, queue, queue_id) { - amdgpu_userqueue_wait_for_last_fence(userq_mgr, queue); - amdgpu_userqueue_unmap_helper(userq_mgr, queue); - amdgpu_userqueue_cleanup(userq_mgr, queue, queue_id); - } - mutex_lock(&adev->userq_mutex); - list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { - if (uqm == userq_mgr) { - list_del(&uqm->list); - break; - } - } - mutex_unlock(&adev->userq_mutex); - idr_destroy(&userq_mgr->userq_idr); - mutex_unlock(&userq_mgr->userq_mutex); - mutex_destroy(&userq_mgr->userq_mutex); -} - -int amdgpu_userq_suspend(struct amdgpu_device *adev) -{ - u32 ip_mask = amdgpu_userqueue_get_supported_ip_mask(adev); - struct amdgpu_usermode_queue *queue; - struct amdgpu_userq_mgr *uqm, *tmp; - int queue_id; - int ret = 0, r; - - if (!ip_mask) - return 0; - - mutex_lock(&adev->userq_mutex); - list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { - cancel_delayed_work_sync(&uqm->resume_work); - idr_for_each_entry(&uqm->userq_idr, queue, queue_id) { - r = amdgpu_userqueue_unmap_helper(uqm, queue); - if (r) - ret = r; - } - } - mutex_unlock(&adev->userq_mutex); - return ret; -} - -int amdgpu_userq_resume(struct amdgpu_device *adev) -{ - u32 ip_mask = amdgpu_userqueue_get_supported_ip_mask(adev); - struct amdgpu_usermode_queue *queue; - struct amdgpu_userq_mgr *uqm, *tmp; - int queue_id; - int ret = 0, r; - - if (!ip_mask) - return 0; - - mutex_lock(&adev->userq_mutex); - list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { - idr_for_each_entry(&uqm->userq_idr, queue, queue_id) { - r = amdgpu_userqueue_map_helper(uqm, queue); - if (r) - ret = r; - } - } - mutex_unlock(&adev->userq_mutex); - return ret; -} - -int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev, - u32 idx) -{ - u32 ip_mask = amdgpu_userqueue_get_supported_ip_mask(adev); - struct amdgpu_usermode_queue *queue; - struct amdgpu_userq_mgr *uqm, *tmp; - int queue_id; - int ret = 0, r; - - /* only need to stop gfx/compute */ - if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE)))) - return 0; - - mutex_lock(&adev->userq_mutex); - if (adev->userq_halt_for_enforce_isolation) - dev_warn(adev->dev, "userq scheduling already stopped!\n"); - adev->userq_halt_for_enforce_isolation = true; - list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { - cancel_delayed_work_sync(&uqm->resume_work); - idr_for_each_entry(&uqm->userq_idr, queue, queue_id) { - if (((queue->queue_type == AMDGPU_HW_IP_GFX) || - (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) && - (queue->xcp_id == idx)) { - r = amdgpu_userqueue_unmap_helper(uqm, queue); - if (r) - ret = r; - } - } - } - mutex_unlock(&adev->userq_mutex); - return ret; -} - -int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev, - u32 idx) -{ - u32 ip_mask = amdgpu_userqueue_get_supported_ip_mask(adev); - struct amdgpu_usermode_queue *queue; - struct amdgpu_userq_mgr *uqm, *tmp; - int queue_id; - int ret = 0, r; - - /* only need to stop gfx/compute */ - if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE)))) - return 0; - - mutex_lock(&adev->userq_mutex); - if (!adev->userq_halt_for_enforce_isolation) - dev_warn(adev->dev, "userq scheduling already started!\n"); - adev->userq_halt_for_enforce_isolation = false; - list_for_each_entry_safe(uqm, tmp, &adev->userq_mgr_list, list) { - idr_for_each_entry(&uqm->userq_idr, queue, queue_id) { - if (((queue->queue_type == AMDGPU_HW_IP_GFX) || - (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) && - (queue->xcp_id == idx)) { - r = amdgpu_userqueue_map_helper(uqm, queue); - if (r) - ret = r; - } - } - } - mutex_unlock(&adev->userq_mutex); - return ret; -} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.h deleted file mode 100644 index a9f0e46bcec0..000000000000 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userqueue.h +++ /dev/null @@ -1,133 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright 2023 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef AMDGPU_USERQUEUE_H_ -#define AMDGPU_USERQUEUE_H_ -#include "amdgpu_eviction_fence.h" - -#define AMDGPU_MAX_USERQ_COUNT 512 - -#define to_ev_fence(f) container_of(f, struct amdgpu_eviction_fence, base) -#define uq_mgr_to_fpriv(u) container_of(u, struct amdgpu_fpriv, userq_mgr) -#define work_to_uq_mgr(w, name) container_of(w, struct amdgpu_userq_mgr, name) - -enum amdgpu_userqueue_state { - AMDGPU_USERQ_STATE_UNMAPPED = 0, - AMDGPU_USERQ_STATE_MAPPED, - AMDGPU_USERQ_STATE_PREEMPTED, - AMDGPU_USERQ_STATE_HUNG, -}; - -struct amdgpu_mqd_prop; - -struct amdgpu_userq_obj { - void *cpu_ptr; - uint64_t gpu_addr; - struct amdgpu_bo *obj; -}; - -struct amdgpu_usermode_queue { - int queue_type; - enum amdgpu_userqueue_state state; - uint64_t doorbell_handle; - uint64_t doorbell_index; - uint64_t flags; - struct amdgpu_mqd_prop *userq_prop; - struct amdgpu_userq_mgr *userq_mgr; - struct amdgpu_vm *vm; - struct amdgpu_userq_obj mqd; - struct amdgpu_userq_obj db_obj; - struct amdgpu_userq_obj fw_obj; - struct amdgpu_userq_obj wptr_obj; - struct xarray fence_drv_xa; - struct amdgpu_userq_fence_driver *fence_drv; - struct dma_fence *last_fence; - u32 xcp_id; - int priority; -}; - -struct amdgpu_userq_funcs { - int (*mqd_create)(struct amdgpu_userq_mgr *uq_mgr, - struct drm_amdgpu_userq_in *args, - struct amdgpu_usermode_queue *queue); - void (*mqd_destroy)(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_usermode_queue *uq); - int (*unmap)(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_usermode_queue *queue); - int (*map)(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_usermode_queue *queue); -}; - -/* Usermode queues for gfx */ -struct amdgpu_userq_mgr { - struct idr userq_idr; - struct mutex userq_mutex; - struct amdgpu_device *adev; - struct delayed_work resume_work; - struct list_head list; -}; - -struct amdgpu_db_info { - uint64_t doorbell_handle; - uint32_t queue_type; - uint32_t doorbell_offset; - struct amdgpu_userq_obj *db_obj; -}; - -int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); - -int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct amdgpu_device *adev); - -void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr); - -int amdgpu_userqueue_create_object(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_userq_obj *userq_obj, - int size); - -void amdgpu_userqueue_destroy_object(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_userq_obj *userq_obj); - -void amdgpu_userqueue_evict(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_eviction_fence *ev_fence); - -int amdgpu_userqueue_active(struct amdgpu_userq_mgr *uq_mgr); - -void amdgpu_userqueue_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr, - struct amdgpu_eviction_fence_mgr *evf_mgr); - -uint64_t amdgpu_userqueue_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr, - struct amdgpu_db_info *db_info, - struct drm_file *filp); - -u32 amdgpu_userqueue_get_supported_ip_mask(struct amdgpu_device *adev); - -int amdgpu_userq_suspend(struct amdgpu_device *adev); -int amdgpu_userq_resume(struct amdgpu_device *adev); - -int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev, - u32 idx); -int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev, - u32 idx); - -#endif diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c index 4c01c3a03095..d6f50b13e2ba 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c @@ -189,7 +189,7 @@ static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr, * for the same. */ size = AMDGPU_USERQ_PROC_CTX_SZ + AMDGPU_USERQ_GANG_CTX_SZ; - r = amdgpu_userqueue_create_object(uq_mgr, ctx, size); + r = amdgpu_userq_create_object(uq_mgr, ctx, size); if (r) { DRM_ERROR("Failed to allocate ctx space bo for userqueue, err:%d\n", r); return r; @@ -222,7 +222,7 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr, goto free_props; } - r = amdgpu_userqueue_create_object(uq_mgr, &queue->mqd, mqd_hw_default->mqd_size); + r = amdgpu_userq_create_object(uq_mgr, &queue->mqd, mqd_hw_default->mqd_size); if (r) { DRM_ERROR("Failed to create MQD object for userqueue\n"); goto free_props; @@ -327,10 +327,10 @@ static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr, return 0; free_ctx: - amdgpu_userqueue_destroy_object(uq_mgr, &queue->fw_obj); + amdgpu_userq_destroy_object(uq_mgr, &queue->fw_obj); free_mqd: - amdgpu_userqueue_destroy_object(uq_mgr, &queue->mqd); + amdgpu_userq_destroy_object(uq_mgr, &queue->mqd); free_props: kfree(userq_props); @@ -342,9 +342,9 @@ static void mes_userq_mqd_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_queue *queue) { - amdgpu_userqueue_destroy_object(uq_mgr, &queue->fw_obj); + amdgpu_userq_destroy_object(uq_mgr, &queue->fw_obj); kfree(queue->userq_prop); - amdgpu_userqueue_destroy_object(uq_mgr, &queue->mqd); + amdgpu_userq_destroy_object(uq_mgr, &queue->mqd); } const struct amdgpu_userq_funcs userq_mes_funcs = { diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.h b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.h index d0a521312ad4..090ae8897770 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.h +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.h @@ -24,7 +24,7 @@ #ifndef MES_USERQ_H #define MES_USERQ_H -#include "amdgpu_userqueue.h" +#include "amdgpu_userq.h" extern const struct amdgpu_userq_funcs userq_mes_funcs; #endif -- cgit v1.2.3 From e125a6e8ce0d747f4ae62234587308cfbbdcf4fa Mon Sep 17 00:00:00 2001 From: Prike Liang Date: Tue, 15 Apr 2025 10:31:15 +0800 Subject: drm/amdgpu: set the evf name to identify the userq case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The evf fence name can clearly identify the userq usage. Signed-off-by: Prike Liang Reviewed-by: Christian König Reviewed-by: Arvind Yadav Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index faa3f59b20c5..d86e611a9ff4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -31,7 +31,7 @@ static const char * amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence) { - return "amdgpu"; + return "amdgpu_eviction_fence"; } static const char * -- cgit v1.2.3 From af7160c25c68520299db40d608892810904fa064 Mon Sep 17 00:00:00 2001 From: Prike Liang Date: Tue, 15 Apr 2025 10:42:19 +0800 Subject: drm/amdgpu: fix the eviction fence dereference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dma_resv_add_fence() already refers to the added fence. So when attaching the evciton fence to the gem bo, it needn't refer to it anymore. Signed-off-by: Prike Liang Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index d86e611a9ff4..1a7469543db5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -189,7 +189,6 @@ void amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr) int amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr, struct amdgpu_bo *bo) { - struct dma_fence *ef; struct amdgpu_eviction_fence *ev_fence; struct dma_resv *resv = bo->tbo.base.resv; int ret; @@ -205,10 +204,8 @@ int amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr, spin_lock(&evf_mgr->ev_fence_lock); ev_fence = evf_mgr->ev_fence; - if (ev_fence) { - ef = dma_fence_get(&ev_fence->base); - dma_resv_add_fence(resv, ef, DMA_RESV_USAGE_BOOKKEEP); - } + if (ev_fence) + dma_resv_add_fence(resv, &ev_fence->base, DMA_RESV_USAGE_BOOKKEEP); spin_unlock(&evf_mgr->ev_fence_lock); return 0; -- cgit v1.2.3 From 13d0724f0ffe92971ef5a0da819f75c76c01d92e Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Thu, 15 May 2025 12:44:46 +0530 Subject: drm/amdgpu: fix use-after-unlock in eviction fence destroy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eviction fence destroy path incorrectly calls dma_fence_put() on evf_mgr->ev_fence after releasing the ev_fence_lock. This introduces a potential use-after-unlock or race because another thread concurrently modifies evf_mgr->ev_fence. Fix this by grabbing a local reference to evf_mgr->ev_fence under the lock and using that for dma_fence_put() after waiting. Cc: Alex Deucher Reviewed-by: Christian König Reviewed-by: Sunil Khatri Signed-off-by: Arvind Yadav Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index 1a7469543db5..73b629b5f56f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -183,7 +183,7 @@ void amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr) dma_fence_wait(&ev_fence->base, false); /* Last unref of ev_fence */ - dma_fence_put(&evf_mgr->ev_fence->base); + dma_fence_put(&ev_fence->base); } int amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr, -- cgit v1.2.3 From b2c11e27080d8556664c20c30ca3527ffa99bec4 Mon Sep 17 00:00:00 2001 From: Prike Liang Date: Wed, 30 Apr 2025 10:34:14 +0800 Subject: drm/amdgpu: lock the eviction fence for wq signals it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lock and refer to the eviction fence before the eviction fence schedules work queue tries to signal it. Suggested-by: Christian König Signed-off-by: Prike Liang Acked-by: Alex Deucher Reviewed-by: Arvind Yadav Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index 73b629b5f56f..8b919ad3af29 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -108,13 +108,22 @@ amdgpu_eviction_fence_suspend_worker(struct work_struct *work) struct amdgpu_eviction_fence *ev_fence; mutex_lock(&uq_mgr->userq_mutex); + spin_lock(&evf_mgr->ev_fence_lock); ev_fence = evf_mgr->ev_fence; - if (!ev_fence) + if (ev_fence) + dma_fence_get(&ev_fence->base); + else goto unlock; + spin_unlock(&evf_mgr->ev_fence_lock); amdgpu_userq_evict(uq_mgr, ev_fence); + mutex_unlock(&uq_mgr->userq_mutex); + dma_fence_put(&ev_fence->base); + return; + unlock: + spin_unlock(&evf_mgr->ev_fence_lock); mutex_unlock(&uq_mgr->userq_mutex); } -- cgit v1.2.3 From bf33a0003d9e3b0546f2d7e91bebfd67af59f275 Mon Sep 17 00:00:00 2001 From: Tvrtko Ursulin Date: Thu, 15 May 2025 10:49:57 +0100 Subject: dma-fence: Use a flag for 64-bit seqnos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the goal of reducing the need for drivers to touch (and dereference) fence->ops, we move the 64-bit seqnos flag from struct dma_fence_ops to the fence->flags. Drivers which were setting this flag are changed to use new dma_fence_init64() instead of dma_fence_init(). v2: * Streamlined init and added kerneldoc. * Rebase for amdgpu userq which landed since. Signed-off-by: Tvrtko Ursulin Reviewed-by: Christian König # v1 Signed-off-by: Tvrtko Ursulin Link: https://lore.kernel.org/r/20250515095004.28318-3-tvrtko.ursulin@igalia.com --- drivers/dma-buf/dma-fence-chain.c | 5 +- drivers/dma-buf/dma-fence.c | 57 +++++++++++++++++----- drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c | 7 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c | 5 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c | 5 +- include/linux/dma-fence.h | 14 ++---- 6 files changed, 58 insertions(+), 35 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c') diff --git a/drivers/dma-buf/dma-fence-chain.c b/drivers/dma-buf/dma-fence-chain.c index 90424f23fd73..a8a90acf4f34 100644 --- a/drivers/dma-buf/dma-fence-chain.c +++ b/drivers/dma-buf/dma-fence-chain.c @@ -218,7 +218,6 @@ static void dma_fence_chain_set_deadline(struct dma_fence *fence, } const struct dma_fence_ops dma_fence_chain_ops = { - .use_64bit_seqno = true, .get_driver_name = dma_fence_chain_get_driver_name, .get_timeline_name = dma_fence_chain_get_timeline_name, .enable_signaling = dma_fence_chain_enable_signaling, @@ -262,8 +261,8 @@ void dma_fence_chain_init(struct dma_fence_chain *chain, seqno = max(prev->seqno, seqno); } - dma_fence_init(&chain->base, &dma_fence_chain_ops, - &chain->lock, context, seqno); + dma_fence_init64(&chain->base, &dma_fence_chain_ops, &chain->lock, + context, seqno); /* * Chaining dma_fence_chain container together is only allowed through diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index f0cdd3e99d36..705b59787731 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -989,6 +989,25 @@ void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq) } EXPORT_SYMBOL(dma_fence_describe); +static void +__dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, + spinlock_t *lock, u64 context, u64 seqno, unsigned long flags) +{ + BUG_ON(!lock); + BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name); + + kref_init(&fence->refcount); + fence->ops = ops; + INIT_LIST_HEAD(&fence->cb_list); + fence->lock = lock; + fence->context = context; + fence->seqno = seqno; + fence->flags = flags; + fence->error = 0; + + trace_dma_fence_init(fence); +} + /** * dma_fence_init - Initialize a custom fence. * @fence: the fence to initialize @@ -1008,18 +1027,30 @@ void dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, spinlock_t *lock, u64 context, u64 seqno) { - BUG_ON(!lock); - BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name); - - kref_init(&fence->refcount); - fence->ops = ops; - INIT_LIST_HEAD(&fence->cb_list); - fence->lock = lock; - fence->context = context; - fence->seqno = seqno; - fence->flags = 0UL; - fence->error = 0; - - trace_dma_fence_init(fence); + __dma_fence_init(fence, ops, lock, context, seqno, 0UL); } EXPORT_SYMBOL(dma_fence_init); + +/** + * dma_fence_init64 - Initialize a custom fence with 64-bit seqno support. + * @fence: the fence to initialize + * @ops: the dma_fence_ops for operations on this fence + * @lock: the irqsafe spinlock to use for locking this fence + * @context: the execution context this fence is run on + * @seqno: a linear increasing sequence number for this context + * + * Initializes an allocated fence, the caller doesn't have to keep its + * refcount after committing with this fence, but it will need to hold a + * refcount again if &dma_fence_ops.enable_signaling gets called. + * + * Context and seqno are used for easy comparison between fences, allowing + * to check which fence is later by simply using dma_fence_later(). + */ +void +dma_fence_init64(struct dma_fence *fence, const struct dma_fence_ops *ops, + spinlock_t *lock, u64 context, u64 seqno) +{ + __dma_fence_init(fence, ops, lock, context, seqno, + BIT(DMA_FENCE_FLAG_SEQNO64_BIT)); +} +EXPORT_SYMBOL(dma_fence_init64); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index 73b629b5f56f..ce621f88ddca 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -134,7 +134,6 @@ static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f) } static const struct dma_fence_ops amdgpu_eviction_fence_ops = { - .use_64bit_seqno = true, .get_driver_name = amdgpu_eviction_fence_get_driver_name, .get_timeline_name = amdgpu_eviction_fence_get_timeline_name, .enable_signaling = amdgpu_eviction_fence_enable_signaling, @@ -160,9 +159,9 @@ amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr) ev_fence->evf_mgr = evf_mgr; get_task_comm(ev_fence->timeline_name, current); spin_lock_init(&ev_fence->lock); - dma_fence_init(&ev_fence->base, &amdgpu_eviction_fence_ops, - &ev_fence->lock, evf_mgr->ev_fence_ctx, - atomic_inc_return(&evf_mgr->ev_fence_seq)); + dma_fence_init64(&ev_fence->base, &amdgpu_eviction_fence_ops, + &ev_fence->lock, evf_mgr->ev_fence_ctx, + atomic_inc_return(&evf_mgr->ev_fence_seq)); return ev_fence; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c index fc4d0d42e223..a3db9442dc37 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c @@ -239,8 +239,8 @@ static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq, fence = &userq_fence->base; userq_fence->fence_drv = fence_drv; - dma_fence_init(fence, &amdgpu_userq_fence_ops, &userq_fence->lock, - fence_drv->context, seq); + dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock, + fence_drv->context, seq); amdgpu_userq_fence_driver_get(fence_drv); dma_fence_get(fence); @@ -334,7 +334,6 @@ static void amdgpu_userq_fence_release(struct dma_fence *f) } static const struct dma_fence_ops amdgpu_userq_fence_ops = { - .use_64bit_seqno = true, .get_driver_name = amdgpu_userq_fence_get_driver_name, .get_timeline_name = amdgpu_userq_fence_get_timeline_name, .signaled = amdgpu_userq_fence_signaled, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c index 51cddfa3f1e8..5d26797356a3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_tlb_fence.c @@ -71,7 +71,6 @@ static void amdgpu_tlb_fence_work(struct work_struct *work) } static const struct dma_fence_ops amdgpu_tlb_fence_ops = { - .use_64bit_seqno = true, .get_driver_name = amdgpu_tlb_fence_get_driver_name, .get_timeline_name = amdgpu_tlb_fence_get_timeline_name }; @@ -101,8 +100,8 @@ void amdgpu_vm_tlb_fence_create(struct amdgpu_device *adev, struct amdgpu_vm *vm INIT_WORK(&f->work, amdgpu_tlb_fence_work); spin_lock_init(&f->lock); - dma_fence_init(&f->base, &amdgpu_tlb_fence_ops, &f->lock, - vm->tlb_fence_context, atomic64_read(&vm->tlb_seq)); + dma_fence_init64(&f->base, &amdgpu_tlb_fence_ops, &f->lock, + vm->tlb_fence_context, atomic64_read(&vm->tlb_seq)); /* TODO: We probably need a separate wq here */ dma_fence_get(&f->base); diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h index c841af28b706..926c01b5b29d 100644 --- a/include/linux/dma-fence.h +++ b/include/linux/dma-fence.h @@ -98,6 +98,7 @@ struct dma_fence { }; enum dma_fence_flag_bits { + DMA_FENCE_FLAG_SEQNO64_BIT, DMA_FENCE_FLAG_SIGNALED_BIT, DMA_FENCE_FLAG_TIMESTAMP_BIT, DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, @@ -125,14 +126,6 @@ struct dma_fence_cb { * */ struct dma_fence_ops { - /** - * @use_64bit_seqno: - * - * True if this dma_fence implementation uses 64bit seqno, false - * otherwise. - */ - bool use_64bit_seqno; - /** * @get_driver_name: * @@ -263,6 +256,9 @@ struct dma_fence_ops { void dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, spinlock_t *lock, u64 context, u64 seqno); +void dma_fence_init64(struct dma_fence *fence, const struct dma_fence_ops *ops, + spinlock_t *lock, u64 context, u64 seqno); + void dma_fence_release(struct kref *kref); void dma_fence_free(struct dma_fence *fence); void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq); @@ -455,7 +451,7 @@ static inline bool __dma_fence_is_later(struct dma_fence *fence, u64 f1, u64 f2) * 32bit sequence numbers. Use a 64bit compare when the driver says to * do so. */ - if (fence->ops->use_64bit_seqno) + if (test_bit(DMA_FENCE_FLAG_SEQNO64_BIT, &fence->flags)) return f1 > f2; return (int)(lower_32_bits(f1) - lower_32_bits(f2)) > 0; -- cgit v1.2.3