From 5d3196605d00b513a735b358f5e4f2d1b65f6c3a Mon Sep 17 00:00:00 2001 From: Christian König Date: Mon, 16 Dec 2019 16:07:06 +0100 Subject: drm/amdgpu: rework job synchronization v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For unlocked page table updates we need to be able to sync to fences of a specific VM. v2: use SYNC_ALWAYS in the UVD code Signed-off-by: Christian König Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_object.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index e3f16b49e970..6b7eb10351e7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -1419,7 +1419,8 @@ int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) int r; amdgpu_sync_create(&sync); - amdgpu_sync_resv(adev, &sync, bo->tbo.base.resv, owner, false); + amdgpu_sync_resv(adev, &sync, bo->tbo.base.resv, + AMDGPU_SYNC_NE_OWNER, owner); r = amdgpu_sync_wait(&sync, intr); amdgpu_sync_free(&sync); -- cgit v1.2.3 From 9f3cc18d19b7b791b8f59bcae51f88e759f73146 Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 23 Jan 2020 14:49:45 +0100 Subject: drm/amdgpu: rework synchronization of VM updates v4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If provided we only sync to the BOs reservation object and no longer to the root PD. v2: update comment, cleanup amdgpu_bo_sync_wait_resv v3: use correct reservation object while clearing v4: fix typo in amdgpu_bo_sync_wait_resv Signed-off-by: Christian König Tested-by: Tom St Denis Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 35 +++++++++++++++++++----- drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 3 +++ drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 7 ----- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 41 +++++++++++++++++------------ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 4 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c | 22 ++++------------ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c | 15 +++-------- 7 files changed, 66 insertions(+), 61 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_object.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 6b7eb10351e7..65176b97401c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -1403,30 +1403,51 @@ void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, } /** - * amdgpu_sync_wait_resv - Wait for BO reservation fences + * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences * - * @bo: buffer object + * @adev: amdgpu device pointer + * @resv: reservation object to sync to + * @sync_mode: synchronization mode * @owner: fence owner * @intr: Whether the wait is interruptible * + * Extract the fences from the reservation object and waits for them to finish. + * * Returns: * 0 on success, errno otherwise. */ -int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) +int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv, + enum amdgpu_sync_mode sync_mode, void *owner, + bool intr) { - struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); struct amdgpu_sync sync; int r; amdgpu_sync_create(&sync); - amdgpu_sync_resv(adev, &sync, bo->tbo.base.resv, - AMDGPU_SYNC_NE_OWNER, owner); + amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner); r = amdgpu_sync_wait(&sync, intr); amdgpu_sync_free(&sync); - return r; } +/** + * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv + * @bo: buffer object to wait for + * @owner: fence owner + * @intr: Whether the wait is interruptible + * + * Wrapper to wait for fences in a BO. + * Returns: + * 0 on success, errno otherwise. + */ +int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + + return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv, + AMDGPU_SYNC_NE_OWNER, owner, intr); +} + /** * amdgpu_bo_gpu_offset - return GPU offset of bo * @bo: amdgpu object for which we query the offset diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 36dec51d1ef1..97c53025e6ac 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -277,6 +277,9 @@ void amdgpu_bo_release_notify(struct ttm_buffer_object *bo); int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo); void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, bool shared); +int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv, + enum amdgpu_sync_mode sync_mode, void *owner, + bool intr); int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr); u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo); int amdgpu_bo_validate(struct amdgpu_bo *bo); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c index 9f42032676da..b86392253696 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c @@ -249,13 +249,6 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync, owner != AMDGPU_FENCE_OWNER_UNDEFINED) continue; - /* VM updates only sync with moves but not with user - * command submissions or KFD evictions fences - */ - if (fence_owner != AMDGPU_FENCE_OWNER_UNDEFINED && - owner == AMDGPU_FENCE_OWNER_VM) - continue; - /* Ignore fences depending on the sync mode */ switch (mode) { case AMDGPU_SYNC_ALWAYS: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 71fc3362580b..3e6ee157cfe6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -797,7 +797,7 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, params.vm = vm; params.direct = direct; - r = vm->update_funcs->prepare(¶ms, AMDGPU_FENCE_OWNER_KFD, NULL); + r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_SYNC_EXPLICIT); if (r) return r; @@ -1293,7 +1293,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, params.vm = vm; params.direct = direct; - r = vm->update_funcs->prepare(¶ms, AMDGPU_FENCE_OWNER_VM, NULL); + r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_SYNC_EXPLICIT); if (r) return r; @@ -1554,7 +1554,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params, * @adev: amdgpu_device pointer * @vm: requested vm * @direct: direct submission in a page fault - * @exclusive: fence we need to sync to + * @resv: fences we need to sync to * @start: start of mapped range * @last: last mapped entry * @flags: flags for the entries @@ -1569,14 +1569,14 @@ static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params, */ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, struct amdgpu_vm *vm, bool direct, - struct dma_fence *exclusive, + struct dma_resv *resv, uint64_t start, uint64_t last, uint64_t flags, uint64_t addr, dma_addr_t *pages_addr, struct dma_fence **fence) { struct amdgpu_vm_update_params params; - void *owner = AMDGPU_FENCE_OWNER_VM; + enum amdgpu_sync_mode sync_mode; int r; memset(¶ms, 0, sizeof(params)); @@ -1585,9 +1585,13 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, params.direct = direct; params.pages_addr = pages_addr; - /* sync to everything except eviction fences on unmapping */ + /* Implicitly sync to command submissions in the same VM before + * unmapping. Sync to moving fences before mapping. + */ if (!(flags & AMDGPU_PTE_VALID)) - owner = AMDGPU_FENCE_OWNER_KFD; + sync_mode = AMDGPU_SYNC_EQ_OWNER; + else + sync_mode = AMDGPU_SYNC_EXPLICIT; amdgpu_vm_eviction_lock(vm); if (vm->evicting) { @@ -1595,7 +1599,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, goto error_unlock; } - r = vm->update_funcs->prepare(¶ms, owner, exclusive); + r = vm->update_funcs->prepare(¶ms, resv, sync_mode); if (r) goto error_unlock; @@ -1614,7 +1618,7 @@ error_unlock: * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks * * @adev: amdgpu_device pointer - * @exclusive: fence we need to sync to + * @resv: fences we need to sync to * @pages_addr: DMA addresses to use for mapping * @vm: requested vm * @mapping: mapped range and flags to use for the update @@ -1630,7 +1634,7 @@ error_unlock: * 0 for success, -EINVAL for failure. */ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, - struct dma_fence *exclusive, + struct dma_resv *resv, dma_addr_t *pages_addr, struct amdgpu_vm *vm, struct amdgpu_bo_va_mapping *mapping, @@ -1706,7 +1710,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev, } last = min((uint64_t)mapping->last, start + max_entries - 1); - r = amdgpu_vm_bo_update_mapping(adev, vm, false, exclusive, + r = amdgpu_vm_bo_update_mapping(adev, vm, false, resv, start, last, flags, addr, dma_addr, fence); if (r) @@ -1745,7 +1749,8 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, dma_addr_t *pages_addr = NULL; struct ttm_mem_reg *mem; struct drm_mm_node *nodes; - struct dma_fence *exclusive, **last_update; + struct dma_fence **last_update; + struct dma_resv *resv; uint64_t flags; struct amdgpu_device *bo_adev = adev; int r; @@ -1753,7 +1758,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, if (clear || !bo) { mem = NULL; nodes = NULL; - exclusive = NULL; + resv = vm->root.base.bo->tbo.base.resv; } else { struct ttm_dma_tt *ttm; @@ -1763,7 +1768,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm); pages_addr = ttm->dma_address; } - exclusive = bo->tbo.moving; + resv = bo->tbo.base.resv; } if (bo) { @@ -1773,7 +1778,8 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, flags = 0x0; } - if (clear || (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv)) + if (clear || (bo && bo->tbo.base.resv == + vm->root.base.bo->tbo.base.resv)) last_update = &vm->last_update; else last_update = &bo_va->last_pt_update; @@ -1787,7 +1793,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, } list_for_each_entry(mapping, &bo_va->invalids, list) { - r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm, + r = amdgpu_vm_bo_split_mapping(adev, resv, pages_addr, vm, mapping, flags, bo_adev, nodes, last_update); if (r) @@ -1982,6 +1988,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct dma_fence **fence) { + struct dma_resv *resv = vm->root.base.bo->tbo.base.resv; struct amdgpu_bo_va_mapping *mapping; uint64_t init_pte_value = 0; struct dma_fence *f = NULL; @@ -1996,7 +2003,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev, mapping->start < AMDGPU_GMC_HOLE_START) init_pte_value = AMDGPU_PTE_DEFAULT_ATC; - r = amdgpu_vm_bo_update_mapping(adev, vm, false, NULL, + r = amdgpu_vm_bo_update_mapping(adev, vm, false, resv, mapping->start, mapping->last, init_pte_value, 0, NULL, &f); amdgpu_vm_free_mapping(adev, vm, mapping, f); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index b4640ab38c95..d00648ee8d54 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -227,8 +227,8 @@ struct amdgpu_vm_update_params { struct amdgpu_vm_update_funcs { int (*map_table)(struct amdgpu_bo *bo); - int (*prepare)(struct amdgpu_vm_update_params *p, void * owner, - struct dma_fence *exclusive); + int (*prepare)(struct amdgpu_vm_update_params *p, struct dma_resv *resv, + enum amdgpu_sync_mode sync_mode); int (*update)(struct amdgpu_vm_update_params *p, struct amdgpu_bo *bo, uint64_t pe, uint64_t addr, unsigned count, uint32_t incr, uint64_t flags); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c index 68b013be3837..e38516304070 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c @@ -44,26 +44,14 @@ static int amdgpu_vm_cpu_map_table(struct amdgpu_bo *table) * Returns: * Negativ errno, 0 for success. */ -static int amdgpu_vm_cpu_prepare(struct amdgpu_vm_update_params *p, void *owner, - struct dma_fence *exclusive) +static int amdgpu_vm_cpu_prepare(struct amdgpu_vm_update_params *p, + struct dma_resv *resv, + enum amdgpu_sync_mode sync_mode) { - int r; - - /* Wait for any BO move to be completed */ - if (exclusive) { - r = dma_fence_wait(exclusive, true); - if (unlikely(r)) - return r; - } - - /* Don't wait for submissions during page fault */ - if (p->direct) + if (!resv) return 0; - /* Wait for PT BOs to be idle. PTs share the same resv. object - * as the root PD BO - */ - return amdgpu_bo_sync_wait(p->vm->root.base.bo, owner, true); + return amdgpu_bo_sync_wait_resv(p->adev, resv, sync_mode, p->vm, true); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c index ab6481751763..4cc7881f438c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c @@ -58,9 +58,9 @@ static int amdgpu_vm_sdma_map_table(struct amdgpu_bo *table) * Negativ errno, 0 for success. */ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p, - void *owner, struct dma_fence *exclusive) + struct dma_resv *resv, + enum amdgpu_sync_mode sync_mode) { - struct amdgpu_bo *root = p->vm->root.base.bo; unsigned int ndw = AMDGPU_VM_SDMA_MIN_NUM_DW; int r; @@ -70,17 +70,10 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p, p->num_dw_left = ndw; - /* Wait for moves to be completed */ - r = amdgpu_sync_fence(&p->job->sync, exclusive, false); - if (r) - return r; - - /* Don't wait for any submissions during page fault handling */ - if (p->direct) + if (!resv) return 0; - return amdgpu_sync_resv(p->adev, &p->job->sync, root->tbo.base.resv, - AMDGPU_SYNC_NE_OWNER, owner); + return amdgpu_sync_resv(p->adev, &p->job->sync, resv, sync_mode, p->vm); } /** -- cgit v1.2.3 From f4a3c42b5c52c8c603fa8433c52aa940a4ad938a Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Tue, 11 Feb 2020 11:28:34 +0800 Subject: drm/amdgpu: Remove kfd eviction fence before release bo (v2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No need to trigger eviction as the memory mapping will not be used anymore. All pt/pd bos share same resv, hence the same shared eviction fence. Everytime page table is freed, the fence will be signled and that cuases kfd unexcepted evictions. v2: squash in 32 bit fix CC: Christian König CC: Felix Kuehling CC: Alex Deucher Acked-by: Christian König Reviewed-by: Felix Kuehling Signed-off-by: xinhui pan Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 5 ++++ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 38 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 5 ++++ 4 files changed, 49 insertions(+) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_object.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index e1c2c182898f..514551da33eb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -673,6 +673,11 @@ void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) { } +int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) +{ + return 0; +} + void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, struct amdgpu_vm *vm) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 9e8db702d878..0ee8aae6c519 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -96,6 +96,7 @@ struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, struct mm_struct *mm); bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm); struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f); +int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo); struct amdkfd_process_info { /* List head of all VMs that belong to a KFD process */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 66bde9e9a4c9..e1d1eed7a25f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -276,6 +276,42 @@ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, return 0; } +int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) +{ + struct amdgpu_bo *root = bo; + struct amdgpu_vm_bo_base *vm_bo; + struct amdgpu_vm *vm; + struct amdkfd_process_info *info; + struct amdgpu_amdkfd_fence *ef; + int ret; + + /* we can always get vm_bo from root PD bo.*/ + while (root->parent) + root = root->parent; + + vm_bo = root->vm_bo; + if (!vm_bo) + return 0; + + vm = vm_bo->vm; + if (!vm) + return 0; + + info = vm->process_info; + if (!info || !info->eviction_fence) + return 0; + + ef = container_of(dma_fence_get(&info->eviction_fence->base), + struct amdgpu_amdkfd_fence, base); + + BUG_ON(!dma_resv_trylock(bo->tbo.base.resv)); + ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef); + dma_resv_unlock(bo->tbo.base.resv); + + dma_fence_put(&ef->base); + return ret; +} + static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, bool wait) { @@ -1044,6 +1080,8 @@ void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, list_del(&vm->vm_list_node); mutex_unlock(&process_info->lock); + vm->process_info = NULL; + /* Release per-process resources when last compute VM is destroyed */ if (!process_info->n_vms) { WARN_ON(!list_empty(&process_info->kfd_bo_list)); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 65176b97401c..e4a8c424d290 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -1307,6 +1307,11 @@ void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) if (abo->kfd_bo) amdgpu_amdkfd_unreserve_memory_limit(abo); + /* We only remove the fence if the resv has individualized. */ + WARN_ON_ONCE(bo->base.resv != &bo->base._resv); + if (bo->base.resv == &bo->base._resv) + amdgpu_amdkfd_remove_fence_on_pt_pd_bos(abo); + if (bo->mem.mem_type != TTM_PL_VRAM || !bo->mem.mm_node || !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) return; -- cgit v1.2.3 From 2d4dad2734e2aca8b4ef501aae9866662edc1afe Mon Sep 17 00:00:00 2001 From: Christian König Date: Wed, 30 May 2018 14:42:24 +0200 Subject: drm/amdgpu: add amdgpu_dma_buf_pin/unpin v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This implements the exporter side of unpinned DMA-buf handling. v2: fix minor coding style issues Signed-off-by: Christian König Acked-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/353999/?series=73646&rev=1 --- drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 53 +++++++++++++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 5 +++ 2 files changed, 51 insertions(+), 7 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_object.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index 7cafc65fd76a..86000c75b133 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -222,6 +222,37 @@ static void amdgpu_dma_buf_detach(struct dma_buf *dmabuf, bo->prime_shared_count--; } +/** + * amdgpu_dma_buf_pin - &dma_buf_ops.pin implementation + * + * @attach: attachment to pin down + * + * Pin the BO which is backing the DMA-buf so that it can't move any more. + */ +static int amdgpu_dma_buf_pin(struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = attach->dmabuf->priv; + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + + /* pin buffer into GTT */ + return amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); +} + +/** + * amdgpu_dma_buf_unpin - &dma_buf_ops.unpin implementation + * + * @attach: attachment to unpin + * + * Unpin a previously pinned BO to make it movable again. + */ +static void amdgpu_dma_buf_unpin(struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = attach->dmabuf->priv; + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + + amdgpu_bo_unpin(bo); +} + /** * amdgpu_dma_buf_map - &dma_buf_ops.map_dma_buf implementation * @attach: DMA-buf attachment @@ -244,9 +275,19 @@ static struct sg_table *amdgpu_dma_buf_map(struct dma_buf_attachment *attach, struct sg_table *sgt; long r; - r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); - if (r) - return ERR_PTR(r); + if (!bo->pin_count) { + /* move buffer into GTT */ + struct ttm_operation_ctx ctx = { false, false }; + + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + if (r) + return ERR_PTR(r); + + } else if (!(amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type) & + AMDGPU_GEM_DOMAIN_GTT)) { + return ERR_PTR(-EBUSY); + } sgt = drm_prime_pages_to_sg(bo->tbo.ttm->pages, bo->tbo.num_pages); if (IS_ERR(sgt)) @@ -277,13 +318,9 @@ static void amdgpu_dma_buf_unmap(struct dma_buf_attachment *attach, struct sg_table *sgt, enum dma_data_direction dir) { - struct drm_gem_object *obj = attach->dmabuf->priv; - struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); - dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir); sg_free_table(sgt); kfree(sgt); - amdgpu_bo_unpin(bo); } /** @@ -330,6 +367,8 @@ const struct dma_buf_ops amdgpu_dmabuf_ops = { .dynamic_mapping = true, .attach = amdgpu_dma_buf_attach, .detach = amdgpu_dma_buf_detach, + .pin = amdgpu_dma_buf_pin, + .unpin = amdgpu_dma_buf_unpin, .map_dma_buf = amdgpu_dma_buf_map, .unmap_dma_buf = amdgpu_dma_buf_unmap, .release = drm_gem_dmabuf_release, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index e3f16b49e970..9de8374bbbab 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -31,6 +31,7 @@ */ #include #include +#include #include #include @@ -1274,6 +1275,10 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, amdgpu_bo_kunmap(abo); + if (abo->tbo.base.dma_buf && !abo->tbo.base.import_attach && + bo->mem.mem_type != TTM_PL_SYSTEM) + dma_buf_move_notify(abo->tbo.base.dma_buf); + /* remember the eviction */ if (evict) atomic64_inc(&adev->num_evictions); -- cgit v1.2.3 From a448cb003edcb4b63d0a9c95f3faab724e6150fb Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 7 Jun 2018 10:28:47 +0200 Subject: drm/amdgpu: implement amdgpu_gem_prime_move_notify v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement the importer side of unpinned DMA-buf handling. v2: update page tables immediately Signed-off-by: Christian König Acked-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/353998/?series=73646&rev=1 --- drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 66 ++++++++++++++++++++++++++++- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 6 +++ 2 files changed, 71 insertions(+), 1 deletion(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_object.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index 86000c75b133..1a040ccf61bf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -451,7 +451,71 @@ error: return ERR_PTR(ret); } +/** + * amdgpu_dma_buf_move_notify - &attach.move_notify implementation + * + * @attach: the DMA-buf attachment + * + * Invalidate the DMA-buf attachment, making sure that the we re-create the + * mapping before the next use. + */ +static void +amdgpu_dma_buf_move_notify(struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = attach->importer_priv; + struct ww_acquire_ctx *ticket = dma_resv_locking_ctx(obj->resv); + struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct ttm_operation_ctx ctx = { false, false }; + struct ttm_placement placement = {}; + struct amdgpu_vm_bo_base *bo_base; + int r; + + if (bo->tbo.mem.mem_type == TTM_PL_SYSTEM) + return; + + r = ttm_bo_validate(&bo->tbo, &placement, &ctx); + if (r) { + DRM_ERROR("Failed to invalidate DMA-buf import (%d))\n", r); + return; + } + + for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { + struct amdgpu_vm *vm = bo_base->vm; + struct dma_resv *resv = vm->root.base.bo->tbo.base.resv; + + if (ticket) { + /* When we get an error here it means that somebody + * else is holding the VM lock and updating page tables + * So we can just continue here. + */ + r = dma_resv_lock(resv, ticket); + if (r) + continue; + + } else { + /* TODO: This is more problematic and we actually need + * to allow page tables updates without holding the + * lock. + */ + if (!dma_resv_trylock(resv)) + continue; + } + + r = amdgpu_vm_clear_freed(adev, vm, NULL); + if (!r) + r = amdgpu_vm_handle_moved(adev, vm); + + if (r && r != -EBUSY) + DRM_ERROR("Failed to invalidate VM page tables (%d))\n", + r); + + dma_resv_unlock(resv); + } +} + static const struct dma_buf_attach_ops amdgpu_dma_buf_attach_ops = { + .move_notify = amdgpu_dma_buf_move_notify }; /** @@ -487,7 +551,7 @@ struct drm_gem_object *amdgpu_gem_prime_import(struct drm_device *dev, return obj; attach = dma_buf_dynamic_attach(dma_buf, dev->dev, - &amdgpu_dma_buf_attach_ops, NULL); + &amdgpu_dma_buf_attach_ops, obj); if (IS_ERR(attach)) { drm_gem_object_put(obj); return ERR_CAST(attach); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 9de8374bbbab..5fa8f59c4ccf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -926,6 +926,9 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, return 0; } + if (bo->tbo.base.import_attach) + dma_buf_pin(bo->tbo.base.import_attach); + bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; /* force to pin into visible video ram */ if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) @@ -1009,6 +1012,9 @@ int amdgpu_bo_unpin(struct amdgpu_bo *bo) amdgpu_bo_subtract_pin_size(bo); + if (bo->tbo.base.import_attach) + dma_buf_unpin(bo->tbo.base.import_attach); + for (i = 0; i < bo->placement.num_placement; i++) { bo->placements[i].lpfn = 0; bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT; -- cgit v1.2.3 From 9fe58d0bbdb013d9712ae4455faeac72216bba0e Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Mon, 9 Mar 2020 22:28:01 +0800 Subject: drm/amdgpu: Correct the condition of warning while bo release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only kernel bo has kfd eviction fence. This warning is to give a notice that kfd only remove eviction fence on individual bos. Tested-by: Nicholas Johnson Reviewed-by: Christian König Signed-off-by: xinhui pan Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_object.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index e4a8c424d290..5ba4f0309910 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -1308,7 +1308,8 @@ void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) amdgpu_amdkfd_unreserve_memory_limit(abo); /* We only remove the fence if the resv has individualized. */ - WARN_ON_ONCE(bo->base.resv != &bo->base._resv); + WARN_ON_ONCE(bo->type == ttm_bo_type_kernel + && bo->base.resv != &bo->base._resv); if (bo->base.resv == &bo->base._resv) amdgpu_amdkfd_remove_fence_on_pt_pd_bos(abo); -- cgit v1.2.3