diff options
| -rw-r--r-- | drivers/gpu/drm/amd/amdgpu/amdgpu.h | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 105 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 53 |
3 files changed, 132 insertions, 32 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index c1b10046317e..148e2c61463c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -588,6 +588,9 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync, struct reservation_object *resv, void *owner); +bool amdgpu_sync_is_idle(struct amdgpu_sync *sync); +int amdgpu_sync_cycle_fences(struct amdgpu_sync *dst, struct amdgpu_sync *src, + struct fence *fence); struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync); int amdgpu_sync_wait(struct amdgpu_sync *sync); void amdgpu_sync_free(struct amdgpu_sync *sync); @@ -875,7 +878,8 @@ struct amdgpu_vm { struct amdgpu_vm_id { struct list_head list; - struct fence *active; + struct fence *first; + struct amdgpu_sync active; atomic_long_t owner; uint64_t pd_gpu_addr; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c index c48b4fce5e57..34a92808bbd4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c @@ -109,6 +109,29 @@ static void amdgpu_sync_keep_later(struct fence **keep, struct fence *fence) } /** + * amdgpu_sync_add_later - add the fence to the hash + * + * @sync: sync object to add the fence to + * @f: fence to add + * + * Tries to add the fence to an existing hash entry. Returns true when an entry + * was found, false otherwise. + */ +static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct fence *f) +{ + struct amdgpu_sync_entry *e; + + hash_for_each_possible(sync->fences, e, node, f->context) { + if (unlikely(e->fence->context != f->context)) + continue; + + amdgpu_sync_keep_later(&e->fence, f); + return true; + } + return false; +} + +/** * amdgpu_sync_fence - remember to sync to this fence * * @sync: sync object to add fence to @@ -127,13 +150,8 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, amdgpu_sync_get_owner(f) == AMDGPU_FENCE_OWNER_VM) amdgpu_sync_keep_later(&sync->last_vm_update, f); - hash_for_each_possible(sync->fences, e, node, f->context) { - if (unlikely(e->fence->context != f->context)) - continue; - - amdgpu_sync_keep_later(&e->fence, f); + if (amdgpu_sync_add_later(sync, f)) return 0; - } e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL); if (!e) @@ -204,6 +222,81 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, return r; } +/** + * amdgpu_sync_is_idle - test if all fences are signaled + * + * @sync: the sync object + * + * Returns true if all fences in the sync object are signaled. + */ +bool amdgpu_sync_is_idle(struct amdgpu_sync *sync) +{ + struct amdgpu_sync_entry *e; + struct hlist_node *tmp; + int i; + + hash_for_each_safe(sync->fences, i, tmp, e, node) { + struct fence *f = e->fence; + + if (fence_is_signaled(f)) { + hash_del(&e->node); + fence_put(f); + kmem_cache_free(amdgpu_sync_slab, e); + continue; + } + + return false; + } + + return true; +} + +/** + * amdgpu_sync_cycle_fences - move fences from one sync object into another + * + * @dst: the destination sync object + * @src: the source sync object + * @fence: fence to add to source + * + * Remove all fences from source and put them into destination and add + * fence as new one into source. + */ +int amdgpu_sync_cycle_fences(struct amdgpu_sync *dst, struct amdgpu_sync *src, + struct fence *fence) +{ + struct amdgpu_sync_entry *e, *newone; + struct hlist_node *tmp; + int i; + + /* Allocate the new entry before moving the old ones */ + newone = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL); + if (!newone) + return -ENOMEM; + + hash_for_each_safe(src->fences, i, tmp, e, node) { + struct fence *f = e->fence; + + hash_del(&e->node); + if (fence_is_signaled(f)) { + fence_put(f); + kmem_cache_free(amdgpu_sync_slab, e); + continue; + } + + if (amdgpu_sync_add_later(dst, f)) { + kmem_cache_free(amdgpu_sync_slab, e); + continue; + } + + hash_add(dst->fences, &e->node, f->context); + } + + hash_add(src->fences, &newone->node, fence->context); + newone->fence = fence_get(fence); + + return 0; +} + struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync) { struct amdgpu_sync_entry *e; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 8a758b4fb3a9..d0cce7c3129a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -189,14 +189,13 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, if (usable) { - r = amdgpu_sync_fence(ring->adev, sync, id->active); - if (r) { - mutex_unlock(&adev->vm_manager.lock); - return r; - } + r = amdgpu_sync_fence(ring->adev, sync, id->first); + if (r) + goto error; - fence_put(id->active); - id->active = fence_get(fence); + r = amdgpu_sync_fence(ring->adev, &id->active, fence); + if (r) + goto error; list_move_tail(&id->list, &adev->vm_manager.ids_lru); @@ -214,13 +213,13 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, struct amdgpu_vm_id, list); - if (id->active && !fence_is_signaled(id->active)) { - struct amdgpu_vm_id *tmp; + if (!amdgpu_sync_is_idle(&id->active)) { struct list_head *head = &adev->vm_manager.ids_lru; + struct amdgpu_vm_id *tmp; list_for_each_entry_safe(id, tmp, &adev->vm_manager.ids_lru, list) { - if (id->active && fence_is_signaled(id->active)) { + if (amdgpu_sync_is_idle(&id->active)) { list_move(&id->list, head); head = &id->list; } @@ -230,25 +229,27 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, list); } - r = amdgpu_sync_fence(ring->adev, sync, id->active); - if (!r) { - fence_put(id->active); - id->active = fence_get(fence); + r = amdgpu_sync_cycle_fences(sync, &id->active, fence); + if (r) + goto error; - fence_put(id->flushed_updates); - id->flushed_updates = fence_get(updates); + fence_put(id->first); + id->first = fence_get(fence); - id->pd_gpu_addr = pd_addr; + fence_put(id->flushed_updates); + id->flushed_updates = fence_get(updates); - list_move_tail(&id->list, &adev->vm_manager.ids_lru); - atomic_long_set(&id->owner, (long)&vm->ids[ring->idx]); - vm->ids[ring->idx] = id; + id->pd_gpu_addr = pd_addr; - *vm_id = id - adev->vm_manager.ids; - *vm_pd_addr = pd_addr; - trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr); - } + list_move_tail(&id->list, &adev->vm_manager.ids_lru); + atomic_long_set(&id->owner, (long)id); + vm->ids[ring->idx] = id; + *vm_id = id - adev->vm_manager.ids; + *vm_pd_addr = pd_addr; + trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr); + +error: mutex_unlock(&adev->vm_manager.lock); return r; } @@ -1467,6 +1468,7 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev) /* skip over VMID 0, since it is the system VM */ for (i = 1; i < adev->vm_manager.num_ids; ++i) { amdgpu_vm_reset_id(adev, i); + amdgpu_sync_create(&adev->vm_manager.ids[i].active); list_add_tail(&adev->vm_manager.ids[i].list, &adev->vm_manager.ids_lru); } @@ -1488,7 +1490,8 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev) for (i = 0; i < AMDGPU_NUM_VM; ++i) { struct amdgpu_vm_id *id = &adev->vm_manager.ids[i]; - fence_put(id->active); + fence_put(adev->vm_manager.ids[i].first); + amdgpu_sync_free(&adev->vm_manager.ids[i].active); fence_put(id->flushed_updates); } } |