drm/amdgpu: use a sync object for VMID fences v2

v2: rebase & cleanup

This way we can store more than one fence as user for each VMID.

Signed-off-by: Christian König <christian.koenig@amd.com>
Acked-by: Alex Deucher <alexander.deucher@amd.com> (v1)
Reviewed-by: Chunming Zhou <david1.zhou@amd.com> (v1)
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

3 files changed, 132 insertions, 32 deletions

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index c1b10046317e..148e2c61463c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -588,6 +588,9 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
 		     struct amdgpu_sync *sync,
 		     struct reservation_object *resv,
 		     void *owner);
+bool amdgpu_sync_is_idle(struct amdgpu_sync *sync);
+int amdgpu_sync_cycle_fences(struct amdgpu_sync *dst, struct amdgpu_sync *src,
+			     struct fence *fence);
 struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync);
 int amdgpu_sync_wait(struct amdgpu_sync *sync);
 void amdgpu_sync_free(struct amdgpu_sync *sync);
@@ -875,7 +878,8 @@ struct amdgpu_vm {
 
 struct amdgpu_vm_id {
 	struct list_head	list;
-	struct fence		*active;
+	struct fence		*first;
+	struct amdgpu_sync	active;
 	atomic_long_t		owner;
 
 	uint64_t		pd_gpu_addr;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index c48b4fce5e57..34a92808bbd4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -109,6 +109,29 @@ static void amdgpu_sync_keep_later(struct fence **keep, struct fence *fence)
 }
 
 /**
+ * amdgpu_sync_add_later - add the fence to the hash
+ *
+ * @sync: sync object to add the fence to
+ * @f: fence to add
+ *
+ * Tries to add the fence to an existing hash entry. Returns true when an entry
+ * was found, false otherwise.
+ */
+static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct fence *f)
+{
+	struct amdgpu_sync_entry *e;
+
+	hash_for_each_possible(sync->fences, e, node, f->context) {
+		if (unlikely(e->fence->context != f->context))
+			continue;
+
+		amdgpu_sync_keep_later(&e->fence, f);
+		return true;
+	}
+	return false;
+}
+
+/**
  * amdgpu_sync_fence - remember to sync to this fence
  *
  * @sync: sync object to add fence to
@@ -127,13 +150,8 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
 	    amdgpu_sync_get_owner(f) == AMDGPU_FENCE_OWNER_VM)
 		amdgpu_sync_keep_later(&sync->last_vm_update, f);
 
-	hash_for_each_possible(sync->fences, e, node, f->context) {
-		if (unlikely(e->fence->context != f->context))
-			continue;
-
-		amdgpu_sync_keep_later(&e->fence, f);
+	if (amdgpu_sync_add_later(sync, f))
 		return 0;
-	}
 
 	e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);
 	if (!e)
@@ -204,6 +222,81 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
 	return r;
 }
 
+/**
+ * amdgpu_sync_is_idle - test if all fences are signaled
+ *
+ * @sync: the sync object
+ *
+ * Returns true if all fences in the sync object are signaled.
+ */
+bool amdgpu_sync_is_idle(struct amdgpu_sync *sync)
+{
+	struct amdgpu_sync_entry *e;
+	struct hlist_node *tmp;
+	int i;
+
+	hash_for_each_safe(sync->fences, i, tmp, e, node) {
+		struct fence *f = e->fence;
+
+		if (fence_is_signaled(f)) {
+			hash_del(&e->node);
+			fence_put(f);
+			kmem_cache_free(amdgpu_sync_slab, e);
+			continue;
+		}
+
+		return false;
+	}
+
+	return true;
+}
+
+/**
+ * amdgpu_sync_cycle_fences - move fences from one sync object into another
+ *
+ * @dst: the destination sync object
+ * @src: the source sync object
+ * @fence: fence to add to source
+ *
+ * Remove all fences from source and put them into destination and add
+ * fence as new one into source.
+ */
+int amdgpu_sync_cycle_fences(struct amdgpu_sync *dst, struct amdgpu_sync *src,
+			     struct fence *fence)
+{
+	struct amdgpu_sync_entry *e, *newone;
+	struct hlist_node *tmp;
+	int i;
+
+	/* Allocate the new entry before moving the old ones */
+	newone = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);
+	if (!newone)
+		return -ENOMEM;
+
+	hash_for_each_safe(src->fences, i, tmp, e, node) {
+		struct fence *f = e->fence;
+
+		hash_del(&e->node);
+		if (fence_is_signaled(f)) {
+			fence_put(f);
+			kmem_cache_free(amdgpu_sync_slab, e);
+			continue;
+		}
+
+		if (amdgpu_sync_add_later(dst, f)) {
+			kmem_cache_free(amdgpu_sync_slab, e);
+			continue;
+		}
+
+		hash_add(dst->fences, &e->node, f->context);
+	}
+
+	hash_add(src->fences, &newone->node, fence->context);
+	newone->fence = fence_get(fence);
+
+	return 0;
+}
+
 struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
 {
 	struct amdgpu_sync_entry *e;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 8a758b4fb3a9..d0cce7c3129a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -189,14 +189,13 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 
 		if (usable) {
 
-			r = amdgpu_sync_fence(ring->adev, sync, id->active);
-			if (r) {
-				mutex_unlock(&adev->vm_manager.lock);
-				return r;
-			}
+			r = amdgpu_sync_fence(ring->adev, sync, id->first);
+			if (r)
+				goto error;
 
-			fence_put(id->active);
-			id->active = fence_get(fence);
+			r = amdgpu_sync_fence(ring->adev, &id->active, fence);
+			if (r)
+				goto error;
 
 			list_move_tail(&id->list, &adev->vm_manager.ids_lru);
 
@@ -214,13 +213,13 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 			      struct amdgpu_vm_id,
 			      list);
 
-	if (id->active && !fence_is_signaled(id->active)) {
-		struct amdgpu_vm_id *tmp;
+	if (!amdgpu_sync_is_idle(&id->active)) {
 		struct list_head *head = &adev->vm_manager.ids_lru;
+		struct amdgpu_vm_id *tmp;
 
 		list_for_each_entry_safe(id, tmp, &adev->vm_manager.ids_lru,
 					 list) {
-			if (id->active && fence_is_signaled(id->active)) {
+			if (amdgpu_sync_is_idle(&id->active)) {
 				list_move(&id->list, head);
 				head = &id->list;
 			}
@@ -230,25 +229,27 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 				      list);
 	}
 
-	r = amdgpu_sync_fence(ring->adev, sync, id->active);
-	if (!r) {
-		fence_put(id->active);
-		id->active = fence_get(fence);
+	r = amdgpu_sync_cycle_fences(sync, &id->active, fence);
+	if (r)
+		goto error;
 
-		fence_put(id->flushed_updates);
-		id->flushed_updates = fence_get(updates);
+	fence_put(id->first);
+	id->first = fence_get(fence);
 
-		id->pd_gpu_addr = pd_addr;
+	fence_put(id->flushed_updates);
+	id->flushed_updates = fence_get(updates);
 
-		list_move_tail(&id->list, &adev->vm_manager.ids_lru);
-		atomic_long_set(&id->owner, (long)&vm->ids[ring->idx]);
-		vm->ids[ring->idx] = id;
+	id->pd_gpu_addr = pd_addr;
 
-		*vm_id = id - adev->vm_manager.ids;
-		*vm_pd_addr = pd_addr;
-		trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr);
-	}
+	list_move_tail(&id->list, &adev->vm_manager.ids_lru);
+	atomic_long_set(&id->owner, (long)id);
+	vm->ids[ring->idx] = id;
 
+	*vm_id = id - adev->vm_manager.ids;
+	*vm_pd_addr = pd_addr;
+	trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr);
+
+error:
 	mutex_unlock(&adev->vm_manager.lock);
 	return r;
 }
@@ -1467,6 +1468,7 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
 	/* skip over VMID 0, since it is the system VM */
 	for (i = 1; i < adev->vm_manager.num_ids; ++i) {
 		amdgpu_vm_reset_id(adev, i);
+		amdgpu_sync_create(&adev->vm_manager.ids[i].active);
 		list_add_tail(&adev->vm_manager.ids[i].list,
 			      &adev->vm_manager.ids_lru);
 	}
@@ -1488,7 +1490,8 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
 	for (i = 0; i < AMDGPU_NUM_VM; ++i) {
 		struct amdgpu_vm_id *id = &adev->vm_manager.ids[i];
 
-		fence_put(id->active);
+		fence_put(adev->vm_manager.ids[i].first);
+		amdgpu_sync_free(&adev->vm_manager.ids[i].active);
 		fence_put(id->flushed_updates);
 	}
 }