From bb977d3711ed1de1601b463e7fd5a43d82a2b077 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Tue, 18 Aug 2015 15:16:40 +0800 Subject: drm/amdgpu: abstract amdgpu_job for scheduler Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu.h') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 2fc58e658986..95d4969369a6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -183,6 +183,7 @@ struct amdgpu_vm; struct amdgpu_ring; struct amdgpu_semaphore; struct amdgpu_cs_parser; +struct amdgpu_job; struct amdgpu_irq_src; struct amdgpu_fpriv; @@ -871,7 +872,7 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, struct amdgpu_ring *ring, struct amdgpu_ib *ibs, unsigned num_ibs, - int (*free_job)(struct amdgpu_cs_parser *), + int (*free_job)(struct amdgpu_job *), void *owner, struct fence **fence); @@ -1040,6 +1041,7 @@ void amdgpu_ctx_fini(struct amdgpu_ctx *ctx); struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id); int amdgpu_ctx_put(struct amdgpu_ctx *ctx); +struct amdgpu_ctx *amdgpu_ctx_get_ref(struct amdgpu_ctx *ctx); uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, struct fence *fence, uint64_t queued_seq); @@ -1265,6 +1267,18 @@ struct amdgpu_cs_parser { struct amd_sched_fence *s_fence; }; +struct amdgpu_job { + struct amd_sched_job base; + struct amdgpu_device *adev; + struct amdgpu_ctx *ctx; + struct drm_file *owner; + struct amdgpu_ib *ibs; + uint32_t num_ibs; + struct mutex job_lock; + struct amdgpu_user_fence uf; + int (*free_job)(struct amdgpu_job *sched_job); +}; + static inline u32 amdgpu_get_ib_value(struct amdgpu_cs_parser *p, uint32_t ib_idx, int idx) { return p->ibs[ib_idx].ptr[idx]; -- cgit v1.2.3 From 3c4adead8480f81df687a181d1e54a39b91b9363 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Tue, 18 Aug 2015 16:19:13 +0800 Subject: drm/amdgpu: some clean up for cs_ioctl Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 8 -------- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 29 +---------------------------- 2 files changed, 1 insertion(+), 36 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu.h') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 95d4969369a6..d050f503d9ff 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1257,14 +1257,6 @@ struct amdgpu_cs_parser { /* user fence */ struct amdgpu_user_fence uf; - - struct amdgpu_ring *ring; - struct mutex job_lock; - struct work_struct job_work; - int (*prepare_job)(struct amdgpu_cs_parser *sched_job); - int (*run_job)(struct amdgpu_cs_parser *sched_job); - int (*free_job)(struct amdgpu_cs_parser *sched_job); - struct amd_sched_fence *s_fence; }; struct amdgpu_job { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 7b4823deef33..77fc7cd0fac3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -827,32 +827,6 @@ static int amdgpu_cs_parser_prepare_job(struct amdgpu_cs_parser *sched_job) return r; } -static struct amdgpu_ring *amdgpu_cs_parser_get_ring( - struct amdgpu_device *adev, - struct amdgpu_cs_parser *parser) -{ - int i, r; - - struct amdgpu_cs_chunk *chunk; - struct drm_amdgpu_cs_chunk_ib *chunk_ib; - struct amdgpu_ring *ring; - for (i = 0; i < parser->nchunks; i++) { - chunk = &parser->chunks[i]; - chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata; - - if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) - continue; - - r = amdgpu_cs_get_ring(adev, chunk_ib->ip_type, - chunk_ib->ip_instance, chunk_ib->ring, - &ring); - if (r) - return NULL; - break; - } - return ring; -} - static int amdgpu_cs_free_job(struct amdgpu_job *sched_job) { int i; @@ -897,8 +871,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) if (amdgpu_enable_scheduler && parser->num_ibs) { struct amdgpu_job *job; - struct amdgpu_ring * ring = - amdgpu_cs_parser_get_ring(adev, parser); + struct amdgpu_ring * ring = parser->ibs->ring; job = kzalloc(sizeof(struct amdgpu_job), GFP_KERNEL); if (!job) return -ENOMEM; -- cgit v1.2.3 From 1aa4051b7f5474cca6009c13868c59d78d06f983 Mon Sep 17 00:00:00 2001 From: Junwei Zhang Date: Wed, 19 Aug 2015 16:24:19 +0800 Subject: drm/amdgpu: modify amdgpu_fence_wait_any() to amdgpu_fence_wait_multiple() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename the function and update the related code with this modified function. Add the new parameter of bool wait_all. If wait_all is true, it will return when all fences are signaled or timeout. If wait_all is false, it will return when any fence is signaled or timeout. Signed-off-by: Junwei Zhang Reviewed-by: Monk Liu Reviewed-by: Jammy Zhou Reviewed-by: Christian König --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 9 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 79 ++++++++++++++++++++++++------- drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c | 3 +- 3 files changed, 69 insertions(+), 22 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu.h') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index d050f503d9ff..4addac5f6763 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -440,9 +440,12 @@ int amdgpu_fence_wait_next(struct amdgpu_ring *ring); int amdgpu_fence_wait_empty(struct amdgpu_ring *ring); unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring); -signed long amdgpu_fence_wait_any(struct amdgpu_device *adev, - struct amdgpu_fence **fences, - bool intr, long t); +signed long amdgpu_fence_wait_multiple(struct amdgpu_device *adev, + struct amdgpu_fence **array, + uint32_t count, + bool wait_all, + bool intr, + signed long t); struct amdgpu_fence *amdgpu_fence_ref(struct amdgpu_fence *fence); void amdgpu_fence_unref(struct amdgpu_fence **fence); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 98500f1756f7..ae014fcf524e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -836,13 +836,12 @@ static inline bool amdgpu_test_signaled(struct amdgpu_fence *fence) return test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags); } -static inline bool amdgpu_test_signaled_any(struct amdgpu_fence **fences) +static bool amdgpu_test_signaled_any(struct amdgpu_fence **fences, uint32_t count) { int idx; struct amdgpu_fence *fence; - idx = 0; - for (idx = 0; idx < AMDGPU_MAX_RINGS; ++idx) { + for (idx = 0; idx < count; ++idx) { fence = fences[idx]; if (fence) { if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags)) @@ -852,6 +851,22 @@ static inline bool amdgpu_test_signaled_any(struct amdgpu_fence **fences) return false; } +static bool amdgpu_test_signaled_all(struct amdgpu_fence **fences, uint32_t count) +{ + int idx; + struct amdgpu_fence *fence; + + for (idx = 0; idx < count; ++idx) { + fence = fences[idx]; + if (fence) { + if (!test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags)) + return false; + } + } + + return true; +} + struct amdgpu_wait_cb { struct fence_cb base; struct task_struct *task; @@ -867,33 +882,56 @@ static void amdgpu_fence_wait_cb(struct fence *fence, struct fence_cb *cb) static signed long amdgpu_fence_default_wait(struct fence *f, bool intr, signed long t) { - struct amdgpu_fence *array[AMDGPU_MAX_RINGS]; struct amdgpu_fence *fence = to_amdgpu_fence(f); struct amdgpu_device *adev = fence->ring->adev; - memset(&array[0], 0, sizeof(array)); - array[0] = fence; - - return amdgpu_fence_wait_any(adev, array, intr, t); + return amdgpu_fence_wait_multiple(adev, &fence, 1, false, intr, t); } -/* wait until any fence in array signaled */ -signed long amdgpu_fence_wait_any(struct amdgpu_device *adev, - struct amdgpu_fence **array, bool intr, signed long t) +/** + * Wait the fence array with timeout + * + * @adev: amdgpu device + * @array: the fence array with amdgpu fence pointer + * @count: the number of the fence array + * @wait_all: the flag of wait all(true) or wait any(false) + * @intr: when sleep, set the current task interruptable or not + * @t: timeout to wait + * + * If wait_all is true, it will return when all fences are signaled or timeout. + * If wait_all is false, it will return when any fence is signaled or timeout. + */ +signed long amdgpu_fence_wait_multiple(struct amdgpu_device *adev, + struct amdgpu_fence **array, + uint32_t count, + bool wait_all, + bool intr, + signed long t) { long idx = 0; - struct amdgpu_wait_cb cb[AMDGPU_MAX_RINGS]; + struct amdgpu_wait_cb *cb; struct amdgpu_fence *fence; BUG_ON(!array); - for (idx = 0; idx < AMDGPU_MAX_RINGS; ++idx) { + cb = kcalloc(count, sizeof(struct amdgpu_wait_cb), GFP_KERNEL); + if (cb == NULL) { + t = -ENOMEM; + goto err_free_cb; + } + + for (idx = 0; idx < count; ++idx) { fence = array[idx]; if (fence) { cb[idx].task = current; if (fence_add_callback(&fence->base, - &cb[idx].base, amdgpu_fence_wait_cb)) - return t; /* return if fence is already signaled */ + &cb[idx].base, amdgpu_fence_wait_cb)) { + /* The fence is already signaled */ + if (wait_all) + continue; + else + goto fence_rm_cb; + } } } @@ -907,7 +945,9 @@ signed long amdgpu_fence_wait_any(struct amdgpu_device *adev, * amdgpu_test_signaled_any must be called after * set_current_state to prevent a race with wake_up_process */ - if (amdgpu_test_signaled_any(array)) + if (!wait_all && amdgpu_test_signaled_any(array, count)) + break; + if (wait_all && amdgpu_test_signaled_all(array, count)) break; if (adev->needs_reset) { @@ -923,13 +963,16 @@ signed long amdgpu_fence_wait_any(struct amdgpu_device *adev, __set_current_state(TASK_RUNNING); - idx = 0; - for (idx = 0; idx < AMDGPU_MAX_RINGS; ++idx) { +fence_rm_cb: + for (idx = 0; idx < count; ++idx) { fence = array[idx]; if (fence) fence_remove_callback(&fence->base, &cb[idx].base); } +err_free_cb: + kfree(cb); + return t; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c index d6398cf45f24..4597899e9758 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c @@ -352,7 +352,8 @@ int amdgpu_sa_bo_new(struct amdgpu_device *adev, } while (amdgpu_sa_bo_next_hole(sa_manager, fences, tries)); spin_unlock(&sa_manager->wq.lock); - t = amdgpu_fence_wait_any(adev, fences, false, MAX_SCHEDULE_TIMEOUT); + t = amdgpu_fence_wait_multiple(adev, fences, AMDGPU_MAX_RINGS, false, false, + MAX_SCHEDULE_TIMEOUT); r = (t > 0) ? 0 : t; spin_lock(&sa_manager->wq.lock); /* if we have nothing to wait for block */ -- cgit v1.2.3 From 4ce9891ee17c6e064cc334e3297f7e992d47f3a6 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Wed, 19 Aug 2015 16:41:19 +0800 Subject: drm/amdgpu: improve sa_bo->fence by kernel fence Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 8 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 22 ++++++------ drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 4 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c | 50 ++++++++++++++++++++------- drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 2 +- 8 files changed, 58 insertions(+), 34 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu.h') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 4addac5f6763..80f2ceaf6af6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -441,7 +441,7 @@ int amdgpu_fence_wait_empty(struct amdgpu_ring *ring); unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring); signed long amdgpu_fence_wait_multiple(struct amdgpu_device *adev, - struct amdgpu_fence **array, + struct fence **array, uint32_t count, bool wait_all, bool intr, @@ -654,7 +654,7 @@ struct amdgpu_sa_bo { struct amdgpu_sa_manager *manager; unsigned soffset; unsigned eoffset; - struct amdgpu_fence *fence; + struct fence *fence; }; /* @@ -696,7 +696,7 @@ bool amdgpu_semaphore_emit_wait(struct amdgpu_ring *ring, struct amdgpu_semaphore *semaphore); void amdgpu_semaphore_free(struct amdgpu_device *adev, struct amdgpu_semaphore **semaphore, - struct amdgpu_fence *fence); + struct fence *fence); /* * Synchronization @@ -717,7 +717,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, int amdgpu_sync_rings(struct amdgpu_sync *sync, struct amdgpu_ring *ring); void amdgpu_sync_free(struct amdgpu_device *adev, struct amdgpu_sync *sync, - struct amdgpu_fence *fence); + struct fence *fence); /* * GART structures, functions & helpers diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index ae014fcf524e..9a87372c3c79 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -836,30 +836,30 @@ static inline bool amdgpu_test_signaled(struct amdgpu_fence *fence) return test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags); } -static bool amdgpu_test_signaled_any(struct amdgpu_fence **fences, uint32_t count) +static bool amdgpu_test_signaled_any(struct fence **fences, uint32_t count) { int idx; - struct amdgpu_fence *fence; + struct fence *fence; for (idx = 0; idx < count; ++idx) { fence = fences[idx]; if (fence) { - if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags)) + if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) return true; } } return false; } -static bool amdgpu_test_signaled_all(struct amdgpu_fence **fences, uint32_t count) +static bool amdgpu_test_signaled_all(struct fence **fences, uint32_t count) { int idx; - struct amdgpu_fence *fence; + struct fence *fence; for (idx = 0; idx < count; ++idx) { fence = fences[idx]; if (fence) { - if (!test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags)) + if (!test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) return false; } } @@ -885,7 +885,7 @@ static signed long amdgpu_fence_default_wait(struct fence *f, bool intr, struct amdgpu_fence *fence = to_amdgpu_fence(f); struct amdgpu_device *adev = fence->ring->adev; - return amdgpu_fence_wait_multiple(adev, &fence, 1, false, intr, t); + return amdgpu_fence_wait_multiple(adev, &f, 1, false, intr, t); } /** @@ -902,7 +902,7 @@ static signed long amdgpu_fence_default_wait(struct fence *f, bool intr, * If wait_all is false, it will return when any fence is signaled or timeout. */ signed long amdgpu_fence_wait_multiple(struct amdgpu_device *adev, - struct amdgpu_fence **array, + struct fence **array, uint32_t count, bool wait_all, bool intr, @@ -910,7 +910,7 @@ signed long amdgpu_fence_wait_multiple(struct amdgpu_device *adev, { long idx = 0; struct amdgpu_wait_cb *cb; - struct amdgpu_fence *fence; + struct fence *fence; BUG_ON(!array); @@ -924,7 +924,7 @@ signed long amdgpu_fence_wait_multiple(struct amdgpu_device *adev, fence = array[idx]; if (fence) { cb[idx].task = current; - if (fence_add_callback(&fence->base, + if (fence_add_callback(fence, &cb[idx].base, amdgpu_fence_wait_cb)) { /* The fence is already signaled */ if (wait_all) @@ -967,7 +967,7 @@ fence_rm_cb: for (idx = 0; idx < count; ++idx) { fence = array[idx]; if (fence) - fence_remove_callback(&fence->base, &cb[idx].base); + fence_remove_callback(fence, &cb[idx].base); } err_free_cb: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 1c237f5e3365..13c5978ac69b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -93,8 +93,8 @@ int amdgpu_ib_get(struct amdgpu_ring *ring, struct amdgpu_vm *vm, */ void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib) { - amdgpu_sync_free(adev, &ib->sync, ib->fence); - amdgpu_sa_bo_free(adev, &ib->sa_bo, ib->fence); + amdgpu_sync_free(adev, &ib->sync, &ib->fence->base); + amdgpu_sa_bo_free(adev, &ib->sa_bo, &ib->fence->base); amdgpu_fence_unref(&ib->fence); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 238465a9ac55..6ea18dcec561 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -193,7 +193,7 @@ int amdgpu_sa_bo_new(struct amdgpu_device *adev, unsigned size, unsigned align); void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo, - struct amdgpu_fence *fence); + struct fence *fence); #if defined(CONFIG_DEBUG_FS) void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, struct seq_file *m); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c index 4597899e9758..b7cbaa9d532e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c @@ -139,6 +139,20 @@ int amdgpu_sa_bo_manager_suspend(struct amdgpu_device *adev, return r; } +static uint32_t amdgpu_sa_get_ring_from_fence(struct fence *f) +{ + struct amdgpu_fence *a_fence; + struct amd_sched_fence *s_fence; + + s_fence = to_amd_sched_fence(f); + if (s_fence) + return s_fence->entity->scheduler->ring_id; + a_fence = to_amdgpu_fence(f); + if (a_fence) + return a_fence->ring->idx; + return 0; +} + static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo) { struct amdgpu_sa_manager *sa_manager = sa_bo->manager; @@ -147,7 +161,7 @@ static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo) } list_del_init(&sa_bo->olist); list_del_init(&sa_bo->flist); - amdgpu_fence_unref(&sa_bo->fence); + fence_put(sa_bo->fence); kfree(sa_bo); } @@ -161,7 +175,7 @@ static void amdgpu_sa_bo_try_free(struct amdgpu_sa_manager *sa_manager) sa_bo = list_entry(sa_manager->hole->next, struct amdgpu_sa_bo, olist); list_for_each_entry_safe_from(sa_bo, tmp, &sa_manager->olist, olist) { if (sa_bo->fence == NULL || - !fence_is_signaled(&sa_bo->fence->base)) { + !fence_is_signaled(sa_bo->fence)) { return; } amdgpu_sa_bo_remove_locked(sa_bo); @@ -246,7 +260,7 @@ static bool amdgpu_sa_event(struct amdgpu_sa_manager *sa_manager, } static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager, - struct amdgpu_fence **fences, + struct fence **fences, unsigned *tries) { struct amdgpu_sa_bo *best_bo = NULL; @@ -275,7 +289,7 @@ static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager, sa_bo = list_first_entry(&sa_manager->flist[i], struct amdgpu_sa_bo, flist); - if (!fence_is_signaled(&sa_bo->fence->base)) { + if (!fence_is_signaled(sa_bo->fence)) { fences[i] = sa_bo->fence; continue; } @@ -299,7 +313,8 @@ static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager, } if (best_bo) { - ++tries[best_bo->fence->ring->idx]; + uint32_t idx = amdgpu_sa_get_ring_from_fence(best_bo->fence); + ++tries[idx]; sa_manager->hole = best_bo->olist.prev; /* we knew that this one is signaled, @@ -315,7 +330,7 @@ int amdgpu_sa_bo_new(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo, unsigned size, unsigned align) { - struct amdgpu_fence *fences[AMDGPU_MAX_RINGS]; + struct fence *fences[AMDGPU_MAX_RINGS]; unsigned tries[AMDGPU_MAX_RINGS]; int i, r; signed long t; @@ -373,7 +388,7 @@ int amdgpu_sa_bo_new(struct amdgpu_device *adev, } void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo, - struct amdgpu_fence *fence) + struct fence *fence) { struct amdgpu_sa_manager *sa_manager; @@ -383,10 +398,11 @@ void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo, sa_manager = (*sa_bo)->manager; spin_lock(&sa_manager->wq.lock); - if (fence && !fence_is_signaled(&fence->base)) { - (*sa_bo)->fence = amdgpu_fence_ref(fence); - list_add_tail(&(*sa_bo)->flist, - &sa_manager->flist[fence->ring->idx]); + if (fence && !fence_is_signaled(fence)) { + uint32_t idx; + (*sa_bo)->fence = fence_get(fence); + idx = amdgpu_sa_get_ring_from_fence(fence); + list_add_tail(&(*sa_bo)->flist, &sa_manager->flist[idx]); } else { amdgpu_sa_bo_remove_locked(*sa_bo); } @@ -413,8 +429,16 @@ void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, seq_printf(m, "[0x%010llx 0x%010llx] size %8lld", soffset, eoffset, eoffset - soffset); if (i->fence) { - seq_printf(m, " protected by 0x%016llx on ring %d", - i->fence->seq, i->fence->ring->idx); + struct amdgpu_fence *a_fence = to_amdgpu_fence(i->fence); + struct amd_sched_fence *s_fence = to_amd_sched_fence(i->fence); + if (a_fence) + seq_printf(m, " protected by 0x%016llx on ring %d", + a_fence->seq, a_fence->ring->idx); + if (s_fence) + seq_printf(m, " protected by 0x%016llx on ring %d", + s_fence->v_seq, + s_fence->entity->scheduler->ring_id); + } seq_printf(m, "\n"); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c index d6d41a42ab65..ff3ca52ec6fe 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c @@ -87,7 +87,7 @@ bool amdgpu_semaphore_emit_wait(struct amdgpu_ring *ring, void amdgpu_semaphore_free(struct amdgpu_device *adev, struct amdgpu_semaphore **semaphore, - struct amdgpu_fence *fence) + struct fence *fence) { if (semaphore == NULL || *semaphore == NULL) { return; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c index 7cb711fc1ee2..ee68eebfded1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c @@ -234,7 +234,7 @@ int amdgpu_sync_rings(struct amdgpu_sync *sync, */ void amdgpu_sync_free(struct amdgpu_device *adev, struct amdgpu_sync *sync, - struct amdgpu_fence *fence) + struct fence *fence) { unsigned i; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index dd3415d2e45d..d7c02e1a309e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1042,7 +1042,7 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, } amdgpu_ring_unlock_commit(ring); - amdgpu_sync_free(adev, &sync, *fence); + amdgpu_sync_free(adev, &sync, &(*fence)->base); return 0; } -- cgit v1.2.3 From ce882e6dc241ab8dded0eeeb33a86482d44a5689 Mon Sep 17 00:00:00 2001 From: Christian König Date: Wed, 19 Aug 2015 15:00:55 +0200 Subject: drm/amdgpu: remove v_seq handling from the scheduler v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simply not used any more. Only keep 32bit atomic for fence sequence numbering. v2: trivial rebase Signed-off-by: Christian König Reviewed-by: Alex Deucher (v1) Reviewed-by: Jammy Zhou (v1) Reviewed-by: Chunming Zhou (v1) --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 6 ++---- drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 20 +++++--------------- drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 6 +----- drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c | 4 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 1 - drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 19 +------------------ drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 7 +------ drivers/gpu/drm/amd/scheduler/sched_fence.c | 13 ++++++++----- 9 files changed, 21 insertions(+), 57 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu.h') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 80f2ceaf6af6..65e0e9406abb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1047,7 +1047,7 @@ int amdgpu_ctx_put(struct amdgpu_ctx *ctx); struct amdgpu_ctx *amdgpu_ctx_get_ref(struct amdgpu_ctx *ctx); uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, - struct fence *fence, uint64_t queued_seq); + struct fence *fence); struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, uint64_t seq); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index dc8d2829c1e9..f91849b12a0f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -866,11 +866,9 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) kfree(job); goto out; } - job->ibs[parser->num_ibs - 1].sequence = + cs->out.handle = amdgpu_ctx_add_fence(job->ctx, ring, - &job->base.s_fence->base, - job->base.s_fence->v_seq); - cs->out.handle = job->base.s_fence->v_seq; + &job->base.s_fence->base); list_sort(NULL, &parser->validated, cmp_size_smaller_first); ttm_eu_fence_buffer_objects(&parser->ticket, &parser->validated, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index 8660c0854a1e..f024effa60f9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -236,17 +236,13 @@ int amdgpu_ctx_put(struct amdgpu_ctx *ctx) } uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, - struct fence *fence, uint64_t queued_seq) + struct fence *fence) { struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx]; - uint64_t seq = 0; + uint64_t seq = cring->sequence; unsigned idx = 0; struct fence *other = NULL; - if (amdgpu_enable_scheduler) - seq = queued_seq; - else - seq = cring->sequence; idx = seq % AMDGPU_CTX_MAX_CS_PENDING; other = cring->fences[idx]; if (other) { @@ -260,8 +256,7 @@ uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, spin_lock(&ctx->ring_lock); cring->fences[idx] = fence; - if (!amdgpu_enable_scheduler) - cring->sequence++; + cring->sequence++; spin_unlock(&ctx->ring_lock); fence_put(other); @@ -274,21 +269,16 @@ struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, { struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx]; struct fence *fence; - uint64_t queued_seq; spin_lock(&ctx->ring_lock); - if (amdgpu_enable_scheduler) - queued_seq = amd_sched_next_queued_seq(&cring->entity); - else - queued_seq = cring->sequence; - if (seq >= queued_seq) { + if (seq >= cring->sequence) { spin_unlock(&ctx->ring_lock); return ERR_PTR(-EINVAL); } - if (seq + AMDGPU_CTX_MAX_CS_PENDING < queued_seq) { + if (seq + AMDGPU_CTX_MAX_CS_PENDING < cring->sequence) { spin_unlock(&ctx->ring_lock); return NULL; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 13c5978ac69b..737c8e3e3a74 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -126,7 +126,6 @@ int amdgpu_ib_schedule(struct amdgpu_device *adev, unsigned num_ibs, struct amdgpu_ring *ring; struct amdgpu_ctx *ctx, *old_ctx; struct amdgpu_vm *vm; - uint64_t sequence; unsigned i; int r = 0; @@ -199,12 +198,9 @@ int amdgpu_ib_schedule(struct amdgpu_device *adev, unsigned num_ibs, return r; } - sequence = amdgpu_enable_scheduler ? ib->sequence : 0; - if (!amdgpu_enable_scheduler && ib->ctx) ib->sequence = amdgpu_ctx_add_fence(ib->ctx, ring, - &ib->fence->base, - sequence); + &ib->fence->base); /* wrap the last IB with fence */ if (ib->user) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c index b7cbaa9d532e..26b17939c9c9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c @@ -435,8 +435,8 @@ void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, seq_printf(m, " protected by 0x%016llx on ring %d", a_fence->seq, a_fence->ring->idx); if (s_fence) - seq_printf(m, " protected by 0x%016llx on ring %d", - s_fence->v_seq, + seq_printf(m, " protected by 0x%016x on ring %d", + s_fence->base.seqno, s_fence->entity->scheduler->ring_id); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 06d7bf51db9a..964b54381feb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -111,7 +111,6 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, kfree(job); return r; } - ibs[num_ibs - 1].sequence = job->base.s_fence->v_seq; *f = fence_get(&job->base.s_fence->base); mutex_unlock(&job->job_lock); } else { diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index 1125aa2e2a9a..f8d46b0b4e3b 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -156,14 +156,12 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched, struct amd_sched_rq *rq, uint32_t jobs) { - uint64_t seq_ring = 0; char name[20]; if (!(sched && entity && rq)) return -EINVAL; memset(entity, 0, sizeof(struct amd_sched_entity)); - seq_ring = ((uint64_t)sched->ring_id) << 60; spin_lock_init(&entity->lock); entity->belongto_rq = rq; entity->scheduler = sched; @@ -179,8 +177,7 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched, return -EINVAL; spin_lock_init(&entity->queue_lock); - atomic64_set(&entity->last_queued_v_seq, seq_ring); - atomic64_set(&entity->last_signaled_v_seq, seq_ring); + atomic_set(&entity->fence_seq, 0); /* Add the entity to the run queue */ amd_sched_rq_add_entity(rq, entity); @@ -299,8 +296,6 @@ static void amd_sched_process_job(struct fence *f, struct fence_cb *cb) unsigned long flags; sched = sched_job->sched; - atomic64_set(&sched_job->s_entity->last_signaled_v_seq, - sched_job->s_fence->v_seq); amd_sched_fence_signal(sched_job->s_fence); spin_lock_irqsave(&sched->queue_lock, flags); list_del(&sched_job->list); @@ -421,15 +416,3 @@ int amd_sched_destroy(struct amd_gpu_scheduler *sched) kfree(sched); return 0; } - -/** - * Get next queued sequence number - * - * @entity The context entity - * - * return the next queued sequence number -*/ -uint64_t amd_sched_next_queued_seq(struct amd_sched_entity *c_entity) -{ - return atomic64_read(&c_entity->last_queued_v_seq) + 1; -} diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index 6597d61266e7..d328e968beb3 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -42,9 +42,7 @@ struct amd_sched_entity { struct list_head list; struct amd_sched_rq *belongto_rq; spinlock_t lock; - /* the virtual_seq is unique per context per ring */ - atomic64_t last_queued_v_seq; - atomic64_t last_signaled_v_seq; + atomic_t fence_seq; /* the job_queue maintains the jobs submitted by clients */ struct kfifo job_queue; spinlock_t queue_lock; @@ -72,7 +70,6 @@ struct amd_sched_fence { struct fence base; struct fence_cb cb; struct amd_sched_entity *entity; - uint64_t v_seq; spinlock_t lock; }; @@ -148,8 +145,6 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched, int amd_sched_entity_fini(struct amd_gpu_scheduler *sched, struct amd_sched_entity *entity); -uint64_t amd_sched_next_queued_seq(struct amd_sched_entity *c_entity); - struct amd_sched_fence *amd_sched_fence_create( struct amd_sched_entity *s_entity); void amd_sched_fence_signal(struct amd_sched_fence *fence); diff --git a/drivers/gpu/drm/amd/scheduler/sched_fence.c b/drivers/gpu/drm/amd/scheduler/sched_fence.c index a4751598c0b4..266ed7bbbc74 100644 --- a/drivers/gpu/drm/amd/scheduler/sched_fence.c +++ b/drivers/gpu/drm/amd/scheduler/sched_fence.c @@ -30,16 +30,19 @@ struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *s_entity) { struct amd_sched_fence *fence = NULL; + unsigned seq; + fence = kzalloc(sizeof(struct amd_sched_fence), GFP_KERNEL); if (fence == NULL) return NULL; - fence->v_seq = atomic64_inc_return(&s_entity->last_queued_v_seq); + fence->entity = s_entity; spin_lock_init(&fence->lock); - fence_init(&fence->base, &amd_sched_fence_ops, - &fence->lock, - s_entity->fence_context, - fence->v_seq); + + seq = atomic_inc_return(&s_entity->fence_seq); + fence_init(&fence->base, &amd_sched_fence_ops, &fence->lock, + s_entity->fence_context, seq); + return fence; } -- cgit v1.2.3 From f91b3a69418120974c9a416939b903ec86607c52 Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 20 Aug 2015 14:47:40 +0800 Subject: drm/amdgpu: fix fence wait in sync_fence, instead should be in sync_rings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 6 +++- drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 56 ++++++++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 3 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu.h') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 65e0e9406abb..3c5487257ef0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -704,6 +704,7 @@ void amdgpu_semaphore_free(struct amdgpu_device *adev, struct amdgpu_sync { struct amdgpu_semaphore *semaphores[AMDGPU_NUM_SYNCS]; struct amdgpu_fence *sync_to[AMDGPU_MAX_RINGS]; + DECLARE_HASHTABLE(fences, 4); struct amdgpu_fence *last_vm_update; }; @@ -716,6 +717,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, void *owner); int amdgpu_sync_rings(struct amdgpu_sync *sync, struct amdgpu_ring *ring); +int amdgpu_sync_wait(struct amdgpu_sync *sync); void amdgpu_sync_free(struct amdgpu_device *adev, struct amdgpu_sync *sync, struct fence *fence); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 737c8e3e3a74..c439735ee670 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -140,7 +140,11 @@ int amdgpu_ib_schedule(struct amdgpu_device *adev, unsigned num_ibs, dev_err(adev->dev, "couldn't schedule ib\n"); return -EINVAL; } - + r = amdgpu_sync_wait(&ibs->sync); + if (r) { + dev_err(adev->dev, "IB sync failed (%d).\n", r); + return r; + } r = amdgpu_ring_lock(ring, (256 + AMDGPU_NUM_SYNCS * 8) * num_ibs); if (r) { dev_err(adev->dev, "scheduling IB failed (%d).\n", r); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c index ee68eebfded1..febbf37b1412 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c @@ -32,6 +32,11 @@ #include "amdgpu.h" #include "amdgpu_trace.h" +struct amdgpu_sync_entry { + struct hlist_node node; + struct fence *fence; +}; + /** * amdgpu_sync_create - zero init sync object * @@ -49,6 +54,7 @@ void amdgpu_sync_create(struct amdgpu_sync *sync) for (i = 0; i < AMDGPU_MAX_RINGS; ++i) sync->sync_to[i] = NULL; + hash_init(sync->fences); sync->last_vm_update = NULL; } @@ -62,6 +68,7 @@ void amdgpu_sync_create(struct amdgpu_sync *sync) int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, struct fence *f) { + struct amdgpu_sync_entry *e; struct amdgpu_fence *fence; struct amdgpu_fence *other; @@ -69,8 +76,27 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, return 0; fence = to_amdgpu_fence(f); - if (!fence || fence->ring->adev != adev) - return fence_wait(f, true); + if (!fence || fence->ring->adev != adev) { + hash_for_each_possible(sync->fences, e, node, f->context) { + struct fence *new; + if (unlikely(e->fence->context != f->context)) + continue; + new = fence_get(fence_later(e->fence, f)); + if (new) { + fence_put(e->fence); + e->fence = new; + } + return 0; + } + + e = kmalloc(sizeof(struct amdgpu_sync_entry), GFP_KERNEL); + if (!e) + return -ENOMEM; + + hash_add(sync->fences, &e->node, f->context); + e->fence = fence_get(f); + return 0; + } other = sync->sync_to[fence->ring->idx]; sync->sync_to[fence->ring->idx] = amdgpu_fence_ref( @@ -147,6 +173,24 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, return r; } +int amdgpu_sync_wait(struct amdgpu_sync *sync) +{ + struct amdgpu_sync_entry *e; + struct hlist_node *tmp; + int i, r; + + hash_for_each_safe(sync->fences, i, tmp, e, node) { + r = fence_wait(e->fence, false); + if (r) + return r; + + hash_del(&e->node); + fence_put(e->fence); + kfree(e); + } + return 0; +} + /** * amdgpu_sync_rings - sync ring to all registered fences * @@ -236,8 +280,16 @@ void amdgpu_sync_free(struct amdgpu_device *adev, struct amdgpu_sync *sync, struct fence *fence) { + struct amdgpu_sync_entry *e; + struct hlist_node *tmp; unsigned i; + hash_for_each_safe(sync->fences, i, tmp, e, node) { + hash_del(&e->node); + fence_put(e->fence); + kfree(e); + } + for (i = 0; i < AMDGPU_NUM_SYNCS; ++i) amdgpu_semaphore_free(adev, &sync->semaphores[i], fence); -- cgit v1.2.3 From 3a185a33a2d291883bd1d0691d22901c345b12c9 Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 20 Aug 2015 17:35:34 +0200 Subject: drm/amdgpu: remove the context from amdgpu_job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 -- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 4 +--- drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 7 ------- 3 files changed, 1 insertion(+), 12 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu.h') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 3c5487257ef0..0088af1efd54 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1046,7 +1046,6 @@ void amdgpu_ctx_fini(struct amdgpu_ctx *ctx); struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id); int amdgpu_ctx_put(struct amdgpu_ctx *ctx); -struct amdgpu_ctx *amdgpu_ctx_get_ref(struct amdgpu_ctx *ctx); uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, struct fence *fence); @@ -1267,7 +1266,6 @@ struct amdgpu_cs_parser { struct amdgpu_job { struct amd_sched_job base; struct amdgpu_device *adev; - struct amdgpu_ctx *ctx; struct drm_file *owner; struct amdgpu_ib *ibs; uint32_t num_ibs; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 5d335522625e..780c0113e8f8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -775,7 +775,6 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev, static int amdgpu_cs_free_job(struct amdgpu_job *sched_job) { int i; - amdgpu_ctx_put(sched_job->ctx); if (sched_job->ibs) for (i = 0; i < sched_job->num_ibs; i++) amdgpu_ib_free(sched_job->adev, &sched_job->ibs[i]); @@ -849,7 +848,6 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) job->ibs = parser->ibs; job->num_ibs = parser->num_ibs; job->owner = parser->filp; - job->ctx = amdgpu_ctx_get_ref(parser->ctx); mutex_init(&job->job_lock); if (job->ibs[job->num_ibs - 1].user) { memcpy(&job->uf, &parser->uf, @@ -867,7 +865,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) goto out; } cs->out.handle = - amdgpu_ctx_add_fence(job->ctx, ring, + amdgpu_ctx_add_fence(parser->ctx, ring, &job->base.s_fence->base); parser->ibs[parser->num_ibs - 1].sequence = cs->out.handle; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index f024effa60f9..20cbc4eb5a6f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -219,13 +219,6 @@ struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id) return ctx; } -struct amdgpu_ctx *amdgpu_ctx_get_ref(struct amdgpu_ctx *ctx) -{ - if (ctx) - kref_get(&ctx->refcount); - return ctx; -} - int amdgpu_ctx_put(struct amdgpu_ctx *ctx) { if (ctx == NULL) -- cgit v1.2.3 From 87e0a87dabdbe9e1698ef85c620c4110b04992a0 Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 20 Aug 2015 17:36:28 +0200 Subject: drm/amdgpu: remove amdgpu_bo_list_clone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not used any more. Signed-off-by: Christian König Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 -- drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c | 33 ----------------------------- 2 files changed, 35 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu.h') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 0088af1efd54..a2d5cf7df56f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1083,8 +1083,6 @@ struct amdgpu_bo_list { struct amdgpu_bo_list_entry *array; }; -struct amdgpu_bo_list * -amdgpu_bo_list_clone(struct amdgpu_bo_list *list); struct amdgpu_bo_list * amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id); void amdgpu_bo_list_put(struct amdgpu_bo_list *list); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c index 7eed523bf28f..f82a2dd83874 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c @@ -62,39 +62,6 @@ static int amdgpu_bo_list_create(struct amdgpu_fpriv *fpriv, return 0; } -struct amdgpu_bo_list * -amdgpu_bo_list_clone(struct amdgpu_bo_list *list) -{ - struct amdgpu_bo_list *result; - unsigned i; - - result = kmalloc(sizeof(struct amdgpu_bo_list), GFP_KERNEL); - if (!result) - return NULL; - - result->array = drm_calloc_large(list->num_entries, - sizeof(struct amdgpu_bo_list_entry)); - if (!result->array) { - kfree(result); - return NULL; - } - - mutex_init(&result->lock); - result->gds_obj = list->gds_obj; - result->gws_obj = list->gws_obj; - result->oa_obj = list->oa_obj; - result->has_userptr = list->has_userptr; - result->num_entries = list->num_entries; - - memcpy(result->array, list->array, list->num_entries * - sizeof(struct amdgpu_bo_list_entry)); - - for (i = 0; i < result->num_entries; ++i) - amdgpu_bo_ref(result->array[i].robj); - - return result; -} - static void amdgpu_bo_list_destroy(struct amdgpu_fpriv *fpriv, int id) { struct amdgpu_bo_list *list; -- cgit v1.2.3 From 84f76ea6b03a766931e5d6d650af5ab980c6c4f4 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Mon, 24 Aug 2015 12:47:36 +0800 Subject: drm/amdgpu: add owner for sched fence Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 - drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 4 ++-- drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 3 ++- drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 4 +++- drivers/gpu/drm/amd/scheduler/sched_fence.c | 4 ++-- 6 files changed, 10 insertions(+), 8 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu.h') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index a2d5cf7df56f..19a8cbead48b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1264,7 +1264,6 @@ struct amdgpu_cs_parser { struct amdgpu_job { struct amd_sched_job base; struct amdgpu_device *adev; - struct drm_file *owner; struct amdgpu_ib *ibs; uint32_t num_ibs; struct mutex job_lock; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 82e14321e9c7..6a206f15635f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -847,7 +847,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) job->adev = parser->adev; job->ibs = parser->ibs; job->num_ibs = parser->num_ibs; - job->owner = parser->filp; + job->base.owner = parser->filp; mutex_init(&job->job_lock); if (job->ibs[job->num_ibs - 1].user) { memcpy(&job->uf, &parser->uf, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 1aa72edbce9a..4f5c0874ad2a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -44,7 +44,7 @@ static struct fence *amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, r = amdgpu_ib_schedule(sched_job->adev, sched_job->num_ibs, sched_job->ibs, - sched_job->owner); + sched_job->base.owner); if (r) goto err; fence = amdgpu_fence_ref(sched_job->ibs[sched_job->num_ibs - 1].fence); @@ -101,7 +101,7 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, job->adev = adev; job->ibs = ibs; job->num_ibs = num_ibs; - job->owner = owner; + job->base.owner = owner; mutex_init(&job->job_lock); job->free_job = free_job; mutex_lock(&job->job_lock); diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index d747f82808a7..24593fd29150 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -232,7 +232,8 @@ static bool amd_sched_entity_in(struct amd_sched_job *job) int amd_sched_entity_push_job(struct amd_sched_job *sched_job) { struct amd_sched_entity *entity = sched_job->s_entity; - struct amd_sched_fence *fence = amd_sched_fence_create(entity); + struct amd_sched_fence *fence = amd_sched_fence_create( + entity, sched_job->owner); int r; if (!fence) diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index 62655f4cf11d..0927864af1f8 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -65,6 +65,7 @@ struct amd_sched_fence { struct fence base; struct amd_gpu_scheduler *scheduler; spinlock_t lock; + void *owner; }; struct amd_sched_job { @@ -72,6 +73,7 @@ struct amd_sched_job { struct amd_gpu_scheduler *sched; struct amd_sched_entity *s_entity; struct amd_sched_fence *s_fence; + void *owner; }; extern const struct fence_ops amd_sched_fence_ops; @@ -126,7 +128,7 @@ int amd_sched_entity_fini(struct amd_gpu_scheduler *sched, int amd_sched_entity_push_job(struct amd_sched_job *sched_job); struct amd_sched_fence *amd_sched_fence_create( - struct amd_sched_entity *s_entity); + struct amd_sched_entity *s_entity, void *owner); void amd_sched_fence_signal(struct amd_sched_fence *fence); diff --git a/drivers/gpu/drm/amd/scheduler/sched_fence.c b/drivers/gpu/drm/amd/scheduler/sched_fence.c index 7f0f57819b62..e62c37920e11 100644 --- a/drivers/gpu/drm/amd/scheduler/sched_fence.c +++ b/drivers/gpu/drm/amd/scheduler/sched_fence.c @@ -27,7 +27,7 @@ #include #include "gpu_scheduler.h" -struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *s_entity) +struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *s_entity, void *owner) { struct amd_sched_fence *fence = NULL; unsigned seq; @@ -35,7 +35,7 @@ struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *s_entity fence = kzalloc(sizeof(struct amd_sched_fence), GFP_KERNEL); if (fence == NULL) return NULL; - + fence->owner = owner; fence->scheduler = s_entity->scheduler; spin_lock_init(&fence->lock); -- cgit v1.2.3 From 3c62338c26bf2677c8285b406cd769b92ee0dc10 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Thu, 20 Aug 2015 18:33:59 +0800 Subject: drm/amdgpu: fix last_vm_update fence is not effetive for sched fence Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 6 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c | 45 +++++++++++++++++++++++++------ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 24 ++++++++++++----- 4 files changed, 58 insertions(+), 18 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu.h') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 19a8cbead48b..c9160430b5ac 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -705,7 +705,7 @@ struct amdgpu_sync { struct amdgpu_semaphore *semaphores[AMDGPU_NUM_SYNCS]; struct amdgpu_fence *sync_to[AMDGPU_MAX_RINGS]; DECLARE_HASHTABLE(fences, 4); - struct amdgpu_fence *last_vm_update; + struct fence *last_vm_update; }; void amdgpu_sync_create(struct amdgpu_sync *sync); @@ -963,7 +963,7 @@ struct amdgpu_vm_id { unsigned id; uint64_t pd_gpu_addr; /* last flushed PD/PT update */ - struct amdgpu_fence *flushed_updates; + struct fence *flushed_updates; /* last use of vmid */ struct amdgpu_fence *last_id_use; }; @@ -2349,7 +2349,7 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, struct amdgpu_sync *sync); void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_vm *vm, - struct amdgpu_fence *updates); + struct fence *updates); void amdgpu_vm_fence(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_fence *fence); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 4f5c0874ad2a..757058d539f4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -119,5 +119,6 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, return r; *f = fence_get(&ibs[num_ibs - 1].fence->base); } + return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c index febbf37b1412..4fffb2539331 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c @@ -58,6 +58,29 @@ void amdgpu_sync_create(struct amdgpu_sync *sync) sync->last_vm_update = NULL; } +static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, struct fence *f) +{ + struct amdgpu_fence *a_fence = to_amdgpu_fence(f); + struct amd_sched_fence *s_fence = to_amd_sched_fence(f); + + if (a_fence) + return a_fence->ring->adev == adev; + if (s_fence) + return (struct amdgpu_device *)s_fence->scheduler->priv == adev; + return false; +} + +static bool amdgpu_sync_test_owner(struct fence *f, void *owner) +{ + struct amdgpu_fence *a_fence = to_amdgpu_fence(f); + struct amd_sched_fence *s_fence = to_amd_sched_fence(f); + if (s_fence) + return s_fence->owner == owner; + if (a_fence) + return a_fence->owner == owner; + return false; +} + /** * amdgpu_sync_fence - remember to sync to this fence * @@ -71,10 +94,23 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, struct amdgpu_sync_entry *e; struct amdgpu_fence *fence; struct amdgpu_fence *other; + struct fence *tmp, *later; if (!f) return 0; + if (amdgpu_sync_same_dev(adev, f) && + amdgpu_sync_test_owner(f, AMDGPU_FENCE_OWNER_VM)) { + if (sync->last_vm_update) { + tmp = sync->last_vm_update; + BUG_ON(f->context != tmp->context); + later = (f->seqno - tmp->seqno <= INT_MAX) ? f : tmp; + sync->last_vm_update = fence_get(later); + fence_put(tmp); + } else + sync->last_vm_update = fence_get(f); + } + fence = to_amdgpu_fence(f); if (!fence || fence->ring->adev != adev) { hash_for_each_possible(sync->fences, e, node, f->context) { @@ -103,13 +139,6 @@ int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, amdgpu_fence_later(fence, other)); amdgpu_fence_unref(&other); - if (fence->owner == AMDGPU_FENCE_OWNER_VM) { - other = sync->last_vm_update; - sync->last_vm_update = amdgpu_fence_ref( - amdgpu_fence_later(fence, other)); - amdgpu_fence_unref(&other); - } - return 0; } @@ -296,5 +325,5 @@ void amdgpu_sync_free(struct amdgpu_device *adev, for (i = 0; i < AMDGPU_MAX_RINGS; ++i) amdgpu_fence_unref(&sync->sync_to[i]); - amdgpu_fence_unref(&sync->last_vm_update); + fence_put(sync->last_vm_update); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 5b99214d0ba6..677266b90c74 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -200,19 +200,29 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, */ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_vm *vm, - struct amdgpu_fence *updates) + struct fence *updates) { uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory); struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; - struct amdgpu_fence *flushed_updates = vm_id->flushed_updates; + struct fence *flushed_updates = vm_id->flushed_updates; + bool is_earlier = false; + + if (flushed_updates && updates) { + BUG_ON(flushed_updates->context != updates->context); + is_earlier = (updates->seqno - flushed_updates->seqno <= + INT_MAX) ? true : false; + } if (pd_addr != vm_id->pd_gpu_addr || !flushed_updates || - (updates && amdgpu_fence_is_earlier(flushed_updates, updates))) { + is_earlier) { trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id->id); - vm_id->flushed_updates = amdgpu_fence_ref( - amdgpu_fence_later(flushed_updates, updates)); - amdgpu_fence_unref(&flushed_updates); + if (is_earlier) { + vm_id->flushed_updates = fence_get(updates); + fence_put(flushed_updates); + } + if (!flushed_updates) + vm_id->flushed_updates = fence_get(updates); vm_id->pd_gpu_addr = pd_addr; amdgpu_ring_emit_vm_flush(ring, vm_id->id, vm_id->pd_gpu_addr); } @@ -1347,7 +1357,7 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) fence_put(vm->page_directory_fence); for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { - amdgpu_fence_unref(&vm->ids[i].flushed_updates); + fence_put(vm->ids[i].flushed_updates); amdgpu_fence_unref(&vm->ids[i].last_id_use); } -- cgit v1.2.3 From c7ae72c01be10f539f385f624713f8ba0aa11a8f Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Tue, 25 Aug 2015 17:23:45 +0800 Subject: drm/amdgpu: use IB for copy buffer of eviction This aids handling buffers moves with the scheduler. Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 8 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c | 8 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_test.c | 12 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 77 ++++++++++++++------------- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 3 +- drivers/gpu/drm/amd/amdgpu/cik_sdma.c | 16 +++--- drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c | 18 +++---- drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 18 +++---- 8 files changed, 81 insertions(+), 79 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu.h') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index c9160430b5ac..aa2dcf578dd6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -247,7 +247,7 @@ struct amdgpu_buffer_funcs { unsigned copy_num_dw; /* used for buffer migration */ - void (*emit_copy_buffer)(struct amdgpu_ring *ring, + void (*emit_copy_buffer)(struct amdgpu_ib *ib, /* src addr in bytes */ uint64_t src_offset, /* dst addr in bytes */ @@ -518,7 +518,7 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t dst_offset, uint32_t byte_count, struct reservation_object *resv, - struct amdgpu_fence **fence); + struct fence **fence); int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma); struct amdgpu_bo_list_entry { @@ -2247,7 +2247,7 @@ static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v) #define amdgpu_display_add_connector(adev, ci, sd, ct, ib, coi, h, r) (adev)->mode_info.funcs->add_connector((adev), (ci), (sd), (ct), (ib), (coi), (h), (r)) #define amdgpu_display_stop_mc_access(adev, s) (adev)->mode_info.funcs->stop_mc_access((adev), (s)) #define amdgpu_display_resume_mc_access(adev, s) (adev)->mode_info.funcs->resume_mc_access((adev), (s)) -#define amdgpu_emit_copy_buffer(adev, r, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((r), (s), (d), (b)) +#define amdgpu_emit_copy_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((ib), (s), (d), (b)) #define amdgpu_emit_fill_buffer(adev, r, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((r), (s), (d), (b)) #define amdgpu_dpm_get_temperature(adev) (adev)->pm.funcs->get_temperature((adev)) #define amdgpu_dpm_pre_set_power_state(adev) (adev)->pm.funcs->pre_set_power_state((adev)) @@ -2379,7 +2379,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, uint64_t addr); void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va); - +int amdgpu_vm_free_job(struct amdgpu_job *job); /* * functions used by amdgpu_encoder.c */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c index 759482e4300d..98d59ee640ce 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c @@ -33,7 +33,7 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size, { unsigned long start_jiffies; unsigned long end_jiffies; - struct amdgpu_fence *fence = NULL; + struct fence *fence = NULL; int i, r; start_jiffies = jiffies; @@ -42,17 +42,17 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size, r = amdgpu_copy_buffer(ring, saddr, daddr, size, NULL, &fence); if (r) goto exit_do_move; - r = fence_wait(&fence->base, false); + r = fence_wait(fence, false); if (r) goto exit_do_move; - amdgpu_fence_unref(&fence); + fence_put(fence); } end_jiffies = jiffies; r = jiffies_to_msecs(end_jiffies - start_jiffies); exit_do_move: if (fence) - amdgpu_fence_unref(&fence); + fence_put(fence); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c index 962dd5552137..f80b1a43be8a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c @@ -77,7 +77,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev) void *gtt_map, *vram_map; void **gtt_start, **gtt_end; void **vram_start, **vram_end; - struct amdgpu_fence *fence = NULL; + struct fence *fence = NULL; r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_GTT, 0, NULL, gtt_obj + i); @@ -116,13 +116,13 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev) goto out_lclean_unpin; } - r = fence_wait(&fence->base, false); + r = fence_wait(fence, false); if (r) { DRM_ERROR("Failed to wait for GTT->VRAM fence %d\n", i); goto out_lclean_unpin; } - amdgpu_fence_unref(&fence); + fence_put(fence); r = amdgpu_bo_kmap(vram_obj, &vram_map); if (r) { @@ -161,13 +161,13 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev) goto out_lclean_unpin; } - r = fence_wait(&fence->base, false); + r = fence_wait(fence, false); if (r) { DRM_ERROR("Failed to wait for VRAM->GTT fence %d\n", i); goto out_lclean_unpin; } - amdgpu_fence_unref(&fence); + fence_put(fence); r = amdgpu_bo_kmap(gtt_obj[i], >t_map); if (r) { @@ -214,7 +214,7 @@ out_lclean: amdgpu_bo_unref(>t_obj[i]); } if (fence) - amdgpu_fence_unref(&fence); + fence_put(fence); break; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 4cb81320b045..399143541d8a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -228,7 +228,7 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo, struct amdgpu_device *adev; struct amdgpu_ring *ring; uint64_t old_start, new_start; - struct amdgpu_fence *fence; + struct fence *fence; int r; adev = amdgpu_get_adev(bo->bdev); @@ -269,9 +269,9 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo, new_mem->num_pages * PAGE_SIZE, /* bytes */ bo->resv, &fence); /* FIXME: handle copy error */ - r = ttm_bo_move_accel_cleanup(bo, &fence->base, + r = ttm_bo_move_accel_cleanup(bo, fence, evict, no_wait_gpu, new_mem); - amdgpu_fence_unref(&fence); + fence_put(fence); return r; } @@ -987,52 +987,48 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t dst_offset, uint32_t byte_count, struct reservation_object *resv, - struct amdgpu_fence **fence) + struct fence **fence) { struct amdgpu_device *adev = ring->adev; - struct amdgpu_sync sync; uint32_t max_bytes; unsigned num_loops, num_dw; + struct amdgpu_ib *ib; unsigned i; int r; - /* sync other rings */ - amdgpu_sync_create(&sync); - if (resv) { - r = amdgpu_sync_resv(adev, &sync, resv, false); - if (r) { - DRM_ERROR("sync failed (%d).\n", r); - amdgpu_sync_free(adev, &sync, NULL); - return r; - } - } - max_bytes = adev->mman.buffer_funcs->copy_max_bytes; num_loops = DIV_ROUND_UP(byte_count, max_bytes); num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw; - /* for fence and sync */ - num_dw += 64 + AMDGPU_NUM_SYNCS * 8; + /* for IB padding */ + while (num_dw & 0x7) + num_dw++; + + ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); + if (!ib) + return -ENOMEM; - r = amdgpu_sync_wait(&sync); + r = amdgpu_ib_get(ring, NULL, num_dw * 4, ib); if (r) { - DRM_ERROR("sync wait failed (%d).\n", r); - amdgpu_sync_free(adev, &sync, NULL); + kfree(ib); return r; } - r = amdgpu_ring_lock(ring, num_dw); - if (r) { - DRM_ERROR("ring lock failed (%d).\n", r); - amdgpu_sync_free(adev, &sync, NULL); - return r; + ib->length_dw = 0; + + if (resv) { + r = amdgpu_sync_resv(adev, &ib->sync, resv, + AMDGPU_FENCE_OWNER_UNDEFINED); + if (r) { + DRM_ERROR("sync failed (%d).\n", r); + goto error_free; + } } - amdgpu_sync_rings(&sync, ring); for (i = 0; i < num_loops; i++) { uint32_t cur_size_in_bytes = min(byte_count, max_bytes); - amdgpu_emit_copy_buffer(adev, ring, src_offset, dst_offset, + amdgpu_emit_copy_buffer(adev, ib, src_offset, dst_offset, cur_size_in_bytes); src_offset += cur_size_in_bytes; @@ -1040,17 +1036,24 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, byte_count -= cur_size_in_bytes; } - r = amdgpu_fence_emit(ring, AMDGPU_FENCE_OWNER_MOVE, fence); - if (r) { - amdgpu_ring_unlock_undo(ring); - amdgpu_sync_free(adev, &sync, NULL); - return r; - } - - amdgpu_ring_unlock_commit(ring); - amdgpu_sync_free(adev, &sync, &(*fence)->base); + amdgpu_vm_pad_ib(adev, ib); + WARN_ON(ib->length_dw > num_dw); + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, + &amdgpu_vm_free_job, + AMDGPU_FENCE_OWNER_MOVE, + fence); + if (r) + goto error_free; + if (!amdgpu_enable_scheduler) { + amdgpu_ib_free(adev, ib); + kfree(ib); + } return 0; +error_free: + amdgpu_ib_free(adev, ib); + kfree(ib); + return r; } #if defined(CONFIG_DEBUG_FS) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 677266b90c74..83b7ce6f5f72 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -316,8 +316,7 @@ static void amdgpu_vm_update_pages(struct amdgpu_device *adev, } } -static int amdgpu_vm_free_job( - struct amdgpu_job *sched_job) +int amdgpu_vm_free_job(struct amdgpu_job *sched_job) { int i; for (i = 0; i < sched_job->num_ibs; i++) diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index af526557ac66..3920c1e346f8 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -1339,18 +1339,18 @@ static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev) * Used by the amdgpu ttm implementation to move pages if * registered as the asic copy callback. */ -static void cik_sdma_emit_copy_buffer(struct amdgpu_ring *ring, +static void cik_sdma_emit_copy_buffer(struct amdgpu_ib *ib, uint64_t src_offset, uint64_t dst_offset, uint32_t byte_count) { - amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0)); - amdgpu_ring_write(ring, byte_count); - amdgpu_ring_write(ring, 0); /* src/dst endian swap */ - amdgpu_ring_write(ring, lower_32_bits(src_offset)); - amdgpu_ring_write(ring, upper_32_bits(src_offset)); - amdgpu_ring_write(ring, lower_32_bits(dst_offset)); - amdgpu_ring_write(ring, upper_32_bits(dst_offset)); + ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0); + ib->ptr[ib->length_dw++] = byte_count; + ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ + ib->ptr[ib->length_dw++] = lower_32_bits(src_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(src_offset); + ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 2b0e89e05b1d..715e02d3bfba 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -1350,19 +1350,19 @@ static void sdma_v2_4_set_irq_funcs(struct amdgpu_device *adev) * Used by the amdgpu ttm implementation to move pages if * registered as the asic copy callback. */ -static void sdma_v2_4_emit_copy_buffer(struct amdgpu_ring *ring, +static void sdma_v2_4_emit_copy_buffer(struct amdgpu_ib *ib, uint64_t src_offset, uint64_t dst_offset, uint32_t byte_count) { - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR)); - amdgpu_ring_write(ring, byte_count); - amdgpu_ring_write(ring, 0); /* src/dst endian swap */ - amdgpu_ring_write(ring, lower_32_bits(src_offset)); - amdgpu_ring_write(ring, upper_32_bits(src_offset)); - amdgpu_ring_write(ring, lower_32_bits(dst_offset)); - amdgpu_ring_write(ring, upper_32_bits(dst_offset)); + ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | + SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR); + ib->ptr[ib->length_dw++] = byte_count; + ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ + ib->ptr[ib->length_dw++] = lower_32_bits(src_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(src_offset); + ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index 6f1df03ebc70..67128c8e78b8 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -1474,19 +1474,19 @@ static void sdma_v3_0_set_irq_funcs(struct amdgpu_device *adev) * Used by the amdgpu ttm implementation to move pages if * registered as the asic copy callback. */ -static void sdma_v3_0_emit_copy_buffer(struct amdgpu_ring *ring, +static void sdma_v3_0_emit_copy_buffer(struct amdgpu_ib *ib, uint64_t src_offset, uint64_t dst_offset, uint32_t byte_count) { - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR)); - amdgpu_ring_write(ring, byte_count); - amdgpu_ring_write(ring, 0); /* src/dst endian swap */ - amdgpu_ring_write(ring, lower_32_bits(src_offset)); - amdgpu_ring_write(ring, upper_32_bits(src_offset)); - amdgpu_ring_write(ring, lower_32_bits(dst_offset)); - amdgpu_ring_write(ring, upper_32_bits(dst_offset)); + ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | + SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR); + ib->ptr[ib->length_dw++] = byte_count; + ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ + ib->ptr[ib->length_dw++] = lower_32_bits(src_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(src_offset); + ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); } /** -- cgit v1.2.3