diff options
-rw-r--r-- | drivers/gpu/drm/i915/i915_debugfs.c | 2 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/i915_drv.h | 4 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/i915_gem.c | 2 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/i915_gem_request.c | 19 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/i915_gem_request.h | 32 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/i915_gpu_error.c | 2 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/i915_guc_submission.c | 4 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/i915_trace.h | 8 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/intel_breadcrumbs.c | 8 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/intel_lrc.c | 4 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/intel_ringbuffer.c | 14 |
11 files changed, 66 insertions, 33 deletions
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 3a0ea5eace37..90bc4a89e0d5 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -637,7 +637,7 @@ static void print_request(struct seq_file *m, rcu_read_lock(); task = pid ? pid_task(pid, PIDTYPE_PID) : NULL; seq_printf(m, "%s%x [%x:%x] @ %d: %s [%d]\n", prefix, - rq->fence.seqno, rq->ctx->hw_id, rq->fence.seqno, + rq->global_seqno, rq->ctx->hw_id, rq->fence.seqno, jiffies_to_msecs(jiffies - rq->emitted_jiffies), task ? task->comm : "<unknown>", task ? task->pid : -1); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index f0f68f64d09c..217674bb1495 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -4050,7 +4050,7 @@ __i915_request_irq_complete(struct drm_i915_gem_request *req) /* Before we do the heavier coherent read of the seqno, * check the value (hopefully) in the CPU cacheline. */ - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; /* Ensure our read of the seqno is coherent so that we @@ -4101,7 +4101,7 @@ __i915_request_irq_complete(struct drm_i915_gem_request *req) wake_up_process(tsk); rcu_read_unlock(); - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; } diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 5ea46a7d991f..f4cfb88bd804 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -2615,7 +2615,7 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine) return; DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n", - engine->name, request->fence.seqno); + engine->name, request->global_seqno); /* Setup the CS to resume from the breadcrumb of the hung request */ engine->reset_hw(engine, request); diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c index 03ae85a1eefb..311cf3fac2e0 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.c +++ b/drivers/gpu/drm/i915/i915_gem_request.c @@ -376,7 +376,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, * of being read by __i915_gem_active_get_rcu(). As such, * we have to be very careful when overwriting the contents. During * the RCU lookup, we change chase the request->engine pointer, - * read the request->fence.seqno and increment the reference count. + * read the request->global_seqno and increment the reference count. * * The reference count is incremented atomically. If it is zero, * the lookup knows the request is unallocated and complete. Otherwise, @@ -418,6 +418,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, INIT_LIST_HEAD(&req->active_list); req->i915 = dev_priv; req->engine = engine; + req->global_seqno = seqno; req->ctx = i915_gem_context_get(ctx); /* No zalloc, must clear what we need by hand */ @@ -475,8 +476,15 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to, return ret < 0 ? ret : 0; } + if (!from->global_seqno) { + ret = i915_sw_fence_await_dma_fence(&to->submit, + &from->fence, 0, + GFP_KERNEL); + return ret < 0 ? ret : 0; + } + idx = intel_engine_sync_index(from->engine, to->engine); - if (from->fence.seqno <= from->engine->semaphore.sync_seqno[idx]) + if (from->global_seqno <= from->engine->semaphore.sync_seqno[idx]) return 0; trace_i915_gem_ring_sync_to(to, from); @@ -494,7 +502,7 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to, return ret; } - from->engine->semaphore.sync_seqno[idx] = from->fence.seqno; + from->engine->semaphore.sync_seqno[idx] = from->global_seqno; return 0; } @@ -774,7 +782,7 @@ bool __i915_spin_request(const struct drm_i915_gem_request *req, timeout_us += local_clock_us(&cpu); do { - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; if (signal_pending_state(state, current)) @@ -883,6 +891,7 @@ long i915_wait_request(struct drm_i915_gem_request *req, GEM_BUG_ON(!i915_sw_fence_done(&req->submit)); } + GEM_BUG_ON(!req->global_seqno); /* Optimistic short spin before touching IRQs */ if (i915_spin_request(req, state, 5)) @@ -892,7 +901,7 @@ long i915_wait_request(struct drm_i915_gem_request *req, if (flags & I915_WAIT_LOCKED) add_wait_queue(&req->i915->gpu_error.wait_queue, &reset); - intel_wait_init(&wait, req->fence.seqno); + intel_wait_init(&wait, req->global_seqno); if (intel_engine_add_wait(req->engine, &wait)) /* In order to check that we haven't missed the interrupt * as we enabled it, we need to kick ourselves to do a diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h index 4ac30ae93e49..75f8360b3421 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.h +++ b/drivers/gpu/drm/i915/i915_gem_request.h @@ -87,6 +87,8 @@ struct drm_i915_gem_request { struct i915_sw_fence submit; wait_queue_t submitq; + u32 global_seqno; + /** GEM sequence number associated with the previous request, * when the HWS breadcrumb is equal to this the GPU is processing * this request. @@ -163,7 +165,7 @@ void i915_gem_request_retire_upto(struct drm_i915_gem_request *req); static inline u32 i915_gem_request_get_seqno(struct drm_i915_gem_request *req) { - return req ? req->fence.seqno : 0; + return req ? req->global_seqno : 0; } static inline struct intel_engine_cs * @@ -248,17 +250,37 @@ static inline bool i915_seqno_passed(u32 seq1, u32 seq2) } static inline bool -i915_gem_request_started(const struct drm_i915_gem_request *req) +__i915_gem_request_started(const struct drm_i915_gem_request *req) { + GEM_BUG_ON(!req->global_seqno); return i915_seqno_passed(intel_engine_get_seqno(req->engine), req->previous_seqno); } static inline bool -i915_gem_request_completed(const struct drm_i915_gem_request *req) +i915_gem_request_started(const struct drm_i915_gem_request *req) { + if (!req->global_seqno) + return false; + + return __i915_gem_request_started(req); +} + +static inline bool +__i915_gem_request_completed(const struct drm_i915_gem_request *req) +{ + GEM_BUG_ON(!req->global_seqno); return i915_seqno_passed(intel_engine_get_seqno(req->engine), - req->fence.seqno); + req->global_seqno); +} + +static inline bool +i915_gem_request_completed(const struct drm_i915_gem_request *req) +{ + if (!req->global_seqno) + return false; + + return __i915_gem_request_completed(req); } bool __i915_spin_request(const struct drm_i915_gem_request *request, @@ -266,7 +288,7 @@ bool __i915_spin_request(const struct drm_i915_gem_request *request, static inline bool i915_spin_request(const struct drm_i915_gem_request *request, int state, unsigned long timeout_us) { - return (i915_gem_request_started(request) && + return (__i915_gem_request_started(request) && __i915_spin_request(request, state, timeout_us)); } diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 12fea57d41fb..9aa197ca6210 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -1176,7 +1176,7 @@ static void record_request(struct drm_i915_gem_request *request, struct drm_i915_error_request *erq) { erq->context = request->ctx->hw_id; - erq->seqno = request->fence.seqno; + erq->seqno = request->global_seqno; erq->jiffies = request->emitted_jiffies; erq->head = request->head; erq->tail = request->tail; diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c index cca250e90845..857ef914cae7 100644 --- a/drivers/gpu/drm/i915/i915_guc_submission.c +++ b/drivers/gpu/drm/i915/i915_guc_submission.c @@ -554,7 +554,7 @@ static void guc_wq_item_append(struct i915_guc_client *gc, wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx, engine); wqi->ring_tail = tail << WQ_RING_TAIL_SHIFT; - wqi->fence_id = rq->fence.seqno; + wqi->fence_id = rq->global_seqno; kunmap_atomic(base); } @@ -655,7 +655,7 @@ static void i915_guc_submit(struct drm_i915_gem_request *rq) client->b_fail += 1; guc->submissions[engine_id] += 1; - guc->last_seqno[engine_id] = rq->fence.seqno; + guc->last_seqno[engine_id] = rq->global_seqno; spin_unlock(&client->wq_lock); } diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index 5c912c25f7d3..c5d210ebaa9a 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -466,7 +466,7 @@ TRACE_EVENT(i915_gem_ring_sync_to, __entry->dev = from->i915->drm.primary->index; __entry->sync_from = from->engine->id; __entry->sync_to = to->engine->id; - __entry->seqno = from->fence.seqno; + __entry->seqno = from->global_seqno; ), TP_printk("dev=%u, sync-from=%u, sync-to=%u, seqno=%u", @@ -489,7 +489,7 @@ TRACE_EVENT(i915_gem_ring_dispatch, TP_fast_assign( __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; - __entry->seqno = req->fence.seqno; + __entry->seqno = req->global_seqno; __entry->flags = flags; dma_fence_enable_sw_signaling(&req->fence); ), @@ -534,7 +534,7 @@ DECLARE_EVENT_CLASS(i915_gem_request, TP_fast_assign( __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; - __entry->seqno = req->fence.seqno; + __entry->seqno = req->global_seqno; ), TP_printk("dev=%u, ring=%u, seqno=%u", @@ -596,7 +596,7 @@ TRACE_EVENT(i915_gem_request_wait_begin, TP_fast_assign( __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; - __entry->seqno = req->fence.seqno; + __entry->seqno = req->global_seqno; __entry->blocking = mutex_is_locked(&req->i915->drm.struct_mutex); ), diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c index 56efcc507ea2..0d5def0d2dfe 100644 --- a/drivers/gpu/drm/i915/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c @@ -504,9 +504,11 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request) /* locked by dma_fence_enable_sw_signaling() */ assert_spin_locked(&request->lock); + if (!request->global_seqno) + return; request->signaling.wait.tsk = b->signaler; - request->signaling.wait.seqno = request->fence.seqno; + request->signaling.wait.seqno = request->global_seqno; i915_gem_request_get(request); spin_lock(&b->lock); @@ -530,8 +532,8 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request) p = &b->signals.rb_node; while (*p) { parent = *p; - if (i915_seqno_passed(request->fence.seqno, - to_signaler(parent)->fence.seqno)) { + if (i915_seqno_passed(request->global_seqno, + to_signaler(parent)->global_seqno)) { p = &parent->rb_right; first = false; } else { diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index cb30549dfd40..e0a9bf81774b 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -1584,7 +1584,7 @@ static int gen8_emit_request(struct drm_i915_gem_request *request) intel_hws_seqno_address(request->engine) | MI_FLUSH_DW_USE_GTT); intel_ring_emit(ring, 0); - intel_ring_emit(ring, request->fence.seqno); + intel_ring_emit(ring, request->global_seqno); intel_ring_emit(ring, MI_USER_INTERRUPT); intel_ring_emit(ring, MI_NOOP); return intel_logical_ring_advance(request); @@ -1613,7 +1613,7 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request) PIPE_CONTROL_QW_WRITE)); intel_ring_emit(ring, intel_hws_seqno_address(request->engine)); intel_ring_emit(ring, 0); - intel_ring_emit(ring, i915_gem_request_get_seqno(request)); + intel_ring_emit(ring, request->global_seqno); /* We're thrashing one dword of HWS. */ intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_USER_INTERRUPT); diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index aaa46d9ffbc1..76c6b70303fb 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -1238,7 +1238,7 @@ static int gen8_rcs_signal(struct drm_i915_gem_request *req) PIPE_CONTROL_CS_STALL); intel_ring_emit(ring, lower_32_bits(gtt_offset)); intel_ring_emit(ring, upper_32_bits(gtt_offset)); - intel_ring_emit(ring, req->fence.seqno); + intel_ring_emit(ring, req->global_seqno); intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_SEMAPHORE_SIGNAL | @@ -1274,7 +1274,7 @@ static int gen8_xcs_signal(struct drm_i915_gem_request *req) lower_32_bits(gtt_offset) | MI_FLUSH_DW_USE_GTT); intel_ring_emit(ring, upper_32_bits(gtt_offset)); - intel_ring_emit(ring, req->fence.seqno); + intel_ring_emit(ring, req->global_seqno); intel_ring_emit(ring, MI_SEMAPHORE_SIGNAL | MI_SEMAPHORE_TARGET(waiter->hw_id)); @@ -1308,7 +1308,7 @@ static int gen6_signal(struct drm_i915_gem_request *req) if (i915_mmio_reg_valid(mbox_reg)) { intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); intel_ring_emit_reg(ring, mbox_reg); - intel_ring_emit(ring, req->fence.seqno); + intel_ring_emit(ring, req->global_seqno); } } @@ -1339,7 +1339,7 @@ static int i9xx_emit_request(struct drm_i915_gem_request *req) intel_ring_emit(ring, MI_STORE_DWORD_INDEX); intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - intel_ring_emit(ring, req->fence.seqno); + intel_ring_emit(ring, req->global_seqno); intel_ring_emit(ring, MI_USER_INTERRUPT); intel_ring_advance(ring); @@ -1389,7 +1389,7 @@ static int gen8_render_emit_request(struct drm_i915_gem_request *req) PIPE_CONTROL_QW_WRITE)); intel_ring_emit(ring, intel_hws_seqno_address(engine)); intel_ring_emit(ring, 0); - intel_ring_emit(ring, i915_gem_request_get_seqno(req)); + intel_ring_emit(ring, req->global_seqno); /* We're thrashing one dword of HWS. */ intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_USER_INTERRUPT); @@ -1427,7 +1427,7 @@ gen8_ring_sync_to(struct drm_i915_gem_request *req, MI_SEMAPHORE_WAIT | MI_SEMAPHORE_GLOBAL_GTT | MI_SEMAPHORE_SAD_GTE_SDD); - intel_ring_emit(ring, signal->fence.seqno); + intel_ring_emit(ring, signal->global_seqno); intel_ring_emit(ring, lower_32_bits(offset)); intel_ring_emit(ring, upper_32_bits(offset)); intel_ring_advance(ring); @@ -1465,7 +1465,7 @@ gen6_ring_sync_to(struct drm_i915_gem_request *req, * seqno is >= the last seqno executed. However for hardware the * comparison is strictly greater than. */ - intel_ring_emit(ring, signal->fence.seqno - 1); + intel_ring_emit(ring, signal->global_seqno - 1); intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); |