From fdcc983e084d4e4c3810db84815474a419e918e7 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Mon, 21 Dec 2015 16:55:43 +0000 Subject: [PATCH] drm/i915: Move execlists interrupt based submission to a bottom-half [ 196.988204] clocksource: timekeeping watchdog: Marking clocksource 'tsc' as unstable because the skew is too large: [ 196.988512] clocksource: 'refined-jiffies' wd_now: ffff9b48 wd_last: ffff9acb mask: ffffffff [ 196.988559] clocksource: 'tsc' cs_now: 4fcfa84354 cs_last: 4f95425e98 mask: ffffffffffffffff [ 196.992115] clocksource: Switched to clocksource refined-jiffies Signed-off-by: Chris Wilson --- drivers/gpu/drm/i915/i915_debugfs.c | 5 +- drivers/gpu/drm/i915/i915_gem.c | 13 +-- drivers/gpu/drm/i915/i915_irq.c | 2 +- drivers/gpu/drm/i915/intel_lrc.c | 147 +++++++++++++++++++------------- drivers/gpu/drm/i915/intel_lrc.h | 1 - drivers/gpu/drm/i915/intel_ringbuffer.h | 1 + 6 files changed, 94 insertions(+), 75 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 137e6743ebbb..ac2c8dc0adcc 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -2076,7 +2076,6 @@ static int i915_execlists(struct seq_file *m, void *data) for_each_ring(ring, dev_priv, ring_id) { struct drm_i915_gem_request *head_req = NULL; int count = 0; - unsigned long flags; seq_printf(m, "%s\n", ring->name); @@ -2103,12 +2102,12 @@ static int i915_execlists(struct seq_file *m, void *data) i, status, ctx_id); } - spin_lock_irqsave(&ring->execlist_lock, flags); + spin_lock(&ring->execlist_lock); list_for_each(cursor, &ring->execlist_queue) count++; head_req = list_first_entry_or_null(&ring->execlist_queue, struct drm_i915_gem_request, execlist_link); - spin_unlock_irqrestore(&ring->execlist_lock, flags); + spin_unlock(&ring->execlist_lock); seq_printf(m, "\t%d requests in queue\n", count); if (head_req) { diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 730a6d2f5163..d1ce86dbe5be 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -2182,13 +2182,13 @@ static void i915_gem_reset_ring_cleanup(struct intel_engine_cs *engine) */ if (i915.enable_execlists) { - spin_lock_irq(&engine->execlist_lock); + spin_lock(&engine->execlist_lock); /* list_splice_tail_init checks for empty lists */ list_splice_tail_init(&engine->execlist_queue, &engine->execlist_retired_req_list); - spin_unlock_irq(&engine->execlist_lock); + spin_unlock(&engine->execlist_lock); intel_execlists_retire_requests(engine); } @@ -2282,15 +2282,8 @@ i915_gem_retire_requests(struct drm_device *dev) for_each_ring(ring, dev_priv, i) { i915_gem_retire_requests_ring(ring); idle &= list_empty(&ring->request_list); - if (i915.enable_execlists) { - unsigned long flags; - - spin_lock_irqsave(&ring->execlist_lock, flags); - idle &= list_empty(&ring->execlist_queue); - spin_unlock_irqrestore(&ring->execlist_lock, flags); - + if (i915.enable_execlists) intel_execlists_retire_requests(ring); - } } if (idle) diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 43f333851561..724478f005f9 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -1314,7 +1314,7 @@ gen8_cs_irq_handler(struct intel_engine_cs *ring, u32 iir, int test_shift) if (iir & (GT_RENDER_USER_INTERRUPT << test_shift)) notify_ring(ring); if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) - intel_lrc_irq_handler(ring); + wake_up_process(ring->execlists_submit); } static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv, diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index 690374379b7e..021afa6bbf0d 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -132,6 +132,8 @@ * */ +#include + #include #include #include "i915_drv.h" @@ -341,7 +343,7 @@ static void execlists_elsp_write(struct drm_i915_gem_request *rq0, rq0->elsp_submitted++; /* You must always write both descriptors in the order below. */ - spin_lock(&dev_priv->uncore.lock); + spin_lock_irq(&dev_priv->uncore.lock); intel_uncore_forcewake_get__locked(dev_priv, FORCEWAKE_ALL); I915_WRITE_FW(RING_ELSP(engine), upper_32_bits(desc[1])); I915_WRITE_FW(RING_ELSP(engine), lower_32_bits(desc[1])); @@ -353,7 +355,7 @@ static void execlists_elsp_write(struct drm_i915_gem_request *rq0, /* ELSP is a wo register, use another nearby reg for posting */ POSTING_READ_FW(RING_EXECLIST_STATUS_LO(engine)); intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL); - spin_unlock(&dev_priv->uncore.lock); + spin_unlock_irq(&dev_priv->uncore.lock); } static int execlists_update_context(struct drm_i915_gem_request *rq) @@ -492,73 +494,84 @@ static bool execlists_check_remove_request(struct intel_engine_cs *ring, return false; } -/** - * intel_lrc_irq_handler() - handle Context Switch interrupts - * @ring: Engine Command Streamer to handle. - * - * Check the unread Context Status Buffers and manage the submission of new - * contexts to the ELSP accordingly. - */ -void intel_lrc_irq_handler(struct intel_engine_cs *ring) +static void set_rtpriority(void) { - struct drm_i915_private *dev_priv = ring->dev->dev_private; - u32 status_pointer; - u8 read_pointer; - u8 write_pointer; - u32 status = 0; - u32 status_id; - u32 submit_contexts = 0; + struct sched_param param = { .sched_priority = 1 }; + sched_setscheduler_nocheck(current, SCHED_FIFO, ¶m); +} + +static int intel_execlists_submit(void *arg) +{ + struct intel_engine_cs *ring = arg; + struct drm_i915_private *dev_priv = ring->i915; - status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(ring)); + set_rtpriority(); - read_pointer = ring->next_context_status_buffer; - write_pointer = status_pointer & GEN8_CSB_PTR_MASK; - if (read_pointer > write_pointer) - write_pointer += GEN8_CSB_ENTRIES; + do { + u32 status; + u32 status_id; + u32 submit_contexts; + u8 head, tail; - spin_lock(&ring->execlist_lock); + set_current_state(TASK_INTERRUPTIBLE); + head = ring->next_context_status_buffer; + tail = I915_READ(RING_CONTEXT_STATUS_PTR(ring)) & GEN8_CSB_PTR_MASK; - while (read_pointer < write_pointer) { - read_pointer++; - status = I915_READ(RING_CONTEXT_STATUS_BUF_LO(ring, read_pointer % GEN8_CSB_ENTRIES)); - status_id = I915_READ(RING_CONTEXT_STATUS_BUF_HI(ring, read_pointer % GEN8_CSB_ENTRIES)); + if (head == tail) { + if (kthread_should_stop()) + return 0; - if (status & GEN8_CTX_STATUS_IDLE_ACTIVE) + schedule(); continue; - - if (status & GEN8_CTX_STATUS_PREEMPTED) { - if (status & GEN8_CTX_STATUS_LITE_RESTORE) { - if (execlists_check_remove_request(ring, status_id)) - WARN(1, "Lite Restored request removed from queue\n"); - } else - WARN(1, "Preemption without Lite Restore\n"); } - if ((status & GEN8_CTX_STATUS_ACTIVE_IDLE) || - (status & GEN8_CTX_STATUS_ELEMENT_SWITCH)) { - if (execlists_check_remove_request(ring, status_id)) - submit_contexts++; + if (head > tail) + tail += GEN8_CSB_ENTRIES; + + status = 0; + submit_contexts = 0; + + spin_lock(&ring->execlist_lock); + + while (head++ < tail) { + status = I915_READ(RING_CONTEXT_STATUS_BUF_LO(ring, head % GEN8_CSB_ENTRIES)); + status_id = I915_READ(RING_CONTEXT_STATUS_BUF_HI(ring, head % GEN8_CSB_ENTRIES)); + + if (status & GEN8_CTX_STATUS_IDLE_ACTIVE) + continue; + + if (status & GEN8_CTX_STATUS_PREEMPTED) { + if (status & GEN8_CTX_STATUS_LITE_RESTORE) { + if (execlists_check_remove_request(ring, status_id)) + WARN(1, "Lite Restored request removed from queue\n"); + } else + WARN(1, "Preemption without Lite Restore\n"); + } + + if ((status & GEN8_CTX_STATUS_ACTIVE_IDLE) || + (status & GEN8_CTX_STATUS_ELEMENT_SWITCH)) { + if (execlists_check_remove_request(ring, status_id)) + submit_contexts++; + } } - } - if (disable_lite_restore_wa(ring)) { - /* Prevent a ctx to preempt itself */ - if ((status & GEN8_CTX_STATUS_ACTIVE_IDLE) && - (submit_contexts != 0)) + if (disable_lite_restore_wa(ring)) { + /* Prevent a ctx to preempt itself */ + if ((status & GEN8_CTX_STATUS_ACTIVE_IDLE) && + (submit_contexts != 0)) + execlists_context_unqueue(ring); + } else if (submit_contexts != 0) { execlists_context_unqueue(ring); - } else if (submit_contexts != 0) { - execlists_context_unqueue(ring); - } - - spin_unlock(&ring->execlist_lock); + } - WARN(submit_contexts > 2, "More than two context complete events?\n"); - ring->next_context_status_buffer = write_pointer % GEN8_CSB_ENTRIES; + spin_unlock(&ring->execlist_lock); - I915_WRITE(RING_CONTEXT_STATUS_PTR(ring), - _MASKED_FIELD(GEN8_CSB_PTR_MASK << 8, - ((u32)ring->next_context_status_buffer & - GEN8_CSB_PTR_MASK) << 8)); + WARN(submit_contexts > 2, "More than two context complete events?\n"); + ring->next_context_status_buffer = tail % GEN8_CSB_ENTRIES; + I915_WRITE(RING_CONTEXT_STATUS_PTR(ring), + _MASKED_FIELD(GEN8_CSB_PTR_MASK << 8, + ring->next_context_status_buffer<<8)); + } while (1); } static int execlists_context_queue(struct drm_i915_gem_request *request) @@ -569,7 +582,7 @@ static int execlists_context_queue(struct drm_i915_gem_request *request) i915_gem_request_reference(request); - spin_lock_irq(&engine->execlist_lock); + spin_lock(&engine->execlist_lock); list_for_each_entry(cursor, &engine->execlist_queue, execlist_link) if (++num_elements > 2) @@ -595,7 +608,7 @@ static int execlists_context_queue(struct drm_i915_gem_request *request) if (num_elements == 0) execlists_context_unqueue(engine); - spin_unlock_irq(&engine->execlist_lock); + spin_unlock(&engine->execlist_lock); return 0; } @@ -648,9 +661,9 @@ void intel_execlists_retire_requests(struct intel_engine_cs *ring) return; INIT_LIST_HEAD(&retired_list); - spin_lock_irq(&ring->execlist_lock); + spin_lock(&ring->execlist_lock); list_replace_init(&ring->execlist_retired_req_list, &retired_list); - spin_unlock_irq(&ring->execlist_lock); + spin_unlock(&ring->execlist_lock); list_for_each_entry_safe(req, tmp, &retired_list, execlist_link) { struct intel_context *ctx = req->ctx; @@ -1494,6 +1507,9 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *ring) if (!intel_engine_initialized(ring)) return; + if (ring->execlists_submit) + kthread_stop(ring->execlists_submit); + if (ring->buffer) { struct drm_i915_private *dev_priv = ring->i915; intel_logical_ring_stop(ring); @@ -1519,13 +1535,15 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *ring) static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring) { + struct drm_i915_private *dev_priv = to_i915(dev); + struct task_struct *task; int ret; /* Intentionally left blank. */ ring->buffer = NULL; ring->dev = dev; - ring->i915 = to_i915(dev); + ring->i915 = dev_priv; ring->fence_context = fence_context_alloc(1); INIT_LIST_HEAD(&ring->request_list); i915_gem_batch_pool_init(dev, &ring->batch_pool); @@ -1556,6 +1574,15 @@ static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *rin goto error; } + ring->next_context_status_buffer = + I915_READ(RING_CONTEXT_STATUS_PTR(ring)) & GEN8_CSB_PTR_MASK; + task = kthread_run(intel_execlists_submit, ring, + "irq/i915:%de", ring->id); + if (IS_ERR(task)) + goto error; + + ring->execlists_submit = task; + return 0; error: diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h index fd88530ac540..635be05bd1e7 100644 --- a/drivers/gpu/drm/i915/intel_lrc.h +++ b/drivers/gpu/drm/i915/intel_lrc.h @@ -70,7 +70,6 @@ int intel_execlists_submission(struct i915_execbuffer_params *params, struct list_head *vmas); u32 intel_execlists_ctx_id(struct drm_i915_gem_object *ctx_obj); -void intel_lrc_irq_handler(struct intel_engine_cs *ring); void intel_execlists_retire_requests(struct intel_engine_cs *ring); #endif /* _INTEL_LRC_H_ */ diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index 5b2cec3db78c..7a1940745880 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -283,6 +283,7 @@ struct intel_engine_cs { } semaphore; /* Execlists */ + struct task_struct *execlists_submit; spinlock_t execlist_lock; struct list_head execlist_queue; struct list_head execlist_retired_req_list; -- 2.6.4