Attachment 108029 Details for Bug 83677 – Backport of requests and PPGTT changes to 3.17.0

[patch] Backport of requests and PPGTT changes to 3.17.0

0001-Backport-drm-i915-from-ickle-s-requests-branch-to-3..patch (text/plain), 886.33 KB, created by Simon Farnsworth on 2014-10-18 14:40:43 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Simon Farnsworth

Created: 2014-10-18 14:40:43 UTC

Size: 886.33 KB

patch

obsolete

>From cd167748adeef1a9e77e54520ade18c520bb11f2 Mon Sep 17 00:00:00 2001
>From: Daniel Vetter <daniel.vetter@ffwll.ch>
>Date: Wed, 6 Aug 2014 15:04:45 +0200
>Subject: [PATCH] Backport drm/i915 from ickle's requests branch to 3.17.0
>
>I've checked out git://people.freedesktop.org/~ickle/linux-2.6
>commit dcee630f5775a1f2268deaa2a3cd011cf8ba941f
>Author: Chris Wilson <chris@chris-wilson.co.uk>
>Date:   Thu Aug 28 10:48:47 2014 +0100
>
>    drm/i915: s/seqno/request/ tracking inside objects
>
>then removed drivers/gpu/drm/i915 from a Fedora 3.17.0 and put in the
>drm/i915 from Chris's tree.
>
>The remaining work is the following:
> * Patch to compile with drm and kernel core from 3.17.0 instead of Chris's tree.
> * Remove unused features that didn't exist in 3.17.0
> * Check all changes to ensure that I've not pulled in something dangerous.
>---
> drivers/gpu/drm/i915/Makefile                |    4 +-
> drivers/gpu/drm/i915/i915_cmd_parser.c       |  160 +-
> drivers/gpu/drm/i915/i915_debugfs.c          |  432 ++--
> drivers/gpu/drm/i915/i915_dma.c              |  979 +-------
> drivers/gpu/drm/i915/i915_drv.c              |  229 +-
> drivers/gpu/drm/i915/i915_drv.h              |  592 +++--
> drivers/gpu/drm/i915/i915_gem.c              | 1832 ++++++--------
> drivers/gpu/drm/i915/i915_gem_context.c      |  682 +++---
> drivers/gpu/drm/i915/i915_gem_debug.c        |  118 -
> drivers/gpu/drm/i915/i915_gem_evict.c        |    4 +-
> drivers/gpu/drm/i915/i915_gem_execbuffer.c   |  694 +++---
> drivers/gpu/drm/i915/i915_gem_gtt.c          |  408 ++--
> drivers/gpu/drm/i915/i915_gem_gtt.h          |   36 +-
> drivers/gpu/drm/i915/i915_gem_render_state.c |   35 +-
> drivers/gpu/drm/i915/i915_gem_request.c      |  666 ++++++
> drivers/gpu/drm/i915/i915_gem_stolen.c       |   13 +-
> drivers/gpu/drm/i915/i915_gem_tiling.c       |   11 +-
> drivers/gpu/drm/i915/i915_gpu_error.c        |  526 ++--
> drivers/gpu/drm/i915/i915_irq.c              |  677 +++---
> drivers/gpu/drm/i915/i915_reg.h              |  287 ++-
> drivers/gpu/drm/i915/i915_sysfs.c            |    2 +-
> drivers/gpu/drm/i915/i915_trace.h            |  215 +-
> drivers/gpu/drm/i915/intel_bios.c            |   13 +-
> drivers/gpu/drm/i915/intel_bios.h            |   13 +-
> drivers/gpu/drm/i915/intel_ddi.c             |  344 +--
> drivers/gpu/drm/i915/intel_display.c         | 2024 +++++++++-------
> drivers/gpu/drm/i915/intel_dp.c              | 1059 ++++++---
> drivers/gpu/drm/i915/intel_drv.h             |   61 +-
> drivers/gpu/drm/i915/intel_dsi.c             |   40 +-
> drivers/gpu/drm/i915/intel_dsi.h             |    2 +
> drivers/gpu/drm/i915/intel_dsi_cmd.c         |    2 +-
> drivers/gpu/drm/i915/intel_dsi_panel_vbt.c   |   38 +-
> drivers/gpu/drm/i915/intel_dsi_pll.c         |   17 +-
> drivers/gpu/drm/i915/intel_dvo.c             |   11 +-
> drivers/gpu/drm/i915/intel_fbdev.c           |   83 +-
> drivers/gpu/drm/i915/intel_hdmi.c            |  170 +-
> drivers/gpu/drm/i915/intel_lvds.c            |    3 +-
> drivers/gpu/drm/i915/intel_opregion.c        |   16 +-
> drivers/gpu/drm/i915/intel_overlay.c         |  200 +-
> drivers/gpu/drm/i915/intel_panel.c           |   59 +-
> drivers/gpu/drm/i915/intel_pm.c              | 1152 ++++++---
> drivers/gpu/drm/i915/intel_ringbuffer.c      | 3291 +++++++++++++-------------
> drivers/gpu/drm/i915/intel_ringbuffer.h      |  379 +--
> drivers/gpu/drm/i915/intel_sprite.c          |  349 ++-
> drivers/gpu/drm/i915/intel_tv.c              |    9 +-
> drivers/gpu/drm/i915/intel_uncore.c          |   23 +-
> 46 files changed, 9728 insertions(+), 8232 deletions(-)
> delete mode 100644 drivers/gpu/drm/i915/i915_gem_debug.c
> create mode 100644 drivers/gpu/drm/i915/i915_gem_request.c
>
>diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
>index 91bd167..5e8dc28 100644
>--- a/drivers/gpu/drm/i915/Makefile
>+++ b/drivers/gpu/drm/i915/Makefile
>@@ -17,14 +17,14 @@ i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o
> 
> # GEM code
> i915-y += i915_cmd_parser.o \
>+	  i915_gem.o \
> 	  i915_gem_context.o \
> 	  i915_gem_render_state.o \
>-	  i915_gem_debug.o \
> 	  i915_gem_dmabuf.o \
> 	  i915_gem_evict.o \
> 	  i915_gem_execbuffer.o \
> 	  i915_gem_gtt.o \
>-	  i915_gem.o \
>+	  i915_gem_request.o \
> 	  i915_gem_stolen.o \
> 	  i915_gem_tiling.o \
> 	  i915_gem_userptr.o \
>diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
>index 4b7ed52..408e0bd 100644
>--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
>+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
>@@ -501,7 +501,7 @@ static u32 gen7_blt_get_cmd_length_mask(u32 cmd_header)
> 	return 0;
> }
> 
>-static bool validate_cmds_sorted(struct intel_engine_cs *ring,
>+static bool validate_cmds_sorted(struct intel_engine_cs *engine,
> 				 const struct drm_i915_cmd_table *cmd_tables,
> 				 int cmd_table_count)
> {
>@@ -523,7 +523,7 @@ static bool validate_cmds_sorted(struct intel_engine_cs *ring,
> 
> 			if (curr < previous) {
> 				DRM_ERROR("CMD: table not sorted ring=%d table=%d entry=%d cmd=0x%08X prev=0x%08X\n",
>-					  ring->id, i, j, curr, previous);
>+					  engine->id, i, j, curr, previous);
> 				ret = false;
> 			}
> 
>@@ -555,11 +555,11 @@ static bool check_sorted(int ring_id, const u32 *reg_table, int reg_count)
> 	return ret;
> }
> 
>-static bool validate_regs_sorted(struct intel_engine_cs *ring)
>+static bool validate_regs_sorted(struct intel_engine_cs *engine)
> {
>-	return check_sorted(ring->id, ring->reg_table, ring->reg_count) &&
>-		check_sorted(ring->id, ring->master_reg_table,
>-			     ring->master_reg_count);
>+	return check_sorted(engine->id, engine->reg_table, engine->reg_count) &&
>+		check_sorted(engine->id, engine->master_reg_table,
>+			     engine->master_reg_count);
> }
> 
> struct cmd_node {
>@@ -583,13 +583,13 @@ struct cmd_node {
>  */
> #define CMD_HASH_MASK STD_MI_OPCODE_MASK
> 
>-static int init_hash_table(struct intel_engine_cs *ring,
>+static int init_hash_table(struct intel_engine_cs *engine,
> 			   const struct drm_i915_cmd_table *cmd_tables,
> 			   int cmd_table_count)
> {
> 	int i, j;
> 
>-	hash_init(ring->cmd_hash);
>+	hash_init(engine->cmd_hash);
> 
> 	for (i = 0; i < cmd_table_count; i++) {
> 		const struct drm_i915_cmd_table *table = &cmd_tables[i];
>@@ -604,7 +604,7 @@ static int init_hash_table(struct intel_engine_cs *ring,
> 				return -ENOMEM;
> 
> 			desc_node->desc = desc;
>-			hash_add(ring->cmd_hash, &desc_node->node,
>+			hash_add(engine->cmd_hash, &desc_node->node,
> 				 desc->cmd.value & CMD_HASH_MASK);
> 		}
> 	}
>@@ -612,21 +612,21 @@ static int init_hash_table(struct intel_engine_cs *ring,
> 	return 0;
> }
> 
>-static void fini_hash_table(struct intel_engine_cs *ring)
>+static void fini_hash_table(struct intel_engine_cs *engine)
> {
> 	struct hlist_node *tmp;
> 	struct cmd_node *desc_node;
> 	int i;
> 
>-	hash_for_each_safe(ring->cmd_hash, i, tmp, desc_node, node) {
>+	hash_for_each_safe(engine->cmd_hash, i, tmp, desc_node, node) {
> 		hash_del(&desc_node->node);
> 		kfree(desc_node);
> 	}
> }
> 
> /**
>- * i915_cmd_parser_init_ring() - set cmd parser related fields for a ringbuffer
>- * @ring: the ringbuffer to initialize
>+ * i915_cmd_parser_init_engine() - set cmd parser related fields for a ringbuffer
>+ * @engine: the ringbuffer to initialize
>  *
>  * Optionally initializes fields related to batch buffer command parsing in the
>  * struct intel_engine_cs based on whether the platform requires software
>@@ -634,18 +634,18 @@ static void fini_hash_table(struct intel_engine_cs *ring)
>  *
>  * Return: non-zero if initialization fails
>  */
>-int i915_cmd_parser_init_ring(struct intel_engine_cs *ring)
>+int i915_cmd_parser_init_engine(struct intel_engine_cs *engine)
> {
> 	const struct drm_i915_cmd_table *cmd_tables;
> 	int cmd_table_count;
> 	int ret;
> 
>-	if (!IS_GEN7(ring->dev))
>+	if (!IS_GEN7(engine->i915))
> 		return 0;
> 
>-	switch (ring->id) {
>+	switch (engine->id) {
> 	case RCS:
>-		if (IS_HASWELL(ring->dev)) {
>+		if (IS_HASWELL(engine->i915)) {
> 			cmd_tables = hsw_render_ring_cmds;
> 			cmd_table_count =
> 				ARRAY_SIZE(hsw_render_ring_cmds);
>@@ -654,26 +654,26 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *ring)
> 			cmd_table_count = ARRAY_SIZE(gen7_render_cmds);
> 		}
> 
>-		ring->reg_table = gen7_render_regs;
>-		ring->reg_count = ARRAY_SIZE(gen7_render_regs);
>+		engine->reg_table = gen7_render_regs;
>+		engine->reg_count = ARRAY_SIZE(gen7_render_regs);
> 
>-		if (IS_HASWELL(ring->dev)) {
>-			ring->master_reg_table = hsw_master_regs;
>-			ring->master_reg_count = ARRAY_SIZE(hsw_master_regs);
>+		if (IS_HASWELL(engine->i915)) {
>+			engine->master_reg_table = hsw_master_regs;
>+			engine->master_reg_count = ARRAY_SIZE(hsw_master_regs);
> 		} else {
>-			ring->master_reg_table = ivb_master_regs;
>-			ring->master_reg_count = ARRAY_SIZE(ivb_master_regs);
>+			engine->master_reg_table = ivb_master_regs;
>+			engine->master_reg_count = ARRAY_SIZE(ivb_master_regs);
> 		}
> 
>-		ring->get_cmd_length_mask = gen7_render_get_cmd_length_mask;
>+		engine->get_cmd_length_mask = gen7_render_get_cmd_length_mask;
> 		break;
> 	case VCS:
> 		cmd_tables = gen7_video_cmds;
> 		cmd_table_count = ARRAY_SIZE(gen7_video_cmds);
>-		ring->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
>+		engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
> 		break;
> 	case BCS:
>-		if (IS_HASWELL(ring->dev)) {
>+		if (IS_HASWELL(engine->i915)) {
> 			cmd_tables = hsw_blt_ring_cmds;
> 			cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmds);
> 		} else {
>@@ -681,70 +681,68 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *ring)
> 			cmd_table_count = ARRAY_SIZE(gen7_blt_cmds);
> 		}
> 
>-		ring->reg_table = gen7_blt_regs;
>-		ring->reg_count = ARRAY_SIZE(gen7_blt_regs);
>+		engine->reg_table = gen7_blt_regs;
>+		engine->reg_count = ARRAY_SIZE(gen7_blt_regs);
> 
>-		if (IS_HASWELL(ring->dev)) {
>-			ring->master_reg_table = hsw_master_regs;
>-			ring->master_reg_count = ARRAY_SIZE(hsw_master_regs);
>+		if (IS_HASWELL(engine->i915)) {
>+			engine->master_reg_table = hsw_master_regs;
>+			engine->master_reg_count = ARRAY_SIZE(hsw_master_regs);
> 		} else {
>-			ring->master_reg_table = ivb_master_regs;
>-			ring->master_reg_count = ARRAY_SIZE(ivb_master_regs);
>+			engine->master_reg_table = ivb_master_regs;
>+			engine->master_reg_count = ARRAY_SIZE(ivb_master_regs);
> 		}
> 
>-		ring->get_cmd_length_mask = gen7_blt_get_cmd_length_mask;
>+		engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask;
> 		break;
> 	case VECS:
> 		cmd_tables = hsw_vebox_cmds;
> 		cmd_table_count = ARRAY_SIZE(hsw_vebox_cmds);
> 		/* VECS can use the same length_mask function as VCS */
>-		ring->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
>+		engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
> 		break;
> 	default:
>-		DRM_ERROR("CMD: cmd_parser_init with unknown ring: %d\n",
>-			  ring->id);
>+		DRM_ERROR("CMD: cmd_parser_init with unknown engine: %d\n",
>+			  engine->id);
> 		BUG();
> 	}
> 
>-	BUG_ON(!validate_cmds_sorted(ring, cmd_tables, cmd_table_count));
>-	BUG_ON(!validate_regs_sorted(ring));
>+	BUG_ON(!validate_cmds_sorted(engine, cmd_tables, cmd_table_count));
>+	BUG_ON(!validate_regs_sorted(engine));
> 
>-	if (hash_empty(ring->cmd_hash)) {
>-		ret = init_hash_table(ring, cmd_tables, cmd_table_count);
>-		if (ret) {
>-			DRM_ERROR("CMD: cmd_parser_init failed!\n");
>-			fini_hash_table(ring);
>-			return ret;
>-		}
>+	ret = init_hash_table(engine, cmd_tables, cmd_table_count);
>+	if (ret) {
>+		DRM_ERROR("CMD: cmd_parser_init failed!\n");
>+		fini_hash_table(engine);
>+		return ret;
> 	}
> 
>-	ring->needs_cmd_parser = true;
>+	engine->needs_cmd_parser = true;
> 
> 	return 0;
> }
> 
> /**
>- * i915_cmd_parser_fini_ring() - clean up cmd parser related fields
>- * @ring: the ringbuffer to clean up
>+ * i915_cmd_parser_fini_engine() - clean up cmd parser related fields
>+ * @engine: the ringbuffer to clean up
>  *
>  * Releases any resources related to command parsing that may have been
>- * initialized for the specified ring.
>+ * initialized for the specified engine.
>  */
>-void i915_cmd_parser_fini_ring(struct intel_engine_cs *ring)
>+void i915_cmd_parser_fini_engine(struct intel_engine_cs *engine)
> {
>-	if (!ring->needs_cmd_parser)
>+	if (!engine->needs_cmd_parser)
> 		return;
> 
>-	fini_hash_table(ring);
>+	fini_hash_table(engine);
> }
> 
> static const struct drm_i915_cmd_descriptor*
>-find_cmd_in_table(struct intel_engine_cs *ring,
>+find_cmd_in_table(struct intel_engine_cs *engine,
> 		  u32 cmd_header)
> {
> 	struct cmd_node *desc_node;
> 
>-	hash_for_each_possible(ring->cmd_hash, desc_node, node,
>+	hash_for_each_possible(engine->cmd_hash, desc_node, node,
> 			       cmd_header & CMD_HASH_MASK) {
> 		const struct drm_i915_cmd_descriptor *desc = desc_node->desc;
> 		u32 masked_cmd = desc->cmd.mask & cmd_header;
>@@ -761,23 +759,23 @@ find_cmd_in_table(struct intel_engine_cs *ring,
>  * Returns a pointer to a descriptor for the command specified by cmd_header.
>  *
>  * The caller must supply space for a default descriptor via the default_desc
>- * parameter. If no descriptor for the specified command exists in the ring's
>+ * parameter. If no descriptor for the specified command exists in the engine's
>  * command parser tables, this function fills in default_desc based on the
>- * ring's default length encoding and returns default_desc.
>+ * engine's default length encoding and returns default_desc.
>  */
> static const struct drm_i915_cmd_descriptor*
>-find_cmd(struct intel_engine_cs *ring,
>+find_cmd(struct intel_engine_cs *engine,
> 	 u32 cmd_header,
> 	 struct drm_i915_cmd_descriptor *default_desc)
> {
> 	const struct drm_i915_cmd_descriptor *desc;
> 	u32 mask;
> 
>-	desc = find_cmd_in_table(ring, cmd_header);
>+	desc = find_cmd_in_table(engine, cmd_header);
> 	if (desc)
> 		return desc;
> 
>-	mask = ring->get_cmd_length_mask(cmd_header);
>+	mask = engine->get_cmd_length_mask(cmd_header);
> 	if (!mask)
> 		return NULL;
> 
>@@ -834,19 +832,17 @@ finish:
> }
> 
> /**
>- * i915_needs_cmd_parser() - should a given ring use software command parsing?
>- * @ring: the ring in question
>+ * i915_needs_cmd_parser() - should a given engine use software command parsing?
>+ * @engine: the engine in question
>  *
>  * Only certain platforms require software batch buffer command parsing, and
>  * only when enabled via module paramter.
>  *
>- * Return: true if the ring requires software command parsing
>+ * Return: true if the engine requires software command parsing
>  */
>-bool i915_needs_cmd_parser(struct intel_engine_cs *ring)
>+bool i915_needs_cmd_parser(struct intel_engine_cs *engine)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>-
>-	if (!ring->needs_cmd_parser)
>+	if (!engine->needs_cmd_parser)
> 		return false;
> 
> 	/*
>@@ -854,13 +850,13 @@ bool i915_needs_cmd_parser(struct intel_engine_cs *ring)
> 	 * disabled. That will cause all of the parser's PPGTT checks to
> 	 * fail. For now, disable parsing when PPGTT is off.
> 	 */
>-	if (!dev_priv->mm.aliasing_ppgtt)
>+	if (USES_PPGTT(engine->dev))
> 		return false;
> 
> 	return (i915.enable_cmd_parser == 1);
> }
> 
>-static bool check_cmd(const struct intel_engine_cs *ring,
>+static bool check_cmd(const struct intel_engine_cs *engine,
> 		      const struct drm_i915_cmd_descriptor *desc,
> 		      const u32 *cmd,
> 		      const bool is_master,
>@@ -897,16 +893,16 @@ static bool check_cmd(const struct intel_engine_cs *ring,
> 				*oacontrol_set = (cmd[2] != 0);
> 		}
> 
>-		if (!valid_reg(ring->reg_table,
>-			       ring->reg_count, reg_addr)) {
>+		if (!valid_reg(engine->reg_table,
>+			       engine->reg_count, reg_addr)) {
> 			if (!is_master ||
>-			    !valid_reg(ring->master_reg_table,
>-				       ring->master_reg_count,
>+			    !valid_reg(engine->master_reg_table,
>+				       engine->master_reg_count,
> 				       reg_addr)) {
>-				DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (ring=%d)\n",
>+				DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (engine=%d)\n",
> 						 reg_addr,
> 						 *cmd,
>-						 ring->id);
>+						 engine->id);
> 				return false;
> 			}
> 		}
>@@ -935,11 +931,11 @@ static bool check_cmd(const struct intel_engine_cs *ring,
> 				desc->bits[i].mask;
> 
> 			if (dword != desc->bits[i].expected) {
>-				DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (ring=%d)\n",
>+				DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (engine=%d)\n",
> 						 *cmd,
> 						 desc->bits[i].mask,
> 						 desc->bits[i].expected,
>-						 dword, ring->id);
>+						 dword, engine->id);
> 				return false;
> 			}
> 		}
>@@ -952,7 +948,7 @@ static bool check_cmd(const struct intel_engine_cs *ring,
> 
> /**
>  * i915_parse_cmds() - parse a submitted batch buffer for privilege violations
>- * @ring: the ring on which the batch is to execute
>+ * @engine: the engine on which the batch is to execute
>  * @batch_obj: the batch buffer in question
>  * @batch_start_offset: byte offset in the batch at which execution starts
>  * @is_master: is the submitting process the drm master?
>@@ -962,7 +958,7 @@ static bool check_cmd(const struct intel_engine_cs *ring,
>  *
>  * Return: non-zero if the parser finds violations or otherwise fails
>  */
>-int i915_parse_cmds(struct intel_engine_cs *ring,
>+int i915_parse_cmds(struct intel_engine_cs *engine,
> 		    struct drm_i915_gem_object *batch_obj,
> 		    u32 batch_start_offset,
> 		    bool is_master)
>@@ -999,7 +995,7 @@ int i915_parse_cmds(struct intel_engine_cs *ring,
> 		if (*cmd == MI_BATCH_BUFFER_END)
> 			break;
> 
>-		desc = find_cmd(ring, *cmd, &default_desc);
>+		desc = find_cmd(engine, *cmd, &default_desc);
> 		if (!desc) {
> 			DRM_DEBUG_DRIVER("CMD: Unrecognized command: 0x%08X\n",
> 					 *cmd);
>@@ -1021,7 +1017,7 @@ int i915_parse_cmds(struct intel_engine_cs *ring,
> 			break;
> 		}
> 
>-		if (!check_cmd(ring, desc, cmd, is_master, &oacontrol_set)) {
>+		if (!check_cmd(engine, desc, cmd, is_master, &oacontrol_set)) {
> 			ret = -EINVAL;
> 			break;
> 		}
>diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
>index 9e737b7..d5f237e 100644
>--- a/drivers/gpu/drm/i915/i915_debugfs.c
>+++ b/drivers/gpu/drm/i915/i915_debugfs.c
>@@ -123,20 +123,23 @@ static void
> describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
> {
> 	struct i915_vma *vma;
>-	int pin_count = 0;
>+	int pin_count = 0, n;
> 
>-	seq_printf(m, "%pK: %s%s%s %8zdKiB %02x %02x %u %u %u%s%s%s",
>+	seq_printf(m, "%pK: %s%s%s %8zdKiB %02x %02x [",
> 		   &obj->base,
> 		   get_pin_flag(obj),
> 		   get_tiling_flag(obj),
> 		   get_global_flag(obj),
> 		   obj->base.size / 1024,
> 		   obj->base.read_domains,
>-		   obj->base.write_domain,
>-		   obj->last_read_seqno,
>-		   obj->last_write_seqno,
>-		   obj->last_fenced_seqno,
>-		   i915_cache_level_str(obj->cache_level),
>+		   obj->base.write_domain);
>+	for (n = 0; n < ARRAY_SIZE(obj->last_read); n++)
>+		seq_printf(m, " %x",
>+			   i915_request_seqno(obj->last_read[n].request));
>+	seq_printf(m, " ] %x %x%s%s%s",
>+		   i915_request_seqno(obj->last_write.request),
>+		   i915_request_seqno(obj->last_fence.request),
>+		   i915_cache_level_str(to_i915(obj->base.dev), obj->cache_level),
> 		   obj->dirty ? " dirty" : "",
> 		   obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
> 	if (obj->base.name)
>@@ -168,15 +171,15 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
> 		*t = '\0';
> 		seq_printf(m, " (%s mappable)", s);
> 	}
>-	if (obj->ring != NULL)
>-		seq_printf(m, " (%s)", obj->ring->name);
>+	if (obj->last_write.request)
>+		seq_printf(m, " (%s)", obj->last_write.request->engine->name);
> 	if (obj->frontbuffer_bits)
> 		seq_printf(m, " (frontbuffer: 0x%03x)", obj->frontbuffer_bits);
> }
> 
> static void describe_ctx(struct seq_file *m, struct intel_context *ctx)
> {
>-	seq_putc(m, ctx->legacy_hw_ctx.initialized ? 'I' : 'i');
>+	seq_putc(m, ctx->ring[RCS].initialized ? 'I' : 'i');
> 	seq_putc(m, ctx->remap_slice ? 'R' : 'r');
> 	seq_putc(m, ' ');
> }
>@@ -333,10 +336,10 @@ static int per_file_stats(int id, void *ptr, void *data)
> 			}
> 
> 			ppgtt = container_of(vma->vm, struct i915_hw_ppgtt, base);
>-			if (ppgtt->ctx && ppgtt->ctx->file_priv != stats->file_priv)
>+			if (ppgtt->file_priv != stats->file_priv)
> 				continue;
> 
>-			if (obj->ring) /* XXX per-vma statistic */
>+			if (obj->active) /* XXX per-vma statistic */
> 				stats->active += obj->base.size;
> 			else
> 				stats->inactive += obj->base.size;
>@@ -346,7 +349,7 @@ static int per_file_stats(int id, void *ptr, void *data)
> 	} else {
> 		if (i915_gem_obj_ggtt_bound(obj)) {
> 			stats->global += obj->base.size;
>-			if (obj->ring)
>+			if (obj->active)
> 				stats->active += obj->base.size;
> 			else
> 				stats->inactive += obj->base.size;
>@@ -515,7 +518,7 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
> {
> 	struct drm_info_node *node = m->private;
> 	struct drm_device *dev = node->minor->dev;
>-	unsigned long flags;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *crtc;
> 	int ret;
> 
>@@ -528,12 +531,14 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
> 		const char plane = plane_name(crtc->plane);
> 		struct intel_unpin_work *work;
> 
>-		spin_lock_irqsave(&dev->event_lock, flags);
>+		spin_lock_irq(&dev->event_lock);
> 		work = crtc->unpin_work;
> 		if (work == NULL) {
> 			seq_printf(m, "No flip due on pipe %c (plane %c)\n",
> 				   pipe, plane);
> 		} else {
>+			u32 addr;
>+
> 			if (atomic_read(&work->pending) < INTEL_FLIP_COMPLETE) {
> 				seq_printf(m, "Flip queued on pipe %c (plane %c)\n",
> 					   pipe, plane);
>@@ -541,26 +546,38 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
> 				seq_printf(m, "Flip pending (waiting for vsync) on pipe %c (plane %c)\n",
> 					   pipe, plane);
> 			}
>+			if (work->flip_queued_request) {
>+				struct i915_gem_request *rq =
>+					work->flip_queued_request;
>+				seq_printf(m, "Flip queued on %s at seqno %u, next seqno %u [current breadcrumb %u], completed? %d\n",
>+					   rq->engine->name,
>+					   rq->seqno, rq->i915->next_seqno,
>+					   rq->engine->get_seqno(rq->engine),
>+					   __i915_request_complete__wa(rq));
>+			} else
>+				seq_printf(m, "Flip not associated with any ring\n");
>+			seq_printf(m, "Flip queued on frame %d, (was ready on frame %d), now %d\n",
>+				   work->flip_queued_vblank,
>+				   work->flip_ready_vblank,
>+				   drm_vblank_count(dev, crtc->pipe));
> 			if (work->enable_stall_check)
> 				seq_puts(m, "Stall check enabled, ");
> 			else
> 				seq_puts(m, "Stall check waiting for page flip ioctl, ");
> 			seq_printf(m, "%d prepares\n", atomic_read(&work->pending));
> 
>-			if (work->old_fb_obj) {
>-				struct drm_i915_gem_object *obj = work->old_fb_obj;
>-				if (obj)
>-					seq_printf(m, "Old framebuffer gtt_offset 0x%08lx\n",
>-						   i915_gem_obj_ggtt_offset(obj));
>-			}
>+			if (INTEL_INFO(dev)->gen >= 4)
>+				addr = I915_HI_DISPBASE(I915_READ(DSPSURF(crtc->plane)));
>+			else
>+				addr = I915_READ(DSPADDR(crtc->plane));
>+			seq_printf(m, "Current scanout address 0x%08x\n", addr);
>+
> 			if (work->pending_flip_obj) {
>-				struct drm_i915_gem_object *obj = work->pending_flip_obj;
>-				if (obj)
>-					seq_printf(m, "New framebuffer gtt_offset 0x%08lx\n",
>-						   i915_gem_obj_ggtt_offset(obj));
>+				seq_printf(m, "New framebuffer address 0x%08lx\n", (long)work->gtt_offset);
>+				seq_printf(m, "MMIO update completed? %d\n",  addr == work->gtt_offset);
> 			}
> 		}
>-		spin_unlock_irqrestore(&dev->event_lock, flags);
>+		spin_unlock_irq(&dev->event_lock);
> 	}
> 
> 	mutex_unlock(&dev->struct_mutex);
>@@ -573,8 +590,8 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
> 	struct drm_info_node *node = m->private;
> 	struct drm_device *dev = node->minor->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>-	struct drm_i915_gem_request *gem_request;
>+	struct intel_engine_cs *engine;
>+	struct i915_gem_request *rq;
> 	int ret, count, i;
> 
> 	ret = mutex_lock_interruptible(&dev->struct_mutex);
>@@ -582,17 +599,15 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
> 		return ret;
> 
> 	count = 0;
>-	for_each_ring(ring, dev_priv, i) {
>-		if (list_empty(&ring->request_list))
>+	for_each_engine(engine, dev_priv, i) {
>+		if (list_empty(&engine->requests))
> 			continue;
> 
>-		seq_printf(m, "%s requests:\n", ring->name);
>-		list_for_each_entry(gem_request,
>-				    &ring->request_list,
>-				    list) {
>+		seq_printf(m, "%s requests:\n", engine->name);
>+		list_for_each_entry(rq, &engine->requests, engine_list) {
> 			seq_printf(m, "    %d @ %d\n",
>-				   gem_request->seqno,
>-				   (int) (jiffies - gem_request->emitted_jiffies));
>+				   rq->seqno,
>+				   (int)(jiffies - rq->emitted_jiffies));
> 		}
> 		count++;
> 	}
>@@ -604,13 +619,17 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
> 	return 0;
> }
> 
>-static void i915_ring_seqno_info(struct seq_file *m,
>-				 struct intel_engine_cs *ring)
>+static void i915_engine_seqno_info(struct seq_file *m,
>+				   struct intel_engine_cs *engine)
> {
>-	if (ring->get_seqno) {
>-		seq_printf(m, "Current sequence (%s): %u\n",
>-			   ring->name, ring->get_seqno(ring, false));
>-	}
>+	seq_printf(m, "Current sequence (%s): seqno=%u, tag=%u [last breadcrumb %u, last request %u], next seqno=%u, next tag=%u\n",
>+		   engine->name,
>+		   engine->get_seqno(engine),
>+		   engine->tag,
>+		   engine->breadcrumb[engine->id],
>+		   engine->last_request ? engine->last_request->seqno : 0,
>+		   engine->i915->next_seqno,
>+		   engine->next_tag);
> }
> 
> static int i915_gem_seqno_info(struct seq_file *m, void *data)
>@@ -618,7 +637,7 @@ static int i915_gem_seqno_info(struct seq_file *m, void *data)
> 	struct drm_info_node *node = m->private;
> 	struct drm_device *dev = node->minor->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	int ret, i;
> 
> 	ret = mutex_lock_interruptible(&dev->struct_mutex);
>@@ -626,8 +645,8 @@ static int i915_gem_seqno_info(struct seq_file *m, void *data)
> 		return ret;
> 	intel_runtime_pm_get(dev_priv);
> 
>-	for_each_ring(ring, dev_priv, i)
>-		i915_ring_seqno_info(m, ring);
>+	for_each_engine(engine, dev_priv, i)
>+		i915_engine_seqno_info(m, engine);
> 
> 	intel_runtime_pm_put(dev_priv);
> 	mutex_unlock(&dev->struct_mutex);
>@@ -641,7 +660,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
> 	struct drm_info_node *node = m->private;
> 	struct drm_device *dev = node->minor->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	int ret, i, pipe;
> 
> 	ret = mutex_lock_interruptible(&dev->struct_mutex);
>@@ -650,7 +669,6 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
> 	intel_runtime_pm_get(dev_priv);
> 
> 	if (IS_CHERRYVIEW(dev)) {
>-		int i;
> 		seq_printf(m, "Master Interrupt Control:\t%08x\n",
> 			   I915_READ(GEN8_MASTER_IRQ));
> 
>@@ -662,7 +680,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
> 			   I915_READ(VLV_IIR_RW));
> 		seq_printf(m, "Display IMR:\t%08x\n",
> 			   I915_READ(VLV_IMR));
>-		for_each_pipe(pipe)
>+		for_each_pipe(dev_priv, pipe)
> 			seq_printf(m, "Pipe %c stat:\t%08x\n",
> 				   pipe_name(pipe),
> 				   I915_READ(PIPESTAT(pipe)));
>@@ -702,7 +720,13 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
> 				   i, I915_READ(GEN8_GT_IER(i)));
> 		}
> 
>-		for_each_pipe(pipe) {
>+		for_each_pipe(dev_priv, pipe) {
>+			if (!intel_display_power_enabled(dev_priv,
>+						POWER_DOMAIN_PIPE(pipe))) {
>+				seq_printf(m, "Pipe %c power disabled\n",
>+					   pipe_name(pipe));
>+				continue;
>+			}
> 			seq_printf(m, "Pipe %c IMR:\t%08x\n",
> 				   pipe_name(pipe),
> 				   I915_READ(GEN8_DE_PIPE_IMR(pipe)));
>@@ -743,7 +767,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
> 			   I915_READ(VLV_IIR_RW));
> 		seq_printf(m, "Display IMR:\t%08x\n",
> 			   I915_READ(VLV_IMR));
>-		for_each_pipe(pipe)
>+		for_each_pipe(dev_priv, pipe)
> 			seq_printf(m, "Pipe %c stat:\t%08x\n",
> 				   pipe_name(pipe),
> 				   I915_READ(PIPESTAT(pipe)));
>@@ -779,7 +803,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
> 			   I915_READ(IIR));
> 		seq_printf(m, "Interrupt mask:      %08x\n",
> 			   I915_READ(IMR));
>-		for_each_pipe(pipe)
>+		for_each_pipe(dev_priv, pipe)
> 			seq_printf(m, "Pipe %c stat:         %08x\n",
> 				   pipe_name(pipe),
> 				   I915_READ(PIPESTAT(pipe)));
>@@ -803,13 +827,13 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
> 		seq_printf(m, "Graphics Interrupt mask:		%08x\n",
> 			   I915_READ(GTIMR));
> 	}
>-	for_each_ring(ring, dev_priv, i) {
>+	for_each_engine(engine, dev_priv, i) {
> 		if (INTEL_INFO(dev)->gen >= 6) {
> 			seq_printf(m,
> 				   "Graphics Interrupt mask (%s):	%08x\n",
>-				   ring->name, I915_READ_IMR(ring));
>+				   engine->name, I915_READ_IMR(engine));
> 		}
>-		i915_ring_seqno_info(m, ring);
>+		i915_engine_seqno_info(m, engine);
> 	}
> 	intel_runtime_pm_put(dev_priv);
> 	mutex_unlock(&dev->struct_mutex);
>@@ -851,12 +875,12 @@ static int i915_hws_info(struct seq_file *m, void *data)
> 	struct drm_info_node *node = m->private;
> 	struct drm_device *dev = node->minor->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	const u32 *hws;
> 	int i;
> 
>-	ring = &dev_priv->ring[(uintptr_t)node->info_ent->data];
>-	hws = ring->status_page.page_addr;
>+	engine = &dev_priv->engine[(uintptr_t)node->info_ent->data];
>+	hws = engine->status_page.page_addr;
> 	if (hws == NULL)
> 		return 0;
> 
>@@ -927,7 +951,7 @@ static ssize_t i915_error_state_read(struct file *file, char __user *userbuf,
> 	ssize_t ret_count = 0;
> 	int ret;
> 
>-	ret = i915_error_state_buf_init(&error_str, count, *pos);
>+	ret = i915_error_state_buf_init(&error_str, to_i915(error_priv->dev), count, *pos);
> 	if (ret)
> 		return ret;
> 
>@@ -980,7 +1004,7 @@ i915_next_seqno_set(void *data, u64 val)
> 	struct drm_device *dev = data;
> 	int ret;
> 
>-	ret = mutex_lock_interruptible(&dev->struct_mutex);
>+	ret = i915_mutex_lock_interruptible(dev);
> 	if (ret)
> 		return ret;
> 
>@@ -1024,6 +1048,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
> 		u32 rpstat, cagf, reqf;
> 		u32 rpupei, rpcurup, rpprevup;
> 		u32 rpdownei, rpcurdown, rpprevdown;
>+		u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask;
> 		int max_freq;
> 
> 		/* RPSTAT1 is in the GT power well */
>@@ -1061,12 +1086,21 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
> 		gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
> 		mutex_unlock(&dev->struct_mutex);
> 
>+		if (IS_GEN6(dev) || IS_GEN7(dev)) {
>+			pm_ier = I915_READ(GEN6_PMIER);
>+			pm_imr = I915_READ(GEN6_PMIMR);
>+			pm_isr = I915_READ(GEN6_PMISR);
>+			pm_iir = I915_READ(GEN6_PMIIR);
>+			pm_mask = I915_READ(GEN6_PMINTRMSK);
>+		} else {
>+			pm_ier = I915_READ(GEN8_GT_IER(2));
>+			pm_imr = I915_READ(GEN8_GT_IMR(2));
>+			pm_isr = I915_READ(GEN8_GT_ISR(2));
>+			pm_iir = I915_READ(GEN8_GT_IIR(2));
>+			pm_mask = I915_READ(GEN6_PMINTRMSK);
>+		}
> 		seq_printf(m, "PM IER=0x%08x IMR=0x%08x ISR=0x%08x IIR=0x%08x, MASK=0x%08x\n",
>-			   I915_READ(GEN6_PMIER),
>-			   I915_READ(GEN6_PMIMR),
>-			   I915_READ(GEN6_PMISR),
>-			   I915_READ(GEN6_PMIIR),
>-			   I915_READ(GEN6_PMINTRMSK));
>+			   pm_ier, pm_imr, pm_isr, pm_iir, pm_mask);
> 		seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
> 		seq_printf(m, "Render p-state ratio: %d\n",
> 			   (gt_perf_status & 0xff00) >> 8);
>@@ -1365,7 +1399,7 @@ static int i915_drpc_info(struct seq_file *m, void *unused)
> 
> 	if (IS_VALLEYVIEW(dev))
> 		return vlv_drpc_info(m);
>-	else if (IS_GEN6(dev) || IS_GEN7(dev))
>+	else if (INTEL_INFO(dev)->gen >= 6)
> 		return gen6_drpc_info(m);
> 	else
> 		return ironlake_drpc_info(m);
>@@ -1433,6 +1467,47 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
> 	return 0;
> }
> 
>+static int i915_fbc_fc_get(void *data, u64 *val)
>+{
>+	struct drm_device *dev = data;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+
>+	if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev))
>+		return -ENODEV;
>+
>+	drm_modeset_lock_all(dev);
>+	*val = dev_priv->fbc.false_color;
>+	drm_modeset_unlock_all(dev);
>+
>+	return 0;
>+}
>+
>+static int i915_fbc_fc_set(void *data, u64 val)
>+{
>+	struct drm_device *dev = data;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	u32 reg;
>+
>+	if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev))
>+		return -ENODEV;
>+
>+	drm_modeset_lock_all(dev);
>+
>+	reg = I915_READ(ILK_DPFC_CONTROL);
>+	dev_priv->fbc.false_color = val;
>+
>+	I915_WRITE(ILK_DPFC_CONTROL, val ?
>+		   (reg | FBC_CTL_FALSE_COLOR) :
>+		   (reg & ~FBC_CTL_FALSE_COLOR));
>+
>+	drm_modeset_unlock_all(dev);
>+	return 0;
>+}
>+
>+DEFINE_SIMPLE_ATTRIBUTE(i915_fbc_fc_fops,
>+			i915_fbc_fc_get, i915_fbc_fc_set,
>+			"%llu\n");
>+
> static int i915_ips_status(struct seq_file *m, void *unused)
> {
> 	struct drm_info_node *node = m->private;
>@@ -1630,12 +1705,18 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
> 	return 0;
> }
> 
>+static void describe_ring(struct seq_file *m, struct intel_ringbuffer *ring)
>+{
>+	seq_printf(m, " (ringbuffer, space: %d, head: %u, tail: %u, last head: %d)",
>+		   ring->space, ring->head, ring->tail, ring->retired_head);
>+}
>+
> static int i915_context_status(struct seq_file *m, void *unused)
> {
> 	struct drm_info_node *node = m->private;
> 	struct drm_device *dev = node->minor->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	struct intel_context *ctx;
> 	int ret, i;
> 
>@@ -1649,23 +1730,28 @@ static int i915_context_status(struct seq_file *m, void *unused)
> 		seq_putc(m, '\n');
> 	}
> 
>-	if (dev_priv->ips.renderctx) {
>-		seq_puts(m, "render context ");
>-		describe_obj(m, dev_priv->ips.renderctx);
>-		seq_putc(m, '\n');
>-	}
>-
> 	list_for_each_entry(ctx, &dev_priv->context_list, link) {
>-		if (ctx->legacy_hw_ctx.rcs_state == NULL)
>-			continue;
>-
> 		seq_puts(m, "HW context ");
> 		describe_ctx(m, ctx);
>-		for_each_ring(ring, dev_priv, i)
>-			if (ring->default_context == ctx)
>-				seq_printf(m, "(default context %s) ", ring->name);
>+		for_each_engine(engine, dev_priv, i) {
>+			if (engine->default_context == ctx)
>+				seq_printf(m, "(default context %s) ",
>+					   engine->name);
>+		}
>+
>+		seq_putc(m, '\n');
>+		for_each_engine(engine, dev_priv, i) {
>+			struct drm_i915_gem_object *obj = ctx->ring[i].state;
>+			struct intel_ringbuffer *ring = ctx->ring[i].ring;
>+
>+			seq_printf(m, "%s: ", engine->name);
>+			if (obj)
>+				describe_obj(m, obj);
>+			if (ring)
>+				describe_ring(m, ring);
>+			seq_putc(m, '\n');
>+		}
> 
>-		describe_obj(m, ctx->legacy_hw_ctx.rcs_state);
> 		seq_putc(m, '\n');
> 	}
> 
>@@ -1774,7 +1860,13 @@ static int per_file_ctx(int id, void *ptr, void *data)
> {
> 	struct intel_context *ctx = ptr;
> 	struct seq_file *m = data;
>-	struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(ctx);
>+	struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
>+
>+	if (!ppgtt) {
>+		seq_printf(m, "  no ppgtt for context %d\n",
>+			   ctx->user_handle);
>+		return 0;
>+	}
> 
> 	if (i915_gem_context_is_default(ctx))
> 		seq_puts(m, "  default context:\n");
>@@ -1788,7 +1880,7 @@ static int per_file_ctx(int id, void *ptr, void *data)
> static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
> 	int unused, i;
> 
>@@ -1797,13 +1889,13 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)
> 
> 	seq_printf(m, "Page directories: %d\n", ppgtt->num_pd_pages);
> 	seq_printf(m, "Page tables: %d\n", ppgtt->num_pd_entries);
>-	for_each_ring(ring, dev_priv, unused) {
>-		seq_printf(m, "%s\n", ring->name);
>+	for_each_engine(engine, dev_priv, unused) {
>+		seq_printf(m, "%s\n", engine->name);
> 		for (i = 0; i < 4; i++) {
> 			u32 offset = 0x270 + i * 8;
>-			u64 pdp = I915_READ(ring->mmio_base + offset + 4);
>+			u64 pdp = I915_READ(engine->mmio_base + offset + 4);
> 			pdp <<= 32;
>-			pdp |= I915_READ(ring->mmio_base + offset);
>+			pdp |= I915_READ(engine->mmio_base + offset);
> 			seq_printf(m, "\tPDP%d 0x%016llx\n", i, pdp);
> 		}
> 	}
>@@ -1812,20 +1904,20 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)
> static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	struct drm_file *file;
> 	int i;
> 
> 	if (INTEL_INFO(dev)->gen == 6)
> 		seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(GFX_MODE));
> 
>-	for_each_ring(ring, dev_priv, i) {
>-		seq_printf(m, "%s\n", ring->name);
>+	for_each_engine(engine, dev_priv, i) {
>+		seq_printf(m, "%s\n", engine->name);
> 		if (INTEL_INFO(dev)->gen == 7)
>-			seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(RING_MODE_GEN7(ring)));
>-		seq_printf(m, "PP_DIR_BASE: 0x%08x\n", I915_READ(RING_PP_DIR_BASE(ring)));
>-		seq_printf(m, "PP_DIR_BASE_READ: 0x%08x\n", I915_READ(RING_PP_DIR_BASE_READ(ring)));
>-		seq_printf(m, "PP_DIR_DCLV: 0x%08x\n", I915_READ(RING_PP_DIR_DCLV(ring)));
>+			seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(RING_MODE_GEN7(engine)));
>+		seq_printf(m, "PP_DIR_BASE: 0x%08x\n", I915_READ(RING_PP_DIR_BASE(engine)));
>+		seq_printf(m, "PP_DIR_BASE_READ: 0x%08x\n", I915_READ(RING_PP_DIR_BASE_READ(engine)));
>+		seq_printf(m, "PP_DIR_DCLV: 0x%08x\n", I915_READ(RING_PP_DIR_DCLV(engine)));
> 	}
> 	if (dev_priv->mm.aliasing_ppgtt) {
> 		struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
>@@ -1834,8 +1926,7 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)
> 		seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd_offset);
> 
> 		ppgtt->debug_dump(ppgtt, m);
>-	} else
>-		return;
>+	}
> 
> 	list_for_each_entry_reverse(file, &dev->filelist, lhead) {
> 		struct drm_i915_file_private *file_priv = file->driver_priv;
>@@ -2313,67 +2404,62 @@ static int i915_semaphore_status(struct seq_file *m, void *unused)
> 	struct drm_info_node *node = (struct drm_info_node *) m->private;
> 	struct drm_device *dev = node->minor->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	int num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
> 	int i, j, ret;
> 
>-	if (!i915_semaphore_is_enabled(dev)) {
>-		seq_puts(m, "Semaphores are disabled\n");
>-		return 0;
>-	}
>-
> 	ret = mutex_lock_interruptible(&dev->struct_mutex);
> 	if (ret)
> 		return ret;
> 	intel_runtime_pm_get(dev_priv);
> 
>-	if (IS_BROADWELL(dev)) {
>-		struct page *page;
>-		uint64_t *seqno;
>+	seq_puts(m, "  Last breadcrumb:");
>+	for_each_engine(engine, dev_priv, i)
>+		for (j = 0; j < num_rings; j++)
>+			seq_printf(m, "0x%08x\n",
>+				   engine->breadcrumb[j]);
>+	seq_putc(m, '\n');
> 
>-		page = i915_gem_object_get_page(dev_priv->semaphore_obj, 0);
>+	if (engine->semaphore.wait) {
>+		if (IS_BROADWELL(dev)) {
>+			struct page *page;
>+			uint64_t *seqno;
> 
>-		seqno = (uint64_t *)kmap_atomic(page);
>-		for_each_ring(ring, dev_priv, i) {
>-			uint64_t offset;
>+			page = i915_gem_object_get_page(dev_priv->semaphore_obj, 0);
> 
>-			seq_printf(m, "%s\n", ring->name);
>+			seqno = (uint64_t *)kmap_atomic(page);
>+			for_each_engine(engine, dev_priv, i) {
>+				uint64_t offset;
> 
>-			seq_puts(m, "  Last signal:");
>-			for (j = 0; j < num_rings; j++) {
>-				offset = i * I915_NUM_RINGS + j;
>-				seq_printf(m, "0x%08llx (0x%02llx) ",
>-					   seqno[offset], offset * 8);
>-			}
>-			seq_putc(m, '\n');
>+				seq_printf(m, "%s\n", engine->name);
> 
>-			seq_puts(m, "  Last wait:  ");
>-			for (j = 0; j < num_rings; j++) {
>-				offset = i + (j * I915_NUM_RINGS);
>-				seq_printf(m, "0x%08llx (0x%02llx) ",
>-					   seqno[offset], offset * 8);
>-			}
>-			seq_putc(m, '\n');
>+				seq_puts(m, "  Last signal:");
>+				for (j = 0; j < num_rings; j++) {
>+					offset = i * I915_NUM_ENGINES + j;
>+					seq_printf(m, "0x%08llx (0x%02llx) ",
>+						   seqno[offset], offset * 8);
>+				}
>+				seq_putc(m, '\n');
> 
>-		}
>-		kunmap_atomic(seqno);
>-	} else {
>-		seq_puts(m, "  Last signal:");
>-		for_each_ring(ring, dev_priv, i)
>-			for (j = 0; j < num_rings; j++)
>-				seq_printf(m, "0x%08x\n",
>-					   I915_READ(ring->semaphore.mbox.signal[j]));
>-		seq_putc(m, '\n');
>-	}
>+				seq_puts(m, "  Last wait:  ");
>+				for (j = 0; j < num_rings; j++) {
>+					offset = i + (j * I915_NUM_ENGINES);
>+					seq_printf(m, "0x%08llx (0x%02llx) ",
>+						   seqno[offset], offset * 8);
>+				}
>+				seq_putc(m, '\n');
> 
>-	seq_puts(m, "\nSync seqno:\n");
>-	for_each_ring(ring, dev_priv, i) {
>-		for (j = 0; j < num_rings; j++) {
>-			seq_printf(m, "  0x%08x ", ring->semaphore.sync_seqno[j]);
>+			}
>+			kunmap_atomic(seqno);
>+		} else {
>+			seq_puts(m, "  Last signal:");
>+			for_each_engine(engine, dev_priv, i)
>+				for (j = 0; j < num_rings; j++)
>+					seq_printf(m, "0x%08x\n",
>+						   I915_READ(engine->semaphore.mbox.signal[j]));
>+			seq_putc(m, '\n');
> 		}
>-		seq_putc(m, '\n');
> 	}
>-	seq_putc(m, '\n');
> 
> 	intel_runtime_pm_put(dev_priv);
> 	mutex_unlock(&dev->struct_mutex);
>@@ -2406,6 +2492,40 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused)
> 	return 0;
> }
> 
>+static int i915_wa_registers(struct seq_file *m, void *unused)
>+{
>+	int i;
>+	int ret;
>+	struct drm_info_node *node = (struct drm_info_node *) m->private;
>+	struct drm_device *dev = node->minor->dev;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+
>+	ret = mutex_lock_interruptible(&dev->struct_mutex);
>+	if (ret)
>+		return ret;
>+
>+	intel_runtime_pm_get(dev_priv);
>+
>+	seq_printf(m, "Workarounds applied: %d\n", dev_priv->num_wa_regs);
>+	for (i = 0; i < dev_priv->num_wa_regs; ++i) {
>+		u32 addr, mask;
>+
>+		addr = dev_priv->intel_wa_regs[i].addr;
>+		mask = dev_priv->intel_wa_regs[i].mask;
>+		dev_priv->intel_wa_regs[i].value = I915_READ(addr) | mask;
>+		if (dev_priv->intel_wa_regs[i].addr)
>+			seq_printf(m, "0x%X: 0x%08X, mask: 0x%08X\n",
>+				   dev_priv->intel_wa_regs[i].addr,
>+				   dev_priv->intel_wa_regs[i].value,
>+				   dev_priv->intel_wa_regs[i].mask);
>+	}
>+
>+	intel_runtime_pm_put(dev_priv);
>+	mutex_unlock(&dev->struct_mutex);
>+
>+	return 0;
>+}
>+
> struct pipe_crc_info {
> 	const char *name;
> 	struct drm_device *dev;
>@@ -2667,8 +2787,7 @@ static int i9xx_pipe_crc_auto_source(struct drm_device *dev, enum pipe pipe,
> 	*source = INTEL_PIPE_CRC_SOURCE_PIPE;
> 
> 	drm_modeset_lock_all(dev);
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>-			    base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		if (!encoder->base.crtc)
> 			continue;
> 
>@@ -3557,9 +3676,6 @@ i915_drop_caches_set(void *data, u64 val)
> {
> 	struct drm_device *dev = data;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_gem_object *obj, *next;
>-	struct i915_address_space *vm;
>-	struct i915_vma *vma, *x;
> 	int ret;
> 
> 	DRM_DEBUG("Dropping caches: 0x%08llx\n", val);
>@@ -3579,29 +3695,11 @@ i915_drop_caches_set(void *data, u64 val)
> 	if (val & (DROP_RETIRE | DROP_ACTIVE))
> 		i915_gem_retire_requests(dev);
> 
>-	if (val & DROP_BOUND) {
>-		list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
>-			list_for_each_entry_safe(vma, x, &vm->inactive_list,
>-						 mm_list) {
>-				if (vma->pin_count)
>-					continue;
>+	if (val & DROP_BOUND)
>+		i915_gem_shrink(dev_priv, LONG_MAX, I915_SHRINK_BOUND);
> 
>-				ret = i915_vma_unbind(vma);
>-				if (ret)
>-					goto unlock;
>-			}
>-		}
>-	}
>-
>-	if (val & DROP_UNBOUND) {
>-		list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
>-					 global_list)
>-			if (obj->pages_pin_count == 0) {
>-				ret = i915_gem_object_put_pages(obj);
>-				if (ret)
>-					goto unlock;
>-			}
>-	}
>+	if (val & DROP_UNBOUND)
>+		i915_gem_shrink(dev_priv, LONG_MAX, I915_SHRINK_UNBOUND);
> 
> unlock:
> 	mutex_unlock(&dev->struct_mutex);
>@@ -3936,6 +4034,7 @@ static const struct drm_info_list i915_debugfs_list[] = {
> 	{"i915_semaphore_status", i915_semaphore_status, 0},
> 	{"i915_shared_dplls_info", i915_shared_dplls_info, 0},
> 	{"i915_dp_mst_info", i915_dp_mst_info, 0},
>+	{"i915_wa_registers", i915_wa_registers, 0},
> };
> #define I915_DEBUGFS_ENTRIES ARRAY_SIZE(i915_debugfs_list)
> 
>@@ -3957,6 +4056,7 @@ static const struct i915_debugfs_files {
> 	{"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
> 	{"i915_spr_wm_latency", &i915_spr_wm_latency_fops},
> 	{"i915_cur_wm_latency", &i915_cur_wm_latency_fops},
>+	{"i915_fbc_false_color", &i915_fbc_fc_fops},
> };
> 
> void intel_display_crc_init(struct drm_device *dev)
>@@ -3964,7 +4064,7 @@ void intel_display_crc_init(struct drm_device *dev)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	enum pipe pipe;
> 
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
> 
> 		pipe_crc->opened = false;
>diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
>index 9933c26..31577d6 100644
>--- a/drivers/gpu/drm/i915/i915_dma.c
>+++ b/drivers/gpu/drm/i915/i915_dma.c
>@@ -28,6 +28,7 @@
> 
> #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
> 
>+#include <linux/async.h>
> #include <drm/drmP.h>
> #include <drm/drm_crtc_helper.h>
> #include <drm/drm_fb_helper.h>
>@@ -48,883 +49,58 @@
> #include <linux/pm_runtime.h>
> #include <linux/oom.h>
> 
>-#define LP_RING(d) (&((struct drm_i915_private *)(d))->ring[RCS])
>-
>-#define BEGIN_LP_RING(n) \
>-	intel_ring_begin(LP_RING(dev_priv), (n))
>-
>-#define OUT_RING(x) \
>-	intel_ring_emit(LP_RING(dev_priv), x)
>-
>-#define ADVANCE_LP_RING() \
>-	__intel_ring_advance(LP_RING(dev_priv))
>-
>-/**
>- * Lock test for when it's just for synchronization of ring access.
>- *
>- * In that case, we don't need to do it when GEM is initialized as nobody else
>- * has access to the ring.
>- */
>-#define RING_LOCK_TEST_WITH_RETURN(dev, file) do {			\
>-	if (LP_RING(dev->dev_private)->buffer->obj == NULL)			\
>-		LOCK_TEST_WITH_RETURN(dev, file);			\
>-} while (0)
>-
>-static inline u32
>-intel_read_legacy_status_page(struct drm_i915_private *dev_priv, int reg)
>-{
>-	if (I915_NEED_GFX_HWS(dev_priv->dev))
>-		return ioread32(dev_priv->dri1.gfx_hws_cpu_addr + reg);
>-	else
>-		return intel_read_status_page(LP_RING(dev_priv), reg);
>-}
>-
>-#define READ_HWSP(dev_priv, reg) intel_read_legacy_status_page(dev_priv, reg)
>-#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX)
>-#define I915_BREADCRUMB_INDEX		0x21
>-
>-void i915_update_dri1_breadcrumb(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_master_private *master_priv;
>-
>-	/*
>-	 * The dri breadcrumb update races against the drm master disappearing.
>-	 * Instead of trying to fix this (this is by far not the only ums issue)
>-	 * just don't do the update in kms mode.
>-	 */
>-	if (drm_core_check_feature(dev, DRIVER_MODESET))
>-		return;
>-
>-	if (dev->primary->master) {
>-		master_priv = dev->primary->master->driver_priv;
>-		if (master_priv->sarea_priv)
>-			master_priv->sarea_priv->last_dispatch =
>-				READ_BREADCRUMB(dev_priv);
>-	}
>-}
>-
>-static void i915_write_hws_pga(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	u32 addr;
>-
>-	addr = dev_priv->status_page_dmah->busaddr;
>-	if (INTEL_INFO(dev)->gen >= 4)
>-		addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
>-	I915_WRITE(HWS_PGA, addr);
>-}
>-
>-/**
>- * Frees the hardware status page, whether it's a physical address or a virtual
>- * address set up by the X Server.
>- */
>-static void i915_free_hws(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = LP_RING(dev_priv);
>-
>-	if (dev_priv->status_page_dmah) {
>-		drm_pci_free(dev, dev_priv->status_page_dmah);
>-		dev_priv->status_page_dmah = NULL;
>-	}
>-
>-	if (ring->status_page.gfx_addr) {
>-		ring->status_page.gfx_addr = 0;
>-		iounmap(dev_priv->dri1.gfx_hws_cpu_addr);
>-	}
>-
>-	/* Need to rewrite hardware status page */
>-	I915_WRITE(HWS_PGA, 0x1ffff000);
>-}
>-
>-void i915_kernel_lost_context(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_master_private *master_priv;
>-	struct intel_engine_cs *ring = LP_RING(dev_priv);
>-	struct intel_ringbuffer *ringbuf = ring->buffer;
>-
>-	/*
>-	 * We should never lose context on the ring with modesetting
>-	 * as we don't expose it to userspace
>-	 */
>-	if (drm_core_check_feature(dev, DRIVER_MODESET))
>-		return;
>-
>-	ringbuf->head = I915_READ_HEAD(ring) & HEAD_ADDR;
>-	ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
>-	ringbuf->space = ringbuf->head - (ringbuf->tail + I915_RING_FREE_SPACE);
>-	if (ringbuf->space < 0)
>-		ringbuf->space += ringbuf->size;
>-
>-	if (!dev->primary->master)
>-		return;
>-
>-	master_priv = dev->primary->master->driver_priv;
>-	if (ringbuf->head == ringbuf->tail && master_priv->sarea_priv)
>-		master_priv->sarea_priv->perf_boxes |= I915_BOX_RING_EMPTY;
>-}
>-
>-static int i915_dma_cleanup(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int i;
>-
>-	/* Make sure interrupts are disabled here because the uninstall ioctl
>-	 * may not have been called from userspace and after dev_private
>-	 * is freed, it's too late.
>-	 */
>-	if (dev->irq_enabled)
>-		drm_irq_uninstall(dev);
>-
>-	mutex_lock(&dev->struct_mutex);
>-	for (i = 0; i < I915_NUM_RINGS; i++)
>-		intel_cleanup_ring_buffer(&dev_priv->ring[i]);
>-	mutex_unlock(&dev->struct_mutex);
>-
>-	/* Clear the HWS virtual address at teardown */
>-	if (I915_NEED_GFX_HWS(dev))
>-		i915_free_hws(dev);
>-
>-	return 0;
>-}
>-
>-static int i915_initialize(struct drm_device *dev, drm_i915_init_t *init)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
>-	int ret;
>-
>-	master_priv->sarea = drm_getsarea(dev);
>-	if (master_priv->sarea) {
>-		master_priv->sarea_priv = (drm_i915_sarea_t *)
>-			((u8 *)master_priv->sarea->handle + init->sarea_priv_offset);
>-	} else {
>-		DRM_DEBUG_DRIVER("sarea not found assuming DRI2 userspace\n");
>-	}
>-
>-	if (init->ring_size != 0) {
>-		if (LP_RING(dev_priv)->buffer->obj != NULL) {
>-			i915_dma_cleanup(dev);
>-			DRM_ERROR("Client tried to initialize ringbuffer in "
>-				  "GEM mode\n");
>-			return -EINVAL;
>-		}
>-
>-		ret = intel_render_ring_init_dri(dev,
>-						 init->ring_start,
>-						 init->ring_size);
>-		if (ret) {
>-			i915_dma_cleanup(dev);
>-			return ret;
>-		}
>-	}
>-
>-	dev_priv->dri1.cpp = init->cpp;
>-	dev_priv->dri1.back_offset = init->back_offset;
>-	dev_priv->dri1.front_offset = init->front_offset;
>-	dev_priv->dri1.current_page = 0;
>-	if (master_priv->sarea_priv)
>-		master_priv->sarea_priv->pf_current_page = 0;
>-
>-	/* Allow hardware batchbuffers unless told otherwise.
>-	 */
>-	dev_priv->dri1.allow_batchbuffer = 1;
>-
>-	return 0;
>-}
>-
>-static int i915_dma_resume(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = LP_RING(dev_priv);
>-
>-	DRM_DEBUG_DRIVER("%s\n", __func__);
>-
>-	if (ring->buffer->virtual_start == NULL) {
>-		DRM_ERROR("can not ioremap virtual address for"
>-			  " ring buffer\n");
>-		return -ENOMEM;
>-	}
>-
>-	/* Program Hardware Status Page */
>-	if (!ring->status_page.page_addr) {
>-		DRM_ERROR("Can not find hardware status page\n");
>-		return -EINVAL;
>-	}
>-	DRM_DEBUG_DRIVER("hw status page @ %p\n",
>-				ring->status_page.page_addr);
>-	if (ring->status_page.gfx_addr != 0)
>-		intel_ring_setup_status_page(ring);
>-	else
>-		i915_write_hws_pga(dev);
>-
>-	DRM_DEBUG_DRIVER("Enabled hardware status page\n");
>-
>-	return 0;
>-}
>-
> static int i915_dma_init(struct drm_device *dev, void *data,
> 			 struct drm_file *file_priv)
> {
>-	drm_i915_init_t *init = data;
>-	int retcode = 0;
>-
>-	if (drm_core_check_feature(dev, DRIVER_MODESET))
>-		return -ENODEV;
>-
>-	switch (init->func) {
>-	case I915_INIT_DMA:
>-		retcode = i915_initialize(dev, init);
>-		break;
>-	case I915_CLEANUP_DMA:
>-		retcode = i915_dma_cleanup(dev);
>-		break;
>-	case I915_RESUME_DMA:
>-		retcode = i915_dma_resume(dev);
>-		break;
>-	default:
>-		retcode = -EINVAL;
>-		break;
>-	}
>-
>-	return retcode;
>-}
>-
>-/* Implement basically the same security restrictions as hardware does
>- * for MI_BATCH_NON_SECURE.  These can be made stricter at any time.
>- *
>- * Most of the calculations below involve calculating the size of a
>- * particular instruction.  It's important to get the size right as
>- * that tells us where the next instruction to check is.  Any illegal
>- * instruction detected will be given a size of zero, which is a
>- * signal to abort the rest of the buffer.
>- */
>-static int validate_cmd(int cmd)
>-{
>-	switch (((cmd >> 29) & 0x7)) {
>-	case 0x0:
>-		switch ((cmd >> 23) & 0x3f) {
>-		case 0x0:
>-			return 1;	/* MI_NOOP */
>-		case 0x4:
>-			return 1;	/* MI_FLUSH */
>-		default:
>-			return 0;	/* disallow everything else */
>-		}
>-		break;
>-	case 0x1:
>-		return 0;	/* reserved */
>-	case 0x2:
>-		return (cmd & 0xff) + 2;	/* 2d commands */
>-	case 0x3:
>-		if (((cmd >> 24) & 0x1f) <= 0x18)
>-			return 1;
>-
>-		switch ((cmd >> 24) & 0x1f) {
>-		case 0x1c:
>-			return 1;
>-		case 0x1d:
>-			switch ((cmd >> 16) & 0xff) {
>-			case 0x3:
>-				return (cmd & 0x1f) + 2;
>-			case 0x4:
>-				return (cmd & 0xf) + 2;
>-			default:
>-				return (cmd & 0xffff) + 2;
>-			}
>-		case 0x1e:
>-			if (cmd & (1 << 23))
>-				return (cmd & 0xffff) + 1;
>-			else
>-				return 1;
>-		case 0x1f:
>-			if ((cmd & (1 << 23)) == 0)	/* inline vertices */
>-				return (cmd & 0x1ffff) + 2;
>-			else if (cmd & (1 << 17))	/* indirect random */
>-				if ((cmd & 0xffff) == 0)
>-					return 0;	/* unknown length, too hard */
>-				else
>-					return (((cmd & 0xffff) + 1) / 2) + 1;
>-			else
>-				return 2;	/* indirect sequential */
>-		default:
>-			return 0;
>-		}
>-	default:
>-		return 0;
>-	}
>-
>-	return 0;
>-}
>-
>-static int i915_emit_cmds(struct drm_device *dev, int *buffer, int dwords)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int i, ret;
>-
>-	if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->buffer->size - 8)
>-		return -EINVAL;
>-
>-	for (i = 0; i < dwords;) {
>-		int sz = validate_cmd(buffer[i]);
>-
>-		if (sz == 0 || i + sz > dwords)
>-			return -EINVAL;
>-		i += sz;
>-	}
>-
>-	ret = BEGIN_LP_RING((dwords+1)&~1);
>-	if (ret)
>-		return ret;
>-
>-	for (i = 0; i < dwords; i++)
>-		OUT_RING(buffer[i]);
>-	if (dwords & 1)
>-		OUT_RING(0);
>-
>-	ADVANCE_LP_RING();
>-
>-	return 0;
>-}
>-
>-int
>-i915_emit_box(struct drm_device *dev,
>-	      struct drm_clip_rect *box,
>-	      int DR1, int DR4)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int ret;
>-
>-	if (box->y2 <= box->y1 || box->x2 <= box->x1 ||
>-	    box->y2 <= 0 || box->x2 <= 0) {
>-		DRM_ERROR("Bad box %d,%d..%d,%d\n",
>-			  box->x1, box->y1, box->x2, box->y2);
>-		return -EINVAL;
>-	}
>-
>-	if (INTEL_INFO(dev)->gen >= 4) {
>-		ret = BEGIN_LP_RING(4);
>-		if (ret)
>-			return ret;
>-
>-		OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
>-		OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
>-		OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
>-		OUT_RING(DR4);
>-	} else {
>-		ret = BEGIN_LP_RING(6);
>-		if (ret)
>-			return ret;
>-
>-		OUT_RING(GFX_OP_DRAWRECT_INFO);
>-		OUT_RING(DR1);
>-		OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
>-		OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
>-		OUT_RING(DR4);
>-		OUT_RING(0);
>-	}
>-	ADVANCE_LP_RING();
>-
>-	return 0;
>-}
>-
>-/* XXX: Emitting the counter should really be moved to part of the IRQ
>- * emit. For now, do it in both places:
>- */
>-
>-static void i915_emit_breadcrumb(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
>-
>-	dev_priv->dri1.counter++;
>-	if (dev_priv->dri1.counter > 0x7FFFFFFFUL)
>-		dev_priv->dri1.counter = 0;
>-	if (master_priv->sarea_priv)
>-		master_priv->sarea_priv->last_enqueue = dev_priv->dri1.counter;
>-
>-	if (BEGIN_LP_RING(4) == 0) {
>-		OUT_RING(MI_STORE_DWORD_INDEX);
>-		OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
>-		OUT_RING(dev_priv->dri1.counter);
>-		OUT_RING(0);
>-		ADVANCE_LP_RING();
>-	}
>-}
>-
>-static int i915_dispatch_cmdbuffer(struct drm_device *dev,
>-				   drm_i915_cmdbuffer_t *cmd,
>-				   struct drm_clip_rect *cliprects,
>-				   void *cmdbuf)
>-{
>-	int nbox = cmd->num_cliprects;
>-	int i = 0, count, ret;
>-
>-	if (cmd->sz & 0x3) {
>-		DRM_ERROR("alignment");
>-		return -EINVAL;
>-	}
>-
>-	i915_kernel_lost_context(dev);
>-
>-	count = nbox ? nbox : 1;
>-
>-	for (i = 0; i < count; i++) {
>-		if (i < nbox) {
>-			ret = i915_emit_box(dev, &cliprects[i],
>-					    cmd->DR1, cmd->DR4);
>-			if (ret)
>-				return ret;
>-		}
>-
>-		ret = i915_emit_cmds(dev, cmdbuf, cmd->sz / 4);
>-		if (ret)
>-			return ret;
>-	}
>-
>-	i915_emit_breadcrumb(dev);
>-	return 0;
>-}
>-
>-static int i915_dispatch_batchbuffer(struct drm_device *dev,
>-				     drm_i915_batchbuffer_t *batch,
>-				     struct drm_clip_rect *cliprects)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int nbox = batch->num_cliprects;
>-	int i, count, ret;
>-
>-	if ((batch->start | batch->used) & 0x7) {
>-		DRM_ERROR("alignment");
>-		return -EINVAL;
>-	}
>-
>-	i915_kernel_lost_context(dev);
>-
>-	count = nbox ? nbox : 1;
>-	for (i = 0; i < count; i++) {
>-		if (i < nbox) {
>-			ret = i915_emit_box(dev, &cliprects[i],
>-					    batch->DR1, batch->DR4);
>-			if (ret)
>-				return ret;
>-		}
>-
>-		if (!IS_I830(dev) && !IS_845G(dev)) {
>-			ret = BEGIN_LP_RING(2);
>-			if (ret)
>-				return ret;
>-
>-			if (INTEL_INFO(dev)->gen >= 4) {
>-				OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965);
>-				OUT_RING(batch->start);
>-			} else {
>-				OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
>-				OUT_RING(batch->start | MI_BATCH_NON_SECURE);
>-			}
>-		} else {
>-			ret = BEGIN_LP_RING(4);
>-			if (ret)
>-				return ret;
>-
>-			OUT_RING(MI_BATCH_BUFFER);
>-			OUT_RING(batch->start | MI_BATCH_NON_SECURE);
>-			OUT_RING(batch->start + batch->used - 4);
>-			OUT_RING(0);
>-		}
>-		ADVANCE_LP_RING();
>-	}
>-
>-
>-	if (IS_G4X(dev) || IS_GEN5(dev)) {
>-		if (BEGIN_LP_RING(2) == 0) {
>-			OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP);
>-			OUT_RING(MI_NOOP);
>-			ADVANCE_LP_RING();
>-		}
>-	}
>-
>-	i915_emit_breadcrumb(dev);
>-	return 0;
>-}
>-
>-static int i915_dispatch_flip(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_master_private *master_priv =
>-		dev->primary->master->driver_priv;
>-	int ret;
>-
>-	if (!master_priv->sarea_priv)
>-		return -EINVAL;
>-
>-	DRM_DEBUG_DRIVER("%s: page=%d pfCurrentPage=%d\n",
>-			  __func__,
>-			 dev_priv->dri1.current_page,
>-			 master_priv->sarea_priv->pf_current_page);
>-
>-	i915_kernel_lost_context(dev);
>-
>-	ret = BEGIN_LP_RING(10);
>-	if (ret)
>-		return ret;
>-
>-	OUT_RING(MI_FLUSH | MI_READ_FLUSH);
>-	OUT_RING(0);
>-
>-	OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
>-	OUT_RING(0);
>-	if (dev_priv->dri1.current_page == 0) {
>-		OUT_RING(dev_priv->dri1.back_offset);
>-		dev_priv->dri1.current_page = 1;
>-	} else {
>-		OUT_RING(dev_priv->dri1.front_offset);
>-		dev_priv->dri1.current_page = 0;
>-	}
>-	OUT_RING(0);
>-
>-	OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
>-	OUT_RING(0);
>-
>-	ADVANCE_LP_RING();
>-
>-	master_priv->sarea_priv->last_enqueue = dev_priv->dri1.counter++;
>-
>-	if (BEGIN_LP_RING(4) == 0) {
>-		OUT_RING(MI_STORE_DWORD_INDEX);
>-		OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
>-		OUT_RING(dev_priv->dri1.counter);
>-		OUT_RING(0);
>-		ADVANCE_LP_RING();
>-	}
>-
>-	master_priv->sarea_priv->pf_current_page = dev_priv->dri1.current_page;
>-	return 0;
>-}
>-
>-static int i915_quiescent(struct drm_device *dev)
>-{
>-	i915_kernel_lost_context(dev);
>-	return intel_ring_idle(LP_RING(dev->dev_private));
>+	return -ENODEV;
> }
> 
> static int i915_flush_ioctl(struct drm_device *dev, void *data,
> 			    struct drm_file *file_priv)
> {
>-	int ret;
>-
>-	if (drm_core_check_feature(dev, DRIVER_MODESET))
>-		return -ENODEV;
>-
>-	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
>-
>-	mutex_lock(&dev->struct_mutex);
>-	ret = i915_quiescent(dev);
>-	mutex_unlock(&dev->struct_mutex);
>-
>-	return ret;
>+	return -ENODEV;
> }
> 
> static int i915_batchbuffer(struct drm_device *dev, void *data,
> 			    struct drm_file *file_priv)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_master_private *master_priv;
>-	drm_i915_sarea_t *sarea_priv;
>-	drm_i915_batchbuffer_t *batch = data;
>-	int ret;
>-	struct drm_clip_rect *cliprects = NULL;
>-
>-	if (drm_core_check_feature(dev, DRIVER_MODESET))
>-		return -ENODEV;
>-
>-	master_priv = dev->primary->master->driver_priv;
>-	sarea_priv = (drm_i915_sarea_t *) master_priv->sarea_priv;
>-
>-	if (!dev_priv->dri1.allow_batchbuffer) {
>-		DRM_ERROR("Batchbuffer ioctl disabled\n");
>-		return -EINVAL;
>-	}
>-
>-	DRM_DEBUG_DRIVER("i915 batchbuffer, start %x used %d cliprects %d\n",
>-			batch->start, batch->used, batch->num_cliprects);
>-
>-	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
>-
>-	if (batch->num_cliprects < 0)
>-		return -EINVAL;
>-
>-	if (batch->num_cliprects) {
>-		cliprects = kcalloc(batch->num_cliprects,
>-				    sizeof(*cliprects),
>-				    GFP_KERNEL);
>-		if (cliprects == NULL)
>-			return -ENOMEM;
>-
>-		ret = copy_from_user(cliprects, batch->cliprects,
>-				     batch->num_cliprects *
>-				     sizeof(struct drm_clip_rect));
>-		if (ret != 0) {
>-			ret = -EFAULT;
>-			goto fail_free;
>-		}
>-	}
>-
>-	mutex_lock(&dev->struct_mutex);
>-	ret = i915_dispatch_batchbuffer(dev, batch, cliprects);
>-	mutex_unlock(&dev->struct_mutex);
>-
>-	if (sarea_priv)
>-		sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
>-
>-fail_free:
>-	kfree(cliprects);
>-
>-	return ret;
>+	return -ENODEV;
> }
> 
> static int i915_cmdbuffer(struct drm_device *dev, void *data,
> 			  struct drm_file *file_priv)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_master_private *master_priv;
>-	drm_i915_sarea_t *sarea_priv;
>-	drm_i915_cmdbuffer_t *cmdbuf = data;
>-	struct drm_clip_rect *cliprects = NULL;
>-	void *batch_data;
>-	int ret;
>-
>-	DRM_DEBUG_DRIVER("i915 cmdbuffer, buf %p sz %d cliprects %d\n",
>-			cmdbuf->buf, cmdbuf->sz, cmdbuf->num_cliprects);
>-
>-	if (drm_core_check_feature(dev, DRIVER_MODESET))
>-		return -ENODEV;
>-
>-	master_priv = dev->primary->master->driver_priv;
>-	sarea_priv = (drm_i915_sarea_t *) master_priv->sarea_priv;
>-
>-	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
>-
>-	if (cmdbuf->num_cliprects < 0)
>-		return -EINVAL;
>-
>-	batch_data = kmalloc(cmdbuf->sz, GFP_KERNEL);
>-	if (batch_data == NULL)
>-		return -ENOMEM;
>-
>-	ret = copy_from_user(batch_data, cmdbuf->buf, cmdbuf->sz);
>-	if (ret != 0) {
>-		ret = -EFAULT;
>-		goto fail_batch_free;
>-	}
>-
>-	if (cmdbuf->num_cliprects) {
>-		cliprects = kcalloc(cmdbuf->num_cliprects,
>-				    sizeof(*cliprects), GFP_KERNEL);
>-		if (cliprects == NULL) {
>-			ret = -ENOMEM;
>-			goto fail_batch_free;
>-		}
>-
>-		ret = copy_from_user(cliprects, cmdbuf->cliprects,
>-				     cmdbuf->num_cliprects *
>-				     sizeof(struct drm_clip_rect));
>-		if (ret != 0) {
>-			ret = -EFAULT;
>-			goto fail_clip_free;
>-		}
>-	}
>-
>-	mutex_lock(&dev->struct_mutex);
>-	ret = i915_dispatch_cmdbuffer(dev, cmdbuf, cliprects, batch_data);
>-	mutex_unlock(&dev->struct_mutex);
>-	if (ret) {
>-		DRM_ERROR("i915_dispatch_cmdbuffer failed\n");
>-		goto fail_clip_free;
>-	}
>-
>-	if (sarea_priv)
>-		sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
>-
>-fail_clip_free:
>-	kfree(cliprects);
>-fail_batch_free:
>-	kfree(batch_data);
>-
>-	return ret;
>-}
>-
>-static int i915_emit_irq(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
>-
>-	i915_kernel_lost_context(dev);
>-
>-	DRM_DEBUG_DRIVER("\n");
>-
>-	dev_priv->dri1.counter++;
>-	if (dev_priv->dri1.counter > 0x7FFFFFFFUL)
>-		dev_priv->dri1.counter = 1;
>-	if (master_priv->sarea_priv)
>-		master_priv->sarea_priv->last_enqueue = dev_priv->dri1.counter;
>-
>-	if (BEGIN_LP_RING(4) == 0) {
>-		OUT_RING(MI_STORE_DWORD_INDEX);
>-		OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
>-		OUT_RING(dev_priv->dri1.counter);
>-		OUT_RING(MI_USER_INTERRUPT);
>-		ADVANCE_LP_RING();
>-	}
>-
>-	return dev_priv->dri1.counter;
>-}
>-
>-static int i915_wait_irq(struct drm_device *dev, int irq_nr)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
>-	int ret = 0;
>-	struct intel_engine_cs *ring = LP_RING(dev_priv);
>-
>-	DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr,
>-		  READ_BREADCRUMB(dev_priv));
>-
>-	if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
>-		if (master_priv->sarea_priv)
>-			master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
>-		return 0;
>-	}
>-
>-	if (master_priv->sarea_priv)
>-		master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
>-
>-	if (ring->irq_get(ring)) {
>-		DRM_WAIT_ON(ret, ring->irq_queue, 3 * HZ,
>-			    READ_BREADCRUMB(dev_priv) >= irq_nr);
>-		ring->irq_put(ring);
>-	} else if (wait_for(READ_BREADCRUMB(dev_priv) >= irq_nr, 3000))
>-		ret = -EBUSY;
>-
>-	if (ret == -EBUSY) {
>-		DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
>-			  READ_BREADCRUMB(dev_priv), (int)dev_priv->dri1.counter);
>-	}
>-
>-	return ret;
>+	return -ENODEV;
> }
> 
>-/* Needs the lock as it touches the ring.
>- */
> static int i915_irq_emit(struct drm_device *dev, void *data,
> 			 struct drm_file *file_priv)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	drm_i915_irq_emit_t *emit = data;
>-	int result;
>-
>-	if (drm_core_check_feature(dev, DRIVER_MODESET))
>-		return -ENODEV;
>-
>-	if (!dev_priv || !LP_RING(dev_priv)->buffer->virtual_start) {
>-		DRM_ERROR("called with no initialization\n");
>-		return -EINVAL;
>-	}
>-
>-	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
>-
>-	mutex_lock(&dev->struct_mutex);
>-	result = i915_emit_irq(dev);
>-	mutex_unlock(&dev->struct_mutex);
>-
>-	if (copy_to_user(emit->irq_seq, &result, sizeof(int))) {
>-		DRM_ERROR("copy_to_user\n");
>-		return -EFAULT;
>-	}
>-
>-	return 0;
>+	return -ENODEV;
> }
> 
>-/* Doesn't need the hardware lock.
>- */
> static int i915_irq_wait(struct drm_device *dev, void *data,
> 			 struct drm_file *file_priv)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	drm_i915_irq_wait_t *irqwait = data;
>-
>-	if (drm_core_check_feature(dev, DRIVER_MODESET))
>-		return -ENODEV;
>-
>-	if (!dev_priv) {
>-		DRM_ERROR("called with no initialization\n");
>-		return -EINVAL;
>-	}
>-
>-	return i915_wait_irq(dev, irqwait->irq_seq);
>+	return -ENODEV;
> }
> 
> static int i915_vblank_pipe_get(struct drm_device *dev, void *data,
> 			 struct drm_file *file_priv)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	drm_i915_vblank_pipe_t *pipe = data;
>-
>-	if (drm_core_check_feature(dev, DRIVER_MODESET))
>-		return -ENODEV;
>-
>-	if (!dev_priv) {
>-		DRM_ERROR("called with no initialization\n");
>-		return -EINVAL;
>-	}
>-
>-	pipe->pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
>-
>-	return 0;
>+	return -ENODEV;
> }
> 
>-/**
>- * Schedule buffer swap at given vertical blank.
>- */
> static int i915_vblank_swap(struct drm_device *dev, void *data,
> 		     struct drm_file *file_priv)
> {
>-	/* The delayed swap mechanism was fundamentally racy, and has been
>-	 * removed.  The model was that the client requested a delayed flip/swap
>-	 * from the kernel, then waited for vblank before continuing to perform
>-	 * rendering.  The problem was that the kernel might wake the client
>-	 * up before it dispatched the vblank swap (since the lock has to be
>-	 * held while touching the ringbuffer), in which case the client would
>-	 * clear and start the next frame before the swap occurred, and
>-	 * flicker would occur in addition to likely missing the vblank.
>-	 *
>-	 * In the absence of this ioctl, userland falls back to a correct path
>-	 * of waiting for a vblank, then dispatching the swap on its own.
>-	 * Context switching to userland and back is plenty fast enough for
>-	 * meeting the requirements of vblank swapping.
>-	 */
>-	return -EINVAL;
>+	return -ENODEV;
> }
> 
> static int i915_flip_bufs(struct drm_device *dev, void *data,
> 			  struct drm_file *file_priv)
> {
>-	int ret;
>-
>-	if (drm_core_check_feature(dev, DRIVER_MODESET))
>-		return -ENODEV;
>-
>-	DRM_DEBUG_DRIVER("%s\n", __func__);
>-
>-	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
>-
>-	mutex_lock(&dev->struct_mutex);
>-	ret = i915_dispatch_flip(dev);
>-	mutex_unlock(&dev->struct_mutex);
>-
>-	return ret;
>+	return -ENODEV;
> }
> 
> static int i915_getparam(struct drm_device *dev, void *data,
>@@ -941,14 +117,11 @@ static int i915_getparam(struct drm_device *dev, void *data,
> 
> 	switch (param->param) {
> 	case I915_PARAM_IRQ_ACTIVE:
>-		value = dev->pdev->irq ? 1 : 0;
>-		break;
>+		return -ENODEV;
> 	case I915_PARAM_ALLOW_BATCHBUFFER:
>-		value = dev_priv->dri1.allow_batchbuffer ? 1 : 0;
>-		break;
>+		return -ENODEV;
> 	case I915_PARAM_LAST_DISPATCH:
>-		value = READ_BREADCRUMB(dev_priv);
>-		break;
>+		return -ENODEV;
> 	case I915_PARAM_CHIPSET_ID:
> 		value = dev->pdev->device;
> 		break;
>@@ -969,13 +142,13 @@ static int i915_getparam(struct drm_device *dev, void *data,
> 		value = 1;
> 		break;
> 	case I915_PARAM_HAS_BSD:
>-		value = intel_ring_initialized(&dev_priv->ring[VCS]);
>+		value = intel_engine_initialized(&dev_priv->engine[VCS]);
> 		break;
> 	case I915_PARAM_HAS_BLT:
>-		value = intel_ring_initialized(&dev_priv->ring[BCS]);
>+		value = intel_engine_initialized(&dev_priv->engine[BCS]);
> 		break;
> 	case I915_PARAM_HAS_VEBOX:
>-		value = intel_ring_initialized(&dev_priv->ring[VECS]);
>+		value = intel_engine_initialized(&dev_priv->engine[VECS]);
> 		break;
> 	case I915_PARAM_HAS_RELAXED_FENCING:
> 		value = 1;
>@@ -999,13 +172,13 @@ static int i915_getparam(struct drm_device *dev, void *data,
> 		value = HAS_WT(dev);
> 		break;
> 	case I915_PARAM_HAS_ALIASING_PPGTT:
>-		value = dev_priv->mm.aliasing_ppgtt || USES_FULL_PPGTT(dev);
>+		value = USES_PPGTT(dev);
> 		break;
> 	case I915_PARAM_HAS_WAIT_TIMEOUT:
> 		value = 1;
> 		break;
> 	case I915_PARAM_HAS_SEMAPHORES:
>-		value = i915_semaphore_is_enabled(dev);
>+		value = RCS_ENGINE(dev_priv)->semaphore.wait != NULL;
> 		break;
> 	case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
> 		value = 1;
>@@ -1051,12 +224,10 @@ static int i915_setparam(struct drm_device *dev, void *data,
> 
> 	switch (param->param) {
> 	case I915_SETPARAM_USE_MI_BATCHBUFFER_START:
>-		break;
> 	case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY:
>-		break;
> 	case I915_SETPARAM_ALLOW_BATCHBUFFER:
>-		dev_priv->dri1.allow_batchbuffer = param->value ? 1 : 0;
>-		break;
>+		return -ENODEV;
>+
> 	case I915_SETPARAM_NUM_USED_FENCES:
> 		if (param->value > dev_priv->num_fence_regs ||
> 		    param->value < 0)
>@@ -1076,49 +247,7 @@ static int i915_setparam(struct drm_device *dev, void *data,
> static int i915_set_status_page(struct drm_device *dev, void *data,
> 				struct drm_file *file_priv)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	drm_i915_hws_addr_t *hws = data;
>-	struct intel_engine_cs *ring;
>-
>-	if (drm_core_check_feature(dev, DRIVER_MODESET))
>-		return -ENODEV;
>-
>-	if (!I915_NEED_GFX_HWS(dev))
>-		return -EINVAL;
>-
>-	if (!dev_priv) {
>-		DRM_ERROR("called with no initialization\n");
>-		return -EINVAL;
>-	}
>-
>-	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
>-		WARN(1, "tried to set status page when mode setting active\n");
>-		return 0;
>-	}
>-
>-	DRM_DEBUG_DRIVER("set status page addr 0x%08x\n", (u32)hws->addr);
>-
>-	ring = LP_RING(dev_priv);
>-	ring->status_page.gfx_addr = hws->addr & (0x1ffff<<12);
>-
>-	dev_priv->dri1.gfx_hws_cpu_addr =
>-		ioremap_wc(dev_priv->gtt.mappable_base + hws->addr, 4096);
>-	if (dev_priv->dri1.gfx_hws_cpu_addr == NULL) {
>-		i915_dma_cleanup(dev);
>-		ring->status_page.gfx_addr = 0;
>-		DRM_ERROR("can not ioremap virtual address for"
>-				" G33 hw status page\n");
>-		return -ENOMEM;
>-	}
>-
>-	memset_io(dev_priv->dri1.gfx_hws_cpu_addr, 0, PAGE_SIZE);
>-	I915_WRITE(HWS_PGA, ring->status_page.gfx_addr);
>-
>-	DRM_DEBUG_DRIVER("load hws HWS_PGA with gfx mem 0x%x\n",
>-			 ring->status_page.gfx_addr);
>-	DRM_DEBUG_DRIVER("load hws at %p\n",
>-			 ring->status_page.page_addr);
>-	return 0;
>+	return -ENODEV;
> }
> 
> static int i915_get_bridge_dev(struct drm_device *dev)
>@@ -1355,8 +484,6 @@ static int i915_load_modeset_init(struct drm_device *dev)
> 	if (ret)
> 		goto cleanup_irq;
> 
>-	INIT_WORK(&dev_priv->console_resume_work, intel_console_resume);
>-
> 	intel_modeset_gem_init(dev);
> 
> 	/* Always safe in the mode setting case. */
>@@ -1382,7 +509,7 @@ static int i915_load_modeset_init(struct drm_device *dev)
> 	 * scanning against hotplug events. Hence do this first and ignore the
> 	 * tiny window where we will loose hotplug notifactions.
> 	 */
>-	intel_fbdev_initial_config(dev);
>+	async_schedule(intel_fbdev_initial_config, dev_priv);
> 
> 	drm_kms_helper_poll_init(dev);
> 
>@@ -1390,10 +517,8 @@ static int i915_load_modeset_init(struct drm_device *dev)
> 
> cleanup_gem:
> 	mutex_lock(&dev->struct_mutex);
>-	i915_gem_cleanup_ringbuffer(dev);
>-	i915_gem_context_fini(dev);
>+	i915_gem_fini(dev);
> 	mutex_unlock(&dev->struct_mutex);
>-	WARN_ON(dev_priv->mm.aliasing_ppgtt);
> cleanup_irq:
> 	drm_irq_uninstall(dev);
> cleanup_gem_stolen:
>@@ -1406,30 +531,6 @@ out:
> 	return ret;
> }
> 
>-int i915_master_create(struct drm_device *dev, struct drm_master *master)
>-{
>-	struct drm_i915_master_private *master_priv;
>-
>-	master_priv = kzalloc(sizeof(*master_priv), GFP_KERNEL);
>-	if (!master_priv)
>-		return -ENOMEM;
>-
>-	master->driver_priv = master_priv;
>-	return 0;
>-}
>-
>-void i915_master_destroy(struct drm_device *dev, struct drm_master *master)
>-{
>-	struct drm_i915_master_private *master_priv = master->driver_priv;
>-
>-	if (!master_priv)
>-		return;
>-
>-	kfree(master_priv);
>-
>-	master->driver_priv = NULL;
>-}
>-
> #if IS_ENABLED(CONFIG_FB)
> static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
> {
>@@ -1536,10 +637,10 @@ static void intel_device_info_runtime_init(struct drm_device *dev)
> 	info = (struct intel_device_info *)&dev_priv->info;
> 
> 	if (IS_VALLEYVIEW(dev))
>-		for_each_pipe(pipe)
>+		for_each_pipe(dev_priv, pipe)
> 			info->num_sprites[pipe] = 2;
> 	else
>-		for_each_pipe(pipe)
>+		for_each_pipe(dev_priv, pipe)
> 			info->num_sprites[pipe] = 1;
> 
> 	if (i915.disable_display) {
>@@ -1601,6 +702,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
> 	if (!drm_core_check_feature(dev, DRIVER_MODESET) && !dev->agp)
> 		return -EINVAL;
> 
>+	BUILD_BUG_ON(I915_NUM_ENGINES >= (1 << I915_NUM_ENGINE_BITS));
>+
> 	dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
> 	if (dev_priv == NULL)
> 		return -ENOMEM;
>@@ -1608,13 +711,14 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
> 	dev->dev_private = dev_priv;
> 	dev_priv->dev = dev;
> 
>-	/* copy initial configuration to dev_priv->info */
>+	/* Setup the write-once "constant" device info */
> 	device_info = (struct intel_device_info *)&dev_priv->info;
>-	*device_info = *info;
>+	memcpy(device_info, info, sizeof(dev_priv->info));
>+	device_info->device_id = dev->pdev->device;
> 
> 	spin_lock_init(&dev_priv->irq_lock);
> 	spin_lock_init(&dev_priv->gpu_error.lock);
>-	spin_lock_init(&dev_priv->backlight_lock);
>+	mutex_init(&dev_priv->backlight_lock);
> 	spin_lock_init(&dev_priv->uncore.lock);
> 	spin_lock_init(&dev_priv->mm.object_stat_lock);
> 	spin_lock_init(&dev_priv->mmio_flip_lock);
>@@ -1822,7 +926,7 @@ out_mtrrfree:
> 	arch_phys_wc_del(dev_priv->gtt.mtrr);
> 	io_mapping_free(dev_priv->gtt.mappable);
> out_gtt:
>-	dev_priv->gtt.base.cleanup(&dev_priv->gtt.base);
>+	i915_global_gtt_cleanup(dev);
> out_regs:
> 	intel_uncore_fini(dev);
> 	pci_iounmap(dev->pdev, dev_priv->regs);
>@@ -1869,7 +973,6 @@ int i915_driver_unload(struct drm_device *dev)
> 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
> 		intel_fbdev_fini(dev);
> 		intel_modeset_cleanup(dev);
>-		cancel_work_sync(&dev_priv->console_resume_work);
> 
> 		/*
> 		 * free the memory space allocated for the child device
>@@ -1900,18 +1003,11 @@ int i915_driver_unload(struct drm_device *dev)
> 		flush_workqueue(dev_priv->wq);
> 
> 		mutex_lock(&dev->struct_mutex);
>-		i915_gem_cleanup_ringbuffer(dev);
>-		i915_gem_context_fini(dev);
>-		WARN_ON(dev_priv->mm.aliasing_ppgtt);
>+		i915_gem_fini(dev);
> 		mutex_unlock(&dev->struct_mutex);
> 		i915_gem_cleanup_stolen(dev);
>-
>-		if (!I915_NEED_GFX_HWS(dev))
>-			i915_free_hws(dev);
> 	}
> 
>-	WARN_ON(!list_empty(&dev_priv->vm_list));
>-
> 	drm_vblank_cleanup(dev);
> 
> 	intel_teardown_gmbus(dev);
>@@ -1921,7 +1017,7 @@ int i915_driver_unload(struct drm_device *dev)
> 	destroy_workqueue(dev_priv->wq);
> 	pm_qos_remove_request(&dev_priv->pm_qos);
> 
>-	dev_priv->gtt.base.cleanup(&dev_priv->gtt.base);
>+	i915_global_gtt_cleanup(dev);
> 
> 	intel_uncore_fini(dev);
> 	if (dev_priv->regs != NULL)
>@@ -1976,8 +1072,6 @@ void i915_driver_lastclose(struct drm_device *dev)
> 	}
> 
> 	i915_gem_lastclose(dev);
>-
>-	i915_dma_cleanup(dev);
> }
> 
> void i915_driver_preclose(struct drm_device *dev, struct drm_file *file)
>@@ -1986,14 +1080,15 @@ void i915_driver_preclose(struct drm_device *dev, struct drm_file *file)
> 	i915_gem_context_close(dev, file);
> 	i915_gem_release(dev, file);
> 	mutex_unlock(&dev->struct_mutex);
>+
>+	if (drm_core_check_feature(dev, DRIVER_MODESET))
>+		intel_modeset_preclose(dev, file);
> }
> 
> void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
> {
> 	struct drm_i915_file_private *file_priv = file->driver_priv;
> 
>-	if (file_priv && file_priv->bsd_ring)
>-		file_priv->bsd_ring = NULL;
> 	kfree(file_priv);
> }
> 
>diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
>index e27cdbe..67084ee 100644
>--- a/drivers/gpu/drm/i915/i915_drv.c
>+++ b/drivers/gpu/drm/i915/i915_drv.c
>@@ -473,26 +473,6 @@ void intel_detect_pch(struct drm_device *dev)
> 	pci_dev_put(pch);
> }
> 
>-bool i915_semaphore_is_enabled(struct drm_device *dev)
>-{
>-	if (INTEL_INFO(dev)->gen < 6)
>-		return false;
>-
>-	if (i915.semaphores >= 0)
>-		return i915.semaphores;
>-
>-	/* Until we get further testing... */
>-	if (IS_GEN8(dev))
>-		return false;
>-
>-#ifdef CONFIG_INTEL_IOMMU
>-	/* Enable semaphores on SNB when IO remapping is off */
>-	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
>-		return false;
>-#endif
>-
>-	return true;
>-}
> 
> void intel_hpd_cancel_work(struct drm_i915_private *dev_priv)
> {
>@@ -505,7 +485,7 @@ void intel_hpd_cancel_work(struct drm_i915_private *dev_priv)
> 	spin_unlock_irq(&dev_priv->irq_lock);
> 
> 	cancel_work_sync(&dev_priv->dig_port_work);
>-	cancel_work_sync(&dev_priv->hotplug_work);
>+	cancel_delayed_work_sync(&dev_priv->hotplug_work);
> 	cancel_delayed_work_sync(&dev_priv->hotplug_reenable_work);
> }
> 
>@@ -524,6 +504,10 @@ static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
> 	drm_modeset_unlock_all(dev);
> }
> 
>+static int intel_suspend_complete(struct drm_i915_private *dev_priv);
>+static int intel_resume_prepare(struct drm_i915_private *dev_priv,
>+				bool rpm_resume);
>+
> static int i915_drm_freeze(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>@@ -591,9 +575,7 @@ static int i915_drm_freeze(struct drm_device *dev)
> 	intel_uncore_forcewake_reset(dev, false);
> 	intel_opregion_fini(dev);
> 
>-	console_lock();
>-	intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED);
>-	console_unlock();
>+	intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true);
> 
> 	dev_priv->suspend_count++;
> 
>@@ -632,30 +614,20 @@ int i915_suspend(struct drm_device *dev, pm_message_t state)
> 	return 0;
> }
> 
>-void intel_console_resume(struct work_struct *work)
>-{
>-	struct drm_i915_private *dev_priv =
>-		container_of(work, struct drm_i915_private,
>-			     console_resume_work);
>-	struct drm_device *dev = dev_priv->dev;
>-
>-	console_lock();
>-	intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING);
>-	console_unlock();
>-}
>-
> static int i915_drm_thaw_early(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>+	int ret;
> 
>-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
>-		hsw_disable_pc8(dev_priv);
>+	ret = intel_resume_prepare(dev_priv, false);
>+	if (ret)
>+		DRM_ERROR("Resume prepare failed: %d,Continuing resume\n", ret);
> 
> 	intel_uncore_early_sanitize(dev, true);
> 	intel_uncore_sanitize(dev);
> 	intel_power_domains_init_hw(dev_priv);
> 
>-	return 0;
>+	return ret;
> }
> 
> static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings)
>@@ -684,16 +656,16 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings)
> 		}
> 		mutex_unlock(&dev->struct_mutex);
> 
>+		/* We need working interrupts for modeset enabling ... */
> 		intel_runtime_pm_restore_interrupts(dev);
> 
> 		intel_modeset_init_hw(dev);
> 
> 		{
>-			unsigned long irqflags;
>-			spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+			spin_lock_irq(&dev_priv->irq_lock);
> 			if (dev_priv->display.hpd_irq_setup)
> 				dev_priv->display.hpd_irq_setup(dev);
>-			spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+			spin_unlock_irq(&dev_priv->irq_lock);
> 		}
> 
> 		intel_dp_mst_resume(dev);
>@@ -714,17 +686,7 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings)
> 
> 	intel_opregion_init(dev);
> 
>-	/*
>-	 * The console lock can be pretty contented on resume due
>-	 * to all the printk activity.  Try to keep it out of the hot
>-	 * path of resume if possible.
>-	 */
>-	if (console_trylock()) {
>-		intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING);
>-		console_unlock();
>-	} else {
>-		schedule_work(&dev_priv->console_resume_work);
>-	}
>+	intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
> 
> 	mutex_lock(&dev_priv->modeset_restore_lock);
> 	dev_priv->modeset_restore = MODESET_DONE;
>@@ -809,7 +771,6 @@ static int i915_resume_legacy(struct drm_device *dev)
> int i915_reset(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	bool simulated;
> 	int ret;
> 
> 	if (!i915.reset)
>@@ -817,14 +778,16 @@ int i915_reset(struct drm_device *dev)
> 
> 	mutex_lock(&dev->struct_mutex);
> 
>-	i915_gem_reset(dev);
>-
>-	simulated = dev_priv->gpu_error.stop_rings != 0;
>-
> 	ret = intel_gpu_reset(dev);
> 
>+	/* Clear the reset counter. Before anyone else
>+	 * can grab the mutex, we will declare whether or
>+	 * not the GPU is wedged.
>+	 */
>+	atomic_inc(&dev_priv->gpu_error.reset_counter);
>+
> 	/* Also reset the gpu hangman. */
>-	if (simulated) {
>+	if (dev_priv->gpu_error.stop_rings) {
> 		DRM_INFO("Simulated gpu hang, resetting stop_rings\n");
> 		dev_priv->gpu_error.stop_rings = 0;
> 		if (ret == -ENODEV) {
>@@ -834,6 +797,8 @@ int i915_reset(struct drm_device *dev)
> 		}
> 	}
> 
>+	i915_gem_reset(dev);
>+
> 	if (ret) {
> 		DRM_ERROR("Failed to reset chip: %i\n", ret);
> 		mutex_unlock(&dev->struct_mutex);
>@@ -857,7 +822,6 @@ int i915_reset(struct drm_device *dev)
> 	if (drm_core_check_feature(dev, DRIVER_MODESET) ||
> 			!dev_priv->ums.mm_suspended) {
> 		dev_priv->ums.mm_suspended = 0;
>-
> 		ret = i915_gem_init_hw(dev);
> 		mutex_unlock(&dev->struct_mutex);
> 		if (ret) {
>@@ -879,8 +843,6 @@ int i915_reset(struct drm_device *dev)
> 		 */
> 		if (INTEL_INFO(dev)->gen > 5)
> 			intel_reset_gt_powersave(dev);
>-
>-		intel_hpd_init(dev);
> 	} else {
> 		mutex_unlock(&dev->struct_mutex);
> 	}
>@@ -941,6 +903,7 @@ static int i915_pm_suspend_late(struct device *dev)
> 	struct pci_dev *pdev = to_pci_dev(dev);
> 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
> 	struct drm_i915_private *dev_priv = drm_dev->dev_private;
>+	int ret;
> 
> 	/*
> 	 * We have a suspedn ordering issue with the snd-hda driver also
>@@ -954,13 +917,16 @@ static int i915_pm_suspend_late(struct device *dev)
> 	if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
> 		return 0;
> 
>-	if (IS_HASWELL(drm_dev) || IS_BROADWELL(drm_dev))
>-		hsw_enable_pc8(dev_priv);
>+	ret = intel_suspend_complete(dev_priv);
> 
>-	pci_disable_device(pdev);
>-	pci_set_power_state(pdev, PCI_D3hot);
>+	if (ret)
>+		DRM_ERROR("Suspend complete failed: %d\n", ret);
>+	else {
>+		pci_disable_device(pdev);
>+		pci_set_power_state(pdev, PCI_D3hot);
>+	}
> 
>-	return 0;
>+	return ret;
> }
> 
> static int i915_pm_resume_early(struct device *dev)
>@@ -1016,23 +982,26 @@ static int i915_pm_poweroff(struct device *dev)
> 	return i915_drm_freeze(drm_dev);
> }
> 
>-static int hsw_runtime_suspend(struct drm_i915_private *dev_priv)
>+static int hsw_suspend_complete(struct drm_i915_private *dev_priv)
> {
> 	hsw_enable_pc8(dev_priv);
> 
> 	return 0;
> }
> 
>-static int snb_runtime_resume(struct drm_i915_private *dev_priv)
>+static int snb_resume_prepare(struct drm_i915_private *dev_priv,
>+				bool rpm_resume)
> {
> 	struct drm_device *dev = dev_priv->dev;
> 
>-	intel_init_pch_refclk(dev);
>+	if (rpm_resume)
>+		intel_init_pch_refclk(dev);
> 
> 	return 0;
> }
> 
>-static int hsw_runtime_resume(struct drm_i915_private *dev_priv)
>+static int hsw_resume_prepare(struct drm_i915_private *dev_priv,
>+				bool rpm_resume)
> {
> 	hsw_disable_pc8(dev_priv);
> 
>@@ -1328,7 +1297,7 @@ static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
> 	I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
> }
> 
>-static int vlv_runtime_suspend(struct drm_i915_private *dev_priv)
>+static int vlv_suspend_complete(struct drm_i915_private *dev_priv)
> {
> 	u32 mask;
> 	int err;
>@@ -1368,7 +1337,8 @@ err1:
> 	return err;
> }
> 
>-static int vlv_runtime_resume(struct drm_i915_private *dev_priv)
>+static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
>+				bool rpm_resume)
> {
> 	struct drm_device *dev = dev_priv->dev;
> 	int err;
>@@ -1393,8 +1363,10 @@ static int vlv_runtime_resume(struct drm_i915_private *dev_priv)
> 
> 	vlv_check_no_gt_access(dev_priv);
> 
>-	intel_init_clock_gating(dev);
>-	i915_gem_restore_fences(dev);
>+	if (rpm_resume) {
>+		intel_init_clock_gating(dev);
>+		i915_gem_restore_fences(dev);
>+	}
> 
> 	return ret;
> }
>@@ -1409,7 +1381,9 @@ static int intel_runtime_suspend(struct device *device)
> 	if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6(dev))))
> 		return -ENODEV;
> 
>-	WARN_ON(!HAS_RUNTIME_PM(dev));
>+	if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
>+		return -ENODEV;
>+
> 	assert_force_wake_inactive(dev_priv);
> 
> 	DRM_DEBUG_KMS("Suspending device\n");
>@@ -1446,17 +1420,7 @@ static int intel_runtime_suspend(struct device *device)
> 	cancel_work_sync(&dev_priv->rps.work);
> 	intel_runtime_pm_disable_interrupts(dev);
> 
>-	if (IS_GEN6(dev)) {
>-		ret = 0;
>-	} else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
>-		ret = hsw_runtime_suspend(dev_priv);
>-	} else if (IS_VALLEYVIEW(dev)) {
>-		ret = vlv_runtime_suspend(dev_priv);
>-	} else {
>-		ret = -ENODEV;
>-		WARN_ON(1);
>-	}
>-
>+	ret = intel_suspend_complete(dev_priv);
> 	if (ret) {
> 		DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
> 		intel_runtime_pm_restore_interrupts(dev);
>@@ -1468,13 +1432,29 @@ static int intel_runtime_suspend(struct device *device)
> 	dev_priv->pm.suspended = true;
> 
> 	/*
>-	 * current versions of firmware which depend on this opregion
>-	 * notification have repurposed the D1 definition to mean
>-	 * "runtime suspended" vs. what you would normally expect (D3)
>-	 * to distinguish it from notifications that might be sent
>-	 * via the suspend path.
>+	 * FIXME: We really should find a document that references the arguments
>+	 * used below!
> 	 */
>-	intel_opregion_notify_adapter(dev, PCI_D1);
>+	if (IS_HASWELL(dev)) {
>+		/*
>+		 * current versions of firmware which depend on this opregion
>+		 * notification have repurposed the D1 definition to mean
>+		 * "runtime suspended" vs. what you would normally expect (D3)
>+		 * to distinguish it from notifications that might be sent via
>+		 * the suspend path.
>+		 */
>+		intel_opregion_notify_adapter(dev, PCI_D1);
>+	} else {
>+		/*
>+		 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
>+		 * being detected, and the call we do at intel_runtime_resume()
>+		 * won't be able to restore them. Since PCI_D3hot matches the
>+		 * actual specification and appears to be working, use it. Let's
>+		 * assume the other non-Haswell platforms will stay the same as
>+		 * Broadwell.
>+		 */
>+		intel_opregion_notify_adapter(dev, PCI_D3hot);
>+	}
> 
> 	DRM_DEBUG_KMS("Device suspended\n");
> 	return 0;
>@@ -1487,24 +1467,15 @@ static int intel_runtime_resume(struct device *device)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	int ret;
> 
>-	WARN_ON(!HAS_RUNTIME_PM(dev));
>+	if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
>+		return -ENODEV;
> 
> 	DRM_DEBUG_KMS("Resuming device\n");
> 
> 	intel_opregion_notify_adapter(dev, PCI_D0);
> 	dev_priv->pm.suspended = false;
> 
>-	if (IS_GEN6(dev)) {
>-		ret = snb_runtime_resume(dev_priv);
>-	} else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
>-		ret = hsw_runtime_resume(dev_priv);
>-	} else if (IS_VALLEYVIEW(dev)) {
>-		ret = vlv_runtime_resume(dev_priv);
>-	} else {
>-		WARN_ON(1);
>-		ret = -ENODEV;
>-	}
>-
>+	ret = intel_resume_prepare(dev_priv, true);
> 	/*
> 	 * No point of rolling back things in case of an error, as the best
> 	 * we can do is to hope that things will still work (and disable RPM).
>@@ -1523,6 +1494,48 @@ static int intel_runtime_resume(struct device *device)
> 	return ret;
> }
> 
>+/*
>+ * This function implements common functionality of runtime and system
>+ * suspend sequence.
>+ */
>+static int intel_suspend_complete(struct drm_i915_private *dev_priv)
>+{
>+	struct drm_device *dev = dev_priv->dev;
>+	int ret;
>+
>+	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
>+		ret = hsw_suspend_complete(dev_priv);
>+	else if (IS_VALLEYVIEW(dev))
>+		ret = vlv_suspend_complete(dev_priv);
>+	else
>+		ret = 0;
>+
>+	return ret;
>+}
>+
>+/*
>+ * This function implements common functionality of runtime and system
>+ * resume sequence. Variable rpm_resume used for implementing different
>+ * code paths.
>+ */
>+static int intel_resume_prepare(struct drm_i915_private *dev_priv,
>+				bool rpm_resume)
>+{
>+	struct drm_device *dev = dev_priv->dev;
>+	int ret;
>+
>+	if (IS_GEN6(dev))
>+		ret = snb_resume_prepare(dev_priv, rpm_resume);
>+	else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
>+		ret = hsw_resume_prepare(dev_priv, rpm_resume);
>+	else if (IS_VALLEYVIEW(dev))
>+		ret = vlv_resume_prepare(dev_priv, rpm_resume);
>+	else
>+		ret = 0;
>+
>+	return ret;
>+}
>+
> static const struct dev_pm_ops i915_pm_ops = {
> 	.suspend = i915_pm_suspend,
> 	.suspend_late = i915_pm_suspend_late,
>@@ -1578,8 +1591,6 @@ static struct drm_driver driver = {
> 	.resume = i915_resume_legacy,
> 
> 	.device_is_agp = i915_driver_device_is_agp,
>-	.master_create = i915_master_create,
>-	.master_destroy = i915_master_destroy,
> #if defined(CONFIG_DEBUG_FS)
> 	.debugfs_init = i915_debugfs_init,
> 	.debugfs_cleanup = i915_debugfs_cleanup,
>@@ -1663,6 +1674,8 @@ static void __exit i915_exit(void)
> module_init(i915_init);
> module_exit(i915_exit);
> 
>-MODULE_AUTHOR(DRIVER_AUTHOR);
>+MODULE_AUTHOR("Tungsten Graphics, Inc.");
>+MODULE_AUTHOR("Intel Corporation");
>+
> MODULE_DESCRIPTION(DRIVER_DESC);
> MODULE_LICENSE("GPL and additional rights");
>diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
>index 3524306..57f03b0 100644
>--- a/drivers/gpu/drm/i915/i915_drv.h
>+++ b/drivers/gpu/drm/i915/i915_drv.h
>@@ -49,11 +49,9 @@
> /* General customization:
>  */
> 
>-#define DRIVER_AUTHOR		"Tungsten Graphics, Inc."
>-
> #define DRIVER_NAME		"i915"
> #define DRIVER_DESC		"Intel Graphics"
>-#define DRIVER_DATE		"20140725"
>+#define DRIVER_DATE		"20140905"
> 
> enum pipe {
> 	INVALID_PIPE = -1,
>@@ -162,7 +160,10 @@ enum hpd_pin {
> 	 I915_GEM_DOMAIN_INSTRUCTION | \
> 	 I915_GEM_DOMAIN_VERTEX)
> 
>-#define for_each_pipe(p) for ((p) = 0; (p) < INTEL_INFO(dev)->num_pipes; (p)++)
>+#define for_each_pipe(__dev_priv, __p) \
>+	for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++)
>+#define for_each_plane(pipe, p) \
>+	for ((p) = 0; (p) < INTEL_INFO(dev)->num_sprites[(pipe)] + 1; (p)++)
> #define for_each_sprite(p, s) for ((s) = 0; (s) < INTEL_INFO(dev)->num_sprites[(p)]; (s)++)
> 
> #define for_each_crtc(dev, crtc) \
>@@ -171,6 +172,11 @@ enum hpd_pin {
> #define for_each_intel_crtc(dev, intel_crtc) \
> 	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head)
> 
>+#define for_each_intel_encoder(dev, intel_encoder)		\
>+	list_for_each_entry(intel_encoder,			\
>+			    &(dev)->mode_config.encoder_list,	\
>+			    base.head)
>+
> #define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \
> 	list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \
> 		if ((intel_encoder)->base.crtc == (__crtc))
>@@ -186,6 +192,7 @@ enum hpd_pin {
> struct drm_i915_private;
> struct i915_mm_struct;
> struct i915_mmu_object;
>+struct i915_gem_request;
> 
> enum intel_dpll_id {
> 	DPLL_ID_PRIVATE = -1, /* non-shared dpll in use */
>@@ -198,10 +205,13 @@ enum intel_dpll_id {
> #define I915_NUM_PLLS 2
> 
> struct intel_dpll_hw_state {
>+	/* i9xx, pch plls */
> 	uint32_t dpll;
> 	uint32_t dpll_md;
> 	uint32_t fp0;
> 	uint32_t fp1;
>+
>+	/* hsw, bdw */
> 	uint32_t wrpll;
> };
> 
>@@ -277,10 +287,6 @@ struct intel_opregion {
> struct intel_overlay;
> struct intel_overlay_error_state;
> 
>-struct drm_i915_master_private {
>-	drm_local_map_t *sarea;
>-	struct _drm_i915_sarea *sarea_priv;
>-};
> #define I915_FENCE_REG_NONE -1
> #define I915_MAX_NUM_FENCES 32
> /* 32 fences + sign bit for FENCE_REG_NONE */
>@@ -316,6 +322,7 @@ struct drm_i915_error_state {
> 	u32 pgtbl_er;
> 	u32 ier;
> 	u32 gtier[4];
>+	u32 gtimr[4];
> 	u32 ccid;
> 	u32 derrmr;
> 	u32 forcewake;
>@@ -333,23 +340,26 @@ struct drm_i915_error_state {
> 	struct drm_i915_error_object *semaphore_obj;
> 
> 	struct drm_i915_error_ring {
>+		int id;
> 		bool valid;
> 		/* Software tracked state */
> 		bool waiting;
> 		int hangcheck_score;
>-		enum intel_ring_hangcheck_action hangcheck_action;
>+		enum intel_engine_hangcheck_action hangcheck_action;
> 		int num_requests;
> 
> 		/* our own tracking of ring head and tail */
> 		u32 cpu_ring_head;
> 		u32 cpu_ring_tail;
>-
>-		u32 semaphore_seqno[I915_NUM_RINGS - 1];
>+		u32 interrupts;
>+		u32 irq_count;
> 
> 		/* Register state */
> 		u32 tail;
> 		u32 head;
>+		u32 start;
> 		u32 ctl;
>+		u32 mode;
> 		u32 hws;
> 		u32 ipeir;
> 		u32 ipehr;
>@@ -357,13 +367,15 @@ struct drm_i915_error_state {
> 		u32 bbstate;
> 		u32 instpm;
> 		u32 instps;
>-		u32 seqno;
>+		u32 seqno, request, tag, hangcheck;
>+		u32 breadcrumb[I915_NUM_ENGINES];
> 		u64 bbaddr;
> 		u64 acthd;
> 		u32 fault_reg;
> 		u64 faddr;
> 		u32 rc_psmi; /* sleep state */
>-		u32 semaphore_mboxes[I915_NUM_RINGS - 1];
>+		u32 semaphore_mboxes[I915_NUM_ENGINES];
>+		u32 semaphore_sync[I915_NUM_ENGINES];
> 
> 		struct drm_i915_error_object {
> 			int page_count;
>@@ -373,8 +385,14 @@ struct drm_i915_error_state {
> 
> 		struct drm_i915_error_request {
> 			long jiffies;
>-			u32 seqno;
>+			long pid;
>+			u32 batch;
>+			u32 head;
> 			u32 tail;
>+			u32 seqno;
>+			u32 breadcrumb[I915_NUM_ENGINES];
>+			u32 complete;
>+			u32 tag;
> 		} *requests;
> 
> 		struct {
>@@ -387,11 +405,12 @@ struct drm_i915_error_state {
> 
> 		pid_t pid;
> 		char comm[TASK_COMM_LEN];
>-	} ring[I915_NUM_RINGS];
>+	} ring[I915_NUM_ENGINES];
>+
> 	struct drm_i915_error_buffer {
> 		u32 size;
> 		u32 name;
>-		u32 rseqno, wseqno;
>+		u32 rseqno[I915_NUM_ENGINES], wseqno, fseqno;
> 		u32 gtt_offset;
> 		u32 read_domains;
> 		u32 write_domain;
>@@ -406,6 +425,7 @@ struct drm_i915_error_state {
> 	} **active_bo, **pinned_bo;
> 
> 	u32 *active_bo_count, *pinned_bo_count;
>+	u32 vm_count;
> };
> 
> struct intel_connector;
>@@ -462,10 +482,10 @@ struct drm_i915_display_funcs {
> 			  struct drm_display_mode *mode);
> 	void (*fdi_link_train)(struct drm_crtc *crtc);
> 	void (*init_clock_gating)(struct drm_device *dev);
>-	int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc,
>+	int (*queue_flip)(struct i915_gem_request *rq,
>+			  struct intel_crtc *crtc,
> 			  struct drm_framebuffer *fb,
> 			  struct drm_i915_gem_object *obj,
>-			  struct intel_engine_cs *ring,
> 			  uint32_t flags);
> 	void (*update_primary_plane)(struct drm_crtc *crtc,
> 				     struct drm_framebuffer *fb,
>@@ -551,6 +571,7 @@ struct intel_uncore {
> 
> struct intel_device_info {
> 	u32 display_mmio_offset;
>+	u16 device_id;
> 	u8 num_pipes:3;
> 	u8 num_sprites[I915_MAX_PIPES];
> 	u8 gen;
>@@ -586,6 +607,11 @@ struct i915_ctx_hang_stats {
> 	/* Time when this context was last blamed for a GPU reset */
> 	unsigned long guilty_ts;
> 
>+	/* If the contexts causes a second GPU hang within this time,
>+	 * it is permanently banned from submitting any more work.
>+	 */
>+	unsigned long ban_period_seconds;
>+
> 	/* This context is banned to submit more work */
> 	bool banned;
> };
>@@ -611,16 +637,18 @@ struct i915_ctx_hang_stats {
>  */
> struct intel_context {
> 	struct kref ref;
>+	struct drm_i915_private *i915;
> 	int user_handle;
> 	uint8_t remap_slice;
> 	struct drm_i915_file_private *file_priv;
> 	struct i915_ctx_hang_stats hang_stats;
>-	struct i915_address_space *vm;
>+	struct i915_hw_ppgtt *ppgtt;
> 
>-	struct {
>-		struct drm_i915_gem_object *rcs_state;
>+	struct intel_engine_context {
>+		struct intel_ringbuffer *ring;
>+		struct drm_i915_gem_object *state;
> 		bool initialized;
>-	} legacy_hw_ctx;
>+	} ring[I915_NUM_ENGINES];
> 
> 	struct list_head link;
> };
>@@ -635,6 +663,8 @@ struct i915_fbc {
> 	struct drm_mm_node compressed_fb;
> 	struct drm_mm_node *compressed_llb;
> 
>+	bool false_color;
>+
> 	struct intel_fbc_work {
> 		struct delayed_work work;
> 		struct drm_crtc *crtc;
>@@ -688,6 +718,7 @@ enum intel_sbi_destination {
> #define QUIRK_LVDS_SSC_DISABLE (1<<1)
> #define QUIRK_INVERT_BRIGHTNESS (1<<2)
> #define QUIRK_BACKLIGHT_PRESENT (1<<3)
>+#define QUIRK_PIPEB_FORCE (1<<4)
> 
> struct intel_fbdev;
> struct intel_fbc_work;
>@@ -917,6 +948,23 @@ struct intel_rps_ei {
> 	u32 media_c0;
> };
> 
>+struct intel_rps_bdw_cal {
>+	u32 it_threshold_pct; /* interrupt, in percentage */
>+	u32 eval_interval; /* evaluation interval, in us */
>+	u32 last_ts;
>+	u32 last_c0;
>+	bool is_up;
>+};
>+
>+struct intel_rps_bdw_turbo {
>+	struct intel_rps_bdw_cal up;
>+	struct intel_rps_bdw_cal down;
>+	struct timer_list flip_timer;
>+	u32 timeout;
>+	atomic_t flip_received;
>+	struct work_struct work_max_freq;
>+};
>+
> struct intel_gen6_power_mgmt {
> 	/* work and pm_iir are protected by dev_priv->irq_lock */
> 	struct work_struct work;
>@@ -950,6 +998,9 @@ struct intel_gen6_power_mgmt {
> 	bool enabled;
> 	struct delayed_work delayed_resume_work;
> 
>+	bool is_bdw_sw_turbo;	/* Switch of BDW software turbo */
>+	struct intel_rps_bdw_turbo sw_turbo; /* Calculate RP interrupt timing */
>+
> 	/* manual wa residency calculations */
> 	struct intel_rps_ei up_ei, down_ei;
> 
>@@ -982,7 +1033,6 @@ struct intel_ilk_power_mgmt {
> 	int r_t;
> 
> 	struct drm_i915_gem_object *pwrctx;
>-	struct drm_i915_gem_object *renderctx;
> };
> 
> struct drm_i915_private;
>@@ -1042,19 +1092,6 @@ struct i915_power_domains {
> 	struct i915_power_well *power_wells;
> };
> 
>-struct i915_dri1_state {
>-	unsigned allow_batchbuffer : 1;
>-	u32 __iomem *gfx_hws_cpu_addr;
>-
>-	unsigned int cpp;
>-	int back_offset;
>-	int front_offset;
>-	int current_page;
>-	int page_flipping;
>-
>-	uint32_t counter;
>-};
>-
> struct i915_ums_state {
> 	/**
> 	 * Flag if the X Server, and thus DRM, is not currently in
>@@ -1147,6 +1184,7 @@ struct i915_gem_mm {
> };
> 
> struct drm_i915_error_state_buf {
>+	struct drm_i915_private *i915;
> 	unsigned bytes;
> 	unsigned size;
> 	int err;
>@@ -1213,7 +1251,7 @@ struct i915_gpu_error {
> 	/* Userspace knobs for gpu hang simulation;
> 	 * combines both a ring mask, and extra flags
> 	 */
>-	u32 stop_rings;
>+	unsigned long stop_rings;
> #define I915_STOP_RING_ALLOW_BAN       (1 << 31)
> #define I915_STOP_RING_ALLOW_WARN      (1 << 30)
> 
>@@ -1228,6 +1266,12 @@ enum modeset_restore {
> };
> 
> struct ddi_vbt_port_info {
>+	/*
>+	 * This is an index in the HDMI/DVI DDI buffer translation table.
>+	 * The special value HDMI_LEVEL_SHIFT_UNKNOWN means the VBT didn't
>+	 * populate this field.
>+	 */
>+#define HDMI_LEVEL_SHIFT_UNKNOWN	0xff
> 	uint8_t hdmi_level_shift;
> 
> 	uint8_t supports_dvi:1;
>@@ -1417,11 +1461,12 @@ struct drm_i915_private {
> 	wait_queue_head_t gmbus_wait_queue;
> 
> 	struct pci_dev *bridge_dev;
>-	struct intel_engine_cs ring[I915_NUM_RINGS];
>+	struct intel_engine_cs engine[I915_NUM_ENGINES];
>+	struct intel_context *default_context;
> 	struct drm_i915_gem_object *semaphore_obj;
>-	uint32_t last_seqno, next_seqno;
>+	uint32_t next_seqno;
> 
>-	drm_dma_handle_t *status_page_dmah;
>+	struct drm_dma_handle *status_page_dmah;
> 	struct resource mch_res;
> 
> 	/* protects the irq masks */
>@@ -1448,7 +1493,7 @@ struct drm_i915_private {
> 	u32 pm_rps_events;
> 	u32 pipestat_irq_mask[I915_MAX_PIPES];
> 
>-	struct work_struct hotplug_work;
>+	struct delayed_work hotplug_work;
> 	struct {
> 		unsigned long hpd_last_jiffies;
> 		int hpd_cnt;
>@@ -1470,11 +1515,14 @@ struct drm_i915_private {
> 	struct intel_overlay *overlay;
> 
> 	/* backlight registers and fields in struct intel_panel */
>-	spinlock_t backlight_lock;
>+	struct mutex backlight_lock;
> 
> 	/* LVDS info */
> 	bool no_aux_handshake;
> 
>+	/* protects panel power sequencer state */
>+	struct mutex pps_mutex;
>+
> 	struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */
> 	int fence_reg_start; /* 4 if userland hasn't ioctl'd us yet */
> 	int num_fence_regs; /* 8 on pre-965, 16 otherwise */
>@@ -1526,6 +1574,20 @@ struct drm_i915_private {
> 	struct intel_shared_dpll shared_dplls[I915_NUM_PLLS];
> 	int dpio_phy_iosf_port[I915_NUM_PHYS_VLV];
> 
>+	/*
>+	 * workarounds are currently applied at different places and
>+	 * changes are being done to consolidate them so exact count is
>+	 * not clear at this point, use a max value for now.
>+	 */
>+#define I915_MAX_WA_REGS  16
>+	struct {
>+		u32 addr;
>+		u32 value;
>+		/* bitmask representing WA bits */
>+		u32 mask;
>+	} intel_wa_regs[I915_MAX_WA_REGS];
>+	u32 num_wa_regs;
>+
> 	/* Reclocking support */
> 	bool render_reclock_avail;
> 	bool lvds_downclock_avail;
>@@ -1561,14 +1623,9 @@ struct drm_i915_private {
> #ifdef CONFIG_DRM_I915_FBDEV
> 	/* list of fbdev register on this device */
> 	struct intel_fbdev *fbdev;
>+	struct work_struct fbdev_suspend_work;
> #endif
> 
>-	/*
>-	 * The console may be contended at resume, but we don't
>-	 * want it to block on it.
>-	 */
>-	struct work_struct console_resume_work;
>-
> 	struct drm_property *broadcast_rgb_property;
> 	struct drm_property *force_audio_property;
> 
>@@ -1614,12 +1671,10 @@ struct drm_i915_private {
> 	 */
> 	struct workqueue_struct *dp_wq;
> 
>-	/* Old dri1 support infrastructure, beware the dragons ya fools entering
>-	 * here! */
>-	struct i915_dri1_state dri1;
>+	uint32_t bios_vgacntr;
>+
> 	/* Old ums support infrastructure, same warning applies. */
> 	struct i915_ums_state ums;
>-
> 	/*
> 	 * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch
> 	 * will be rejected. Instead look for a better place.
>@@ -1632,9 +1687,11 @@ static inline struct drm_i915_private *to_i915(const struct drm_device *dev)
> }
> 
> /* Iterate over initialised rings */
>-#define for_each_ring(ring__, dev_priv__, i__) \
>-	for ((i__) = 0; (i__) < I915_NUM_RINGS; (i__)++) \
>-		if (((ring__) = &(dev_priv__)->ring[(i__)]), intel_ring_initialized((ring__)))
>+#define for_each_engine(engine__, dev_priv__, i__) \
>+	for ((i__) = 0; (i__) < I915_NUM_ENGINES; (i__)++) \
>+		if (((engine__) = &(dev_priv__)->engine[(i__)]), intel_engine_initialized((engine__)))
>+
>+#define RCS_ENGINE(x) (&__I915__(x)->engine[RCS])
> 
> enum hdmi_force_audio {
> 	HDMI_AUDIO_OFF_DVI = -2,	/* no aux data for HDMI-DVI converter */
>@@ -1699,16 +1756,15 @@ struct drm_i915_gem_object {
> 	struct drm_mm_node *stolen;
> 	struct list_head global_list;
> 
>-	struct list_head ring_list;
> 	/** Used in execbuf to temporarily hold a ref */
> 	struct list_head obj_exec_link;
> 
> 	/**
> 	 * This is set if the object is on the active lists (has pending
>-	 * rendering and so a non-zero seqno), and is not set if it i s on
>-	 * inactive (ready to be unbound) list.
>+	 * rendering and so a submitted request), and is not set if it is on
>+	 * inactive (ready to be unbound) list. We track activity per engine.
> 	 */
>-	unsigned int active:1;
>+	unsigned int active:I915_NUM_ENGINE_BITS;
> 
> 	/**
> 	 * This is set if the object has been written to since last bound
>@@ -1761,13 +1817,6 @@ struct drm_i915_gem_object {
> 	 * Only honoured if hardware has relevant pte bit
> 	 */
> 	unsigned long gt_ro:1;
>-
>-	/*
>-	 * Is the GPU currently using a fence to access this buffer,
>-	 */
>-	unsigned int pending_fenced_gpu_access:1;
>-	unsigned int fenced_gpu_access:1;
>-
> 	unsigned int cache_level:3;
> 
> 	unsigned int has_aliasing_ppgtt_mapping:1;
>@@ -1783,13 +1832,11 @@ struct drm_i915_gem_object {
> 	void *dma_buf_vmapping;
> 	int vmapping_count;
> 
>-	struct intel_engine_cs *ring;
>-
>-	/** Breadcrumb of last rendering to the buffer. */
>-	uint32_t last_read_seqno;
>-	uint32_t last_write_seqno;
>-	/** Breadcrumb of last fenced GPU access to the buffer. */
>-	uint32_t last_fenced_seqno;
>+	/** Breadcrumbs of last rendering to the buffer. */
>+	struct {
>+		struct i915_gem_request *request;
>+		struct list_head engine_list;
>+	} last_write, last_read[I915_NUM_ENGINES], last_fence;
> 
> 	/** Current tiling stride for the object, if it's tiled. */
> 	uint32_t stride;
>@@ -1805,7 +1852,7 @@ struct drm_i915_gem_object {
> 	struct drm_file *pin_filp;
> 
> 	/** for phy allocated objects */
>-	drm_dma_handle_t *phys_handle;
>+	struct drm_dma_handle *phys_handle;
> 
> 	union {
> 		struct i915_gem_userptr {
>@@ -1827,44 +1874,13 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
> 		       unsigned frontbuffer_bits);
> 
> /**
>- * Request queue structure.
>- *
>- * The request queue allows us to note sequence numbers that have been emitted
>- * and may be associated with active buffers to be retired.
>- *
>- * By keeping this list, we can avoid having to do questionable
>- * sequence-number comparisons on buffer last_rendering_seqnos, and associate
>- * an emission time with seqnos for tracking how far ahead of the GPU we are.
>+ * Returns true if seq1 is later than seq2.
>  */
>-struct drm_i915_gem_request {
>-	/** On Which ring this request was generated */
>-	struct intel_engine_cs *ring;
>-
>-	/** GEM sequence number associated with this request. */
>-	uint32_t seqno;
>-
>-	/** Position in the ringbuffer of the start of the request */
>-	u32 head;
>-
>-	/** Position in the ringbuffer of the end of the request */
>-	u32 tail;
>-
>-	/** Context related to this request */
>-	struct intel_context *ctx;
>-
>-	/** Batch buffer related to this request if any */
>-	struct drm_i915_gem_object *batch_obj;
>-
>-	/** Time at which this request was emitted, in jiffies. */
>-	unsigned long emitted_jiffies;
>-
>-	/** global list entry for this request */
>-	struct list_head list;
>-
>-	struct drm_i915_file_private *file_priv;
>-	/** file_priv list entry for this request */
>-	struct list_head client_list;
>-};
>+static inline bool
>+__i915_seqno_passed(uint32_t seq1, uint32_t seq2)
>+{
>+	return (int32_t)(seq1 - seq2) >= 0;
>+}
> 
> struct drm_i915_file_private {
> 	struct drm_i915_private *dev_priv;
>@@ -1878,7 +1894,7 @@ struct drm_i915_file_private {
> 	struct idr context_idr;
> 
> 	atomic_t rps_wait_boost;
>-	struct  intel_engine_cs *bsd_ring;
>+	struct intel_engine_cs *bsd_engine;
> };
> 
> /*
>@@ -1971,51 +1987,63 @@ struct drm_i915_cmd_table {
> 	int count;
> };
> 
>-#define INTEL_INFO(dev)	(&to_i915(dev)->info)
>-
>-#define IS_I830(dev)		((dev)->pdev->device == 0x3577)
>-#define IS_845G(dev)		((dev)->pdev->device == 0x2562)
>+/* Note that the (struct drm_i915_private *) cast is just to shut up gcc. */
>+#define __I915__(p) ({ \
>+	struct drm_i915_private *__p; \
>+	if (__builtin_types_compatible_p(typeof(*p), struct drm_i915_private)) \
>+		__p = (struct drm_i915_private *)p; \
>+	else if (__builtin_types_compatible_p(typeof(*p), struct drm_device)) \
>+		__p = to_i915((struct drm_device *)p); \
>+	else \
>+		BUILD_BUG(); \
>+	__p; \
>+})
>+#define INTEL_INFO(p) 	(&__I915__(p)->info)
>+#define INTEL_DEVID(p)	(INTEL_INFO(p)->device_id)
>+
>+#define IS_I830(dev)		(INTEL_DEVID(dev) == 0x3577)
>+#define IS_845G(dev)		(INTEL_DEVID(dev) == 0x2562)
> #define IS_I85X(dev)		(INTEL_INFO(dev)->is_i85x)
>-#define IS_I865G(dev)		((dev)->pdev->device == 0x2572)
>+#define IS_I865G(dev)		(INTEL_DEVID(dev) == 0x2572)
> #define IS_I915G(dev)		(INTEL_INFO(dev)->is_i915g)
>-#define IS_I915GM(dev)		((dev)->pdev->device == 0x2592)
>-#define IS_I945G(dev)		((dev)->pdev->device == 0x2772)
>+#define IS_I915GM(dev)		(INTEL_DEVID(dev) == 0x2592)
>+#define IS_I945G(dev)		(INTEL_DEVID(dev) == 0x2772)
> #define IS_I945GM(dev)		(INTEL_INFO(dev)->is_i945gm)
> #define IS_BROADWATER(dev)	(INTEL_INFO(dev)->is_broadwater)
> #define IS_CRESTLINE(dev)	(INTEL_INFO(dev)->is_crestline)
>-#define IS_GM45(dev)		((dev)->pdev->device == 0x2A42)
>+#define IS_GM45(dev)		(INTEL_DEVID(dev) == 0x2A42)
> #define IS_G4X(dev)		(INTEL_INFO(dev)->is_g4x)
>-#define IS_PINEVIEW_G(dev)	((dev)->pdev->device == 0xa001)
>-#define IS_PINEVIEW_M(dev)	((dev)->pdev->device == 0xa011)
>+#define IS_PINEVIEW_G(dev)	(INTEL_DEVID(dev) == 0xa001)
>+#define IS_PINEVIEW_M(dev)	(INTEL_DEVID(dev) == 0xa011)
> #define IS_PINEVIEW(dev)	(INTEL_INFO(dev)->is_pineview)
> #define IS_G33(dev)		(INTEL_INFO(dev)->is_g33)
>-#define IS_IRONLAKE_M(dev)	((dev)->pdev->device == 0x0046)
>+#define IS_IRONLAKE_M(dev)	(INTEL_DEVID(dev) == 0x0046)
> #define IS_IVYBRIDGE(dev)	(INTEL_INFO(dev)->is_ivybridge)
>-#define IS_IVB_GT1(dev)		((dev)->pdev->device == 0x0156 || \
>-				 (dev)->pdev->device == 0x0152 || \
>-				 (dev)->pdev->device == 0x015a)
>-#define IS_SNB_GT1(dev)		((dev)->pdev->device == 0x0102 || \
>-				 (dev)->pdev->device == 0x0106 || \
>-				 (dev)->pdev->device == 0x010A)
>+#define IS_IVB_GT1(dev)		(INTEL_DEVID(dev) == 0x0156 || \
>+				 INTEL_DEVID(dev) == 0x0152 || \
>+				 INTEL_DEVID(dev) == 0x015a)
>+#define IS_SNB_GT1(dev)		(INTEL_DEVID(dev) == 0x0102 || \
>+				 INTEL_DEVID(dev) == 0x0106 || \
>+				 INTEL_DEVID(dev) == 0x010A)
> #define IS_VALLEYVIEW(dev)	(INTEL_INFO(dev)->is_valleyview)
> #define IS_CHERRYVIEW(dev)	(INTEL_INFO(dev)->is_valleyview && IS_GEN8(dev))
> #define IS_HASWELL(dev)	(INTEL_INFO(dev)->is_haswell)
> #define IS_BROADWELL(dev)	(!INTEL_INFO(dev)->is_valleyview && IS_GEN8(dev))
> #define IS_MOBILE(dev)		(INTEL_INFO(dev)->is_mobile)
> #define IS_HSW_EARLY_SDV(dev)	(IS_HASWELL(dev) && \
>-				 ((dev)->pdev->device & 0xFF00) == 0x0C00)
>+				 (INTEL_DEVID(dev) & 0xFF00) == 0x0C00)
> #define IS_BDW_ULT(dev)		(IS_BROADWELL(dev) && \
>-				 (((dev)->pdev->device & 0xf) == 0x2  || \
>-				 ((dev)->pdev->device & 0xf) == 0x6 || \
>-				 ((dev)->pdev->device & 0xf) == 0xe))
>+				 ((INTEL_DEVID(dev) & 0xf) == 0x2  || \
>+				 (INTEL_DEVID(dev) & 0xf) == 0x6 || \
>+				 (INTEL_DEVID(dev) & 0xf) == 0xe))
> #define IS_HSW_ULT(dev)		(IS_HASWELL(dev) && \
>-				 ((dev)->pdev->device & 0xFF00) == 0x0A00)
>+				 (INTEL_DEVID(dev) & 0xFF00) == 0x0A00)
> #define IS_ULT(dev)		(IS_HSW_ULT(dev) || IS_BDW_ULT(dev))
> #define IS_HSW_GT3(dev)		(IS_HASWELL(dev) && \
>-				 ((dev)->pdev->device & 0x00F0) == 0x0020)
>+				 (INTEL_DEVID(dev) & 0x00F0) == 0x0020)
> /* ULX machines are also considered ULT. */
>-#define IS_HSW_ULX(dev)		((dev)->pdev->device == 0x0A0E || \
>-				 (dev)->pdev->device == 0x0A1E)
>+#define IS_HSW_ULX(dev)		(INTEL_DEVID(dev) == 0x0A0E || \
>+				 INTEL_DEVID(dev) == 0x0A1E)
> #define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary)
> 
> /*
>@@ -2046,11 +2074,12 @@ struct drm_i915_cmd_table {
> 				 to_i915(dev)->ellc_size)
> #define I915_NEED_GFX_HWS(dev)	(INTEL_INFO(dev)->need_gfx_hws)
> 
>-#define HAS_HW_CONTEXTS(dev)	(INTEL_INFO(dev)->gen >= 6)
>+#define HAS_HW_CONTEXTS(dev)	(INTEL_INFO(dev)->gen >= 5)
>+#define HAS_LOGICAL_RING_CONTEXTS(dev)	(INTEL_INFO(dev)->gen >= 8)
> #define HAS_ALIASING_PPGTT(dev)	(INTEL_INFO(dev)->gen >= 6)
> #define HAS_PPGTT(dev)		(INTEL_INFO(dev)->gen >= 7 && !IS_GEN8(dev))
>-#define USES_PPGTT(dev)		intel_enable_ppgtt(dev, false)
>-#define USES_FULL_PPGTT(dev)	intel_enable_ppgtt(dev, true)
>+#define USES_PPGTT(dev)		(i915.enable_ppgtt)
>+#define USES_FULL_PPGTT(dev)	(i915.enable_ppgtt == 2)
> 
> #define HAS_OVERLAY(dev)		(INTEL_INFO(dev)->has_overlay)
> #define OVERLAY_NEEDS_PHYSICAL(dev)	(INTEL_INFO(dev)->overlay_needs_physical)
>@@ -2152,9 +2181,7 @@ struct i915_params {
> };
> extern struct i915_params i915 __read_mostly;
> 
>-				/* i915_dma.c */
>-void i915_update_dri1_breadcrumb(struct drm_device *dev);
>-extern void i915_kernel_lost_context(struct drm_device * dev);
>+/* i915_dma.c */
> extern int i915_driver_load(struct drm_device *, unsigned long flags);
> extern int i915_driver_unload(struct drm_device *);
> extern int i915_driver_open(struct drm_device *dev, struct drm_file *file);
>@@ -2168,9 +2195,6 @@ extern int i915_driver_device_is_agp(struct drm_device * dev);
> extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
> 			      unsigned long arg);
> #endif
>-extern int i915_emit_box(struct drm_device *dev,
>-			 struct drm_clip_rect *box,
>-			 int DR1, int DR4);
> extern int intel_gpu_reset(struct drm_device *dev);
> extern int i915_reset(struct drm_device *dev);
> extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
>@@ -2180,8 +2204,6 @@ extern void i915_update_gfx_val(struct drm_i915_private *dev_priv);
> int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on);
> void intel_hpd_cancel_work(struct drm_i915_private *dev_priv);
> 
>-extern void intel_console_resume(struct work_struct *work);
>-
> /* i915_irq.c */
> void i915_queue_hangcheck(struct drm_device *dev);
> __printf(3, 4)
>@@ -2263,6 +2285,12 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
> int i915_gem_wait_ioctl(struct drm_device *dev, void *data,
> 			struct drm_file *file_priv);
> void i915_gem_load(struct drm_device *dev);
>+unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
>+			      long target,
>+			      unsigned flags);
>+#define I915_SHRINK_PURGEABLE 0x1
>+#define I915_SHRINK_UNBOUND 0x2
>+#define I915_SHRINK_BOUND 0x4
> void *i915_gem_object_alloc(struct drm_device *dev);
> void i915_gem_object_free(struct drm_i915_gem_object *obj);
> void i915_gem_object_init(struct drm_i915_gem_object *obj,
>@@ -2315,22 +2343,12 @@ static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
> 
> int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
> int i915_gem_object_sync(struct drm_i915_gem_object *obj,
>-			 struct intel_engine_cs *to);
>-void i915_vma_move_to_active(struct i915_vma *vma,
>-			     struct intel_engine_cs *ring);
>+			 struct i915_gem_request *rq);
> int i915_gem_dumb_create(struct drm_file *file_priv,
> 			 struct drm_device *dev,
> 			 struct drm_mode_create_dumb *args);
> int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev,
> 		      uint32_t handle, uint64_t *offset);
>-/**
>- * Returns true if seq1 is later than seq2.
>- */
>-static inline bool
>-i915_seqno_passed(uint32_t seq1, uint32_t seq2)
>-{
>-	return (int32_t)(seq1 - seq2) >= 0;
>-}
> 
> int __must_check i915_gem_get_seqno(struct drm_device *dev, u32 *seqno);
> int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno);
>@@ -2340,24 +2358,33 @@ int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj);
> bool i915_gem_object_pin_fence(struct drm_i915_gem_object *obj);
> void i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj);
> 
>-struct drm_i915_gem_request *
>-i915_gem_find_active_request(struct intel_engine_cs *ring);
>-
> bool i915_gem_retire_requests(struct drm_device *dev);
>-void i915_gem_retire_requests_ring(struct intel_engine_cs *ring);
>-int __must_check i915_gem_check_wedge(struct i915_gpu_error *error,
>-				      bool interruptible);
>-int __must_check i915_gem_check_olr(struct intel_engine_cs *ring, u32 seqno);
>+void i915_gem_retire_requests__engine(struct intel_engine_cs *engine);
>+
>+static inline bool __i915_reset_in_progress(unsigned x)
>+{
>+	return unlikely(x & I915_RESET_IN_PROGRESS_FLAG);
>+}
> 
> static inline bool i915_reset_in_progress(struct i915_gpu_error *error)
> {
>-	return unlikely(atomic_read(&error->reset_counter)
>-			& (I915_RESET_IN_PROGRESS_FLAG | I915_WEDGED));
>+	return __i915_reset_in_progress(atomic_read(&error->reset_counter));
>+}
>+
>+static inline bool __i915_terminally_wedged(unsigned x)
>+{
>+	return unlikely(x & I915_WEDGED);
> }
> 
> static inline bool i915_terminally_wedged(struct i915_gpu_error *error)
> {
>-	return atomic_read(&error->reset_counter) & I915_WEDGED;
>+	return __i915_terminally_wedged(atomic_read(&error->reset_counter));
>+}
>+
>+static inline bool i915_recovery_pending(struct i915_gpu_error *error)
>+{
>+	unsigned x = atomic_read(&error->reset_counter);
>+	return __i915_reset_in_progress(x) && !__i915_terminally_wedged(x);
> }
> 
> static inline u32 i915_reset_count(struct i915_gpu_error *error)
>@@ -2382,19 +2409,10 @@ bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force);
> int __must_check i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj);
> int __must_check i915_gem_init(struct drm_device *dev);
> int __must_check i915_gem_init_hw(struct drm_device *dev);
>-int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice);
>+void i915_gem_fini(struct drm_device *dev);
> void i915_gem_init_swizzling(struct drm_device *dev);
>-void i915_gem_cleanup_ringbuffer(struct drm_device *dev);
> int __must_check i915_gpu_idle(struct drm_device *dev);
> int __must_check i915_gem_suspend(struct drm_device *dev);
>-int __i915_add_request(struct intel_engine_cs *ring,
>-		       struct drm_file *file,
>-		       struct drm_i915_gem_object *batch_obj,
>-		       u32 *seqno);
>-#define i915_add_request(ring, seqno) \
>-	__i915_add_request(ring, NULL, NULL, seqno)
>-int __must_check i915_wait_seqno(struct intel_engine_cs *ring,
>-				 uint32_t seqno);
> int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
> int __must_check
> i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
>@@ -2404,7 +2422,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write);
> int __must_check
> i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
> 				     u32 alignment,
>-				     struct intel_engine_cs *pipelined);
>+				     struct i915_gem_request *pipelined);
> void i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj);
> int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
> 				int align);
>@@ -2451,30 +2469,36 @@ static inline bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) {
> }
> 
> /* Some GGTT VM helpers */
>-#define obj_to_ggtt(obj) \
>-	(&((struct drm_i915_private *)(obj)->base.dev->dev_private)->gtt.base)
>+#define i915_obj_to_ggtt(obj) (&to_i915((obj)->base.dev)->gtt.base)
> static inline bool i915_is_ggtt(struct i915_address_space *vm)
> {
>-	struct i915_address_space *ggtt =
>-		&((struct drm_i915_private *)(vm)->dev->dev_private)->gtt.base;
>-	return vm == ggtt;
>+	return vm == &to_i915(vm->dev)->gtt.base;
> }
> 
>+static inline struct i915_hw_ppgtt *
>+i915_vm_to_ppgtt(struct i915_address_space *vm)
>+{
>+	WARN_ON(i915_is_ggtt(vm));
>+
>+	return container_of(vm, struct i915_hw_ppgtt, base);
>+}
>+
>+
> static inline bool i915_gem_obj_ggtt_bound(struct drm_i915_gem_object *obj)
> {
>-	return i915_gem_obj_bound(obj, obj_to_ggtt(obj));
>+	return i915_gem_obj_bound(obj, i915_obj_to_ggtt(obj));
> }
> 
> static inline unsigned long
> i915_gem_obj_ggtt_offset(struct drm_i915_gem_object *obj)
> {
>-	return i915_gem_obj_offset(obj, obj_to_ggtt(obj));
>+	return i915_gem_obj_offset(obj, i915_obj_to_ggtt(obj));
> }
> 
> static inline unsigned long
> i915_gem_obj_ggtt_size(struct drm_i915_gem_object *obj)
> {
>-	return i915_gem_obj_size(obj, obj_to_ggtt(obj));
>+	return i915_gem_obj_size(obj, i915_obj_to_ggtt(obj));
> }
> 
> static inline int __must_check
>@@ -2482,7 +2506,8 @@ i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj,
> 		      uint32_t alignment,
> 		      unsigned flags)
> {
>-	return i915_gem_object_pin(obj, obj_to_ggtt(obj), alignment, flags | PIN_GLOBAL);
>+	return i915_gem_object_pin(obj, i915_obj_to_ggtt(obj),
>+				   alignment, flags | PIN_GLOBAL);
> }
> 
> static inline int
>@@ -2494,18 +2519,19 @@ i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
> void i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj);
> 
> /* i915_gem_context.c */
>-#define ctx_to_ppgtt(ctx) container_of((ctx)->vm, struct i915_hw_ppgtt, base)
> int __must_check i915_gem_context_init(struct drm_device *dev);
> void i915_gem_context_fini(struct drm_device *dev);
>-void i915_gem_context_reset(struct drm_device *dev);
> int i915_gem_context_open(struct drm_device *dev, struct drm_file *file);
> int i915_gem_context_enable(struct drm_i915_private *dev_priv);
> void i915_gem_context_close(struct drm_device *dev, struct drm_file *file);
>-int i915_switch_context(struct intel_engine_cs *ring,
>-			struct intel_context *to);
>+int i915_request_switch_context(struct i915_gem_request *rq);
>+void i915_request_switch_context__commit(struct i915_gem_request *rq);
>+void i915_request_switch_context__undo(struct i915_gem_request *rq);
> struct intel_context *
> i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id);
> void i915_gem_context_free(struct kref *ctx_ref);
>+struct drm_i915_gem_object *
>+i915_gem_alloc_context_obj(struct drm_device *dev, size_t size);
> static inline void i915_gem_context_reference(struct intel_context *ctx)
> {
> 	kref_get(&ctx->ref);
>@@ -2527,7 +2553,7 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
> 				   struct drm_file *file);
> 
> /* i915_gem_render_state.c */
>-int i915_gem_render_state_init(struct intel_engine_cs *ring);
>+int i915_gem_render_state_init(struct i915_gem_request *rq);
> /* i915_gem_evict.c */
> int __must_check i915_gem_evict_something(struct drm_device *dev,
> 					  struct i915_address_space *vm,
>@@ -2547,6 +2573,160 @@ static inline void i915_gem_chipset_flush(struct drm_device *dev)
> 		intel_gtt_chipset_flush();
> }
> 
>+/* i915_gem_request.c */
>+
>+/**
>+ * Request queue structure.
>+ *
>+ * The request queue allows us to note sequence numbers that have been emitted
>+ * and may be associated with active buffers to be retired.
>+ *
>+ * By keeping this list, we can avoid having to do questionable
>+ * sequence-number comparisons on buffer last_rendering_seqnos, and associate
>+ * an emission time with seqnos for tracking how far ahead of the GPU we are.
>+ */
>+struct i915_gem_request {
>+	struct kref kref;
>+
>+	/** On which ring/engine/ctx this request was generated */
>+	struct drm_i915_private *i915;
>+	struct intel_context *ctx;
>+	struct intel_engine_cs *engine;
>+	struct intel_ringbuffer *ring;
>+
>+	/** How many GPU resets ago was this request first constructed? */
>+	unsigned reset_counter;
>+
>+	/** GEM sequence number/breadcrumb associated with this request. */
>+	u32 seqno;
>+	u32 breadcrumb[I915_NUM_ENGINES];
>+	u32 semaphore[I915_NUM_ENGINES];
>+
>+	/** Position in the ringbuffer of the request */
>+	u32 head, tail;
>+
>+	/** Batch buffer and objects related to this request if any */
>+	struct i915_vma *batch;
>+	struct list_head vmas;
>+
>+	/** Time at which this request was emitted, in jiffies. */
>+	unsigned long emitted_jiffies;
>+
>+	/** global list entry for this request */
>+	struct list_head engine_list;
>+	struct list_head breadcrumb_link;
>+
>+	struct drm_i915_file_private *file_priv;
>+	/** file_priv list entry for this request */
>+	struct list_head client_list;
>+
>+	u16 tag;
>+	unsigned remap_l3:8;
>+	unsigned pending_flush:4;
>+	bool outstanding:1;
>+	bool has_ctx_switch:1;
>+
>+	bool completed; /* kept separate for atomicity */
>+};
>+
>+static inline struct intel_engine_cs *i915_request_engine(struct i915_gem_request *rq)
>+{
>+	return rq ? rq->engine : NULL;
>+}
>+
>+static inline int i915_request_engine_id(struct i915_gem_request *rq)
>+{
>+	return rq ? rq->engine->id : -1;
>+}
>+
>+static inline u32 i915_request_seqno(struct i915_gem_request *rq)
>+{
>+	return rq ? rq->seqno : 0;
>+}
>+
>+bool __i915_request_complete__wa(struct i915_gem_request *rq);
>+
>+static inline bool
>+i915_request_complete(struct i915_gem_request *rq)
>+{
>+	if (!rq->completed && rq->engine->is_complete(rq)) {
>+		trace_i915_gem_request_complete(rq);
>+		rq->completed = true;
>+	}
>+	return rq->completed;
>+}
>+
>+static inline struct i915_gem_request *
>+i915_request_get(struct i915_gem_request *rq)
>+{
>+	if (rq)
>+		kref_get(&rq->kref);
>+	return rq;
>+}
>+
>+void __i915_request_free(struct kref *kref);
>+
>+static inline void
>+i915_request_put(struct i915_gem_request *rq)
>+{
>+	if (rq == NULL)
>+		return;
>+
>+	lockdep_assert_held(&rq->i915->dev->struct_mutex);
>+	kref_put(&rq->kref, __i915_request_free);
>+}
>+
>+static inline void
>+i915_request_put__unlocked(struct i915_gem_request *rq)
>+{
>+	if (!atomic_add_unless(&rq->kref.refcount, -1, 1)) {
>+		struct drm_device *dev = rq->i915->dev;
>+
>+		mutex_lock(&dev->struct_mutex);
>+		if (likely(atomic_dec_and_test(&rq->kref.refcount)))
>+			__i915_request_free(&rq->kref);
>+		mutex_unlock(&dev->struct_mutex);
>+	}
>+}
>+
>+int __must_check
>+i915_request_add_vma(struct i915_gem_request *rq,
>+		     struct i915_vma *vma,
>+		     unsigned fenced);
>+#define VMA_IS_FENCED 0x1
>+#define VMA_HAS_FENCE 0x2
>+int __must_check
>+i915_request_emit_flush(struct i915_gem_request *rq,
>+			unsigned flags);
>+int __must_check
>+__i915_request_emit_breadcrumb(struct i915_gem_request *rq, int id);
>+static inline int __must_check
>+i915_request_emit_breadcrumb(struct i915_gem_request *rq)
>+{
>+	return __i915_request_emit_breadcrumb(rq, rq->engine->id);
>+}
>+static inline int __must_check
>+i915_request_emit_semaphore(struct i915_gem_request *rq, int id)
>+{
>+	return __i915_request_emit_breadcrumb(rq, id);
>+}
>+int __must_check
>+i915_request_emit_batchbuffer(struct i915_gem_request *rq,
>+			      struct i915_vma *batch,
>+			      uint64_t start, uint32_t len,
>+			      unsigned flags);
>+int __must_check
>+i915_request_commit(struct i915_gem_request *rq);
>+struct i915_gem_request *
>+i915_request_get_breadcrumb(struct i915_gem_request *rq);
>+int __must_check
>+i915_request_wait(struct i915_gem_request *rq);
>+int __i915_request_wait(struct i915_gem_request *rq,
>+			bool interruptible,
>+			s64 *timeout,
>+			struct drm_i915_file_private *file);
>+void i915_request_retire(struct i915_gem_request *rq);
>+
> /* i915_gem_stolen.c */
> int i915_gem_init_stolen(struct drm_device *dev);
> int i915_gem_stolen_setup_compression(struct drm_device *dev, int size, int fb_cpp);
>@@ -2573,13 +2753,6 @@ void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
> void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj);
> void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
> 
>-/* i915_gem_debug.c */
>-#if WATCH_LISTS
>-int i915_verify_lists(struct drm_device *dev);
>-#else
>-#define i915_verify_lists(dev) 0
>-#endif
>-
> /* i915_debugfs.c */
> int i915_debugfs_init(struct drm_minor *minor);
> void i915_debugfs_cleanup(struct drm_minor *minor);
>@@ -2595,6 +2768,7 @@ void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...);
> int i915_error_state_to_str(struct drm_i915_error_state_buf *estr,
> 			    const struct i915_error_state_file_priv *error);
> int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb,
>+			      struct drm_i915_private *i915,
> 			      size_t count, loff_t pos);
> static inline void i915_error_state_buf_release(
> 	struct drm_i915_error_state_buf *eb)
>@@ -2609,14 +2783,14 @@ void i915_error_state_put(struct i915_error_state_file_priv *error_priv);
> void i915_destroy_error_state(struct drm_device *dev);
> 
> void i915_get_extra_instdone(struct drm_device *dev, uint32_t *instdone);
>-const char *i915_cache_level_str(int type);
>+const char *i915_cache_level_str(struct drm_i915_private *i915, int type);
> 
> /* i915_cmd_parser.c */
> int i915_cmd_parser_get_version(void);
>-int i915_cmd_parser_init_ring(struct intel_engine_cs *ring);
>-void i915_cmd_parser_fini_ring(struct intel_engine_cs *ring);
>-bool i915_needs_cmd_parser(struct intel_engine_cs *ring);
>-int i915_parse_cmds(struct intel_engine_cs *ring,
>+int i915_cmd_parser_init_engine(struct intel_engine_cs *engine);
>+void i915_cmd_parser_fini_engine(struct intel_engine_cs *engine);
>+bool i915_needs_cmd_parser(struct intel_engine_cs *engine);
>+int i915_parse_cmds(struct intel_engine_cs *engine,
> 		    struct drm_i915_gem_object *batch_obj,
> 		    u32 batch_start_offset,
> 		    bool is_master);
>@@ -2701,10 +2875,13 @@ extern void intel_modeset_setup_hw_state(struct drm_device *dev,
> extern void i915_redisable_vga(struct drm_device *dev);
> extern void i915_redisable_vga_power_on(struct drm_device *dev);
> extern bool intel_fbc_enabled(struct drm_device *dev);
>+extern void gen8_fbc_sw_flush(struct drm_device *dev, u32 value);
> extern void intel_disable_fbc(struct drm_device *dev);
> extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
> extern void intel_init_pch_refclk(struct drm_device *dev);
> extern void gen6_set_rps(struct drm_device *dev, u8 val);
>+extern void bdw_software_turbo(struct drm_device *dev);
>+extern void gen8_flip_interrupt(struct drm_device *dev);
> extern void valleyview_set_rps(struct drm_device *dev, u8 val);
> extern void intel_set_memory_cxsr(struct drm_i915_private *dev_priv,
> 				  bool enable);
>@@ -2712,14 +2889,11 @@ extern void intel_detect_pch(struct drm_device *dev);
> extern int intel_trans_dp_port_sel(struct drm_crtc *crtc);
> extern int intel_enable_rc6(const struct drm_device *dev);
> 
>-extern bool i915_semaphore_is_enabled(struct drm_device *dev);
> int i915_reg_read_ioctl(struct drm_device *dev, void *data,
> 			struct drm_file *file);
> int i915_get_reset_stats_ioctl(struct drm_device *dev, void *data,
> 			       struct drm_file *file);
> 
>-void intel_notify_mmio_flip(struct intel_engine_cs *ring);
>-
> /* overlay */
> extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev);
> extern void intel_overlay_print_error_state(struct drm_i915_error_state_buf *e,
>diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
>index ad55b06..98a78a4 100644
>--- a/drivers/gpu/drm/i915/i915_gem.c
>+++ b/drivers/gpu/drm/i915/i915_gem.c
>@@ -38,15 +38,14 @@
> #include <linux/pci.h>
> #include <linux/dma-buf.h>
> 
>+#define RQ_BUG_ON(x) BUG_ON(x)
>+
> static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
> static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
> 						   bool force);
> static __must_check int
> i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
> 			       bool readonly);
>-static void
>-i915_gem_object_retire(struct drm_i915_gem_object *obj);
>-
> static void i915_gem_write_fence(struct drm_device *dev, int reg,
> 				 struct drm_i915_gem_object *obj);
> static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
>@@ -60,7 +59,6 @@ static unsigned long i915_gem_shrinker_scan(struct shrinker *shrinker,
> static int i915_gem_shrinker_oom(struct notifier_block *nb,
> 				 unsigned long event,
> 				 void *ptr);
>-static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
> static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
> 
> static bool cpu_cache_is_coherent(struct drm_device *dev,
>@@ -108,23 +106,108 @@ static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
> 	spin_unlock(&dev_priv->mm.object_stat_lock);
> }
> 
>+static void
>+i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
>+{
>+	RQ_BUG_ON(obj->active == 0);
>+	RQ_BUG_ON(obj->last_write.request == NULL);
>+	RQ_BUG_ON(list_empty(&obj->last_write.engine_list));
>+
>+	intel_fb_obj_flush(obj, true);
>+	list_del_init(&obj->last_write.engine_list);
>+	i915_request_put(obj->last_write.request);
>+	obj->last_write.request = NULL;
>+}
>+
>+static void
>+i915_gem_object_retire__fence(struct drm_i915_gem_object *obj)
>+{
>+	RQ_BUG_ON(obj->active == 0);
>+	RQ_BUG_ON(obj->last_fence.request == NULL);
>+	RQ_BUG_ON(list_empty(&obj->last_fence.engine_list));
>+
>+	list_del_init(&obj->last_fence.engine_list);
>+	i915_request_put(obj->last_fence.request);
>+	obj->last_fence.request = NULL;
>+}
>+
>+static void
>+i915_gem_object_retire__read(struct drm_i915_gem_object *obj,
>+			     struct intel_engine_cs *engine)
>+{
>+	struct i915_vma *vma;
>+
>+	RQ_BUG_ON(obj->active == 0);
>+	RQ_BUG_ON(obj->last_read[engine->id].request == NULL);
>+	RQ_BUG_ON(list_empty(&obj->last_read[engine->id].engine_list));
>+
>+	list_del_init(&obj->last_read[engine->id].engine_list);
>+	i915_request_put(obj->last_read[engine->id].request);
>+	obj->last_read[engine->id].request = NULL;
>+
>+	if (obj->last_write.request &&
>+	    obj->last_write.request->engine == engine)
>+		i915_gem_object_retire__write(obj);
>+
>+	if (obj->last_fence.request &&
>+	    obj->last_fence.request->engine == engine)
>+		i915_gem_object_retire__fence(obj);
>+
>+	if (--obj->active)
>+		return;
>+
>+	RQ_BUG_ON(obj->last_write.request);
>+	RQ_BUG_ON(obj->last_fence.request);
>+
>+	list_for_each_entry(vma, &obj->vma_list, vma_link)
>+		if (!list_empty(&vma->mm_list))
>+			list_move_tail(&vma->mm_list, &vma->vm->inactive_list);
>+
>+	drm_gem_object_unreference(&obj->base);
>+}
>+
>+static void
>+i915_gem_object_retire(struct drm_i915_gem_object *obj)
>+{
>+	struct i915_gem_request *rq;
>+	int i;
>+
>+	/* We should only be called from code paths where we know we
>+	 * hold both the active reference *and* a user reference.
>+	 * Therefore we can safely access the object after retiring as
>+	 * we will hold a second reference and not free the object.
>+	 */
>+
>+	rq = obj->last_write.request;
>+	if (rq && i915_request_complete(rq))
>+		i915_gem_object_retire__write(obj);
>+
>+	rq = obj->last_fence.request;
>+	if (rq && i915_request_complete(rq))
>+		i915_gem_object_retire__fence(obj);
>+
>+	for (i = 0; i < I915_NUM_ENGINES; i++) {
>+		rq = obj->last_read[i].request;
>+		if (rq && i915_request_complete(rq))
>+			i915_gem_object_retire__read(obj, rq->engine);
>+	}
>+
>+	if (!obj->active)
>+		i915_gem_retire_requests(obj->base.dev);
>+}
>+
> static int
> i915_gem_wait_for_error(struct i915_gpu_error *error)
> {
> 	int ret;
> 
>-#define EXIT_COND (!i915_reset_in_progress(error) || \
>-		   i915_terminally_wedged(error))
>-	if (EXIT_COND)
>-		return 0;
>-
> 	/*
> 	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
> 	 * userspace. If it takes that long something really bad is going on and
> 	 * we should simply try to bail out and fail as gracefully as possible.
> 	 */
> 	ret = wait_event_interruptible_timeout(error->reset_queue,
>-					       EXIT_COND,
>+					       !i915_recovery_pending(error),
> 					       10*HZ);
> 	if (ret == 0) {
> 		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
>@@ -132,7 +215,6 @@ i915_gem_wait_for_error(struct i915_gpu_error *error)
> 	} else if (ret < 0) {
> 		return ret;
> 	}
>-#undef EXIT_COND
> 
> 	return 0;
> }
>@@ -150,7 +232,6 @@ int i915_mutex_lock_interruptible(struct drm_device *dev)
> 	if (ret)
> 		return ret;
> 
>-	WARN_ON(i915_verify_lists(dev));
> 	return 0;
> }
> 
>@@ -474,8 +555,6 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
> 		ret = i915_gem_object_wait_rendering(obj, true);
> 		if (ret)
> 			return ret;
>-
>-		i915_gem_object_retire(obj);
> 	}
> 
> 	ret = i915_gem_object_get_pages(obj);
>@@ -891,8 +970,6 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
> 		ret = i915_gem_object_wait_rendering(obj, false);
> 		if (ret)
> 			return ret;
>-
>-		i915_gem_object_retire(obj);
> 	}
> 	/* Same trick applies to invalidate partially written cachelines read
> 	 * before writing. */
>@@ -1071,229 +1148,6 @@ unlock:
> 	return ret;
> }
> 
>-int
>-i915_gem_check_wedge(struct i915_gpu_error *error,
>-		     bool interruptible)
>-{
>-	if (i915_reset_in_progress(error)) {
>-		/* Non-interruptible callers can't handle -EAGAIN, hence return
>-		 * -EIO unconditionally for these. */
>-		if (!interruptible)
>-			return -EIO;
>-
>-		/* Recovery complete, but the reset failed ... */
>-		if (i915_terminally_wedged(error))
>-			return -EIO;
>-
>-		return -EAGAIN;
>-	}
>-
>-	return 0;
>-}
>-
>-/*
>- * Compare seqno against outstanding lazy request. Emit a request if they are
>- * equal.
>- */
>-int
>-i915_gem_check_olr(struct intel_engine_cs *ring, u32 seqno)
>-{
>-	int ret;
>-
>-	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
>-
>-	ret = 0;
>-	if (seqno == ring->outstanding_lazy_seqno)
>-		ret = i915_add_request(ring, NULL);
>-
>-	return ret;
>-}
>-
>-static void fake_irq(unsigned long data)
>-{
>-	wake_up_process((struct task_struct *)data);
>-}
>-
>-static bool missed_irq(struct drm_i915_private *dev_priv,
>-		       struct intel_engine_cs *ring)
>-{
>-	return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
>-}
>-
>-static bool can_wait_boost(struct drm_i915_file_private *file_priv)
>-{
>-	if (file_priv == NULL)
>-		return true;
>-
>-	return !atomic_xchg(&file_priv->rps_wait_boost, true);
>-}
>-
>-/**
>- * __wait_seqno - wait until execution of seqno has finished
>- * @ring: the ring expected to report seqno
>- * @seqno: duh!
>- * @reset_counter: reset sequence associated with the given seqno
>- * @interruptible: do an interruptible wait (normally yes)
>- * @timeout: in - how long to wait (NULL forever); out - how much time remaining
>- *
>- * Note: It is of utmost importance that the passed in seqno and reset_counter
>- * values have been read by the caller in an smp safe manner. Where read-side
>- * locks are involved, it is sufficient to read the reset_counter before
>- * unlocking the lock that protects the seqno. For lockless tricks, the
>- * reset_counter _must_ be read before, and an appropriate smp_rmb must be
>- * inserted.
>- *
>- * Returns 0 if the seqno was found within the alloted time. Else returns the
>- * errno with remaining time filled in timeout argument.
>- */
>-static int __wait_seqno(struct intel_engine_cs *ring, u32 seqno,
>-			unsigned reset_counter,
>-			bool interruptible,
>-			s64 *timeout,
>-			struct drm_i915_file_private *file_priv)
>-{
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	const bool irq_test_in_progress =
>-		ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
>-	DEFINE_WAIT(wait);
>-	unsigned long timeout_expire;
>-	s64 before, now;
>-	int ret;
>-
>-	WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
>-
>-	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
>-		return 0;
>-
>-	timeout_expire = timeout ? jiffies + nsecs_to_jiffies((u64)*timeout) : 0;
>-
>-	if (INTEL_INFO(dev)->gen >= 6 && ring->id == RCS && can_wait_boost(file_priv)) {
>-		gen6_rps_boost(dev_priv);
>-		if (file_priv)
>-			mod_delayed_work(dev_priv->wq,
>-					 &file_priv->mm.idle_work,
>-					 msecs_to_jiffies(100));
>-	}
>-
>-	if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring)))
>-		return -ENODEV;
>-
>-	/* Record current time in case interrupted by signal, or wedged */
>-	trace_i915_gem_request_wait_begin(ring, seqno);
>-	before = ktime_get_raw_ns();
>-	for (;;) {
>-		struct timer_list timer;
>-
>-		prepare_to_wait(&ring->irq_queue, &wait,
>-				interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
>-
>-		/* We need to check whether any gpu reset happened in between
>-		 * the caller grabbing the seqno and now ... */
>-		if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
>-			/* ... but upgrade the -EAGAIN to an -EIO if the gpu
>-			 * is truely gone. */
>-			ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
>-			if (ret == 0)
>-				ret = -EAGAIN;
>-			break;
>-		}
>-
>-		if (i915_seqno_passed(ring->get_seqno(ring, false), seqno)) {
>-			ret = 0;
>-			break;
>-		}
>-
>-		if (interruptible && signal_pending(current)) {
>-			ret = -ERESTARTSYS;
>-			break;
>-		}
>-
>-		if (timeout && time_after_eq(jiffies, timeout_expire)) {
>-			ret = -ETIME;
>-			break;
>-		}
>-
>-		timer.function = NULL;
>-		if (timeout || missed_irq(dev_priv, ring)) {
>-			unsigned long expire;
>-
>-			setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
>-			expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
>-			mod_timer(&timer, expire);
>-		}
>-
>-		io_schedule();
>-
>-		if (timer.function) {
>-			del_singleshot_timer_sync(&timer);
>-			destroy_timer_on_stack(&timer);
>-		}
>-	}
>-	now = ktime_get_raw_ns();
>-	trace_i915_gem_request_wait_end(ring, seqno);
>-
>-	if (!irq_test_in_progress)
>-		ring->irq_put(ring);
>-
>-	finish_wait(&ring->irq_queue, &wait);
>-
>-	if (timeout) {
>-		s64 tres = *timeout - (now - before);
>-
>-		*timeout = tres < 0 ? 0 : tres;
>-	}
>-
>-	return ret;
>-}
>-
>-/**
>- * Waits for a sequence number to be signaled, and cleans up the
>- * request and object lists appropriately for that event.
>- */
>-int
>-i915_wait_seqno(struct intel_engine_cs *ring, uint32_t seqno)
>-{
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	bool interruptible = dev_priv->mm.interruptible;
>-	int ret;
>-
>-	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
>-	BUG_ON(seqno == 0);
>-
>-	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
>-	if (ret)
>-		return ret;
>-
>-	ret = i915_gem_check_olr(ring, seqno);
>-	if (ret)
>-		return ret;
>-
>-	return __wait_seqno(ring, seqno,
>-			    atomic_read(&dev_priv->gpu_error.reset_counter),
>-			    interruptible, NULL, NULL);
>-}
>-
>-static int
>-i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
>-				     struct intel_engine_cs *ring)
>-{
>-	if (!obj->active)
>-		return 0;
>-
>-	/* Manually manage the write flush as we may have not yet
>-	 * retired the buffer.
>-	 *
>-	 * Note that the last_write_seqno is always the earlier of
>-	 * the two (read/write) seqno, so if we haved successfully waited,
>-	 * we know we have passed the last write.
>-	 */
>-	obj->last_write_seqno = 0;
>-
>-	return 0;
>-}
>-
> /**
>  * Ensures that all rendering to the object has completed and the object is
>  * safe to unbind from the GTT or access from the CPU.
>@@ -1302,19 +1156,30 @@ static __must_check int
> i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
> 			       bool readonly)
> {
>-	struct intel_engine_cs *ring = obj->ring;
>-	u32 seqno;
>-	int ret;
>+	int i, ret;
> 
>-	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
>-	if (seqno == 0)
>+	if (!obj->active)
> 		return 0;
> 
>-	ret = i915_wait_seqno(ring, seqno);
>-	if (ret)
>-		return ret;
>+	if (readonly) {
>+		if (obj->last_write.request) {
>+			ret = i915_request_wait(obj->last_write.request);
>+			if (ret)
>+				return ret;
>+		}
>+	} else {
>+		for (i = 0; i < I915_NUM_ENGINES; i++) {
>+			if (obj->last_read[i].request == NULL)
>+				continue;
>+
>+			ret = i915_request_wait(obj->last_read[i].request);
>+			if (ret)
>+				return ret;
>+		}
>+	}
> 
>-	return i915_gem_object_wait_rendering__tail(obj, ring);
>+	i915_gem_object_retire(obj);
>+	return 0;
> }
> 
> /* A nonblocking variant of the above wait. This is a highly dangerous routine
>@@ -1327,34 +1192,51 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
> {
> 	struct drm_device *dev = obj->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = obj->ring;
>-	unsigned reset_counter;
>-	u32 seqno;
>-	int ret;
>+	struct i915_gem_request *rq[I915_NUM_ENGINES] = {};
>+	int i, n, ret = 0;
> 
> 	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
> 	BUG_ON(!dev_priv->mm.interruptible);
> 
>-	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
>-	if (seqno == 0)
>+	n = 0;
>+	if (readonly) {
>+		if (obj->last_write.request) {
>+			rq[n] = i915_request_get_breadcrumb(obj->last_write.request);
>+			if (IS_ERR(rq[n]))
>+				return PTR_ERR(rq[n]);
>+			n++;
>+		}
>+	} else {
>+		for (i = 0; i < I915_NUM_ENGINES; i++) {
>+			if (obj->last_read[i].request == NULL)
>+				continue;
>+
>+			rq[n] = i915_request_get_breadcrumb(obj->last_read[i].request);
>+			if (IS_ERR(rq[n])) {
>+				ret = PTR_ERR(rq[n]);
>+				goto out;
>+			}
>+			n++;
>+		}
>+	}
>+	if (n == 0)
> 		return 0;
> 
>-	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
>-	if (ret)
>-		return ret;
>+	mutex_unlock(&dev->struct_mutex);
> 
>-	ret = i915_gem_check_olr(ring, seqno);
>-	if (ret)
>-		return ret;
>+	for (i = 0; i < n; i++) {
>+		ret = __i915_request_wait(rq[i], true, NULL, file_priv);
>+		if (ret)
>+			break;
>+	}
> 
>-	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
>-	mutex_unlock(&dev->struct_mutex);
>-	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file_priv);
> 	mutex_lock(&dev->struct_mutex);
>-	if (ret)
>-		return ret;
> 
>-	return i915_gem_object_wait_rendering__tail(obj, ring);
>+out:
>+	for (i = 0; i < n; i++)
>+		i915_request_put(rq[i]);
>+
>+	return ret;
> }
> 
> /**
>@@ -1735,7 +1617,11 @@ static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
> 	 * offsets on purgeable objects by truncating it and marking it purged,
> 	 * which prevents userspace from ever using that object again.
> 	 */
>-	i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
>+	i915_gem_shrink(dev_priv,
>+			obj->base.size >> PAGE_SHIFT,
>+			I915_SHRINK_BOUND |
>+			I915_SHRINK_UNBOUND |
>+			I915_SHRINK_PURGEABLE);
> 	ret = drm_gem_create_mmap_offset(&obj->base);
> 	if (ret != -ENOSPC)
> 		goto out;
>@@ -1932,12 +1818,11 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
> 	return 0;
> }
> 
>-static unsigned long
>-__i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
>-		  bool purgeable_only)
>+unsigned long
>+i915_gem_shrink(struct drm_i915_private *dev_priv,
>+		long target, unsigned flags)
> {
>-	struct list_head still_in_list;
>-	struct drm_i915_gem_object *obj;
>+	const bool purgeable_only = flags & I915_SHRINK_PURGEABLE;
> 	unsigned long count = 0;
> 
> 	/*
>@@ -1959,62 +1844,68 @@ __i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
> 	 * dev->struct_mutex and so we won't ever be able to observe an
> 	 * object on the bound_list with a reference count equals 0.
> 	 */
>-	INIT_LIST_HEAD(&still_in_list);
>-	while (count < target && !list_empty(&dev_priv->mm.unbound_list)) {
>-		obj = list_first_entry(&dev_priv->mm.unbound_list,
>-				       typeof(*obj), global_list);
>-		list_move_tail(&obj->global_list, &still_in_list);
>+	if (flags & I915_SHRINK_UNBOUND) {
>+		struct list_head still_in_list;
> 
>-		if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
>-			continue;
>+		INIT_LIST_HEAD(&still_in_list);
>+		while (count < target && !list_empty(&dev_priv->mm.unbound_list)) {
>+			struct drm_i915_gem_object *obj;
> 
>-		drm_gem_object_reference(&obj->base);
>+			obj = list_first_entry(&dev_priv->mm.unbound_list,
>+					       typeof(*obj), global_list);
>+			list_move_tail(&obj->global_list, &still_in_list);
> 
>-		if (i915_gem_object_put_pages(obj) == 0)
>-			count += obj->base.size >> PAGE_SHIFT;
>+			if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
>+				continue;
>+
>+			drm_gem_object_reference(&obj->base);
>+
>+			if (i915_gem_object_put_pages(obj) == 0)
>+				count += obj->base.size >> PAGE_SHIFT;
> 
>-		drm_gem_object_unreference(&obj->base);
>+			drm_gem_object_unreference(&obj->base);
>+		}
>+		list_splice(&still_in_list, &dev_priv->mm.unbound_list);
> 	}
>-	list_splice(&still_in_list, &dev_priv->mm.unbound_list);
> 
>-	INIT_LIST_HEAD(&still_in_list);
>-	while (count < target && !list_empty(&dev_priv->mm.bound_list)) {
>-		struct i915_vma *vma, *v;
>+	if (flags & I915_SHRINK_BOUND) {
>+		struct list_head still_in_list;
> 
>-		obj = list_first_entry(&dev_priv->mm.bound_list,
>-				       typeof(*obj), global_list);
>-		list_move_tail(&obj->global_list, &still_in_list);
>+		INIT_LIST_HEAD(&still_in_list);
>+		while (count < target && !list_empty(&dev_priv->mm.bound_list)) {
>+			struct drm_i915_gem_object *obj;
>+			struct i915_vma *vma, *v;
> 
>-		if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
>-			continue;
>+			obj = list_first_entry(&dev_priv->mm.bound_list,
>+					       typeof(*obj), global_list);
>+			list_move_tail(&obj->global_list, &still_in_list);
> 
>-		drm_gem_object_reference(&obj->base);
>+			if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
>+				continue;
> 
>-		list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link)
>-			if (i915_vma_unbind(vma))
>-				break;
>+			drm_gem_object_reference(&obj->base);
> 
>-		if (i915_gem_object_put_pages(obj) == 0)
>-			count += obj->base.size >> PAGE_SHIFT;
>+			list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link)
>+				if (i915_vma_unbind(vma))
>+					break;
> 
>-		drm_gem_object_unreference(&obj->base);
>+			if (i915_gem_object_put_pages(obj) == 0)
>+				count += obj->base.size >> PAGE_SHIFT;
>+
>+			drm_gem_object_unreference(&obj->base);
>+		}
>+		list_splice(&still_in_list, &dev_priv->mm.bound_list);
> 	}
>-	list_splice(&still_in_list, &dev_priv->mm.bound_list);
> 
> 	return count;
> }
> 
> static unsigned long
>-i915_gem_purge(struct drm_i915_private *dev_priv, long target)
>-{
>-	return __i915_gem_shrink(dev_priv, target, true);
>-}
>-
>-static unsigned long
> i915_gem_shrink_all(struct drm_i915_private *dev_priv)
> {
> 	i915_gem_evict_everything(dev_priv->dev);
>-	return __i915_gem_shrink(dev_priv, LONG_MAX, false);
>+	return i915_gem_shrink(dev_priv, LONG_MAX,
>+			       I915_SHRINK_BOUND | I915_SHRINK_UNBOUND);
> }
> 
> static int
>@@ -2061,7 +1952,11 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
> 	for (i = 0; i < page_count; i++) {
> 		page = shmem_read_mapping_page_gfp(mapping, i, gfp);
> 		if (IS_ERR(page)) {
>-			i915_gem_purge(dev_priv, page_count);
>+			i915_gem_shrink(dev_priv,
>+					page_count,
>+					I915_SHRINK_BOUND |
>+					I915_SHRINK_UNBOUND |
>+					I915_SHRINK_PURGEABLE);
> 			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
> 		}
> 		if (IS_ERR(page)) {
>@@ -2159,405 +2054,75 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
> 	return 0;
> }
> 
>-static void
>-i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
>-			       struct intel_engine_cs *ring)
>+int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
> {
>-	struct drm_device *dev = obj->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	u32 seqno = intel_ring_get_seqno(ring);
>-
>-	BUG_ON(ring == NULL);
>-	if (obj->ring != ring && obj->last_write_seqno) {
>-		/* Keep the seqno relative to the current ring */
>-		obj->last_write_seqno = seqno;
>-	}
>-	obj->ring = ring;
>-
>-	/* Add a reference if we're newly entering the active list. */
>-	if (!obj->active) {
>-		drm_gem_object_reference(&obj->base);
>-		obj->active = 1;
>-	}
>-
>-	list_move_tail(&obj->ring_list, &ring->active_list);
>+	struct intel_engine_cs *signaller, *waiter;
>+	int ret, i, j;
> 
>-	obj->last_read_seqno = seqno;
>+	if (seqno == 0)
>+		return -EINVAL;
> 
>-	if (obj->fenced_gpu_access) {
>-		obj->last_fenced_seqno = seqno;
>+	if (seqno == dev_priv->next_seqno)
>+		return 0;
> 
>-		/* Bump MRU to take account of the delayed flush */
>-		if (obj->fence_reg != I915_FENCE_REG_NONE) {
>-			struct drm_i915_fence_reg *reg;
>+	do {
>+		/* Flush the breadcrumbs */
>+		ret = i915_gpu_idle(dev);
>+		if (ret)
>+			return ret;
> 
>-			reg = &dev_priv->fence_regs[obj->fence_reg];
>-			list_move_tail(&reg->lru_list,
>-				       &dev_priv->mm.fence_list);
>-		}
>-	}
>-}
>+		if (!i915_gem_retire_requests(dev))
>+			return -EIO;
> 
>-void i915_vma_move_to_active(struct i915_vma *vma,
>-			     struct intel_engine_cs *ring)
>-{
>-	list_move_tail(&vma->mm_list, &vma->vm->active_list);
>-	return i915_gem_object_move_to_active(vma->obj, ring);
>-}
>+		/* Update all semaphores to the current value */
>+		for_each_engine(signaller, to_i915(dev), i) {
>+			struct i915_gem_request *rq;
> 
>-static void
>-i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
>-{
>-	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
>-	struct i915_address_space *vm;
>-	struct i915_vma *vma;
>+			if (!signaller->semaphore.signal)
>+				continue;
> 
>-	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
>-	BUG_ON(!obj->active);
>+			rq = intel_engine_alloc_request(signaller,
>+							signaller->default_context);
>+			if (IS_ERR(rq))
>+				return PTR_ERR(rq);
> 
>-	list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
>-		vma = i915_gem_obj_to_vma(obj, vm);
>-		if (vma && !list_empty(&vma->mm_list))
>-			list_move_tail(&vma->mm_list, &vm->inactive_list);
>-	}
>+			for_each_engine(waiter, to_i915(dev), j) {
>+				if (signaller == waiter)
>+					continue;
> 
>-	intel_fb_obj_flush(obj, true);
>+				if (!waiter->semaphore.wait)
>+					continue;
> 
>-	list_del_init(&obj->ring_list);
>-	obj->ring = NULL;
>+				ret = i915_request_emit_semaphore(rq, waiter->id);
>+				if (ret)
>+					break;
>+			}
> 
>-	obj->last_read_seqno = 0;
>-	obj->last_write_seqno = 0;
>-	obj->base.write_domain = 0;
>+			if (ret == 0)
>+				ret = i915_request_commit(rq);
>+			i915_request_put(rq);
>+			if (ret)
>+				return ret;
>+		}
> 
>-	obj->last_fenced_seqno = 0;
>-	obj->fenced_gpu_access = false;
>+		/* We can only roll seqno forwards across a wraparound.
>+		 * This ship is not for turning!
>+		 */
>+		if (!__i915_seqno_passed(dev_priv->next_seqno, seqno))
>+			break;
> 
>-	obj->active = 0;
>-	drm_gem_object_unreference(&obj->base);
>+		dev_priv->next_seqno += 0x40000000;
>+	}while (1);
> 
>-	WARN_ON(i915_verify_lists(dev));
>+	dev_priv->next_seqno = seqno;
>+	return 0;
> }
> 
>-static void
>-i915_gem_object_retire(struct drm_i915_gem_object *obj)
>+void i915_gem_restore_fences(struct drm_device *dev)
> {
>-	struct intel_engine_cs *ring = obj->ring;
>-
>-	if (ring == NULL)
>-		return;
>-
>-	if (i915_seqno_passed(ring->get_seqno(ring, true),
>-			      obj->last_read_seqno))
>-		i915_gem_object_move_to_inactive(obj);
>-}
>-
>-static int
>-i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>-	int ret, i, j;
>-
>-	/* Carefully retire all requests without writing to the rings */
>-	for_each_ring(ring, dev_priv, i) {
>-		ret = intel_ring_idle(ring);
>-		if (ret)
>-			return ret;
>-	}
>-	i915_gem_retire_requests(dev);
>-
>-	/* Finally reset hw state */
>-	for_each_ring(ring, dev_priv, i) {
>-		intel_ring_init_seqno(ring, seqno);
>-
>-		for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
>-			ring->semaphore.sync_seqno[j] = 0;
>-	}
>-
>-	return 0;
>-}
>-
>-int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int ret;
>-
>-	if (seqno == 0)
>-		return -EINVAL;
>-
>-	/* HWS page needs to be set less than what we
>-	 * will inject to ring
>-	 */
>-	ret = i915_gem_init_seqno(dev, seqno - 1);
>-	if (ret)
>-		return ret;
>-
>-	/* Carefully set the last_seqno value so that wrap
>-	 * detection still works
>-	 */
>-	dev_priv->next_seqno = seqno;
>-	dev_priv->last_seqno = seqno - 1;
>-	if (dev_priv->last_seqno == 0)
>-		dev_priv->last_seqno--;
>-
>-	return 0;
>-}
>-
>-int
>-i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-
>-	/* reserve 0 for non-seqno */
>-	if (dev_priv->next_seqno == 0) {
>-		int ret = i915_gem_init_seqno(dev, 0);
>-		if (ret)
>-			return ret;
>-
>-		dev_priv->next_seqno = 1;
>-	}
>-
>-	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
>-	return 0;
>-}
>-
>-int __i915_add_request(struct intel_engine_cs *ring,
>-		       struct drm_file *file,
>-		       struct drm_i915_gem_object *obj,
>-		       u32 *out_seqno)
>-{
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>-	struct drm_i915_gem_request *request;
>-	u32 request_ring_position, request_start;
>-	int ret;
>-
>-	request_start = intel_ring_get_tail(ring->buffer);
>-	/*
>-	 * Emit any outstanding flushes - execbuf can fail to emit the flush
>-	 * after having emitted the batchbuffer command. Hence we need to fix
>-	 * things up similar to emitting the lazy request. The difference here
>-	 * is that the flush _must_ happen before the next request, no matter
>-	 * what.
>-	 */
>-	ret = intel_ring_flush_all_caches(ring);
>-	if (ret)
>-		return ret;
>-
>-	request = ring->preallocated_lazy_request;
>-	if (WARN_ON(request == NULL))
>-		return -ENOMEM;
>-
>-	/* Record the position of the start of the request so that
>-	 * should we detect the updated seqno part-way through the
>-	 * GPU processing the request, we never over-estimate the
>-	 * position of the head.
>-	 */
>-	request_ring_position = intel_ring_get_tail(ring->buffer);
>-
>-	ret = ring->add_request(ring);
>-	if (ret)
>-		return ret;
>-
>-	request->seqno = intel_ring_get_seqno(ring);
>-	request->ring = ring;
>-	request->head = request_start;
>-	request->tail = request_ring_position;
>-
>-	/* Whilst this request exists, batch_obj will be on the
>-	 * active_list, and so will hold the active reference. Only when this
>-	 * request is retired will the the batch_obj be moved onto the
>-	 * inactive_list and lose its active reference. Hence we do not need
>-	 * to explicitly hold another reference here.
>-	 */
>-	request->batch_obj = obj;
>-
>-	/* Hold a reference to the current context so that we can inspect
>-	 * it later in case a hangcheck error event fires.
>-	 */
>-	request->ctx = ring->last_context;
>-	if (request->ctx)
>-		i915_gem_context_reference(request->ctx);
>-
>-	request->emitted_jiffies = jiffies;
>-	list_add_tail(&request->list, &ring->request_list);
>-	request->file_priv = NULL;
>-
>-	if (file) {
>-		struct drm_i915_file_private *file_priv = file->driver_priv;
>-
>-		spin_lock(&file_priv->mm.lock);
>-		request->file_priv = file_priv;
>-		list_add_tail(&request->client_list,
>-			      &file_priv->mm.request_list);
>-		spin_unlock(&file_priv->mm.lock);
>-	}
>-
>-	trace_i915_gem_request_add(ring, request->seqno);
>-	ring->outstanding_lazy_seqno = 0;
>-	ring->preallocated_lazy_request = NULL;
>-
>-	if (!dev_priv->ums.mm_suspended) {
>-		i915_queue_hangcheck(ring->dev);
>-
>-		cancel_delayed_work_sync(&dev_priv->mm.idle_work);
>-		queue_delayed_work(dev_priv->wq,
>-				   &dev_priv->mm.retire_work,
>-				   round_jiffies_up_relative(HZ));
>-		intel_mark_busy(dev_priv->dev);
>-	}
>-
>-	if (out_seqno)
>-		*out_seqno = request->seqno;
>-	return 0;
>-}
>-
>-static inline void
>-i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
>-{
>-	struct drm_i915_file_private *file_priv = request->file_priv;
>-
>-	if (!file_priv)
>-		return;
>-
>-	spin_lock(&file_priv->mm.lock);
>-	list_del(&request->client_list);
>-	request->file_priv = NULL;
>-	spin_unlock(&file_priv->mm.lock);
>-}
>-
>-static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
>-				   const struct intel_context *ctx)
>-{
>-	unsigned long elapsed;
>-
>-	elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
>-
>-	if (ctx->hang_stats.banned)
>-		return true;
>-
>-	if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
>-		if (!i915_gem_context_is_default(ctx)) {
>-			DRM_DEBUG("context hanging too fast, banning!\n");
>-			return true;
>-		} else if (i915_stop_ring_allow_ban(dev_priv)) {
>-			if (i915_stop_ring_allow_warn(dev_priv))
>-				DRM_ERROR("gpu hanging too fast, banning!\n");
>-			return true;
>-		}
>-	}
>-
>-	return false;
>-}
>-
>-static void i915_set_reset_status(struct drm_i915_private *dev_priv,
>-				  struct intel_context *ctx,
>-				  const bool guilty)
>-{
>-	struct i915_ctx_hang_stats *hs;
>-
>-	if (WARN_ON(!ctx))
>-		return;
>-
>-	hs = &ctx->hang_stats;
>-
>-	if (guilty) {
>-		hs->banned = i915_context_is_banned(dev_priv, ctx);
>-		hs->batch_active++;
>-		hs->guilty_ts = get_seconds();
>-	} else {
>-		hs->batch_pending++;
>-	}
>-}
>-
>-static void i915_gem_free_request(struct drm_i915_gem_request *request)
>-{
>-	list_del(&request->list);
>-	i915_gem_request_remove_from_client(request);
>-
>-	if (request->ctx)
>-		i915_gem_context_unreference(request->ctx);
>-
>-	kfree(request);
>-}
>-
>-struct drm_i915_gem_request *
>-i915_gem_find_active_request(struct intel_engine_cs *ring)
>-{
>-	struct drm_i915_gem_request *request;
>-	u32 completed_seqno;
>-
>-	completed_seqno = ring->get_seqno(ring, false);
>-
>-	list_for_each_entry(request, &ring->request_list, list) {
>-		if (i915_seqno_passed(completed_seqno, request->seqno))
>-			continue;
>-
>-		return request;
>-	}
>-
>-	return NULL;
>-}
>-
>-static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
>-				       struct intel_engine_cs *ring)
>-{
>-	struct drm_i915_gem_request *request;
>-	bool ring_hung;
>-
>-	request = i915_gem_find_active_request(ring);
>-
>-	if (request == NULL)
>-		return;
>-
>-	ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
>-
>-	i915_set_reset_status(dev_priv, request->ctx, ring_hung);
>-
>-	list_for_each_entry_continue(request, &ring->request_list, list)
>-		i915_set_reset_status(dev_priv, request->ctx, false);
>-}
>-
>-static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
>-					struct intel_engine_cs *ring)
>-{
>-	while (!list_empty(&ring->active_list)) {
>-		struct drm_i915_gem_object *obj;
>-
>-		obj = list_first_entry(&ring->active_list,
>-				       struct drm_i915_gem_object,
>-				       ring_list);
>-
>-		i915_gem_object_move_to_inactive(obj);
>-	}
>-
>-	/*
>-	 * We must free the requests after all the corresponding objects have
>-	 * been moved off active lists. Which is the same order as the normal
>-	 * retire_requests function does. This is important if object hold
>-	 * implicit references on things like e.g. ppgtt address spaces through
>-	 * the request.
>-	 */
>-	while (!list_empty(&ring->request_list)) {
>-		struct drm_i915_gem_request *request;
>-
>-		request = list_first_entry(&ring->request_list,
>-					   struct drm_i915_gem_request,
>-					   list);
>-
>-		i915_gem_free_request(request);
>-	}
>-
>-	/* These may not have been flush before the reset, do so now */
>-	kfree(ring->preallocated_lazy_request);
>-	ring->preallocated_lazy_request = NULL;
>-	ring->outstanding_lazy_seqno = 0;
>-}
>-
>-void i915_gem_restore_fences(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int i;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	int i;
> 
> 	for (i = 0; i < dev_priv->num_fence_regs; i++) {
> 		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
>@@ -2578,21 +2143,25 @@ void i915_gem_restore_fences(struct drm_device *dev)
> void i915_gem_reset(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	int i;
> 
>-	/*
>-	 * Before we free the objects from the requests, we need to inspect
>-	 * them for finding the guilty party. As the requests only borrow
>-	 * their reference to the objects, the inspection must be done first.
>-	 */
>-	for_each_ring(ring, dev_priv, i)
>-		i915_gem_reset_ring_status(dev_priv, ring);
>+	for_each_engine(engine, dev_priv, i) {
>+		/* Clearing the read list will also clear the write
>+		 * and fence lists, 3 birds with one stone.
>+		 */
>+		while (!list_empty(&engine->read_list)) {
>+			struct drm_i915_gem_object *obj;
> 
>-	for_each_ring(ring, dev_priv, i)
>-		i915_gem_reset_ring_cleanup(dev_priv, ring);
>+			obj = list_first_entry(&engine->read_list,
>+					       struct drm_i915_gem_object,
>+					       last_read[i].engine_list);
> 
>-	i915_gem_context_reset(dev);
>+			i915_gem_object_retire__read(obj, engine);
>+		}
>+
>+		intel_engine_reset(engine);
>+	}
> 
> 	i915_gem_restore_fences(dev);
> }
>@@ -2601,82 +2170,71 @@ void i915_gem_reset(struct drm_device *dev)
>  * This function clears the request list as sequence numbers are passed.
>  */
> void
>-i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
>+i915_gem_retire_requests__engine(struct intel_engine_cs *engine)
> {
>-	uint32_t seqno;
>-
>-	if (list_empty(&ring->request_list))
>+	if (engine->last_request == NULL)
> 		return;
> 
>-	WARN_ON(i915_verify_lists(ring->dev));
>-
>-	seqno = ring->get_seqno(ring, true);
>+	if (!intel_engine_retire(engine, engine->get_seqno(engine)))
>+		return;
> 
>-	/* Move any buffers on the active list that are no longer referenced
>-	 * by the ringbuffer to the flushing/inactive lists as appropriate,
>-	 * before we free the context associated with the requests.
>-	 */
>-	while (!list_empty(&ring->active_list)) {
>+	while (!list_empty(&engine->read_list)) {
> 		struct drm_i915_gem_object *obj;
> 
>-		obj = list_first_entry(&ring->active_list,
>-				      struct drm_i915_gem_object,
>-				      ring_list);
>+		obj = list_first_entry(&engine->read_list,
>+				       struct drm_i915_gem_object,
>+				       last_read[engine->id].engine_list);
> 
>-		if (!i915_seqno_passed(seqno, obj->last_read_seqno))
>+		if (!obj->last_read[engine->id].request->completed)
> 			break;
> 
>-		i915_gem_object_move_to_inactive(obj);
>+		i915_gem_object_retire__read(obj, engine);
> 	}
> 
>+	while (!list_empty(&engine->fence_list)) {
>+		struct drm_i915_gem_object *obj;
> 
>-	while (!list_empty(&ring->request_list)) {
>-		struct drm_i915_gem_request *request;
>-
>-		request = list_first_entry(&ring->request_list,
>-					   struct drm_i915_gem_request,
>-					   list);
>+		obj = list_first_entry(&engine->fence_list,
>+				       struct drm_i915_gem_object,
>+				       last_fence.engine_list);
> 
>-		if (!i915_seqno_passed(seqno, request->seqno))
>+		if (!obj->last_fence.request->completed)
> 			break;
> 
>-		trace_i915_gem_request_retire(ring, request->seqno);
>-		/* We know the GPU must have read the request to have
>-		 * sent us the seqno + interrupt, so use the position
>-		 * of tail of the request to update the last known position
>-		 * of the GPU head.
>-		 */
>-		ring->buffer->last_retired_head = request->tail;
>-
>-		i915_gem_free_request(request);
>+		i915_gem_object_retire__fence(obj);
> 	}
> 
>-	if (unlikely(ring->trace_irq_seqno &&
>-		     i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
>-		ring->irq_put(ring);
>-		ring->trace_irq_seqno = 0;
>-	}
>+	while (!list_empty(&engine->write_list)) {
>+		struct drm_i915_gem_object *obj;
> 
>-	WARN_ON(i915_verify_lists(ring->dev));
>+		obj = list_first_entry(&engine->write_list,
>+				       struct drm_i915_gem_object,
>+				       last_write.engine_list);
>+
>+		if (!obj->last_write.request->completed)
>+			break;
>+
>+		i915_gem_object_retire__write(obj);
>+	}
> }
> 
> bool
> i915_gem_retire_requests(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	bool idle = true;
> 	int i;
> 
>-	for_each_ring(ring, dev_priv, i) {
>-		i915_gem_retire_requests_ring(ring);
>-		idle &= list_empty(&ring->request_list);
>+	for_each_engine(engine, dev_priv, i) {
>+		i915_gem_retire_requests__engine(engine);
>+		idle &= engine->last_request == NULL;
> 	}
> 
> 	if (idle)
> 		mod_delayed_work(dev_priv->wq,
>-				   &dev_priv->mm.idle_work,
>-				   msecs_to_jiffies(100));
>+				 &dev_priv->mm.idle_work,
>+				 msecs_to_jiffies(100));
> 
> 	return idle;
> }
>@@ -2696,7 +2254,8 @@ i915_gem_retire_work_handler(struct work_struct *work)
> 		mutex_unlock(&dev->struct_mutex);
> 	}
> 	if (!idle)
>-		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
>+		queue_delayed_work(dev_priv->wq,
>+				   &dev_priv->mm.retire_work,
> 				   round_jiffies_up_relative(HZ));
> }
> 
>@@ -2719,14 +2278,16 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
> {
> 	int ret;
> 
>-	if (obj->active) {
>-		ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
>+	if (!obj->active)
>+		return 0;
>+
>+	if (obj->last_write.request) {
>+		ret = i915_request_emit_breadcrumb(obj->last_write.request);
> 		if (ret)
> 			return ret;
>-
>-		i915_gem_retire_requests_ring(obj->ring);
> 	}
> 
>+	i915_gem_object_retire(obj);
> 	return 0;
> }
> 
>@@ -2755,13 +2316,10 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
> int
> i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct drm_i915_gem_wait *args = data;
> 	struct drm_i915_gem_object *obj;
>-	struct intel_engine_cs *ring = NULL;
>-	unsigned reset_counter;
>-	u32 seqno = 0;
>-	int ret = 0;
>+	struct i915_gem_request *rq[I915_NUM_ENGINES] = {};
>+	int i, n, ret = 0;
> 
> 	ret = i915_mutex_lock_interruptible(dev);
> 	if (ret)
>@@ -2778,13 +2336,8 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> 	if (ret)
> 		goto out;
> 
>-	if (obj->active) {
>-		seqno = obj->last_read_seqno;
>-		ring = obj->ring;
>-	}
>-
>-	if (seqno == 0)
>-		 goto out;
>+	if (!obj->active)
>+		goto out;
> 
> 	/* Do this after OLR check to make sure we make forward progress polling
> 	 * on this IOCTL with a timeout <=0 (like busy ioctl)
>@@ -2794,12 +2347,31 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
> 		goto out;
> 	}
> 
>+	for (i = n = 0; i < I915_NUM_ENGINES; i++) {
>+		if (obj->last_read[i].request == NULL)
>+			continue;
>+
>+		rq[n] = i915_request_get_breadcrumb(obj->last_read[i].request);
>+		if (IS_ERR(rq[n])) {
>+			ret = PTR_ERR(rq[n]);
>+			break;
>+		}
>+		n++;
>+	}
>+
> 	drm_gem_object_unreference(&obj->base);
>-	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
> 	mutex_unlock(&dev->struct_mutex);
> 
>-	return __wait_seqno(ring, seqno, reset_counter, true, &args->timeout_ns,
>-			    file->driver_priv);
>+	for (i = 0; i < n; i++) {
>+		if (ret == 0)
>+			ret = __i915_request_wait(rq[i], true,
>+						  &args->timeout_ns,
>+						  file->driver_priv);
>+
>+		i915_request_put__unlocked(rq[i]);
>+	}
>+
>+	return ret;
> 
> out:
> 	drm_gem_object_unreference(&obj->base);
>@@ -2807,6 +2379,49 @@ out:
> 	return ret;
> }
> 
>+static int
>+__i915_request_sync(struct i915_gem_request *waiter,
>+		    struct i915_gem_request *signaller,
>+		    bool *retire)
>+{
>+	int ret;
>+
>+	if (signaller == NULL || i915_request_complete(signaller))
>+		return 0;
>+
>+	if (waiter == NULL)
>+		goto wait;
>+
>+	/* XXX still true with execlists? */
>+	if (waiter->engine == signaller->engine)
>+		return 0;
>+
>+	if (!waiter->engine->semaphore.wait)
>+		goto wait;
>+
>+	/* Try to emit only one wait per request per ring */
>+	if (waiter->semaphore[signaller->engine->id] &&
>+	    __i915_seqno_passed(waiter->semaphore[signaller->engine->id],
>+				signaller->seqno))
>+		return 0;
>+
>+	ret = i915_request_emit_semaphore(signaller, waiter->engine->id);
>+	if (ret)
>+		goto wait;
>+
>+	trace_i915_gem_ring_wait(signaller, waiter);
>+	if (waiter->engine->semaphore.wait(waiter, signaller))
>+		goto wait;
>+
>+	waiter->pending_flush &= ~I915_COMMAND_BARRIER;
>+	waiter->semaphore[signaller->engine->id] = signaller->breadcrumb[waiter->engine->id];
>+	return 0;
>+
>+wait:
>+	*retire = true;
>+	return i915_request_wait(signaller);
>+}
>+
> /**
>  * i915_gem_object_sync - sync an object to a ring.
>  *
>@@ -2821,38 +2436,23 @@ out:
>  */
> int
> i915_gem_object_sync(struct drm_i915_gem_object *obj,
>-		     struct intel_engine_cs *to)
>+		     struct i915_gem_request *rq)
> {
>-	struct intel_engine_cs *from = obj->ring;
>-	u32 seqno;
>-	int ret, idx;
>-
>-	if (from == NULL || to == from)
>-		return 0;
>-
>-	if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
>-		return i915_gem_object_wait_rendering(obj, false);
>-
>-	idx = intel_ring_sync_index(from, to);
>-
>-	seqno = obj->last_read_seqno;
>-	/* Optimization: Avoid semaphore sync when we are sure we already
>-	 * waited for an object with higher seqno */
>-	if (seqno <= from->semaphore.sync_seqno[idx])
>-		return 0;
>+	int ret = 0, i;
>+	bool retire = false;
> 
>-	ret = i915_gem_check_olr(obj->ring, seqno);
>-	if (ret)
>-		return ret;
>+	if (obj->base.pending_write_domain == 0) {
>+		ret = __i915_request_sync(rq, obj->last_write.request, &retire);
>+	} else {
>+		for (i = 0; i < I915_NUM_ENGINES; i++) {
>+			ret = __i915_request_sync(rq, obj->last_read[i].request, &retire);
>+			if (ret)
>+				break;
>+		}
>+	}
> 
>-	trace_i915_gem_ring_sync_to(from, to, seqno);
>-	ret = to->semaphore.sync_to(to, from, seqno);
>-	if (!ret)
>-		/* We use last_read_seqno because sync_to()
>-		 * might have just caused seqno wrap under
>-		 * the radar.
>-		 */
>-		from->semaphore.sync_seqno[idx] = obj->last_read_seqno;
>+	if (retire)
>+		i915_gem_object_retire(obj);
> 
> 	return ret;
> }
>@@ -2922,9 +2522,8 @@ int i915_vma_unbind(struct i915_vma *vma)
> 	vma->unbind_vma(vma);
> 
> 	list_del_init(&vma->mm_list);
>-	/* Avoid an unnecessary call to unbind on rebind. */
> 	if (i915_is_ggtt(vma->vm))
>-		obj->map_and_fenceable = true;
>+		obj->map_and_fenceable = false;
> 
> 	drm_mm_remove_node(&vma->node);
> 	i915_gem_vma_destroy(vma);
>@@ -2947,17 +2546,22 @@ int i915_vma_unbind(struct i915_vma *vma)
> 
> int i915_gpu_idle(struct drm_device *dev)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>-	int ret, i;
>+	struct intel_engine_cs *engine;
>+	int i;
> 
>-	/* Flush everything onto the inactive list. */
>-	for_each_ring(ring, dev_priv, i) {
>-		ret = i915_switch_context(ring, ring->default_context);
>-		if (ret)
>-			return ret;
>+	/* Flush everything including contexts onto the inactive list. */
>+	for_each_engine(engine, to_i915(dev), i) {
>+		struct i915_gem_request *rq;
>+		int ret;
>+
>+		rq = intel_engine_alloc_request(engine,
>+						engine->default_context);
>+		if (IS_ERR(rq))
>+			return PTR_ERR(rq);
>+
>+		ret = i915_request_wait(rq);
>+		i915_request_put(rq);
> 
>-		ret = intel_ring_idle(ring);
> 		if (ret)
> 			return ret;
> 	}
>@@ -3161,15 +2765,16 @@ static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
> static int
> i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
> {
>-	if (obj->last_fenced_seqno) {
>-		int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
>-		if (ret)
>-			return ret;
>+	int ret;
> 
>-		obj->last_fenced_seqno = 0;
>-	}
>+	if (obj->last_fence.request == NULL)
>+		return 0;
> 
>-	obj->fenced_gpu_access = false;
>+	ret = i915_request_wait(obj->last_fence.request);
>+	if (ret)
>+		return ret;
>+
>+	i915_gem_object_retire__fence(obj);
> 	return 0;
> }
> 
>@@ -3276,6 +2881,9 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
> 			return 0;
> 		}
> 	} else if (enable) {
>+		if (WARN_ON(!obj->map_and_fenceable))
>+			return -EINVAL;
>+
> 		reg = i915_find_fence_reg(dev);
> 		if (IS_ERR(reg))
> 			return PTR_ERR(reg);
>@@ -3297,17 +2905,20 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
> 	return 0;
> }
> 
>-static bool i915_gem_valid_gtt_space(struct drm_device *dev,
>-				     struct drm_mm_node *gtt_space,
>+static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
> 				     unsigned long cache_level)
> {
>+	struct drm_mm_node *gtt_space = &vma->node;
> 	struct drm_mm_node *other;
> 
>-	/* On non-LLC machines we have to be careful when putting differing
>-	 * types of snoopable memory together to avoid the prefetcher
>-	 * crossing memory domains and dying.
>+	/*
>+	 * On some machines we have to be careful when putting differing types
>+	 * of snoopable memory together to avoid the prefetcher crossing memory
>+	 * domains and dying. During vm initialisation, we decide whether or not
>+	 * these constraints apply and set the drm_mm.color_adjust
>+	 * appropriately.
> 	 */
>-	if (HAS_LLC(dev))
>+	if (vma->vm->mm.color_adjust == NULL)
> 		return true;
> 
> 	if (!drm_mm_node_allocated(gtt_space))
>@@ -3327,46 +2938,6 @@ static bool i915_gem_valid_gtt_space(struct drm_device *dev,
> 	return true;
> }
> 
>-static void i915_gem_verify_gtt(struct drm_device *dev)
>-{
>-#if WATCH_GTT
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_gem_object *obj;
>-	int err = 0;
>-
>-	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
>-		if (obj->gtt_space == NULL) {
>-			printk(KERN_ERR "object found on GTT list with no space reserved\n");
>-			err++;
>-			continue;
>-		}
>-
>-		if (obj->cache_level != obj->gtt_space->color) {
>-			printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
>-			       i915_gem_obj_ggtt_offset(obj),
>-			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
>-			       obj->cache_level,
>-			       obj->gtt_space->color);
>-			err++;
>-			continue;
>-		}
>-
>-		if (!i915_gem_valid_gtt_space(dev,
>-					      obj->gtt_space,
>-					      obj->cache_level)) {
>-			printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
>-			       i915_gem_obj_ggtt_offset(obj),
>-			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
>-			       obj->cache_level);
>-			err++;
>-			continue;
>-		}
>-	}
>-
>-	WARN_ON(err);
>-#endif
>-}
>-
> /**
>  * Finds free space in the GTT aperture and binds the object there.
>  */
>@@ -3445,8 +3016,7 @@ search_free:
> 
> 		goto err_free_vma;
> 	}
>-	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
>-					      obj->cache_level))) {
>+	if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
> 		ret = -EINVAL;
> 		goto err_remove_node;
> 	}
>@@ -3476,7 +3046,6 @@ search_free:
> 	vma->bind_vma(vma, obj->cache_level,
> 		      flags & (PIN_MAPPABLE | PIN_GLOBAL) ? GLOBAL_BIND : 0);
> 
>-	i915_gem_verify_gtt(dev);
> 	return vma;
> 
> err_remove_node:
>@@ -3586,11 +3155,12 @@ int
> i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
> {
> 	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
>+	struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
> 	uint32_t old_write_domain, old_read_domains;
> 	int ret;
> 
> 	/* Not valid to be called on unbound objects. */
>-	if (!i915_gem_obj_bound_any(obj))
>+	if (vma == NULL)
> 		return -EINVAL;
> 
> 	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
>@@ -3600,7 +3170,6 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
> 	if (ret)
> 		return ret;
> 
>-	i915_gem_object_retire(obj);
> 	i915_gem_object_flush_cpu_write_domain(obj, false);
> 
> 	/* Serialise direct access to this object with the barriers for
>@@ -3619,26 +3188,20 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
> 	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
> 	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
> 	if (write) {
>+		intel_fb_obj_invalidate(obj, NULL);
> 		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
> 		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
> 		obj->dirty = 1;
> 	}
> 
>-	if (write)
>-		intel_fb_obj_invalidate(obj, NULL);
>-
> 	trace_i915_gem_object_change_domain(obj,
> 					    old_read_domains,
> 					    old_write_domain);
> 
> 	/* And bump the LRU for this access */
>-	if (i915_gem_object_is_inactive(obj)) {
>-		struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
>-		if (vma)
>-			list_move_tail(&vma->mm_list,
>-				       &dev_priv->gtt.base.inactive_list);
>-
>-	}
>+	if (i915_gem_object_is_inactive(obj))
>+		list_move_tail(&vma->mm_list,
>+			       &dev_priv->gtt.base.inactive_list);
> 
> 	return 0;
> }
>@@ -3659,7 +3222,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
> 	}
> 
> 	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
>-		if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
>+		if (!i915_gem_valid_gtt_space(vma, cache_level)) {
> 			ret = i915_vma_unbind(vma);
> 			if (ret)
> 				return ret;
>@@ -3702,7 +3265,6 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
> 		 * in obj->write_domain and have been skipping the clflushes.
> 		 * Just set it to the CPU cache for now.
> 		 */
>-		i915_gem_object_retire(obj);
> 		WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
> 
> 		old_read_domains = obj->base.read_domains;
>@@ -3716,7 +3278,6 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
> 						    old_write_domain);
> 	}
> 
>-	i915_gem_verify_gtt(dev);
> 	return 0;
> }
> 
>@@ -3802,9 +3363,6 @@ static bool is_pin_display(struct drm_i915_gem_object *obj)
> {
> 	struct i915_vma *vma;
> 
>-	if (list_empty(&obj->vma_list))
>-		return false;
>-
> 	vma = i915_gem_obj_to_ggtt(obj);
> 	if (!vma)
> 		return false;
>@@ -3831,17 +3389,15 @@ static bool is_pin_display(struct drm_i915_gem_object *obj)
> int
> i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
> 				     u32 alignment,
>-				     struct intel_engine_cs *pipelined)
>+				     struct i915_gem_request *pipelined)
> {
> 	u32 old_read_domains, old_write_domain;
> 	bool was_pin_display;
> 	int ret;
> 
>-	if (pipelined != obj->ring) {
>-		ret = i915_gem_object_sync(obj, pipelined);
>-		if (ret)
>-			return ret;
>-	}
>+	ret = i915_gem_object_sync(obj, pipelined);
>+	if (ret)
>+		return ret;
> 
> 	/* Mark the pin_display early so that we account for the
> 	 * display coherency whilst setting up the cache domains.
>@@ -3937,7 +3493,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
> 	if (ret)
> 		return ret;
> 
>-	i915_gem_object_retire(obj);
> 	i915_gem_object_flush_gtt_write_domain(obj);
> 
> 	old_write_domain = obj->base.write_domain;
>@@ -3959,13 +3514,11 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
> 	 * need to be invalidated at next use.
> 	 */
> 	if (write) {
>+		intel_fb_obj_invalidate(obj, NULL);
> 		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
> 		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
> 	}
> 
>-	if (write)
>-		intel_fb_obj_invalidate(obj, NULL);
>-
> 	trace_i915_gem_object_change_domain(obj,
> 					    old_read_domains,
> 					    old_write_domain);
>@@ -3973,57 +3526,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
> 	return 0;
> }
> 
>-/* Throttle our rendering by waiting until the ring has completed our requests
>- * emitted over 20 msec ago.
>- *
>- * Note that if we were to use the current jiffies each time around the loop,
>- * we wouldn't escape the function with any frames outstanding if the time to
>- * render a frame was over 20ms.
>- *
>- * This should get us reasonable parallelism between CPU and GPU but also
>- * relatively low latency when blocking on a particular request to finish.
>- */
>-static int
>-i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_file_private *file_priv = file->driver_priv;
>-	unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
>-	struct drm_i915_gem_request *request;
>-	struct intel_engine_cs *ring = NULL;
>-	unsigned reset_counter;
>-	u32 seqno = 0;
>-	int ret;
>-
>-	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
>-	if (ret)
>-		return ret;
>-
>-	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
>-	if (ret)
>-		return ret;
>-
>-	spin_lock(&file_priv->mm.lock);
>-	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
>-		if (time_after_eq(request->emitted_jiffies, recent_enough))
>-			break;
>-
>-		ring = request->ring;
>-		seqno = request->seqno;
>-	}
>-	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
>-	spin_unlock(&file_priv->mm.lock);
>-
>-	if (seqno == 0)
>-		return 0;
>-
>-	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, NULL);
>-	if (ret == 0)
>-		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
>-
>-	return ret;
>-}
>-
> static bool
> i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
> {
>@@ -4234,7 +3736,7 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
> {
> 	struct drm_i915_gem_busy *args = data;
> 	struct drm_i915_gem_object *obj;
>-	int ret;
>+	int ret, i;
> 
> 	ret = i915_mutex_lock_interruptible(dev);
> 	if (ret)
>@@ -4253,10 +3755,16 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
> 	 */
> 	ret = i915_gem_object_flush_active(obj);
> 
>-	args->busy = obj->active;
>-	if (obj->ring) {
>-		BUILD_BUG_ON(I915_NUM_RINGS > 16);
>-		args->busy |= intel_ring_flag(obj->ring) << 16;
>+	args->busy = 0;
>+	if (obj->active) {
>+		BUILD_BUG_ON(I915_NUM_ENGINES > 16);
>+		args->busy |= 1;
>+		for (i = 0; i < I915_NUM_ENGINES; i++)  {
>+			if (obj->last_read[i].request == NULL)
>+				continue;
>+
>+			args->busy |= 1 << (16 + i);
>+		}
> 	}
> 
> 	drm_gem_object_unreference(&obj->base);
>@@ -4265,11 +3773,58 @@ unlock:
> 	return ret;
> }
> 
>+/* Throttle our rendering by waiting until the ring has completed our requests
>+ * emitted over 20 msec ago.
>+ *
>+ * Note that if we were to use the current jiffies each time around the loop,
>+ * we wouldn't escape the function with any frames outstanding if the time to
>+ * render a frame was over 20ms.
>+ *
>+ * This should get us reasonable parallelism between CPU and GPU but also
>+ * relatively low latency when blocking on a particular request to finish.
>+ */
> int
> i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
>-			struct drm_file *file_priv)
>+			struct drm_file *file)
> {
>-	return i915_gem_ring_throttle(dev, file_priv);
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_file_private *file_priv = file->driver_priv;
>+	unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
>+	struct i915_gem_request *rq, *tmp;
>+	int ret;
>+
>+	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
>+	if (ret)
>+		return ret;
>+
>+	/* used for querying whethering the GPU is wedged by legacy userspace */
>+	if (i915_terminally_wedged(&dev_priv->gpu_error))
>+		return -EIO;
>+
>+	spin_lock(&file_priv->mm.lock);
>+	rq = NULL;
>+	list_for_each_entry(tmp, &file_priv->mm.request_list, client_list) {
>+		if (time_after_eq(tmp->emitted_jiffies, recent_enough))
>+			break;
>+		rq = tmp;
>+	}
>+	rq = i915_request_get(rq);
>+	spin_unlock(&file_priv->mm.lock);
>+
>+	if (rq != NULL) {
>+		if (rq->breadcrumb[rq->engine->id] == 0) {
>+			ret = i915_mutex_lock_interruptible(dev);
>+			if (ret == 0) {
>+				ret = i915_request_emit_breadcrumb(rq);
>+				mutex_unlock(&dev->struct_mutex);
>+			}
>+		}
>+		if (ret == 0)
>+			ret = __i915_request_wait(rq, true, NULL, NULL);
>+		i915_request_put__unlocked(rq);
>+	}
>+
>+	return ret;
> }
> 
> int
>@@ -4322,8 +3877,13 @@ unlock:
> void i915_gem_object_init(struct drm_i915_gem_object *obj,
> 			  const struct drm_i915_gem_object_ops *ops)
> {
>+	int i;
>+
> 	INIT_LIST_HEAD(&obj->global_list);
>-	INIT_LIST_HEAD(&obj->ring_list);
>+	INIT_LIST_HEAD(&obj->last_fence.engine_list);
>+	INIT_LIST_HEAD(&obj->last_write.engine_list);
>+	for (i = 0; i < I915_NUM_ENGINES; i++)
>+		INIT_LIST_HEAD(&obj->last_read[i].engine_list);
> 	INIT_LIST_HEAD(&obj->obj_exec_link);
> 	INIT_LIST_HEAD(&obj->vma_list);
> 
>@@ -4331,8 +3891,6 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
> 
> 	obj->fence_reg = I915_FENCE_REG_NONE;
> 	obj->madv = I915_MADV_WILLNEED;
>-	/* Avoid an unnecessary call to unbind on the first bind. */
>-	obj->map_and_fenceable = true;
> 
> 	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
> }
>@@ -4493,26 +4051,156 @@ struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
> 
> void i915_gem_vma_destroy(struct i915_vma *vma)
> {
>+	struct i915_address_space *vm = NULL;
> 	WARN_ON(vma->node.allocated);
> 
> 	/* Keep the vma as a placeholder in the execbuffer reservation lists */
> 	if (!list_empty(&vma->exec_list))
> 		return;
> 
>+	vm = vma->vm;
>+
>+	if (!i915_is_ggtt(vm))
>+		i915_ppgtt_put(i915_vm_to_ppgtt(vm));
>+
> 	list_del(&vma->vma_link);
> 
> 	kfree(vma);
> }
> 
> static void
>-i915_gem_stop_ringbuffers(struct drm_device *dev)
>+i915_gem_cleanup_engines(struct drm_device *dev)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
> 	int i;
> 
>-	for_each_ring(ring, dev_priv, i)
>-		intel_stop_ring_buffer(ring);
>+	/* Not the regular for_each_engine so we can cleanup a failed setup */
>+	for (i =0; i < I915_NUM_ENGINES; i++) {
>+		struct intel_engine_cs *engine = &to_i915(dev)->engine[i];
>+
>+		if (engine->i915 == NULL)
>+			continue;
>+
>+		intel_engine_cleanup(engine);
>+	}
>+}
>+
>+static int
>+i915_gem_resume_engines(struct drm_device *dev)
>+{
>+	struct intel_engine_cs *engine;
>+	int i, ret;
>+
>+	for_each_engine(engine, to_i915(dev), i) {
>+		ret = intel_engine_resume(engine);
>+		if (ret)
>+			return ret;
>+	}
>+
>+	return 0;
>+}
>+
>+static int
>+i915_gem_suspend_engines(struct drm_device *dev)
>+{
>+	struct intel_engine_cs *engine;
>+	int i, ret;
>+
>+	for_each_engine(engine, to_i915(dev), i) {
>+		ret = intel_engine_suspend(engine);
>+		if (ret)
>+			return ret;
>+	}
>+
>+	return 0;
>+}
>+
>+static bool
>+intel_enable_blt(struct drm_i915_private *dev_priv)
>+{
>+	if (!HAS_BLT(dev_priv))
>+		return false;
>+
>+	/* The blitter was dysfunctional on early prototypes */
>+	if (IS_GEN6(dev_priv) && dev_priv->dev->pdev->revision < 8) {
>+		DRM_INFO("BLT not supported on this pre-production hardware;"
>+			 " graphics performance will be degraded.\n");
>+		return false;
>+	}
>+
>+	return true;
>+}
>+
>+static void stop_unused_ring(struct drm_i915_private *dev_priv, u32 base)
>+{
>+	I915_WRITE(RING_CTL(base), 0);
>+	I915_WRITE(RING_HEAD(base), 0);
>+	I915_WRITE(RING_TAIL(base), 0);
>+	I915_WRITE(RING_START(base), 0);
>+}
>+
>+static void stop_unused_rings(struct drm_i915_private *dev_priv)
>+{
>+	if (IS_I830(dev_priv)) {
>+		stop_unused_ring(dev_priv, PRB1_BASE);
>+		stop_unused_ring(dev_priv, SRB0_BASE);
>+		stop_unused_ring(dev_priv, SRB1_BASE);
>+		stop_unused_ring(dev_priv, SRB2_BASE);
>+		stop_unused_ring(dev_priv, SRB3_BASE);
>+	} else if (IS_GEN2(dev_priv)) {
>+		stop_unused_ring(dev_priv, SRB0_BASE);
>+		stop_unused_ring(dev_priv, SRB1_BASE);
>+	} else if (IS_GEN3(dev_priv)) {
>+		stop_unused_ring(dev_priv, PRB1_BASE);
>+		stop_unused_ring(dev_priv, PRB2_BASE);
>+	}
>+}
>+
>+static int i915_gem_setup_engines(struct drm_device *dev)
>+{
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	int ret;
>+
>+	/*
>+	 * At least 830 can leave some of the unused rings
>+	 * "active" (ie. head != tail) after resume which
>+	 * will prevent c3 entry. Makes sure all unused rings
>+	 * are totally idle.
>+	 */
>+	stop_unused_rings(dev_priv);
>+
>+	ret = intel_init_render_engine(dev_priv);
>+	if (ret)
>+		goto cleanup;
>+
>+	if (HAS_BSD(dev_priv)) {
>+		ret = intel_init_bsd_engine(dev_priv);
>+		if (ret)
>+			goto cleanup;
>+	}
>+
>+	if (intel_enable_blt(dev_priv)) {
>+		ret = intel_init_blt_engine(dev_priv);
>+		if (ret)
>+			goto cleanup;
>+	}
>+
>+	if (HAS_VEBOX(dev_priv)) {
>+		ret = intel_init_vebox_engine(dev_priv);
>+		if (ret)
>+			goto cleanup;
>+	}
>+
>+	if (HAS_BSD2(dev_priv)) {
>+		ret = intel_init_bsd2_engine(dev_priv);
>+		if (ret)
>+			goto cleanup;
>+	}
>+
>+	return 0;
>+
>+cleanup:
>+	i915_gem_cleanup_engines(dev);
>+	return ret;
> }
> 
> int
>@@ -4535,8 +4223,9 @@ i915_gem_suspend(struct drm_device *dev)
> 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
> 		i915_gem_evict_everything(dev);
> 
>-	i915_kernel_lost_context(dev);
>-	i915_gem_stop_ringbuffers(dev);
>+	ret = i915_gem_suspend_engines(dev);
>+	if (ret)
>+		goto err;
> 
> 	/* Hack!  Don't let anybody do execbuf while we don't control the chip.
> 	 * We need to replace this with a semaphore, or something.
>@@ -4557,37 +4246,6 @@ err:
> 	return ret;
> }
> 
>-int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice)
>-{
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
>-	u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
>-	int i, ret;
>-
>-	if (!HAS_L3_DPF(dev) || !remap_info)
>-		return 0;
>-
>-	ret = intel_ring_begin(ring, GEN7_L3LOG_SIZE / 4 * 3);
>-	if (ret)
>-		return ret;
>-
>-	/*
>-	 * Note: We do not worry about the concurrent register cacheline hang
>-	 * here because no other code should access these registers other than
>-	 * at initialization time.
>-	 */
>-	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
>-		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
>-		intel_ring_emit(ring, reg_base + i);
>-		intel_ring_emit(ring, remap_info[i/4]);
>-	}
>-
>-	intel_ring_advance(ring);
>-
>-	return ret;
>-}
>-
> void i915_gem_init_swizzling(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>@@ -4613,80 +4271,11 @@ void i915_gem_init_swizzling(struct drm_device *dev)
> 		BUG();
> }
> 
>-static bool
>-intel_enable_blt(struct drm_device *dev)
>-{
>-	if (!HAS_BLT(dev))
>-		return false;
>-
>-	/* The blitter was dysfunctional on early prototypes */
>-	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
>-		DRM_INFO("BLT not supported on this pre-production hardware;"
>-			 " graphics performance will be degraded.\n");
>-		return false;
>-	}
>-
>-	return true;
>-}
>-
>-static int i915_gem_init_rings(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int ret;
>-
>-	ret = intel_init_render_ring_buffer(dev);
>-	if (ret)
>-		return ret;
>-
>-	if (HAS_BSD(dev)) {
>-		ret = intel_init_bsd_ring_buffer(dev);
>-		if (ret)
>-			goto cleanup_render_ring;
>-	}
>-
>-	if (intel_enable_blt(dev)) {
>-		ret = intel_init_blt_ring_buffer(dev);
>-		if (ret)
>-			goto cleanup_bsd_ring;
>-	}
>-
>-	if (HAS_VEBOX(dev)) {
>-		ret = intel_init_vebox_ring_buffer(dev);
>-		if (ret)
>-			goto cleanup_blt_ring;
>-	}
>-
>-	if (HAS_BSD2(dev)) {
>-		ret = intel_init_bsd2_ring_buffer(dev);
>-		if (ret)
>-			goto cleanup_vebox_ring;
>-	}
>-
>-	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
>-	if (ret)
>-		goto cleanup_bsd2_ring;
>-
>-	return 0;
>-
>-cleanup_bsd2_ring:
>-	intel_cleanup_ring_buffer(&dev_priv->ring[VCS2]);
>-cleanup_vebox_ring:
>-	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
>-cleanup_blt_ring:
>-	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
>-cleanup_bsd_ring:
>-	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
>-cleanup_render_ring:
>-	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
>-
>-	return ret;
>-}
>-
> int
> i915_gem_init_hw(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int ret, i;
>+	int ret;
> 
> 	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
> 		return -EIO;
>@@ -4712,25 +4301,11 @@ i915_gem_init_hw(struct drm_device *dev)
> 
> 	i915_gem_init_swizzling(dev);
> 
>-	ret = i915_gem_init_rings(dev);
>-	if (ret)
>-		return ret;
>-
>-	for (i = 0; i < NUM_L3_SLICES(dev); i++)
>-		i915_gem_l3_remap(&dev_priv->ring[RCS], i);
>-
>-	/*
>-	 * XXX: Contexts should only be initialized once. Doing a switch to the
>-	 * default context switch however is something we'd like to do after
>-	 * reset or thaw (the latter may not actually be necessary for HW, but
>-	 * goes with our code better). Context switching requires rings (for
>-	 * the do_switch), but before enabling PPGTT. So don't move this.
>-	 */
>-	ret = i915_gem_context_enable(dev_priv);
>-	if (ret && ret != -EIO) {
>-		DRM_ERROR("Context enable failed %d\n", ret);
>-		i915_gem_cleanup_ringbuffer(dev);
>-	}
>+	ret = i915_ppgtt_init_hw(dev);
>+	if (ret == 0)
>+		ret = i915_gem_context_enable(dev_priv);
>+	if (ret == 0)
>+		ret = i915_gem_resume_engines(dev);
> 
> 	return ret;
> }
>@@ -4750,16 +4325,20 @@ int i915_gem_init(struct drm_device *dev)
> 			DRM_DEBUG_DRIVER("allow wake ack timed out\n");
> 	}
> 
>-	i915_gem_init_userptr(dev);
>-	i915_gem_init_global_gtt(dev);
>-
>-	ret = i915_gem_context_init(dev);
>+	ret = i915_gem_init_userptr(dev);
> 	if (ret) {
> 		mutex_unlock(&dev->struct_mutex);
> 		return ret;
> 	}
> 
>-	ret = i915_gem_init_hw(dev);
>+	i915_gem_init_global_gtt(dev);
>+
>+	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
>+	ret = i915_gem_setup_engines(dev);
>+	if (ret == 0)
>+		ret = i915_gem_context_init(dev);
>+	if (ret == 0)
>+		ret = i915_gem_init_hw(dev);
> 	if (ret == -EIO) {
> 		/* Allow ring initialisation to fail by marking the GPU as
> 		 * wedged. But we only want to do this where the GPU is angry,
>@@ -4769,23 +4348,16 @@ int i915_gem_init(struct drm_device *dev)
> 		atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
> 		ret = 0;
> 	}
>+	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
> 	mutex_unlock(&dev->struct_mutex);
> 
>-	/* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
>-	if (!drm_core_check_feature(dev, DRIVER_MODESET))
>-		dev_priv->dri1.allow_batchbuffer = 1;
> 	return ret;
> }
> 
>-void
>-i915_gem_cleanup_ringbuffer(struct drm_device *dev)
>+void i915_gem_fini(struct drm_device *dev)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>-	int i;
>-
>-	for_each_ring(ring, dev_priv, i)
>-		intel_cleanup_ring_buffer(ring);
>+	i915_gem_context_fini(dev);
>+	i915_gem_cleanup_engines(dev);
> }
> 
> int
>@@ -4804,26 +4376,12 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
> 	}
> 
> 	mutex_lock(&dev->struct_mutex);
>-	dev_priv->ums.mm_suspended = 0;
>-
> 	ret = i915_gem_init_hw(dev);
>-	if (ret != 0) {
>-		mutex_unlock(&dev->struct_mutex);
>-		return ret;
>-	}
>-
>+	if (ret == 0)
>+		ret = drm_irq_install(dev, dev->pdev->irq);
>+	if (ret == 0)
>+		dev_priv->ums.mm_suspended = 0;
> 	BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
>-
>-	ret = drm_irq_install(dev, dev->pdev->irq);
>-	if (ret)
>-		goto cleanup_ringbuffer;
>-	mutex_unlock(&dev->struct_mutex);
>-
>-	return 0;
>-
>-cleanup_ringbuffer:
>-	i915_gem_cleanup_ringbuffer(dev);
>-	dev_priv->ums.mm_suspended = 1;
> 	mutex_unlock(&dev->struct_mutex);
> 
> 	return ret;
>@@ -4857,10 +4415,13 @@ i915_gem_lastclose(struct drm_device *dev)
> }
> 
> static void
>-init_ring_lists(struct intel_engine_cs *ring)
>+init_null_engine(struct intel_engine_cs *engine)
> {
>-	INIT_LIST_HEAD(&ring->active_list);
>-	INIT_LIST_HEAD(&ring->request_list);
>+	INIT_LIST_HEAD(&engine->read_list);
>+	INIT_LIST_HEAD(&engine->write_list);
>+	INIT_LIST_HEAD(&engine->fence_list);
>+	INIT_LIST_HEAD(&engine->requests);
>+	INIT_LIST_HEAD(&engine->rings);
> }
> 
> void i915_init_vm(struct drm_i915_private *dev_priv,
>@@ -4894,8 +4455,8 @@ i915_gem_load(struct drm_device *dev)
> 	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
> 	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
> 	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
>-	for (i = 0; i < I915_NUM_RINGS; i++)
>-		init_ring_lists(&dev_priv->ring[i]);
>+	for (i = 0; i < I915_NUM_ENGINES; i++)
>+		init_null_engine(&dev_priv->engine[i]);
> 	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
> 		INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
> 	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
>@@ -4955,13 +4516,13 @@ void i915_gem_release(struct drm_device *dev, struct drm_file *file)
> 	 */
> 	spin_lock(&file_priv->mm.lock);
> 	while (!list_empty(&file_priv->mm.request_list)) {
>-		struct drm_i915_gem_request *request;
>+		struct i915_gem_request *rq;
> 
>-		request = list_first_entry(&file_priv->mm.request_list,
>-					   struct drm_i915_gem_request,
>-					   client_list);
>-		list_del(&request->client_list);
>-		request->file_priv = NULL;
>+		rq = list_first_entry(&file_priv->mm.request_list,
>+				      struct i915_gem_request,
>+				      client_list);
>+		list_del(&rq->client_list);
>+		rq->file_priv = NULL;
> 	}
> 	spin_unlock(&file_priv->mm.lock);
> }
>@@ -5097,9 +4658,7 @@ unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
> 	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
> 	struct i915_vma *vma;
> 
>-	if (!dev_priv->mm.aliasing_ppgtt ||
>-	    vm == &dev_priv->mm.aliasing_ppgtt->base)
>-		vm = &dev_priv->gtt.base;
>+	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
> 
> 	list_for_each_entry(vma, &o->vma_list, vma_link) {
> 		if (vma->vm == vm)
>@@ -5140,9 +4699,7 @@ unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
> 	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
> 	struct i915_vma *vma;
> 
>-	if (!dev_priv->mm.aliasing_ppgtt ||
>-	    vm == &dev_priv->mm.aliasing_ppgtt->base)
>-		vm = &dev_priv->gtt.base;
>+	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
> 
> 	BUG_ON(list_empty(&o->vma_list));
> 
>@@ -5165,11 +4722,16 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
> 	if (!i915_gem_shrinker_lock(dev, &unlock))
> 		return SHRINK_STOP;
> 
>-	freed = i915_gem_purge(dev_priv, sc->nr_to_scan);
>+	freed = i915_gem_shrink(dev_priv,
>+				sc->nr_to_scan,
>+				I915_SHRINK_BOUND |
>+				I915_SHRINK_UNBOUND |
>+				I915_SHRINK_PURGEABLE);
> 	if (freed < sc->nr_to_scan)
>-		freed += __i915_gem_shrink(dev_priv,
>-					   sc->nr_to_scan - freed,
>-					   false);
>+		freed += i915_gem_shrink(dev_priv,
>+					 sc->nr_to_scan - freed,
>+					 I915_SHRINK_BOUND |
>+					 I915_SHRINK_UNBOUND);
> 	if (unlock)
> 		mutex_unlock(&dev->struct_mutex);
> 
>@@ -5247,14 +4809,8 @@ struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj)
> {
> 	struct i915_vma *vma;
> 
>-	/* This WARN has probably outlived its usefulness (callers already
>-	 * WARN if they don't find the GGTT vma they expect). When removing,
>-	 * remember to remove the pre-check in is_pin_display() as well */
>-	if (WARN_ON(list_empty(&obj->vma_list)))
>-		return NULL;
>-
> 	vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
>-	if (vma->vm != obj_to_ggtt(obj))
>+	if (vma->vm != i915_obj_to_ggtt(obj))
> 		return NULL;
> 
> 	return vma;
>diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
>index 3b99390..841056c 100644
>--- a/drivers/gpu/drm/i915/i915_gem_context.c
>+++ b/drivers/gpu/drm/i915/i915_gem_context.c
>@@ -96,53 +96,9 @@
> #define GEN6_CONTEXT_ALIGN (64<<10)
> #define GEN7_CONTEXT_ALIGN 4096
> 
>-static void do_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt)
>+static size_t get_context_alignment(struct drm_i915_private *i915)
> {
>-	struct drm_device *dev = ppgtt->base.dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct i915_address_space *vm = &ppgtt->base;
>-
>-	if (ppgtt == dev_priv->mm.aliasing_ppgtt ||
>-	    (list_empty(&vm->active_list) && list_empty(&vm->inactive_list))) {
>-		ppgtt->base.cleanup(&ppgtt->base);
>-		return;
>-	}
>-
>-	/*
>-	 * Make sure vmas are unbound before we take down the drm_mm
>-	 *
>-	 * FIXME: Proper refcounting should take care of this, this shouldn't be
>-	 * needed at all.
>-	 */
>-	if (!list_empty(&vm->active_list)) {
>-		struct i915_vma *vma;
>-
>-		list_for_each_entry(vma, &vm->active_list, mm_list)
>-			if (WARN_ON(list_empty(&vma->vma_link) ||
>-				    list_is_singular(&vma->vma_link)))
>-				break;
>-
>-		i915_gem_evict_vm(&ppgtt->base, true);
>-	} else {
>-		i915_gem_retire_requests(dev);
>-		i915_gem_evict_vm(&ppgtt->base, false);
>-	}
>-
>-	ppgtt->base.cleanup(&ppgtt->base);
>-}
>-
>-static void ppgtt_release(struct kref *kref)
>-{
>-	struct i915_hw_ppgtt *ppgtt =
>-		container_of(kref, struct i915_hw_ppgtt, ref);
>-
>-	do_ppgtt_cleanup(ppgtt);
>-	kfree(ppgtt);
>-}
>-
>-static size_t get_context_alignment(struct drm_device *dev)
>-{
>-	if (IS_GEN6(dev))
>+	if (IS_GEN6(i915))
> 		return GEN6_CONTEXT_ALIGN;
> 
> 	return GEN7_CONTEXT_ALIGN;
>@@ -155,6 +111,9 @@ static int get_context_size(struct drm_device *dev)
> 	u32 reg;
> 
> 	switch (INTEL_INFO(dev)->gen) {
>+	case 5:
>+		ret = ILK_CXT_TOTAL_SIZE;
>+		break;
> 	case 6:
> 		reg = I915_READ(CXT_SIZE);
> 		ret = GEN6_CXT_TOTAL_SIZE(reg) * 64;
>@@ -178,25 +137,27 @@ static int get_context_size(struct drm_device *dev)
> 
> void i915_gem_context_free(struct kref *ctx_ref)
> {
>-	struct intel_context *ctx = container_of(ctx_ref,
>-						   typeof(*ctx), ref);
>-	struct i915_hw_ppgtt *ppgtt = NULL;
>-
>-	if (ctx->legacy_hw_ctx.rcs_state) {
>-		/* We refcount even the aliasing PPGTT to keep the code symmetric */
>-		if (USES_PPGTT(ctx->legacy_hw_ctx.rcs_state->base.dev))
>-			ppgtt = ctx_to_ppgtt(ctx);
>+	struct intel_context *ctx =
>+		container_of(ctx_ref, typeof(*ctx), ref);
>+	struct drm_i915_private *dev_priv = ctx->i915;
>+	int i;
>+
>+	i915_ppgtt_put(ctx->ppgtt);
>+
>+	for (i = 0; i < I915_NUM_ENGINES; i++) {
>+		if (intel_engine_initialized(&dev_priv->engine[i]) &&
>+		    ctx->ring[i].ring != NULL)
>+			dev_priv->engine[i].put_ring(ctx->ring[i].ring, ctx);
>+
>+		if (ctx->ring[i].state != NULL)
>+			drm_gem_object_unreference(&ctx->ring[i].state->base);
> 	}
> 
>-	if (ppgtt)
>-		kref_put(&ppgtt->ref, ppgtt_release);
>-	if (ctx->legacy_hw_ctx.rcs_state)
>-		drm_gem_object_unreference(&ctx->legacy_hw_ctx.rcs_state->base);
> 	list_del(&ctx->link);
> 	kfree(ctx);
> }
> 
>-static struct drm_i915_gem_object *
>+struct drm_i915_gem_object *
> i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
> {
> 	struct drm_i915_gem_object *obj;
>@@ -226,49 +187,32 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
> 	return obj;
> }
> 
>-static struct i915_hw_ppgtt *
>-create_vm_for_ctx(struct drm_device *dev, struct intel_context *ctx)
>-{
>-	struct i915_hw_ppgtt *ppgtt;
>-	int ret;
>-
>-	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
>-	if (!ppgtt)
>-		return ERR_PTR(-ENOMEM);
>-
>-	ret = i915_gem_init_ppgtt(dev, ppgtt);
>-	if (ret) {
>-		kfree(ppgtt);
>-		return ERR_PTR(ret);
>-	}
>-
>-	ppgtt->ctx = ctx;
>-	return ppgtt;
>-}
>-
> static struct intel_context *
>-__create_hw_context(struct drm_device *dev,
>-		  struct drm_i915_file_private *file_priv)
>+i915_gem_create_context(struct drm_device *dev,
>+			struct drm_i915_file_private *file_priv)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = to_i915(dev);
> 	struct intel_context *ctx;
> 	int ret;
> 
>+	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
>+
> 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
> 	if (ctx == NULL)
> 		return ERR_PTR(-ENOMEM);
> 
> 	kref_init(&ctx->ref);
> 	list_add_tail(&ctx->link, &dev_priv->context_list);
>+	ctx->i915 = dev_priv;
> 
> 	if (dev_priv->hw_context_size) {
> 		struct drm_i915_gem_object *obj =
>-				i915_gem_alloc_context_obj(dev, dev_priv->hw_context_size);
>+			i915_gem_alloc_context_obj(dev, dev_priv->hw_context_size);
> 		if (IS_ERR(obj)) {
> 			ret = PTR_ERR(obj);
>-			goto err_out;
>+			goto err;
> 		}
>-		ctx->legacy_hw_ctx.rcs_state = obj;
>+		ctx->ring[RCS].state = obj;
> 	}
> 
> 	/* Default context will never have a file_priv */
>@@ -276,7 +220,7 @@ __create_hw_context(struct drm_device *dev,
> 		ret = idr_alloc(&file_priv->context_idr, ctx,
> 				DEFAULT_CONTEXT_HANDLE, 0, GFP_KERNEL);
> 		if (ret < 0)
>-			goto err_out;
>+			goto err;
> 	} else
> 		ret = DEFAULT_CONTEXT_HANDLE;
> 
>@@ -287,135 +231,33 @@ __create_hw_context(struct drm_device *dev,
> 	 * is no remap info, it will be a NOP. */
> 	ctx->remap_slice = (1 << NUM_L3_SLICES(dev)) - 1;
> 
>-	return ctx;
>-
>-err_out:
>-	i915_gem_context_unreference(ctx);
>-	return ERR_PTR(ret);
>-}
>-
>-/**
>- * The default context needs to exist per ring that uses contexts. It stores the
>- * context state of the GPU for applications that don't utilize HW contexts, as
>- * well as an idle case.
>- */
>-static struct intel_context *
>-i915_gem_create_context(struct drm_device *dev,
>-			struct drm_i915_file_private *file_priv,
>-			bool create_vm)
>-{
>-	const bool is_global_default_ctx = file_priv == NULL;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_context *ctx;
>-	int ret = 0;
>-
>-	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
>+	ctx->hang_stats.ban_period_seconds = DRM_I915_CTX_BAN_PERIOD;
> 
>-	ctx = __create_hw_context(dev, file_priv);
>-	if (IS_ERR(ctx))
>-		return ctx;
>-
>-	if (is_global_default_ctx && ctx->legacy_hw_ctx.rcs_state) {
>-		/* We may need to do things with the shrinker which
>-		 * require us to immediately switch back to the default
>-		 * context. This can cause a problem as pinning the
>-		 * default context also requires GTT space which may not
>-		 * be available. To avoid this we always pin the default
>-		 * context.
>-		 */
>-		ret = i915_gem_obj_ggtt_pin(ctx->legacy_hw_ctx.rcs_state,
>-					    get_context_alignment(dev), 0);
>-		if (ret) {
>-			DRM_DEBUG_DRIVER("Couldn't pin %d\n", ret);
>-			goto err_destroy;
>-		}
>-	}
>-
>-	if (create_vm) {
>-		struct i915_hw_ppgtt *ppgtt = create_vm_for_ctx(dev, ctx);
>+	if (USES_FULL_PPGTT(dev)) {
>+		struct i915_hw_ppgtt *ppgtt = i915_ppgtt_create(dev, file_priv);
> 
> 		if (IS_ERR_OR_NULL(ppgtt)) {
> 			DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n",
> 					 PTR_ERR(ppgtt));
> 			ret = PTR_ERR(ppgtt);
>-			goto err_unpin;
>-		} else
>-			ctx->vm = &ppgtt->base;
>-
>-		/* This case is reserved for the global default context and
>-		 * should only happen once. */
>-		if (is_global_default_ctx) {
>-			if (WARN_ON(dev_priv->mm.aliasing_ppgtt)) {
>-				ret = -EEXIST;
>-				goto err_unpin;
>-			}
>-
>-			dev_priv->mm.aliasing_ppgtt = ppgtt;
>+			goto err;
> 		}
>-	} else if (USES_PPGTT(dev)) {
>-		/* For platforms which only have aliasing PPGTT, we fake the
>-		 * address space and refcounting. */
>-		ctx->vm = &dev_priv->mm.aliasing_ppgtt->base;
>-		kref_get(&dev_priv->mm.aliasing_ppgtt->ref);
>-	} else
>-		ctx->vm = &dev_priv->gtt.base;
>+
>+		ctx->ppgtt = ppgtt;
>+	}
> 
> 	return ctx;
> 
>-err_unpin:
>-	if (is_global_default_ctx && ctx->legacy_hw_ctx.rcs_state)
>-		i915_gem_object_ggtt_unpin(ctx->legacy_hw_ctx.rcs_state);
>-err_destroy:
>+err:
> 	i915_gem_context_unreference(ctx);
> 	return ERR_PTR(ret);
> }
> 
>-void i915_gem_context_reset(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int i;
>-
>-	/* Prevent the hardware from restoring the last context (which hung) on
>-	 * the next switch */
>-	for (i = 0; i < I915_NUM_RINGS; i++) {
>-		struct intel_engine_cs *ring = &dev_priv->ring[i];
>-		struct intel_context *dctx = ring->default_context;
>-		struct intel_context *lctx = ring->last_context;
>-
>-		/* Do a fake switch to the default context */
>-		if (lctx == dctx)
>-			continue;
>-
>-		if (!lctx)
>-			continue;
>-
>-		if (dctx->legacy_hw_ctx.rcs_state && i == RCS) {
>-			WARN_ON(i915_gem_obj_ggtt_pin(dctx->legacy_hw_ctx.rcs_state,
>-						      get_context_alignment(dev), 0));
>-			/* Fake a finish/inactive */
>-			dctx->legacy_hw_ctx.rcs_state->base.write_domain = 0;
>-			dctx->legacy_hw_ctx.rcs_state->active = 0;
>-		}
>-
>-		if (lctx->legacy_hw_ctx.rcs_state && i == RCS)
>-			i915_gem_object_ggtt_unpin(lctx->legacy_hw_ctx.rcs_state);
>-
>-		i915_gem_context_unreference(lctx);
>-		i915_gem_context_reference(dctx);
>-		ring->last_context = dctx;
>-	}
>-}
>-
> int i915_gem_context_init(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_context *ctx;
>-	int i;
>-
>-	/* Init should only be called once per module load. Eventually the
>-	 * restriction on the context_disabled check can be loosened. */
>-	if (WARN_ON(dev_priv->ring[RCS].default_context))
>-		return 0;
>+	int i, ret;
> 
> 	if (HAS_HW_CONTEXTS(dev)) {
> 		dev_priv->hw_context_size = round_up(get_context_size(dev), 4096);
>@@ -426,86 +268,111 @@ int i915_gem_context_init(struct drm_device *dev)
> 		}
> 	}
> 
>-	ctx = i915_gem_create_context(dev, NULL, USES_PPGTT(dev));
>+	/**
>+	 * The default context needs to exist per ring that uses contexts.
>+	 * It stores the context state of the GPU for applications that don't
>+	 * utilize HW contexts or per-process VM, as well as an idle case.
>+	 */
>+	ctx = i915_gem_create_context(dev, NULL);
> 	if (IS_ERR(ctx)) {
> 		DRM_ERROR("Failed to create default global context (error %ld)\n",
> 			  PTR_ERR(ctx));
> 		return PTR_ERR(ctx);
> 	}
> 
>-	/* NB: RCS will hold a ref for all rings */
>-	for (i = 0; i < I915_NUM_RINGS; i++)
>-		dev_priv->ring[i].default_context = ctx;
>+	if (dev_priv->hw_context_size) {
>+		/* We may need to do things with the shrinker which
>+		 * require us to immediately switch back to the default
>+		 * context. This can cause a problem as pinning the
>+		 * default context also requires GTT space which may not
>+		 * be available. To avoid this we always pin the default
>+		 * context.
>+		 */
>+		ret = i915_gem_obj_ggtt_pin(ctx->ring[RCS].state,
>+					    get_context_alignment(dev_priv), 0);
>+		if (ret) {
>+			DRM_ERROR("Failed to pin global default context\n");
>+			i915_gem_context_unreference(ctx);
>+			return ret;
>+		}
>+	}
>+
>+	for (i = 0; i < I915_NUM_ENGINES; i++) {
>+		struct intel_engine_cs *engine = &dev_priv->engine[i];
>+
>+		if (engine->i915 == NULL)
>+			continue;
>+
>+		engine->default_context = ctx;
>+		i915_gem_context_reference(ctx);
>+	}
>+
>+	dev_priv->default_context = ctx;
> 
>-	DRM_DEBUG_DRIVER("%s context support initialized\n", dev_priv->hw_context_size ? "HW" : "fake");
>+	DRM_DEBUG_DRIVER("%s context support initialized\n",
>+			 dev_priv->hw_context_size ? "HW" : "fake");
> 	return 0;
> }
> 
> void i915_gem_context_fini(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_context *dctx = dev_priv->ring[RCS].default_context;
>+	struct intel_engine_cs *engine;
> 	int i;
> 
>-	if (dctx->legacy_hw_ctx.rcs_state) {
>+	if (dev_priv->hw_context_size)
> 		/* The only known way to stop the gpu from accessing the hw context is
> 		 * to reset it. Do this as the very last operation to avoid confusing
> 		 * other code, leading to spurious errors. */
> 		intel_gpu_reset(dev);
> 
>-		/* When default context is created and switched to, base object refcount
>-		 * will be 2 (+1 from object creation and +1 from do_switch()).
>-		 * i915_gem_context_fini() will be called after gpu_idle() has switched
>-		 * to default context. So we need to unreference the base object once
>-		 * to offset the do_switch part, so that i915_gem_context_unreference()
>-		 * can then free the base object correctly. */
>-		WARN_ON(!dev_priv->ring[RCS].last_context);
>-		if (dev_priv->ring[RCS].last_context == dctx) {
>-			/* Fake switch to NULL context */
>-			WARN_ON(dctx->legacy_hw_ctx.rcs_state->active);
>-			i915_gem_object_ggtt_unpin(dctx->legacy_hw_ctx.rcs_state);
>-			i915_gem_context_unreference(dctx);
>-			dev_priv->ring[RCS].last_context = NULL;
>-		}
>-
>-		i915_gem_object_ggtt_unpin(dctx->legacy_hw_ctx.rcs_state);
>+	for_each_engine(engine, dev_priv, i) {
>+		i915_gem_context_unreference(engine->default_context);
>+		engine->default_context = NULL;
> 	}
> 
>-	for (i = 0; i < I915_NUM_RINGS; i++) {
>-		struct intel_engine_cs *ring = &dev_priv->ring[i];
>-
>-		if (ring->last_context)
>-			i915_gem_context_unreference(ring->last_context);
>-
>-		ring->default_context = NULL;
>-		ring->last_context = NULL;
>+	if (dev_priv->default_context) {
>+		if (dev_priv->hw_context_size)
>+			i915_gem_object_ggtt_unpin(dev_priv->default_context->ring[RCS].state);
>+		i915_gem_context_unreference(dev_priv->default_context);
>+		dev_priv->default_context = NULL;
> 	}
>-
>-	i915_gem_context_unreference(dctx);
> }
> 
> int i915_gem_context_enable(struct drm_i915_private *dev_priv)
> {
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	int ret, i;
> 
>-	/* This is the only place the aliasing PPGTT gets enabled, which means
>-	 * it has to happen before we bail on reset */
>-	if (dev_priv->mm.aliasing_ppgtt) {
>-		struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
>-		ppgtt->enable(ppgtt);
>-	}
>+	for_each_engine(engine, dev_priv, i) {
>+		struct intel_context *ctx = engine->default_context;
>+		struct i915_gem_request *rq;
> 
>-	/* FIXME: We should make this work, even in reset */
>-	if (i915_reset_in_progress(&dev_priv->gpu_error))
>-		return 0;
>+		if (HAS_L3_DPF(dev_priv))
>+			ctx->remap_slice = (1 << NUM_L3_SLICES(dev_priv)) - 1;
> 
>-	BUG_ON(!dev_priv->ring[RCS].default_context);
>+		rq = intel_engine_alloc_request(engine, ctx);
>+		if (IS_ERR(rq)) {
>+			ret = PTR_ERR(rq);
>+			goto err;
>+		}
> 
>-	for_each_ring(ring, dev_priv, i) {
>-		ret = i915_switch_context(ring, ring->default_context);
>-		if (ret)
>+		ret = 0;
>+		/*
>+		 * Workarounds applied in this fn are part of register state context,
>+		 * they need to be re-initialized followed by gpu reset, suspend/resume,
>+		 * module reload.
>+		 */
>+		if (engine->init_context)
>+			ret = engine->init_context(rq);
>+		if (ret == 0)
>+			ret = i915_request_commit(rq);
>+		i915_request_put(rq);
>+		if (ret) {
>+err:
>+			DRM_ERROR("failed to enabled contexts (%s): %d\n", engine->name, ret);
> 			return ret;
>+		}
> 	}
> 
> 	return 0;
>@@ -515,7 +382,9 @@ static int context_idr_cleanup(int id, void *p, void *data)
> {
> 	struct intel_context *ctx = p;
> 
>+	ctx->file_priv = NULL;
> 	i915_gem_context_unreference(ctx);
>+
> 	return 0;
> }
> 
>@@ -527,7 +396,7 @@ int i915_gem_context_open(struct drm_device *dev, struct drm_file *file)
> 	idr_init(&file_priv->context_idr);
> 
> 	mutex_lock(&dev->struct_mutex);
>-	ctx = i915_gem_create_context(dev, file_priv, USES_FULL_PPGTT(dev));
>+	ctx = i915_gem_create_context(dev, file_priv);
> 	mutex_unlock(&dev->struct_mutex);
> 
> 	if (IS_ERR(ctx)) {
>@@ -559,147 +428,212 @@ i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id)
> }
> 
> static inline int
>-mi_set_context(struct intel_engine_cs *ring,
>-	       struct intel_context *new_context,
>-	       u32 hw_flags)
>+mi_set_context(struct i915_gem_request *rq,
>+	       struct intel_engine_context *new_context,
>+	       u32 flags)
> {
>-	int ret;
>+	struct intel_ringbuffer *ring;
>+	int len;
> 
> 	/* w/a: If Flush TLB Invalidation Mode is enabled, driver must do a TLB
> 	 * invalidation prior to MI_SET_CONTEXT. On GEN6 we don't set the value
>-	 * explicitly, so we rely on the value at ring init, stored in
>+	 * explicitly, so we rely on the value at engine init, stored in
> 	 * itlb_before_ctx_switch.
> 	 */
>-	if (IS_GEN6(ring->dev)) {
>-		ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, 0);
>-		if (ret)
>-			return ret;
>+	if (IS_GEN6(rq->i915))
>+		rq->pending_flush |= I915_INVALIDATE_CACHES;
>+
>+	len = 3;
>+	switch (INTEL_INFO(rq->i915)->gen) {
>+	case 8:
>+	case 7:
>+	case 5: len += 2;
>+		break;
> 	}
> 
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, len);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
>-	/* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
>-	if (INTEL_INFO(ring->dev)->gen >= 7)
>+	switch (INTEL_INFO(rq->i915)->gen) {
>+	case 8:
>+	case 7:
>+		/* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
> 		intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_DISABLE);
>-	else
>-		intel_ring_emit(ring, MI_NOOP);
>+		break;
>+	case 5:
>+		intel_ring_emit(ring, MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
>+		break;
>+	}
> 
>-	intel_ring_emit(ring, MI_NOOP);
> 	intel_ring_emit(ring, MI_SET_CONTEXT);
>-	intel_ring_emit(ring, i915_gem_obj_ggtt_offset(new_context->legacy_hw_ctx.rcs_state) |
>+	intel_ring_emit(ring,
>+			i915_gem_obj_ggtt_offset(new_context->state) |
> 			MI_MM_SPACE_GTT |
>-			MI_SAVE_EXT_STATE_EN |
>-			MI_RESTORE_EXT_STATE_EN |
>-			hw_flags);
>+			flags);
> 	/*
> 	 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
> 	 * WaMiSetContext_Hang:snb,ivb,vlv
> 	 */
> 	intel_ring_emit(ring, MI_NOOP);
> 
>-	if (INTEL_INFO(ring->dev)->gen >= 7)
>+	switch (INTEL_INFO(rq->i915)->gen) {
>+	case 8:
>+	case 7:
> 		intel_ring_emit(ring, MI_ARB_ON_OFF | MI_ARB_ENABLE);
>-	else
>-		intel_ring_emit(ring, MI_NOOP);
>+		break;
>+	case 5:
>+		intel_ring_emit(ring, MI_SUSPEND_FLUSH);
>+		break;
>+	}
> 
> 	intel_ring_advance(ring);
> 
>-	return ret;
>+	rq->pending_flush &= ~I915_COMMAND_BARRIER;
>+	return 0;
> }
> 
>-static int do_switch(struct intel_engine_cs *ring,
>-		     struct intel_context *to)
>+static int l3_remap(struct i915_gem_request *rq, int slice)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>-	struct intel_context *from = ring->last_context;
>-	struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(to);
>-	u32 hw_flags = 0;
>-	bool uninitialized = false;
>-	int ret, i;
>+	const u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
>+	const u32 *remap_info;
>+	struct intel_ringbuffer *ring;
>+	int i;
>+
>+	remap_info = rq->i915->l3_parity.remap_info[slice];
>+	if (remap_info == NULL)
>+		return 0;
>+
>+	ring = intel_ring_begin(rq, GEN7_L3LOG_SIZE / 4 * 3);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
>-	if (from != NULL && ring == &dev_priv->ring[RCS]) {
>-		BUG_ON(from->legacy_hw_ctx.rcs_state == NULL);
>-		BUG_ON(!i915_gem_obj_is_pinned(from->legacy_hw_ctx.rcs_state));
>+	/*
>+	 * Note: We do not worry about the concurrent register cacheline hang
>+	 * here because no other code should access these registers other than
>+	 * at initialization time.
>+	 */
>+	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
>+		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
>+		intel_ring_emit(ring, reg_base + i);
>+		intel_ring_emit(ring, remap_info[i/4]);
> 	}
> 
>-	if (from == to && !to->remap_slice)
>+	intel_ring_advance(ring);
>+	return 0;
>+}
>+
>+/**
>+ * i915_request_switch_context() - perform a GPU context switch.
>+ * @rq: request and ring/ctx for which we'll execute the context switch
>+ *
>+ * The context life cycle is simple. The context refcount is incremented and
>+ * decremented by 1 and create and destroy. If the context is in use by the GPU,
>+ * it will have a refoucnt > 1. This allows us to destroy the context abstract
>+ * object while letting the normal object tracking destroy the backing BO.
>+ */
>+int i915_request_switch_context(struct i915_gem_request *rq)
>+{
>+	struct intel_context *to = rq->ctx;
>+	struct intel_engine_context *ctx = &to->ring[rq->engine->id];
>+	struct intel_context *from;
>+	int ret, i;
>+
>+	lockdep_assert_held(&rq->i915->dev->struct_mutex);
>+
>+	if (rq->ring->last_context == to && !to->remap_slice)
> 		return 0;
> 
>-	/* Trying to pin first makes error handling easier. */
>-	if (ring == &dev_priv->ring[RCS]) {
>-		ret = i915_gem_obj_ggtt_pin(to->legacy_hw_ctx.rcs_state,
>-					    get_context_alignment(ring->dev), 0);
>+	if (ctx->state != NULL) {
>+		/* Trying to pin first makes error handling easier. */
>+		ret = i915_gem_obj_ggtt_pin(ctx->state,
>+					    get_context_alignment(rq->i915), 0);
> 		if (ret)
> 			return ret;
>-	}
>-
>-	/*
>-	 * Pin can switch back to the default context if we end up calling into
>-	 * evict_everything - as a last ditch gtt defrag effort that also
>-	 * switches to the default context. Hence we need to reload from here.
>-	 */
>-	from = ring->last_context;
> 
>-	if (USES_FULL_PPGTT(ring->dev)) {
>-		ret = ppgtt->switch_mm(ppgtt, ring, false);
>+		/*
>+		 * Clear this page out of any CPU caches for coherent swap-in/out. Note
>+		 * that thanks to write = false in this call and us not setting any gpu
>+		 * write domains when putting a context object onto the active list
>+		 * (when switching away from it), this won't block.
>+		 *
>+		 * XXX: We need a real interface to do this instead of trickery.
>+		 */
>+		ret = i915_gem_object_set_to_gtt_domain(ctx->state, false);
> 		if (ret)
> 			goto unpin_out;
> 	}
> 
>-	if (ring != &dev_priv->ring[RCS]) {
>-		if (from)
>-			i915_gem_context_unreference(from);
>-		goto done;
>-	}
>+	if (ctx->state != NULL) {
>+		u32 flags;
> 
>-	/*
>-	 * Clear this page out of any CPU caches for coherent swap-in/out. Note
>-	 * that thanks to write = false in this call and us not setting any gpu
>-	 * write domains when putting a context object onto the active list
>-	 * (when switching away from it), this won't block.
>-	 *
>-	 * XXX: We need a real interface to do this instead of trickery.
>-	 */
>-	ret = i915_gem_object_set_to_gtt_domain(to->legacy_hw_ctx.rcs_state, false);
>-	if (ret)
>-		goto unpin_out;
>+		if (!ctx->state->has_global_gtt_mapping) {
>+			struct i915_vma *vma =
>+				i915_gem_obj_to_vma(ctx->state,
>+						    &rq->i915->gtt.base);
>+			vma->bind_vma(vma,
>+				      ctx->state->cache_level,
>+				      GLOBAL_BIND);
>+		}
> 
>-	if (!to->legacy_hw_ctx.rcs_state->has_global_gtt_mapping) {
>-		struct i915_vma *vma = i915_gem_obj_to_vma(to->legacy_hw_ctx.rcs_state,
>-							   &dev_priv->gtt.base);
>-		vma->bind_vma(vma, to->legacy_hw_ctx.rcs_state->cache_level, GLOBAL_BIND);
>-	}
>+		flags = 0;
>+		if (!ctx->initialized || i915_gem_context_is_default(to))
>+			flags |= MI_RESTORE_INHIBIT;
>+
>+		/* These flags are for resource streamer on HSW+ */
>+		if (!IS_HASWELL(rq->i915) && INTEL_INFO(rq->i915)->gen < 8) {
>+			if (ctx->initialized)
>+				flags |= MI_RESTORE_EXT_STATE_EN;
>+			flags |= MI_SAVE_EXT_STATE_EN;
>+		}
> 
>-	if (!to->legacy_hw_ctx.initialized || i915_gem_context_is_default(to))
>-		hw_flags |= MI_RESTORE_INHIBIT;
>+		trace_i915_gem_ring_switch_context(rq->engine, to, flags);
>+		ret = mi_set_context(rq, ctx, flags);
>+		if (ret)
>+			goto unpin_out;
> 
>-	ret = mi_set_context(ring, to, hw_flags);
>-	if (ret)
>-		goto unpin_out;
>+		rq->pending_flush &= ~I915_COMMAND_BARRIER;
>+	}
>+
>+	if (to->ppgtt) {
>+		ret = to->ppgtt->switch_mm(rq, to->ppgtt);
>+		if (ret)
>+			goto unpin_out;
>+	}
> 
> 	for (i = 0; i < MAX_L3_SLICES; i++) {
>-		if (!(to->remap_slice & (1<<i)))
>+		if (!(to->remap_slice & (1 << i)))
> 			continue;
> 
>-		ret = i915_gem_l3_remap(ring, i);
> 		/* If it failed, try again next round */
>-		if (ret)
>-			DRM_DEBUG_DRIVER("L3 remapping failed\n");
>-		else
>-			to->remap_slice &= ~(1<<i);
>+		if (l3_remap(rq, i) == 0)
>+			rq->remap_l3 |= 1 << i;
> 	}
> 
>+	/*
>+	 * Pin can switch back to the default context if we end up calling into
>+	 * evict_everything - as a last ditch gtt defrag effort that also
>+	 * switches to the default context. Hence we need to reload from here.
>+	 */
>+	from = rq->ring->last_context;
>+
> 	/* The backing object for the context is done after switching to the
> 	 * *next* context. Therefore we cannot retire the previous context until
> 	 * the next context has already started running. In fact, the below code
> 	 * is a bit suboptimal because the retiring can occur simply after the
> 	 * MI_SET_CONTEXT instead of when the next seqno has completed.
> 	 */
>-	if (from != NULL) {
>-		from->legacy_hw_ctx.rcs_state->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION;
>-		i915_vma_move_to_active(i915_gem_obj_to_ggtt(from->legacy_hw_ctx.rcs_state), ring);
>+	if (from && from->ring[rq->engine->id].state) {
>+		struct drm_i915_gem_object *from_obj = from->ring[rq->engine->id].state;
>+
>+		from_obj->base.pending_read_domains = I915_GEM_DOMAIN_INSTRUCTION;
>+		/* obj is kept alive until the next request by its active ref */
>+		ret = i915_request_add_vma(rq,
>+					   i915_gem_obj_to_ggtt(from_obj),
>+					   0);
>+		if (ret)
>+			goto unpin_out;
>+
> 		/* As long as MI_SET_CONTEXT is serializing, ie. it flushes the
> 		 * whole damn pipeline, we don't need to explicitly mark the
> 		 * object dirty. The only exception is that the context must be
>@@ -707,68 +641,60 @@ static int do_switch(struct intel_engine_cs *ring,
> 		 * able to defer doing this until we know the object would be
> 		 * swapped, but there is no way to do that yet.
> 		 */
>-		from->legacy_hw_ctx.rcs_state->dirty = 1;
>-		BUG_ON(from->legacy_hw_ctx.rcs_state->ring != ring);
>-
>-		/* obj is kept alive until the next request by its active ref */
>-		i915_gem_object_ggtt_unpin(from->legacy_hw_ctx.rcs_state);
>-		i915_gem_context_unreference(from);
>-	}
>-
>-	uninitialized = !to->legacy_hw_ctx.initialized && from == NULL;
>-	to->legacy_hw_ctx.initialized = true;
>-
>-done:
>-	i915_gem_context_reference(to);
>-	ring->last_context = to;
>-
>-	if (uninitialized) {
>-		ret = i915_gem_render_state_init(ring);
>-		if (ret)
>-			DRM_ERROR("init render state: %d\n", ret);
>+		from_obj->dirty = 1;
> 	}
> 
>+	rq->has_ctx_switch = true;
> 	return 0;
> 
> unpin_out:
>-	if (ring->id == RCS)
>-		i915_gem_object_ggtt_unpin(to->legacy_hw_ctx.rcs_state);
>+	if (ctx->state)
>+		i915_gem_object_ggtt_unpin(ctx->state);
> 	return ret;
> }
> 
> /**
>- * i915_switch_context() - perform a GPU context switch.
>- * @ring: ring for which we'll execute the context switch
>- * @to: the context to switch to
>- *
>- * The context life cycle is simple. The context refcount is incremented and
>- * decremented by 1 and create and destroy. If the context is in use by the GPU,
>- * it will have a refoucnt > 1. This allows us to destroy the context abstract
>- * object while letting the normal object tracking destroy the backing BO.
>+ * i915_request_switch_context__commit() - commit the context sitch
>+ * @rq: request for which we have executed the context switch
>  */
>-int i915_switch_context(struct intel_engine_cs *ring,
>-			struct intel_context *to)
>+void i915_request_switch_context__commit(struct i915_gem_request *rq)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>+	struct intel_context *ctx;
> 
>-	WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
>+	lockdep_assert_held(&rq->i915->dev->struct_mutex);
> 
>-	if (to->legacy_hw_ctx.rcs_state == NULL) { /* We have the fake context */
>-		if (to != ring->last_context) {
>-			i915_gem_context_reference(to);
>-			if (ring->last_context)
>-				i915_gem_context_unreference(ring->last_context);
>-			ring->last_context = to;
>-		}
>-		return 0;
>-	}
>+	if (!rq->has_ctx_switch)
>+		return;
>+
>+	ctx = rq->ring->last_context;
>+	if (ctx && ctx->ring[rq->engine->id].state)
>+		i915_gem_object_ggtt_unpin(ctx->ring[rq->engine->id].state);
>+
>+	ctx = rq->ctx;
>+	ctx->remap_slice &= ~rq->remap_l3;
>+	ctx->ring[rq->engine->id].initialized = true;
>+
>+	rq->has_ctx_switch = false;
>+}
>+
>+/**
>+ * i915_request_switch_context__undo() - unwind the context sitch
>+ * @rq: request for which we have executed the context switch
>+ */
>+void i915_request_switch_context__undo(struct i915_gem_request *rq)
>+{
>+	lockdep_assert_held(&rq->i915->dev->struct_mutex);
>+
>+	if (!rq->has_ctx_switch)
>+		return;
> 
>-	return do_switch(ring, to);
>+	if (rq->ctx->ring[rq->engine->id].state)
>+		i915_gem_object_ggtt_unpin(rq->ctx->ring[rq->engine->id].state);
> }
> 
>-static bool hw_context_enabled(struct drm_device *dev)
>+static bool contexts_enabled(struct drm_i915_private *dev_priv)
> {
>-	return to_i915(dev)->hw_context_size;
>+	return dev_priv->hw_context_size;
> }
> 
> int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
>@@ -779,14 +705,14 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
> 	struct intel_context *ctx;
> 	int ret;
> 
>-	if (!hw_context_enabled(dev))
>+	if (!contexts_enabled(to_i915(dev)))
> 		return -ENODEV;
> 
> 	ret = i915_mutex_lock_interruptible(dev);
> 	if (ret)
> 		return ret;
> 
>-	ctx = i915_gem_create_context(dev, file_priv, USES_FULL_PPGTT(dev));
>+	ctx = i915_gem_create_context(dev, file_priv);
> 	mutex_unlock(&dev->struct_mutex);
> 	if (IS_ERR(ctx))
> 		return PTR_ERR(ctx);
>diff --git a/drivers/gpu/drm/i915/i915_gem_debug.c b/drivers/gpu/drm/i915/i915_gem_debug.c
>deleted file mode 100644
>index f462d1b..0000000
>--- a/drivers/gpu/drm/i915/i915_gem_debug.c
>+++ /dev/null
>@@ -1,118 +0,0 @@
>-/*
>- * Copyright Â© 2008 Intel Corporation
>- *
>- * Permission is hereby granted, free of charge, to any person obtaining a
>- * copy of this software and associated documentation files (the "Software"),
>- * to deal in the Software without restriction, including without limitation
>- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
>- * and/or sell copies of the Software, and to permit persons to whom the
>- * Software is furnished to do so, subject to the following conditions:
>- *
>- * The above copyright notice and this permission notice (including the next
>- * paragraph) shall be included in all copies or substantial portions of the
>- * Software.
>- *
>- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
>- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
>- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
>- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
>- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
>- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
>- * IN THE SOFTWARE.
>- *
>- * Authors:
>- *    Keith Packard <keithp@keithp.com>
>- *
>- */
>-
>-#include <drm/drmP.h>
>-#include <drm/i915_drm.h>
>-#include "i915_drv.h"
>-
>-#if WATCH_LISTS
>-int
>-i915_verify_lists(struct drm_device *dev)
>-{
>-	static int warned;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_gem_object *obj;
>-	int err = 0;
>-
>-	if (warned)
>-		return 0;
>-
>-	list_for_each_entry(obj, &dev_priv->render_ring.active_list, list) {
>-		if (obj->base.dev != dev ||
>-		    !atomic_read(&obj->base.refcount.refcount)) {
>-			DRM_ERROR("freed render active %p\n", obj);
>-			err++;
>-			break;
>-		} else if (!obj->active ||
>-			   (obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) {
>-			DRM_ERROR("invalid render active %p (a %d r %x)\n",
>-				  obj,
>-				  obj->active,
>-				  obj->base.read_domains);
>-			err++;
>-		} else if (obj->base.write_domain && list_empty(&obj->gpu_write_list)) {
>-			DRM_ERROR("invalid render active %p (w %x, gwl %d)\n",
>-				  obj,
>-				  obj->base.write_domain,
>-				  !list_empty(&obj->gpu_write_list));
>-			err++;
>-		}
>-	}
>-
>-	list_for_each_entry(obj, &dev_priv->mm.flushing_list, list) {
>-		if (obj->base.dev != dev ||
>-		    !atomic_read(&obj->base.refcount.refcount)) {
>-			DRM_ERROR("freed flushing %p\n", obj);
>-			err++;
>-			break;
>-		} else if (!obj->active ||
>-			   (obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0 ||
>-			   list_empty(&obj->gpu_write_list)) {
>-			DRM_ERROR("invalid flushing %p (a %d w %x gwl %d)\n",
>-				  obj,
>-				  obj->active,
>-				  obj->base.write_domain,
>-				  !list_empty(&obj->gpu_write_list));
>-			err++;
>-		}
>-	}
>-
>-	list_for_each_entry(obj, &dev_priv->mm.gpu_write_list, gpu_write_list) {
>-		if (obj->base.dev != dev ||
>-		    !atomic_read(&obj->base.refcount.refcount)) {
>-			DRM_ERROR("freed gpu write %p\n", obj);
>-			err++;
>-			break;
>-		} else if (!obj->active ||
>-			   (obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) {
>-			DRM_ERROR("invalid gpu write %p (a %d w %x)\n",
>-				  obj,
>-				  obj->active,
>-				  obj->base.write_domain);
>-			err++;
>-		}
>-	}
>-
>-	list_for_each_entry(obj, &i915_gtt_vm->inactive_list, list) {
>-		if (obj->base.dev != dev ||
>-		    !atomic_read(&obj->base.refcount.refcount)) {
>-			DRM_ERROR("freed inactive %p\n", obj);
>-			err++;
>-			break;
>-		} else if (obj->pin_count || obj->active ||
>-			   (obj->base.write_domain & I915_GEM_GPU_DOMAINS)) {
>-			DRM_ERROR("invalid inactive %p (p %d a %d w %x)\n",
>-				  obj,
>-				  obj->pin_count, obj->active,
>-				  obj->base.write_domain);
>-			err++;
>-		}
>-	}
>-
>-	return warned = err;
>-}
>-#endif /* WATCH_LIST */
>diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
>index bbf4b12..886ff2e 100644
>--- a/drivers/gpu/drm/i915/i915_gem_evict.c
>+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
>@@ -243,7 +243,7 @@ int
> i915_gem_evict_everything(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct i915_address_space *vm;
>+	struct i915_address_space *vm, *v;
> 	bool lists_empty = true;
> 	int ret;
> 
>@@ -270,7 +270,7 @@ i915_gem_evict_everything(struct drm_device *dev)
> 	i915_gem_retire_requests(dev);
> 
> 	/* Having flushed everything, unbind() should never raise an error */
>-	list_for_each_entry(vm, &dev_priv->vm_list, global_link)
>+	list_for_each_entry_safe(vm, v, &dev_priv->vm_list, global_link)
> 		WARN_ON(i915_gem_evict_vm(vm, false));
> 
> 	return 0;
>diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
>index 60998fc..182c0b3 100644
>--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
>+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
>@@ -35,12 +35,14 @@
> 
> #define  __EXEC_OBJECT_HAS_PIN (1<<31)
> #define  __EXEC_OBJECT_HAS_FENCE (1<<30)
>+#define  __EXEC_OBJECT_NEEDS_MAP (1<<29)
> #define  __EXEC_OBJECT_NEEDS_BIAS (1<<28)
> 
> #define BATCH_OFFSET_BIAS (256*1024)
> 
> struct eb_vmas {
> 	struct list_head vmas;
>+	struct i915_vma *batch;
> 	int and;
> 	union {
> 		struct i915_vma *lut[0];
>@@ -87,6 +89,26 @@ eb_reset(struct eb_vmas *eb)
> 		memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
> }
> 
>+static struct i915_vma *
>+eb_get_batch(struct eb_vmas *eb)
>+{
>+	struct i915_vma *vma =
>+		list_entry(eb->vmas.prev, typeof(*vma), exec_list);
>+
>+	/*
>+	 * SNA is doing fancy tricks with compressing batch buffers, which leads
>+	 * to negative relocation deltas. Usually that works out ok since the
>+	 * relocate address is still positive, except when the batch is placed
>+	 * very low in the GTT. Ensure this doesn't happen.
>+	 *
>+	 * Note that actual hangs have only been observed on gen7, but for
>+	 * paranoia do it everywhere.
>+	 */
>+	vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
>+
>+	return vma;
>+}
>+
> static int
> eb_lookup_vmas(struct eb_vmas *eb,
> 	       struct drm_i915_gem_exec_object2 *exec,
>@@ -94,7 +116,6 @@ eb_lookup_vmas(struct eb_vmas *eb,
> 	       struct i915_address_space *vm,
> 	       struct drm_file *file)
> {
>-	struct drm_i915_private *dev_priv = vm->dev->dev_private;
> 	struct drm_i915_gem_object *obj;
> 	struct list_head objects;
> 	int i, ret;
>@@ -129,20 +150,6 @@ eb_lookup_vmas(struct eb_vmas *eb,
> 	i = 0;
> 	while (!list_empty(&objects)) {
> 		struct i915_vma *vma;
>-		struct i915_address_space *bind_vm = vm;
>-
>-		if (exec[i].flags & EXEC_OBJECT_NEEDS_GTT &&
>-		    USES_FULL_PPGTT(vm->dev)) {
>-			ret = -EINVAL;
>-			goto err;
>-		}
>-
>-		/* If we have secure dispatch, or the userspace assures us that
>-		 * they know what they're doing, use the GGTT VM.
>-		 */
>-		if (((args->flags & I915_EXEC_SECURE) &&
>-		    (i == (args->buffer_count - 1))))
>-			bind_vm = &dev_priv->gtt.base;
> 
> 		obj = list_first_entry(&objects,
> 				       struct drm_i915_gem_object,
>@@ -156,7 +163,7 @@ eb_lookup_vmas(struct eb_vmas *eb,
> 		 * from the (obj, vm) we don't run the risk of creating
> 		 * duplicated vmas for the same vm.
> 		 */
>-		vma = i915_gem_obj_lookup_or_create_vma(obj, bind_vm);
>+		vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
> 		if (IS_ERR(vma)) {
> 			DRM_DEBUG("Failed to lookup VMA\n");
> 			ret = PTR_ERR(vma);
>@@ -179,6 +186,9 @@ eb_lookup_vmas(struct eb_vmas *eb,
> 		++i;
> 	}
> 
>+	/* take note of the batch buffer before we might reorder the lists */
>+	eb->batch = eb_get_batch(eb);
>+
> 	return 0;
> 
> 
>@@ -270,7 +280,7 @@ relocate_entry_cpu(struct drm_i915_gem_object *obj,
> {
> 	struct drm_device *dev = obj->base.dev;
> 	uint32_t page_offset = offset_in_page(reloc->offset);
>-	uint64_t delta = reloc->delta + target_offset;
>+	uint64_t delta = (int)reloc->delta + target_offset;
> 	char *vaddr;
> 	int ret;
> 
>@@ -306,8 +316,8 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj,
> {
> 	struct drm_device *dev = obj->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	uint64_t delta = reloc->delta + target_offset;
>-	uint32_t __iomem *reloc_entry;
>+	uint64_t delta = (int)reloc->delta + target_offset;
>+	uint64_t offset;
> 	void __iomem *reloc_page;
> 	int ret;
> 
>@@ -320,25 +330,24 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj,
> 		return ret;
> 
> 	/* Map the page containing the relocation we're going to perform.  */
>-	reloc->offset += i915_gem_obj_ggtt_offset(obj);
>+	offset = i915_gem_obj_ggtt_offset(obj);
>+	offset += reloc->offset;
> 	reloc_page = io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
>-			reloc->offset & PAGE_MASK);
>-	reloc_entry = (uint32_t __iomem *)
>-		(reloc_page + offset_in_page(reloc->offset));
>-	iowrite32(lower_32_bits(delta), reloc_entry);
>+					      offset & PAGE_MASK);
>+	iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));
> 
> 	if (INTEL_INFO(dev)->gen >= 8) {
>-		reloc_entry += 1;
>+		offset += sizeof(uint32_t);
> 
>-		if (offset_in_page(reloc->offset + sizeof(uint32_t)) == 0) {
>+		if (offset_in_page(offset) == 0) {
> 			io_mapping_unmap_atomic(reloc_page);
>-			reloc_page = io_mapping_map_atomic_wc(
>-					dev_priv->gtt.mappable,
>-					reloc->offset + sizeof(uint32_t));
>-			reloc_entry = reloc_page;
>+			reloc_page =
>+				io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
>+							 offset);
> 		}
> 
>-		iowrite32(upper_32_bits(delta), reloc_entry);
>+		iowrite32(upper_32_bits(delta),
>+			  reloc_page + offset_in_page(offset));
> 	}
> 
> 	io_mapping_unmap_atomic(reloc_page);
>@@ -437,13 +446,11 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
> 		ret = relocate_entry_cpu(obj, reloc, target_offset);
> 	else
> 		ret = relocate_entry_gtt(obj, reloc, target_offset);
>-
> 	if (ret)
> 		return ret;
> 
> 	/* and update the user's relocation entry */
> 	reloc->presumed_offset = target_offset;
>-
> 	return 0;
> }
> 
>@@ -535,34 +542,18 @@ i915_gem_execbuffer_relocate(struct eb_vmas *eb)
> }
> 
> static int
>-need_reloc_mappable(struct i915_vma *vma)
>-{
>-	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
>-	return entry->relocation_count && !use_cpu_reloc(vma->obj) &&
>-		i915_is_ggtt(vma->vm);
>-}
>-
>-static int
> i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
>-				struct intel_engine_cs *ring,
>+				struct intel_engine_cs *engine,
> 				bool *need_reloc)
> {
> 	struct drm_i915_gem_object *obj = vma->obj;
> 	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
>-	bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
>-	bool need_fence;
> 	uint64_t flags;
> 	int ret;
> 
> 	flags = 0;
>-
>-	need_fence =
>-		has_fenced_gpu_access &&
>-		entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
>-		obj->tiling_mode != I915_TILING_NONE;
>-	if (need_fence || need_reloc_mappable(vma))
>+	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
> 		flags |= PIN_MAPPABLE;
>-
> 	if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
> 		flags |= PIN_GLOBAL;
> 	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
>@@ -574,17 +565,13 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
> 
> 	entry->flags |= __EXEC_OBJECT_HAS_PIN;
> 
>-	if (has_fenced_gpu_access) {
>-		if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
>-			ret = i915_gem_object_get_fence(obj);
>-			if (ret)
>-				return ret;
>-
>-			if (i915_gem_object_pin_fence(obj))
>-				entry->flags |= __EXEC_OBJECT_HAS_FENCE;
>+	if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
>+		ret = i915_gem_object_get_fence(obj);
>+		if (ret)
>+			return ret;
> 
>-			obj->pending_fenced_gpu_access = true;
>-		}
>+		if (i915_gem_object_pin_fence(obj))
>+			entry->flags |= __EXEC_OBJECT_HAS_FENCE;
> 	}
> 
> 	if (entry->offset != vma->node.start) {
>@@ -601,26 +588,40 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
> }
> 
> static bool
>-eb_vma_misplaced(struct i915_vma *vma, bool has_fenced_gpu_access)
>+need_reloc_mappable(struct i915_vma *vma)
> {
> 	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
>-	struct drm_i915_gem_object *obj = vma->obj;
>-	bool need_fence, need_mappable;
> 
>-	need_fence =
>-		has_fenced_gpu_access &&
>-		entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
>-		obj->tiling_mode != I915_TILING_NONE;
>-	need_mappable = need_fence || need_reloc_mappable(vma);
>+	if (entry->relocation_count == 0)
>+		return false;
>+
>+	if (!i915_is_ggtt(vma->vm))
>+		return false;
>+
>+	/* See also use_cpu_reloc() */
>+	if (HAS_LLC(vma->obj->base.dev))
>+		return false;
>+
>+	if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
>+		return false;
>+
>+	return true;
>+}
> 
>-	WARN_ON((need_mappable || need_fence) &&
>+static bool
>+eb_vma_misplaced(struct i915_vma *vma)
>+{
>+	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
>+	struct drm_i915_gem_object *obj = vma->obj;
>+
>+	WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
> 	       !i915_is_ggtt(vma->vm));
> 
> 	if (entry->alignment &&
> 	    vma->node.start & (entry->alignment - 1))
> 		return true;
> 
>-	if (need_mappable && !obj->map_and_fenceable)
>+	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
> 		return true;
> 
> 	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
>@@ -631,7 +632,7 @@ eb_vma_misplaced(struct i915_vma *vma, bool has_fenced_gpu_access)
> }
> 
> static int
>-i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
>+i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
> 			    struct list_head *vmas,
> 			    bool *need_relocs)
> {
>@@ -639,13 +640,10 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
> 	struct i915_vma *vma;
> 	struct i915_address_space *vm;
> 	struct list_head ordered_vmas;
>-	bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
>+	bool has_fenced_gpu_access = INTEL_INFO(engine->i915)->gen < 4;
> 	int retry;
> 
>-	if (list_empty(vmas))
>-		return 0;
>-
>-	i915_gem_retire_requests_ring(ring);
>+	i915_gem_retire_requests__engine(engine);
> 
> 	vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
> 
>@@ -658,20 +656,21 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
> 		obj = vma->obj;
> 		entry = vma->exec_entry;
> 
>+		if (!has_fenced_gpu_access)
>+			entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
> 		need_fence =
>-			has_fenced_gpu_access &&
> 			entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
> 			obj->tiling_mode != I915_TILING_NONE;
> 		need_mappable = need_fence || need_reloc_mappable(vma);
> 
>-		if (need_mappable)
>+		if (need_mappable) {
>+			entry->flags |= __EXEC_OBJECT_NEEDS_MAP;
> 			list_move(&vma->exec_list, &ordered_vmas);
>-		else
>+		} else
> 			list_move_tail(&vma->exec_list, &ordered_vmas);
> 
> 		obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
> 		obj->base.pending_write_domain = 0;
>-		obj->pending_fenced_gpu_access = false;
> 	}
> 	list_splice(&ordered_vmas, vmas);
> 
>@@ -696,10 +695,10 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
> 			if (!drm_mm_node_allocated(&vma->node))
> 				continue;
> 
>-			if (eb_vma_misplaced(vma, has_fenced_gpu_access))
>+			if (eb_vma_misplaced(vma))
> 				ret = i915_vma_unbind(vma);
> 			else
>-				ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
>+				ret = i915_gem_execbuffer_reserve_vma(vma, engine, need_relocs);
> 			if (ret)
> 				goto err;
> 		}
>@@ -709,7 +708,7 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
> 			if (drm_mm_node_allocated(&vma->node))
> 				continue;
> 
>-			ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
>+			ret = i915_gem_execbuffer_reserve_vma(vma, engine, need_relocs);
> 			if (ret)
> 				goto err;
> 		}
>@@ -729,10 +728,10 @@ err:
> }
> 
> static int
>-i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
>+i915_gem_execbuffer_relocate_slow(struct drm_i915_private *i915,
> 				  struct drm_i915_gem_execbuffer2 *args,
> 				  struct drm_file *file,
>-				  struct intel_engine_cs *ring,
>+				  struct intel_engine_cs *engine,
> 				  struct eb_vmas *eb,
> 				  struct drm_i915_gem_exec_object2 *exec)
> {
>@@ -744,9 +743,6 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
> 	int i, total, ret;
> 	unsigned count = args->buffer_count;
> 
>-	if (WARN_ON(list_empty(&eb->vmas)))
>-		return 0;
>-
> 	vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
> 
> 	/* We may process another execbuffer during the unlock... */
>@@ -757,7 +753,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
> 		drm_gem_object_unreference(&vma->obj->base);
> 	}
> 
>-	mutex_unlock(&dev->struct_mutex);
>+	mutex_unlock(&i915->dev->struct_mutex);
> 
> 	total = 0;
> 	for (i = 0; i < count; i++)
>@@ -768,7 +764,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
> 	if (reloc == NULL || reloc_offset == NULL) {
> 		drm_free_large(reloc);
> 		drm_free_large(reloc_offset);
>-		mutex_lock(&dev->struct_mutex);
>+		mutex_lock(&i915->dev->struct_mutex);
> 		return -ENOMEM;
> 	}
> 
>@@ -783,7 +779,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
> 		if (copy_from_user(reloc+total, user_relocs,
> 				   exec[i].relocation_count * sizeof(*reloc))) {
> 			ret = -EFAULT;
>-			mutex_lock(&dev->struct_mutex);
>+			mutex_lock(&i915->dev->struct_mutex);
> 			goto err;
> 		}
> 
>@@ -801,7 +797,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
> 					   &invalid_offset,
> 					   sizeof(invalid_offset))) {
> 				ret = -EFAULT;
>-				mutex_lock(&dev->struct_mutex);
>+				mutex_lock(&i915->dev->struct_mutex);
> 				goto err;
> 			}
> 		}
>@@ -810,9 +806,9 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
> 		total += exec[i].relocation_count;
> 	}
> 
>-	ret = i915_mutex_lock_interruptible(dev);
>+	ret = i915_mutex_lock_interruptible(i915->dev);
> 	if (ret) {
>-		mutex_lock(&dev->struct_mutex);
>+		mutex_lock(&i915->dev->struct_mutex);
> 		goto err;
> 	}
> 
>@@ -823,7 +819,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
> 		goto err;
> 
> 	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
>-	ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs);
>+	ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, &need_relocs);
> 	if (ret)
> 		goto err;
> 
>@@ -848,17 +844,19 @@ err:
> }
> 
> static int
>-i915_gem_execbuffer_move_to_gpu(struct intel_engine_cs *ring,
>-				struct list_head *vmas)
>+vmas_move_to_rq(struct eb_vmas *eb,
>+		struct i915_gem_request *rq)
> {
> 	struct i915_vma *vma;
> 	uint32_t flush_domains = 0;
> 	bool flush_chipset = false;
> 	int ret;
> 
>-	list_for_each_entry(vma, vmas, exec_list) {
>+	/* 1: flush/serialise damage from other sources */
>+	list_for_each_entry(vma, &eb->vmas, exec_list) {
> 		struct drm_i915_gem_object *obj = vma->obj;
>-		ret = i915_gem_object_sync(obj, ring);
>+
>+		ret = i915_gem_object_sync(obj, rq);
> 		if (ret)
> 			return ret;
> 
>@@ -866,18 +864,39 @@ i915_gem_execbuffer_move_to_gpu(struct intel_engine_cs *ring,
> 			flush_chipset |= i915_gem_clflush_object(obj, false);
> 
> 		flush_domains |= obj->base.write_domain;
>+		if (obj->last_read[rq->engine->id].request == NULL)
>+			rq->pending_flush |= I915_INVALIDATE_CACHES;
> 	}
> 
> 	if (flush_chipset)
>-		i915_gem_chipset_flush(ring->dev);
>+		i915_gem_chipset_flush(rq->i915->dev);
> 
> 	if (flush_domains & I915_GEM_DOMAIN_GTT)
> 		wmb();
> 
>-	/* Unconditionally invalidate gpu caches and ensure that we do flush
>-	 * any residual writes from the previous batch.
>-	 */
>-	return intel_ring_invalidate_all_caches(ring);
>+	/* 2: invalidate the caches from this ring after emitting semaphores */
>+	ret = i915_request_emit_flush(rq, I915_INVALIDATE_CACHES);
>+	if (ret)
>+		return ret;
>+
>+	/* 3: track flushes and objects for this rq */
>+	list_for_each_entry(vma, &eb->vmas, exec_list) {
>+		struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
>+		unsigned fenced;
>+
>+		fenced = 0;
>+		if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
>+			fenced |= VMA_IS_FENCED;
>+			if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
>+				fenced |= VMA_HAS_FENCE;
>+		}
>+
>+		ret = i915_request_add_vma(rq, vma, fenced);
>+		if (ret)
>+			return ret;
>+	}
>+
>+	return 0;
> }
> 
> static bool
>@@ -890,18 +909,24 @@ i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
> }
> 
> static int
>-validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
>+validate_exec_list(struct drm_i915_private *dev_priv,
>+		   struct drm_i915_gem_exec_object2 *exec,
> 		   int count)
> {
>-	int i;
> 	unsigned relocs_total = 0;
> 	unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
>+	unsigned invalid_flags;
>+	int i;
>+
>+	invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
>+	if (USES_FULL_PPGTT(dev_priv))
>+		invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
> 
> 	for (i = 0; i < count; i++) {
> 		char __user *ptr = to_user_ptr(exec[i].relocs_ptr);
> 		int length; /* limited by fault_in_pages_readable() */
> 
>-		if (exec[i].flags & __EXEC_OBJECT_UNKNOWN_FLAGS)
>+		if (exec[i].flags & invalid_flags)
> 			return -EINVAL;
> 
> 		/* First check for malicious input causing overflow in
>@@ -932,13 +957,14 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
> }
> 
> static struct intel_context *
>-i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
>-			  struct intel_engine_cs *ring, const u32 ctx_id)
>+i915_gem_validate_context(struct drm_file *file,
>+			  struct intel_engine_cs *engine,
>+			  const u32 ctx_id)
> {
> 	struct intel_context *ctx = NULL;
> 	struct i915_ctx_hang_stats *hs;
> 
>-	if (ring->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)
>+	if (engine->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)
> 		return ERR_PTR(-EINVAL);
> 
> 	ctx = i915_gem_context_get(file->driver_priv, ctx_id);
>@@ -954,109 +980,152 @@ i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
> 	return ctx;
> }
> 
>-static void
>-i915_gem_execbuffer_move_to_active(struct list_head *vmas,
>-				   struct intel_engine_cs *ring)
>+static int
>+reset_sol_offsets(struct i915_gem_request *rq)
> {
>-	struct i915_vma *vma;
>+	struct intel_ringbuffer *ring;
>+	int i;
> 
>-	list_for_each_entry(vma, vmas, exec_list) {
>-		struct drm_i915_gem_object *obj = vma->obj;
>-		u32 old_read = obj->base.read_domains;
>-		u32 old_write = obj->base.write_domain;
>+	if (!IS_GEN7(rq->i915) || rq->engine->id != RCS) {
>+		DRM_DEBUG("sol reset is gen7/rcs only\n");
>+		return -EINVAL;
>+	}
> 
>-		obj->base.write_domain = obj->base.pending_write_domain;
>-		if (obj->base.write_domain == 0)
>-			obj->base.pending_read_domains |= obj->base.read_domains;
>-		obj->base.read_domains = obj->base.pending_read_domains;
>-		obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
>+	ring = intel_ring_begin(rq, 4 * 3);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
>-		i915_vma_move_to_active(vma, ring);
>-		if (obj->base.write_domain) {
>-			obj->dirty = 1;
>-			obj->last_write_seqno = intel_ring_get_seqno(ring);
>+	for (i = 0; i < 4; i++) {
>+		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
>+		intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
>+		intel_ring_emit(ring, 0);
>+	}
> 
>-			intel_fb_obj_invalidate(obj, ring);
>+	intel_ring_advance(ring);
>+	return 0;
>+}
> 
>-			/* update for the implicit flush after a batch */
>-			obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
>-		}
>+static int
>+emit_box(struct i915_gem_request *rq,
>+	 struct drm_clip_rect *box,
>+	 int DR1, int DR4)
>+{
>+	struct intel_ringbuffer *ring;
> 
>-		trace_i915_gem_object_change_domain(obj, old_read, old_write);
>+	if (box->y2 <= box->y1 || box->x2 <= box->x1 ||
>+	    box->y2 <= 0 || box->x2 <= 0) {
>+		DRM_DEBUG("Bad box %d,%d..%d,%d\n",
>+			  box->x1, box->y1, box->x2, box->y2);
>+		return -EINVAL;
> 	}
>-}
> 
>-static void
>-i915_gem_execbuffer_retire_commands(struct drm_device *dev,
>-				    struct drm_file *file,
>-				    struct intel_engine_cs *ring,
>-				    struct drm_i915_gem_object *obj)
>-{
>-	/* Unconditionally force add_request to emit a full flush. */
>-	ring->gpu_caches_dirty = true;
>+	if (INTEL_INFO(rq->i915)->gen >= 4) {
>+		ring = intel_ring_begin(rq, 4);
>+		if (IS_ERR(ring))
>+			return PTR_ERR(ring);
> 
>-	/* Add a breadcrumb for the completion of the batch buffer */
>-	(void)__i915_add_request(ring, file, obj, NULL);
>+		intel_ring_emit(ring, GFX_OP_DRAWRECT_INFO_I965);
>+	} else {
>+		ring = intel_ring_begin(rq, 5);
>+		if (IS_ERR(ring))
>+			return PTR_ERR(ring);
>+
>+		intel_ring_emit(ring, GFX_OP_DRAWRECT_INFO);
>+		intel_ring_emit(ring, DR1);
>+	}
>+	intel_ring_emit(ring, (box->x1 & 0xffff) | box->y1 << 16);
>+	intel_ring_emit(ring, ((box->x2 - 1) & 0xffff) | (box->y2 - 1) << 16);
>+	intel_ring_emit(ring, DR4);
>+	intel_ring_advance(ring);
>+
>+	return 0;
> }
> 
>-static int
>-i915_reset_gen7_sol_offsets(struct drm_device *dev,
>-			    struct intel_engine_cs *ring)
>+static int set_contants_base(struct i915_gem_request *rq,
>+			     struct drm_i915_gem_execbuffer2 *args)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int ret, i;
>+	int mode = args->flags & I915_EXEC_CONSTANTS_MASK;
>+	u32 mask = I915_EXEC_CONSTANTS_MASK;
> 
>-	if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS]) {
>-		DRM_DEBUG("sol reset is gen7/rcs only\n");
>+	switch (mode) {
>+	case I915_EXEC_CONSTANTS_REL_GENERAL:
>+	case I915_EXEC_CONSTANTS_ABSOLUTE:
>+	case I915_EXEC_CONSTANTS_REL_SURFACE:
>+		if (mode != 0 && rq->engine->id != RCS) {
>+			DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
>+			return -EINVAL;
>+		}
>+
>+		if (mode != rq->engine->i915->relative_constants_mode) {
>+			if (INTEL_INFO(rq->engine->i915)->gen < 4) {
>+				DRM_DEBUG("no rel constants on pre-gen4\n");
>+				return -EINVAL;
>+			}
>+
>+			if (INTEL_INFO(rq->engine->i915)->gen > 5 &&
>+			    mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
>+				DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
>+				return -EINVAL;
>+			}
>+
>+			/* The HW changed the meaning on this bit on gen6 */
>+			if (INTEL_INFO(rq->i915)->gen >= 6)
>+				mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
>+		}
>+		break;
>+	default:
>+		DRM_DEBUG("execbuf with unknown constants: %d\n", mode);
> 		return -EINVAL;
> 	}
> 
>-	ret = intel_ring_begin(ring, 4 * 3);
>-	if (ret)
>-		return ret;
>+	/* XXX INSTPM is per-context not global etc */
>+	if (rq->engine->id == RCS && mode != rq->i915->relative_constants_mode) {
>+		struct intel_ringbuffer *ring;
>+
>+		ring = intel_ring_begin(rq, 3);
>+		if (IS_ERR(ring))
>+			return PTR_ERR(ring);
> 
>-	for (i = 0; i < 4; i++) {
> 		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
>-		intel_ring_emit(ring, GEN7_SO_WRITE_OFFSET(i));
>-		intel_ring_emit(ring, 0);
>-	}
>+		intel_ring_emit(ring, INSTPM);
>+		intel_ring_emit(ring, mask << 16 | mode);
>+		intel_ring_advance(ring);
> 
>-	intel_ring_advance(ring);
>+		rq->i915->relative_constants_mode = mode;
>+	}
> 
> 	return 0;
> }
> 
> static int
>-legacy_ringbuffer_submission(struct drm_device *dev, struct drm_file *file,
>-			     struct intel_engine_cs *ring,
>-			     struct intel_context *ctx,
>-			     struct drm_i915_gem_execbuffer2 *args,
>-			     struct list_head *vmas,
>-			     struct drm_i915_gem_object *batch_obj,
>-			     u64 exec_start, u32 flags)
>+submit_execbuf(struct intel_engine_cs *engine,
>+	       struct intel_context *ctx,
>+	       struct drm_i915_gem_execbuffer2 *args,
>+	       struct eb_vmas *eb,
>+	       u64 exec_start, u32 flags)
> {
> 	struct drm_clip_rect *cliprects = NULL;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	u64 exec_len;
>-	int instp_mode;
>-	u32 instp_mask;
>-	int i, ret = 0;
>+	struct i915_gem_request *rq = NULL;
>+	int i, ret;
> 
> 	if (args->num_cliprects != 0) {
>-		if (ring != &dev_priv->ring[RCS]) {
>+		if (engine->id != RCS) {
> 			DRM_DEBUG("clip rectangles are only valid with the render ring\n");
>-			return -EINVAL;
>+			ret = -EINVAL;
>+			goto out;
> 		}
> 
>-		if (INTEL_INFO(dev)->gen >= 5) {
>+		if (INTEL_INFO(engine->i915)->gen >= 5) {
> 			DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
>-			return -EINVAL;
>+			ret = -EINVAL;
>+			goto out;
> 		}
> 
> 		if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) {
> 			DRM_DEBUG("execbuf with %u cliprects\n",
> 				  args->num_cliprects);
>-			return -EINVAL;
>+			ret = -EINVAL;
>+			goto out;
> 		}
> 
> 		cliprects = kcalloc(args->num_cliprects,
>@@ -1064,14 +1133,14 @@ legacy_ringbuffer_submission(struct drm_device *dev, struct drm_file *file,
> 				    GFP_KERNEL);
> 		if (cliprects == NULL) {
> 			ret = -ENOMEM;
>-			goto error;
>+			goto out;
> 		}
> 
> 		if (copy_from_user(cliprects,
> 				   to_user_ptr(args->cliprects_ptr),
> 				   sizeof(*cliprects)*args->num_cliprects)) {
> 			ret = -EFAULT;
>-			goto error;
>+			goto out_cliprects;
> 		}
> 	} else {
> 		if (args->DR4 == 0xffffffff) {
>@@ -1081,172 +1150,115 @@ legacy_ringbuffer_submission(struct drm_device *dev, struct drm_file *file,
> 
> 		if (args->DR1 || args->DR4 || args->cliprects_ptr) {
> 			DRM_DEBUG("0 cliprects but dirt in cliprects fields\n");
>-			return -EINVAL;
>-		}
>-	}
>-
>-	ret = i915_gem_execbuffer_move_to_gpu(ring, vmas);
>-	if (ret)
>-		goto error;
>-
>-	ret = i915_switch_context(ring, ctx);
>-	if (ret)
>-		goto error;
>-
>-	instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
>-	instp_mask = I915_EXEC_CONSTANTS_MASK;
>-	switch (instp_mode) {
>-	case I915_EXEC_CONSTANTS_REL_GENERAL:
>-	case I915_EXEC_CONSTANTS_ABSOLUTE:
>-	case I915_EXEC_CONSTANTS_REL_SURFACE:
>-		if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) {
>-			DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
> 			ret = -EINVAL;
>-			goto error;
>-		}
>-
>-		if (instp_mode != dev_priv->relative_constants_mode) {
>-			if (INTEL_INFO(dev)->gen < 4) {
>-				DRM_DEBUG("no rel constants on pre-gen4\n");
>-				ret = -EINVAL;
>-				goto error;
>-			}
>-
>-			if (INTEL_INFO(dev)->gen > 5 &&
>-			    instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
>-				DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
>-				ret = -EINVAL;
>-				goto error;
>-			}
>-
>-			/* The HW changed the meaning on this bit on gen6 */
>-			if (INTEL_INFO(dev)->gen >= 6)
>-				instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
>+			goto out;
> 		}
>-		break;
>-	default:
>-		DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
>-		ret = -EINVAL;
>-		goto error;
> 	}
> 
>-	if (ring == &dev_priv->ring[RCS] &&
>-			instp_mode != dev_priv->relative_constants_mode) {
>-		ret = intel_ring_begin(ring, 4);
>-		if (ret)
>-			goto error;
>+	rq = intel_engine_alloc_request(engine, ctx);
>+	if (IS_ERR(rq)) {
>+		ret = PTR_ERR(rq);
>+		goto out_cliprects;
>+	}
> 
>-		intel_ring_emit(ring, MI_NOOP);
>-		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
>-		intel_ring_emit(ring, INSTPM);
>-		intel_ring_emit(ring, instp_mask << 16 | instp_mode);
>-		intel_ring_advance(ring);
>+	ret = vmas_move_to_rq(eb, rq);
>+	if (ret)
>+		goto out_rq;
> 
>-		dev_priv->relative_constants_mode = instp_mode;
>-	}
>+	ret = set_contants_base(rq, args);
>+	if (ret)
>+		goto out_rq;
> 
> 	if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
>-		ret = i915_reset_gen7_sol_offsets(dev, ring);
>+		ret = reset_sol_offsets(rq);
> 		if (ret)
>-			goto error;
>+			goto out_rq;
> 	}
> 
>-	exec_len = args->batch_len;
> 	if (cliprects) {
> 		for (i = 0; i < args->num_cliprects; i++) {
>-			ret = i915_emit_box(dev, &cliprects[i],
>-					    args->DR1, args->DR4);
>+			ret = emit_box(rq, &cliprects[i],
>+				       args->DR1, args->DR4);
> 			if (ret)
>-				goto error;
>+				goto out_rq;
> 
>-			ret = ring->dispatch_execbuffer(ring,
>-							exec_start, exec_len,
>-							flags);
>+			ret = i915_request_emit_batchbuffer(rq, eb->batch,
>+							    exec_start,
>+							    args->batch_len,
>+							    flags);
> 			if (ret)
>-				goto error;
>+				goto out_rq;
> 		}
> 	} else {
>-		ret = ring->dispatch_execbuffer(ring,
>-						exec_start, exec_len,
>-						flags);
>+		ret = i915_request_emit_batchbuffer(rq, eb->batch,
>+						    exec_start,
>+						    args->batch_len,
>+						    flags);
> 		if (ret)
>-			return ret;
>+			goto out_rq;
> 	}
> 
>-	trace_i915_gem_ring_dispatch(ring, intel_ring_get_seqno(ring), flags);
>+	ret = i915_request_commit(rq);
>+	if (ret)
>+		goto out_rq;
>+
>+	i915_queue_hangcheck(rq->i915->dev);
> 
>-	i915_gem_execbuffer_move_to_active(vmas, ring);
>-	i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj);
>+	cancel_delayed_work_sync(&rq->i915->mm.idle_work);
>+	queue_delayed_work(rq->i915->wq,
>+			   &rq->i915->mm.retire_work,
>+			   round_jiffies_up_relative(HZ));
>+	intel_mark_busy(rq->i915->dev);
> 
>-error:
>+out_rq:
>+	i915_request_put(rq);
>+out_cliprects:
> 	kfree(cliprects);
>+out:
> 	return ret;
> }
> 
> /**
>  * Find one BSD ring to dispatch the corresponding BSD command.
>- * The Ring ID is returned.
>  */
>-static int gen8_dispatch_bsd_ring(struct drm_device *dev,
>-				  struct drm_file *file)
>+static struct intel_engine_cs *
>+gen8_select_bsd_engine(struct drm_i915_private *dev_priv,
>+		       struct drm_file *file)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct drm_i915_file_private *file_priv = file->driver_priv;
> 
>-	/* Check whether the file_priv is using one ring */
>-	if (file_priv->bsd_ring)
>-		return file_priv->bsd_ring->id;
>-	else {
>-		/* If no, use the ping-pong mechanism to select one ring */
>-		int ring_id;
>+	/* Use the ping-pong mechanism to select one ring for this client */
>+	if (file_priv->bsd_engine == NULL) {
>+		int id;
> 
>-		mutex_lock(&dev->struct_mutex);
>+		mutex_lock(&dev_priv->dev->struct_mutex);
> 		if (dev_priv->mm.bsd_ring_dispatch_index == 0) {
>-			ring_id = VCS;
>+			id = VCS;
> 			dev_priv->mm.bsd_ring_dispatch_index = 1;
> 		} else {
>-			ring_id = VCS2;
>+			id = VCS2;
> 			dev_priv->mm.bsd_ring_dispatch_index = 0;
> 		}
>-		file_priv->bsd_ring = &dev_priv->ring[ring_id];
>-		mutex_unlock(&dev->struct_mutex);
>-		return ring_id;
>+		file_priv->bsd_engine = &dev_priv->engine[id];
>+		mutex_unlock(&dev_priv->dev->struct_mutex);
> 	}
>-}
>-
>-static struct drm_i915_gem_object *
>-eb_get_batch(struct eb_vmas *eb)
>-{
>-	struct i915_vma *vma = list_entry(eb->vmas.prev, typeof(*vma), exec_list);
> 
>-	/*
>-	 * SNA is doing fancy tricks with compressing batch buffers, which leads
>-	 * to negative relocation deltas. Usually that works out ok since the
>-	 * relocate address is still positive, except when the batch is placed
>-	 * very low in the GTT. Ensure this doesn't happen.
>-	 *
>-	 * Note that actual hangs have only been observed on gen7, but for
>-	 * paranoia do it everywhere.
>-	 */
>-	vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
>-
>-	return vma->obj;
>+	return file_priv->bsd_engine;
> }
> 
> static int
>-i915_gem_do_execbuffer(struct drm_device *dev, void *data,
>+i915_gem_do_execbuffer(struct drm_i915_private *dev_priv, void *data,
> 		       struct drm_file *file,
> 		       struct drm_i915_gem_execbuffer2 *args,
> 		       struct drm_i915_gem_exec_object2 *exec)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct eb_vmas *eb;
>-	struct drm_i915_gem_object *batch_obj;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	struct intel_context *ctx;
> 	struct i915_address_space *vm;
> 	const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
> 	u64 exec_start = args->batch_start_offset;
>+	struct drm_i915_gem_object *batch;
> 	u32 flags;
> 	int ret;
> 	bool need_relocs;
>@@ -1254,7 +1266,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
> 	if (!i915_gem_check_execbuffer(args))
> 		return -EINVAL;
> 
>-	ret = validate_exec_list(exec, args->buffer_count);
>+	ret = validate_exec_list(dev_priv, exec, args->buffer_count);
> 	if (ret)
> 		return ret;
> 
>@@ -1275,18 +1287,16 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
> 	}
> 
> 	if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_DEFAULT)
>-		ring = &dev_priv->ring[RCS];
>+		engine = &dev_priv->engine[RCS];
> 	else if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_BSD) {
>-		if (HAS_BSD2(dev)) {
>-			int ring_id;
>-			ring_id = gen8_dispatch_bsd_ring(dev, file);
>-			ring = &dev_priv->ring[ring_id];
>-		} else
>-			ring = &dev_priv->ring[VCS];
>+		if (HAS_BSD2(dev_priv))
>+			engine = gen8_select_bsd_engine(dev_priv, file);
>+		else
>+			engine = &dev_priv->engine[VCS];
> 	} else
>-		ring = &dev_priv->ring[(args->flags & I915_EXEC_RING_MASK) - 1];
>+		engine = &dev_priv->engine[(args->flags & I915_EXEC_RING_MASK) - 1];
> 
>-	if (!intel_ring_initialized(ring)) {
>+	if (!intel_engine_initialized(engine)) {
> 		DRM_DEBUG("execbuf with invalid ring: %d\n",
> 			  (int)(args->flags & I915_EXEC_RING_MASK));
> 		return -EINVAL;
>@@ -1299,33 +1309,34 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
> 
> 	intel_runtime_pm_get(dev_priv);
> 
>-	ret = i915_mutex_lock_interruptible(dev);
>+	ret = i915_mutex_lock_interruptible(dev_priv->dev);
> 	if (ret)
> 		goto pre_mutex_err;
> 
> 	if (dev_priv->ums.mm_suspended) {
>-		mutex_unlock(&dev->struct_mutex);
>+		mutex_unlock(&dev_priv->dev->struct_mutex);
> 		ret = -EBUSY;
> 		goto pre_mutex_err;
> 	}
> 
>-	ctx = i915_gem_validate_context(dev, file, ring, ctx_id);
>+	ctx = i915_gem_validate_context(file, engine, ctx_id);
> 	if (IS_ERR(ctx)) {
>-		mutex_unlock(&dev->struct_mutex);
>+		mutex_unlock(&dev_priv->dev->struct_mutex);
> 		ret = PTR_ERR(ctx);
> 		goto pre_mutex_err;
> 	}
> 
> 	i915_gem_context_reference(ctx);
> 
>-	vm = ctx->vm;
>-	if (!USES_FULL_PPGTT(dev))
>+	if (ctx->ppgtt)
>+		vm = &ctx->ppgtt->base;
>+	else
> 		vm = &dev_priv->gtt.base;
> 
> 	eb = eb_create(args);
> 	if (eb == NULL) {
> 		i915_gem_context_unreference(ctx);
>-		mutex_unlock(&dev->struct_mutex);
>+		mutex_unlock(&dev_priv->dev->struct_mutex);
> 		ret = -ENOMEM;
> 		goto pre_mutex_err;
> 	}
>@@ -1335,12 +1346,9 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
> 	if (ret)
> 		goto err;
> 
>-	/* take note of the batch buffer before we might reorder the lists */
>-	batch_obj = eb_get_batch(eb);
>-
> 	/* Move the objects en-masse into the GTT, evicting if necessary. */
> 	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
>-	ret = i915_gem_execbuffer_reserve(ring, &eb->vmas, &need_relocs);
>+	ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, &need_relocs);
> 	if (ret)
> 		goto err;
> 
>@@ -1349,25 +1357,25 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
> 		ret = i915_gem_execbuffer_relocate(eb);
> 	if (ret) {
> 		if (ret == -EFAULT) {
>-			ret = i915_gem_execbuffer_relocate_slow(dev, args, file, ring,
>+			ret = i915_gem_execbuffer_relocate_slow(dev_priv, args, file, engine,
> 								eb, exec);
>-			BUG_ON(!mutex_is_locked(&dev->struct_mutex));
>+			BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
> 		}
> 		if (ret)
> 			goto err;
> 	}
> 
> 	/* Set the pending read domains for the batch buffer to COMMAND */
>-	if (batch_obj->base.pending_write_domain) {
>+	batch = eb->batch->obj;
>+	if (batch->base.pending_write_domain) {
> 		DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
> 		ret = -EINVAL;
> 		goto err;
> 	}
>-	batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
>+	batch->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
> 
>-	if (i915_needs_cmd_parser(ring)) {
>-		ret = i915_parse_cmds(ring,
>-				      batch_obj,
>+	if (i915_needs_cmd_parser(engine)) {
>+		ret = i915_parse_cmds(engine, batch,
> 				      args->batch_start_offset,
> 				      file->is_master);
> 		if (ret)
>@@ -1386,31 +1394,41 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
> 	/* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
> 	 * batch" bit. Hence we need to pin secure batches into the global gtt.
> 	 * hsw should have this fixed, but bdw mucks it up again. */
>-	if (flags & I915_DISPATCH_SECURE &&
>-	    !batch_obj->has_global_gtt_mapping) {
>-		/* When we have multiple VMs, we'll need to make sure that we
>-		 * allocate space first */
>-		struct i915_vma *vma = i915_gem_obj_to_ggtt(batch_obj);
>-		BUG_ON(!vma);
>-		vma->bind_vma(vma, batch_obj->cache_level, GLOBAL_BIND);
>-	}
>+	if (flags & I915_DISPATCH_SECURE) {
>+		/*
>+		 * So on first glance it looks freaky that we pin the batch here
>+		 * outside of the reservation loop. But:
>+		 * - The batch is already pinned into the relevant ppgtt, so we
>+		 *   already have the backing storage fully allocated.
>+		 * - No other BO uses the global gtt (well contexts, but meh),
>+		 *   so we don't really have issues with mutliple objects not
>+		 *   fitting due to fragmentation.
>+		 * So this is actually safe.
>+		 */
>+		ret = i915_gem_obj_ggtt_pin(batch, 0, 0);
>+		if (ret)
>+			goto err;
> 
>-	if (flags & I915_DISPATCH_SECURE)
>-		exec_start += i915_gem_obj_ggtt_offset(batch_obj);
>-	else
>-		exec_start += i915_gem_obj_offset(batch_obj, vm);
>+		exec_start += i915_gem_obj_ggtt_offset(batch);
>+	} else
>+		exec_start += i915_gem_obj_offset(batch, vm);
> 
>-	ret = legacy_ringbuffer_submission(dev, file, ring, ctx,
>-			args, &eb->vmas, batch_obj, exec_start, flags);
>-	if (ret)
>-		goto err;
>+	ret = submit_execbuf(engine, ctx, args, eb, exec_start, flags);
> 
>+	/*
>+	 * FIXME: We crucially rely upon the active tracking for the (ppgtt)
>+	 * batch vma for correctness. For less ugly and less fragility this
>+	 * needs to be adjusted to also track the ggtt batch vma properly as
>+	 * active.
>+	 */
>+	if (flags & I915_DISPATCH_SECURE)
>+		i915_gem_object_ggtt_unpin(batch);
> err:
> 	/* the request owns the ref now */
> 	i915_gem_context_unreference(ctx);
> 	eb_destroy(eb);
> 
>-	mutex_unlock(&dev->struct_mutex);
>+	mutex_unlock(&dev_priv->dev->struct_mutex);
> 
> pre_mutex_err:
> 	/* intel_gpu_busy should also get a ref, so it will free when the device
>@@ -1482,7 +1500,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
> 	exec2.flags = I915_EXEC_RENDER;
> 	i915_execbuffer2_set_context_id(exec2, 0);
> 
>-	ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
>+	ret = i915_gem_do_execbuffer(to_i915(dev), data, file, &exec2, exec2_list);
> 	if (!ret) {
> 		struct drm_i915_gem_exec_object __user *user_exec_list =
> 			to_user_ptr(args->buffers_ptr);
>@@ -1546,7 +1564,7 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
> 		return -EFAULT;
> 	}
> 
>-	ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
>+	ret = i915_gem_do_execbuffer(to_i915(dev), data, file, args, exec2_list);
> 	if (!ret) {
> 		/* Copy the new buffer offsets back to the user's exec list. */
> 		struct drm_i915_gem_exec_object2 __user *user_exec_list =
>diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
>index e42925f..91f6962 100644
>--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
>+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
>@@ -33,17 +33,6 @@
> static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv);
> static void chv_setup_private_ppat(struct drm_i915_private *dev_priv);
> 
>-bool intel_enable_ppgtt(struct drm_device *dev, bool full)
>-{
>-	if (i915.enable_ppgtt == 0)
>-		return false;
>-
>-	if (i915.enable_ppgtt == 1 && full)
>-		return false;
>-
>-	return true;
>-}
>-
> static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
> {
> 	if (enable_ppgtt == 0 || !HAS_ALIASING_PPGTT(dev))
>@@ -70,7 +59,7 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
> 		return 0;
> 	}
> 
>-	return HAS_ALIASING_PPGTT(dev) ? 1 : 0;
>+	return HAS_PPGTT(dev) ? 2 : HAS_ALIASING_PPGTT(dev) ? 1 : 0;
> }
> 
> 
>@@ -78,7 +67,6 @@ static void ppgtt_bind_vma(struct i915_vma *vma,
> 			   enum i915_cache_level cache_level,
> 			   u32 flags);
> static void ppgtt_unbind_vma(struct i915_vma *vma);
>-static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt);
> 
> static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
> 					     enum i915_cache_level level,
>@@ -215,38 +203,28 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
> }
> 
> /* Broadwell Page Directory Pointer Descriptors */
>-static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,
>-			   uint64_t val, bool synchronous)
>+static int gen8_write_pdp(struct i915_gem_request *rq, unsigned entry, uint64_t val)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>-	int ret;
>+	struct intel_ringbuffer *ring;
> 
> 	BUG_ON(entry >= 4);
> 
>-	if (synchronous) {
>-		I915_WRITE(GEN8_RING_PDP_UDW(ring, entry), val >> 32);
>-		I915_WRITE(GEN8_RING_PDP_LDW(ring, entry), (u32)val);
>-		return 0;
>-	}
>-
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 5);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
>-	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
>-	intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
>+	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
>+	intel_ring_emit(ring, GEN8_RING_PDP_UDW(rq->engine, entry));
> 	intel_ring_emit(ring, (u32)(val >> 32));
>-	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
>-	intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
>+	intel_ring_emit(ring, GEN8_RING_PDP_LDW(rq->engine, entry));
> 	intel_ring_emit(ring, (u32)(val));
> 	intel_ring_advance(ring);
> 
> 	return 0;
> }
> 
>-static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
>-			  struct intel_engine_cs *ring,
>-			  bool synchronous)
>+static int gen8_mm_switch(struct i915_gem_request *rq,
>+			  struct i915_hw_ppgtt *ppgtt)
> {
> 	int i, ret;
> 
>@@ -255,7 +233,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
> 
> 	for (i = used_pd - 1; i >= 0; i--) {
> 		dma_addr_t addr = ppgtt->pd_dma_addr[i];
>-		ret = gen8_write_pdp(ring, i, addr, synchronous);
>+		ret = gen8_write_pdp(rq, i, addr);
> 		if (ret)
> 			return ret;
> 	}
>@@ -403,9 +381,6 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
> 	struct i915_hw_ppgtt *ppgtt =
> 		container_of(vm, struct i915_hw_ppgtt, base);
> 
>-	list_del(&vm->global_link);
>-	drm_mm_takedown(&vm->mm);
>-
> 	gen8_ppgtt_unmap_pages(ppgtt);
> 	gen8_ppgtt_free(ppgtt);
> }
>@@ -615,7 +590,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
> 		kunmap_atomic(pd_vaddr);
> 	}
> 
>-	ppgtt->enable = gen8_ppgtt_enable;
> 	ppgtt->switch_mm = gen8_mm_switch;
> 	ppgtt->base.clear_range = gen8_ppgtt_clear_range;
> 	ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
>@@ -723,154 +697,53 @@ static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
> 	return (ppgtt->pd_offset / 64) << 16;
> }
> 
>-static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
>-			 struct intel_engine_cs *ring,
>-			 bool synchronous)
>+static int gen7_mm_switch(struct i915_gem_request *rq,
>+			  struct i915_hw_ppgtt *ppgtt)
> {
>-	struct drm_device *dev = ppgtt->base.dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct intel_ringbuffer *ring;
> 	int ret;
> 
>-	/* If we're in reset, we can assume the GPU is sufficiently idle to
>-	 * manually frob these bits. Ideally we could use the ring functions,
>-	 * except our error handling makes it quite difficult (can't use
>-	 * intel_ring_begin, ring->flush, or intel_ring_advance)
>-	 *
>-	 * FIXME: We should try not to special case reset
>-	 */
>-	if (synchronous ||
>-	    i915_reset_in_progress(&dev_priv->gpu_error)) {
>-		WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt);
>-		I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
>-		I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
>-		POSTING_READ(RING_PP_DIR_BASE(ring));
>-		return 0;
>-	}
>-
>-	/* NB: TLBs must be flushed and invalidated before a switch */
>-	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
>+	ret = i915_request_emit_flush(rq, I915_COMMAND_BARRIER);
> 	if (ret)
> 		return ret;
> 
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 5);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
>-	intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
>+	intel_ring_emit(ring, RING_PP_DIR_DCLV(rq->engine));
> 	intel_ring_emit(ring, PP_DIR_DCLV_2G);
>-	intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
>+	intel_ring_emit(ring, RING_PP_DIR_BASE(rq->engine));
> 	intel_ring_emit(ring, get_pd_offset(ppgtt));
>-	intel_ring_emit(ring, MI_NOOP);
> 	intel_ring_advance(ring);
> 
>-	return 0;
>-}
>-
>-static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
>-			  struct intel_engine_cs *ring,
>-			  bool synchronous)
>-{
>-	struct drm_device *dev = ppgtt->base.dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int ret;
>-
>-	/* If we're in reset, we can assume the GPU is sufficiently idle to
>-	 * manually frob these bits. Ideally we could use the ring functions,
>-	 * except our error handling makes it quite difficult (can't use
>-	 * intel_ring_begin, ring->flush, or intel_ring_advance)
>-	 *
>-	 * FIXME: We should try not to special case reset
>-	 */
>-	if (synchronous ||
>-	    i915_reset_in_progress(&dev_priv->gpu_error)) {
>-		WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt);
>-		I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
>-		I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
>-		POSTING_READ(RING_PP_DIR_BASE(ring));
>-		return 0;
>-	}
>-
>-	/* NB: TLBs must be flushed and invalidated before a switch */
>-	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
>-	if (ret)
>-		return ret;
>-
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret)
>-		return ret;
>-
>-	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
>-	intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
>-	intel_ring_emit(ring, PP_DIR_DCLV_2G);
>-	intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
>-	intel_ring_emit(ring, get_pd_offset(ppgtt));
>-	intel_ring_emit(ring, MI_NOOP);
>-	intel_ring_advance(ring);
>-
>-	/* XXX: RCS is the only one to auto invalidate the TLBs? */
>-	if (ring->id != RCS) {
>-		ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
>-		if (ret)
>-			return ret;
>-	}
>+	rq->pending_flush |= I915_INVALIDATE_CACHES;
> 
> 	return 0;
> }
> 
>-static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
>-			  struct intel_engine_cs *ring,
>-			  bool synchronous)
>+static int gen6_mm_switch(struct i915_gem_request *rq,
>+			  struct i915_hw_ppgtt *ppgtt)
> {
>-	struct drm_device *dev = ppgtt->base.dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-
>-	if (!synchronous)
>-		return 0;
>-
>-	I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
>-	I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
>-
>-	POSTING_READ(RING_PP_DIR_DCLV(ring));
>-
>-	return 0;
>+	return -ENODEV;
> }
> 
>-static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
>+static void gen8_ppgtt_enable(struct drm_device *dev)
> {
>-	struct drm_device *dev = ppgtt->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>-	int j, ret;
>+	struct intel_engine_cs *engine;
>+	int j;
> 
>-	for_each_ring(ring, dev_priv, j) {
>-		I915_WRITE(RING_MODE_GEN7(ring),
>+	for_each_engine(engine, dev_priv, j)
>+		I915_WRITE(RING_MODE_GEN7(engine),
> 			   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
>-
>-		/* We promise to do a switch later with FULL PPGTT. If this is
>-		 * aliasing, this is the one and only switch we'll do */
>-		if (USES_FULL_PPGTT(dev))
>-			continue;
>-
>-		ret = ppgtt->switch_mm(ppgtt, ring, true);
>-		if (ret)
>-			goto err_out;
>-	}
>-
>-	return 0;
>-
>-err_out:
>-	for_each_ring(ring, dev_priv, j)
>-		I915_WRITE(RING_MODE_GEN7(ring),
>-			   _MASKED_BIT_DISABLE(GFX_PPGTT_ENABLE));
>-	return ret;
> }
> 
>-static int gen7_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
>+static void gen7_ppgtt_enable(struct drm_device *dev)
> {
>-	struct drm_device *dev = ppgtt->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	uint32_t ecochk, ecobits;
> 	int i;
> 
>@@ -886,31 +759,25 @@ static int gen7_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
> 	}
> 	I915_WRITE(GAM_ECOCHK, ecochk);
> 
>-	for_each_ring(ring, dev_priv, i) {
>-		int ret;
>+	for_each_engine(engine, dev_priv, i) {
> 		/* GFX_MODE is per-ring on gen7+ */
>-		I915_WRITE(RING_MODE_GEN7(ring),
>+		I915_WRITE(RING_MODE_GEN7(engine),
> 			   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
> 
>-		/* We promise to do a switch later with FULL PPGTT. If this is
>-		 * aliasing, this is the one and only switch we'll do */
>-		if (USES_FULL_PPGTT(dev))
>-			continue;
>+		if (dev_priv->mm.aliasing_ppgtt) {
>+			I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
>+			I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(dev_priv->mm.aliasing_ppgtt));
>+		}
> 
>-		ret = ppgtt->switch_mm(ppgtt, ring, true);
>-		if (ret)
>-			return ret;
> 	}
>-
>-	return 0;
>+	POSTING_READ(RING_PP_DIR_DCLV(RCS_ENGINE(dev_priv)));
> }
> 
>-static int gen6_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
>+static void gen6_ppgtt_enable(struct drm_device *dev)
> {
>-	struct drm_device *dev = ppgtt->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
> 	uint32_t ecochk, gab_ctl, ecobits;
>+	struct intel_engine_cs *engine;
> 	int i;
> 
> 	ecobits = I915_READ(GAC_ECO_BITS);
>@@ -925,13 +792,14 @@ static int gen6_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
> 
> 	I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
> 
>-	for_each_ring(ring, dev_priv, i) {
>-		int ret = ppgtt->switch_mm(ppgtt, ring, true);
>-		if (ret)
>-			return ret;
>-	}
>+	for_each_engine(engine, dev_priv, i) {
>+		if (dev_priv->mm.aliasing_ppgtt) {
>+			I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
>+			I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(dev_priv->mm.aliasing_ppgtt));
>+		}
> 
>-	return 0;
>+	}
>+	POSTING_READ(RING_PP_DIR_DCLV(RCS_ENGINE(dev_priv)));
> }
> 
> /* PPGTT support for Sandybdrige/Gen6 and later */
>@@ -1029,8 +897,6 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
> 	struct i915_hw_ppgtt *ppgtt =
> 		container_of(vm, struct i915_hw_ppgtt, base);
> 
>-	list_del(&vm->global_link);
>-	drm_mm_takedown(&ppgtt->base.mm);
> 	drm_mm_remove_node(&ppgtt->node);
> 
> 	gen6_ppgtt_unmap_pages(ppgtt);
>@@ -1151,13 +1017,8 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
> 
> 	ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
> 	if (IS_GEN6(dev)) {
>-		ppgtt->enable = gen6_ppgtt_enable;
> 		ppgtt->switch_mm = gen6_mm_switch;
>-	} else if (IS_HASWELL(dev)) {
>-		ppgtt->enable = gen7_ppgtt_enable;
>-		ppgtt->switch_mm = hsw_mm_switch;
> 	} else if (IS_GEN7(dev)) {
>-		ppgtt->enable = gen7_ppgtt_enable;
> 		ppgtt->switch_mm = gen7_mm_switch;
> 	} else
> 		BUG();
>@@ -1188,40 +1049,96 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
> 			 ppgtt->node.size >> 20,
> 			 ppgtt->node.start / PAGE_SIZE);
> 
>+	gen6_write_pdes(ppgtt);
>+	DRM_DEBUG("Adding PPGTT at offset %x\n",
>+		  ppgtt->pd_offset << 10);
>+
> 	return 0;
> }
> 
>-int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
>+static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int ret = 0;
> 
> 	ppgtt->base.dev = dev;
> 	ppgtt->base.scratch = dev_priv->gtt.base.scratch;
> 
> 	if (INTEL_INFO(dev)->gen < 8)
>-		ret = gen6_ppgtt_init(ppgtt);
>+		return gen6_ppgtt_init(ppgtt);
> 	else if (IS_GEN8(dev))
>-		ret = gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
>+		return gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
> 	else
> 		BUG();
>+}
>+int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
>+{
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	int ret = 0;
> 
>-	if (!ret) {
>-		struct drm_i915_private *dev_priv = dev->dev_private;
>+	ret = __hw_ppgtt_init(dev, ppgtt);
>+	if (ret == 0) {
> 		kref_init(&ppgtt->ref);
> 		drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
> 			    ppgtt->base.total);
> 		i915_init_vm(dev_priv, &ppgtt->base);
>-		if (INTEL_INFO(dev)->gen < 8) {
>-			gen6_write_pdes(ppgtt);
>-			DRM_DEBUG("Adding PPGTT at offset %x\n",
>-				  ppgtt->pd_offset << 10);
>-		}
> 	}
> 
> 	return ret;
> }
> 
>+int i915_ppgtt_init_hw(struct drm_device *dev)
>+{
>+	if (!USES_PPGTT(dev))
>+		return 0;
>+
>+	if (IS_GEN6(dev))
>+		gen6_ppgtt_enable(dev);
>+	else if (IS_GEN7(dev))
>+		gen7_ppgtt_enable(dev);
>+	else if (INTEL_INFO(dev)->gen >= 8)
>+		gen8_ppgtt_enable(dev);
>+	else
>+		WARN_ON(1);
>+
>+	return 0;
>+}
>+struct i915_hw_ppgtt *
>+i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
>+{
>+	struct i915_hw_ppgtt *ppgtt;
>+	int ret;
>+
>+	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
>+	if (!ppgtt)
>+		return ERR_PTR(-ENOMEM);
>+
>+	ret = i915_ppgtt_init(dev, ppgtt);
>+	if (ret) {
>+		kfree(ppgtt);
>+		return ERR_PTR(ret);
>+	}
>+
>+	ppgtt->file_priv = fpriv;
>+
>+	return ppgtt;
>+}
>+
>+void  i915_ppgtt_release(struct kref *kref)
>+{
>+	struct i915_hw_ppgtt *ppgtt =
>+		container_of(kref, struct i915_hw_ppgtt, ref);
>+
>+	/* vmas should already be unbound */
>+	WARN_ON(!list_empty(&ppgtt->base.active_list));
>+	WARN_ON(!list_empty(&ppgtt->base.inactive_list));
>+
>+	list_del(&ppgtt->base.global_link);
>+	drm_mm_takedown(&ppgtt->base.mm);
>+
>+	ppgtt->base.cleanup(&ppgtt->base);
>+	kfree(ppgtt);
>+}
>+
> static void
> ppgtt_bind_vma(struct i915_vma *vma,
> 	       enum i915_cache_level cache_level,
>@@ -1233,6 +1150,7 @@ ppgtt_bind_vma(struct i915_vma *vma,
> 
> 	vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
> 				cache_level, flags);
>+	vma->vm->dirty = true;
> }
> 
> static void ppgtt_unbind_vma(struct i915_vma *vma)
>@@ -1284,15 +1202,15 @@ static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
> void i915_check_and_clear_faults(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	int i;
> 
> 	if (INTEL_INFO(dev)->gen < 6)
> 		return;
> 
>-	for_each_ring(ring, dev_priv, i) {
>+	for_each_engine(engine, dev_priv, i) {
> 		u32 fault_reg;
>-		fault_reg = I915_READ(RING_FAULT_REG(ring));
>+		fault_reg = I915_READ(RING_FAULT_REG(engine));
> 		if (fault_reg & RING_FAULT_VALID) {
> 			DRM_DEBUG_DRIVER("Unexpected fault\n"
> 					 "\tAddr: 0x%08lx\\n"
>@@ -1303,21 +1221,11 @@ void i915_check_and_clear_faults(struct drm_device *dev)
> 					 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
> 					 RING_FAULT_SRCID(fault_reg),
> 					 RING_FAULT_FAULT_TYPE(fault_reg));
>-			I915_WRITE(RING_FAULT_REG(ring),
>+			I915_WRITE(RING_FAULT_REG(engine),
> 				   fault_reg & ~RING_FAULT_VALID);
> 		}
> 	}
>-	POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
>-}
>-
>-static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
>-{
>-	if (INTEL_INFO(dev_priv->dev)->gen < 6) {
>-		intel_gtt_chipset_flush();
>-	} else {
>-		I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
>-		POSTING_READ(GFX_FLSH_CNTL_GEN6);
>-	}
>+	POSTING_READ(RING_FAULT_REG(RCS_ENGINE(dev_priv)));
> }
> 
> void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
>@@ -1336,8 +1244,6 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
> 				       dev_priv->gtt.base.start,
> 				       dev_priv->gtt.base.total,
> 				       true);
>-
>-	i915_ggtt_flush(dev_priv);
> }
> 
> void i915_gem_restore_gtt_mappings(struct drm_device *dev)
>@@ -1390,7 +1296,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
> 		gen6_write_pdes(container_of(vm, struct i915_hw_ppgtt, base));
> 	}
> 
>-	i915_ggtt_flush(dev_priv);
>+	i915_gem_chipset_flush(dev);
> }
> 
> int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
>@@ -1562,6 +1468,7 @@ static void i915_ggtt_bind_vma(struct i915_vma *vma,
> 
> 	BUG_ON(!i915_is_ggtt(vma->vm));
> 	intel_gtt_insert_sg_entries(vma->obj->pages, entry, flags);
>+	vma->vm->dirty = true;
> 	vma->obj->has_global_gtt_mapping = 1;
> }
> 
>@@ -1572,6 +1479,7 @@ static void i915_ggtt_clear_range(struct i915_address_space *vm,
> {
> 	unsigned first_entry = start >> PAGE_SHIFT;
> 	unsigned num_entries = length >> PAGE_SHIFT;
>+
> 	intel_gtt_clear_range(first_entry, num_entries);
> }
> 
>@@ -1614,6 +1522,7 @@ static void ggtt_bind_vma(struct i915_vma *vma,
> 			vma->vm->insert_entries(vma->vm, obj->pages,
> 						vma->node.start,
> 						cache_level, flags);
>+			vma->vm->dirty = true;
> 			obj->has_global_gtt_mapping = 1;
> 		}
> 	}
>@@ -1626,6 +1535,7 @@ static void ggtt_bind_vma(struct i915_vma *vma,
> 					    vma->obj->pages,
> 					    vma->node.start,
> 					    cache_level, flags);
>+		vma->vm->dirty = true;
> 		vma->obj->has_aliasing_ppgtt_mapping = 1;
> 	}
> }
>@@ -1687,10 +1597,10 @@ static void i915_gtt_color_adjust(struct drm_mm_node *node,
> 	}
> }
> 
>-void i915_gem_setup_global_gtt(struct drm_device *dev,
>-			       unsigned long start,
>-			       unsigned long mappable_end,
>-			       unsigned long end)
>+int i915_gem_setup_global_gtt(struct drm_device *dev,
>+			      unsigned long start,
>+			      unsigned long mappable_end,
>+			      unsigned long end)
> {
> 	/* Let GEM Manage all of the aperture.
> 	 *
>@@ -1706,6 +1616,7 @@ void i915_gem_setup_global_gtt(struct drm_device *dev,
> 	struct drm_mm_node *entry;
> 	struct drm_i915_gem_object *obj;
> 	unsigned long hole_start, hole_end;
>+	int ret;
> 
> 	BUG_ON(mappable_end > end);
> 
>@@ -1717,14 +1628,16 @@ void i915_gem_setup_global_gtt(struct drm_device *dev,
> 	/* Mark any preallocated objects as occupied */
> 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
> 		struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
>-		int ret;
>+
> 		DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
> 			      i915_gem_obj_ggtt_offset(obj), obj->base.size);
> 
> 		WARN_ON(i915_gem_obj_ggtt_bound(obj));
> 		ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
>-		if (ret)
>-			DRM_DEBUG_KMS("Reservation failed\n");
>+		if (ret) {
>+			DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
>+			return ret;
>+		}
> 		obj->has_global_gtt_mapping = 1;
> 	}
> 
>@@ -1741,6 +1654,22 @@ void i915_gem_setup_global_gtt(struct drm_device *dev,
> 
> 	/* And finally clear the reserved guard page */
> 	ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
>+
>+	if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
>+		struct i915_hw_ppgtt *ppgtt;
>+
>+		ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
>+		if (!ppgtt)
>+			return -ENOMEM;
>+
>+		ret = __hw_ppgtt_init(dev, ppgtt);
>+		if (ret != 0)
>+			return ret;
>+
>+		dev_priv->mm.aliasing_ppgtt = ppgtt;
>+	}
>+
>+	return 0;
> }
> 
> void i915_gem_init_global_gtt(struct drm_device *dev)
>@@ -1754,6 +1683,25 @@ void i915_gem_init_global_gtt(struct drm_device *dev)
> 	i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
> }
> 
>+void i915_global_gtt_cleanup(struct drm_device *dev)
>+{
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct i915_address_space *vm = &dev_priv->gtt.base;
>+
>+	if (dev_priv->mm.aliasing_ppgtt) {
>+		struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
>+
>+		ppgtt->base.cleanup(&ppgtt->base);
>+	}
>+
>+	if (drm_mm_initialized(&vm->mm)) {
>+		drm_mm_takedown(&vm->mm);
>+		list_del(&vm->global_link);
>+	}
>+
>+	vm->cleanup(vm);
>+}
>+
> static int setup_scratch_page(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>@@ -1763,7 +1711,6 @@ static int setup_scratch_page(struct drm_device *dev)
> 	page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
> 	if (page == NULL)
> 		return -ENOMEM;
>-	get_page(page);
> 	set_pages_uc(page, 1);
> 
> #ifdef CONFIG_INTEL_IOMMU
>@@ -1788,7 +1735,6 @@ static void teardown_scratch_page(struct drm_device *dev)
> 	set_pages_wb(page, 1);
> 	pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
> 		       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
>-	put_page(page);
> 	__free_page(page);
> }
> 
>@@ -2022,10 +1968,6 @@ static void gen6_gmch_remove(struct i915_address_space *vm)
> 
> 	struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
> 
>-	if (drm_mm_initialized(&vm->mm)) {
>-		drm_mm_takedown(&vm->mm);
>-		list_del(&vm->global_link);
>-	}
> 	iounmap(gtt->gsm);
> 	teardown_scratch_page(vm->dev);
> }
>@@ -2058,10 +2000,6 @@ static int i915_gmch_probe(struct drm_device *dev,
> 
> static void i915_gmch_remove(struct i915_address_space *vm)
> {
>-	if (drm_mm_initialized(&vm->mm)) {
>-		drm_mm_takedown(&vm->mm);
>-		list_del(&vm->global_link);
>-	}
> 	intel_gmch_remove();
> }
> 
>@@ -2160,8 +2098,10 @@ static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
> 	/* Keep GGTT vmas first to make debug easier */
> 	if (i915_is_ggtt(vm))
> 		list_add(&vma->vma_link, &obj->vma_list);
>-	else
>+	else {
> 		list_add_tail(&vma->vma_link, &obj->vma_list);
>+		i915_ppgtt_get(i915_vm_to_ppgtt(vm));
>+	}
> 
> 	return vma;
> }
>diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
>index 8d6f7c1..208b96b 100644
>--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
>+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
>@@ -34,6 +34,8 @@
> #ifndef __I915_GEM_GTT_H__
> #define __I915_GEM_GTT_H__
> 
>+struct drm_i915_file_private;
>+
> typedef uint32_t gen6_gtt_pte_t;
> typedef uint64_t gen8_gtt_pte_t;
> typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;
>@@ -167,6 +169,8 @@ struct i915_address_space {
> 	unsigned long start;		/* Start offset always 0 for dri2 */
> 	size_t total;		/* size addr space maps (ex. 2GB for ggtt) */
> 
>+	bool dirty;
>+
> 	struct {
> 		dma_addr_t addr;
> 		struct page *page;
>@@ -258,22 +262,36 @@ struct i915_hw_ppgtt {
> 		dma_addr_t *gen8_pt_dma_addr[4];
> 	};
> 
>-	struct intel_context *ctx;
>+	struct drm_i915_file_private *file_priv;
> 
> 	int (*enable)(struct i915_hw_ppgtt *ppgtt);
>-	int (*switch_mm)(struct i915_hw_ppgtt *ppgtt,
>-			 struct intel_engine_cs *ring,
>-			 bool synchronous);
>+	int (*switch_mm)(struct i915_gem_request *rq,
>+			 struct i915_hw_ppgtt *ppgtt);
> 	void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m);
> };
> 
> int i915_gem_gtt_init(struct drm_device *dev);
> void i915_gem_init_global_gtt(struct drm_device *dev);
>-void i915_gem_setup_global_gtt(struct drm_device *dev, unsigned long start,
>-			       unsigned long mappable_end, unsigned long end);
>-
>-bool intel_enable_ppgtt(struct drm_device *dev, bool full);
>-int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt);
>+int i915_gem_setup_global_gtt(struct drm_device *dev, unsigned long start,
>+			      unsigned long mappable_end, unsigned long end);
>+void i915_global_gtt_cleanup(struct drm_device *dev);
>+
>+
>+int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt);
>+int i915_ppgtt_init_hw(struct drm_device *dev);
>+void i915_ppgtt_release(struct kref *kref);
>+struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_device *dev,
>+					struct drm_i915_file_private *fpriv);
>+static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt)
>+{
>+	if (ppgtt)
>+		kref_get(&ppgtt->ref);
>+}
>+static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt)
>+{
>+	if (ppgtt)
>+		kref_put(&ppgtt->ref, i915_ppgtt_release);
>+}
> 
> void i915_check_and_clear_faults(struct drm_device *dev);
> void i915_gem_suspend_gtt_mappings(struct drm_device *dev);
>diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
>index e60be3f..fffd26d 100644
>--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
>+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
>@@ -36,7 +36,7 @@ struct render_state {
> };
> 
> static const struct intel_renderstate_rodata *
>-render_state_get_rodata(struct drm_device *dev, const int gen)
>+render_state_get_rodata(const int gen)
> {
> 	switch (gen) {
> 	case 6:
>@@ -50,19 +50,19 @@ render_state_get_rodata(struct drm_device *dev, const int gen)
> 	return NULL;
> }
> 
>-static int render_state_init(struct render_state *so, struct drm_device *dev)
>+static int render_state_init(struct render_state *so, struct i915_gem_request *rq)
> {
> 	int ret;
> 
>-	so->gen = INTEL_INFO(dev)->gen;
>-	so->rodata = render_state_get_rodata(dev, so->gen);
>+	so->gen = INTEL_INFO(rq->i915)->gen;
>+	so->rodata = render_state_get_rodata(so->gen);
> 	if (so->rodata == NULL)
> 		return 0;
> 
> 	if (so->rodata->batch_items * 4 > 4096)
> 		return -EINVAL;
> 
>-	so->obj = i915_gem_alloc_object(dev, 4096);
>+	so->obj = i915_gem_alloc_object(rq->i915->dev, 4096);
> 	if (so->obj == NULL)
> 		return -ENOMEM;
> 
>@@ -115,10 +115,6 @@ static int render_state_setup(struct render_state *so)
> 	}
> 	kunmap(page);
> 
>-	ret = i915_gem_object_set_to_gtt_domain(so->obj, false);
>-	if (ret)
>-		return ret;
>-
> 	if (rodata->reloc[reloc_index] != -1) {
> 		DRM_ERROR("only %d relocs resolved\n", reloc_index);
> 		return -EINVAL;
>@@ -133,15 +129,15 @@ static void render_state_fini(struct render_state *so)
> 	drm_gem_object_unreference(&so->obj->base);
> }
> 
>-int i915_gem_render_state_init(struct intel_engine_cs *ring)
>+int i915_gem_render_state_init(struct i915_gem_request *rq)
> {
> 	struct render_state so;
> 	int ret;
> 
>-	if (WARN_ON(ring->id != RCS))
>+	if (WARN_ON(rq->engine->id != RCS))
> 		return -ENOENT;
> 
>-	ret = render_state_init(&so, ring->dev);
>+	ret = render_state_init(&so, rq);
> 	if (ret)
> 		return ret;
> 
>@@ -152,16 +148,19 @@ int i915_gem_render_state_init(struct intel_engine_cs *ring)
> 	if (ret)
> 		goto out;
> 
>-	ret = ring->dispatch_execbuffer(ring,
>-					so.ggtt_offset,
>-					so.rodata->batch_items * 4,
>-					I915_DISPATCH_SECURE);
>+	if (i915_gem_clflush_object(so.obj, false))
>+		i915_gem_chipset_flush(rq->i915->dev);
>+
>+	ret = i915_request_emit_batchbuffer(rq, NULL,
>+					    so.ggtt_offset,
>+					    so.rodata->batch_items * 4,
>+					    I915_DISPATCH_SECURE);
> 	if (ret)
> 		goto out;
> 
>-	i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), ring);
>+	so.obj->base.pending_read_domains = I915_GEM_DOMAIN_COMMAND;
>+	ret = i915_request_add_vma(rq, i915_gem_obj_to_ggtt(so.obj), 0);
> 
>-	ret = __i915_add_request(ring, NULL, so.obj, NULL);
> 	/* __i915_add_request moves object to inactive if it fails */
> out:
> 	render_state_fini(&so);
>diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
>new file mode 100644
>index 0000000..c85b526
>--- /dev/null
>+++ b/drivers/gpu/drm/i915/i915_gem_request.c
>@@ -0,0 +1,666 @@
>+/*
>+ * Copyright Â© 2014 Intel Corporation
>+ *
>+ * Permission is hereby granted, free of charge, to any person obtaining a
>+ * copy of this software and associated documentation files (the "Software"),
>+ * to deal in the Software without restriction, including without limitation
>+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
>+ * and/or sell copies of the Software, and to permit persons to whom the
>+ * Software is furnished to do so, subject to the following conditions:
>+ *
>+ * The above copyright notice and this permission notice (including the next
>+ * paragraph) shall be included in all copies or substantial portions of the
>+ * Software.
>+ *
>+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
>+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
>+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
>+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
>+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
>+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
>+ * IN THE SOFTWARE.
>+ *
>+ */
>+
>+#include <drm/drmP.h>
>+#include "i915_drv.h"
>+#include <drm/i915_drm.h>
>+#include "i915_trace.h"
>+#include "intel_drv.h"
>+
>+struct i915_gem_request__vma {
>+	struct list_head link;
>+	struct i915_vma *vma;
>+	struct drm_i915_gem_object *obj;
>+	u32 write, fence;
>+};
>+
>+static bool check_reset(struct i915_gem_request *rq)
>+{
>+	unsigned reset = atomic_read(&rq->i915->gpu_error.reset_counter);
>+	return likely(reset == rq->reset_counter);
>+}
>+
>+int
>+i915_request_add_vma(struct i915_gem_request *rq,
>+		     struct i915_vma *vma,
>+		     unsigned fenced)
>+{
>+	struct drm_i915_gem_object *obj = vma->obj;
>+	u32 old_read = obj->base.read_domains;
>+	u32 old_write = obj->base.write_domain;
>+	struct i915_gem_request__vma *ref;
>+
>+	lockdep_assert_held(&rq->i915->dev->struct_mutex);
>+	BUG_ON(!rq->outstanding);
>+
>+	obj->base.write_domain = obj->base.pending_write_domain;
>+	if (obj->base.write_domain == 0)
>+		obj->base.pending_read_domains |= obj->base.read_domains;
>+	obj->base.read_domains = obj->base.pending_read_domains;
>+
>+	obj->base.pending_read_domains = 0;
>+	obj->base.pending_write_domain = 0;
>+
>+	trace_i915_gem_object_change_domain(obj, old_read, old_write);
>+	if (obj->base.read_domains == 0)
>+		return 0;
>+
>+	ref = kmalloc(sizeof(*ref), GFP_KERNEL);
>+	if (ref == NULL)
>+		return -ENOMEM;
>+
>+	list_add(&ref->link, &rq->vmas);
>+	ref->vma = vma;
>+	ref->obj = obj;
>+	drm_gem_object_reference(&obj->base);
>+	ref->write = obj->base.write_domain & I915_GEM_GPU_DOMAINS;
>+	ref->fence = fenced;
>+
>+	if (ref->write) {
>+		rq->pending_flush |= I915_FLUSH_CACHES;
>+		intel_fb_obj_invalidate(obj, rq);
>+	}
>+
>+	/* update for the implicit flush after the rq */
>+	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
>+	return 0;
>+}
>+
>+static void vma_free(struct i915_gem_request__vma *ref)
>+{
>+	drm_gem_object_unreference(&ref->obj->base);
>+	list_del(&ref->link);
>+	kfree(ref);
>+}
>+
>+int
>+i915_request_emit_flush(struct i915_gem_request *rq,
>+			unsigned flags)
>+{
>+	struct intel_engine_cs *engine = rq->engine;
>+	int ret;
>+
>+	lockdep_assert_held(&rq->i915->dev->struct_mutex);
>+	BUG_ON(!rq->outstanding);
>+
>+	if ((flags & rq->pending_flush) == 0)
>+		return 0;
>+
>+	trace_i915_gem_request_emit_flush(rq);
>+	ret = engine->emit_flush(rq, rq->pending_flush);
>+	if (ret)
>+		return ret;
>+
>+	rq->pending_flush = 0;
>+	return 0;
>+}
>+
>+int
>+__i915_request_emit_breadcrumb(struct i915_gem_request *rq, int id)
>+{
>+	struct intel_engine_cs *engine = rq->engine;
>+	u32 seqno;
>+	int ret;
>+
>+	lockdep_assert_held(&rq->i915->dev->struct_mutex);
>+
>+	if (rq->breadcrumb[id])
>+		return 0;
>+
>+	if (rq->outstanding) {
>+		ret = i915_request_emit_flush(rq, I915_COMMAND_BARRIER);
>+		if (ret)
>+			return ret;
>+
>+		trace_i915_gem_request_emit_breadcrumb(rq);
>+		if (id == engine->id)
>+			ret = engine->emit_breadcrumb(rq);
>+		else
>+			ret = engine->semaphore.signal(rq, id);
>+		if (ret)
>+			return ret;
>+
>+		seqno = rq->seqno;
>+	} else if (engine->breadcrumb[id] == 0 ||
>+		   __i915_seqno_passed(rq->seqno, engine->breadcrumb[id])) {
>+		struct i915_gem_request *tmp;
>+
>+		tmp = intel_engine_alloc_request(engine,
>+						 rq->ring->last_context);
>+		if (IS_ERR(tmp))
>+			return PTR_ERR(tmp);
>+
>+		/* Masquerade as a continuation of the earlier request */
>+		tmp->reset_counter = rq->reset_counter;
>+
>+		ret = __i915_request_emit_breadcrumb(tmp, id);
>+		if (ret == 0 && id != engine->id) {
>+			/* semaphores are unstable across a wrap */
>+			if (tmp->seqno < engine->breadcrumb[id])
>+				ret = i915_request_wait(tmp);
>+		}
>+		if (ret == 0)
>+			ret = i915_request_commit(tmp);
>+
>+		i915_request_put(tmp);
>+		if (ret)
>+			return ret;
>+
>+		seqno = tmp->seqno;
>+	} else
>+		seqno = engine->breadcrumb[id];
>+
>+	rq->breadcrumb[id] = seqno;
>+	return 0;
>+}
>+
>+int
>+i915_request_emit_batchbuffer(struct i915_gem_request *rq,
>+			      struct i915_vma *batch,
>+			      uint64_t start, uint32_t len,
>+			      unsigned flags)
>+{
>+	struct intel_engine_cs *engine = rq->engine;
>+	int ret;
>+
>+	lockdep_assert_held(&rq->i915->dev->struct_mutex);
>+	BUG_ON(!rq->outstanding);
>+	BUG_ON(rq->breadcrumb[rq->engine->id]);
>+
>+	trace_i915_gem_request_emit_batch(rq);
>+	rq->batch = batch;
>+
>+	ret = engine->emit_batchbuffer(rq, start, len, flags);
>+	if (ret)
>+		return ret;
>+
>+	/* We track the associated batch vma for debugging and error capture.
>+	 * Whilst this request exists, the batch obj will be on the active_list,
>+	 * and so will hold the active reference. Only when this request is
>+	 * retired will the the batch be moved onto the inactive_list and lose
>+	 * its active reference. Hence we do not need to explicitly hold
>+	 * another reference here.
>+	 */
>+	rq->pending_flush |= I915_COMMAND_BARRIER;
>+	return 0;
>+}
>+
>+/* Track the batches submitted by clients for throttling */
>+static void
>+add_to_client(struct i915_gem_request *rq)
>+{
>+	struct drm_i915_file_private *file_priv = rq->ctx->file_priv;
>+
>+	if (file_priv) {
>+		spin_lock(&file_priv->mm.lock);
>+		list_add_tail(&rq->client_list,
>+			      &file_priv->mm.request_list);
>+		rq->file_priv = file_priv;
>+		spin_unlock(&file_priv->mm.lock);
>+	}
>+}
>+
>+static void
>+remove_from_client(struct i915_gem_request *rq)
>+{
>+	struct drm_i915_file_private *file_priv = rq->file_priv;
>+
>+	if (!file_priv)
>+		return;
>+
>+	spin_lock(&file_priv->mm.lock);
>+	if (rq->file_priv) {
>+		list_del(&rq->client_list);
>+		rq->file_priv = NULL;
>+	}
>+	spin_unlock(&file_priv->mm.lock);
>+}
>+
>+/* Activity tracking on the object so that we can serialise CPU access to
>+ * the object's memory with the GPU.
>+ */
>+static void
>+add_to_obj(struct i915_gem_request *rq,
>+	   struct i915_gem_request__vma *ref)
>+{
>+	struct drm_i915_gem_object *obj = ref->obj;
>+	struct intel_engine_cs *engine = rq->engine;
>+
>+	/* Add a reference if we're newly entering the active list. */
>+	if (obj->last_read[engine->id].request == NULL && obj->active++ == 0)
>+		drm_gem_object_reference(&obj->base);
>+
>+	if (ref->write) {
>+		obj->dirty = 1;
>+		i915_request_put(obj->last_write.request);
>+		obj->last_write.request = i915_request_get(rq);
>+		list_move_tail(&obj->last_write.engine_list,
>+			       &engine->write_list);
>+
>+		if (obj->active > 1) {
>+			int i;
>+
>+			for (i = 0; i < I915_NUM_ENGINES; i++) {
>+				if (obj->last_read[i].request == NULL)
>+					continue;
>+
>+				list_del_init(&obj->last_read[i].engine_list);
>+				i915_request_put(obj->last_read[i].request);
>+				obj->last_read[i].request = NULL;
>+			}
>+
>+			obj->active = 1;
>+		}
>+	}
>+
>+	if (ref->fence & VMA_IS_FENCED) {
>+		i915_request_put(obj->last_fence.request);
>+		obj->last_fence.request = i915_request_get(rq);
>+		list_move_tail(&obj->last_fence.engine_list,
>+			       &engine->fence_list);
>+		if (ref->fence & VMA_HAS_FENCE)
>+			list_move_tail(&rq->i915->fence_regs[obj->fence_reg].lru_list,
>+					&rq->i915->mm.fence_list);
>+	}
>+
>+	i915_request_put(obj->last_read[engine->id].request);
>+	obj->last_read[engine->id].request = i915_request_get(rq);
>+	list_move_tail(&obj->last_read[engine->id].engine_list,
>+		       &engine->read_list);
>+
>+	list_move_tail(&ref->vma->mm_list, &ref->vma->vm->active_list);
>+}
>+
>+static bool leave_breadcrumb(struct i915_gem_request *rq)
>+{
>+	if (rq->breadcrumb[rq->engine->id])
>+		return false;
>+
>+	/* Auto-report HEAD every 4k to make sure that we can always wait on
>+	 * some available ring space in the future. This also caps the
>+	 * latency of future waits for missed breadcrumbs.
>+	 */
>+	if (__intel_ring_space(rq->ring->tail, rq->ring->breadcrumb_tail,
>+			       rq->ring->size, 0) >= PAGE_SIZE)
>+		return true;
>+
>+	return false;
>+}
>+
>+static bool simulated_hang(struct intel_engine_cs *engine)
>+{
>+	return test_and_clear_bit(engine->id,
>+				  &engine->i915->gpu_error.stop_rings);
>+}
>+
>+int i915_request_commit(struct i915_gem_request *rq)
>+{
>+	int ret, n;
>+
>+	lockdep_assert_held(&rq->i915->dev->struct_mutex);
>+
>+	if (!rq->outstanding)
>+		return 0;
>+
>+	if (rq->head == rq->ring->tail) {
>+		rq->completed = true;
>+		goto done;
>+	}
>+
>+	if (simulated_hang(rq->engine))
>+		i915_handle_error(rq->i915->dev, true, "Simulated hang");
>+
>+	if (!check_reset(rq))
>+		return rq->i915->mm.interruptible ? -EAGAIN : -EIO;
>+
>+	if (leave_breadcrumb(rq)) {
>+		ret = i915_request_emit_breadcrumb(rq);
>+		if (ret)
>+			return ret;
>+	}
>+
>+	/* TAIL must be aligned to a qword */
>+	if ((rq->ring->tail / sizeof (uint32_t)) & 1) {
>+		intel_ring_emit(rq->ring, MI_NOOP);
>+		intel_ring_advance(rq->ring);
>+	}
>+	rq->tail = rq->ring->tail;
>+	rq->emitted_jiffies = jiffies;
>+
>+	intel_runtime_pm_get(rq->i915);
>+
>+	trace_i915_gem_request_commit(rq);
>+	ret = rq->engine->add_request(rq);
>+	if (ret) {
>+		intel_runtime_pm_put(rq->i915);
>+		return ret;
>+	}
>+
>+	i915_request_get(rq);
>+
>+	rq->outstanding = false;
>+	if (rq->breadcrumb[rq->engine->id]) {
>+		list_add_tail(&rq->breadcrumb_link, &rq->ring->breadcrumbs);
>+		rq->ring->breadcrumb_tail = rq->tail;
>+	}
>+
>+	memcpy(rq->engine->semaphore.sync,
>+	       rq->semaphore,
>+	       sizeof(rq->semaphore));
>+	for (n = 0; n < ARRAY_SIZE(rq->breadcrumb); n++)
>+		if (rq->breadcrumb[n])
>+			rq->engine->breadcrumb[n] = rq->breadcrumb[n];
>+
>+	rq->ring->pending_flush = rq->pending_flush;
>+
>+	if (rq->batch) {
>+		add_to_client(rq);
>+		while (!list_empty(&rq->vmas)) {
>+			struct i915_gem_request__vma *ref =
>+				list_first_entry(&rq->vmas, typeof(*ref), link);
>+
>+			add_to_obj(rq, ref);
>+			vma_free(ref);
>+		}
>+		rq->batch->vm->dirty = false;
>+	}
>+
>+	i915_request_switch_context__commit(rq);
>+
>+	rq->engine->last_request = rq;
>+done:
>+	rq->ring->last_context = rq->ctx;
>+	return 0;
>+}
>+
>+static void fake_irq(unsigned long data)
>+{
>+	wake_up_process((struct task_struct *)data);
>+}
>+
>+static bool missed_irq(struct i915_gem_request *rq)
>+{
>+	return test_bit(rq->engine->id, &rq->i915->gpu_error.missed_irq_rings);
>+}
>+
>+static bool can_wait_boost(struct drm_i915_file_private *file_priv)
>+{
>+	if (file_priv == NULL)
>+		return true;
>+
>+	return !atomic_xchg(&file_priv->rps_wait_boost, true);
>+}
>+
>+bool __i915_request_complete__wa(struct i915_gem_request *rq)
>+{
>+	struct drm_i915_private *dev_priv = rq->i915;
>+	unsigned head, tail;
>+
>+	if (i915_request_complete(rq))
>+		return true;
>+
>+	/* With execlists, we rely on interrupts to track request completion */
>+	if (rq->engine->execlists_enabled)
>+		return false;
>+
>+	/* As we may not emit a breadcrumb with every request, we
>+	 * often have unflushed requests. In the event of an emergency,
>+	 * just assume that if the RING_HEAD has reached the tail, then
>+	 * the request is complete. However, note that the RING_HEAD
>+	 * advances before the instruction completes, so this is quite lax,
>+	 * and should only be used carefully.
>+	 *
>+	 * As we treat this as only an advisory completion, we forgo
>+	 * marking the request as actually complete.
>+	 */
>+	head = __intel_ring_space(I915_READ_HEAD(rq->engine) & HEAD_ADDR,
>+				  rq->ring->tail, rq->ring->size, 0);
>+	tail = __intel_ring_space(rq->tail,
>+				  rq->ring->tail, rq->ring->size, 0);
>+	return head >= tail;
>+}
>+
>+/**
>+ * __i915_request_wait - wait until execution of request has finished
>+ * @request: the request to wait upon
>+ * @interruptible: do an interruptible wait (normally yes)
>+ * @timeout_ns: in - how long to wait (NULL forever); out - how much time remaining
>+ *
>+ * Returns 0 if the request was completed within the alloted time. Else returns the
>+ * errno with remaining time filled in timeout argument.
>+ */
>+int __i915_request_wait(struct i915_gem_request *rq,
>+			bool interruptible,
>+			s64 *timeout_ns,
>+			struct drm_i915_file_private *file_priv)
>+{
>+	const bool irq_test_in_progress =
>+		ACCESS_ONCE(rq->i915->gpu_error.test_irq_rings) & intel_engine_flag(rq->engine);
>+	DEFINE_WAIT(wait);
>+	unsigned long timeout_expire;
>+	unsigned long before, now;
>+	int ret = 0;
>+
>+	WARN(!intel_irqs_enabled(rq->i915), "IRQs disabled");
>+
>+	if (i915_request_complete(rq))
>+		return 0;
>+
>+	timeout_expire = timeout_ns ? jiffies + nsecs_to_jiffies((u64)*timeout_ns) : 0;
>+
>+	if (INTEL_INFO(rq->i915)->gen >= 6 && rq->engine->id == RCS && can_wait_boost(file_priv)) {
>+		gen6_rps_boost(rq->i915);
>+		if (file_priv)
>+			mod_delayed_work(rq->i915->wq,
>+					 &file_priv->mm.idle_work,
>+					 msecs_to_jiffies(100));
>+	}
>+
>+	if (!irq_test_in_progress) {
>+		if (WARN_ON(!intel_irqs_enabled(rq->i915)))
>+			return -ENODEV;
>+
>+		rq->engine->irq_get(rq->engine);
>+	}
>+
>+	/* Record current time in case interrupted by signal, or wedged */
>+	trace_i915_gem_request_wait_begin(rq);
>+	before = jiffies;
>+	for (;;) {
>+		struct timer_list timer;
>+
>+		prepare_to_wait(&rq->engine->irq_queue, &wait,
>+				interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
>+
>+		if (!check_reset(rq))
>+			break;
>+
>+		rq->engine->irq_barrier(rq->engine);
>+
>+		if (i915_request_complete(rq))
>+			break;
>+
>+		if (timeout_ns && time_after_eq(jiffies, timeout_expire)) {
>+			ret = -ETIME;
>+			break;
>+		}
>+
>+		if (interruptible && signal_pending(current)) {
>+			ret = -ERESTARTSYS;
>+			break;
>+		}
>+
>+		/* Paranoid kick of hangcheck so that we never wait forever */
>+		i915_queue_hangcheck(rq->i915->dev);
>+
>+		timer.function = NULL;
>+		if (timeout_ns || missed_irq(rq)) {
>+			unsigned long expire;
>+
>+			setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
>+			expire = missed_irq(rq) ? jiffies + 1 : timeout_expire;
>+			mod_timer(&timer, expire);
>+		}
>+
>+		io_schedule();
>+
>+		if (timer.function) {
>+			del_singleshot_timer_sync(&timer);
>+			destroy_timer_on_stack(&timer);
>+		}
>+	}
>+	now = jiffies;
>+	trace_i915_gem_request_wait_end(rq);
>+
>+	if (!irq_test_in_progress)
>+		rq->engine->irq_put(rq->engine);
>+
>+	finish_wait(&rq->engine->irq_queue, &wait);
>+
>+	if (timeout_ns) {
>+		s64 tres = *timeout_ns - jiffies_to_nsecs(now - before);
>+		*timeout_ns = tres <= 0 ? 0 : tres;
>+	}
>+
>+	return ret;
>+}
>+
>+struct i915_gem_request *
>+i915_request_get_breadcrumb(struct i915_gem_request *rq)
>+{
>+	struct list_head *list;
>+	u32 seqno;
>+	int ret;
>+
>+	/* Writes are only coherent from the cpu (in the general case) when
>+	 * the interrupt following the write to memory is complete. That is
>+	 * when the breadcrumb after the write request is complete.
>+	 *
>+	 * Reads are only complete when then command streamer barrier is
>+	 * passed.
>+	 *
>+	 * In both cases, the CPU needs to wait upon the subsequent breadcrumb,
>+	 * which ensures that all pending flushes have been emitted and are
>+	 * complete, before reporting that the request is finished and
>+	 * the CPU's view of memory is coherent with the GPU.
>+	 */
>+
>+	ret = i915_request_emit_breadcrumb(rq);
>+	if (ret)
>+		return ERR_PTR(ret);
>+
>+	ret = i915_request_commit(rq);
>+	if (ret)
>+		return ERR_PTR(ret);
>+
>+	if (!list_empty(&rq->breadcrumb_link))
>+		return i915_request_get(rq);
>+
>+	seqno = rq->breadcrumb[rq->engine->id];
>+	list = &rq->ring->breadcrumbs;
>+	list_for_each_entry_reverse(rq, list, breadcrumb_link) {
>+		if (rq->seqno == seqno)
>+			return i915_request_get(rq);
>+	}
>+
>+	return ERR_PTR(-EIO);
>+}
>+
>+int
>+i915_request_wait(struct i915_gem_request *rq)
>+{
>+	int ret;
>+
>+	lockdep_assert_held(&rq->i915->dev->struct_mutex);
>+
>+	rq = i915_request_get_breadcrumb(rq);
>+	if (IS_ERR(rq))
>+		return PTR_ERR(rq);
>+
>+	ret = __i915_request_wait(rq, rq->i915->mm.interruptible,
>+				  NULL, NULL);
>+	i915_request_put(rq);
>+
>+	return ret;
>+}
>+
>+void
>+i915_request_retire(struct i915_gem_request *rq)
>+{
>+	lockdep_assert_held(&rq->i915->dev->struct_mutex);
>+
>+	if (!rq->completed) {
>+		trace_i915_gem_request_complete(rq);
>+		rq->completed = true;
>+	}
>+	trace_i915_gem_request_retire(rq);
>+
>+	/* We know the GPU must have read the request to have
>+	 * sent us the seqno + interrupt, we can use the position
>+	 * of tail of the request to update the last known position
>+	 * of the GPU head.
>+	 */
>+	if (!list_empty(&rq->breadcrumb_link))
>+		rq->ring->retired_head = rq->tail;
>+
>+	rq->batch = NULL;
>+
>+	/* We need to protect against simultaneous hangcheck/capture */
>+	spin_lock(&rq->engine->lock);
>+	if (rq->engine->last_request == rq)
>+		rq->engine->last_request = NULL;
>+	list_del(&rq->engine_list);
>+	spin_unlock(&rq->engine->lock);
>+
>+	list_del(&rq->breadcrumb_link);
>+	remove_from_client(rq);
>+
>+	intel_runtime_pm_put(rq->i915);
>+	i915_request_put(rq);
>+}
>+
>+void
>+__i915_request_free(struct kref *kref)
>+{
>+	struct i915_gem_request *rq = container_of(kref, struct i915_gem_request, kref);
>+
>+	lockdep_assert_held(&rq->i915->dev->struct_mutex);
>+
>+	if (rq->outstanding) {
>+		/* Rollback this partial transaction as we never committed
>+		 * the request to the hardware queue.
>+		 */
>+		rq->ring->tail = rq->head;
>+		rq->ring->space = intel_ring_space(rq->ring);
>+	}
>+
>+	while (!list_empty(&rq->vmas))
>+		vma_free(list_first_entry(&rq->vmas,
>+					  struct i915_gem_request__vma,
>+					  link));
>+
>+	i915_request_switch_context__undo(rq);
>+	i915_gem_context_unreference(rq->ctx);
>+	kfree(rq);
>+}
>diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
>index 21c025a..85fda6b 100644
>--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
>+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
>@@ -289,6 +289,7 @@ void i915_gem_cleanup_stolen(struct drm_device *dev)
> int i915_gem_init_stolen(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>+	u32 tmp;
> 	int bios_reserved = 0;
> 
> #ifdef CONFIG_INTEL_IOMMU
>@@ -308,8 +309,16 @@ int i915_gem_init_stolen(struct drm_device *dev)
> 	DRM_DEBUG_KMS("found %zd bytes of stolen memory at %08lx\n",
> 		      dev_priv->gtt.stolen_size, dev_priv->mm.stolen_base);
> 
>-	if (IS_VALLEYVIEW(dev))
>-		bios_reserved = 1024*1024; /* top 1M on VLV/BYT */
>+	if (INTEL_INFO(dev)->gen >= 8) {
>+		tmp = I915_READ(GEN7_BIOS_RESERVED);
>+		tmp >>= GEN8_BIOS_RESERVED_SHIFT;
>+		tmp &= GEN8_BIOS_RESERVED_MASK;
>+		bios_reserved = (1024*1024) << tmp;
>+	} else if (IS_GEN7(dev)) {
>+		tmp = I915_READ(GEN7_BIOS_RESERVED);
>+		bios_reserved = tmp & GEN7_BIOS_RESERVED_256K ?
>+			256*1024 : 1024*1024;
>+	}
> 
> 	if (WARN_ON(bios_reserved > dev_priv->gtt.stolen_size))
> 		return 0;
>diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
>index cb150e8..a48355c 100644
>--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
>+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
>@@ -91,7 +91,14 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
> 	uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
> 	uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
> 
>-	if (IS_VALLEYVIEW(dev)) {
>+	if (INTEL_INFO(dev)->gen >= 8 || IS_VALLEYVIEW(dev)) {
>+		/*
>+		 * On BDW+, swizzling is not used. We leave the CPU memory
>+		 * controller in charge of optimizing memory accesses without
>+		 * the extra address manipulation GPU side.
>+		 *
>+		 * VLV and CHV don't have GPU swizzling.
>+		 */
> 		swizzle_x = I915_BIT_6_SWIZZLE_NONE;
> 		swizzle_y = I915_BIT_6_SWIZZLE_NONE;
> 	} else if (INTEL_INFO(dev)->gen >= 6) {
>@@ -376,7 +383,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
> 
> 		if (ret == 0) {
> 			obj->fence_dirty =
>-				obj->fenced_gpu_access ||
>+				obj->last_fence.request ||
> 				obj->fence_reg != I915_FENCE_REG_NONE;
> 
> 			obj->tiling_mode = args->tiling_mode;
>diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
>index eab41f9..1f320c8 100644
>--- a/drivers/gpu/drm/i915/i915_gpu_error.c
>+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
>@@ -192,15 +192,18 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
> 				struct drm_i915_error_buffer *err,
> 				int count)
> {
>-	err_printf(m, "%s [%d]:\n", name, count);
>+	int n;
> 
>+	err_printf(m, "  %s [%d]:\n", name, count);
> 	while (count--) {
>-		err_printf(m, "  %08x %8u %02x %02x %x %x",
>+		err_printf(m, "    %08x %8u %02x %02x [",
> 			   err->gtt_offset,
> 			   err->size,
> 			   err->read_domains,
>-			   err->write_domain,
>-			   err->rseqno, err->wseqno);
>+			   err->write_domain);
>+		for (n = 0; n < ARRAY_SIZE(err->rseqno); n++)
>+			err_printf(m, " %x", err->rseqno[n]);
>+		err_printf(m, " ] %x %x ", err->wseqno, err->fseqno);
> 		err_puts(m, pin_flag(err->pinned));
> 		err_puts(m, tiling_flag(err->tiling));
> 		err_puts(m, dirty_flag(err->dirty));
>@@ -208,7 +211,7 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
> 		err_puts(m, err->userptr ? " userptr" : "");
> 		err_puts(m, err->ring != -1 ? " " : "");
> 		err_puts(m, ring_str(err->ring));
>-		err_puts(m, i915_cache_level_str(err->cache_level));
>+		err_puts(m, i915_cache_level_str(m->i915, err->cache_level));
> 
> 		if (err->name)
> 			err_printf(m, " (name: %d)", err->name);
>@@ -220,11 +223,13 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
> 	}
> }
> 
>-static const char *hangcheck_action_to_str(enum intel_ring_hangcheck_action a)
>+static const char *hangcheck_action_to_str(enum intel_engine_hangcheck_action a)
> {
> 	switch (a) {
> 	case HANGCHECK_IDLE:
> 		return "idle";
>+	case HANGCHECK_IDLE_WAITERS:
>+		return "idle (with waiters)";
> 	case HANGCHECK_WAIT:
> 		return "wait";
> 	case HANGCHECK_ACTIVE:
>@@ -244,13 +249,19 @@ static void i915_ring_error_state(struct drm_i915_error_state_buf *m,
> 				  struct drm_device *dev,
> 				  struct drm_i915_error_ring *ring)
> {
>+	int n;
>+
> 	if (!ring->valid)
> 		return;
> 
>-	err_printf(m, "  HEAD: 0x%08x\n", ring->head);
>-	err_printf(m, "  TAIL: 0x%08x\n", ring->tail);
>-	err_printf(m, "  CTL: 0x%08x\n", ring->ctl);
>-	err_printf(m, "  HWS: 0x%08x\n", ring->hws);
>+	err_printf(m, "%s command stream:\n", ring_str(ring->id));
>+
>+	err_printf(m, "  START: 0x%08x\n", ring->start);
>+	err_printf(m, "  HEAD:  0x%08x\n", ring->head);
>+	err_printf(m, "  TAIL:  0x%08x\n", ring->tail);
>+	err_printf(m, "  CTL:   0x%08x\n", ring->ctl);
>+	err_printf(m, "  MODE:  0x%08x [idle? %d]\n", ring->mode, !!(ring->mode & MODE_IDLE));
>+	err_printf(m, "  HWS:   0x%08x\n", ring->hws);
> 	err_printf(m, "  ACTHD: 0x%08x %08x\n", (u32)(ring->acthd>>32), (u32)ring->acthd);
> 	err_printf(m, "  IPEIR: 0x%08x\n", ring->ipeir);
> 	err_printf(m, "  IPEHR: 0x%08x\n", ring->ipehr);
>@@ -266,17 +277,13 @@ static void i915_ring_error_state(struct drm_i915_error_state_buf *m,
> 	if (INTEL_INFO(dev)->gen >= 6) {
> 		err_printf(m, "  RC PSMI: 0x%08x\n", ring->rc_psmi);
> 		err_printf(m, "  FAULT_REG: 0x%08x\n", ring->fault_reg);
>-		err_printf(m, "  SYNC_0: 0x%08x [last synced 0x%08x]\n",
>-			   ring->semaphore_mboxes[0],
>-			   ring->semaphore_seqno[0]);
>-		err_printf(m, "  SYNC_1: 0x%08x [last synced 0x%08x]\n",
>-			   ring->semaphore_mboxes[1],
>-			   ring->semaphore_seqno[1]);
>-		if (HAS_VEBOX(dev)) {
>-			err_printf(m, "  SYNC_2: 0x%08x [last synced 0x%08x]\n",
>-				   ring->semaphore_mboxes[2],
>-				   ring->semaphore_seqno[2]);
>-		}
>+		err_printf(m, "  SYNC_0: 0x%08x\n",
>+			   ring->semaphore_mboxes[0]);
>+		err_printf(m, "  SYNC_1: 0x%08x\n",
>+			   ring->semaphore_mboxes[1]);
>+		if (HAS_VEBOX(dev))
>+			err_printf(m, "  SYNC_2: 0x%08x\n",
>+				   ring->semaphore_mboxes[2]);
> 	}
> 	if (USES_PPGTT(dev)) {
> 		err_printf(m, "  GFX_MODE: 0x%08x\n", ring->vm_info.gfx_mode);
>@@ -291,8 +298,20 @@ static void i915_ring_error_state(struct drm_i915_error_state_buf *m,
> 				   ring->vm_info.pp_dir_base);
> 		}
> 	}
>-	err_printf(m, "  seqno: 0x%08x\n", ring->seqno);
>-	err_printf(m, "  waiting: %s\n", yesno(ring->waiting));
>+	err_printf(m, "  tag: 0x%04x\n", ring->tag);
>+	err_printf(m, "  seqno: 0x%08x [hangcheck 0x%08x, breadcrumb 0x%08x, request 0x%08x]\n",
>+		   ring->seqno, ring->hangcheck, ring->breadcrumb[ring->id], ring->request);
>+	err_printf(m, "  sem.signal: [");
>+	for (n = 0; n < ARRAY_SIZE(ring->breadcrumb); n++)
>+		err_printf(m, " %s%08x", n == ring->id ? "*" : "", ring->breadcrumb[n]);
>+	err_printf(m, " ]\n");
>+	err_printf(m, "  sem.waited: [");
>+	for (n = 0; n < ARRAY_SIZE(ring->semaphore_sync); n++)
>+		err_printf(m, " %s%08x", n == ring->id ? "*" : "", ring->semaphore_sync[n]);
>+	err_printf(m, " ]\n");
>+	err_printf(m, "  waiting: %s [irq count %d]\n",
>+		   yesno(ring->waiting), ring->irq_count);
>+	err_printf(m, "  interrupts: %d\n", ring->interrupts);
> 	err_printf(m, "  ring->head: 0x%08x\n", ring->cpu_ring_head);
> 	err_printf(m, "  ring->tail: 0x%08x\n", ring->cpu_ring_tail);
> 	err_printf(m, "  hangcheck: %s [%d]\n",
>@@ -362,11 +381,16 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
> 	err_printf(m, "EIR: 0x%08x\n", error->eir);
> 	err_printf(m, "IER: 0x%08x\n", error->ier);
> 	if (INTEL_INFO(dev)->gen >= 8) {
>-		for (i = 0; i < 4; i++)
>+		for (i = 0; i < 4; i++) {
> 			err_printf(m, "GTIER gt %d: 0x%08x\n", i,
> 				   error->gtier[i]);
>-	} else if (HAS_PCH_SPLIT(dev) || IS_VALLEYVIEW(dev))
>+			err_printf(m, "GTIMR gt %d: 0x%08x\n", i,
>+				   error->gtimr[i]);
>+		}
>+	} else if (HAS_PCH_SPLIT(dev) || IS_VALLEYVIEW(dev)) {
> 		err_printf(m, "GTIER: 0x%08x\n", error->gtier[0]);
>+		err_printf(m, "GTIMR: 0x%08x\n", error->gtimr[0]);
>+	}
> 	err_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
> 	err_printf(m, "FORCEWAKE: 0x%08x\n", error->forcewake);
> 	err_printf(m, "DERRMR: 0x%08x\n", error->derrmr);
>@@ -388,64 +412,69 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
> 	if (INTEL_INFO(dev)->gen == 7)
> 		err_printf(m, "ERR_INT: 0x%08x\n", error->err_int);
> 
>-	for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
>-		err_printf(m, "%s command stream:\n", ring_str(i));
>+	for (i = 0; i < ARRAY_SIZE(error->ring); i++)
> 		i915_ring_error_state(m, dev, &error->ring[i]);
>-	}
> 
>-	if (error->active_bo)
>+	for (i = 0; i < error->vm_count; i++) {
>+		err_printf(m, "vm[%d]\n", i);
>+
> 		print_error_buffers(m, "Active",
>-				    error->active_bo[0],
>-				    error->active_bo_count[0]);
>+				    error->active_bo[i],
>+				    error->active_bo_count[i]);
> 
>-	if (error->pinned_bo)
> 		print_error_buffers(m, "Pinned",
>-				    error->pinned_bo[0],
>-				    error->pinned_bo_count[0]);
>+				    error->pinned_bo[i],
>+				    error->pinned_bo_count[i]);
>+	}
> 
> 	for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
>-		obj = error->ring[i].batchbuffer;
>+		const struct drm_i915_error_ring *ering = &error->ring[i];
>+		const char *name = dev_priv->engine[ering->id].name;
>+
>+		obj = ering->batchbuffer;
> 		if (obj) {
>-			err_puts(m, dev_priv->ring[i].name);
>-			if (error->ring[i].pid != -1)
>+			err_puts(m, name);
>+			if (ering->pid != -1)
> 				err_printf(m, " (submitted by %s [%d])",
>-					   error->ring[i].comm,
>-					   error->ring[i].pid);
>+					   ering->comm, ering->pid);
> 			err_printf(m, " --- gtt_offset = 0x%08x\n",
> 				   obj->gtt_offset);
> 			print_error_obj(m, obj);
> 		}
> 
>-		obj = error->ring[i].wa_batchbuffer;
>+		obj = ering->wa_batchbuffer;
> 		if (obj) {
> 			err_printf(m, "%s (w/a) --- gtt_offset = 0x%08x\n",
>-				   dev_priv->ring[i].name, obj->gtt_offset);
>+				   name, obj->gtt_offset);
> 			print_error_obj(m, obj);
> 		}
> 
>-		if (error->ring[i].num_requests) {
>+		if (ering->num_requests) {
> 			err_printf(m, "%s --- %d requests\n",
>-				   dev_priv->ring[i].name,
>-				   error->ring[i].num_requests);
>-			for (j = 0; j < error->ring[i].num_requests; j++) {
>-				err_printf(m, "  seqno 0x%08x, emitted %ld, tail 0x%08x\n",
>-					   error->ring[i].requests[j].seqno,
>-					   error->ring[i].requests[j].jiffies,
>-					   error->ring[i].requests[j].tail);
>+				   name, ering->num_requests);
>+			for (j = 0; j < ering->num_requests; j++) {
>+				err_printf(m, "  pid %ld, seqno 0x%08x, tag 0x%04x, emitted %dms ago (at %ld jiffies), head 0x%08x, tail 0x%08x, batch 0x%08x, complete? %d\n",
>+					   ering->requests[j].pid,
>+					   ering->requests[j].seqno,
>+					   ering->requests[j].tag,
>+					   jiffies_to_usecs(jiffies - ering->requests[j].jiffies) / 1000,
>+					   ering->requests[j].jiffies,
>+					   ering->requests[j].head,
>+					   ering->requests[j].tail,
>+					   ering->requests[j].batch,
>+					   ering->requests[j].complete);
> 			}
> 		}
> 
>-		if ((obj = error->ring[i].ringbuffer)) {
>+		if ((obj = ering->ringbuffer)) {
> 			err_printf(m, "%s --- ringbuffer = 0x%08x\n",
>-				   dev_priv->ring[i].name,
>-				   obj->gtt_offset);
>+				   name, obj->gtt_offset);
> 			print_error_obj(m, obj);
> 		}
> 
>-		if ((obj = error->ring[i].hws_page)) {
>+		if ((obj = ering->hws_page)) {
> 			err_printf(m, "%s --- HW Status = 0x%08x\n",
>-				   dev_priv->ring[i].name,
>-				   obj->gtt_offset);
>+				   name, obj->gtt_offset);
> 			offset = 0;
> 			for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
> 				err_printf(m, "[%04x] %08x %08x %08x %08x\n",
>@@ -460,8 +489,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
> 
> 		if ((obj = error->ring[i].ctx)) {
> 			err_printf(m, "%s --- HW Context = 0x%08x\n",
>-				   dev_priv->ring[i].name,
>-				   obj->gtt_offset);
>+				   name, obj->gtt_offset);
> 			print_error_obj(m, obj);
> 		}
> 	}
>@@ -492,9 +520,11 @@ out:
> }
> 
> int i915_error_state_buf_init(struct drm_i915_error_state_buf *ebuf,
>+			      struct drm_i915_private *i915,
> 			      size_t count, loff_t pos)
> {
> 	memset(ebuf, 0, sizeof(*ebuf));
>+	ebuf->i915 = i915;
> 
> 	/* We need to have enough room to store any i915_error_state printf
> 	 * so that we can move it to start position.
>@@ -556,24 +586,55 @@ static void i915_error_state_free(struct kref *error_ref)
> }
> 
> static struct drm_i915_error_object *
>-i915_error_object_create_sized(struct drm_i915_private *dev_priv,
>-			       struct drm_i915_gem_object *src,
>-			       struct i915_address_space *vm,
>-			       const int num_pages)
>+i915_error_object_create(struct drm_i915_private *dev_priv,
>+			 struct i915_vma *vma)
> {
>+	struct drm_i915_gem_object *src;
> 	struct drm_i915_error_object *dst;
>-	int i;
>+	int num_pages;
>+	bool use_ggtt;
>+	int i = 0;
> 	u32 reloc_offset;
> 
>-	if (src == NULL || src->pages == NULL)
>+	if (vma == NULL)
> 		return NULL;
> 
>+	src = vma->obj;
>+	if (src->pages == NULL)
>+		return NULL;
>+
>+	num_pages = src->base.size >> PAGE_SHIFT;
>+
> 	dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *), GFP_ATOMIC);
> 	if (dst == NULL)
> 		return NULL;
> 
>-	reloc_offset = dst->gtt_offset = i915_gem_obj_offset(src, vm);
>-	for (i = 0; i < num_pages; i++) {
>+	dst->gtt_offset = vma->node.start;
>+
>+	reloc_offset = dst->gtt_offset;
>+	use_ggtt = (src->cache_level == I915_CACHE_NONE &&
>+		    i915_is_ggtt(vma->vm) &&
>+		    src->has_global_gtt_mapping &&
>+		    reloc_offset + num_pages * PAGE_SIZE <= dev_priv->gtt.mappable_end);
>+
>+	/* Cannot access stolen address directly, try to use the aperture */
>+	if (src->stolen) {
>+		use_ggtt = true;
>+
>+		if (!src->has_global_gtt_mapping)
>+			goto unwind;
>+
>+		reloc_offset = i915_gem_obj_ggtt_offset(src);
>+		if (reloc_offset + num_pages * PAGE_SIZE > dev_priv->gtt.mappable_end)
>+			goto unwind;
>+	}
>+
>+	/* Cannot access snooped pages through the aperture */
>+	if (use_ggtt && src->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv->dev))
>+		goto unwind;
>+
>+	dst->page_count = num_pages;
>+	while (num_pages--) {
> 		unsigned long flags;
> 		void *d;
> 
>@@ -582,10 +643,7 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv,
> 			goto unwind;
> 
> 		local_irq_save(flags);
>-		if (src->cache_level == I915_CACHE_NONE &&
>-		    reloc_offset < dev_priv->gtt.mappable_end &&
>-		    src->has_global_gtt_mapping &&
>-		    i915_is_ggtt(vm)) {
>+		if (use_ggtt) {
> 			void __iomem *s;
> 
> 			/* Simply ignore tiling or any overlapping fence.
>@@ -597,14 +655,6 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv,
> 						     reloc_offset);
> 			memcpy_fromio(d, s, PAGE_SIZE);
> 			io_mapping_unmap_atomic(s);
>-		} else if (src->stolen) {
>-			unsigned long offset;
>-
>-			offset = dev_priv->mm.stolen_base;
>-			offset += src->stolen->start;
>-			offset += i << PAGE_SHIFT;
>-
>-			memcpy_fromio(d, (void __iomem *) offset, PAGE_SIZE);
> 		} else {
> 			struct page *page;
> 			void *s;
>@@ -621,11 +671,9 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv,
> 		}
> 		local_irq_restore(flags);
> 
>-		dst->pages[i] = d;
>-
>+		dst->pages[i++] = d;
> 		reloc_offset += PAGE_SIZE;
> 	}
>-	dst->page_count = num_pages;
> 
> 	return dst;
> 
>@@ -635,22 +683,32 @@ unwind:
> 	kfree(dst);
> 	return NULL;
> }
>-#define i915_error_object_create(dev_priv, src, vm) \
>-	i915_error_object_create_sized((dev_priv), (src), (vm), \
>-				       (src)->base.size>>PAGE_SHIFT)
> 
>-#define i915_error_ggtt_object_create(dev_priv, src) \
>-	i915_error_object_create_sized((dev_priv), (src), &(dev_priv)->gtt.base, \
>-				       (src)->base.size>>PAGE_SHIFT)
>+static inline struct drm_i915_error_object *
>+i915_error_ggtt_object_create(struct drm_i915_private *i915,
>+			      struct drm_i915_gem_object *src)
>+{
>+	if (src == NULL)
>+		return NULL;
>+
>+	return i915_error_object_create(i915,
>+					i915_gem_obj_to_vma(src,
>+							    &i915->gtt.base));
>+}
> 
> static void capture_bo(struct drm_i915_error_buffer *err,
>-		       struct drm_i915_gem_object *obj)
>+		       struct i915_vma *vma)
> {
>+	struct drm_i915_gem_object *obj = vma->obj;
>+	int n;
>+
> 	err->size = obj->base.size;
> 	err->name = obj->base.name;
>-	err->rseqno = obj->last_read_seqno;
>-	err->wseqno = obj->last_write_seqno;
>-	err->gtt_offset = i915_gem_obj_ggtt_offset(obj);
>+	for (n = 0; n < ARRAY_SIZE(obj->last_read); n++)
>+		err->rseqno[n] = i915_request_seqno(obj->last_read[n].request);
>+	err->wseqno = i915_request_seqno(obj->last_write.request);
>+	err->fseqno = i915_request_seqno(obj->last_fence.request);
>+	err->gtt_offset = vma->node.start;
> 	err->read_domains = obj->base.read_domains;
> 	err->write_domain = obj->base.write_domain;
> 	err->fence_reg = obj->fence_reg;
>@@ -663,7 +721,7 @@ static void capture_bo(struct drm_i915_error_buffer *err,
> 	err->dirty = obj->dirty;
> 	err->purgeable = obj->madv != I915_MADV_WILLNEED;
> 	err->userptr = obj->userptr.mm != NULL;
>-	err->ring = obj->ring ? obj->ring->id : -1;
>+	err->ring = i915_request_engine_id(obj->last_write.request);
> 	err->cache_level = obj->cache_level;
> }
> 
>@@ -674,7 +732,7 @@ static u32 capture_active_bo(struct drm_i915_error_buffer *err,
> 	int i = 0;
> 
> 	list_for_each_entry(vma, head, mm_list) {
>-		capture_bo(err++, vma->obj);
>+		capture_bo(err++, vma);
> 		if (++i == count)
> 			break;
> 	}
>@@ -683,21 +741,27 @@ static u32 capture_active_bo(struct drm_i915_error_buffer *err,
> }
> 
> static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
>-			     int count, struct list_head *head)
>+			     int count, struct list_head *head,
>+			     struct i915_address_space *vm)
> {
> 	struct drm_i915_gem_object *obj;
>-	int i = 0;
>+	struct drm_i915_error_buffer * const first = err;
>+	struct drm_i915_error_buffer * const last = err + count;
> 
> 	list_for_each_entry(obj, head, global_list) {
>-		if (!i915_gem_obj_is_pinned(obj))
>-			continue;
>+		struct i915_vma *vma;
> 
>-		capture_bo(err++, obj);
>-		if (++i == count)
>+		if (err == last)
> 			break;
>+
>+		list_for_each_entry(vma, &obj->vma_list, vma_link)
>+			if (vma->vm == vm && vma->pin_count > 0) {
>+				capture_bo(err++, vma);
>+				break;
>+			}
> 	}
> 
>-	return i;
>+	return err - first;
> }
> 
> /* Generate a semi-unique error code. The code is not meant to have meaning, The
>@@ -721,7 +785,7 @@ static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv,
> 	 * synchronization commands which almost always appear in the case
> 	 * strictly a client bug. Use instdone to differentiate those some.
> 	 */
>-	for (i = 0; i < I915_NUM_RINGS; i++) {
>+	for (i = 0; i < I915_NUM_ENGINES; i++) {
> 		if (error->ring[i].hangcheck_action == HANGCHECK_HUNG) {
> 			if (ring_id)
> 				*ring_id = i;
>@@ -769,83 +833,77 @@ static void i915_gem_record_fences(struct drm_device *dev,
> 
> static void gen8_record_semaphore_state(struct drm_i915_private *dev_priv,
> 					struct drm_i915_error_state *error,
>-					struct intel_engine_cs *ring,
>+					struct intel_engine_cs *engine,
> 					struct drm_i915_error_ring *ering)
> {
> 	struct intel_engine_cs *to;
>+	u32 *mbox;
> 	int i;
> 
>-	if (!i915_semaphore_is_enabled(dev_priv->dev))
>+	if (dev_priv->semaphore_obj == NULL)
> 		return;
> 
>-	if (!error->semaphore_obj)
>+	if (error->semaphore_obj == NULL)
> 		error->semaphore_obj =
>-			i915_error_object_create(dev_priv,
>-						 dev_priv->semaphore_obj,
>-						 &dev_priv->gtt.base);
>-
>-	for_each_ring(to, dev_priv, i) {
>-		int idx;
>-		u16 signal_offset;
>-		u32 *tmp;
>+			i915_error_ggtt_object_create(dev_priv,
>+						      dev_priv->semaphore_obj);
>+	if (error->semaphore_obj == NULL)
>+		return;
> 
>-		if (ring == to)
>+	mbox = error->semaphore_obj->pages[0];
>+	for_each_engine(to, dev_priv, i) {
>+		if (engine == to)
> 			continue;
> 
>-		signal_offset = (GEN8_SIGNAL_OFFSET(ring, i) & (PAGE_SIZE - 1))
>-				/ 4;
>-		tmp = error->semaphore_obj->pages[0];
>-		idx = intel_ring_sync_index(ring, to);
>-
>-		ering->semaphore_mboxes[idx] = tmp[signal_offset];
>-		ering->semaphore_seqno[idx] = ring->semaphore.sync_seqno[idx];
>+		ering->semaphore_mboxes[i] =
>+			mbox[GEN8_SEMAPHORE_OFFSET(dev_priv,
>+						   engine->id,
>+						   i) & (PAGE_SIZE - 1) / 4];
> 	}
> }
> 
> static void gen6_record_semaphore_state(struct drm_i915_private *dev_priv,
>-					struct intel_engine_cs *ring,
>+					struct intel_engine_cs *engine,
> 					struct drm_i915_error_ring *ering)
> {
>-	ering->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(ring->mmio_base));
>-	ering->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(ring->mmio_base));
>-	ering->semaphore_seqno[0] = ring->semaphore.sync_seqno[0];
>-	ering->semaphore_seqno[1] = ring->semaphore.sync_seqno[1];
>-
>+	ering->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(engine->mmio_base));
>+	ering->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(engine->mmio_base));
> 	if (HAS_VEBOX(dev_priv->dev)) {
> 		ering->semaphore_mboxes[2] =
>-			I915_READ(RING_SYNC_2(ring->mmio_base));
>-		ering->semaphore_seqno[2] = ring->semaphore.sync_seqno[2];
>+			I915_READ(RING_SYNC_2(engine->mmio_base));
> 	}
> }
> 
> static void i915_record_ring_state(struct drm_device *dev,
> 				   struct drm_i915_error_state *error,
>-				   struct intel_engine_cs *ring,
>+				   struct intel_engine_cs *engine,
>+				   struct i915_gem_request *rq,
> 				   struct drm_i915_error_ring *ering)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct intel_ringbuffer *ring;
> 
> 	if (INTEL_INFO(dev)->gen >= 6) {
>-		ering->rc_psmi = I915_READ(ring->mmio_base + 0x50);
>-		ering->fault_reg = I915_READ(RING_FAULT_REG(ring));
>+		ering->rc_psmi = I915_READ(engine->mmio_base + 0x50);
>+		ering->fault_reg = I915_READ(RING_FAULT_REG(engine));
> 		if (INTEL_INFO(dev)->gen >= 8)
>-			gen8_record_semaphore_state(dev_priv, error, ring, ering);
>+			gen8_record_semaphore_state(dev_priv, error, engine, ering);
> 		else
>-			gen6_record_semaphore_state(dev_priv, ring, ering);
>+			gen6_record_semaphore_state(dev_priv, engine, ering);
> 	}
> 
> 	if (INTEL_INFO(dev)->gen >= 4) {
>-		ering->faddr = I915_READ(RING_DMA_FADD(ring->mmio_base));
>-		ering->ipeir = I915_READ(RING_IPEIR(ring->mmio_base));
>-		ering->ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
>-		ering->instdone = I915_READ(RING_INSTDONE(ring->mmio_base));
>-		ering->instps = I915_READ(RING_INSTPS(ring->mmio_base));
>-		ering->bbaddr = I915_READ(RING_BBADDR(ring->mmio_base));
>+		ering->faddr = I915_READ(RING_DMA_FADD(engine->mmio_base));
>+		ering->ipeir = I915_READ(RING_IPEIR(engine->mmio_base));
>+		ering->ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
>+		ering->instdone = I915_READ(RING_INSTDONE(engine->mmio_base));
>+		ering->instps = I915_READ(RING_INSTPS(engine->mmio_base));
>+		ering->bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
> 		if (INTEL_INFO(dev)->gen >= 8) {
>-			ering->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(ring->mmio_base)) << 32;
>-			ering->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(ring->mmio_base)) << 32;
>+			ering->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(engine->mmio_base)) << 32;
>+			ering->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(engine->mmio_base)) << 32;
> 		}
>-		ering->bbstate = I915_READ(RING_BBSTATE(ring->mmio_base));
>+		ering->bbstate = I915_READ(RING_BBSTATE(engine->mmio_base));
> 	} else {
> 		ering->faddr = I915_READ(DMA_FADD_I8XX);
> 		ering->ipeir = I915_READ(IPEIR);
>@@ -853,19 +911,29 @@ static void i915_record_ring_state(struct drm_device *dev,
> 		ering->instdone = I915_READ(INSTDONE);
> 	}
> 
>-	ering->waiting = waitqueue_active(&ring->irq_queue);
>-	ering->instpm = I915_READ(RING_INSTPM(ring->mmio_base));
>-	ering->seqno = ring->get_seqno(ring, false);
>-	ering->acthd = intel_ring_get_active_head(ring);
>-	ering->head = I915_READ_HEAD(ring);
>-	ering->tail = I915_READ_TAIL(ring);
>-	ering->ctl = I915_READ_CTL(ring);
>+	ering->waiting = waitqueue_active(&engine->irq_queue);
>+	ering->instpm = I915_READ(RING_INSTPM(engine->mmio_base));
>+	ering->acthd = intel_engine_get_active_head(engine);
>+	ering->seqno = engine->get_seqno(engine);
>+	ering->request = engine->last_request ? engine->last_request->seqno : 0;
>+	ering->hangcheck = engine->hangcheck.seqno;
>+	memcpy(ering->breadcrumb, engine->breadcrumb, sizeof(ering->breadcrumb));
>+	memcpy(ering->semaphore_sync, engine->semaphore.sync, sizeof(ering->semaphore_sync));
>+	ering->tag = engine->tag;
>+	ering->interrupts = atomic_read(&engine->interrupts);
>+	ering->irq_count = engine->irq_refcount;
>+	ering->start = I915_READ_START(engine);
>+	ering->head = I915_READ_HEAD(engine);
>+	ering->tail = I915_READ_TAIL(engine);
>+	ering->ctl = I915_READ_CTL(engine);
>+	if (!IS_GEN2(dev_priv))
>+		ering->mode = I915_READ_MODE(engine);
> 
> 	if (I915_NEED_GFX_HWS(dev)) {
> 		int mmio;
> 
> 		if (IS_GEN7(dev)) {
>-			switch (ring->id) {
>+			switch (engine->id) {
> 			default:
> 			case RCS:
> 				mmio = RENDER_HWS_PGA_GEN7;
>@@ -880,59 +948,67 @@ static void i915_record_ring_state(struct drm_device *dev,
> 				mmio = VEBOX_HWS_PGA_GEN7;
> 				break;
> 			}
>-		} else if (IS_GEN6(ring->dev)) {
>-			mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
>+		} else if (IS_GEN6(engine->i915)) {
>+			mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
> 		} else {
> 			/* XXX: gen8 returns to sanity */
>-			mmio = RING_HWS_PGA(ring->mmio_base);
>+			mmio = RING_HWS_PGA(engine->mmio_base);
> 		}
> 
> 		ering->hws = I915_READ(mmio);
> 	}
> 
>-	ering->cpu_ring_head = ring->buffer->head;
>-	ering->cpu_ring_tail = ring->buffer->tail;
>+	ring = rq ? rq->ctx->ring[engine->id].ring : engine->default_context->ring[engine->id].ring;
>+	if (ring) {
>+		ering->cpu_ring_head = ring->head;
>+		ering->cpu_ring_tail = ring->tail;
>+		ering->ringbuffer =
>+			i915_error_ggtt_object_create(dev_priv, ring->obj);
>+	}
>+
>+	ering->hws_page =
>+		i915_error_ggtt_object_create(dev_priv,
>+					      engine->status_page.obj);
> 
>-	ering->hangcheck_score = ring->hangcheck.score;
>-	ering->hangcheck_action = ring->hangcheck.action;
>+	ering->hangcheck_score = engine->hangcheck.score;
>+	ering->hangcheck_action = engine->hangcheck.action;
> 
> 	if (USES_PPGTT(dev)) {
> 		int i;
> 
>-		ering->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(ring));
>+		ering->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(engine));
> 
> 		switch (INTEL_INFO(dev)->gen) {
> 		case 8:
> 			for (i = 0; i < 4; i++) {
> 				ering->vm_info.pdp[i] =
>-					I915_READ(GEN8_RING_PDP_UDW(ring, i));
>+					I915_READ(GEN8_RING_PDP_UDW(engine, i));
> 				ering->vm_info.pdp[i] <<= 32;
> 				ering->vm_info.pdp[i] |=
>-					I915_READ(GEN8_RING_PDP_LDW(ring, i));
>+					I915_READ(GEN8_RING_PDP_LDW(engine, i));
> 			}
> 			break;
> 		case 7:
> 			ering->vm_info.pp_dir_base =
>-				I915_READ(RING_PP_DIR_BASE(ring));
>+				I915_READ(RING_PP_DIR_BASE(engine));
> 			break;
> 		case 6:
> 			ering->vm_info.pp_dir_base =
>-				I915_READ(RING_PP_DIR_BASE_READ(ring));
>+				I915_READ(RING_PP_DIR_BASE_READ(engine));
> 			break;
> 		}
> 	}
> }
> 
>-
>-static void i915_gem_record_active_context(struct intel_engine_cs *ring,
>+static void i915_gem_record_active_context(struct intel_engine_cs *engine,
> 					   struct drm_i915_error_state *error,
> 					   struct drm_i915_error_ring *ering)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	struct drm_i915_gem_object *obj;
> 
> 	/* Currently render ring is the only HW context user */
>-	if (ring->id != RCS || !error->ccid)
>+	if (engine->id != RCS || !error->ccid)
> 		return;
> 
> 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
>@@ -950,45 +1026,40 @@ static void i915_gem_record_rings(struct drm_device *dev,
> 				  struct drm_i915_error_state *error)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_i915_gem_request *request;
>+	struct i915_gem_request *rq;
> 	int i, count;
> 
>-	for (i = 0; i < I915_NUM_RINGS; i++) {
>-		struct intel_engine_cs *ring = &dev_priv->ring[i];
>+	for (i = 0; i < I915_NUM_ENGINES; i++) {
>+		struct intel_engine_cs *engine = &dev_priv->engine[i];
> 
> 		error->ring[i].pid = -1;
> 
>-		if (ring->dev == NULL)
>+		if (engine->i915 == NULL)
> 			continue;
> 
> 		error->ring[i].valid = true;
>+		error->ring[i].id = i;
> 
>-		i915_record_ring_state(dev, error, ring, &error->ring[i]);
>-
>-		request = i915_gem_find_active_request(ring);
>-		if (request) {
>+		spin_lock(&engine->lock);
>+		rq = intel_engine_find_active_batch(engine);
>+		if (rq) {
> 			/* We need to copy these to an anonymous buffer
> 			 * as the simplest method to avoid being overwritten
> 			 * by userspace.
> 			 */
> 			error->ring[i].batchbuffer =
>-				i915_error_object_create(dev_priv,
>-							 request->batch_obj,
>-							 request->ctx ?
>-							 request->ctx->vm :
>-							 &dev_priv->gtt.base);
>-
>-			if (HAS_BROKEN_CS_TLB(dev_priv->dev) &&
>-			    ring->scratch.obj)
>+				i915_error_object_create(dev_priv, rq->batch);
>+
>+			if (HAS_BROKEN_CS_TLB(dev_priv->dev))
> 				error->ring[i].wa_batchbuffer =
> 					i915_error_ggtt_object_create(dev_priv,
>-							     ring->scratch.obj);
>+							     engine->scratch.obj);
> 
>-			if (request->file_priv) {
>+			if (rq->file_priv) {
> 				struct task_struct *task;
> 
> 				rcu_read_lock();
>-				task = pid_task(request->file_priv->file->pid,
>+				task = pid_task(rq->file_priv->file->pid,
> 						PIDTYPE_PID);
> 				if (task) {
> 					strcpy(error->ring[i].comm, task->comm);
>@@ -998,17 +1069,12 @@ static void i915_gem_record_rings(struct drm_device *dev,
> 			}
> 		}
> 
>-		error->ring[i].ringbuffer =
>-			i915_error_ggtt_object_create(dev_priv, ring->buffer->obj);
>-
>-		if (ring->status_page.obj)
>-			error->ring[i].hws_page =
>-				i915_error_ggtt_object_create(dev_priv, ring->status_page.obj);
>+		i915_record_ring_state(dev, error, engine, rq, &error->ring[i]);
> 
>-		i915_gem_record_active_context(ring, error, &error->ring[i]);
>+		i915_gem_record_active_context(engine, error, &error->ring[i]);
> 
> 		count = 0;
>-		list_for_each_entry(request, &ring->request_list, list)
>+		list_for_each_entry(rq, &engine->requests, engine_list)
> 			count++;
> 
> 		error->ring[i].num_requests = count;
>@@ -1021,14 +1087,28 @@ static void i915_gem_record_rings(struct drm_device *dev,
> 		}
> 
> 		count = 0;
>-		list_for_each_entry(request, &ring->request_list, list) {
>+		list_for_each_entry(rq, &engine->requests, engine_list) {
> 			struct drm_i915_error_request *erq;
>+			struct task_struct *task;
> 
> 			erq = &error->ring[i].requests[count++];
>-			erq->seqno = request->seqno;
>-			erq->jiffies = request->emitted_jiffies;
>-			erq->tail = request->tail;
>+			erq->seqno = rq->seqno;
>+			erq->jiffies = rq->emitted_jiffies;
>+			erq->head = rq->head;
>+			erq->tail = rq->tail;
>+			erq->batch = 0;
>+			if (rq->batch)
>+				erq->batch = rq->batch->node.start;
>+			memcpy(erq->breadcrumb, rq->breadcrumb, sizeof(rq->breadcrumb));
>+			erq->complete = i915_request_complete(rq);
>+			erq->tag = rq->tag;
>+
>+			rcu_read_lock();
>+			task = rq->file_priv ? pid_task(rq->file_priv->file->pid, PIDTYPE_PID) : NULL;
>+			erq->pid = task ? task->pid : 0;
>+			rcu_read_unlock();
> 		}
>+		spin_unlock(&engine->lock);
> 	}
> }
> 
>@@ -1049,9 +1129,14 @@ static void i915_gem_capture_vm(struct drm_i915_private *dev_priv,
> 	list_for_each_entry(vma, &vm->active_list, mm_list)
> 		i++;
> 	error->active_bo_count[ndx] = i;
>-	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
>-		if (i915_gem_obj_is_pinned(obj))
>-			i++;
>+
>+	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
>+		list_for_each_entry(vma, &obj->vma_list, vma_link)
>+			if (vma->vm == vm && vma->pin_count > 0) {
>+				i++;
>+				break;
>+			}
>+	}
> 	error->pinned_bo_count[ndx] = i - error->active_bo_count[ndx];
> 
> 	if (i) {
>@@ -1070,7 +1155,7 @@ static void i915_gem_capture_vm(struct drm_i915_private *dev_priv,
> 		error->pinned_bo_count[ndx] =
> 			capture_pinned_bo(pinned_bo,
> 					  error->pinned_bo_count[ndx],
>-					  &dev_priv->mm.bound_list);
>+					  &dev_priv->mm.bound_list, vm);
> 	error->active_bo[ndx] = active_bo;
> 	error->pinned_bo[ndx] = pinned_bo;
> }
>@@ -1091,8 +1176,25 @@ static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv,
> 	error->pinned_bo_count = kcalloc(cnt, sizeof(*error->pinned_bo_count),
> 					 GFP_ATOMIC);
> 
>-	list_for_each_entry(vm, &dev_priv->vm_list, global_link)
>-		i915_gem_capture_vm(dev_priv, error, vm, i++);
>+	if (error->active_bo == NULL ||
>+	    error->pinned_bo == NULL ||
>+	    error->active_bo_count == NULL ||
>+	    error->pinned_bo_count == NULL) {
>+		kfree(error->active_bo);
>+		kfree(error->active_bo_count);
>+		kfree(error->pinned_bo);
>+		kfree(error->pinned_bo_count);
>+
>+		error->active_bo = NULL;
>+		error->active_bo_count = NULL;
>+		error->pinned_bo = NULL;
>+		error->pinned_bo_count = NULL;
>+	} else {
>+		list_for_each_entry(vm, &dev_priv->vm_list, global_link)
>+			i915_gem_capture_vm(dev_priv, error, vm, i++);
>+
>+		error->vm_count = cnt;
>+	}
> }
> 
> /* Capture all registers which don't fit into another category. */
>@@ -1113,6 +1215,7 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
> 	/* 1: Registers specific to a single generation */
> 	if (IS_VALLEYVIEW(dev)) {
> 		error->gtier[0] = I915_READ(GTIER);
>+		error->gtimr[0] = I915_READ(GTIMR);
> 		error->ier = I915_READ(VLV_IER);
> 		error->forcewake = I915_READ(FORCEWAKE_VLV);
> 	}
>@@ -1148,11 +1251,14 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
> 
> 	if (INTEL_INFO(dev)->gen >= 8) {
> 		error->ier = I915_READ(GEN8_DE_MISC_IER);
>-		for (i = 0; i < 4; i++)
>+		for (i = 0; i < 4; i++) {
> 			error->gtier[i] = I915_READ(GEN8_GT_IER(i));
>+			error->gtimr[i] = I915_READ(GEN8_GT_IMR(i));
>+		}
> 	} else if (HAS_PCH_SPLIT(dev)) {
> 		error->ier = I915_READ(DEIER);
> 		error->gtier[0] = I915_READ(GTIER);
>+		error->gtimr[0] = I915_READ(GTIMR);
> 	} else if (IS_GEN2(dev)) {
> 		error->ier = I915_READ16(IER);
> 	} else if (!IS_VALLEYVIEW(dev)) {
>@@ -1264,13 +1370,12 @@ void i915_error_state_get(struct drm_device *dev,
> 			  struct i915_error_state_file_priv *error_priv)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	unsigned long flags;
> 
>-	spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
>+	spin_lock_irq(&dev_priv->gpu_error.lock);
> 	error_priv->error = dev_priv->gpu_error.first_error;
> 	if (error_priv->error)
> 		kref_get(&error_priv->error->ref);
>-	spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
>+	spin_unlock_irq(&dev_priv->gpu_error.lock);
> 
> }
> 
>@@ -1284,22 +1389,21 @@ void i915_destroy_error_state(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct drm_i915_error_state *error;
>-	unsigned long flags;
> 
>-	spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
>+	spin_lock_irq(&dev_priv->gpu_error.lock);
> 	error = dev_priv->gpu_error.first_error;
> 	dev_priv->gpu_error.first_error = NULL;
>-	spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
>+	spin_unlock_irq(&dev_priv->gpu_error.lock);
> 
> 	if (error)
> 		kref_put(&error->ref, i915_error_state_free);
> }
> 
>-const char *i915_cache_level_str(int type)
>+const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
> {
> 	switch (type) {
> 	case I915_CACHE_NONE: return " uncached";
>-	case I915_CACHE_LLC: return " snooped or LLC";
>+	case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
> 	case I915_CACHE_L3_LLC: return " L3+LLC";
> 	case I915_CACHE_WT: return " WT";
> 	default: return "";
>diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
>index 0050ee9..348f207 100644
>--- a/drivers/gpu/drm/i915/i915_irq.c
>+++ b/drivers/gpu/drm/i915/i915_irq.c
>@@ -151,7 +151,7 @@ ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
> {
> 	assert_spin_locked(&dev_priv->irq_lock);
> 
>-	if (!intel_irqs_enabled(dev_priv))
>+	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
> 		return;
> 
> 	if ((dev_priv->irq_mask & mask) != mask) {
>@@ -238,7 +238,7 @@ static bool ivb_can_enable_err_int(struct drm_device *dev)
> 
> 	assert_spin_locked(&dev_priv->irq_lock);
> 
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
> 
> 		if (crtc->cpu_fifo_underrun_disabled)
>@@ -296,7 +296,7 @@ static bool cpt_can_enable_serr_int(struct drm_device *dev)
> 
> 	assert_spin_locked(&dev_priv->irq_lock);
> 
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
> 
> 		if (crtc->pch_fifo_underrun_disabled)
>@@ -310,9 +310,8 @@ void i9xx_check_fifo_underruns(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *crtc;
>-	unsigned long flags;
> 
>-	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 
> 	for_each_intel_crtc(dev, crtc) {
> 		u32 reg = PIPESTAT(crtc->pipe);
>@@ -331,7 +330,7 @@ void i9xx_check_fifo_underruns(struct drm_device *dev)
> 		DRM_ERROR("pipe %c underrun\n", pipe_name(crtc->pipe));
> 	}
> 
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> }
> 
> static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev,
>@@ -497,7 +496,7 @@ static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
> 	old = !intel_crtc->cpu_fifo_underrun_disabled;
> 	intel_crtc->cpu_fifo_underrun_disabled = !enable;
> 
>-	if (INTEL_INFO(dev)->gen < 5 || IS_VALLEYVIEW(dev))
>+	if (HAS_GMCH_DISPLAY(dev))
> 		i9xx_set_fifo_underrun_reporting(dev, pipe, enable, old);
> 	else if (IS_GEN5(dev) || IS_GEN6(dev))
> 		ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
>@@ -589,6 +588,7 @@ __i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
> 	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
> 
> 	assert_spin_locked(&dev_priv->irq_lock);
>+	WARN_ON(!intel_irqs_enabled(dev_priv));
> 
> 	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
> 		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
>@@ -615,6 +615,7 @@ __i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
> 	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
> 
> 	assert_spin_locked(&dev_priv->irq_lock);
>+	WARN_ON(!intel_irqs_enabled(dev_priv));
> 
> 	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
> 		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
>@@ -694,19 +695,18 @@ i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
> static void i915_enable_asle_pipestat(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	unsigned long irqflags;
> 
> 	if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
> 		return;
> 
>-	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 
> 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
> 	if (INTEL_INFO(dev)->gen >= 4)
> 		i915_enable_pipestat(dev_priv, PIPE_A,
> 				     PIPE_LEGACY_BLC_EVENT_STATUS);
> 
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> }
> 
> /**
>@@ -1094,18 +1094,17 @@ static void i915_digport_work_func(struct work_struct *work)
> {
> 	struct drm_i915_private *dev_priv =
> 		container_of(work, struct drm_i915_private, dig_port_work);
>-	unsigned long irqflags;
> 	u32 long_port_mask, short_port_mask;
> 	struct intel_digital_port *intel_dig_port;
> 	int i, ret;
> 	u32 old_bits = 0;
> 
>-	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 	long_port_mask = dev_priv->long_hpd_port_mask;
> 	dev_priv->long_hpd_port_mask = 0;
> 	short_port_mask = dev_priv->short_hpd_port_mask;
> 	dev_priv->short_hpd_port_mask = 0;
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> 
> 	for (i = 0; i < I915_MAX_PORTS; i++) {
> 		bool valid = false;
>@@ -1130,10 +1129,13 @@ static void i915_digport_work_func(struct work_struct *work)
> 	}
> 
> 	if (old_bits) {
>-		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+		spin_lock_irq(&dev_priv->irq_lock);
> 		dev_priv->hpd_event_bits |= old_bits;
>-		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>-		schedule_work(&dev_priv->hotplug_work);
>+		spin_unlock_irq(&dev_priv->irq_lock);
>+
>+		mod_delayed_work(system_wq,
>+				 &dev_priv->hotplug_work,
>+				 msecs_to_jiffies(50));
> 	}
> }
> 
>@@ -1145,13 +1147,12 @@ static void i915_digport_work_func(struct work_struct *work)
> static void i915_hotplug_work_func(struct work_struct *work)
> {
> 	struct drm_i915_private *dev_priv =
>-		container_of(work, struct drm_i915_private, hotplug_work);
>+		container_of(work, struct drm_i915_private, hotplug_work.work);
> 	struct drm_device *dev = dev_priv->dev;
> 	struct drm_mode_config *mode_config = &dev->mode_config;
> 	struct intel_connector *intel_connector;
> 	struct intel_encoder *intel_encoder;
> 	struct drm_connector *connector;
>-	unsigned long irqflags;
> 	bool hpd_disabled = false;
> 	bool changed = false;
> 	u32 hpd_event_bits;
>@@ -1159,7 +1160,7 @@ static void i915_hotplug_work_func(struct work_struct *work)
> 	mutex_lock(&mode_config->mutex);
> 	DRM_DEBUG_KMS("running encoder hotplug functions\n");
> 
>-	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 
> 	hpd_event_bits = dev_priv->hpd_event_bits;
> 	dev_priv->hpd_event_bits = 0;
>@@ -1193,7 +1194,7 @@ static void i915_hotplug_work_func(struct work_struct *work)
> 				 msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
> 	}
> 
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> 
> 	list_for_each_entry(connector, &mode_config->connector_list, head) {
> 		intel_connector = to_intel_connector(connector);
>@@ -1253,17 +1254,15 @@ static void ironlake_rps_change_irq_handler(struct drm_device *dev)
> }
> 
> static void notify_ring(struct drm_device *dev,
>-			struct intel_engine_cs *ring)
>+			struct intel_engine_cs *engine)
> {
>-	if (!intel_ring_initialized(ring))
>+	if (!intel_engine_initialized(engine))
> 		return;
> 
>-	trace_i915_gem_request_complete(ring);
>-
>-	if (drm_core_check_feature(dev, DRIVER_MODESET))
>-		intel_notify_mmio_flip(ring);
>+	trace_i915_gem_ring_complete(engine);
>+	atomic_inc(&engine->interrupts);
> 
>-	wake_up_all(&ring->irq_queue);
>+	wake_up_all(&engine->irq_queue);
> 	i915_queue_hangcheck(dev);
> }
> 
>@@ -1322,10 +1321,10 @@ static u32 vlv_c0_residency(struct drm_i915_private *dev_priv,
>  * @dev_priv: DRM device private
>  *
>  */
>-static u32 vlv_calc_delay_from_C0_counters(struct drm_i915_private *dev_priv)
>+static int vlv_calc_delay_from_C0_counters(struct drm_i915_private *dev_priv)
> {
> 	u32 residency_C0_up = 0, residency_C0_down = 0;
>-	u8 new_delay, adj;
>+	int new_delay, adj;
> 
> 	dev_priv->rps.ei_interrupt_count++;
> 
>@@ -1488,7 +1487,6 @@ static void ivybridge_parity_work(struct work_struct *work)
> 	u32 error_status, row, bank, subbank;
> 	char *parity_event[6];
> 	uint32_t misccpctl;
>-	unsigned long flags;
> 	uint8_t slice = 0;
> 
> 	/* We must turn off DOP level clock gating to access the L3 registers.
>@@ -1547,9 +1545,9 @@ static void ivybridge_parity_work(struct work_struct *work)
> 
> out:
> 	WARN_ON(dev_priv->l3_parity.which_slice);
>-	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> 
> 	mutex_unlock(&dev_priv->dev->struct_mutex);
> }
>@@ -1581,9 +1579,9 @@ static void ilk_gt_irq_handler(struct drm_device *dev,
> {
> 	if (gt_iir &
> 	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
>-		notify_ring(dev, &dev_priv->ring[RCS]);
>+		notify_ring(dev, &dev_priv->engine[RCS]);
> 	if (gt_iir & ILK_BSD_USER_INTERRUPT)
>-		notify_ring(dev, &dev_priv->ring[VCS]);
>+		notify_ring(dev, &dev_priv->engine[VCS]);
> }
> 
> static void snb_gt_irq_handler(struct drm_device *dev,
>@@ -1593,11 +1591,11 @@ static void snb_gt_irq_handler(struct drm_device *dev,
> 
> 	if (gt_iir &
> 	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
>-		notify_ring(dev, &dev_priv->ring[RCS]);
>+		notify_ring(dev, &dev_priv->engine[RCS]);
> 	if (gt_iir & GT_BSD_USER_INTERRUPT)
>-		notify_ring(dev, &dev_priv->ring[VCS]);
>+		notify_ring(dev, &dev_priv->engine[VCS]);
> 	if (gt_iir & GT_BLT_USER_INTERRUPT)
>-		notify_ring(dev, &dev_priv->ring[BCS]);
>+		notify_ring(dev, &dev_priv->engine[BCS]);
> 
> 	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
> 		      GT_BSD_CS_ERROR_INTERRUPT |
>@@ -1627,6 +1625,7 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
> 				       struct drm_i915_private *dev_priv,
> 				       u32 master_ctl)
> {
>+	struct intel_engine_cs *engine;
> 	u32 rcs, bcs, vcs;
> 	uint32_t tmp = 0;
> 	irqreturn_t ret = IRQ_NONE;
>@@ -1636,12 +1635,16 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
> 		if (tmp) {
> 			I915_WRITE(GEN8_GT_IIR(0), tmp);
> 			ret = IRQ_HANDLED;
>+
> 			rcs = tmp >> GEN8_RCS_IRQ_SHIFT;
>-			bcs = tmp >> GEN8_BCS_IRQ_SHIFT;
>+			engine = &dev_priv->engine[RCS];
> 			if (rcs & GT_RENDER_USER_INTERRUPT)
>-				notify_ring(dev, &dev_priv->ring[RCS]);
>+				notify_ring(dev, engine);
>+
>+			bcs = tmp >> GEN8_BCS_IRQ_SHIFT;
>+			engine = &dev_priv->engine[BCS];
> 			if (bcs & GT_RENDER_USER_INTERRUPT)
>-				notify_ring(dev, &dev_priv->ring[BCS]);
>+				notify_ring(dev, engine);
> 		} else
> 			DRM_ERROR("The master control interrupt lied (GT0)!\n");
> 	}
>@@ -1651,12 +1654,16 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
> 		if (tmp) {
> 			I915_WRITE(GEN8_GT_IIR(1), tmp);
> 			ret = IRQ_HANDLED;
>+
> 			vcs = tmp >> GEN8_VCS1_IRQ_SHIFT;
>+			engine = &dev_priv->engine[VCS];
> 			if (vcs & GT_RENDER_USER_INTERRUPT)
>-				notify_ring(dev, &dev_priv->ring[VCS]);
>+				notify_ring(dev, engine);
>+
> 			vcs = tmp >> GEN8_VCS2_IRQ_SHIFT;
>+			engine = &dev_priv->engine[VCS2];
> 			if (vcs & GT_RENDER_USER_INTERRUPT)
>-				notify_ring(dev, &dev_priv->ring[VCS2]);
>+				notify_ring(dev, engine);
> 		} else
> 			DRM_ERROR("The master control interrupt lied (GT1)!\n");
> 	}
>@@ -1677,9 +1684,11 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
> 		if (tmp) {
> 			I915_WRITE(GEN8_GT_IIR(3), tmp);
> 			ret = IRQ_HANDLED;
>+
> 			vcs = tmp >> GEN8_VECS_IRQ_SHIFT;
>+			engine = &dev_priv->engine[VECS];
> 			if (vcs & GT_RENDER_USER_INTERRUPT)
>-				notify_ring(dev, &dev_priv->ring[VECS]);
>+				notify_ring(dev, engine);
> 		} else
> 			DRM_ERROR("The master control interrupt lied (GT3)!\n");
> 	}
>@@ -1772,7 +1781,9 @@ static inline void intel_hpd_irq_handler(struct drm_device *dev,
> 				long_hpd = (dig_hotplug_reg >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT;
> 			}
> 
>-			DRM_DEBUG_DRIVER("digital hpd port %d %d\n", port, long_hpd);
>+			DRM_DEBUG_DRIVER("digital hpd port %c - %s\n",
>+					 port_name(port),
>+					 long_hpd ? "long" : "short");
> 			/* for long HPD pulses we want to have the digital queue happen,
> 			   but we still want HPD storm detection to function. */
> 			if (long_hpd) {
>@@ -1843,7 +1854,9 @@ static inline void intel_hpd_irq_handler(struct drm_device *dev,
> 	if (queue_dig)
> 		queue_work(dev_priv->dp_wq, &dev_priv->dig_port_work);
> 	if (queue_hp)
>-		schedule_work(&dev_priv->hotplug_work);
>+		mod_delayed_work(system_wq,
>+				 &dev_priv->hotplug_work,
>+				 msecs_to_jiffies(50));
> }
> 
> static void gmbus_irq_handler(struct drm_device *dev)
>@@ -1956,6 +1969,27 @@ static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
> 				     res1, res2);
> }
> 
>+void gen8_flip_interrupt(struct drm_device *dev)
>+{
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+
>+	if (!dev_priv->rps.is_bdw_sw_turbo)
>+		return;
>+
>+	if(atomic_read(&dev_priv->rps.sw_turbo.flip_received)) {
>+		mod_timer(&dev_priv->rps.sw_turbo.flip_timer,
>+				usecs_to_jiffies(dev_priv->rps.sw_turbo.timeout) + jiffies);
>+	}
>+	else {
>+		dev_priv->rps.sw_turbo.flip_timer.expires =
>+				usecs_to_jiffies(dev_priv->rps.sw_turbo.timeout) + jiffies;
>+		add_timer(&dev_priv->rps.sw_turbo.flip_timer);
>+		atomic_set(&dev_priv->rps.sw_turbo.flip_received, true);
>+	}
>+
>+	bdw_software_turbo(dev);
>+}
>+
> /* The RPS events need forcewake, so we add them to a work queue and mask their
>  * IMR bits until the work is done. Other interrupts can be processed without
>  * the work queue. */
>@@ -1972,7 +2006,7 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
> 
> 	if (HAS_VEBOX(dev_priv->dev)) {
> 		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
>-			notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
>+			notify_ring(dev_priv->dev, &dev_priv->engine[VECS]);
> 
> 		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
> 			i915_handle_error(dev_priv->dev, false,
>@@ -1984,14 +2018,9 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
> 
> static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
> {
>-	struct intel_crtc *crtc;
>-
> 	if (!drm_handle_vblank(dev, pipe))
> 		return false;
> 
>-	crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe));
>-	wake_up(&crtc->vbl_wait);
>-
> 	return true;
> }
> 
>@@ -2002,7 +2031,7 @@ static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
> 	int pipe;
> 
> 	spin_lock(&dev_priv->irq_lock);
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		int reg;
> 		u32 mask, iir_bit = 0;
> 
>@@ -2047,9 +2076,10 @@ static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
> 	}
> 	spin_unlock(&dev_priv->irq_lock);
> 
>-	for_each_pipe(pipe) {
>-		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
>-			intel_pipe_handle_vblank(dev, pipe);
>+	for_each_pipe(dev_priv, pipe) {
>+		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
>+		    intel_pipe_handle_vblank(dev, pipe))
>+			intel_check_page_flip(dev, pipe);
> 
> 		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
> 			intel_prepare_page_flip(dev, pipe);
>@@ -2216,7 +2246,7 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
> 		DRM_ERROR("PCH poison interrupt\n");
> 
> 	if (pch_iir & SDE_FDI_MASK)
>-		for_each_pipe(pipe)
>+		for_each_pipe(dev_priv, pipe)
> 			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
> 					 pipe_name(pipe),
> 					 I915_READ(FDI_RX_IIR(pipe)));
>@@ -2247,7 +2277,7 @@ static void ivb_err_int_handler(struct drm_device *dev)
> 	if (err_int & ERR_INT_POISON)
> 		DRM_ERROR("Poison interrupt\n");
> 
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe)) {
> 			if (intel_set_cpu_fifo_underrun_reporting(dev, pipe,
> 								  false))
>@@ -2324,7 +2354,7 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
> 		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
> 
> 	if (pch_iir & SDE_FDI_MASK_CPT)
>-		for_each_pipe(pipe)
>+		for_each_pipe(dev_priv, pipe)
> 			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
> 					 pipe_name(pipe),
> 					 I915_READ(FDI_RX_IIR(pipe)));
>@@ -2347,9 +2377,10 @@ static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
> 	if (de_iir & DE_POISON)
> 		DRM_ERROR("Poison interrupt\n");
> 
>-	for_each_pipe(pipe) {
>-		if (de_iir & DE_PIPE_VBLANK(pipe))
>-			intel_pipe_handle_vblank(dev, pipe);
>+	for_each_pipe(dev_priv, pipe) {
>+		if (de_iir & DE_PIPE_VBLANK(pipe) &&
>+		    intel_pipe_handle_vblank(dev, pipe))
>+			intel_check_page_flip(dev, pipe);
> 
> 		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
> 			if (intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
>@@ -2397,9 +2428,10 @@ static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
> 	if (de_iir & DE_GSE_IVB)
> 		intel_opregion_asle_intr(dev);
> 
>-	for_each_pipe(pipe) {
>-		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
>-			intel_pipe_handle_vblank(dev, pipe);
>+	for_each_pipe(dev_priv, pipe) {
>+		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
>+		    intel_pipe_handle_vblank(dev, pipe))
>+			intel_check_page_flip(dev, pipe);
> 
> 		/* plane/pipes map 1:1 on ilk+ */
> 		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
>@@ -2544,7 +2576,7 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
> 			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
> 	}
> 
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		uint32_t pipe_iir;
> 
> 		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
>@@ -2554,8 +2586,9 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
> 		if (pipe_iir) {
> 			ret = IRQ_HANDLED;
> 			I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
>-			if (pipe_iir & GEN8_PIPE_VBLANK)
>-				intel_pipe_handle_vblank(dev, pipe);
>+			if (pipe_iir & GEN8_PIPE_VBLANK &&
>+			    intel_pipe_handle_vblank(dev, pipe))
>+				intel_check_page_flip(dev, pipe);
> 
> 			if (pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE) {
> 				intel_prepare_page_flip(dev, pipe);
>@@ -2606,7 +2639,7 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
> static void i915_error_wake_up(struct drm_i915_private *dev_priv,
> 			       bool reset_completed)
> {
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	int i;
> 
> 	/*
>@@ -2617,8 +2650,8 @@ static void i915_error_wake_up(struct drm_i915_private *dev_priv,
> 	 */
> 
> 	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
>-	for_each_ring(ring, dev_priv, i)
>-		wake_up_all(&ring->irq_queue);
>+	for_each_engine(engine, dev_priv, i)
>+		wake_up_all(&engine->irq_queue);
> 
> 	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
> 	wake_up_all(&dev_priv->pending_flip_queue);
>@@ -2662,7 +2695,7 @@ static void i915_error_work_func(struct work_struct *work)
> 	 * the reset in-progress bit is only ever set by code outside of this
> 	 * work we don't need to worry about any other races.
> 	 */
>-	if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
>+	if (i915_recovery_pending(error)) {
> 		DRM_DEBUG_DRIVER("resetting chip\n");
> 		kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
> 				   reset_event);
>@@ -2698,9 +2731,7 @@ static void i915_error_work_func(struct work_struct *work)
> 			 * updates before
> 			 * the counter increment.
> 			 */
>-			smp_mb__before_atomic();
>-			atomic_inc(&dev_priv->gpu_error.reset_counter);
>-
>+			smp_mb__after_atomic();
> 			kobject_uevent_env(&dev->primary->kdev->kobj,
> 					   KOBJ_CHANGE, reset_done_event);
> 		} else {
>@@ -2763,7 +2794,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
> 
> 	if (eir & I915_ERROR_MEMORY_REFRESH) {
> 		pr_err("memory refresh error:\n");
>-		for_each_pipe(pipe)
>+		for_each_pipe(dev_priv, pipe)
> 			pr_err("pipe %c stat: 0x%08x\n",
> 			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
> 		/* pipestat has already been acked */
>@@ -2860,52 +2891,6 @@ void i915_handle_error(struct drm_device *dev, bool wedged,
> 	schedule_work(&dev_priv->gpu_error.work);
> }
> 
>-static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
>-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>-	struct drm_i915_gem_object *obj;
>-	struct intel_unpin_work *work;
>-	unsigned long flags;
>-	bool stall_detected;
>-
>-	/* Ignore early vblank irqs */
>-	if (intel_crtc == NULL)
>-		return;
>-
>-	spin_lock_irqsave(&dev->event_lock, flags);
>-	work = intel_crtc->unpin_work;
>-
>-	if (work == NULL ||
>-	    atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
>-	    !work->enable_stall_check) {
>-		/* Either the pending flip IRQ arrived, or we're too early. Don't check */
>-		spin_unlock_irqrestore(&dev->event_lock, flags);
>-		return;
>-	}
>-
>-	/* Potential stall - if we see that the flip has happened, assume a missed interrupt */
>-	obj = work->pending_flip_obj;
>-	if (INTEL_INFO(dev)->gen >= 4) {
>-		int dspsurf = DSPSURF(intel_crtc->plane);
>-		stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
>-					i915_gem_obj_ggtt_offset(obj);
>-	} else {
>-		int dspaddr = DSPADDR(intel_crtc->plane);
>-		stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) +
>-							crtc->y * crtc->primary->fb->pitches[0] +
>-							crtc->x * crtc->primary->fb->bits_per_pixel/8);
>-	}
>-
>-	spin_unlock_irqrestore(&dev->event_lock, flags);
>-
>-	if (stall_detected) {
>-		DRM_DEBUG_DRIVER("Pageflip stall detected\n");
>-		intel_prepare_page_flip(dev, intel_crtc->plane);
>-	}
>-}
>-
> /* Called from drm generic code, passed 'crtc' which
>  * we use as a pipe index
>  */
>@@ -3031,24 +3016,28 @@ static void gen8_disable_vblank(struct drm_device *dev, int pipe)
> 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
> }
> 
>-static u32
>-ring_last_seqno(struct intel_engine_cs *ring)
>-{
>-	return list_entry(ring->request_list.prev,
>-			  struct drm_i915_gem_request, list)->seqno;
>-}
>-
> static bool
>-ring_idle(struct intel_engine_cs *ring, u32 seqno)
>+engine_idle(struct intel_engine_cs *engine)
> {
>-	return (list_empty(&ring->request_list) ||
>-		i915_seqno_passed(seqno, ring_last_seqno(ring)));
>+	bool ret = true;
>+
>+	spin_lock(&engine->lock);
>+	if (engine->last_request) {
>+		/* poke to make sure we retire before we wake up again */
>+		queue_delayed_work(engine->i915->wq,
>+				   &engine->i915->mm.retire_work,
>+				   round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES/2));
>+		ret = __i915_request_complete__wa(engine->last_request);
>+	}
>+	spin_unlock(&engine->lock);
>+
>+	return ret;
> }
> 
> static bool
>-ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
>+ipehr_is_semaphore_wait(struct drm_i915_private *i915, u32 ipehr)
> {
>-	if (INTEL_INFO(dev)->gen >= 8) {
>+	if (INTEL_INFO(i915)->gen >= 8) {
> 		return (ipehr >> 23) == 0x1c;
> 	} else {
> 		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
>@@ -3058,48 +3047,54 @@ ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
> }
> 
> static struct intel_engine_cs *
>-semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
>+semaphore_wait_to_signaller_engine(struct intel_engine_cs *engine, u32 ipehr, u64 offset)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	struct intel_engine_cs *signaller;
> 	int i;
> 
> 	if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
>-		for_each_ring(signaller, dev_priv, i) {
>-			if (ring == signaller)
>+		for_each_engine(signaller, dev_priv, i) {
>+			if (engine == signaller)
> 				continue;
> 
>-			if (offset == signaller->semaphore.signal_ggtt[ring->id])
>+			if (offset == GEN8_SEMAPHORE_OFFSET(dev_priv, signaller->id, engine->id))
> 				return signaller;
> 		}
> 	} else {
> 		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
> 
>-		for_each_ring(signaller, dev_priv, i) {
>-			if(ring == signaller)
>+		for_each_engine(signaller, dev_priv, i) {
>+			if(engine == signaller)
> 				continue;
> 
>-			if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
>+			if (sync_bits == signaller->semaphore.mbox.wait[engine->id])
> 				return signaller;
> 		}
> 	}
> 
> 	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
>-		  ring->id, ipehr, offset);
>+		  engine->id, ipehr, offset);
> 
> 	return NULL;
> }
> 
> static struct intel_engine_cs *
>-semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
>+semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
>+	struct intel_ringbuffer *ring;
> 	u32 cmd, ipehr, head;
> 	u64 offset = 0;
> 	int i, backwards;
> 
>-	ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
>-	if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
>+	ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
>+	if (!ipehr_is_semaphore_wait(engine->i915, ipehr))
>+		return NULL;
>+
>+	/* XXX execlists */
>+	ring = engine->default_context->ring[RCS].ring;
>+	if (ring == NULL)
> 		return NULL;
> 
> 	/*
>@@ -3110,19 +3105,19 @@ semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
> 	 * point at at batch, and semaphores are always emitted into the
> 	 * ringbuffer itself.
> 	 */
>-	head = I915_READ_HEAD(ring) & HEAD_ADDR;
>-	backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
>+	head = I915_READ_HEAD(engine) & HEAD_ADDR;
>+	backwards = (INTEL_INFO(dev_priv)->gen >= 8) ? 5 : 4;
> 
> 	for (i = backwards; i; --i) {
> 		/*
> 		 * Be paranoid and presume the hw has gone off into the wild -
>-		 * our ring is smaller than what the hardware (and hence
>+		 * our engine is smaller than what the hardware (and hence
> 		 * HEAD_ADDR) allows. Also handles wrap-around.
> 		 */
>-		head &= ring->buffer->size - 1;
>+		head &= ring->size - 1;
> 
> 		/* This here seems to blow up */
>-		cmd = ioread32(ring->buffer->virtual_start + head);
>+		cmd = ioread32(ring->virtual_start + head);
> 		if (cmd == ipehr)
> 			break;
> 
>@@ -3132,32 +3127,37 @@ semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
> 	if (!i)
> 		return NULL;
> 
>-	*seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
>-	if (INTEL_INFO(ring->dev)->gen >= 8) {
>-		offset = ioread32(ring->buffer->virtual_start + head + 12);
>+	*seqno = ioread32(ring->virtual_start + head + 4) + 1;
>+	if (INTEL_INFO(dev_priv)->gen >= 8) {
>+		offset = ioread32(ring->virtual_start + head + 12);
> 		offset <<= 32;
>-		offset = ioread32(ring->buffer->virtual_start + head + 8);
>+		offset = ioread32(ring->virtual_start + head + 8);
> 	}
>-	return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
>+	return semaphore_wait_to_signaller_engine(engine, ipehr, offset);
> }
> 
>-static int semaphore_passed(struct intel_engine_cs *ring)
>+static int semaphore_passed(struct intel_engine_cs *engine)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	struct intel_engine_cs *signaller;
>+	struct i915_gem_request *rq;
> 	u32 seqno;
> 
>-	ring->hangcheck.deadlock++;
>+	engine->hangcheck.deadlock++;
>+
>+	if (engine->semaphore.wait == NULL)
>+		return -1;
> 
>-	signaller = semaphore_waits_for(ring, &seqno);
>+	signaller = semaphore_waits_for(engine, &seqno);
> 	if (signaller == NULL)
> 		return -1;
> 
> 	/* Prevent pathological recursion due to driver bugs */
>-	if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
>+	if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES)
> 		return -1;
> 
>-	if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
>+	rq = intel_engine_seqno_to_request(engine, seqno);
>+	if (rq == NULL || i915_request_complete(rq))
> 		return 1;
> 
> 	/* cursory check for an unkickable deadlock */
>@@ -3170,30 +3170,29 @@ static int semaphore_passed(struct intel_engine_cs *ring)
> 
> static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
> {
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	int i;
> 
>-	for_each_ring(ring, dev_priv, i)
>-		ring->hangcheck.deadlock = 0;
>+	for_each_engine(engine, dev_priv, i)
>+		engine->hangcheck.deadlock = 0;
> }
> 
>-static enum intel_ring_hangcheck_action
>-ring_stuck(struct intel_engine_cs *ring, u64 acthd)
>+static enum intel_engine_hangcheck_action
>+engine_stuck(struct intel_engine_cs *engine, u64 acthd)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	u32 tmp;
> 
>-	if (acthd != ring->hangcheck.acthd) {
>-		if (acthd > ring->hangcheck.max_acthd) {
>-			ring->hangcheck.max_acthd = acthd;
>+	if (acthd != engine->hangcheck.acthd) {
>+		if (acthd > engine->hangcheck.max_acthd) {
>+			engine->hangcheck.max_acthd = acthd;
> 			return HANGCHECK_ACTIVE;
> 		}
> 
> 		return HANGCHECK_ACTIVE_LOOP;
> 	}
> 
>-	if (IS_GEN2(dev))
>+	if (IS_GEN2(dev_priv))
> 		return HANGCHECK_HUNG;
> 
> 	/* Is the chip hanging on a WAIT_FOR_EVENT?
>@@ -3201,24 +3200,24 @@ ring_stuck(struct intel_engine_cs *ring, u64 acthd)
> 	 * and break the hang. This should work on
> 	 * all but the second generation chipsets.
> 	 */
>-	tmp = I915_READ_CTL(ring);
>+	tmp = I915_READ_CTL(engine);
> 	if (tmp & RING_WAIT) {
>-		i915_handle_error(dev, false,
>+		i915_handle_error(dev_priv->dev, false,
> 				  "Kicking stuck wait on %s",
>-				  ring->name);
>-		I915_WRITE_CTL(ring, tmp);
>+				  engine->name);
>+		I915_WRITE_CTL(engine, tmp);
> 		return HANGCHECK_KICK;
> 	}
> 
>-	if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
>-		switch (semaphore_passed(ring)) {
>+	if (INTEL_INFO(dev_priv)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
>+		switch (semaphore_passed(engine)) {
> 		default:
> 			return HANGCHECK_HUNG;
> 		case 1:
>-			i915_handle_error(dev, false,
>+			i915_handle_error(dev_priv->dev, false,
> 					  "Kicking stuck semaphore on %s",
>-					  ring->name);
>-			I915_WRITE_CTL(ring, tmp);
>+					  engine->name);
>+			I915_WRITE_CTL(engine, tmp);
> 			return HANGCHECK_KICK;
> 		case 0:
> 			return HANGCHECK_WAIT;
>@@ -3230,7 +3229,7 @@ ring_stuck(struct intel_engine_cs *ring, u64 acthd)
> 
> /**
>  * This is called when the chip hasn't reported back with completed
>- * batchbuffers in a long time. We keep track per ring seqno progress and
>+ * batchbuffers in a long time. We keep track per engine seqno progress and
>  * if there are no progress, hangcheck score for that ring is increased.
>  * Further, acthd is inspected to see if the ring is stuck. On stuck case
>  * we kick the ring. If we see no progress on three subsequent calls
>@@ -3238,12 +3237,11 @@ ring_stuck(struct intel_engine_cs *ring, u64 acthd)
>  */
> static void i915_hangcheck_elapsed(unsigned long data)
> {
>-	struct drm_device *dev = (struct drm_device *)data;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct drm_i915_private *dev_priv = (struct drm_i915_private *)data;
>+	struct intel_engine_cs *engine;
> 	int i;
> 	int busy_count = 0, rings_hung = 0;
>-	bool stuck[I915_NUM_RINGS] = { 0 };
>+	bool stuck[I915_NUM_ENGINES] = { 0 };
> #define BUSY 1
> #define KICK 5
> #define HUNG 20
>@@ -3251,104 +3249,108 @@ static void i915_hangcheck_elapsed(unsigned long data)
> 	if (!i915.enable_hangcheck)
> 		return;
> 
>-	for_each_ring(ring, dev_priv, i) {
>+	for_each_engine(engine, dev_priv, i) {
> 		u64 acthd;
> 		u32 seqno;
>+		u32 interrupts;
> 		bool busy = true;
> 
> 		semaphore_clear_deadlocks(dev_priv);
> 
>-		seqno = ring->get_seqno(ring, false);
>-		acthd = intel_ring_get_active_head(ring);
>-
>-		if (ring->hangcheck.seqno == seqno) {
>-			if (ring_idle(ring, seqno)) {
>-				ring->hangcheck.action = HANGCHECK_IDLE;
>-
>-				if (waitqueue_active(&ring->irq_queue)) {
>-					/* Issue a wake-up to catch stuck h/w. */
>-					if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
>-						if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
>-							DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
>-								  ring->name);
>-						else
>-							DRM_INFO("Fake missed irq on %s\n",
>-								 ring->name);
>-						wake_up_all(&ring->irq_queue);
>-					}
>-					/* Safeguard against driver failure */
>-					ring->hangcheck.score += BUSY;
>-				} else
>-					busy = false;
>+		acthd = intel_engine_get_active_head(engine);
>+		seqno = engine->get_seqno(engine);
>+		interrupts = atomic_read(&engine->interrupts);
>+
>+		if (engine_idle(engine)) {
>+			if (waitqueue_active(&engine->irq_queue)) {
>+				/* Issue a wake-up to catch stuck h/w. */
>+				if (engine->hangcheck.action == HANGCHECK_IDLE_WAITERS &&
>+						engine->hangcheck.interrupts == interrupts &&
>+						!test_and_set_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings)) {
>+					if (!(dev_priv->gpu_error.test_irq_rings & intel_engine_flag(engine)))
>+						DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
>+								engine->name);
>+					else
>+						DRM_INFO("Fake missed irq on %s\n",
>+								engine->name);
>+					wake_up_all(&engine->irq_queue);
>+				}
>+
>+				/* Safeguard against driver failure */
>+				engine->hangcheck.score += BUSY;
>+				engine->hangcheck.action = HANGCHECK_IDLE_WAITERS;
> 			} else {
>-				/* We always increment the hangcheck score
>-				 * if the ring is busy and still processing
>-				 * the same request, so that no single request
>-				 * can run indefinitely (such as a chain of
>-				 * batches). The only time we do not increment
>-				 * the hangcheck score on this ring, if this
>-				 * ring is in a legitimate wait for another
>-				 * ring. In that case the waiting ring is a
>-				 * victim and we want to be sure we catch the
>-				 * right culprit. Then every time we do kick
>-				 * the ring, add a small increment to the
>-				 * score so that we can catch a batch that is
>-				 * being repeatedly kicked and so responsible
>-				 * for stalling the machine.
>-				 */
>-				ring->hangcheck.action = ring_stuck(ring,
>-								    acthd);
>-
>-				switch (ring->hangcheck.action) {
>+				busy = false;
>+				engine->hangcheck.action = HANGCHECK_IDLE;
>+			}
>+		} else if (engine->hangcheck.seqno == seqno) {
>+			/* We always increment the hangcheck score
>+			 * if the ring is busy and still processing
>+			 * the same request, so that no single request
>+			 * can run indefinitely (such as a chain of
>+			 * batches). The only time we do not increment
>+			 * the hangcheck score on this ring, if this
>+			 * ring is in a legitimate wait for another
>+			 * ring. In that case the waiting ring is a
>+			 * victim and we want to be sure we catch the
>+			 * right culprit. Then every time we do kick
>+			 * the ring, add a small increment to the
>+			 * score so that we can catch a batch that is
>+			 * being repeatedly kicked and so responsible
>+			 * for stalling the machine.
>+			 */
>+			engine->hangcheck.action = engine_stuck(engine, acthd);
>+			switch (engine->hangcheck.action) {
> 				case HANGCHECK_IDLE:
>+				case HANGCHECK_IDLE_WAITERS:
> 				case HANGCHECK_WAIT:
> 				case HANGCHECK_ACTIVE:
> 					break;
> 				case HANGCHECK_ACTIVE_LOOP:
>-					ring->hangcheck.score += BUSY;
>+					engine->hangcheck.score += BUSY;
> 					break;
> 				case HANGCHECK_KICK:
>-					ring->hangcheck.score += KICK;
>+					engine->hangcheck.score += KICK;
> 					break;
> 				case HANGCHECK_HUNG:
>-					ring->hangcheck.score += HUNG;
>+					engine->hangcheck.score += HUNG;
> 					stuck[i] = true;
> 					break;
>-				}
> 			}
> 		} else {
>-			ring->hangcheck.action = HANGCHECK_ACTIVE;
>+			engine->hangcheck.action = HANGCHECK_ACTIVE;
> 
> 			/* Gradually reduce the count so that we catch DoS
> 			 * attempts across multiple batches.
> 			 */
>-			if (ring->hangcheck.score > 0)
>-				ring->hangcheck.score--;
>+			if (engine->hangcheck.score > 0)
>+				engine->hangcheck.score--;
> 
>-			ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
>+			engine->hangcheck.acthd = engine->hangcheck.max_acthd = 0;
> 		}
> 
>-		ring->hangcheck.seqno = seqno;
>-		ring->hangcheck.acthd = acthd;
>+		engine->hangcheck.interrupts = interrupts;
>+		engine->hangcheck.seqno = seqno;
>+		engine->hangcheck.acthd = acthd;
> 		busy_count += busy;
> 	}
> 
>-	for_each_ring(ring, dev_priv, i) {
>-		if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
>+	for_each_engine(engine, dev_priv, i) {
>+		if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
> 			DRM_INFO("%s on %s\n",
> 				 stuck[i] ? "stuck" : "no progress",
>-				 ring->name);
>+				 engine->name);
> 			rings_hung++;
> 		}
> 	}
> 
> 	if (rings_hung)
>-		return i915_handle_error(dev, true, "Ring hung");
>+		return i915_handle_error(dev_priv->dev, true, "Ring hung");
> 
> 	if (busy_count)
> 		/* Reset timer case chip hangs without another request
> 		 * being added */
>-		i915_queue_hangcheck(dev);
>+		i915_queue_hangcheck(dev_priv->dev);
> }
> 
> void i915_queue_hangcheck(struct drm_device *dev)
>@@ -3441,7 +3443,7 @@ static void valleyview_irq_preinstall(struct drm_device *dev)
> 
> 	I915_WRITE(PORT_HOTPLUG_EN, 0);
> 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		I915_WRITE(PIPESTAT(pipe), 0xffff);
> 	I915_WRITE(VLV_IIR, 0xffffffff);
> 	I915_WRITE(VLV_IMR, 0xffffffff);
>@@ -3467,7 +3469,7 @@ static void gen8_irq_reset(struct drm_device *dev)
> 
> 	gen8_gt_irq_reset(dev_priv);
> 
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		if (intel_display_power_enabled(dev_priv,
> 						POWER_DOMAIN_PIPE(pipe)))
> 			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
>@@ -3481,14 +3483,12 @@ static void gen8_irq_reset(struct drm_device *dev)
> 
> void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv)
> {
>-	unsigned long irqflags;
>-
>-	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B, dev_priv->de_irq_mask[PIPE_B],
> 			  ~dev_priv->de_irq_mask[PIPE_B]);
> 	GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C, dev_priv->de_irq_mask[PIPE_C],
> 			  ~dev_priv->de_irq_mask[PIPE_C]);
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> }
> 
> static void cherryview_irq_preinstall(struct drm_device *dev)
>@@ -3510,7 +3510,7 @@ static void cherryview_irq_preinstall(struct drm_device *dev)
> 	I915_WRITE(PORT_HOTPLUG_EN, 0);
> 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
> 
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		I915_WRITE(PIPESTAT(pipe), 0xffff);
> 
> 	I915_WRITE(VLV_IMR, 0xffffffff);
>@@ -3522,18 +3522,17 @@ static void cherryview_irq_preinstall(struct drm_device *dev)
> static void ibx_hpd_irq_setup(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_mode_config *mode_config = &dev->mode_config;
> 	struct intel_encoder *intel_encoder;
> 	u32 hotplug_irqs, hotplug, enabled_irqs = 0;
> 
> 	if (HAS_PCH_IBX(dev)) {
> 		hotplug_irqs = SDE_HOTPLUG_MASK;
>-		list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
>+		for_each_intel_encoder(dev, intel_encoder)
> 			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
> 				enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
> 	} else {
> 		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
>-		list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
>+		for_each_intel_encoder(dev, intel_encoder)
> 			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
> 				enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
> 	}
>@@ -3608,7 +3607,6 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev)
> 
> static int ironlake_irq_postinstall(struct drm_device *dev)
> {
>-	unsigned long irqflags;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	u32 display_mask, extra_mask;
> 
>@@ -3647,9 +3645,9 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
> 		 * spinlocking not required here for correctness since interrupt
> 		 * setup is guaranteed to run in single-threaded context. But we
> 		 * need it to make the assert_spin_locked happy. */
>-		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+		spin_lock_irq(&dev_priv->irq_lock);
> 		ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
>-		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+		spin_unlock_irq(&dev_priv->irq_lock);
> 	}
> 
> 	return 0;
>@@ -3725,7 +3723,7 @@ void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
> 
> 	dev_priv->display_irqs_enabled = true;
> 
>-	if (dev_priv->dev->irq_enabled)
>+	if (intel_irqs_enabled(dev_priv))
> 		valleyview_display_irqs_install(dev_priv);
> }
> 
>@@ -3738,14 +3736,13 @@ void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
> 
> 	dev_priv->display_irqs_enabled = false;
> 
>-	if (dev_priv->dev->irq_enabled)
>+	if (intel_irqs_enabled(dev_priv))
> 		valleyview_display_irqs_uninstall(dev_priv);
> }
> 
> static int valleyview_irq_postinstall(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	unsigned long irqflags;
> 
> 	dev_priv->irq_mask = ~0;
> 
>@@ -3759,10 +3756,10 @@ static int valleyview_irq_postinstall(struct drm_device *dev)
> 
> 	/* Interrupt setup is already guaranteed to be single-threaded, this is
> 	 * just to make the assert_spin_locked check happy. */
>-	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 	if (dev_priv->display_irqs_enabled)
> 		valleyview_display_irqs_install(dev_priv);
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> 
> 	I915_WRITE(VLV_IIR, 0xffffffff);
> 	I915_WRITE(VLV_IIR, 0xffffffff);
>@@ -3782,28 +3779,31 @@ static int valleyview_irq_postinstall(struct drm_device *dev)
> 
> static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
> {
>-	int i;
>-
> 	/* These are interrupts we'll toggle with the ring mask register */
> 	uint32_t gt_interrupts[] = {
> 		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
>+			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
> 			GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
>-			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
>+			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
>+			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
> 		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
>-			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
>+			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
>+			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
>+			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
> 		0,
>-		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT
>+		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
>+			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
> 		};
> 
>-	for (i = 0; i < ARRAY_SIZE(gt_interrupts); i++)
>-		GEN8_IRQ_INIT_NDX(GT, i, ~gt_interrupts[i], gt_interrupts[i]);
>-
> 	dev_priv->pm_irq_mask = 0xffffffff;
>+	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
>+	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
>+	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, dev_priv->pm_rps_events);
>+	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
> }
> 
> static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
> {
>-	struct drm_device *dev = dev_priv->dev;
> 	uint32_t de_pipe_masked = GEN8_PIPE_PRIMARY_FLIP_DONE |
> 		GEN8_PIPE_CDCLK_CRC_DONE |
> 		GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
>@@ -3814,7 +3814,7 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
> 	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
> 	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
> 
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		if (intel_display_power_enabled(dev_priv,
> 				POWER_DOMAIN_PIPE(pipe)))
> 			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
>@@ -3850,7 +3850,6 @@ static int cherryview_irq_postinstall(struct drm_device *dev)
> 		I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
> 	u32 pipestat_enable = PLANE_FLIP_DONE_INT_STATUS_VLV |
> 		PIPE_CRC_DONE_INTERRUPT_STATUS;
>-	unsigned long irqflags;
> 	int pipe;
> 
> 	/*
>@@ -3859,14 +3858,14 @@ static int cherryview_irq_postinstall(struct drm_device *dev)
> 	 */
> 	dev_priv->irq_mask = ~enable_mask;
> 
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		I915_WRITE(PIPESTAT(pipe), 0xffff);
> 
>-	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		i915_enable_pipestat(dev_priv, pipe, pipestat_enable);
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> 
> 	I915_WRITE(VLV_IIR, 0xffffffff);
> 	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
>@@ -3893,7 +3892,6 @@ static void gen8_irq_uninstall(struct drm_device *dev)
> static void valleyview_irq_uninstall(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	unsigned long irqflags;
> 	int pipe;
> 
> 	if (!dev_priv)
>@@ -3901,17 +3899,19 @@ static void valleyview_irq_uninstall(struct drm_device *dev)
> 
> 	I915_WRITE(VLV_MASTER_IER, 0);
> 
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		I915_WRITE(PIPESTAT(pipe), 0xffff);
> 
> 	I915_WRITE(HWSTAM, 0xffffffff);
> 	I915_WRITE(PORT_HOTPLUG_EN, 0);
> 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
> 
>-	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+	/* Interrupt setup is already guaranteed to be single-threaded, this is
>+	 * just to make the assert_spin_locked check happy. */
>+	spin_lock_irq(&dev_priv->irq_lock);
> 	if (dev_priv->display_irqs_enabled)
> 		valleyview_display_irqs_uninstall(dev_priv);
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> 
> 	dev_priv->irq_mask = 0;
> 
>@@ -3963,7 +3963,7 @@ do {							\
> 	I915_WRITE(PORT_HOTPLUG_EN, 0);
> 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
> 
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		I915_WRITE(PIPESTAT(pipe), 0xffff);
> 
> 	I915_WRITE(VLV_IMR, 0xffffffff);
>@@ -3987,7 +3987,7 @@ static void i8xx_irq_preinstall(struct drm_device * dev)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	int pipe;
> 
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		I915_WRITE(PIPESTAT(pipe), 0);
> 	I915_WRITE16(IMR, 0xffff);
> 	I915_WRITE16(IER, 0x0);
>@@ -3997,7 +3997,6 @@ static void i8xx_irq_preinstall(struct drm_device * dev)
> static int i8xx_irq_postinstall(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	unsigned long irqflags;
> 
> 	I915_WRITE16(EMR,
> 		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
>@@ -4020,10 +4019,10 @@ static int i8xx_irq_postinstall(struct drm_device *dev)
> 
> 	/* Interrupt setup is already guaranteed to be single-threaded, this is
> 	 * just to make the assert_spin_locked check happy. */
>-	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
> 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> 
> 	return 0;
> }
>@@ -4041,7 +4040,7 @@ static bool i8xx_handle_vblank(struct drm_device *dev,
> 		return false;
> 
> 	if ((iir & flip_pending) == 0)
>-		return false;
>+		goto check_page_flip;
> 
> 	intel_prepare_page_flip(dev, plane);
> 
>@@ -4052,11 +4051,14 @@ static bool i8xx_handle_vblank(struct drm_device *dev,
> 	 * an interrupt per se, we watch for the change at vblank.
> 	 */
> 	if (I915_READ16(ISR) & flip_pending)
>-		return false;
>+		goto check_page_flip;
> 
> 	intel_finish_page_flip(dev, pipe);
>-
> 	return true;
>+
>+check_page_flip:
>+	intel_check_page_flip(dev, pipe);
>+	return false;
> }
> 
> static irqreturn_t i8xx_irq_handler(int irq, void *arg)
>@@ -4065,7 +4067,6 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	u16 iir, new_iir;
> 	u32 pipe_stats[2];
>-	unsigned long irqflags;
> 	int pipe;
> 	u16 flip_mask =
> 		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
>@@ -4081,13 +4082,13 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
> 		 * It doesn't set the bit in iir again, but it still produces
> 		 * interrupts (for non-MSI).
> 		 */
>-		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+		spin_lock(&dev_priv->irq_lock);
> 		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
> 			i915_handle_error(dev, false,
> 					  "Command parser error, iir 0x%08x",
> 					  iir);
> 
>-		for_each_pipe(pipe) {
>+		for_each_pipe(dev_priv, pipe) {
> 			int reg = PIPESTAT(pipe);
> 			pipe_stats[pipe] = I915_READ(reg);
> 
>@@ -4097,17 +4098,15 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
> 			if (pipe_stats[pipe] & 0x8000ffff)
> 				I915_WRITE(reg, pipe_stats[pipe]);
> 		}
>-		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+		spin_unlock(&dev_priv->irq_lock);
> 
> 		I915_WRITE16(IIR, iir & ~flip_mask);
> 		new_iir = I915_READ16(IIR); /* Flush posted writes */
> 
>-		i915_update_dri1_breadcrumb(dev);
>-
> 		if (iir & I915_USER_INTERRUPT)
>-			notify_ring(dev, &dev_priv->ring[RCS]);
>+			notify_ring(dev, &dev_priv->engine[RCS]);
> 
>-		for_each_pipe(pipe) {
>+		for_each_pipe(dev_priv, pipe) {
> 			int plane = pipe;
> 			if (HAS_FBC(dev))
> 				plane = !plane;
>@@ -4135,7 +4134,7 @@ static void i8xx_irq_uninstall(struct drm_device * dev)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	int pipe;
> 
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		/* Clear enable bits; then clear status bits */
> 		I915_WRITE(PIPESTAT(pipe), 0);
> 		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
>@@ -4156,7 +4155,7 @@ static void i915_irq_preinstall(struct drm_device * dev)
> 	}
> 
> 	I915_WRITE16(HWSTAM, 0xeffe);
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		I915_WRITE(PIPESTAT(pipe), 0);
> 	I915_WRITE(IMR, 0xffffffff);
> 	I915_WRITE(IER, 0x0);
>@@ -4167,7 +4166,6 @@ static int i915_irq_postinstall(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	u32 enable_mask;
>-	unsigned long irqflags;
> 
> 	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
> 
>@@ -4205,10 +4203,10 @@ static int i915_irq_postinstall(struct drm_device *dev)
> 
> 	/* Interrupt setup is already guaranteed to be single-threaded, this is
> 	 * just to make the assert_spin_locked check happy. */
>-	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
> 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> 
> 	return 0;
> }
>@@ -4226,7 +4224,7 @@ static bool i915_handle_vblank(struct drm_device *dev,
> 		return false;
> 
> 	if ((iir & flip_pending) == 0)
>-		return false;
>+		goto check_page_flip;
> 
> 	intel_prepare_page_flip(dev, plane);
> 
>@@ -4237,11 +4235,14 @@ static bool i915_handle_vblank(struct drm_device *dev,
> 	 * an interrupt per se, we watch for the change at vblank.
> 	 */
> 	if (I915_READ(ISR) & flip_pending)
>-		return false;
>+		goto check_page_flip;
> 
> 	intel_finish_page_flip(dev, pipe);
>-
> 	return true;
>+
>+check_page_flip:
>+	intel_check_page_flip(dev, pipe);
>+	return false;
> }
> 
> static irqreturn_t i915_irq_handler(int irq, void *arg)
>@@ -4249,7 +4250,6 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
> 	struct drm_device *dev = arg;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
>-	unsigned long irqflags;
> 	u32 flip_mask =
> 		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
> 		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
>@@ -4265,13 +4265,13 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
> 		 * It doesn't set the bit in iir again, but it still produces
> 		 * interrupts (for non-MSI).
> 		 */
>-		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+		spin_lock(&dev_priv->irq_lock);
> 		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
> 			i915_handle_error(dev, false,
> 					  "Command parser error, iir 0x%08x",
> 					  iir);
> 
>-		for_each_pipe(pipe) {
>+		for_each_pipe(dev_priv, pipe) {
> 			int reg = PIPESTAT(pipe);
> 			pipe_stats[pipe] = I915_READ(reg);
> 
>@@ -4281,7 +4281,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
> 				irq_received = true;
> 			}
> 		}
>-		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+		spin_unlock(&dev_priv->irq_lock);
> 
> 		if (!irq_received)
> 			break;
>@@ -4295,9 +4295,9 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
> 		new_iir = I915_READ(IIR); /* Flush posted writes */
> 
> 		if (iir & I915_USER_INTERRUPT)
>-			notify_ring(dev, &dev_priv->ring[RCS]);
>+			notify_ring(dev, &dev_priv->engine[RCS]);
> 
>-		for_each_pipe(pipe) {
>+		for_each_pipe(dev_priv, pipe) {
> 			int plane = pipe;
> 			if (HAS_FBC(dev))
> 				plane = !plane;
>@@ -4339,8 +4339,6 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
> 		iir = new_iir;
> 	} while (iir & ~flip_mask);
> 
>-	i915_update_dri1_breadcrumb(dev);
>-
> 	return ret;
> }
> 
>@@ -4355,7 +4353,7 @@ static void i915_irq_uninstall(struct drm_device * dev)
> 	}
> 
> 	I915_WRITE16(HWSTAM, 0xffff);
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		/* Clear enable bits; then clear status bits */
> 		I915_WRITE(PIPESTAT(pipe), 0);
> 		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
>@@ -4375,7 +4373,7 @@ static void i965_irq_preinstall(struct drm_device * dev)
> 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
> 
> 	I915_WRITE(HWSTAM, 0xeffe);
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		I915_WRITE(PIPESTAT(pipe), 0);
> 	I915_WRITE(IMR, 0xffffffff);
> 	I915_WRITE(IER, 0x0);
>@@ -4387,7 +4385,6 @@ static int i965_irq_postinstall(struct drm_device *dev)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	u32 enable_mask;
> 	u32 error_mask;
>-	unsigned long irqflags;
> 
> 	/* Unmask the interrupts that we always want on. */
> 	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
>@@ -4408,11 +4405,11 @@ static int i965_irq_postinstall(struct drm_device *dev)
> 
> 	/* Interrupt setup is already guaranteed to be single-threaded, this is
> 	 * just to make the assert_spin_locked check happy. */
>-	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
> 	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
> 	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> 
> 	/*
> 	 * Enable some error detection, note the instruction error mask
>@@ -4444,7 +4441,6 @@ static int i965_irq_postinstall(struct drm_device *dev)
> static void i915_hpd_irq_setup(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_mode_config *mode_config = &dev->mode_config;
> 	struct intel_encoder *intel_encoder;
> 	u32 hotplug_en;
> 
>@@ -4455,7 +4451,7 @@ static void i915_hpd_irq_setup(struct drm_device *dev)
> 		hotplug_en &= ~HOTPLUG_INT_EN_MASK;
> 		/* Note HDMI and DP share hotplug bits */
> 		/* enable bits are the same for all generations */
>-		list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
>+		for_each_intel_encoder(dev, intel_encoder)
> 			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
> 				hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
> 		/* Programming the CRT detection parameters tends
>@@ -4478,7 +4474,6 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	u32 iir, new_iir;
> 	u32 pipe_stats[I915_MAX_PIPES];
>-	unsigned long irqflags;
> 	int ret = IRQ_NONE, pipe;
> 	u32 flip_mask =
> 		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
>@@ -4495,13 +4490,13 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
> 		 * It doesn't set the bit in iir again, but it still produces
> 		 * interrupts (for non-MSI).
> 		 */
>-		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+		spin_lock(&dev_priv->irq_lock);
> 		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
> 			i915_handle_error(dev, false,
> 					  "Command parser error, iir 0x%08x",
> 					  iir);
> 
>-		for_each_pipe(pipe) {
>+		for_each_pipe(dev_priv, pipe) {
> 			int reg = PIPESTAT(pipe);
> 			pipe_stats[pipe] = I915_READ(reg);
> 
>@@ -4513,7 +4508,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
> 				irq_received = true;
> 			}
> 		}
>-		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+		spin_unlock(&dev_priv->irq_lock);
> 
> 		if (!irq_received)
> 			break;
>@@ -4528,11 +4523,11 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
> 		new_iir = I915_READ(IIR); /* Flush posted writes */
> 
> 		if (iir & I915_USER_INTERRUPT)
>-			notify_ring(dev, &dev_priv->ring[RCS]);
>+			notify_ring(dev, &dev_priv->engine[RCS]);
> 		if (iir & I915_BSD_USER_INTERRUPT)
>-			notify_ring(dev, &dev_priv->ring[VCS]);
>+			notify_ring(dev, &dev_priv->engine[VCS]);
> 
>-		for_each_pipe(pipe) {
>+		for_each_pipe(dev_priv, pipe) {
> 			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
> 			    i915_handle_vblank(dev, pipe, pipe, iir))
> 				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
>@@ -4572,8 +4567,6 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
> 		iir = new_iir;
> 	}
> 
>-	i915_update_dri1_breadcrumb(dev);
>-
> 	return ret;
> }
> 
>@@ -4589,30 +4582,29 @@ static void i965_irq_uninstall(struct drm_device * dev)
> 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
> 
> 	I915_WRITE(HWSTAM, 0xffffffff);
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		I915_WRITE(PIPESTAT(pipe), 0);
> 	I915_WRITE(IMR, 0xffffffff);
> 	I915_WRITE(IER, 0x0);
> 
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		I915_WRITE(PIPESTAT(pipe),
> 			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
> 	I915_WRITE(IIR, I915_READ(IIR));
> }
> 
>-static void intel_hpd_irq_reenable(struct work_struct *work)
>+static void intel_hpd_irq_reenable_work(struct work_struct *work)
> {
> 	struct drm_i915_private *dev_priv =
> 		container_of(work, typeof(*dev_priv),
> 			     hotplug_reenable_work.work);
> 	struct drm_device *dev = dev_priv->dev;
> 	struct drm_mode_config *mode_config = &dev->mode_config;
>-	unsigned long irqflags;
> 	int i;
> 
> 	intel_runtime_pm_get(dev_priv);
> 
>-	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 	for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
> 		struct drm_connector *connector;
> 
>@@ -4636,7 +4628,7 @@ static void intel_hpd_irq_reenable(struct work_struct *work)
> 	}
> 	if (dev_priv->display.hpd_irq_setup)
> 		dev_priv->display.hpd_irq_setup(dev);
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> 
> 	intel_runtime_pm_put(dev_priv);
> }
>@@ -4645,24 +4637,24 @@ void intel_irq_init(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 
>-	INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
>+	INIT_DELAYED_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
> 	INIT_WORK(&dev_priv->dig_port_work, i915_digport_work_func);
> 	INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
> 	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
> 	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
> 
> 	/* Let's track the enabled rps events */
>-	if (IS_VALLEYVIEW(dev))
>-		/* WaGsvRC0ResidenncyMethod:VLV */
>+	if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev))
>+		/* WaGsvRC0ResidencyMethod:vlv */
> 		dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
> 	else
> 		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
> 
> 	setup_timer(&dev_priv->gpu_error.hangcheck_timer,
> 		    i915_hangcheck_elapsed,
>-		    (unsigned long) dev);
>+		    (unsigned long) dev_priv);
> 	INIT_DELAYED_WORK(&dev_priv->hotplug_reenable_work,
>-			  intel_hpd_irq_reenable);
>+			  intel_hpd_irq_reenable_work);
> 
> 	pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
> 
>@@ -4746,7 +4738,6 @@ void intel_hpd_init(struct drm_device *dev)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct drm_mode_config *mode_config = &dev->mode_config;
> 	struct drm_connector *connector;
>-	unsigned long irqflags;
> 	int i;
> 
> 	for (i = 1; i < HPD_NUM_PINS; i++) {
>@@ -4764,10 +4755,10 @@ void intel_hpd_init(struct drm_device *dev)
> 
> 	/* Interrupt setup is already guaranteed to be single-threaded, this is
> 	 * just to make the assert_spin_locked checks happy. */
>-	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+	spin_lock_irq(&dev_priv->irq_lock);
> 	if (dev_priv->display.hpd_irq_setup)
> 		dev_priv->display.hpd_irq_setup(dev);
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+	spin_unlock_irq(&dev_priv->irq_lock);
> }
> 
> /* Disable interrupts so we can allow runtime PM. */
>diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
>index f29b44c..44aaabd 100644
>--- a/drivers/gpu/drm/i915/i915_reg.h
>+++ b/drivers/gpu/drm/i915/i915_reg.h
>@@ -143,6 +143,14 @@
> #define GAB_CTL				0x24000
> #define   GAB_CTL_CONT_AFTER_PAGEFAULT	(1<<8)
> 
>+#define GEN7_BIOS_RESERVED		0x1082C0
>+#define GEN7_BIOS_RESERVED_1M		(0 << 5)
>+#define GEN7_BIOS_RESERVED_256K		(1 << 5)
>+#define GEN8_BIOS_RESERVED_SHIFT       7
>+#define GEN7_BIOS_RESERVED_MASK        0x1
>+#define GEN8_BIOS_RESERVED_MASK        0x3
>+
>+
> /* VGA stuff */
> 
> #define VGA_ST01_MDA 0x3ba
>@@ -272,6 +280,7 @@
> #define   MI_SEMAPHORE_POLL		(1<<15)
> #define   MI_SEMAPHORE_SAD_GTE_SDD	(1<<12)
> #define MI_STORE_DWORD_IMM	MI_INSTR(0x20, 1)
>+#define MI_STORE_DWORD_IMM_GEN8	MI_INSTR(0x20, 2)
> #define   MI_MEM_VIRTUAL	(1 << 22) /* 965+ only */
> #define MI_STORE_DWORD_INDEX	MI_INSTR(0x21, 1)
> #define   MI_STORE_DWORD_INDEX_SHIFT 2
>@@ -282,10 +291,11 @@
>  *   address/value pairs. Don't overdue it, though, x <= 2^4 must hold!
>  */
> #define MI_LOAD_REGISTER_IMM(x)	MI_INSTR(0x22, 2*(x)-1)
>+#define   MI_LRI_FORCE_POSTED		(1<<12)
> #define MI_STORE_REGISTER_MEM(x) MI_INSTR(0x24, 2*(x)-1)
> #define MI_STORE_REGISTER_MEM_GEN8(x) MI_INSTR(0x24, 3*(x)-1)
> #define   MI_SRM_LRM_GLOBAL_GTT		(1<<22)
>-#define MI_FLUSH_DW		MI_INSTR(0x26, 1) /* for GEN6 */
>+#define MI_FLUSH_DW		MI_INSTR(0x26, 0) /* for GEN6 */
> #define   MI_FLUSH_DW_STORE_INDEX	(1<<21)
> #define   MI_INVALIDATE_TLB		(1<<18)
> #define   MI_FLUSH_DW_OP_STOREDW	(1<<14)
>@@ -501,10 +511,26 @@
> #define BUNIT_REG_BISOC				0x11
> 
> #define PUNIT_REG_DSPFREQ			0x36
>+#define   DSPFREQSTAT_SHIFT_CHV			24
>+#define   DSPFREQSTAT_MASK_CHV			(0x1f << DSPFREQSTAT_SHIFT_CHV)
>+#define   DSPFREQGUAR_SHIFT_CHV			8
>+#define   DSPFREQGUAR_MASK_CHV			(0x1f << DSPFREQGUAR_SHIFT_CHV)
> #define   DSPFREQSTAT_SHIFT			30
> #define   DSPFREQSTAT_MASK			(0x3 << DSPFREQSTAT_SHIFT)
> #define   DSPFREQGUAR_SHIFT			14
> #define   DSPFREQGUAR_MASK			(0x3 << DSPFREQGUAR_SHIFT)
>+#define   _DP_SSC(val, pipe)			((val) << (2 * (pipe)))
>+#define   DP_SSC_MASK(pipe)			_DP_SSC(0x3, (pipe))
>+#define   DP_SSC_PWR_ON(pipe)			_DP_SSC(0x0, (pipe))
>+#define   DP_SSC_CLK_GATE(pipe)			_DP_SSC(0x1, (pipe))
>+#define   DP_SSC_RESET(pipe)			_DP_SSC(0x2, (pipe))
>+#define   DP_SSC_PWR_GATE(pipe)			_DP_SSC(0x3, (pipe))
>+#define   _DP_SSS(val, pipe)			((val) << (2 * (pipe) + 16))
>+#define   DP_SSS_MASK(pipe)			_DP_SSS(0x3, (pipe))
>+#define   DP_SSS_PWR_ON(pipe)			_DP_SSS(0x0, (pipe))
>+#define   DP_SSS_CLK_GATE(pipe)			_DP_SSS(0x1, (pipe))
>+#define   DP_SSS_RESET(pipe)			_DP_SSS(0x2, (pipe))
>+#define   DP_SSS_PWR_GATE(pipe)			_DP_SSS(0x3, (pipe))
> 
> /* See the PUNIT HAS v0.8 for the below bits */
> enum punit_power_well {
>@@ -518,6 +544,11 @@ enum punit_power_well {
> 	PUNIT_POWER_WELL_DPIO_TX_C_LANES_23	= 9,
> 	PUNIT_POWER_WELL_DPIO_RX0		= 10,
> 	PUNIT_POWER_WELL_DPIO_RX1		= 11,
>+	PUNIT_POWER_WELL_DPIO_CMN_D		= 12,
>+	/* FIXME: guesswork below */
>+	PUNIT_POWER_WELL_DPIO_TX_D_LANES_01	= 13,
>+	PUNIT_POWER_WELL_DPIO_TX_D_LANES_23	= 14,
>+	PUNIT_POWER_WELL_DPIO_RX2		= 15,
> 
> 	PUNIT_POWER_WELL_NUM,
> };
>@@ -838,8 +869,8 @@ enum punit_power_well {
> 
> #define _VLV_TX_DW2_CH0			0x8288
> #define _VLV_TX_DW2_CH1			0x8488
>-#define   DPIO_SWING_MARGIN_SHIFT	16
>-#define   DPIO_SWING_MARGIN_MASK	(0xff << DPIO_SWING_MARGIN_SHIFT)
>+#define   DPIO_SWING_MARGIN000_SHIFT	16
>+#define   DPIO_SWING_MARGIN000_MASK	(0xff << DPIO_SWING_MARGIN000_SHIFT)
> #define   DPIO_UNIQ_TRANS_SCALE_SHIFT	8
> #define VLV_TX_DW2(ch) _PORT(ch, _VLV_TX_DW2_CH0, _VLV_TX_DW2_CH1)
> 
>@@ -847,12 +878,16 @@ enum punit_power_well {
> #define _VLV_TX_DW3_CH1			0x848c
> /* The following bit for CHV phy */
> #define   DPIO_TX_UNIQ_TRANS_SCALE_EN	(1<<27)
>+#define   DPIO_SWING_MARGIN101_SHIFT	16
>+#define   DPIO_SWING_MARGIN101_MASK	(0xff << DPIO_SWING_MARGIN101_SHIFT)
> #define VLV_TX_DW3(ch) _PORT(ch, _VLV_TX_DW3_CH0, _VLV_TX_DW3_CH1)
> 
> #define _VLV_TX_DW4_CH0			0x8290
> #define _VLV_TX_DW4_CH1			0x8490
> #define   DPIO_SWING_DEEMPH9P5_SHIFT	24
> #define   DPIO_SWING_DEEMPH9P5_MASK	(0xff << DPIO_SWING_DEEMPH9P5_SHIFT)
>+#define   DPIO_SWING_DEEMPH6P0_SHIFT	16
>+#define   DPIO_SWING_DEEMPH6P0_MASK	(0xff << DPIO_SWING_DEEMPH6P0_SHIFT)
> #define VLV_TX_DW4(ch) _PORT(ch, _VLV_TX_DW4_CH0, _VLV_TX_DW4_CH1)
> 
> #define _VLV_TX3_DW4_CH0		0x690
>@@ -1003,6 +1038,13 @@ enum punit_power_well {
> #define   PGTBL_ADDRESS_LO_MASK	0xfffff000 /* bits [31:12] */
> #define   PGTBL_ADDRESS_HI_MASK	0x000000f0 /* bits [35:32] (gen4) */
> #define PGTBL_ER	0x02024
>+#define PRB0_BASE (0x2030-0x30)
>+#define PRB1_BASE (0x2040-0x30) /* 830,gen3 */
>+#define PRB2_BASE (0x2050-0x30) /* gen3 */
>+#define SRB0_BASE (0x2100-0x30) /* gen2 */
>+#define SRB1_BASE (0x2110-0x30) /* gen2 */
>+#define SRB2_BASE (0x2120-0x30) /* 830 */
>+#define SRB3_BASE (0x2130-0x30) /* 830 */
> #define RENDER_RING_BASE	0x02000
> #define BSD_RING_BASE		0x04000
> #define GEN6_BSD_RING_BASE	0x12000
>@@ -1064,6 +1106,7 @@ enum punit_power_well {
> #define RING_ACTHD_UDW(base)	((base)+0x5c)
> #define RING_NOPID(base)	((base)+0x94)
> #define RING_IMR(base)		((base)+0xa8)
>+#define RING_HWSTAM(base)	((base)+0x98)
> #define RING_TIMESTAMP(base)	((base)+0x358)
> #define   TAIL_ADDR		0x001FFFF8
> #define   HEAD_WRAP_COUNT	0xFFE00000
>@@ -1248,6 +1291,10 @@ enum punit_power_well {
> #define   INSTPM_TLB_INVALIDATE	(1<<9)
> #define   INSTPM_SYNC_FLUSH	(1<<5)
> #define ACTHD	        0x020c8
>+#define MEM_MODE	0x020cc
>+#define   MEM_DISPLAY_B_TRICKLE_FEED_DISABLE (1<<3) /* 830 only */
>+#define   MEM_DISPLAY_A_TRICKLE_FEED_DISABLE (1<<2) /* 830/845 only */
>+#define   MEM_DISPLAY_TRICKLE_FEED_DISABLE (1<<2) /* 85x only */
> #define FW_BLC		0x020d8
> #define FW_BLC2		0x020dc
> #define FW_BLC_SELF	0x020e0 /* 915+ only */
>@@ -1380,6 +1427,7 @@ enum punit_power_well {
> #define GT_BSD_CS_ERROR_INTERRUPT		(1 << 15)
> #define GT_BSD_USER_INTERRUPT			(1 << 12)
> #define GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1	(1 << 11) /* hsw+; rsvd on snb, ivb, vlv */
>+#define GT_CONTEXT_SWITCH_INTERRUPT		(1 <<  8)
> #define GT_RENDER_L3_PARITY_ERROR_INTERRUPT	(1 <<  5) /* !snb */
> #define GT_RENDER_PIPECTL_NOTIFY_INTERRUPT	(1 <<  4)
> #define GT_RENDER_CS_MASTER_ERROR_INTERRUPT	(1 <<  3)
>@@ -1519,6 +1567,7 @@ enum punit_power_well {
> /* Framebuffer compression for Ironlake */
> #define ILK_DPFC_CB_BASE	0x43200
> #define ILK_DPFC_CONTROL	0x43208
>+#define   FBC_CTL_FALSE_COLOR	(1<<10)
> /* The bit 28-8 is reserved */
> #define   DPFC_RESERVED		(0x1FFFFF00)
> #define ILK_DPFC_RECOMP_CTL	0x4320c
>@@ -1675,12 +1724,9 @@ enum punit_power_well {
> #define DPIO_PHY_STATUS			(VLV_DISPLAY_BASE + 0x6240)
> #define   DPLL_PORTD_READY_MASK		(0xf)
> #define DISPLAY_PHY_CONTROL (VLV_DISPLAY_BASE + 0x60100)
>-#define   PHY_COM_LANE_RESET_DEASSERT(phy, val) \
>-				((phy == DPIO_PHY0) ? (val | 1) : (val | 2))
>-#define   PHY_COM_LANE_RESET_ASSERT(phy, val) \
>-				((phy == DPIO_PHY0) ? (val & ~1) : (val & ~2))
>+#define   PHY_COM_LANE_RESET_DEASSERT(phy) (1 << (phy))
> #define DISPLAY_PHY_STATUS (VLV_DISPLAY_BASE + 0x60104)
>-#define   PHY_POWERGOOD(phy)	((phy == DPIO_PHY0) ? (1<<31) : (1<<30))
>+#define   PHY_POWERGOOD(phy)	(((phy) == DPIO_PHY0) ? (1<<31) : (1<<30))
> 
> /*
>  * The i830 generation, in LVDS mode, defines P1 as the bit number set within
>@@ -2261,6 +2307,7 @@ enum punit_power_well {
>  *   doesn't need saving on GT1
>  */
> #define CXT_SIZE		0x21a0
>+#define ILK_CXT_TOTAL_SIZE		(1 * PAGE_SIZE)
> #define GEN6_CXT_POWER_SIZE(cxt_reg)	((cxt_reg >> 24) & 0x3f)
> #define GEN6_CXT_RING_SIZE(cxt_reg)	((cxt_reg >> 18) & 0x3f)
> #define GEN6_CXT_RENDER_SIZE(cxt_reg)	((cxt_reg >> 12) & 0x3f)
>@@ -3476,6 +3523,8 @@ enum punit_power_well {
> #define   DP_LINK_TRAIN_OFF		(3 << 28)
> #define   DP_LINK_TRAIN_MASK		(3 << 28)
> #define   DP_LINK_TRAIN_SHIFT		28
>+#define   DP_LINK_TRAIN_PAT_3_CHV	(1 << 14)
>+#define   DP_LINK_TRAIN_MASK_CHV	((3 << 28)|(1<<14))
> 
> /* CPT Link training mode */
> #define   DP_LINK_TRAIN_PAT_1_CPT	(0 << 8)
>@@ -3732,7 +3781,6 @@ enum punit_power_well {
> #define   PIPE_VSYNC_INTERRUPT_STATUS		(1UL<<9)
> #define   PIPE_DISPLAY_LINE_COMPARE_STATUS	(1UL<<8)
> #define   PIPE_DPST_EVENT_STATUS		(1UL<<7)
>-#define   PIPE_LEGACY_BLC_EVENT_STATUS		(1UL<<6)
> #define   PIPE_A_PSR_STATUS_VLV			(1UL<<6)
> #define   PIPE_LEGACY_BLC_EVENT_STATUS		(1UL<<6)
> #define   PIPE_ODD_FIELD_INTERRUPT_STATUS	(1UL<<5)
>@@ -3842,73 +3890,151 @@ enum punit_power_well {
> #define   DSPARB_BEND_SHIFT	9 /* on 855 */
> #define   DSPARB_AEND_SHIFT	0
> 
>+/* pnv/gen4/g4x/vlv/chv */
> #define DSPFW1			(dev_priv->info.display_mmio_offset + 0x70034)
>-#define   DSPFW_SR_SHIFT	23
>-#define   DSPFW_SR_MASK		(0x1ff<<23)
>-#define   DSPFW_CURSORB_SHIFT	16
>-#define   DSPFW_CURSORB_MASK	(0x3f<<16)
>-#define   DSPFW_PLANEB_SHIFT	8
>-#define   DSPFW_PLANEB_MASK	(0x7f<<8)
>-#define   DSPFW_PLANEA_MASK	(0x7f)
>+#define   DSPFW_SR_SHIFT		23
>+#define   DSPFW_SR_MASK			(0x1ff<<23)
>+#define   DSPFW_CURSORB_SHIFT		16
>+#define   DSPFW_CURSORB_MASK		(0x3f<<16)
>+#define   DSPFW_PLANEB_SHIFT		8
>+#define   DSPFW_PLANEB_MASK		(0x7f<<8)
>+#define   DSPFW_PLANEB_MASK_VLV		(0xff<<8) /* vlv/chv */
>+#define   DSPFW_PLANEA_SHIFT		0
>+#define   DSPFW_PLANEA_MASK		(0x7f<<0)
>+#define   DSPFW_PLANEA_MASK_VLV		(0xff<<0) /* vlv/chv */
> #define DSPFW2			(dev_priv->info.display_mmio_offset + 0x70038)
>-#define   DSPFW_CURSORA_MASK	0x00003f00
>-#define   DSPFW_CURSORA_SHIFT	8
>-#define   DSPFW_PLANEC_MASK	(0x7f)
>+#define   DSPFW_FBC_SR_EN		(1<<31)	  /* g4x */
>+#define   DSPFW_FBC_SR_SHIFT		28
>+#define   DSPFW_FBC_SR_MASK		(0x7<<28) /* g4x */
>+#define   DSPFW_FBC_HPLL_SR_SHIFT	24
>+#define   DSPFW_FBC_HPLL_SR_MASK	(0xf<<24) /* g4x */
>+#define   DSPFW_SPRITEB_SHIFT		(16)
>+#define   DSPFW_SPRITEB_MASK		(0x7f<<16) /* g4x */
>+#define   DSPFW_SPRITEB_MASK_VLV	(0xff<<16) /* vlv/chv */
>+#define   DSPFW_CURSORA_SHIFT		8
>+#define   DSPFW_CURSORA_MASK		(0x3f<<8)
>+#define   DSPFW_PLANEC_SHIFT_OLD	0
>+#define   DSPFW_PLANEC_MASK_OLD		(0x7f<<0) /* pre-gen4 sprite C */
>+#define   DSPFW_SPRITEA_SHIFT		0
>+#define   DSPFW_SPRITEA_MASK		(0x7f<<0) /* g4x */
>+#define   DSPFW_SPRITEA_MASK_VLV	(0xff<<0) /* vlv/chv */
> #define DSPFW3			(dev_priv->info.display_mmio_offset + 0x7003c)
>-#define   DSPFW_HPLL_SR_EN	(1<<31)
>-#define   DSPFW_CURSOR_SR_SHIFT	24
>+#define   DSPFW_HPLL_SR_EN		(1<<31)
> #define   PINEVIEW_SELF_REFRESH_EN	(1<<30)
>+#define   DSPFW_CURSOR_SR_SHIFT		24
> #define   DSPFW_CURSOR_SR_MASK		(0x3f<<24)
> #define   DSPFW_HPLL_CURSOR_SHIFT	16
> #define   DSPFW_HPLL_CURSOR_MASK	(0x3f<<16)
>-#define   DSPFW_HPLL_SR_MASK		(0x1ff)
>-#define DSPFW4			(dev_priv->info.display_mmio_offset + 0x70070)
>-#define DSPFW7			(dev_priv->info.display_mmio_offset + 0x7007c)
>+#define   DSPFW_HPLL_SR_SHIFT		0
>+#define   DSPFW_HPLL_SR_MASK		(0x1ff<<0)
>+
>+/* vlv/chv */
>+#define DSPFW4			(VLV_DISPLAY_BASE + 0x70070)
>+#define   DSPFW_SPRITEB_WM1_SHIFT	16
>+#define   DSPFW_SPRITEB_WM1_MASK	(0xff<<16)
>+#define   DSPFW_CURSORA_WM1_SHIFT	8
>+#define   DSPFW_CURSORA_WM1_MASK	(0x3f<<8)
>+#define   DSPFW_SPRITEA_WM1_SHIFT	0
>+#define   DSPFW_SPRITEA_WM1_MASK	(0xff<<0)
>+#define DSPFW5			(VLV_DISPLAY_BASE + 0x70074)
>+#define   DSPFW_PLANEB_WM1_SHIFT	24
>+#define   DSPFW_PLANEB_WM1_MASK		(0xff<<24)
>+#define   DSPFW_PLANEA_WM1_SHIFT	16
>+#define   DSPFW_PLANEA_WM1_MASK		(0xff<<16)
>+#define   DSPFW_CURSORB_WM1_SHIFT	8
>+#define   DSPFW_CURSORB_WM1_MASK	(0x3f<<8)
>+#define   DSPFW_CURSOR_SR_WM1_SHIFT	0
>+#define   DSPFW_CURSOR_SR_WM1_MASK	(0x3f<<0)
>+#define DSPFW6			(VLV_DISPLAY_BASE + 0x70078)
>+#define   DSPFW_SR_WM1_SHIFT		0
>+#define   DSPFW_SR_WM1_MASK		(0x1ff<<0)
>+#define DSPFW7			(VLV_DISPLAY_BASE + 0x7007c)
>+#define DSPFW7_CHV		(VLV_DISPLAY_BASE + 0x700b4) /* wtf #1? */
>+#define   DSPFW_SPRITED_WM1_SHIFT	24
>+#define   DSPFW_SPRITED_WM1_MASK	(0xff<<24)
>+#define   DSPFW_SPRITED_SHIFT		16
>+#define   DSPFW_SPRITED_MASK		(0xff<<16)
>+#define   DSPFW_SPRITEC_WM1_SHIFT	8
>+#define   DSPFW_SPRITEC_WM1_MASK	(0xff<<8)
>+#define   DSPFW_SPRITEC_SHIFT		0
>+#define   DSPFW_SPRITEC_MASK		(0xff<<0)
>+#define DSPFW8_CHV		(VLV_DISPLAY_BASE + 0x700b8)
>+#define   DSPFW_SPRITEF_WM1_SHIFT	24
>+#define   DSPFW_SPRITEF_WM1_MASK	(0xff<<24)
>+#define   DSPFW_SPRITEF_SHIFT		16
>+#define   DSPFW_SPRITEF_MASK		(0xff<<16)
>+#define   DSPFW_SPRITEE_WM1_SHIFT	8
>+#define   DSPFW_SPRITEE_WM1_MASK	(0xff<<8)
>+#define   DSPFW_SPRITEE_SHIFT		0
>+#define   DSPFW_SPRITEE_MASK		(0xff<<0)
>+#define DSPFW9_CHV		(VLV_DISPLAY_BASE + 0x7007c) /* wtf #2? */
>+#define   DSPFW_PLANEC_WM1_SHIFT	24
>+#define   DSPFW_PLANEC_WM1_MASK		(0xff<<24)
>+#define   DSPFW_PLANEC_SHIFT		16
>+#define   DSPFW_PLANEC_MASK		(0xff<<16)
>+#define   DSPFW_CURSORC_WM1_SHIFT	8
>+#define   DSPFW_CURSORC_WM1_MASK	(0x3f<<16)
>+#define   DSPFW_CURSORC_SHIFT		0
>+#define   DSPFW_CURSORC_MASK		(0x3f<<0)
>+
>+/* vlv/chv high order bits */
>+#define DSPHOWM			(VLV_DISPLAY_BASE + 0x70064)
>+#define   DSPFW_SR_HI_SHIFT		24
>+#define   DSPFW_SR_HI_MASK		(1<<24)
>+#define   DSPFW_SPRITEF_HI_SHIFT	23
>+#define   DSPFW_SPRITEF_HI_MASK		(1<<23)
>+#define   DSPFW_SPRITEE_HI_SHIFT	22
>+#define   DSPFW_SPRITEE_HI_MASK		(1<<22)
>+#define   DSPFW_PLANEC_HI_SHIFT		21
>+#define   DSPFW_PLANEC_HI_MASK		(1<<21)
>+#define   DSPFW_SPRITED_HI_SHIFT	20
>+#define   DSPFW_SPRITED_HI_MASK		(1<<20)
>+#define   DSPFW_SPRITEC_HI_SHIFT	16
>+#define   DSPFW_SPRITEC_HI_MASK		(1<<16)
>+#define   DSPFW_PLANEB_HI_SHIFT		12
>+#define   DSPFW_PLANEB_HI_MASK		(1<<12)
>+#define   DSPFW_SPRITEB_HI_SHIFT	8
>+#define   DSPFW_SPRITEB_HI_MASK		(1<<8)
>+#define   DSPFW_SPRITEA_HI_SHIFT	4
>+#define   DSPFW_SPRITEA_HI_MASK		(1<<4)
>+#define   DSPFW_PLANEA_HI_SHIFT		0
>+#define   DSPFW_PLANEA_HI_MASK		(1<<0)
>+#define DSPHOWM1		(VLV_DISPLAY_BASE + 0x70068)
>+#define   DSPFW_SR_WM1_HI_SHIFT		24
>+#define   DSPFW_SR_WM1_HI_MASK		(1<<24)
>+#define   DSPFW_SPRITEF_WM1_HI_SHIFT	23
>+#define   DSPFW_SPRITEF_WM1_HI_MASK	(1<<23)
>+#define   DSPFW_SPRITEE_WM1_HI_SHIFT	22
>+#define   DSPFW_SPRITEE_WM1_HI_MASK	(1<<22)
>+#define   DSPFW_PLANEC_WM1_HI_SHIFT	21
>+#define   DSPFW_PLANEC_WM1_HI_MASK	(1<<21)
>+#define   DSPFW_SPRITED_WM1_HI_SHIFT	20
>+#define   DSPFW_SPRITED_WM1_HI_MASK	(1<<20)
>+#define   DSPFW_SPRITEC_WM1_HI_SHIFT	16
>+#define   DSPFW_SPRITEC_WM1_HI_MASK	(1<<16)
>+#define   DSPFW_PLANEB_WM1_HI_SHIFT	12
>+#define   DSPFW_PLANEB_WM1_HI_MASK	(1<<12)
>+#define   DSPFW_SPRITEB_WM1_HI_SHIFT	8
>+#define   DSPFW_SPRITEB_WM1_HI_MASK	(1<<8)
>+#define   DSPFW_SPRITEA_WM1_HI_SHIFT	4
>+#define   DSPFW_SPRITEA_WM1_HI_MASK	(1<<4)
>+#define   DSPFW_PLANEA_WM1_HI_SHIFT	0
>+#define   DSPFW_PLANEA_WM1_HI_MASK	(1<<0)
> 
> /* drain latency register values*/
> #define DRAIN_LATENCY_PRECISION_32	32
> #define DRAIN_LATENCY_PRECISION_64	64
>-#define VLV_DDL1			(VLV_DISPLAY_BASE + 0x70050)
>-#define DDL_CURSORA_PRECISION_64	(1<<31)
>-#define DDL_CURSORA_PRECISION_32	(0<<31)
>-#define DDL_CURSORA_SHIFT		24
>-#define DDL_SPRITEB_PRECISION_64	(1<<23)
>-#define DDL_SPRITEB_PRECISION_32	(0<<23)
>-#define DDL_SPRITEB_SHIFT		16
>-#define DDL_SPRITEA_PRECISION_64	(1<<15)
>-#define DDL_SPRITEA_PRECISION_32	(0<<15)
>-#define DDL_SPRITEA_SHIFT		8
>-#define DDL_PLANEA_PRECISION_64		(1<<7)
>-#define DDL_PLANEA_PRECISION_32		(0<<7)
>-#define DDL_PLANEA_SHIFT		0
>-
>-#define VLV_DDL2			(VLV_DISPLAY_BASE + 0x70054)
>-#define DDL_CURSORB_PRECISION_64	(1<<31)
>-#define DDL_CURSORB_PRECISION_32	(0<<31)
>-#define DDL_CURSORB_SHIFT		24
>-#define DDL_SPRITED_PRECISION_64	(1<<23)
>-#define DDL_SPRITED_PRECISION_32	(0<<23)
>-#define DDL_SPRITED_SHIFT		16
>-#define DDL_SPRITEC_PRECISION_64	(1<<15)
>-#define DDL_SPRITEC_PRECISION_32	(0<<15)
>-#define DDL_SPRITEC_SHIFT		8
>-#define DDL_PLANEB_PRECISION_64		(1<<7)
>-#define DDL_PLANEB_PRECISION_32		(0<<7)
>-#define DDL_PLANEB_SHIFT		0
>-
>-#define VLV_DDL3			(VLV_DISPLAY_BASE + 0x70058)
>-#define DDL_CURSORC_PRECISION_64	(1<<31)
>-#define DDL_CURSORC_PRECISION_32	(0<<31)
>-#define DDL_CURSORC_SHIFT		24
>-#define DDL_SPRITEF_PRECISION_64	(1<<23)
>-#define DDL_SPRITEF_PRECISION_32	(0<<23)
>-#define DDL_SPRITEF_SHIFT		16
>-#define DDL_SPRITEE_PRECISION_64	(1<<15)
>-#define DDL_SPRITEE_PRECISION_32	(0<<15)
>-#define DDL_SPRITEE_SHIFT		8
>-#define DDL_PLANEC_PRECISION_64		(1<<7)
>-#define DDL_PLANEC_PRECISION_32		(0<<7)
>-#define DDL_PLANEC_SHIFT		0
>+#define VLV_DDL(pipe)			(VLV_DISPLAY_BASE + 0x70050 + 4 * (pipe))
>+#define DDL_CURSOR_PRECISION_64		(1<<31)
>+#define DDL_CURSOR_PRECISION_32		(0<<31)
>+#define DDL_CURSOR_SHIFT		24
>+#define DDL_SPRITE_PRECISION_64(sprite)	(1<<(15+8*(sprite)))
>+#define DDL_SPRITE_PRECISION_32(sprite)	(0<<(15+8*(sprite)))
>+#define DDL_SPRITE_SHIFT(sprite)	(8+8*(sprite))
>+#define DDL_PLANE_PRECISION_64		(1<<7)
>+#define DDL_PLANE_PRECISION_32		(0<<7)
>+#define DDL_PLANE_SHIFT			0
>+#define DRAIN_LATENCY_MASK		0x7f
> 
> /* FIFO watermark sizes etc */
> #define G4X_FIFO_LINE_SIZE	64
>@@ -4026,7 +4152,8 @@ enum punit_power_well {
> /* Old style CUR*CNTR flags (desktop 8xx) */
> #define   CURSOR_ENABLE		0x80000000
> #define   CURSOR_GAMMA_ENABLE	0x40000000
>-#define   CURSOR_STRIDE_MASK	0x30000000
>+#define   CURSOR_STRIDE_SHIFT	28
>+#define   CURSOR_STRIDE(x)	((ffs(x)-9) << CURSOR_STRIDE_SHIFT) /* 256,512,1k,2k */
> #define   CURSOR_PIPE_CSC_ENABLE (1<<24)
> #define   CURSOR_FORMAT_SHIFT	24
> #define   CURSOR_FORMAT_MASK	(0x07 << CURSOR_FORMAT_SHIFT)
>@@ -4111,6 +4238,7 @@ enum punit_power_well {
> #define   DISPPLANE_NO_LINE_DOUBLE		0
> #define   DISPPLANE_STEREO_POLARITY_FIRST	0
> #define   DISPPLANE_STEREO_POLARITY_SECOND	(1<<18)
>+#define   DISPPLANE_ROTATE_180         (1<<15)
> #define   DISPPLANE_TRICKLE_FEED_DISABLE	(1<<14) /* Ironlake */
> #define   DISPPLANE_TILED			(1<<10)
> #define _DSPAADDR				0x70184
>@@ -4195,6 +4323,7 @@ enum punit_power_well {
> #define   DVS_YUV_ORDER_UYVY	(1<<16)
> #define   DVS_YUV_ORDER_YVYU	(2<<16)
> #define   DVS_YUV_ORDER_VYUY	(3<<16)
>+#define   DVS_ROTATE_180	(1<<15)
> #define   DVS_DEST_KEY		(1<<2)
> #define   DVS_TRICKLE_FEED_DISABLE (1<<14)
> #define   DVS_TILED		(1<<10)
>@@ -4265,6 +4394,7 @@ enum punit_power_well {
> #define   SPRITE_YUV_ORDER_UYVY		(1<<16)
> #define   SPRITE_YUV_ORDER_YVYU		(2<<16)
> #define   SPRITE_YUV_ORDER_VYUY		(3<<16)
>+#define   SPRITE_ROTATE_180		(1<<15)
> #define   SPRITE_TRICKLE_FEED_DISABLE	(1<<14)
> #define   SPRITE_INT_GAMMA_ENABLE	(1<<13)
> #define   SPRITE_TILED			(1<<10)
>@@ -4338,6 +4468,7 @@ enum punit_power_well {
> #define   SP_YUV_ORDER_UYVY		(1<<16)
> #define   SP_YUV_ORDER_YVYU		(2<<16)
> #define   SP_YUV_ORDER_VYUY		(3<<16)
>+#define   SP_ROTATE_180			(1<<15)
> #define   SP_TILED			(1<<10)
> #define _SPALINOFF		(VLV_DISPLAY_BASE + 0x72184)
> #define _SPASTRIDE		(VLV_DISPLAY_BASE + 0x72188)
>@@ -5246,8 +5377,7 @@ enum punit_power_well {
> #define PIPEA_PP_STATUS         (VLV_DISPLAY_BASE + 0x61200)
> #define PIPEA_PP_CONTROL        (VLV_DISPLAY_BASE + 0x61204)
> #define PIPEA_PP_ON_DELAYS      (VLV_DISPLAY_BASE + 0x61208)
>-#define  PANEL_PORT_SELECT_DPB_VLV	(1 << 30)
>-#define  PANEL_PORT_SELECT_DPC_VLV	(2 << 30)
>+#define  PANEL_PORT_SELECT_VLV(port)	((port) << 30)
> #define PIPEA_PP_OFF_DELAYS     (VLV_DISPLAY_BASE + 0x6120c)
> #define PIPEA_PP_DIVISOR        (VLV_DISPLAY_BASE + 0x61210)
> 
>@@ -5407,7 +5537,6 @@ enum punit_power_well {
> #define   VLV_GTLC_ALLOWWAKEERR			(1 << 1)
> #define   VLV_GTLC_PW_MEDIA_STATUS_MASK		(1 << 5)
> #define   VLV_GTLC_PW_RENDER_STATUS_MASK	(1 << 7)
>-#define VLV_GTLC_SURVIVABILITY_REG              0x130098
> #define  FORCEWAKE_MT				0xa188 /* multi-threaded */
> #define   FORCEWAKE_KERNEL			0x1
> #define   FORCEWAKE_USER			0x2
>@@ -5457,6 +5586,10 @@ enum punit_power_well {
> #define GEN8_UCGCTL6				0x9430
> #define   GEN8_SDEUNIT_CLOCK_GATE_DISABLE	(1<<14)
> 
>+#define TIMESTAMP_CTR		0x44070
>+#define FREQ_1_28_US(us)	(((us) * 100) >> 7)
>+#define MCHBAR_PCU_C0		(MCHBAR_MIRROR_BASE_SNB + 0x5960)
>+
> #define GEN6_GFXPAUSE				0xA000
> #define GEN6_RPNSWREQ				0xA008
> #define   GEN6_TURBO_DISABLE			(1<<31)
>@@ -5545,12 +5678,6 @@ enum punit_power_well {
> 						 GEN6_PM_RP_DOWN_THRESHOLD | \
> 						 GEN6_PM_RP_DOWN_TIMEOUT)
> 
>-#define CHV_CZ_CLOCK_FREQ_MODE_200			200
>-#define CHV_CZ_CLOCK_FREQ_MODE_267			267
>-#define CHV_CZ_CLOCK_FREQ_MODE_320			320
>-#define CHV_CZ_CLOCK_FREQ_MODE_333			333
>-#define CHV_CZ_CLOCK_FREQ_MODE_400			400
>-
> #define GEN7_GT_SCRATCH_BASE			0x4F100
> #define GEN7_GT_SCRATCH_REG_NUM			8
> 
>@@ -5866,15 +5993,7 @@ enum punit_power_well {
> #define DDI_BUF_CTL_B				0x64100
> #define DDI_BUF_CTL(port) _PORT(port, DDI_BUF_CTL_A, DDI_BUF_CTL_B)
> #define  DDI_BUF_CTL_ENABLE			(1<<31)
>-#define  DDI_BUF_EMP_400MV_0DB_HSW		(0<<24)   /* Sel0 */
>-#define  DDI_BUF_EMP_400MV_3_5DB_HSW		(1<<24)   /* Sel1 */
>-#define  DDI_BUF_EMP_400MV_6DB_HSW		(2<<24)   /* Sel2 */
>-#define  DDI_BUF_EMP_400MV_9_5DB_HSW		(3<<24)   /* Sel3 */
>-#define  DDI_BUF_EMP_600MV_0DB_HSW		(4<<24)   /* Sel4 */
>-#define  DDI_BUF_EMP_600MV_3_5DB_HSW		(5<<24)   /* Sel5 */
>-#define  DDI_BUF_EMP_600MV_6DB_HSW		(6<<24)   /* Sel6 */
>-#define  DDI_BUF_EMP_800MV_0DB_HSW		(7<<24)   /* Sel7 */
>-#define  DDI_BUF_EMP_800MV_3_5DB_HSW		(8<<24)   /* Sel8 */
>+#define  DDI_BUF_TRANS_SELECT(n)	((n) << 24)
> #define  DDI_BUF_EMP_MASK			(0xf<<24)
> #define  DDI_BUF_PORT_REVERSAL			(1<<16)
> #define  DDI_BUF_IS_IDLE			(1<<7)
>diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
>index ae7fd8f..503847f 100644
>--- a/drivers/gpu/drm/i915/i915_sysfs.c
>+++ b/drivers/gpu/drm/i915/i915_sysfs.c
>@@ -540,7 +540,7 @@ static ssize_t error_state_read(struct file *filp, struct kobject *kobj,
> 
> 	memset(&error_priv, 0, sizeof(error_priv));
> 
>-	ret = i915_error_state_buf_init(&error_str, count, off);
>+	ret = i915_error_state_buf_init(&error_str, to_i915(dev), count, off);
> 	if (ret)
> 		return ret;
> 
>diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
>index f5aa006..8bb51dc 100644
>--- a/drivers/gpu/drm/i915/i915_trace.h
>+++ b/drivers/gpu/drm/i915/i915_trace.h
>@@ -325,11 +325,10 @@ TRACE_EVENT(i915_gem_evict_vm,
> 	    TP_printk("dev=%d, vm=%p", __entry->dev, __entry->vm)
> );
> 
>-TRACE_EVENT(i915_gem_ring_sync_to,
>-	    TP_PROTO(struct intel_engine_cs *from,
>-		     struct intel_engine_cs *to,
>-		     u32 seqno),
>-	    TP_ARGS(from, to, seqno),
>+TRACE_EVENT(i915_gem_ring_wait,
>+	    TP_PROTO(struct i915_gem_request *waiter,
>+		     struct i915_gem_request *signaller),
>+	    TP_ARGS(waiter, signaller),
> 
> 	    TP_STRUCT__entry(
> 			     __field(u32, dev)
>@@ -339,18 +338,40 @@ TRACE_EVENT(i915_gem_ring_sync_to,
> 			     ),
> 
> 	    TP_fast_assign(
>-			   __entry->dev = from->dev->primary->index;
>-			   __entry->sync_from = from->id;
>-			   __entry->sync_to = to->id;
>-			   __entry->seqno = seqno;
>+			   __entry->dev = waiter->i915->dev->primary->index;
>+			   __entry->sync_from = waiter->engine->id;
>+			   __entry->sync_to = signaller->engine->id;
>+			   __entry->seqno = signaller->breadcrumb[waiter->engine->id];
> 			   ),
> 
>-	    TP_printk("dev=%u, sync-from=%u, sync-to=%u, seqno=%u",
>+	    TP_printk("dev=%u, sync-from=%u, sync-to=%u, seqno=%x",
> 		      __entry->dev,
> 		      __entry->sync_from, __entry->sync_to,
> 		      __entry->seqno)
> );
> 
>+TRACE_EVENT(i915_gem_ring_switch_context,
>+	    TP_PROTO(struct intel_engine_cs *engine, struct intel_context *ctx, u32 flags),
>+	    TP_ARGS(engine, ctx, flags),
>+
>+	    TP_STRUCT__entry(
>+			     __field(u32, dev)
>+			     __field(u32, ring)
>+			     __field(u32, ctx)
>+			     __field(u32, flags)
>+			     ),
>+
>+	    TP_fast_assign(
>+			   __entry->dev = engine->i915->dev->primary->index;
>+			   __entry->ring = engine->id;
>+			   __entry->ctx = ctx->file_priv ? ctx->user_handle : -1;
>+			   __entry->flags = flags;
>+			   ),
>+
>+	    TP_printk("dev=%u, ring=%u, ctx=%d, flags=0x%08x",
>+		      __entry->dev, __entry->ring, __entry->ctx, __entry->flags)
>+);
>+
> TRACE_EVENT(i915_gem_ring_dispatch,
> 	    TP_PROTO(struct intel_engine_cs *ring, u32 seqno, u32 flags),
> 	    TP_ARGS(ring, seqno, flags),
>@@ -363,66 +384,84 @@ TRACE_EVENT(i915_gem_ring_dispatch,
> 			     ),
> 
> 	    TP_fast_assign(
>-			   __entry->dev = ring->dev->primary->index;
>+			   __entry->dev = ring->i915->dev->primary->index;
> 			   __entry->ring = ring->id;
> 			   __entry->seqno = seqno;
> 			   __entry->flags = flags;
> 			   i915_trace_irq_get(ring, seqno);
> 			   ),
> 
>-	    TP_printk("dev=%u, ring=%u, seqno=%u, flags=%x",
>+	    TP_printk("dev=%u, ring=%u, seqno=%x, flags=%x",
> 		      __entry->dev, __entry->ring, __entry->seqno, __entry->flags)
> );
> 
>-TRACE_EVENT(i915_gem_ring_flush,
>-	    TP_PROTO(struct intel_engine_cs *ring, u32 invalidate, u32 flush),
>-	    TP_ARGS(ring, invalidate, flush),
>+TRACE_EVENT(intel_ringbuffer_begin,
>+	    TP_PROTO(struct intel_ringbuffer *ring, int need),
>+	    TP_ARGS(ring, need),
> 
> 	    TP_STRUCT__entry(
> 			     __field(u32, dev)
> 			     __field(u32, ring)
>-			     __field(u32, invalidate)
>-			     __field(u32, flush)
>+			     __field(u32, need)
>+			     __field(u32, space)
> 			     ),
> 
> 	    TP_fast_assign(
>-			   __entry->dev = ring->dev->primary->index;
>-			   __entry->ring = ring->id;
>-			   __entry->invalidate = invalidate;
>-			   __entry->flush = flush;
>+			   __entry->dev = ring->engine->i915->dev->primary->index;
>+			   __entry->ring = ring->engine->id;
>+			   __entry->need = need;
>+			   __entry->space = intel_ring_space(ring);
> 			   ),
> 
>-	    TP_printk("dev=%u, ring=%x, invalidate=%04x, flush=%04x",
>-		      __entry->dev, __entry->ring,
>-		      __entry->invalidate, __entry->flush)
>+	    TP_printk("dev=%u, ring=%u, need=%u, space=%u",
>+		      __entry->dev, __entry->ring, __entry->need, __entry->space)
> );
> 
>-DECLARE_EVENT_CLASS(i915_gem_request,
>-	    TP_PROTO(struct intel_engine_cs *ring, u32 seqno),
>-	    TP_ARGS(ring, seqno),
>+TRACE_EVENT(intel_ringbuffer_wait,
>+	    TP_PROTO(struct intel_ringbuffer *ring, int need),
>+	    TP_ARGS(ring, need),
> 
> 	    TP_STRUCT__entry(
> 			     __field(u32, dev)
> 			     __field(u32, ring)
>-			     __field(u32, seqno)
>+			     __field(u32, need)
>+			     __field(u32, space)
> 			     ),
> 
> 	    TP_fast_assign(
>-			   __entry->dev = ring->dev->primary->index;
>-			   __entry->ring = ring->id;
>-			   __entry->seqno = seqno;
>+			   __entry->dev = ring->engine->i915->dev->primary->index;
>+			   __entry->ring = ring->engine->id;
>+			   __entry->need = need;
>+			   __entry->space = intel_ring_space(ring);
> 			   ),
> 
>-	    TP_printk("dev=%u, ring=%u, seqno=%u",
>-		      __entry->dev, __entry->ring, __entry->seqno)
>+	    TP_printk("dev=%u, ring=%u, need=%u, space=%u",
>+		      __entry->dev, __entry->ring, __entry->need, __entry->space)
> );
> 
>-DEFINE_EVENT(i915_gem_request, i915_gem_request_add,
>-	    TP_PROTO(struct intel_engine_cs *ring, u32 seqno),
>-	    TP_ARGS(ring, seqno)
>+TRACE_EVENT(intel_ringbuffer_wrap,
>+	    TP_PROTO(struct intel_ringbuffer *ring, int rem),
>+	    TP_ARGS(ring, rem),
>+
>+	    TP_STRUCT__entry(
>+			     __field(u32, dev)
>+			     __field(u32, ring)
>+			     __field(u32, rem)
>+			     __field(u32, size)
>+			     ),
>+
>+	    TP_fast_assign(
>+			   __entry->dev = ring->engine->i915->dev->primary->index;
>+			   __entry->ring = ring->engine->id;
>+			   __entry->rem = rem;
>+			   __entry->size = ring->effective_size;
>+			   ),
>+
>+	    TP_printk("dev=%u, ring=%u, rem=%u, size=%u",
>+		      __entry->dev, __entry->ring, __entry->rem, __entry->size)
> );
> 
>-TRACE_EVENT(i915_gem_request_complete,
>+TRACE_EVENT(i915_gem_ring_complete,
> 	    TP_PROTO(struct intel_engine_cs *ring),
> 	    TP_ARGS(ring),
> 
>@@ -433,23 +472,68 @@ TRACE_EVENT(i915_gem_request_complete,
> 			     ),
> 
> 	    TP_fast_assign(
>-			   __entry->dev = ring->dev->primary->index;
>+			   __entry->dev = ring->i915->dev->primary->index;
> 			   __entry->ring = ring->id;
>-			   __entry->seqno = ring->get_seqno(ring, false);
>+			   __entry->seqno = ring->get_seqno(ring);
> 			   ),
> 
>-	    TP_printk("dev=%u, ring=%u, seqno=%u",
>+	    TP_printk("dev=%u, ring=%u, seqno=%x",
> 		      __entry->dev, __entry->ring, __entry->seqno)
> );
> 
>+DECLARE_EVENT_CLASS(i915_gem_request,
>+	    TP_PROTO(struct i915_gem_request *rq),
>+	    TP_ARGS(rq),
>+
>+	    TP_STRUCT__entry(
>+			     __field(u32, dev)
>+			     __field(u32, ring)
>+			     __field(u32, seqno)
>+			     ),
>+
>+	    TP_fast_assign(
>+			   __entry->dev = rq->i915->dev->primary->index;
>+			   __entry->ring = rq->engine->id;
>+			   __entry->seqno = rq->seqno;
>+			   ),
>+
>+	    TP_printk("dev=%u, ring=%u, seqno=%x",
>+		      __entry->dev, __entry->ring, __entry->seqno)
>+);
>+
>+DEFINE_EVENT(i915_gem_request, i915_gem_request_emit_flush,
>+	    TP_PROTO(struct i915_gem_request *rq),
>+	    TP_ARGS(rq)
>+);
>+
>+DEFINE_EVENT(i915_gem_request, i915_gem_request_emit_batch,
>+	    TP_PROTO(struct i915_gem_request *rq),
>+	    TP_ARGS(rq)
>+);
>+
>+DEFINE_EVENT(i915_gem_request, i915_gem_request_emit_breadcrumb,
>+	    TP_PROTO(struct i915_gem_request *rq),
>+	    TP_ARGS(rq)
>+);
>+
>+DEFINE_EVENT(i915_gem_request, i915_gem_request_commit,
>+	    TP_PROTO(struct i915_gem_request *rq),
>+	    TP_ARGS(rq)
>+);
>+
>+DEFINE_EVENT(i915_gem_request, i915_gem_request_complete,
>+	    TP_PROTO(struct i915_gem_request *rq),
>+	    TP_ARGS(rq)
>+);
>+
> DEFINE_EVENT(i915_gem_request, i915_gem_request_retire,
>-	    TP_PROTO(struct intel_engine_cs *ring, u32 seqno),
>-	    TP_ARGS(ring, seqno)
>+	    TP_PROTO(struct i915_gem_request *rq),
>+	    TP_ARGS(rq)
> );
> 
> TRACE_EVENT(i915_gem_request_wait_begin,
>-	    TP_PROTO(struct intel_engine_cs *ring, u32 seqno),
>-	    TP_ARGS(ring, seqno),
>+	    TP_PROTO(struct i915_gem_request *rq),
>+	    TP_ARGS(rq),
> 
> 	    TP_STRUCT__entry(
> 			     __field(u32, dev)
>@@ -465,47 +549,38 @@ TRACE_EVENT(i915_gem_request_wait_begin,
> 	     * less desirable.
> 	     */
> 	    TP_fast_assign(
>-			   __entry->dev = ring->dev->primary->index;
>-			   __entry->ring = ring->id;
>-			   __entry->seqno = seqno;
>-			   __entry->blocking = mutex_is_locked(&ring->dev->struct_mutex);
>+			   __entry->dev = rq->i915->dev->primary->index;
>+			   __entry->ring = rq->engine->id;
>+			   __entry->seqno = rq->seqno;
>+			   __entry->blocking = mutex_is_locked(&rq->i915->dev->struct_mutex);
> 			   ),
> 
>-	    TP_printk("dev=%u, ring=%u, seqno=%u, blocking=%s",
>+	    TP_printk("dev=%u, ring=%u, seqno=%x, blocking?=%s",
> 		      __entry->dev, __entry->ring, __entry->seqno,
> 		      __entry->blocking ?  "yes (NB)" : "no")
> );
> 
>-DEFINE_EVENT(i915_gem_request, i915_gem_request_wait_end,
>-	    TP_PROTO(struct intel_engine_cs *ring, u32 seqno),
>-	    TP_ARGS(ring, seqno)
>-);
>-
>-DECLARE_EVENT_CLASS(i915_ring,
>-	    TP_PROTO(struct intel_engine_cs *ring),
>-	    TP_ARGS(ring),
>+TRACE_EVENT(i915_gem_request_wait_end,
>+	    TP_PROTO(struct i915_gem_request *rq),
>+	    TP_ARGS(rq),
> 
> 	    TP_STRUCT__entry(
> 			     __field(u32, dev)
> 			     __field(u32, ring)
>+			     __field(u32, seqno)
>+			     __field(bool, completed)
> 			     ),
> 
> 	    TP_fast_assign(
>-			   __entry->dev = ring->dev->primary->index;
>-			   __entry->ring = ring->id;
>+			   __entry->dev = rq->i915->dev->primary->index;
>+			   __entry->ring = rq->engine->id;
>+			   __entry->seqno = rq->seqno;
>+			   __entry->completed = rq->completed;
> 			   ),
> 
>-	    TP_printk("dev=%u, ring=%u", __entry->dev, __entry->ring)
>-);
>-
>-DEFINE_EVENT(i915_ring, i915_ring_wait_begin,
>-	    TP_PROTO(struct intel_engine_cs *ring),
>-	    TP_ARGS(ring)
>-);
>-
>-DEFINE_EVENT(i915_ring, i915_ring_wait_end,
>-	    TP_PROTO(struct intel_engine_cs *ring),
>-	    TP_ARGS(ring)
>+	    TP_printk("dev=%u, ring=%u, seqno=%x, completed=%s",
>+		      __entry->dev, __entry->ring, __entry->seqno,
>+		      __entry->completed ?  "yes" : "no")
> );
> 
> TRACE_EVENT(i915_flip_request,
>diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
>index eee79e1..32d2233 100644
>--- a/drivers/gpu/drm/i915/intel_bios.c
>+++ b/drivers/gpu/drm/i915/intel_bios.c
>@@ -976,12 +976,10 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
> 	if (bdb->version >= 158) {
> 		/* The VBT HDMI level shift values match the table we have. */
> 		hdmi_level_shift = child->raw[7] & 0xF;
>-		if (hdmi_level_shift < 0xC) {
>-			DRM_DEBUG_KMS("VBT HDMI level shift for port %c: %d\n",
>-				      port_name(port),
>-				      hdmi_level_shift);
>-			info->hdmi_level_shift = hdmi_level_shift;
>-		}
>+		DRM_DEBUG_KMS("VBT HDMI level shift for port %c: %d\n",
>+			      port_name(port),
>+			      hdmi_level_shift);
>+		info->hdmi_level_shift = hdmi_level_shift;
> 	}
> }
> 
>@@ -1114,8 +1112,7 @@ init_vbt_defaults(struct drm_i915_private *dev_priv)
> 		struct ddi_vbt_port_info *info =
> 			&dev_priv->vbt.ddi_port_info[port];
> 
>-		/* Recommended BSpec default: 800mV 0dB. */
>-		info->hdmi_level_shift = 6;
>+		info->hdmi_level_shift = HDMI_LEVEL_SHIFT_UNKNOWN;
> 
> 		info->supports_dvi = (port != PORT_A && port != PORT_E);
> 		info->supports_hdmi = info->supports_dvi;
>diff --git a/drivers/gpu/drm/i915/intel_bios.h b/drivers/gpu/drm/i915/intel_bios.h
>index b986677..7603765 100644
>--- a/drivers/gpu/drm/i915/intel_bios.h
>+++ b/drivers/gpu/drm/i915/intel_bios.h
>@@ -46,7 +46,7 @@ struct bdb_header {
> 	u16 version;			/**< decimal */
> 	u16 header_size;		/**< in bytes */
> 	u16 bdb_size;			/**< in bytes */
>-};
>+} __packed;
> 
> /* strictly speaking, this is a "skip" block, but it has interesting info */
> struct vbios_data {
>@@ -252,7 +252,7 @@ union child_device_config {
> 	/* This one should also be safe to use anywhere, even without version
> 	 * checks. */
> 	struct common_child_dev_config common;
>-};
>+} __packed;
> 
> struct bdb_general_definitions {
> 	/* DDC GPIO */
>@@ -802,7 +802,8 @@ struct mipi_config {
> 
> 	u16 rsvd4;
> 
>-	u8 rsvd5[5];
>+	u8 rsvd5;
>+	u32 target_burst_mode_freq;
> 	u32 dsi_ddr_clk;
> 	u32 bridge_ref_clk;
> 
>@@ -887,12 +888,12 @@ struct mipi_pps_data {
> 	u16 bl_disable_delay;
> 	u16 panel_off_delay;
> 	u16 panel_power_cycle_delay;
>-};
>+} __packed;
> 
> struct bdb_mipi_config {
> 	struct mipi_config config[MAX_MIPI_CONFIGURATIONS];
> 	struct mipi_pps_data pps[MAX_MIPI_CONFIGURATIONS];
>-};
>+} __packed;
> 
> /* Block 53 contains MIPI sequences as needed by the panel
>  * for enabling it. This block can be variable in size and
>@@ -901,7 +902,7 @@ struct bdb_mipi_config {
> struct bdb_mipi_sequence {
> 	u8 version;
> 	u8 data[0];
>-};
>+} __packed;
> 
> /* MIPI Sequnece Block definitions */
> enum mipi_seq {
>diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
>index 5db0b55..b63d4fa 100644
>--- a/drivers/gpu/drm/i915/intel_ddi.c
>+++ b/drivers/gpu/drm/i915/intel_ddi.c
>@@ -28,87 +28,103 @@
> #include "i915_drv.h"
> #include "intel_drv.h"
> 
>+struct ddi_buf_trans {
>+	u32 trans1;	/* balance leg enable, de-emph level */
>+	u32 trans2;	/* vref sel, vswing */
>+};
>+
> /* HDMI/DVI modes ignore everything but the last 2 items. So we share
>  * them for both DP and FDI transports, allowing those ports to
>  * automatically adapt to HDMI connections as well
>  */
>-static const u32 hsw_ddi_translations_dp[] = {
>-	0x00FFFFFF, 0x0006000E,		/* DP parameters */
>-	0x00D75FFF, 0x0005000A,
>-	0x00C30FFF, 0x00040006,
>-	0x80AAAFFF, 0x000B0000,
>-	0x00FFFFFF, 0x0005000A,
>-	0x00D75FFF, 0x000C0004,
>-	0x80C30FFF, 0x000B0000,
>-	0x00FFFFFF, 0x00040006,
>-	0x80D75FFF, 0x000B0000,
>+static const struct ddi_buf_trans hsw_ddi_translations_dp[] = {
>+	{ 0x00FFFFFF, 0x0006000E },
>+	{ 0x00D75FFF, 0x0005000A },
>+	{ 0x00C30FFF, 0x00040006 },
>+	{ 0x80AAAFFF, 0x000B0000 },
>+	{ 0x00FFFFFF, 0x0005000A },
>+	{ 0x00D75FFF, 0x000C0004 },
>+	{ 0x80C30FFF, 0x000B0000 },
>+	{ 0x00FFFFFF, 0x00040006 },
>+	{ 0x80D75FFF, 0x000B0000 },
> };
> 
>-static const u32 hsw_ddi_translations_fdi[] = {
>-	0x00FFFFFF, 0x0007000E,		/* FDI parameters */
>-	0x00D75FFF, 0x000F000A,
>-	0x00C30FFF, 0x00060006,
>-	0x00AAAFFF, 0x001E0000,
>-	0x00FFFFFF, 0x000F000A,
>-	0x00D75FFF, 0x00160004,
>-	0x00C30FFF, 0x001E0000,
>-	0x00FFFFFF, 0x00060006,
>-	0x00D75FFF, 0x001E0000,
>+static const struct ddi_buf_trans hsw_ddi_translations_fdi[] = {
>+	{ 0x00FFFFFF, 0x0007000E },
>+	{ 0x00D75FFF, 0x000F000A },
>+	{ 0x00C30FFF, 0x00060006 },
>+	{ 0x00AAAFFF, 0x001E0000 },
>+	{ 0x00FFFFFF, 0x000F000A },
>+	{ 0x00D75FFF, 0x00160004 },
>+	{ 0x00C30FFF, 0x001E0000 },
>+	{ 0x00FFFFFF, 0x00060006 },
>+	{ 0x00D75FFF, 0x001E0000 },
> };
> 
>-static const u32 hsw_ddi_translations_hdmi[] = {
>-				/* Idx	NT mV diff	T mV diff	db  */
>-	0x00FFFFFF, 0x0006000E, /* 0:	400		400		0   */
>-	0x00E79FFF, 0x000E000C, /* 1:	400		500		2   */
>-	0x00D75FFF, 0x0005000A, /* 2:	400		600		3.5 */
>-	0x00FFFFFF, 0x0005000A, /* 3:	600		600		0   */
>-	0x00E79FFF, 0x001D0007, /* 4:	600		750		2   */
>-	0x00D75FFF, 0x000C0004, /* 5:	600		900		3.5 */
>-	0x00FFFFFF, 0x00040006, /* 6:	800		800		0   */
>-	0x80E79FFF, 0x00030002, /* 7:	800		1000		2   */
>-	0x00FFFFFF, 0x00140005, /* 8:	850		850		0   */
>-	0x00FFFFFF, 0x000C0004, /* 9:	900		900		0   */
>-	0x00FFFFFF, 0x001C0003, /* 10:	950		950		0   */
>-	0x80FFFFFF, 0x00030002, /* 11:	1000		1000		0   */
>+static const struct ddi_buf_trans hsw_ddi_translations_hdmi[] = {
>+					/* Idx	NT mV d	T mV d	db	*/
>+	{ 0x00FFFFFF, 0x0006000E },	/* 0:	400	400	0	*/
>+	{ 0x00E79FFF, 0x000E000C },	/* 1:	400	500	2	*/
>+	{ 0x00D75FFF, 0x0005000A },	/* 2:	400	600	3.5	*/
>+	{ 0x00FFFFFF, 0x0005000A },	/* 3:	600	600	0	*/
>+	{ 0x00E79FFF, 0x001D0007 },	/* 4:	600	750	2	*/
>+	{ 0x00D75FFF, 0x000C0004 },	/* 5:	600	900	3.5	*/
>+	{ 0x00FFFFFF, 0x00040006 },	/* 6:	800	800	0	*/
>+	{ 0x80E79FFF, 0x00030002 },	/* 7:	800	1000	2	*/
>+	{ 0x00FFFFFF, 0x00140005 },	/* 8:	850	850	0	*/
>+	{ 0x00FFFFFF, 0x000C0004 },	/* 9:	900	900	0	*/
>+	{ 0x00FFFFFF, 0x001C0003 },	/* 10:	950	950	0	*/
>+	{ 0x80FFFFFF, 0x00030002 },	/* 11:	1000	1000	0	*/
> };
> 
>-static const u32 bdw_ddi_translations_edp[] = {
>-	0x00FFFFFF, 0x00000012,		/* eDP parameters */
>-	0x00EBAFFF, 0x00020011,
>-	0x00C71FFF, 0x0006000F,
>-	0x00AAAFFF, 0x000E000A,
>-	0x00FFFFFF, 0x00020011,
>-	0x00DB6FFF, 0x0005000F,
>-	0x00BEEFFF, 0x000A000C,
>-	0x00FFFFFF, 0x0005000F,
>-	0x00DB6FFF, 0x000A000C,
>-	0x00FFFFFF, 0x00140006		/* HDMI parameters 800mV 0dB*/
>+static const struct ddi_buf_trans bdw_ddi_translations_edp[] = {
>+	{ 0x00FFFFFF, 0x00000012 },
>+	{ 0x00EBAFFF, 0x00020011 },
>+	{ 0x00C71FFF, 0x0006000F },
>+	{ 0x00AAAFFF, 0x000E000A },
>+	{ 0x00FFFFFF, 0x00020011 },
>+	{ 0x00DB6FFF, 0x0005000F },
>+	{ 0x00BEEFFF, 0x000A000C },
>+	{ 0x00FFFFFF, 0x0005000F },
>+	{ 0x00DB6FFF, 0x000A000C },
> };
> 
>-static const u32 bdw_ddi_translations_dp[] = {
>-	0x00FFFFFF, 0x0007000E,		/* DP parameters */
>-	0x00D75FFF, 0x000E000A,
>-	0x00BEFFFF, 0x00140006,
>-	0x80B2CFFF, 0x001B0002,
>-	0x00FFFFFF, 0x000E000A,
>-	0x00D75FFF, 0x00180004,
>-	0x80CB2FFF, 0x001B0002,
>-	0x00F7DFFF, 0x00180004,
>-	0x80D75FFF, 0x001B0002,
>-	0x00FFFFFF, 0x00140006		/* HDMI parameters 800mV 0dB*/
>+static const struct ddi_buf_trans bdw_ddi_translations_dp[] = {
>+	{ 0x00FFFFFF, 0x0007000E },
>+	{ 0x00D75FFF, 0x000E000A },
>+	{ 0x00BEFFFF, 0x00140006 },
>+	{ 0x80B2CFFF, 0x001B0002 },
>+	{ 0x00FFFFFF, 0x000E000A },
>+	{ 0x00D75FFF, 0x00180004 },
>+	{ 0x80CB2FFF, 0x001B0002 },
>+	{ 0x00F7DFFF, 0x00180004 },
>+	{ 0x80D75FFF, 0x001B0002 },
> };
> 
>-static const u32 bdw_ddi_translations_fdi[] = {
>-	0x00FFFFFF, 0x0001000E,		/* FDI parameters */
>-	0x00D75FFF, 0x0004000A,
>-	0x00C30FFF, 0x00070006,
>-	0x00AAAFFF, 0x000C0000,
>-	0x00FFFFFF, 0x0004000A,
>-	0x00D75FFF, 0x00090004,
>-	0x00C30FFF, 0x000C0000,
>-	0x00FFFFFF, 0x00070006,
>-	0x00D75FFF, 0x000C0000,
>-	0x00FFFFFF, 0x00140006		/* HDMI parameters 800mV 0dB*/
>+static const struct ddi_buf_trans bdw_ddi_translations_fdi[] = {
>+	{ 0x00FFFFFF, 0x0001000E },
>+	{ 0x00D75FFF, 0x0004000A },
>+	{ 0x00C30FFF, 0x00070006 },
>+	{ 0x00AAAFFF, 0x000C0000 },
>+	{ 0x00FFFFFF, 0x0004000A },
>+	{ 0x00D75FFF, 0x00090004 },
>+	{ 0x00C30FFF, 0x000C0000 },
>+	{ 0x00FFFFFF, 0x00070006 },
>+	{ 0x00D75FFF, 0x000C0000 },
>+};
>+
>+static const struct ddi_buf_trans bdw_ddi_translations_hdmi[] = {
>+					/* Idx	NT mV d	T mV df	db	*/
>+	{ 0x00FFFFFF, 0x0007000E },	/* 0:	400	400	0	*/
>+	{ 0x00D75FFF, 0x000E000A },	/* 1:	400	600	3.5	*/
>+	{ 0x00BEFFFF, 0x00140006 },	/* 2:	400	800	6	*/
>+	{ 0x00FFFFFF, 0x0009000D },	/* 3:	450	450	0	*/
>+	{ 0x00FFFFFF, 0x000E000A },	/* 4:	600	600	0	*/
>+	{ 0x00D7FFFF, 0x00140006 },	/* 5:	600	800	2.5	*/
>+	{ 0x80CB2FFF, 0x001B0002 },	/* 6:	600	1000	4.5	*/
>+	{ 0x00FFFFFF, 0x00140006 },	/* 7:	800	800	0	*/
>+	{ 0x80E79FFF, 0x001B0002 },	/* 8:	800	1000	2	*/
>+	{ 0x80FFFFFF, 0x001B0002 },	/* 9:	1000	1000	0	*/
> };
> 
> enum port intel_ddi_get_encoder_port(struct intel_encoder *intel_encoder)
>@@ -145,26 +161,36 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	u32 reg;
>-	int i;
>+	int i, n_hdmi_entries, hdmi_800mV_0dB;
> 	int hdmi_level = dev_priv->vbt.ddi_port_info[port].hdmi_level_shift;
>-	const u32 *ddi_translations_fdi;
>-	const u32 *ddi_translations_dp;
>-	const u32 *ddi_translations_edp;
>-	const u32 *ddi_translations;
>+	const struct ddi_buf_trans *ddi_translations_fdi;
>+	const struct ddi_buf_trans *ddi_translations_dp;
>+	const struct ddi_buf_trans *ddi_translations_edp;
>+	const struct ddi_buf_trans *ddi_translations_hdmi;
>+	const struct ddi_buf_trans *ddi_translations;
> 
> 	if (IS_BROADWELL(dev)) {
> 		ddi_translations_fdi = bdw_ddi_translations_fdi;
> 		ddi_translations_dp = bdw_ddi_translations_dp;
> 		ddi_translations_edp = bdw_ddi_translations_edp;
>+		ddi_translations_hdmi = bdw_ddi_translations_hdmi;
>+		n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi);
>+		hdmi_800mV_0dB = 7;
> 	} else if (IS_HASWELL(dev)) {
> 		ddi_translations_fdi = hsw_ddi_translations_fdi;
> 		ddi_translations_dp = hsw_ddi_translations_dp;
> 		ddi_translations_edp = hsw_ddi_translations_dp;
>+		ddi_translations_hdmi = hsw_ddi_translations_hdmi;
>+		n_hdmi_entries = ARRAY_SIZE(hsw_ddi_translations_hdmi);
>+		hdmi_800mV_0dB = 6;
> 	} else {
> 		WARN(1, "ddi translation table missing\n");
> 		ddi_translations_edp = bdw_ddi_translations_dp;
> 		ddi_translations_fdi = bdw_ddi_translations_fdi;
> 		ddi_translations_dp = bdw_ddi_translations_dp;
>+		ddi_translations_hdmi = bdw_ddi_translations_hdmi;
>+		n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi);
>+		hdmi_800mV_0dB = 7;
> 	}
> 
> 	switch (port) {
>@@ -190,14 +216,22 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port)
> 
> 	for (i = 0, reg = DDI_BUF_TRANS(port);
> 	     i < ARRAY_SIZE(hsw_ddi_translations_fdi); i++) {
>-		I915_WRITE(reg, ddi_translations[i]);
>+		I915_WRITE(reg, ddi_translations[i].trans1);
> 		reg += 4;
>-	}
>-	/* Entry 9 is for HDMI: */
>-	for (i = 0; i < 2; i++) {
>-		I915_WRITE(reg, hsw_ddi_translations_hdmi[hdmi_level * 2 + i]);
>+		I915_WRITE(reg, ddi_translations[i].trans2);
> 		reg += 4;
> 	}
>+
>+	/* Choose a good default if VBT is badly populated */
>+	if (hdmi_level == HDMI_LEVEL_SHIFT_UNKNOWN ||
>+	    hdmi_level >= n_hdmi_entries)
>+		hdmi_level = hdmi_800mV_0dB;
>+
>+	/* Entry 9 is for HDMI: */
>+	I915_WRITE(reg, ddi_translations_hdmi[hdmi_level].trans1);
>+	reg += 4;
>+	I915_WRITE(reg, ddi_translations_hdmi[hdmi_level].trans2);
>+	reg += 4;
> }
> 
> /* Program DDI buffers translations for DP. By default, program ports A-D in DP
>@@ -214,18 +248,6 @@ void intel_prepare_ddi(struct drm_device *dev)
> 		intel_prepare_ddi_buffers(dev, port);
> }
> 
>-static const long hsw_ddi_buf_ctl_values[] = {
>-	DDI_BUF_EMP_400MV_0DB_HSW,
>-	DDI_BUF_EMP_400MV_3_5DB_HSW,
>-	DDI_BUF_EMP_400MV_6DB_HSW,
>-	DDI_BUF_EMP_400MV_9_5DB_HSW,
>-	DDI_BUF_EMP_600MV_0DB_HSW,
>-	DDI_BUF_EMP_600MV_3_5DB_HSW,
>-	DDI_BUF_EMP_600MV_6DB_HSW,
>-	DDI_BUF_EMP_800MV_0DB_HSW,
>-	DDI_BUF_EMP_800MV_3_5DB_HSW
>-};
>-
> static void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv,
> 				    enum port port)
> {
>@@ -285,7 +307,7 @@ void hsw_fdi_link_train(struct drm_crtc *crtc)
> 
> 	/* Start the training iterating through available voltages and emphasis,
> 	 * testing each value twice. */
>-	for (i = 0; i < ARRAY_SIZE(hsw_ddi_buf_ctl_values) * 2; i++) {
>+	for (i = 0; i < ARRAY_SIZE(hsw_ddi_translations_fdi) * 2; i++) {
> 		/* Configure DP_TP_CTL with auto-training */
> 		I915_WRITE(DP_TP_CTL(PORT_E),
> 					DP_TP_CTL_FDI_AUTOTRAIN |
>@@ -300,7 +322,7 @@ void hsw_fdi_link_train(struct drm_crtc *crtc)
> 		I915_WRITE(DDI_BUF_CTL(PORT_E),
> 			   DDI_BUF_CTL_ENABLE |
> 			   ((intel_crtc->config.fdi_lanes - 1) << 1) |
>-			   hsw_ddi_buf_ctl_values[i / 2]);
>+			   DDI_BUF_TRANS_SELECT(i / 2));
> 		POSTING_READ(DDI_BUF_CTL(PORT_E));
> 
> 		udelay(600);
>@@ -375,7 +397,7 @@ void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder)
> 		enc_to_dig_port(&encoder->base);
> 
> 	intel_dp->DP = intel_dig_port->saved_port_bits |
>-		DDI_BUF_CTL_ENABLE | DDI_BUF_EMP_400MV_0DB_HSW;
>+		DDI_BUF_CTL_ENABLE | DDI_BUF_TRANS_SELECT(0);
> 	intel_dp->DP |= DDI_PORT_WIDTH(intel_dp->lane_count);
> 
> }
>@@ -402,7 +424,7 @@ intel_ddi_get_crtc_encoder(struct drm_crtc *crtc)
> }
> 
> #define LC_FREQ 2700
>-#define LC_FREQ_2K (LC_FREQ * 2000)
>+#define LC_FREQ_2K U64_C(LC_FREQ * 2000)
> 
> #define P_MIN 2
> #define P_MAX 64
>@@ -414,7 +436,11 @@ intel_ddi_get_crtc_encoder(struct drm_crtc *crtc)
> #define VCO_MIN 2400
> #define VCO_MAX 4800
> 
>-#define ABS_DIFF(a, b) ((a > b) ? (a - b) : (b - a))
>+#define abs_diff(a, b) ({			\
>+	typeof(a) __a = (a);			\
>+	typeof(b) __b = (b);			\
>+	(void) (&__a == &__b);			\
>+	__a > __b ? (__a - __b) : (__b - __a); })
> 
> struct wrpll_rnp {
> 	unsigned p, n2, r2;
>@@ -524,9 +550,9 @@ static void wrpll_update_rnp(uint64_t freq2k, unsigned budget,
> 	 */
> 	a = freq2k * budget * p * r2;
> 	b = freq2k * budget * best->p * best->r2;
>-	diff = ABS_DIFF((freq2k * p * r2), (LC_FREQ_2K * n2));
>-	diff_best = ABS_DIFF((freq2k * best->p * best->r2),
>-			     (LC_FREQ_2K * best->n2));
>+	diff = abs_diff(freq2k * p * r2, LC_FREQ_2K * n2);
>+	diff_best = abs_diff(freq2k * best->p * best->r2,
>+			     LC_FREQ_2K * best->n2);
> 	c = 1000000 * diff;
> 	d = 1000000 * diff_best;
> 
>@@ -587,8 +613,8 @@ static int intel_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv,
> 	return (refclk * n * 100) / (p * r);
> }
> 
>-void intel_ddi_clock_get(struct intel_encoder *encoder,
>-			 struct intel_crtc_config *pipe_config)
>+static void hsw_ddi_clock_get(struct intel_encoder *encoder,
>+			      struct intel_crtc_config *pipe_config)
> {
> 	struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
> 	int link_clock = 0;
>@@ -643,9 +669,15 @@ void intel_ddi_clock_get(struct intel_encoder *encoder,
> 		pipe_config->adjusted_mode.crtc_clock = pipe_config->port_clock;
> }
> 
>+void intel_ddi_clock_get(struct intel_encoder *encoder,
>+			 struct intel_crtc_config *pipe_config)
>+{
>+	hsw_ddi_clock_get(encoder, pipe_config);
>+}
>+
> static void
>-intel_ddi_calculate_wrpll(int clock /* in Hz */,
>-			  unsigned *r2_out, unsigned *n2_out, unsigned *p_out)
>+hsw_ddi_calculate_wrpll(int clock /* in Hz */,
>+			unsigned *r2_out, unsigned *n2_out, unsigned *p_out)
> {
> 	uint64_t freq2k;
> 	unsigned p, n2, r2;
>@@ -708,27 +740,17 @@ intel_ddi_calculate_wrpll(int clock /* in Hz */,
> 	*r2_out = best.r2;
> }
> 
>-/*
>- * Tries to find a PLL for the CRTC. If it finds, it increases the refcount and
>- * stores it in intel_crtc->ddi_pll_sel, so other mode sets won't be able to
>- * steal the selected PLL. You need to call intel_ddi_pll_enable to actually
>- * enable the PLL.
>- */
>-bool intel_ddi_pll_select(struct intel_crtc *intel_crtc)
>+static bool
>+hsw_ddi_pll_select(struct intel_crtc *intel_crtc,
>+		   struct intel_encoder *intel_encoder,
>+		   int clock)
> {
>-	struct drm_crtc *crtc = &intel_crtc->base;
>-	struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
>-	int type = intel_encoder->type;
>-	int clock = intel_crtc->config.port_clock;
>-
>-	intel_put_shared_dpll(intel_crtc);
>-
>-	if (type == INTEL_OUTPUT_HDMI) {
>+	if (intel_encoder->type == INTEL_OUTPUT_HDMI) {
> 		struct intel_shared_dpll *pll;
> 		uint32_t val;
> 		unsigned p, n2, r2;
> 
>-		intel_ddi_calculate_wrpll(clock * 1000, &r2, &n2, &p);
>+		hsw_ddi_calculate_wrpll(clock * 1000, &r2, &n2, &p);
> 
> 		val = WRPLL_PLL_ENABLE | WRPLL_PLL_LCPLL |
> 		      WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) |
>@@ -749,6 +771,25 @@ bool intel_ddi_pll_select(struct intel_crtc *intel_crtc)
> 	return true;
> }
> 
>+
>+/*
>+ * Tries to find a *shared* PLL for the CRTC and store it in
>+ * intel_crtc->ddi_pll_sel.
>+ *
>+ * For private DPLLs, compute_config() should do the selection for us. This
>+ * function should be folded into compute_config() eventually.
>+ */
>+bool intel_ddi_pll_select(struct intel_crtc *intel_crtc)
>+{
>+	struct drm_crtc *crtc = &intel_crtc->base;
>+	struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
>+	int clock = intel_crtc->config.port_clock;
>+
>+	intel_put_shared_dpll(intel_crtc);
>+
>+	return hsw_ddi_pll_select(intel_crtc, intel_encoder, clock);
>+}
>+
> void intel_ddi_set_pipe_settings(struct drm_crtc *crtc)
> {
> 	struct drm_i915_private *dev_priv = crtc->dev->dev_private;
>@@ -1183,31 +1224,52 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder)
> 	}
> }
> 
>-int intel_ddi_get_cdclk_freq(struct drm_i915_private *dev_priv)
>+static int bdw_get_cdclk_freq(struct drm_i915_private *dev_priv)
>+{
>+	uint32_t lcpll = I915_READ(LCPLL_CTL);
>+	uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
>+
>+	if (lcpll & LCPLL_CD_SOURCE_FCLK)
>+		return 800000;
>+	else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
>+		return 450000;
>+	else if (freq == LCPLL_CLK_FREQ_450)
>+		return 450000;
>+	else if (freq == LCPLL_CLK_FREQ_54O_BDW)
>+		return 540000;
>+	else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
>+		return 337500;
>+	else
>+		return 675000;
>+}
>+
>+static int hsw_get_cdclk_freq(struct drm_i915_private *dev_priv)
> {
> 	struct drm_device *dev = dev_priv->dev;
> 	uint32_t lcpll = I915_READ(LCPLL_CTL);
> 	uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
> 
>-	if (lcpll & LCPLL_CD_SOURCE_FCLK) {
>+	if (lcpll & LCPLL_CD_SOURCE_FCLK)
> 		return 800000;
>-	} else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT) {
>+	else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
> 		return 450000;
>-	} else if (freq == LCPLL_CLK_FREQ_450) {
>+	else if (freq == LCPLL_CLK_FREQ_450)
> 		return 450000;
>-	} else if (IS_HASWELL(dev)) {
>-		if (IS_ULT(dev))
>-			return 337500;
>-		else
>-			return 540000;
>-	} else {
>-		if (freq == LCPLL_CLK_FREQ_54O_BDW)
>-			return 540000;
>-		else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
>-			return 337500;
>-		else
>-			return 675000;
>-	}
>+	else if (IS_ULT(dev))
>+		return 337500;
>+	else
>+		return 540000;
>+}
>+
>+int intel_ddi_get_cdclk_freq(struct drm_i915_private *dev_priv)
>+{
>+	struct drm_device *dev = dev_priv->dev;
>+
>+	if (IS_BROADWELL(dev))
>+		return bdw_get_cdclk_freq(dev_priv);
>+
>+	/* Haswell */
>+	return hsw_get_cdclk_freq(dev_priv);
> }
> 
> static void hsw_ddi_pll_enable(struct drm_i915_private *dev_priv,
>@@ -1248,10 +1310,8 @@ static const char * const hsw_ddi_pll_names[] = {
> 	"WRPLL 2",
> };
> 
>-void intel_ddi_pll_init(struct drm_device *dev)
>+static void hsw_shared_dplls_init(struct drm_i915_private *dev_priv)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	uint32_t val = I915_READ(LCPLL_CTL);
> 	int i;
> 
> 	dev_priv->num_shared_dpll = 2;
>@@ -1264,6 +1324,14 @@ void intel_ddi_pll_init(struct drm_device *dev)
> 		dev_priv->shared_dplls[i].get_hw_state =
> 			hsw_ddi_pll_get_hw_state;
> 	}
>+}
>+
>+void intel_ddi_pll_init(struct drm_device *dev)
>+{
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	uint32_t val = I915_READ(LCPLL_CTL);
>+
>+	hsw_shared_dplls_init(dev_priv);
> 
> 	/* The LCPLL register should be turned on by the BIOS. For now let's
> 	 * just check its state and print errors in case something is wrong.
>@@ -1444,7 +1512,7 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
> 		dev_priv->vbt.edp_bpp = pipe_config->pipe_bpp;
> 	}
> 
>-	intel_ddi_clock_get(encoder, pipe_config);
>+	hsw_ddi_clock_get(encoder, pipe_config);
> }
> 
> static void intel_ddi_destroy(struct drm_encoder *encoder)
>diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
>index ee0ca36..add26d0 100644
>--- a/drivers/gpu/drm/i915/intel_display.c
>+++ b/drivers/gpu/drm/i915/intel_display.c
>@@ -91,15 +91,16 @@ static int intel_framebuffer_init(struct drm_device *dev,
> 				  struct intel_framebuffer *ifb,
> 				  struct drm_mode_fb_cmd2 *mode_cmd,
> 				  struct drm_i915_gem_object *obj);
>-static void intel_dp_set_m_n(struct intel_crtc *crtc);
> static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc);
> static void intel_set_pipe_timings(struct intel_crtc *intel_crtc);
> static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
>-					 struct intel_link_m_n *m_n);
>+					 struct intel_link_m_n *m_n,
>+					 struct intel_link_m_n *m2_n2);
> static void ironlake_set_pipeconf(struct drm_crtc *crtc);
> static void haswell_set_pipeconf(struct drm_crtc *crtc);
> static void intel_set_pipe_csc(struct drm_crtc *crtc);
> static void vlv_prepare_pll(struct intel_crtc *crtc);
>+static void chv_prepare_pll(struct intel_crtc *crtc);
> 
> static struct intel_encoder *intel_find_encoder(struct intel_connector *connector, int pipe)
> {
>@@ -964,8 +965,7 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
> 
> /*
>  * intel_wait_for_pipe_off - wait for pipe to turn off
>- * @dev: drm device
>- * @pipe: pipe to wait for
>+ * @crtc: crtc whose pipe to wait for
>  *
>  * After disabling a pipe, we can't wait for vblank in the usual way,
>  * spinning on the vblank interrupt status bit, since we won't actually
>@@ -979,11 +979,12 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
>  *   ends up stopping at the start of the next frame).
>  *
>  */
>-void intel_wait_for_pipe_off(struct drm_device *dev, int pipe)
>+static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
> {
>+	struct drm_device *dev = crtc->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
>-								      pipe);
>+	enum transcoder cpu_transcoder = crtc->config.cpu_transcoder;
>+	enum pipe pipe = crtc->pipe;
> 
> 	if (INTEL_INFO(dev)->gen >= 4) {
> 		int reg = PIPECONF(cpu_transcoder);
>@@ -1192,27 +1193,40 @@ void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
> static void assert_panel_unlocked(struct drm_i915_private *dev_priv,
> 				  enum pipe pipe)
> {
>-	int pp_reg, lvds_reg;
>+	struct drm_device *dev = dev_priv->dev;
>+	int pp_reg;
> 	u32 val;
> 	enum pipe panel_pipe = PIPE_A;
> 	bool locked = true;
> 
>-	if (HAS_PCH_SPLIT(dev_priv->dev)) {
>+	if (WARN_ON(HAS_DDI(dev)))
>+		return;
>+
>+	if (HAS_PCH_SPLIT(dev)) {
>+		u32 port_sel;
>+
> 		pp_reg = PCH_PP_CONTROL;
>-		lvds_reg = PCH_LVDS;
>+		port_sel = I915_READ(PCH_PP_ON_DELAYS) & PANEL_PORT_SELECT_MASK;
>+
>+		if (port_sel == PANEL_PORT_SELECT_LVDS &&
>+		    I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT)
>+			panel_pipe = PIPE_B;
>+		/* XXX: else fix for eDP */
>+	} else if (IS_VALLEYVIEW(dev)) {
>+		/* presumably write lock depends on pipe, not port select */
>+		pp_reg = VLV_PIPE_PP_CONTROL(pipe);
>+		panel_pipe = pipe;
> 	} else {
> 		pp_reg = PP_CONTROL;
>-		lvds_reg = LVDS;
>+		if (I915_READ(LVDS) & LVDS_PIPEB_SELECT)
>+			panel_pipe = PIPE_B;
> 	}
> 
> 	val = I915_READ(pp_reg);
> 	if (!(val & PANEL_POWER_ON) ||
>-	    ((val & PANEL_UNLOCK_REGS) == PANEL_UNLOCK_REGS))
>+	    ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
> 		locked = false;
> 
>-	if (I915_READ(lvds_reg) & LVDS_PIPEB_SELECT)
>-		panel_pipe = PIPE_B;
>-
> 	WARN(panel_pipe == pipe && locked,
> 	     "panel assertion failure, pipe %c regs locked\n",
> 	     pipe_name(pipe));
>@@ -1245,8 +1259,9 @@ void assert_pipe(struct drm_i915_private *dev_priv,
> 	enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
> 								      pipe);
> 
>-	/* if we need the pipe A quirk it must be always on */
>-	if (pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE)
>+	/* if we need the pipe quirk it must be always on */
>+	if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
>+	    (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
> 		state = true;
> 
> 	if (!intel_display_power_enabled(dev_priv,
>@@ -1300,7 +1315,7 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv,
> 	}
> 
> 	/* Need to check both planes against the pipe */
>-	for_each_pipe(i) {
>+	for_each_pipe(dev_priv, i) {
> 		reg = DSPCNTR(i);
> 		val = I915_READ(reg);
> 		cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
>@@ -1341,6 +1356,12 @@ static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
> 	}
> }
> 
>+static void assert_vblank_disabled(struct drm_crtc *crtc)
>+{
>+	if (drm_crtc_vblank_get(crtc) == 0)
>+		drm_crtc_vblank_put(crtc);
>+}
>+
> static void ibx_assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
> {
> 	u32 val;
>@@ -1513,34 +1534,6 @@ static void intel_init_dpio(struct drm_device *dev)
> 	}
> }
> 
>-static void intel_reset_dpio(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-
>-	if (IS_CHERRYVIEW(dev)) {
>-		enum dpio_phy phy;
>-		u32 val;
>-
>-		for (phy = DPIO_PHY0; phy < I915_NUM_PHYS_VLV; phy++) {
>-			/* Poll for phypwrgood signal */
>-			if (wait_for(I915_READ(DISPLAY_PHY_STATUS) &
>-						PHY_POWERGOOD(phy), 1))
>-				DRM_ERROR("Display PHY %d is not power up\n", phy);
>-
>-			/*
>-			 * Deassert common lane reset for PHY.
>-			 *
>-			 * This should only be done on init and resume from S3
>-			 * with both PLLs disabled, or we risk losing DPIO and
>-			 * PLL synchronization.
>-			 */
>-			val = I915_READ(DISPLAY_PHY_CONTROL);
>-			I915_WRITE(DISPLAY_PHY_CONTROL,
>-				PHY_COM_LANE_RESET_DEASSERT(phy, val));
>-		}
>-	}
>-}
>-
> static void vlv_enable_pll(struct intel_crtc *crtc)
> {
> 	struct drm_device *dev = crtc->base.dev;
>@@ -1554,7 +1547,7 @@ static void vlv_enable_pll(struct intel_crtc *crtc)
> 	BUG_ON(!IS_VALLEYVIEW(dev_priv->dev));
> 
> 	/* PLL is protected by panel, make sure we can write it */
>-	if (IS_MOBILE(dev_priv->dev) && !IS_I830(dev_priv->dev))
>+	if (IS_MOBILE(dev_priv->dev))
> 		assert_panel_unlocked(dev_priv, crtc->pipe);
> 
> 	I915_WRITE(reg, dpll);
>@@ -1617,6 +1610,18 @@ static void chv_enable_pll(struct intel_crtc *crtc)
> 	mutex_unlock(&dev_priv->dpio_lock);
> }
> 
>+static int intel_num_dvo_pipes(struct drm_device *dev)
>+{
>+	struct intel_crtc *crtc;
>+	int count = 0;
>+
>+	for_each_intel_crtc(dev, crtc)
>+		count += crtc->active &&
>+			intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DVO);
>+
>+	return count;
>+}
>+
> static void i9xx_enable_pll(struct intel_crtc *crtc)
> {
> 	struct drm_device *dev = crtc->base.dev;
>@@ -1633,7 +1638,18 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
> 	if (IS_MOBILE(dev) && !IS_I830(dev))
> 		assert_panel_unlocked(dev_priv, crtc->pipe);
> 
>-	I915_WRITE(reg, dpll);
>+	/* Enable DVO 2x clock on both PLLs if necessary */
>+	if (IS_I830(dev) && intel_num_dvo_pipes(dev) > 0) {
>+		/*
>+		 * It appears to be important that we don't enable this
>+		 * for the current pipe before otherwise configuring the
>+		 * PLL. No idea how this should be handled if multiple
>+		 * DVO outputs are enabled simultaneosly.
>+		 */
>+		dpll |= DPLL_DVO_2X_MODE;
>+		I915_WRITE(DPLL(!crtc->pipe),
>+			   I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE);
>+	}
> 
> 	/* Wait for the clocks to stabilize. */
> 	POSTING_READ(reg);
>@@ -1672,10 +1688,25 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
>  *
>  * Note!  This is for pre-ILK only.
>  */
>-static void i9xx_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
>+static void i9xx_disable_pll(struct intel_crtc *crtc)
> {
>-	/* Don't disable pipe A or pipe A PLLs if needed */
>-	if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
>+	struct drm_device *dev = crtc->base.dev;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	enum pipe pipe = crtc->pipe;
>+
>+	/* Disable DVO 2x clock on both PLLs if necessary */
>+	if (IS_I830(dev) &&
>+	    intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DVO) &&
>+	    intel_num_dvo_pipes(dev) == 1) {
>+		I915_WRITE(DPLL(PIPE_B),
>+			   I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
>+		I915_WRITE(DPLL(PIPE_A),
>+			   I915_READ(DPLL(PIPE_A)) & ~DPLL_DVO_2X_MODE);
>+	}
>+
>+	/* Don't disable pipe or pipe PLLs if needed */
>+	if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
>+	    (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
> 		return;
> 
> 	/* Make sure the pipe isn't still relying on us */
>@@ -1712,7 +1743,7 @@ static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
> 	assert_pipe_disabled(dev_priv, pipe);
> 
> 	/* Set PLL en = 0 */
>-	val = DPLL_SSC_REF_CLOCK_CHV;
>+	val = DPLL_SSC_REF_CLOCK_CHV | DPLL_REFA_CLK_ENABLE_VLV;
> 	if (pipe != PIPE_A)
> 		val |= DPLL_INTEGRATED_CRI_CLK_VLV;
> 	I915_WRITE(DPLL(pipe), val);
>@@ -1806,7 +1837,7 @@ static void intel_enable_shared_dpll(struct intel_crtc *crtc)
> 	if (WARN_ON(pll->refcount == 0))
> 		return;
> 
>-	DRM_DEBUG_KMS("enable %s (active %d, on? %d)for crtc %d\n",
>+	DRM_DEBUG_KMS("enable %s (active %d, on? %d) for crtc %d\n",
> 		      pll->name, pll->active, pll->on,
> 		      crtc->base.base.id);
> 
>@@ -1824,7 +1855,7 @@ static void intel_enable_shared_dpll(struct intel_crtc *crtc)
> 	pll->on = true;
> }
> 
>-void intel_disable_shared_dpll(struct intel_crtc *crtc)
>+static void intel_disable_shared_dpll(struct intel_crtc *crtc)
> {
> 	struct drm_device *dev = crtc->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>@@ -1868,7 +1899,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
> 	uint32_t reg, val, pipeconf_val;
> 
> 	/* PCH only available on ILK+ */
>-	BUG_ON(INTEL_INFO(dev)->gen < 5);
>+	BUG_ON(!HAS_PCH_SPLIT(dev));
> 
> 	/* Make sure PCH DPLL is enabled */
> 	assert_shared_dpll_enabled(dev_priv,
>@@ -1921,7 +1952,7 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
> 	u32 val, pipeconf_val;
> 
> 	/* PCH only available on ILK+ */
>-	BUG_ON(INTEL_INFO(dev_priv->dev)->gen < 5);
>+	BUG_ON(!HAS_PCH_SPLIT(dev_priv->dev));
> 
> 	/* FDI must be feeding us bits for PCH ports */
> 	assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
>@@ -2043,8 +2074,8 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
> 	reg = PIPECONF(cpu_transcoder);
> 	val = I915_READ(reg);
> 	if (val & PIPECONF_ENABLE) {
>-		WARN_ON(!(pipe == PIPE_A &&
>-			  dev_priv->quirks & QUIRK_PIPEA_FORCE));
>+		WARN_ON(!((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
>+			  (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)));
> 		return;
> 	}
> 
>@@ -2054,21 +2085,19 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
> 
> /**
>  * intel_disable_pipe - disable a pipe, asserting requirements
>- * @dev_priv: i915 private structure
>- * @pipe: pipe to disable
>- *
>- * Disable @pipe, making sure that various hardware specific requirements
>- * are met, if applicable, e.g. plane disabled, panel fitter off, etc.
>+ * @crtc: crtc whose pipes is to be disabled
>  *
>- * @pipe should be %PIPE_A or %PIPE_B.
>+ * Disable the pipe of @crtc, making sure that various hardware
>+ * specific requirements are met, if applicable, e.g. plane
>+ * disabled, panel fitter off, etc.
>  *
>  * Will wait until the pipe has shut down before returning.
>  */
>-static void intel_disable_pipe(struct drm_i915_private *dev_priv,
>-			       enum pipe pipe)
>+static void intel_disable_pipe(struct intel_crtc *crtc)
> {
>-	enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
>-								      pipe);
>+	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
>+	enum transcoder cpu_transcoder = crtc->config.cpu_transcoder;
>+	enum pipe pipe = crtc->pipe;
> 	int reg;
> 	u32 val;
> 
>@@ -2080,17 +2109,26 @@ static void intel_disable_pipe(struct drm_i915_private *dev_priv,
> 	assert_cursor_disabled(dev_priv, pipe);
> 	assert_sprites_disabled(dev_priv, pipe);
> 
>-	/* Don't disable pipe A or pipe A PLLs if needed */
>-	if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
>-		return;
>-
> 	reg = PIPECONF(cpu_transcoder);
> 	val = I915_READ(reg);
> 	if ((val & PIPECONF_ENABLE) == 0)
> 		return;
> 
>-	I915_WRITE(reg, val & ~PIPECONF_ENABLE);
>-	intel_wait_for_pipe_off(dev_priv->dev, pipe);
>+	/*
>+	 * Double wide has implications for planes
>+	 * so best keep it disabled when not needed.
>+	 */
>+	if (crtc->config.double_wide)
>+		val &= ~PIPECONF_DOUBLE_WIDE;
>+
>+	/* Don't disable pipe or pipe PLLs if needed */
>+	if (!(pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) &&
>+	    !(pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
>+		val &= ~PIPECONF_ENABLE;
>+
>+	I915_WRITE(reg, val);
>+	if ((val & PIPECONF_ENABLE) == 0)
>+		intel_wait_for_pipe_off(crtc);
> }
> 
> /*
>@@ -2109,35 +2147,28 @@ void intel_flush_primary_plane(struct drm_i915_private *dev_priv,
> 
> /**
>  * intel_enable_primary_hw_plane - enable the primary plane on a given pipe
>- * @dev_priv: i915 private structure
>- * @plane: plane to enable
>- * @pipe: pipe being fed
>+ * @plane:  plane to be enabled
>+ * @crtc: crtc for the plane
>  *
>- * Enable @plane on @pipe, making sure that @pipe is running first.
>+ * Enable @plane on @crtc, making sure that the pipe is running first.
>  */
>-static void intel_enable_primary_hw_plane(struct drm_i915_private *dev_priv,
>-					  enum plane plane, enum pipe pipe)
>+static void intel_enable_primary_hw_plane(struct drm_plane *plane,
>+					  struct drm_crtc *crtc)
> {
>-	struct drm_device *dev = dev_priv->dev;
>-	struct intel_crtc *intel_crtc =
>-		to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
>-	int reg;
>-	u32 val;
>+	struct drm_device *dev = plane->dev;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 
> 	/* If the pipe isn't enabled, we can't pump pixels and may hang */
>-	assert_pipe_enabled(dev_priv, pipe);
>+	assert_pipe_enabled(dev_priv, intel_crtc->pipe);
> 
> 	if (intel_crtc->primary_enabled)
> 		return;
> 
> 	intel_crtc->primary_enabled = true;
> 
>-	reg = DSPCNTR(plane);
>-	val = I915_READ(reg);
>-	WARN_ON(val & DISPLAY_PLANE_ENABLE);
>-
>-	I915_WRITE(reg, val | DISPLAY_PLANE_ENABLE);
>-	intel_flush_primary_plane(dev_priv, plane);
>+	dev_priv->display.update_primary_plane(crtc, plane->fb,
>+					       crtc->x, crtc->y);
> 
> 	/*
> 	 * BDW signals flip done immediately if the plane
>@@ -2150,31 +2181,27 @@ static void intel_enable_primary_hw_plane(struct drm_i915_private *dev_priv,
> 
> /**
>  * intel_disable_primary_hw_plane - disable the primary hardware plane
>- * @dev_priv: i915 private structure
>- * @plane: plane to disable
>- * @pipe: pipe consuming the data
>+ * @plane: plane to be disabled
>+ * @crtc: crtc for the plane
>  *
>- * Disable @plane; should be an independent operation.
>+ * Disable @plane on @crtc, making sure that the pipe is running first.
>  */
>-static void intel_disable_primary_hw_plane(struct drm_i915_private *dev_priv,
>-					   enum plane plane, enum pipe pipe)
>+static void intel_disable_primary_hw_plane(struct drm_plane *plane,
>+					   struct drm_crtc *crtc)
> {
>-	struct intel_crtc *intel_crtc =
>-		to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
>-	int reg;
>-	u32 val;
>+	struct drm_device *dev = plane->dev;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+
>+	assert_pipe_enabled(dev_priv, intel_crtc->pipe);
> 
> 	if (!intel_crtc->primary_enabled)
> 		return;
> 
> 	intel_crtc->primary_enabled = false;
> 
>-	reg = DSPCNTR(plane);
>-	val = I915_READ(reg);
>-	WARN_ON((val & DISPLAY_PLANE_ENABLE) == 0);
>-
>-	I915_WRITE(reg, val & ~DISPLAY_PLANE_ENABLE);
>-	intel_flush_primary_plane(dev_priv, plane);
>+	dev_priv->display.update_primary_plane(crtc, plane->fb,
>+					       crtc->x, crtc->y);
> }
> 
> static bool need_vtd_wa(struct drm_device *dev)
>@@ -2197,7 +2224,7 @@ static int intel_align_height(struct drm_device *dev, int height, bool tiled)
> int
> intel_pin_and_fence_fb_obj(struct drm_device *dev,
> 			   struct drm_i915_gem_object *obj,
>-			   struct intel_engine_cs *pipelined)
>+			   struct i915_gem_request *pipelined)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	u32 alignment;
>@@ -2422,16 +2449,46 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc,
> 	struct drm_device *dev = crtc->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>-	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
>+	struct drm_i915_gem_object *obj;
> 	int plane = intel_crtc->plane;
> 	unsigned long linear_offset;
> 	u32 dspcntr;
>-	u32 reg;
>+	u32 reg = DSPCNTR(plane);
>+	int pixel_size;
>+
>+	if (!intel_crtc->primary_enabled) {
>+		I915_WRITE(reg, 0);
>+		if (INTEL_INFO(dev)->gen >= 4)
>+			I915_WRITE(DSPSURF(plane), 0);
>+		else
>+			I915_WRITE(DSPADDR(plane), 0);
>+		POSTING_READ(reg);
>+		return;
>+	}
>+
>+	obj = intel_fb_obj(fb);
>+	if (WARN_ON(obj == NULL))
>+		return;
>+
>+	pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
>+
>+	dspcntr = DISPPLANE_GAMMA_ENABLE;
>+
>+	dspcntr |= DISPLAY_PLANE_ENABLE;
>+
>+	if (INTEL_INFO(dev)->gen < 4) {
>+		if (intel_crtc->pipe == PIPE_B)
>+			dspcntr |= DISPPLANE_SEL_PIPE_B;
>+
>+		/* pipesrc and dspsize control the size that is scaled from,
>+		 * which should always be the user's requested size.
>+		 */
>+		I915_WRITE(DSPSIZE(plane),
>+			   ((intel_crtc->config.pipe_src_h - 1) << 16) |
>+			   (intel_crtc->config.pipe_src_w - 1));
>+		I915_WRITE(DSPPOS(plane), 0);
>+	}
> 
>-	reg = DSPCNTR(plane);
>-	dspcntr = I915_READ(reg);
>-	/* Mask out pixel format bits in case we change it */
>-	dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
> 	switch (fb->pixel_format) {
> 	case DRM_FORMAT_C8:
> 		dspcntr |= DISPPLANE_8BPP;
>@@ -2463,30 +2520,40 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc,
> 		BUG();
> 	}
> 
>-	if (INTEL_INFO(dev)->gen >= 4) {
>-		if (obj->tiling_mode != I915_TILING_NONE)
>-			dspcntr |= DISPPLANE_TILED;
>-		else
>-			dspcntr &= ~DISPPLANE_TILED;
>-	}
>+	if (INTEL_INFO(dev)->gen >= 4 &&
>+	    obj->tiling_mode != I915_TILING_NONE)
>+		dspcntr |= DISPPLANE_TILED;
> 
> 	if (IS_G4X(dev))
> 		dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
> 
>-	I915_WRITE(reg, dspcntr);
>-
>-	linear_offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8);
>+	linear_offset = y * fb->pitches[0] + x * pixel_size;
> 
> 	if (INTEL_INFO(dev)->gen >= 4) {
> 		intel_crtc->dspaddr_offset =
> 			intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode,
>-						       fb->bits_per_pixel / 8,
>+						       pixel_size,
> 						       fb->pitches[0]);
> 		linear_offset -= intel_crtc->dspaddr_offset;
> 	} else {
> 		intel_crtc->dspaddr_offset = linear_offset;
> 	}
> 
>+	if (to_intel_plane(crtc->primary)->rotation == BIT(DRM_ROTATE_180)) {
>+		dspcntr |= DISPPLANE_ROTATE_180;
>+
>+		x += (intel_crtc->config.pipe_src_w - 1);
>+		y += (intel_crtc->config.pipe_src_h - 1);
>+
>+		/* Finding the last pixel of the last line of the display
>+		data and adding to linear_offset*/
>+		linear_offset +=
>+			(intel_crtc->config.pipe_src_h - 1) * fb->pitches[0] +
>+			(intel_crtc->config.pipe_src_w - 1) * pixel_size;
>+	}
>+
>+	I915_WRITE(reg, dspcntr);
>+
> 	DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
> 		      i915_gem_obj_ggtt_offset(obj), linear_offset, x, y,
> 		      fb->pitches[0]);
>@@ -2508,16 +2575,33 @@ static void ironlake_update_primary_plane(struct drm_crtc *crtc,
> 	struct drm_device *dev = crtc->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>-	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
>+	struct drm_i915_gem_object *obj;
> 	int plane = intel_crtc->plane;
> 	unsigned long linear_offset;
> 	u32 dspcntr;
>-	u32 reg;
>+	u32 reg = DSPCNTR(plane);
>+	int pixel_size;
>+
>+	if (!intel_crtc->primary_enabled) {
>+		I915_WRITE(reg, 0);
>+		I915_WRITE(DSPSURF(plane), 0);
>+		POSTING_READ(reg);
>+		return;
>+	}
>+
>+	obj = intel_fb_obj(fb);
>+	if (WARN_ON(obj == NULL))
>+		return;
>+
>+	pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
>+
>+	dspcntr = DISPPLANE_GAMMA_ENABLE;
>+
>+	dspcntr |= DISPLAY_PLANE_ENABLE;
>+
>+	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
>+		dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
> 
>-	reg = DSPCNTR(plane);
>-	dspcntr = I915_READ(reg);
>-	/* Mask out pixel format bits in case we change it */
>-	dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
> 	switch (fb->pixel_format) {
> 	case DRM_FORMAT_C8:
> 		dspcntr |= DISPPLANE_8BPP;
>@@ -2547,22 +2631,32 @@ static void ironlake_update_primary_plane(struct drm_crtc *crtc,
> 
> 	if (obj->tiling_mode != I915_TILING_NONE)
> 		dspcntr |= DISPPLANE_TILED;
>-	else
>-		dspcntr &= ~DISPPLANE_TILED;
> 
>-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
>-		dspcntr &= ~DISPPLANE_TRICKLE_FEED_DISABLE;
>-	else
>+	if (!IS_HASWELL(dev) && !IS_BROADWELL(dev))
> 		dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
> 
>-	I915_WRITE(reg, dspcntr);
>-
>-	linear_offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8);
>+	linear_offset = y * fb->pitches[0] + x * pixel_size;
> 	intel_crtc->dspaddr_offset =
> 		intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode,
>-					       fb->bits_per_pixel / 8,
>+					       pixel_size,
> 					       fb->pitches[0]);
> 	linear_offset -= intel_crtc->dspaddr_offset;
>+	if (to_intel_plane(crtc->primary)->rotation == BIT(DRM_ROTATE_180)) {
>+		dspcntr |= DISPPLANE_ROTATE_180;
>+
>+		if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
>+			x += (intel_crtc->config.pipe_src_w - 1);
>+			y += (intel_crtc->config.pipe_src_h - 1);
>+
>+			/* Finding the last pixel of the last line of the display
>+			data and adding to linear_offset*/
>+			linear_offset +=
>+				(intel_crtc->config.pipe_src_h - 1) * fb->pitches[0] +
>+				(intel_crtc->config.pipe_src_w - 1) * pixel_size;
>+		}
>+	}
>+
>+	I915_WRITE(reg, dspcntr);
> 
> 	DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
> 		      i915_gem_obj_ggtt_offset(obj), linear_offset, x, y,
>@@ -2669,20 +2763,58 @@ static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
> 	struct drm_device *dev = crtc->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>-	unsigned long flags;
> 	bool pending;
> 
> 	if (i915_reset_in_progress(&dev_priv->gpu_error) ||
> 	    intel_crtc->reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
> 		return false;
> 
>-	spin_lock_irqsave(&dev->event_lock, flags);
>+	spin_lock_irq(&dev->event_lock);
> 	pending = to_intel_crtc(crtc)->unpin_work != NULL;
>-	spin_unlock_irqrestore(&dev->event_lock, flags);
>+	spin_unlock_irq(&dev->event_lock);
> 
> 	return pending;
> }
> 
>+static void intel_update_pipe_size(struct intel_crtc *crtc)
>+{
>+	struct drm_device *dev = crtc->base.dev;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	const struct drm_display_mode *adjusted_mode;
>+
>+	if (!i915.fastboot)
>+		return;
>+
>+	/*
>+	 * Update pipe size and adjust fitter if needed: the reason for this is
>+	 * that in compute_mode_changes we check the native mode (not the pfit
>+	 * mode) to see if we can flip rather than do a full mode set. In the
>+	 * fastboot case, we'll flip, but if we don't update the pipesrc and
>+	 * pfit state, we'll end up with a big fb scanned out into the wrong
>+	 * sized surface.
>+	 *
>+	 * To fix this properly, we need to hoist the checks up into
>+	 * compute_mode_changes (or above), check the actual pfit state and
>+	 * whether the platform allows pfit disable with pipe active, and only
>+	 * then update the pipesrc and pfit state, even on the flip path.
>+	 */
>+
>+	adjusted_mode = &crtc->config.adjusted_mode;
>+
>+	I915_WRITE(PIPESRC(crtc->pipe),
>+		   ((adjusted_mode->crtc_hdisplay - 1) << 16) |
>+		   (adjusted_mode->crtc_vdisplay - 1));
>+	if (!crtc->config.pch_pfit.enabled &&
>+	    (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_LVDS) ||
>+	     intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_EDP))) {
>+		I915_WRITE(PF_CTL(crtc->pipe), 0);
>+		I915_WRITE(PF_WIN_POS(crtc->pipe), 0);
>+		I915_WRITE(PF_WIN_SZ(crtc->pipe), 0);
>+	}
>+	crtc->config.pipe_src_w = adjusted_mode->crtc_hdisplay;
>+	crtc->config.pipe_src_h = adjusted_mode->crtc_vdisplay;
>+}
>+
> static int
> intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
> 		    struct drm_framebuffer *fb)
>@@ -2725,36 +2857,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
> 		return ret;
> 	}
> 
>-	/*
>-	 * Update pipe size and adjust fitter if needed: the reason for this is
>-	 * that in compute_mode_changes we check the native mode (not the pfit
>-	 * mode) to see if we can flip rather than do a full mode set. In the
>-	 * fastboot case, we'll flip, but if we don't update the pipesrc and
>-	 * pfit state, we'll end up with a big fb scanned out into the wrong
>-	 * sized surface.
>-	 *
>-	 * To fix this properly, we need to hoist the checks up into
>-	 * compute_mode_changes (or above), check the actual pfit state and
>-	 * whether the platform allows pfit disable with pipe active, and only
>-	 * then update the pipesrc and pfit state, even on the flip path.
>-	 */
>-	if (i915.fastboot) {
>-		const struct drm_display_mode *adjusted_mode =
>-			&intel_crtc->config.adjusted_mode;
>-
>-		I915_WRITE(PIPESRC(intel_crtc->pipe),
>-			   ((adjusted_mode->crtc_hdisplay - 1) << 16) |
>-			   (adjusted_mode->crtc_vdisplay - 1));
>-		if (!intel_crtc->config.pch_pfit.enabled &&
>-		    (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) ||
>-		     intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))) {
>-			I915_WRITE(PF_CTL(intel_crtc->pipe), 0);
>-			I915_WRITE(PF_WIN_POS(intel_crtc->pipe), 0);
>-			I915_WRITE(PF_WIN_SZ(intel_crtc->pipe), 0);
>-		}
>-		intel_crtc->config.pipe_src_w = adjusted_mode->crtc_hdisplay;
>-		intel_crtc->config.pipe_src_h = adjusted_mode->crtc_vdisplay;
>-	}
>+	intel_update_pipe_size(intel_crtc);
> 
> 	dev_priv->display.update_primary_plane(crtc, fb, x, y);
> 
>@@ -3346,23 +3449,53 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev)
> 	return false;
> }
> 
>+static void page_flip_completed(struct intel_crtc *intel_crtc)
>+{
>+	struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
>+	struct intel_unpin_work *work = intel_crtc->unpin_work;
>+
>+	/* ensure that the unpin work is consistent wrt ->pending. */
>+	smp_rmb();
>+	intel_crtc->unpin_work = NULL;
>+
>+	if (work->event)
>+		drm_send_vblank_event(intel_crtc->base.dev,
>+				      intel_crtc->pipe,
>+				      work->event);
>+
>+	drm_crtc_vblank_put(&intel_crtc->base);
>+
>+	wake_up_all(&dev_priv->pending_flip_queue);
>+	queue_work(dev_priv->wq, &work->work);
>+
>+	trace_i915_flip_complete(intel_crtc->plane,
>+				 work->pending_flip_obj);
>+}
>+
> void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
> {
> 	struct drm_device *dev = crtc->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 
>-	if (crtc->primary->fb == NULL)
>-		return;
>-
> 	WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue));
>+	if (WARN_ON(wait_event_timeout(dev_priv->pending_flip_queue,
>+				       !intel_crtc_has_pending_flip(crtc),
>+				       60*HZ) == 0)) {
>+		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 
>-	WARN_ON(wait_event_timeout(dev_priv->pending_flip_queue,
>-				   !intel_crtc_has_pending_flip(crtc),
>-				   60*HZ) == 0);
>+		spin_lock_irq(&dev->event_lock);
>+		if (intel_crtc->unpin_work) {
>+			WARN_ONCE(1, "Removing stuck page flip\n");
>+			page_flip_completed(intel_crtc);
>+		}
>+		spin_unlock_irq(&dev->event_lock);
>+	}
> 
>-	mutex_lock(&dev->struct_mutex);
>-	intel_finish_fb(crtc->primary->fb);
>-	mutex_unlock(&dev->struct_mutex);
>+	if (crtc->primary->fb) {
>+		mutex_lock(&dev->struct_mutex);
>+		intel_finish_fb(crtc->primary->fb);
>+		mutex_unlock(&dev->struct_mutex);
>+	}
> }
> 
> /* Program iCLKIP clock to the desired frequency */
>@@ -3911,14 +4044,10 @@ static void intel_crtc_dpms_overlay(struct intel_crtc *intel_crtc, bool enable)
> static void intel_crtc_enable_planes(struct drm_crtc *crtc)
> {
> 	struct drm_device *dev = crtc->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 	int pipe = intel_crtc->pipe;
>-	int plane = intel_crtc->plane;
>-
>-	drm_vblank_on(dev, pipe);
> 
>-	intel_enable_primary_hw_plane(dev_priv, plane, pipe);
>+	intel_enable_primary_hw_plane(crtc->primary, crtc);
> 	intel_enable_planes(crtc);
> 	intel_crtc_update_cursor(crtc, true);
> 	intel_crtc_dpms_overlay(intel_crtc, true);
>@@ -3955,7 +4084,7 @@ static void intel_crtc_disable_planes(struct drm_crtc *crtc)
> 	intel_crtc_dpms_overlay(intel_crtc, false);
> 	intel_crtc_update_cursor(crtc, false);
> 	intel_disable_planes(crtc);
>-	intel_disable_primary_hw_plane(dev_priv, plane, pipe);
>+	intel_disable_primary_hw_plane(crtc->primary, crtc);
> 
> 	/*
> 	 * FIXME: Once we grow proper nuclear flip support out of this we need
>@@ -3963,8 +4092,6 @@ static void intel_crtc_disable_planes(struct drm_crtc *crtc)
> 	 * consider this a flip to a NULL plane.
> 	 */
> 	intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_ALL_MASK(pipe));
>-
>-	drm_vblank_off(dev, pipe);
> }
> 
> static void ironlake_crtc_enable(struct drm_crtc *crtc)
>@@ -3974,7 +4101,6 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 	struct intel_encoder *encoder;
> 	int pipe = intel_crtc->pipe;
>-	enum plane plane = intel_crtc->plane;
> 
> 	WARN_ON(!crtc->enabled);
> 
>@@ -3991,18 +4117,11 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
> 
> 	if (intel_crtc->config.has_pch_encoder) {
> 		intel_cpu_transcoder_set_m_n(intel_crtc,
>-					     &intel_crtc->config.fdi_m_n);
>+				     &intel_crtc->config.fdi_m_n, NULL);
> 	}
> 
> 	ironlake_set_pipeconf(crtc);
> 
>-	/* Set up the display plane register */
>-	I915_WRITE(DSPCNTR(plane), DISPPLANE_GAMMA_ENABLE);
>-	POSTING_READ(DSPCNTR(plane));
>-
>-	dev_priv->display.update_primary_plane(crtc, crtc->primary->fb,
>-					       crtc->x, crtc->y);
>-
> 	intel_crtc->active = true;
> 
> 	intel_set_cpu_fifo_underrun_reporting(dev, pipe, true);
>@@ -4042,6 +4161,9 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
> 	if (HAS_PCH_CPT(dev))
> 		cpt_verify_modeset(dev, intel_crtc->pipe);
> 
>+	assert_vblank_disabled(crtc);
>+	drm_crtc_vblank_on(crtc);
>+
> 	intel_crtc_enable_planes(crtc);
> }
> 
>@@ -4087,7 +4209,6 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 	struct intel_encoder *encoder;
> 	int pipe = intel_crtc->pipe;
>-	enum plane plane = intel_crtc->plane;
> 
> 	WARN_ON(!crtc->enabled);
> 
>@@ -4104,20 +4225,13 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
> 
> 	if (intel_crtc->config.has_pch_encoder) {
> 		intel_cpu_transcoder_set_m_n(intel_crtc,
>-					     &intel_crtc->config.fdi_m_n);
>+				     &intel_crtc->config.fdi_m_n, NULL);
> 	}
> 
> 	haswell_set_pipeconf(crtc);
> 
> 	intel_set_pipe_csc(crtc);
> 
>-	/* Set up the display plane register */
>-	I915_WRITE(DSPCNTR(plane), DISPPLANE_GAMMA_ENABLE | DISPPLANE_PIPE_CSC_ENABLE);
>-	POSTING_READ(DSPCNTR(plane));
>-
>-	dev_priv->display.update_primary_plane(crtc, crtc->primary->fb,
>-					       crtc->x, crtc->y);
>-
> 	intel_crtc->active = true;
> 
> 	intel_set_cpu_fifo_underrun_reporting(dev, pipe, true);
>@@ -4157,6 +4271,9 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
> 		intel_opregion_notify_encoder(encoder, true);
> 	}
> 
>+	assert_vblank_disabled(crtc);
>+	drm_crtc_vblank_on(crtc);
>+
> 	/* If we change the relative order between pipe/planes enabling, we need
> 	 * to change the workaround. */
> 	haswell_mode_set_planes_workaround(intel_crtc);
>@@ -4192,13 +4309,17 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
> 
> 	intel_crtc_disable_planes(crtc);
> 
>+	drm_crtc_vblank_off(crtc);
>+	assert_vblank_disabled(crtc);
>+
> 	for_each_encoder_on_crtc(dev, crtc, encoder)
> 		encoder->disable(encoder);
> 
> 	if (intel_crtc->config.has_pch_encoder)
> 		intel_set_pch_fifo_underrun_reporting(dev, pipe, false);
> 
>-	intel_disable_pipe(dev_priv, pipe);
>+	intel_disable_pipe(intel_crtc);
>+
> 	ironlake_pfit_disable(intel_crtc);
> 
> 	for_each_encoder_on_crtc(dev, crtc, encoder)
>@@ -4246,7 +4367,6 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 	struct intel_encoder *encoder;
>-	int pipe = intel_crtc->pipe;
> 	enum transcoder cpu_transcoder = intel_crtc->config.cpu_transcoder;
> 
> 	if (!intel_crtc->active)
>@@ -4254,6 +4374,9 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
> 
> 	intel_crtc_disable_planes(crtc);
> 
>+	drm_crtc_vblank_off(crtc);
>+	assert_vblank_disabled(crtc);
>+
> 	for_each_encoder_on_crtc(dev, crtc, encoder) {
> 		intel_opregion_notify_encoder(encoder, false);
> 		encoder->disable(encoder);
>@@ -4261,7 +4384,7 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
> 
> 	if (intel_crtc->config.has_pch_encoder)
> 		intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A, false);
>-	intel_disable_pipe(dev_priv, pipe);
>+	intel_disable_pipe(intel_crtc);
> 
> 	if (intel_crtc->config.dp_encoder_is_mst)
> 		intel_ddi_set_vc_payload_alloc(crtc, false);
>@@ -4539,12 +4662,57 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk)
> 	vlv_update_cdclk(dev);
> }
> 
>+static void cherryview_set_cdclk(struct drm_device *dev, int cdclk)
>+{
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	u32 val, cmd;
>+
>+	WARN_ON(dev_priv->display.get_display_clock_speed(dev) != dev_priv->vlv_cdclk_freq);
>+
>+	switch (cdclk) {
>+	case 400000:
>+		cmd = 3;
>+		break;
>+	case 333333:
>+	case 320000:
>+		cmd = 2;
>+		break;
>+	case 266667:
>+		cmd = 1;
>+		break;
>+	case 200000:
>+		cmd = 0;
>+		break;
>+	default:
>+		WARN_ON(1);
>+		return;
>+	}
>+
>+	mutex_lock(&dev_priv->rps.hw_lock);
>+	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
>+	val &= ~DSPFREQGUAR_MASK_CHV;
>+	val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
>+	vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
>+	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) &
>+		      DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
>+		     50)) {
>+		DRM_ERROR("timed out waiting for CDclk change\n");
>+	}
>+	mutex_unlock(&dev_priv->rps.hw_lock);
>+
>+	vlv_update_cdclk(dev);
>+}
>+
> static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv,
> 				 int max_pixclk)
> {
> 	int vco = valleyview_get_vco(dev_priv);
> 	int freq_320 = (vco <<  1) % 320000 != 0 ? 333333 : 320000;
> 
>+	/* FIXME: Punit isn't quite ready yet */
>+	if (IS_CHERRYVIEW(dev_priv->dev))
>+		return 400000;
>+
> 	/*
> 	 * Really only a few cases to deal with, as only 4 CDclks are supported:
> 	 *   200MHz
>@@ -4607,21 +4775,23 @@ static void valleyview_modeset_global_resources(struct drm_device *dev)
> 	int max_pixclk = intel_mode_max_pixclk(dev_priv);
> 	int req_cdclk = valleyview_calc_cdclk(dev_priv, max_pixclk);
> 
>-	if (req_cdclk != dev_priv->vlv_cdclk_freq)
>-		valleyview_set_cdclk(dev, req_cdclk);
>+	if (req_cdclk != dev_priv->vlv_cdclk_freq) {
>+		if (IS_CHERRYVIEW(dev))
>+			cherryview_set_cdclk(dev, req_cdclk);
>+		else
>+			valleyview_set_cdclk(dev, req_cdclk);
>+	}
>+
> 	modeset_update_crtc_power_domains(dev);
> }
> 
> static void valleyview_crtc_enable(struct drm_crtc *crtc)
> {
> 	struct drm_device *dev = crtc->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 	struct intel_encoder *encoder;
> 	int pipe = intel_crtc->pipe;
>-	int plane = intel_crtc->plane;
> 	bool is_dsi;
>-	u32 dspcntr;
> 
> 	WARN_ON(!crtc->enabled);
> 
>@@ -4630,33 +4800,20 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
> 
> 	is_dsi = intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI);
> 
>-	if (!is_dsi && !IS_CHERRYVIEW(dev))
>-		vlv_prepare_pll(intel_crtc);
>-
>-	/* Set up the display plane register */
>-	dspcntr = DISPPLANE_GAMMA_ENABLE;
>+	if (!is_dsi) {
>+		if (IS_CHERRYVIEW(dev))
>+			chv_prepare_pll(intel_crtc);
>+		else
>+			vlv_prepare_pll(intel_crtc);
>+	}
> 
> 	if (intel_crtc->config.has_dp_encoder)
> 		intel_dp_set_m_n(intel_crtc);
> 
> 	intel_set_pipe_timings(intel_crtc);
> 
>-	/* pipesrc and dspsize control the size that is scaled from,
>-	 * which should always be the user's requested size.
>-	 */
>-	I915_WRITE(DSPSIZE(plane),
>-		   ((intel_crtc->config.pipe_src_h - 1) << 16) |
>-		   (intel_crtc->config.pipe_src_w - 1));
>-	I915_WRITE(DSPPOS(plane), 0);
>-
> 	i9xx_set_pipeconf(intel_crtc);
> 
>-	I915_WRITE(DSPCNTR(plane), dspcntr);
>-	POSTING_READ(DSPCNTR(plane));
>-
>-	dev_priv->display.update_primary_plane(crtc, crtc->primary->fb,
>-					       crtc->x, crtc->y);
>-
> 	intel_crtc->active = true;
> 
> 	intel_set_cpu_fifo_underrun_reporting(dev, pipe, true);
>@@ -4686,6 +4843,9 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
> 	for_each_encoder_on_crtc(dev, crtc, encoder)
> 		encoder->enable(encoder);
> 
>+	assert_vblank_disabled(crtc);
>+	drm_crtc_vblank_on(crtc);
>+
> 	intel_crtc_enable_planes(crtc);
> 
> 	/* Underruns don't raise interrupts, so check manually. */
>@@ -4704,49 +4864,24 @@ static void i9xx_set_pll_dividers(struct intel_crtc *crtc)
> static void i9xx_crtc_enable(struct drm_crtc *crtc)
> {
> 	struct drm_device *dev = crtc->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 	struct intel_encoder *encoder;
> 	int pipe = intel_crtc->pipe;
>-	int plane = intel_crtc->plane;
>-	u32 dspcntr;
> 
> 	WARN_ON(!crtc->enabled);
> 
> 	if (intel_crtc->active)
> 		return;
> 
>-	i9xx_set_pll_dividers(intel_crtc);
>-
>-	/* Set up the display plane register */
>-	dspcntr = DISPPLANE_GAMMA_ENABLE;
>-
>-	if (pipe == 0)
>-		dspcntr &= ~DISPPLANE_SEL_PIPE_MASK;
>-	else
>-		dspcntr |= DISPPLANE_SEL_PIPE_B;
>+	i9xx_set_pll_dividers(intel_crtc);
> 
> 	if (intel_crtc->config.has_dp_encoder)
> 		intel_dp_set_m_n(intel_crtc);
> 
> 	intel_set_pipe_timings(intel_crtc);
> 
>-	/* pipesrc and dspsize control the size that is scaled from,
>-	 * which should always be the user's requested size.
>-	 */
>-	I915_WRITE(DSPSIZE(plane),
>-		   ((intel_crtc->config.pipe_src_h - 1) << 16) |
>-		   (intel_crtc->config.pipe_src_w - 1));
>-	I915_WRITE(DSPPOS(plane), 0);
>-
> 	i9xx_set_pipeconf(intel_crtc);
> 
>-	I915_WRITE(DSPCNTR(plane), dspcntr);
>-	POSTING_READ(DSPCNTR(plane));
>-
>-	dev_priv->display.update_primary_plane(crtc, crtc->primary->fb,
>-					       crtc->x, crtc->y);
>-
> 	intel_crtc->active = true;
> 
> 	if (!IS_GEN2(dev))
>@@ -4768,6 +4903,9 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
> 	for_each_encoder_on_crtc(dev, crtc, encoder)
> 		encoder->enable(encoder);
> 
>+	assert_vblank_disabled(crtc);
>+	drm_crtc_vblank_on(crtc);
>+
> 	intel_crtc_enable_planes(crtc);
> 
> 	/*
>@@ -4831,9 +4969,6 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
> 	intel_set_memory_cxsr(dev_priv, false);
> 	intel_crtc_disable_planes(crtc);
> 
>-	for_each_encoder_on_crtc(dev, crtc, encoder)
>-		encoder->disable(encoder);
>-
> 	/*
> 	 * On gen2 planes are double buffered but the pipe isn't, so we must
> 	 * wait for planes to fully turn off before disabling the pipe.
>@@ -4842,7 +4977,13 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
> 	 */
> 	intel_wait_for_vblank(dev, pipe);
> 
>-	intel_disable_pipe(dev_priv, pipe);
>+	drm_crtc_vblank_off(crtc);
>+	assert_vblank_disabled(crtc);
>+
>+	for_each_encoder_on_crtc(dev, crtc, encoder)
>+		encoder->disable(encoder);
>+
>+	intel_disable_pipe(intel_crtc);
> 
> 	i9xx_pfit_disable(intel_crtc);
> 
>@@ -4856,7 +4997,7 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
> 		else if (IS_VALLEYVIEW(dev))
> 			vlv_disable_pll(dev_priv, pipe);
> 		else
>-			i9xx_disable_pll(dev_priv, pipe);
>+			i9xx_disable_pll(intel_crtc);
> 	}
> 
> 	if (!IS_GEN2(dev))
>@@ -4874,36 +5015,6 @@ static void i9xx_crtc_off(struct drm_crtc *crtc)
> {
> }
> 
>-static void intel_crtc_update_sarea(struct drm_crtc *crtc,
>-				    bool enabled)
>-{
>-	struct drm_device *dev = crtc->dev;
>-	struct drm_i915_master_private *master_priv;
>-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>-	int pipe = intel_crtc->pipe;
>-
>-	if (!dev->primary->master)
>-		return;
>-
>-	master_priv = dev->primary->master->driver_priv;
>-	if (!master_priv->sarea_priv)
>-		return;
>-
>-	switch (pipe) {
>-	case 0:
>-		master_priv->sarea_priv->pipeA_w = enabled ? crtc->mode.hdisplay : 0;
>-		master_priv->sarea_priv->pipeA_h = enabled ? crtc->mode.vdisplay : 0;
>-		break;
>-	case 1:
>-		master_priv->sarea_priv->pipeB_w = enabled ? crtc->mode.hdisplay : 0;
>-		master_priv->sarea_priv->pipeB_h = enabled ? crtc->mode.vdisplay : 0;
>-		break;
>-	default:
>-		DRM_ERROR("Can't update pipe %c in SAREA\n", pipe_name(pipe));
>-		break;
>-	}
>-}
>-
> /* Master function to enable/disable CRTC and corresponding power wells */
> void intel_crtc_control(struct drm_crtc *crtc, bool enable)
> {
>@@ -4947,8 +5058,6 @@ void intel_crtc_update_dpms(struct drm_crtc *crtc)
> 		enable |= intel_encoder->connectors_active;
> 
> 	intel_crtc_control(crtc, enable);
>-
>-	intel_crtc_update_sarea(crtc, enable);
> }
> 
> static void intel_crtc_disable(struct drm_crtc *crtc)
>@@ -4963,7 +5072,6 @@ static void intel_crtc_disable(struct drm_crtc *crtc)
> 	WARN_ON(!crtc->enabled);
> 
> 	dev_priv->display.crtc_disable(crtc);
>-	intel_crtc_update_sarea(crtc, false);
> 	dev_priv->display.off(crtc);
> 
> 	if (crtc->primary->fb) {
>@@ -5190,12 +5298,29 @@ retry:
> 	return setup_ok ? 0 : -EINVAL;
> }
> 
>+static bool pipe_config_supports_ips(struct drm_i915_private *dev_priv,
>+				     struct intel_crtc_config *pipe_config)
>+{
>+	if (pipe_config->pipe_bpp > 24)
>+		return false;
>+
>+	/* HSW can handle pixel rate up to cdclk? */
>+	if (IS_HASWELL(dev_priv->dev))
>+		return true;
>+
>+	return ilk_pipe_pixel_rate(pipe_config) <=
>+		intel_ddi_get_cdclk_freq(dev_priv) * 95 / 100;
>+}
>+
> static void hsw_compute_ips_config(struct intel_crtc *crtc,
> 				   struct intel_crtc_config *pipe_config)
> {
>+	struct drm_device *dev = crtc->base.dev;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+
> 	pipe_config->ips_enabled = i915.enable_ips &&
>-				   hsw_crtc_supports_ips(crtc) &&
>-				   pipe_config->pipe_bpp <= 24;
>+		hsw_crtc_supports_ips(crtc) &&
>+		pipe_config_supports_ips(dev_priv, pipe_config);
> }
> 
> static int intel_crtc_compute_config(struct intel_crtc *crtc,
>@@ -5275,6 +5400,10 @@ static int valleyview_get_display_clock_speed(struct drm_device *dev)
> 	u32 val;
> 	int divider;
> 
>+	/* FIXME: Punit isn't quite ready yet */
>+	if (IS_CHERRYVIEW(dev))
>+		return 400000;
>+
> 	mutex_lock(&dev_priv->dpio_lock);
> 	val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
> 	mutex_unlock(&dev_priv->dpio_lock);
>@@ -5519,7 +5648,8 @@ static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc,
> }
> 
> static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
>-					 struct intel_link_m_n *m_n)
>+					 struct intel_link_m_n *m_n,
>+					 struct intel_link_m_n *m2_n2)
> {
> 	struct drm_device *dev = crtc->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>@@ -5531,6 +5661,18 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
> 		I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
> 		I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
> 		I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
>+		/* M2_N2 registers to be set only for gen < 8 (M2_N2 available
>+		 * for gen < 8) and if DRRS is supported (to make sure the
>+		 * registers are not unnecessarily accessed).
>+		 */
>+		if (m2_n2 && INTEL_INFO(dev)->gen < 8 &&
>+			crtc->config.has_drrs) {
>+			I915_WRITE(PIPE_DATA_M2(transcoder),
>+					TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
>+			I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
>+			I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
>+			I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
>+		}
> 	} else {
> 		I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
> 		I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
>@@ -5539,12 +5681,13 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
> 	}
> }
> 
>-static void intel_dp_set_m_n(struct intel_crtc *crtc)
>+void intel_dp_set_m_n(struct intel_crtc *crtc)
> {
> 	if (crtc->config.has_pch_encoder)
> 		intel_pch_transcoder_set_m_n(crtc, &crtc->config.dp_m_n);
> 	else
>-		intel_cpu_transcoder_set_m_n(crtc, &crtc->config.dp_m_n);
>+		intel_cpu_transcoder_set_m_n(crtc, &crtc->config.dp_m_n,
>+						   &crtc->config.dp_m2_n2);
> }
> 
> static void vlv_update_pll(struct intel_crtc *crtc)
>@@ -5662,6 +5805,18 @@ static void vlv_prepare_pll(struct intel_crtc *crtc)
> 
> static void chv_update_pll(struct intel_crtc *crtc)
> {
>+	crtc->config.dpll_hw_state.dpll = DPLL_SSC_REF_CLOCK_CHV |
>+		DPLL_REFA_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS |
>+		DPLL_VCO_ENABLE;
>+	if (crtc->pipe != PIPE_A)
>+		crtc->config.dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
>+
>+	crtc->config.dpll_hw_state.dpll_md =
>+		(crtc->config.pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
>+}
>+
>+static void chv_prepare_pll(struct intel_crtc *crtc)
>+{
> 	struct drm_device *dev = crtc->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	int pipe = crtc->pipe;
>@@ -5671,15 +5826,6 @@ static void chv_update_pll(struct intel_crtc *crtc)
> 	u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
> 	int refclk;
> 
>-	crtc->config.dpll_hw_state.dpll = DPLL_SSC_REF_CLOCK_CHV |
>-		DPLL_REFA_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS |
>-		DPLL_VCO_ENABLE;
>-	if (pipe != PIPE_A)
>-		crtc->config.dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
>-
>-	crtc->config.dpll_hw_state.dpll_md =
>-		(crtc->config.pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
>-
> 	bestn = crtc->config.dpll.n;
> 	bestm2_frac = crtc->config.dpll.m2 & 0x3fffff;
> 	bestm1 = crtc->config.dpll.m1;
>@@ -5839,7 +5985,7 @@ static void i8xx_update_pll(struct intel_crtc *crtc,
> 			dpll |= PLL_P2_DIVIDE_BY_4;
> 	}
> 
>-	if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DVO))
>+	if (!IS_I830(dev) && intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DVO))
> 		dpll |= DPLL_DVO_2X_MODE;
> 
> 	if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_LVDS) &&
>@@ -5990,9 +6136,9 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
> 
> 	pipeconf = 0;
> 
>-	if (dev_priv->quirks & QUIRK_PIPEA_FORCE &&
>-	    I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE)
>-		pipeconf |= PIPECONF_ENABLE;
>+	if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
>+	    (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
>+		pipeconf |= I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE;
> 
> 	if (intel_crtc->config.double_wide)
> 		pipeconf |= PIPECONF_DOUBLE_WIDE;
>@@ -6235,7 +6381,7 @@ static void i9xx_get_plane_config(struct intel_crtc *crtc,
> 	crtc->base.primary->fb->height = ((val >> 0) & 0xfff) + 1;
> 
> 	val = I915_READ(DSPSTRIDE(pipe));
>-	crtc->base.primary->fb->pitches[0] = val & 0xffffff80;
>+	crtc->base.primary->fb->pitches[0] = val & 0xffffffc0;
> 
> 	aligned_height = intel_align_height(dev, crtc->base.primary->fb->height,
> 					    plane_config->tiled);
>@@ -6345,6 +6491,14 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
> 	}
> 	pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
> 	if (!IS_VALLEYVIEW(dev)) {
>+		/*
>+		 * DPLL_DVO_2X_MODE must be enabled for both DPLLs
>+		 * on 830. Filter it out here so that we don't
>+		 * report errors due to that.
>+		 */
>+		if (IS_I830(dev))
>+			pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
>+
> 		pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
> 		pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
> 	} else {
>@@ -6367,7 +6521,6 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
> static void ironlake_init_pch_refclk(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct drm_mode_config *mode_config = &dev->mode_config;
> 	struct intel_encoder *encoder;
> 	u32 val, final;
> 	bool has_lvds = false;
>@@ -6377,8 +6530,7 @@ static void ironlake_init_pch_refclk(struct drm_device *dev)
> 	bool can_ssc = false;
> 
> 	/* We need to take the global config into account */
>-	list_for_each_entry(encoder, &mode_config->encoder_list,
>-			    base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		switch (encoder->type) {
> 		case INTEL_OUTPUT_LVDS:
> 			has_panel = true;
>@@ -6685,11 +6837,10 @@ static void lpt_disable_clkout_dp(struct drm_device *dev)
> 
> static void lpt_init_pch_refclk(struct drm_device *dev)
> {
>-	struct drm_mode_config *mode_config = &dev->mode_config;
> 	struct intel_encoder *encoder;
> 	bool has_vga = false;
> 
>-	list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		switch (encoder->type) {
> 		case INTEL_OUTPUT_ANALOG:
> 			has_vga = true;
>@@ -7145,7 +7296,8 @@ static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
> 
> static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
> 					 enum transcoder transcoder,
>-					 struct intel_link_m_n *m_n)
>+					 struct intel_link_m_n *m_n,
>+					 struct intel_link_m_n *m2_n2)
> {
> 	struct drm_device *dev = crtc->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>@@ -7159,6 +7311,20 @@ static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
> 		m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
> 		m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
> 			    & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
>+		/* Read M2_N2 registers only for gen < 8 (M2_N2 available for
>+		 * gen < 8) and if DRRS is supported (to make sure the
>+		 * registers are not unnecessarily read).
>+		 */
>+		if (m2_n2 && INTEL_INFO(dev)->gen < 8 &&
>+			crtc->config.has_drrs) {
>+			m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
>+			m2_n2->link_n =	I915_READ(PIPE_LINK_N2(transcoder));
>+			m2_n2->gmch_m =	I915_READ(PIPE_DATA_M2(transcoder))
>+					& ~TU_SIZE_MASK;
>+			m2_n2->gmch_n =	I915_READ(PIPE_DATA_N2(transcoder));
>+			m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
>+					& TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
>+		}
> 	} else {
> 		m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
> 		m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
>@@ -7177,14 +7343,15 @@ void intel_dp_get_m_n(struct intel_crtc *crtc,
> 		intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
> 	else
> 		intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
>-					     &pipe_config->dp_m_n);
>+					     &pipe_config->dp_m_n,
>+					     &pipe_config->dp_m2_n2);
> }
> 
> static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
> 					struct intel_crtc_config *pipe_config)
> {
> 	intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
>-				     &pipe_config->fdi_m_n);
>+				     &pipe_config->fdi_m_n, NULL);
> }
> 
> static void ironlake_get_pfit_config(struct intel_crtc *crtc,
>@@ -7255,7 +7422,7 @@ static void ironlake_get_plane_config(struct intel_crtc *crtc,
> 	crtc->base.primary->fb->height = ((val >> 0) & 0xfff) + 1;
> 
> 	val = I915_READ(DSPSTRIDE(pipe));
>-	crtc->base.primary->fb->pitches[0] = val & 0xffffff80;
>+	crtc->base.primary->fb->pitches[0] = val & 0xffffffc0;
> 
> 	aligned_height = intel_align_height(dev, crtc->base.primary->fb->height,
> 					    plane_config->tiled);
>@@ -7472,7 +7639,6 @@ static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
> static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
> {
> 	uint32_t val;
>-	unsigned long irqflags;
> 
> 	val = I915_READ(LCPLL_CTL);
> 
>@@ -7492,10 +7658,10 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
> 	 * to call special forcewake code that doesn't touch runtime PM and
> 	 * doesn't enable the forcewake delayed work.
> 	 */
>-	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
>+	spin_lock_irq(&dev_priv->uncore.lock);
> 	if (dev_priv->uncore.forcewake_count++ == 0)
> 		dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL);
>-	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
>+	spin_unlock_irq(&dev_priv->uncore.lock);
> 
> 	if (val & LCPLL_POWER_DOWN_ALLOW) {
> 		val &= ~LCPLL_POWER_DOWN_ALLOW;
>@@ -7526,10 +7692,10 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
> 	}
> 
> 	/* See the big comment above. */
>-	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
>+	spin_lock_irq(&dev_priv->uncore.lock);
> 	if (--dev_priv->uncore.forcewake_count == 0)
> 		dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
>-	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
>+	spin_unlock_irq(&dev_priv->uncore.lock);
> }
> 
> /*
>@@ -7615,6 +7781,22 @@ static int haswell_crtc_mode_set(struct drm_crtc *crtc,
> 	return 0;
> }
> 
>+static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
>+				enum port port,
>+				struct intel_crtc_config *pipe_config)
>+{
>+	pipe_config->ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
>+
>+	switch (pipe_config->ddi_pll_sel) {
>+	case PORT_CLK_SEL_WRPLL1:
>+		pipe_config->shared_dpll = DPLL_ID_WRPLL1;
>+		break;
>+	case PORT_CLK_SEL_WRPLL2:
>+		pipe_config->shared_dpll = DPLL_ID_WRPLL2;
>+		break;
>+	}
>+}
>+
> static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
> 				       struct intel_crtc_config *pipe_config)
> {
>@@ -7628,16 +7810,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
> 
> 	port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
> 
>-	pipe_config->ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
>-
>-	switch (pipe_config->ddi_pll_sel) {
>-	case PORT_CLK_SEL_WRPLL1:
>-		pipe_config->shared_dpll = DPLL_ID_WRPLL1;
>-		break;
>-	case PORT_CLK_SEL_WRPLL2:
>-		pipe_config->shared_dpll = DPLL_ID_WRPLL2;
>-		break;
>-	}
>+	haswell_get_ddi_pll(dev_priv, port, pipe_config);
> 
> 	if (pipe_config->shared_dpll >= 0) {
> 		pll = &dev_priv->shared_dplls[pipe_config->shared_dpll];
>@@ -8037,28 +8210,56 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base)
> 	struct drm_device *dev = crtc->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>-	uint32_t cntl;
>+	uint32_t cntl = 0, size = 0;
> 
>-	if (base != intel_crtc->cursor_base) {
>-		/* On these chipsets we can only modify the base whilst
>-		 * the cursor is disabled.
>-		 */
>-		if (intel_crtc->cursor_cntl) {
>-			I915_WRITE(_CURACNTR, 0);
>-			POSTING_READ(_CURACNTR);
>-			intel_crtc->cursor_cntl = 0;
>+	if (base) {
>+		unsigned int width = intel_crtc->cursor_width;
>+		unsigned int height = intel_crtc->cursor_height;
>+		unsigned int stride = roundup_pow_of_two(width) * 4;
>+
>+		switch (stride) {
>+		default:
>+			WARN_ONCE(1, "Invalid cursor width/stride, width=%u, stride=%u\n",
>+				  width, stride);
>+			stride = 256;
>+			/* fallthrough */
>+		case 256:
>+		case 512:
>+		case 1024:
>+		case 2048:
>+			break;
> 		}
> 
>+		cntl |= CURSOR_ENABLE |
>+			CURSOR_GAMMA_ENABLE |
>+			CURSOR_FORMAT_ARGB |
>+			CURSOR_STRIDE(stride);
>+
>+		size = (height << 12) | width;
>+	}
>+
>+	if (intel_crtc->cursor_cntl != 0 &&
>+	    (intel_crtc->cursor_base != base ||
>+	     intel_crtc->cursor_size != size ||
>+	     intel_crtc->cursor_cntl != cntl)) {
>+		/* On these chipsets we can only modify the base/size/stride
>+		 * whilst the cursor is disabled.
>+		 */
>+		I915_WRITE(_CURACNTR, 0);
>+		POSTING_READ(_CURACNTR);
>+		intel_crtc->cursor_cntl = 0;
>+	}
>+
>+	if (intel_crtc->cursor_base != base) {
> 		I915_WRITE(_CURABASE, base);
>-		POSTING_READ(_CURABASE);
>+		intel_crtc->cursor_base = base;
>+	}
>+
>+	if (intel_crtc->cursor_size != size) {
>+		I915_WRITE(CURSIZE, size);
>+		intel_crtc->cursor_size = size;
> 	}
> 
>-	/* XXX width must be 64, stride 256 => 0x00 << 28 */
>-	cntl = 0;
>-	if (base)
>-		cntl = (CURSOR_ENABLE |
>-			CURSOR_GAMMA_ENABLE |
>-			CURSOR_FORMAT_ARGB);
> 	if (intel_crtc->cursor_cntl != cntl) {
> 		I915_WRITE(_CURACNTR, cntl);
> 		POSTING_READ(_CURACNTR);
>@@ -8092,46 +8293,10 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
> 				return;
> 		}
> 		cntl |= pipe << 28; /* Connect to correct pipe */
>-	}
>-	if (intel_crtc->cursor_cntl != cntl) {
>-		I915_WRITE(CURCNTR(pipe), cntl);
>-		POSTING_READ(CURCNTR(pipe));
>-		intel_crtc->cursor_cntl = cntl;
>-	}
>-
>-	/* and commit changes on next vblank */
>-	I915_WRITE(CURBASE(pipe), base);
>-	POSTING_READ(CURBASE(pipe));
>-}
>-
>-static void ivb_update_cursor(struct drm_crtc *crtc, u32 base)
>-{
>-	struct drm_device *dev = crtc->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>-	int pipe = intel_crtc->pipe;
>-	uint32_t cntl;
> 
>-	cntl = 0;
>-	if (base) {
>-		cntl = MCURSOR_GAMMA_ENABLE;
>-		switch (intel_crtc->cursor_width) {
>-			case 64:
>-				cntl |= CURSOR_MODE_64_ARGB_AX;
>-				break;
>-			case 128:
>-				cntl |= CURSOR_MODE_128_ARGB_AX;
>-				break;
>-			case 256:
>-				cntl |= CURSOR_MODE_256_ARGB_AX;
>-				break;
>-			default:
>-				WARN_ON(1);
>-				return;
>-		}
>+		if (IS_HASWELL(dev) || IS_BROADWELL(dev))
>+			cntl |= CURSOR_PIPE_CSC_ENABLE;
> 	}
>-	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
>-		cntl |= CURSOR_PIPE_CSC_ENABLE;
> 
> 	if (intel_crtc->cursor_cntl != cntl) {
> 		I915_WRITE(CURCNTR(pipe), cntl);
>@@ -8142,6 +8307,8 @@ static void ivb_update_cursor(struct drm_crtc *crtc, u32 base)
> 	/* and commit changes on next vblank */
> 	I915_WRITE(CURBASE(pipe), base);
> 	POSTING_READ(CURBASE(pipe));
>+
>+	intel_crtc->cursor_base = base;
> }
> 
> /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */
>@@ -8188,13 +8355,47 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
> 
> 	I915_WRITE(CURPOS(pipe), pos);
> 
>-	if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev))
>-		ivb_update_cursor(crtc, base);
>-	else if (IS_845G(dev) || IS_I865G(dev))
>+	if (IS_845G(dev) || IS_I865G(dev))
> 		i845_update_cursor(crtc, base);
> 	else
> 		i9xx_update_cursor(crtc, base);
>-	intel_crtc->cursor_base = base;
>+}
>+
>+static bool cursor_size_ok(struct drm_device *dev,
>+			   uint32_t width, uint32_t height)
>+{
>+	if (width == 0 || height == 0)
>+		return false;
>+
>+	/*
>+	 * 845g/865g are special in that they are only limited by
>+	 * the width of their cursors, the height is arbitrary up to
>+	 * the precision of the register. Everything else requires
>+	 * square cursors, limited to a few power-of-two sizes.
>+	 */
>+	if (IS_845G(dev) || IS_I865G(dev)) {
>+		if ((width & 63) != 0)
>+			return false;
>+
>+		if (width > (IS_845G(dev) ? 64 : 512))
>+			return false;
>+
>+		if (height > 1023)
>+			return false;
>+	} else {
>+		switch (width | height) {
>+		case 256:
>+		case 128:
>+			if (IS_GEN2(dev))
>+				return false;
>+		case 64:
>+			break;
>+		default:
>+			return false;
>+		}
>+	}
>+
>+	return true;
> }
> 
> /*
>@@ -8209,10 +8410,10 @@ static int intel_crtc_cursor_set_obj(struct drm_crtc *crtc,
> 				     uint32_t width, uint32_t height)
> {
> 	struct drm_device *dev = crtc->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = to_i915(dev);
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 	enum pipe pipe = intel_crtc->pipe;
>-	unsigned old_width;
>+	unsigned old_width, stride;
> 	uint32_t addr;
> 	int ret;
> 
>@@ -8220,20 +8421,18 @@ static int intel_crtc_cursor_set_obj(struct drm_crtc *crtc,
> 	if (!obj) {
> 		DRM_DEBUG_KMS("cursor off\n");
> 		addr = 0;
>-		obj = NULL;
> 		mutex_lock(&dev->struct_mutex);
> 		goto finish;
> 	}
> 
> 	/* Check for which cursor types we support */
>-	if (!((width == 64 && height == 64) ||
>-			(width == 128 && height == 128 && !IS_GEN2(dev)) ||
>-			(width == 256 && height == 256 && !IS_GEN2(dev)))) {
>+	if (!cursor_size_ok(dev, width, height)) {
> 		DRM_DEBUG("Cursor dimension not supported\n");
> 		return -EINVAL;
> 	}
> 
>-	if (obj->base.size < width * height * 4) {
>+	stride = roundup_pow_of_two(width) * 4;
>+	if (obj->base.size < stride * height) {
> 		DRM_DEBUG_KMS("buffer is too small\n");
> 		ret = -ENOMEM;
> 		goto fail;
>@@ -8295,9 +8494,6 @@ static int intel_crtc_cursor_set_obj(struct drm_crtc *crtc,
> 		addr = obj->phys_handle->busaddr;
> 	}
> 
>-	if (IS_GEN2(dev))
>-		I915_WRITE(CURSIZE, (height << 12) | width);
>-
>  finish:
> 	if (intel_crtc->cursor_bo) {
> 		if (!INTEL_INFO(dev)->cursor_needs_physical)
>@@ -8942,36 +9138,37 @@ out:
>  */
> static void intel_mark_fb_busy(struct drm_device *dev,
> 			       unsigned frontbuffer_bits,
>-			       struct intel_engine_cs *ring)
>+			       struct i915_gem_request *rq)
> {
>+	struct drm_i915_private *dev_priv = dev->dev_private;
> 	enum pipe pipe;
> 
> 	if (!i915.powersave)
> 		return;
> 
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		if (!(frontbuffer_bits & INTEL_FRONTBUFFER_ALL_MASK(pipe)))
> 			continue;
> 
> 		intel_increase_pllclock(dev, pipe);
>-		if (ring && intel_fbc_enabled(dev))
>-			ring->fbc_dirty = true;
>+		if (rq && intel_fbc_enabled(dev))
>+			rq->pending_flush |= I915_KICK_FBC;
> 	}
> }
> 
> /**
>  * intel_fb_obj_invalidate - invalidate frontbuffer object
>  * @obj: GEM object to invalidate
>- * @ring: set for asynchronous rendering
>+ * @rq: set for asynchronous rendering
>  *
>  * This function gets called every time rendering on the given object starts and
>  * frontbuffer caching (fbc, low refresh rate for DRRS, panel self refresh) must
>- * be invalidated. If @ring is non-NULL any subsequent invalidation will be delayed
>+ * be invalidated. If @engine is non-NULL any subsequent invalidation will be delayed
>  * until the rendering completes or a flip on this frontbuffer plane is
>  * scheduled.
>  */
> void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
>-			     struct intel_engine_cs *ring)
>+			     struct i915_gem_request *rq)
> {
> 	struct drm_device *dev = obj->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>@@ -8981,7 +9178,7 @@ void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
> 	if (!obj->frontbuffer_bits)
> 		return;
> 
>-	if (ring) {
>+	if (rq) {
> 		mutex_lock(&dev_priv->fb_tracking.lock);
> 		dev_priv->fb_tracking.busy_bits
> 			|= obj->frontbuffer_bits;
>@@ -8990,7 +9187,7 @@ void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
> 		mutex_unlock(&dev_priv->fb_tracking.lock);
> 	}
> 
>-	intel_mark_fb_busy(dev, obj->frontbuffer_bits, ring);
>+	intel_mark_fb_busy(dev, obj->frontbuffer_bits, rq);
> 
> 	intel_edp_psr_invalidate(dev, obj->frontbuffer_bits);
> }
>@@ -9019,6 +9216,14 @@ void intel_frontbuffer_flush(struct drm_device *dev,
> 	intel_mark_fb_busy(dev, frontbuffer_bits, NULL);
> 
> 	intel_edp_psr_flush(dev, frontbuffer_bits);
>+
>+	/*
>+	 * FIXME: Unconditional fbc flushing here is a rather gross hack and
>+	 * needs to be reworked into a proper frontbuffer tracking scheme like
>+	 * psr employs.
>+	 */
>+	if (IS_BROADWELL(dev))
>+		gen8_fbc_sw_flush(dev, FBC_REND_CACHE_CLEAN);
> }
> 
> /**
>@@ -9108,12 +9313,11 @@ static void intel_crtc_destroy(struct drm_crtc *crtc)
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 	struct drm_device *dev = crtc->dev;
> 	struct intel_unpin_work *work;
>-	unsigned long flags;
> 
>-	spin_lock_irqsave(&dev->event_lock, flags);
>+	spin_lock_irq(&dev->event_lock);
> 	work = intel_crtc->unpin_work;
> 	intel_crtc->unpin_work = NULL;
>-	spin_unlock_irqrestore(&dev->event_lock, flags);
>+	spin_unlock_irq(&dev->event_lock);
> 
> 	if (work) {
> 		cancel_work_sync(&work->work);
>@@ -9136,6 +9340,7 @@ static void intel_unpin_work_fn(struct work_struct *__work)
> 	intel_unpin_fb_obj(work->old_fb_obj);
> 	drm_gem_object_unreference(&work->pending_flip_obj->base);
> 	drm_gem_object_unreference(&work->old_fb_obj->base);
>+	i915_request_put(work->flip_queued_request);
> 
> 	intel_update_fbc(dev);
> 	mutex_unlock(&dev->struct_mutex);
>@@ -9151,7 +9356,6 @@ static void intel_unpin_work_fn(struct work_struct *__work)
> static void do_intel_finish_page_flip(struct drm_device *dev,
> 				      struct drm_crtc *crtc)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 	struct intel_unpin_work *work;
> 	unsigned long flags;
>@@ -9160,6 +9364,10 @@ static void do_intel_finish_page_flip(struct drm_device *dev,
> 	if (intel_crtc == NULL)
> 		return;
> 
>+	/*
>+	 * This is called both by irq handlers and the reset code (to complete
>+	 * lost pageflips) so needs the full irqsave spinlocks.
>+	 */
> 	spin_lock_irqsave(&dev->event_lock, flags);
> 	work = intel_crtc->unpin_work;
> 
>@@ -9171,23 +9379,9 @@ static void do_intel_finish_page_flip(struct drm_device *dev,
> 		return;
> 	}
> 
>-	/* and that the unpin work is consistent wrt ->pending. */
>-	smp_rmb();
>-
>-	intel_crtc->unpin_work = NULL;
>-
>-	if (work->event)
>-		drm_send_vblank_event(dev, intel_crtc->pipe, work->event);
>-
>-	drm_crtc_vblank_put(crtc);
>+	page_flip_completed(intel_crtc);
> 
> 	spin_unlock_irqrestore(&dev->event_lock, flags);
>-
>-	wake_up_all(&dev_priv->pending_flip_queue);
>-
>-	queue_work(dev_priv->wq, &work->work);
>-
>-	trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj);
> }
> 
> void intel_finish_page_flip(struct drm_device *dev, int pipe)
>@@ -9255,7 +9449,12 @@ void intel_prepare_page_flip(struct drm_device *dev, int plane)
> 		to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]);
> 	unsigned long flags;
> 
>-	/* NB: An MMIO update of the plane base pointer will also
>+
>+	/*
>+	 * This is called both by irq handlers and the reset code (to complete
>+	 * lost pageflips) so needs the full irqsave spinlocks.
>+	 *
>+	 * NB: An MMIO update of the plane base pointer will also
> 	 * generate a page-flip completion irq, i.e. every modeset
> 	 * is also accompanied by a spurious intel_prepare_page_flip().
> 	 */
>@@ -9274,97 +9473,86 @@ static inline void intel_mark_page_flip_active(struct intel_crtc *intel_crtc)
> 	smp_wmb();
> }
> 
>-static int intel_gen2_queue_flip(struct drm_device *dev,
>-				 struct drm_crtc *crtc,
>+static int intel_gen2_queue_flip(struct i915_gem_request *rq,
>+				 struct intel_crtc *crtc,
> 				 struct drm_framebuffer *fb,
> 				 struct drm_i915_gem_object *obj,
>-				 struct intel_engine_cs *ring,
> 				 uint32_t flags)
> {
>-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+	struct intel_ringbuffer *ring;
> 	u32 flip_mask;
>-	int ret;
> 
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 5);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	/* Can't queue multiple flips, so wait for the previous
> 	 * one to finish before executing the next.
> 	 */
>-	if (intel_crtc->plane)
>+	if (crtc->plane)
> 		flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
> 	else
> 		flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
> 	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
>-	intel_ring_emit(ring, MI_NOOP);
> 	intel_ring_emit(ring, MI_DISPLAY_FLIP |
>-			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
>+			MI_DISPLAY_FLIP_PLANE(crtc->plane));
> 	intel_ring_emit(ring, fb->pitches[0]);
>-	intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
>+	intel_ring_emit(ring, crtc->unpin_work->gtt_offset);
> 	intel_ring_emit(ring, 0); /* aux display base address, unused */
>+	intel_ring_advance(ring);
> 
>-	intel_mark_page_flip_active(intel_crtc);
>-	__intel_ring_advance(ring);
> 	return 0;
> }
> 
>-static int intel_gen3_queue_flip(struct drm_device *dev,
>-				 struct drm_crtc *crtc,
>+static int intel_gen3_queue_flip(struct i915_gem_request *rq,
>+				 struct intel_crtc *crtc,
> 				 struct drm_framebuffer *fb,
> 				 struct drm_i915_gem_object *obj,
>-				 struct intel_engine_cs *ring,
> 				 uint32_t flags)
> {
>-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+	struct intel_ringbuffer *ring;
> 	u32 flip_mask;
>-	int ret;
> 
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 4);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
>-	if (intel_crtc->plane)
>+	if (crtc->plane)
> 		flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
> 	else
> 		flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
> 	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
>-	intel_ring_emit(ring, MI_NOOP);
> 	intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 |
>-			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
>+			MI_DISPLAY_FLIP_PLANE(crtc->plane));
> 	intel_ring_emit(ring, fb->pitches[0]);
>-	intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
>-	intel_ring_emit(ring, MI_NOOP);
>+	intel_ring_emit(ring, crtc->unpin_work->gtt_offset);
>+	intel_ring_advance(ring);
> 
>-	intel_mark_page_flip_active(intel_crtc);
>-	__intel_ring_advance(ring);
> 	return 0;
> }
> 
>-static int intel_gen4_queue_flip(struct drm_device *dev,
>-				 struct drm_crtc *crtc,
>+static int intel_gen4_queue_flip(struct i915_gem_request *rq,
>+				 struct intel_crtc *crtc,
> 				 struct drm_framebuffer *fb,
> 				 struct drm_i915_gem_object *obj,
>-				 struct intel_engine_cs *ring,
> 				 uint32_t flags)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+	struct drm_i915_private *dev_priv = rq->i915;
>+	struct intel_ringbuffer *ring;
> 	uint32_t pf, pipesrc;
>-	int ret;
> 
>-	ret = intel_ring_begin(ring, 4);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 4);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	/* i965+ uses the linear or tiled offsets from the
> 	 * Display Registers (which do not change across a page-flip)
> 	 * so we need only reprogram the base address.
> 	 */
> 	intel_ring_emit(ring, MI_DISPLAY_FLIP |
>-			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
>+			MI_DISPLAY_FLIP_PLANE(crtc->plane));
> 	intel_ring_emit(ring, fb->pitches[0]);
>-	intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset |
>+	intel_ring_emit(ring, crtc->unpin_work->gtt_offset |
> 			obj->tiling_mode);
> 
> 	/* XXX Enabling the panel-fitter across page-flip is so far
>@@ -9372,62 +9560,57 @@ static int intel_gen4_queue_flip(struct drm_device *dev,
> 	 * pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
> 	 */
> 	pf = 0;
>-	pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
>+	pipesrc = I915_READ(PIPESRC(crtc->pipe)) & 0x0fff0fff;
> 	intel_ring_emit(ring, pf | pipesrc);
>+	intel_ring_advance(ring);
> 
>-	intel_mark_page_flip_active(intel_crtc);
>-	__intel_ring_advance(ring);
> 	return 0;
> }
> 
>-static int intel_gen6_queue_flip(struct drm_device *dev,
>-				 struct drm_crtc *crtc,
>+static int intel_gen6_queue_flip(struct i915_gem_request *rq,
>+				 struct intel_crtc *crtc,
> 				 struct drm_framebuffer *fb,
> 				 struct drm_i915_gem_object *obj,
>-				 struct intel_engine_cs *ring,
> 				 uint32_t flags)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+	struct drm_i915_private *dev_priv = rq->i915;
>+	struct intel_ringbuffer *ring;
> 	uint32_t pf, pipesrc;
>-	int ret;
> 
>-	ret = intel_ring_begin(ring, 4);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 4);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring, MI_DISPLAY_FLIP |
>-			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
>+			MI_DISPLAY_FLIP_PLANE(crtc->plane));
> 	intel_ring_emit(ring, fb->pitches[0] | obj->tiling_mode);
>-	intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
>+	intel_ring_emit(ring, crtc->unpin_work->gtt_offset);
> 
> 	/* Contrary to the suggestions in the documentation,
> 	 * "Enable Panel Fitter" does not seem to be required when page
> 	 * flipping with a non-native mode, and worse causes a normal
> 	 * modeset to fail.
>-	 * pf = I915_READ(PF_CTL(intel_crtc->pipe)) & PF_ENABLE;
>+	 * pf = I915_READ(PF_CTL(crtc->pipe)) & PF_ENABLE;
> 	 */
> 	pf = 0;
>-	pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
>+	pipesrc = I915_READ(PIPESRC(crtc->pipe)) & 0x0fff0fff;
> 	intel_ring_emit(ring, pf | pipesrc);
>+	intel_ring_advance(ring);
> 
>-	intel_mark_page_flip_active(intel_crtc);
>-	__intel_ring_advance(ring);
> 	return 0;
> }
> 
>-static int intel_gen7_queue_flip(struct drm_device *dev,
>-				 struct drm_crtc *crtc,
>+static int intel_gen7_queue_flip(struct i915_gem_request *rq,
>+				 struct intel_crtc *crtc,
> 				 struct drm_framebuffer *fb,
> 				 struct drm_i915_gem_object *obj,
>-				 struct intel_engine_cs *ring,
> 				 uint32_t flags)
> {
>-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+	struct intel_ringbuffer *ring;
> 	uint32_t plane_bit = 0;
> 	int len, ret;
> 
>-	switch (intel_crtc->plane) {
>+	switch (crtc->plane) {
> 	case PLANE_A:
> 		plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_A;
> 		break;
>@@ -9442,16 +9625,16 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
> 		return -ENODEV;
> 	}
> 
>-	len = 4;
>-	if (ring->id == RCS) {
>+	len = 3;
>+	if (rq->engine->id == RCS) {
> 		len += 6;
> 		/*
> 		 * On Gen 8, SRM is now taking an extra dword to accommodate
> 		 * 48bits addresses, and we need a NOOP for the batch size to
> 		 * stay even.
> 		 */
>-		if (IS_GEN8(dev))
>-			len += 2;
>+		if (IS_GEN8(rq->i915))
>+			len += 1;
> 	}
> 
> 	/*
>@@ -9464,13 +9647,13 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
> 	 * then do the cacheline alignment, and finally emit the
> 	 * MI_DISPLAY_FLIP.
> 	 */
>-	ret = intel_ring_cacheline_align(ring);
>+	ret = intel_ring_cacheline_align(rq);
> 	if (ret)
> 		return ret;
> 
>-	ret = intel_ring_begin(ring, len);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, len);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	/* Unmask the flip-done completion message. Note that the bspec says that
> 	 * we should do this for both the BCS and RCS, and that we must not unmask
>@@ -9481,37 +9664,33 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
> 	 * for the RCS also doesn't appear to drop events. Setting the DERRMR
> 	 * to zero does lead to lockups within MI_DISPLAY_FLIP.
> 	 */
>-	if (ring->id == RCS) {
>+	if (rq->engine->id == RCS) {
> 		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
> 		intel_ring_emit(ring, DERRMR);
> 		intel_ring_emit(ring, ~(DERRMR_PIPEA_PRI_FLIP_DONE |
> 					DERRMR_PIPEB_PRI_FLIP_DONE |
> 					DERRMR_PIPEC_PRI_FLIP_DONE));
>-		if (IS_GEN8(dev))
>+		if (IS_GEN8(rq->i915))
> 			intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8(1) |
> 					      MI_SRM_LRM_GLOBAL_GTT);
> 		else
> 			intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) |
> 					      MI_SRM_LRM_GLOBAL_GTT);
> 		intel_ring_emit(ring, DERRMR);
>-		intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
>-		if (IS_GEN8(dev)) {
>+		intel_ring_emit(ring, rq->engine->scratch.gtt_offset + 256);
>+		if (IS_GEN8(rq->i915))
> 			intel_ring_emit(ring, 0);
>-			intel_ring_emit(ring, MI_NOOP);
>-		}
> 	}
> 
> 	intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | plane_bit);
>-	intel_ring_emit(ring, (fb->pitches[0] | obj->tiling_mode));
>-	intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
>-	intel_ring_emit(ring, (MI_NOOP));
>+	intel_ring_emit(ring, fb->pitches[0] | obj->tiling_mode);
>+	intel_ring_emit(ring, crtc->unpin_work->gtt_offset);
>+	intel_ring_advance(ring);
> 
>-	intel_mark_page_flip_active(intel_crtc);
>-	__intel_ring_advance(ring);
> 	return 0;
> }
> 
>-static bool use_mmio_flip(struct intel_engine_cs *ring,
>+static bool use_mmio_flip(struct intel_engine_cs *engine,
> 			  struct drm_i915_gem_object *obj)
> {
> 	/*
>@@ -9522,10 +9701,10 @@ static bool use_mmio_flip(struct intel_engine_cs *ring,
> 	 * So using MMIO flips there would disrupt this mechanism.
> 	 */
> 
>-	if (ring == NULL)
>+	if (engine == NULL)
> 		return true;
> 
>-	if (INTEL_INFO(ring->dev)->gen < 5)
>+	if (INTEL_INFO(engine->i915)->gen < 5)
> 		return false;
> 
> 	if (i915.use_mmio_flip < 0)
>@@ -9533,7 +9712,7 @@ static bool use_mmio_flip(struct intel_engine_cs *ring,
> 	else if (i915.use_mmio_flip > 0)
> 		return true;
> 	else
>-		return ring != obj->ring;
>+		return engine != i915_request_engine(obj->last_write.request);
> }
> 
> static void intel_do_mmio_flip(struct intel_crtc *intel_crtc)
>@@ -9564,105 +9743,124 @@ static void intel_do_mmio_flip(struct intel_crtc *intel_crtc)
> 	POSTING_READ(DSPSURF(intel_crtc->plane));
> }
> 
>-static int intel_postpone_flip(struct drm_i915_gem_object *obj)
>-{
>-	struct intel_engine_cs *ring;
>-	int ret;
>+struct flip_work {
>+	struct work_struct work;
>+	struct i915_gem_request *rq;
>+	struct intel_crtc *crtc;
>+};
> 
>-	lockdep_assert_held(&obj->base.dev->struct_mutex);
>+static void intel_mmio_flip_work(struct work_struct *work)
>+{
>+	struct flip_work *flip = container_of(work, struct flip_work, work);
> 
>-	if (!obj->last_write_seqno)
>-		return 0;
>+	if (__i915_request_wait(flip->rq, false, NULL, NULL) == 0)
>+		intel_do_mmio_flip(flip->crtc);
> 
>-	ring = obj->ring;
>+	i915_request_put__unlocked(flip->rq);
>+	kfree(flip);
>+}
> 
>-	if (i915_seqno_passed(ring->get_seqno(ring, true),
>-			      obj->last_write_seqno))
>-		return 0;
>+static int intel_queue_mmio_flip(struct intel_crtc *crtc,
>+				 struct i915_gem_request *rq)
>+{
>+	struct flip_work *flip;
> 
>-	ret = i915_gem_check_olr(ring, obj->last_write_seqno);
>-	if (ret)
>-		return ret;
>+	if (WARN_ON(crtc->mmio_flip))
>+		return -EBUSY;
> 
>-	if (WARN_ON(!ring->irq_get(ring)))
>+	if (rq == NULL) {
>+		intel_do_mmio_flip(crtc);
> 		return 0;
>+	}
> 
>-	return 1;
>-}
>-
>-void intel_notify_mmio_flip(struct intel_engine_cs *ring)
>-{
>-	struct drm_i915_private *dev_priv = to_i915(ring->dev);
>-	struct intel_crtc *intel_crtc;
>-	unsigned long irq_flags;
>-	u32 seqno;
>-
>-	seqno = ring->get_seqno(ring, false);
>+	if (i915_request_complete(rq)) {
>+		intel_do_mmio_flip(crtc);
>+		return 0;
>+	}
> 
>-	spin_lock_irqsave(&dev_priv->mmio_flip_lock, irq_flags);
>-	for_each_intel_crtc(ring->dev, intel_crtc) {
>-		struct intel_mmio_flip *mmio_flip;
>+	flip = kmalloc(sizeof(*flip), GFP_KERNEL);
>+	if (flip == NULL)
>+		return -ENOMEM;
> 
>-		mmio_flip = &intel_crtc->mmio_flip;
>-		if (mmio_flip->seqno == 0)
>-			continue;
>+	INIT_WORK(&flip->work, intel_mmio_flip_work);
>+	flip->crtc = crtc;
>+	flip->rq = i915_request_get_breadcrumb(rq);
>+	if (IS_ERR(flip->rq)) {
>+		int ret = PTR_ERR(flip->rq);
>+		kfree(flip);
>+		return ret;
>+	}
> 
>-		if (ring->id != mmio_flip->ring_id)
>-			continue;
>+	schedule_work(&flip->work);
>+	return 0;
>+}
> 
>-		if (i915_seqno_passed(seqno, mmio_flip->seqno)) {
>-			intel_do_mmio_flip(intel_crtc);
>-			mmio_flip->seqno = 0;
>-			ring->irq_put(ring);
>-		}
>-	}
>-	spin_unlock_irqrestore(&dev_priv->mmio_flip_lock, irq_flags);
>+static int intel_default_queue_flip(struct i915_gem_request *rq,
>+				    struct intel_crtc *crtc,
>+				    struct drm_framebuffer *fb,
>+				    struct drm_i915_gem_object *obj,
>+				    uint32_t flags)
>+{
>+	return -ENODEV;
> }
> 
>-static int intel_queue_mmio_flip(struct drm_device *dev,
>-				 struct drm_crtc *crtc,
>-				 struct drm_framebuffer *fb,
>-				 struct drm_i915_gem_object *obj,
>-				 struct intel_engine_cs *ring,
>-				 uint32_t flags)
>+static bool __intel_pageflip_stall_check(struct drm_device *dev,
>+					 struct drm_crtc *crtc)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>-	unsigned long irq_flags;
>-	int ret;
>+	struct intel_unpin_work *work = intel_crtc->unpin_work;
>+	u32 addr;
> 
>-	if (WARN_ON(intel_crtc->mmio_flip.seqno))
>-		return -EBUSY;
>+	if (atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE)
>+		return true;
> 
>-	ret = intel_postpone_flip(obj);
>-	if (ret < 0)
>-		return ret;
>-	if (ret == 0) {
>-		intel_do_mmio_flip(intel_crtc);
>-		return 0;
>+	if (!work->enable_stall_check)
>+		return false;
>+
>+	if (work->flip_ready_vblank == 0) {
>+		if (work->flip_queued_request &&
>+		    !i915_request_complete(work->flip_queued_request))
>+			return false;
>+
>+		work->flip_ready_vblank = drm_vblank_count(dev, intel_crtc->pipe);
> 	}
> 
>-	spin_lock_irqsave(&dev_priv->mmio_flip_lock, irq_flags);
>-	intel_crtc->mmio_flip.seqno = obj->last_write_seqno;
>-	intel_crtc->mmio_flip.ring_id = obj->ring->id;
>-	spin_unlock_irqrestore(&dev_priv->mmio_flip_lock, irq_flags);
>+	if (drm_vblank_count(dev, intel_crtc->pipe) - work->flip_ready_vblank < 3)
>+		return false;
> 
>-	/*
>-	 * Double check to catch cases where irq fired before
>-	 * mmio flip data was ready
>+	/* Potential stall - if we see that the flip has happened,
>+	 * assume a missed interrupt. */
>+	if (INTEL_INFO(dev)->gen >= 4)
>+		addr = I915_HI_DISPBASE(I915_READ(DSPSURF(intel_crtc->plane)));
>+	else
>+		addr = I915_READ(DSPADDR(intel_crtc->plane));
>+
>+	/* There is a potential issue here with a false positive after a flip
>+	 * to the same address. We could address this by checking for a
>+	 * non-incrementing frame counter.
> 	 */
>-	intel_notify_mmio_flip(obj->ring);
>-	return 0;
>+	return addr == work->gtt_offset;
> }
> 
>-static int intel_default_queue_flip(struct drm_device *dev,
>-				    struct drm_crtc *crtc,
>-				    struct drm_framebuffer *fb,
>-				    struct drm_i915_gem_object *obj,
>-				    struct intel_engine_cs *ring,
>-				    uint32_t flags)
>+void intel_check_page_flip(struct drm_device *dev, int pipe)
> {
>-	return -ENODEV;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
>+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+
>+	WARN_ON(!in_irq());
>+
>+	if (crtc == NULL)
>+		return;
>+
>+	spin_lock(&dev->event_lock);
>+	if (intel_crtc->unpin_work && __intel_pageflip_stall_check(dev, crtc)) {
>+		WARN_ONCE(1, "Kicking stuck page flip: queued at %d, now %d\n",
>+			 intel_crtc->unpin_work->flip_queued_vblank, drm_vblank_count(dev, pipe));
>+		page_flip_completed(intel_crtc);
>+	}
>+	spin_unlock(&dev->event_lock);
> }
> 
> static int intel_crtc_page_flip(struct drm_crtc *crtc,
>@@ -9677,10 +9875,13 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 	enum pipe pipe = intel_crtc->pipe;
> 	struct intel_unpin_work *work;
>-	struct intel_engine_cs *ring;
>-	unsigned long flags;
>+	struct intel_engine_cs *engine;
>+	struct i915_gem_request *rq;
> 	int ret;
> 
>+	//trigger software GT busyness calculation
>+	gen8_flip_interrupt(dev);
>+
> 	/*
> 	 * drm_mode_page_flip_ioctl() should already catch this, but double
> 	 * check to be safe.  In the future we may enable pageflipping from
>@@ -9719,17 +9920,25 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
> 		goto free_work;
> 
> 	/* We borrow the event spin lock for protecting unpin_work */
>-	spin_lock_irqsave(&dev->event_lock, flags);
>+	spin_lock_irq(&dev->event_lock);
> 	if (intel_crtc->unpin_work) {
>-		spin_unlock_irqrestore(&dev->event_lock, flags);
>-		kfree(work);
>-		drm_crtc_vblank_put(crtc);
>+		/* Before declaring the flip queue wedged, check if
>+		 * the hardware completed the operation behind our backs.
>+		 */
>+		if (__intel_pageflip_stall_check(dev, crtc)) {
>+			DRM_DEBUG_DRIVER("flip queue: previous flip completed, continuing\n");
>+			page_flip_completed(intel_crtc);
>+		} else {
>+			DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
>+			spin_unlock_irq(&dev->event_lock);
> 
>-		DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
>-		return -EBUSY;
>+			drm_crtc_vblank_put(crtc);
>+			kfree(work);
>+			return -EBUSY;
>+		}
> 	}
> 	intel_crtc->unpin_work = work;
>-	spin_unlock_irqrestore(&dev->event_lock, flags);
>+	spin_unlock_irq(&dev->event_lock);
> 
> 	if (atomic_read(&intel_crtc->unpin_work_count) >= 2)
> 		flush_workqueue(dev_priv->wq);
>@@ -9746,8 +9955,6 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
> 
> 	work->pending_flip_obj = obj;
> 
>-	work->enable_stall_check = true;
>-
> 	atomic_inc(&intel_crtc->unpin_work_count);
> 	intel_crtc->reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
> 
>@@ -9755,35 +9962,65 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
> 		work->flip_count = I915_READ(PIPE_FLIPCOUNT_GM45(pipe)) + 1;
> 
> 	if (IS_VALLEYVIEW(dev)) {
>-		ring = &dev_priv->ring[BCS];
>+		engine = &dev_priv->engine[BCS];
> 		if (obj->tiling_mode != work->old_fb_obj->tiling_mode)
> 			/* vlv: DISPLAY_FLIP fails to change tiling */
>-			ring = NULL;
>+			engine = NULL;
> 	} else if (IS_IVYBRIDGE(dev)) {
>-		ring = &dev_priv->ring[BCS];
>+		engine = &dev_priv->engine[BCS];
> 	} else if (INTEL_INFO(dev)->gen >= 7) {
>-		ring = obj->ring;
>-		if (ring == NULL || ring->id != RCS)
>-			ring = &dev_priv->ring[BCS];
>+		engine = i915_request_engine(obj->last_write.request);
>+		if (engine == NULL || engine->id != RCS)
>+			engine = &dev_priv->engine[BCS];
> 	} else {
>-		ring = &dev_priv->ring[RCS];
>+		engine = &dev_priv->engine[RCS];
> 	}
> 
>-	ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
>-	if (ret)
>-		goto cleanup_pending;
>+	if (use_mmio_flip(engine, obj)) {
>+		rq = i915_request_get(obj->last_write.request);
> 
>-	work->gtt_offset =
>-		i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset;
>+		ret = intel_pin_and_fence_fb_obj(dev, obj, rq);
>+		if (ret)
>+			goto cleanup_rq;
> 
>-	if (use_mmio_flip(ring, obj))
>-		ret = intel_queue_mmio_flip(dev, crtc, fb, obj, ring,
>-					    page_flip_flags);
>-	else
>-		ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, ring,
>-				page_flip_flags);
>-	if (ret)
>-		goto cleanup_unpin;
>+		work->gtt_offset =
>+			i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset;
>+
>+		ret = intel_queue_mmio_flip(intel_crtc, rq);
>+		if (ret)
>+			goto cleanup_unpin;
>+	} else {
>+		struct intel_context *ctx = engine->default_context;
>+		if (obj->last_write.request)
>+			ctx = obj->last_write.request->ctx;
>+		rq = intel_engine_alloc_request(engine, ctx);
>+		if (IS_ERR(rq)) {
>+			ret = PTR_ERR(rq);
>+			goto cleanup_pending;
>+		}
>+
>+		ret = intel_pin_and_fence_fb_obj(dev, obj, rq);
>+		if (ret)
>+			goto cleanup_rq;
>+
>+		work->gtt_offset =
>+			i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset;
>+
>+		ret = dev_priv->display.queue_flip(rq, intel_crtc, fb, obj,
>+						   page_flip_flags);
>+		if (ret)
>+			goto cleanup_unpin;
>+
>+		intel_mark_page_flip_active(intel_crtc);
>+
>+		ret = i915_request_commit(rq);
>+		if (ret)
>+			goto cleanup_unpin;
>+	}
>+
>+	work->flip_queued_request = rq;
>+	work->flip_queued_vblank = drm_vblank_count(dev, intel_crtc->pipe);
>+	work->enable_stall_check = true;
> 
> 	i915_gem_track_fb(work->old_fb_obj, obj,
> 			  INTEL_FRONTBUFFER_PRIMARY(pipe));
>@@ -9798,6 +10035,8 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
> 
> cleanup_unpin:
> 	intel_unpin_fb_obj(obj);
>+cleanup_rq:
>+	i915_request_put(rq);
> cleanup_pending:
> 	atomic_dec(&intel_crtc->unpin_work_count);
> 	crtc->primary->fb = old_fb;
>@@ -9806,9 +10045,9 @@ cleanup_pending:
> 	mutex_unlock(&dev->struct_mutex);
> 
> cleanup:
>-	spin_lock_irqsave(&dev->event_lock, flags);
>+	spin_lock_irq(&dev->event_lock);
> 	intel_crtc->unpin_work = NULL;
>-	spin_unlock_irqrestore(&dev->event_lock, flags);
>+	spin_unlock_irq(&dev->event_lock);
> 
> 	drm_crtc_vblank_put(crtc);
> free_work:
>@@ -9818,8 +10057,11 @@ free_work:
> out_hang:
> 		intel_crtc_wait_for_pending_flips(crtc);
> 		ret = intel_pipe_set_base(crtc, crtc->x, crtc->y, fb);
>-		if (ret == 0 && event)
>+		if (ret == 0 && event) {
>+			spin_lock_irq(&dev->event_lock);
> 			drm_send_vblank_event(dev, pipe, event);
>+			spin_unlock_irq(&dev->event_lock);
>+		}
> 	}
> 	return ret;
> }
>@@ -9847,8 +10089,7 @@ static void intel_modeset_update_staged_output_state(struct drm_device *dev)
> 			to_intel_encoder(connector->base.encoder);
> 	}
> 
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>-			    base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		encoder->new_crtc =
> 			to_intel_crtc(encoder->base.crtc);
> 	}
>@@ -9879,8 +10120,7 @@ static void intel_modeset_commit_output_state(struct drm_device *dev)
> 		connector->base.encoder = &connector->new_encoder->base;
> 	}
> 
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>-			    base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		encoder->base.crtc = &encoder->new_crtc->base;
> 	}
> 
>@@ -10007,6 +10247,15 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
> 		      pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n,
> 		      pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n,
> 		      pipe_config->dp_m_n.tu);
>+
>+	DRM_DEBUG_KMS("dp: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n",
>+		      pipe_config->has_dp_encoder,
>+		      pipe_config->dp_m2_n2.gmch_m,
>+		      pipe_config->dp_m2_n2.gmch_n,
>+		      pipe_config->dp_m2_n2.link_m,
>+		      pipe_config->dp_m2_n2.link_n,
>+		      pipe_config->dp_m2_n2.tu);
>+
> 	DRM_DEBUG_KMS("requested mode:\n");
> 	drm_mode_debug_printmodeline(&pipe_config->requested_mode);
> 	DRM_DEBUG_KMS("adjusted mode:\n");
>@@ -10041,8 +10290,7 @@ static bool check_single_encoder_cloning(struct intel_crtc *crtc,
> 	struct drm_device *dev = crtc->base.dev;
> 	struct intel_encoder *source_encoder;
> 
>-	list_for_each_entry(source_encoder,
>-			    &dev->mode_config.encoder_list, base.head) {
>+	for_each_intel_encoder(dev, source_encoder) {
> 		if (source_encoder->new_crtc != crtc)
> 			continue;
> 
>@@ -10058,8 +10306,7 @@ static bool check_encoder_cloning(struct intel_crtc *crtc)
> 	struct drm_device *dev = crtc->base.dev;
> 	struct intel_encoder *encoder;
> 
>-	list_for_each_entry(encoder,
>-			    &dev->mode_config.encoder_list, base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		if (encoder->new_crtc != crtc)
> 			continue;
> 
>@@ -10143,8 +10390,7 @@ encoder_retry:
> 	 * adjust it according to limitations or connector properties, and also
> 	 * a chance to reject the mode entirely.
> 	 */
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>-			    base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 
> 		if (&encoder->new_crtc->base != crtc)
> 			continue;
>@@ -10222,8 +10468,7 @@ intel_modeset_affected_pipes(struct drm_crtc *crtc, unsigned *modeset_pipes,
> 				1 << connector->new_encoder->new_crtc->pipe;
> 	}
> 
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>-			    base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		if (encoder->base.crtc == &encoder->new_crtc->base)
> 			continue;
> 
>@@ -10297,8 +10542,7 @@ intel_modeset_update_state(struct drm_device *dev, unsigned prepare_pipes)
> 	struct intel_crtc *intel_crtc;
> 	struct drm_connector *connector;
> 
>-	list_for_each_entry(intel_encoder, &dev->mode_config.encoder_list,
>-			    base.head) {
>+	for_each_intel_encoder(dev, intel_encoder) {
> 		if (!intel_encoder->base.crtc)
> 			continue;
> 
>@@ -10387,6 +10631,22 @@ intel_pipe_config_compare(struct drm_device *dev,
> 		return false; \
> 	}
> 
>+/* This is required for BDW+ where there is only one set of registers for
>+ * switching between high and low RR.
>+ * This macro can be used whenever a comparison has to be made between one
>+ * hw state and multiple sw state variables.
>+ */
>+#define PIPE_CONF_CHECK_I_ALT(name, alt_name) \
>+	if ((current_config->name != pipe_config->name) && \
>+		(current_config->alt_name != pipe_config->name)) { \
>+			DRM_ERROR("mismatch in " #name " " \
>+				  "(expected %i or %i, found %i)\n", \
>+				  current_config->name, \
>+				  current_config->alt_name, \
>+				  pipe_config->name); \
>+			return false; \
>+	}
>+
> #define PIPE_CONF_CHECK_FLAGS(name, mask)	\
> 	if ((current_config->name ^ pipe_config->name) & (mask)) { \
> 		DRM_ERROR("mismatch in " #name "(" #mask ") "	   \
>@@ -10419,11 +10679,28 @@ intel_pipe_config_compare(struct drm_device *dev,
> 	PIPE_CONF_CHECK_I(fdi_m_n.tu);
> 
> 	PIPE_CONF_CHECK_I(has_dp_encoder);
>-	PIPE_CONF_CHECK_I(dp_m_n.gmch_m);
>-	PIPE_CONF_CHECK_I(dp_m_n.gmch_n);
>-	PIPE_CONF_CHECK_I(dp_m_n.link_m);
>-	PIPE_CONF_CHECK_I(dp_m_n.link_n);
>-	PIPE_CONF_CHECK_I(dp_m_n.tu);
>+
>+	if (INTEL_INFO(dev)->gen < 8) {
>+		PIPE_CONF_CHECK_I(dp_m_n.gmch_m);
>+		PIPE_CONF_CHECK_I(dp_m_n.gmch_n);
>+		PIPE_CONF_CHECK_I(dp_m_n.link_m);
>+		PIPE_CONF_CHECK_I(dp_m_n.link_n);
>+		PIPE_CONF_CHECK_I(dp_m_n.tu);
>+
>+		if (current_config->has_drrs) {
>+			PIPE_CONF_CHECK_I(dp_m2_n2.gmch_m);
>+			PIPE_CONF_CHECK_I(dp_m2_n2.gmch_n);
>+			PIPE_CONF_CHECK_I(dp_m2_n2.link_m);
>+			PIPE_CONF_CHECK_I(dp_m2_n2.link_n);
>+			PIPE_CONF_CHECK_I(dp_m2_n2.tu);
>+		}
>+	} else {
>+		PIPE_CONF_CHECK_I_ALT(dp_m_n.gmch_m, dp_m2_n2.gmch_m);
>+		PIPE_CONF_CHECK_I_ALT(dp_m_n.gmch_n, dp_m2_n2.gmch_n);
>+		PIPE_CONF_CHECK_I_ALT(dp_m_n.link_m, dp_m2_n2.link_m);
>+		PIPE_CONF_CHECK_I_ALT(dp_m_n.link_n, dp_m2_n2.link_n);
>+		PIPE_CONF_CHECK_I_ALT(dp_m_n.tu, dp_m2_n2.tu);
>+	}
> 
> 	PIPE_CONF_CHECK_I(adjusted_mode.crtc_hdisplay);
> 	PIPE_CONF_CHECK_I(adjusted_mode.crtc_htotal);
>@@ -10509,6 +10786,7 @@ intel_pipe_config_compare(struct drm_device *dev,
> 
> #undef PIPE_CONF_CHECK_X
> #undef PIPE_CONF_CHECK_I
>+#undef PIPE_CONF_CHECK_I_ALT
> #undef PIPE_CONF_CHECK_FLAGS
> #undef PIPE_CONF_CHECK_CLOCK_FUZZY
> #undef PIPE_CONF_QUIRK
>@@ -10538,8 +10816,7 @@ check_encoder_state(struct drm_device *dev)
> 	struct intel_encoder *encoder;
> 	struct intel_connector *connector;
> 
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>-			    base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		bool enabled = false;
> 		bool active = false;
> 		enum pipe pipe, tracked_pipe;
>@@ -10618,8 +10895,7 @@ check_crtc_state(struct drm_device *dev)
> 		WARN(crtc->active && !crtc->base.enabled,
> 		     "active crtc, but not enabled in sw tracking\n");
> 
>-		list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>-				    base.head) {
>+		for_each_intel_encoder(dev, encoder) {
> 			if (encoder->base.crtc != &crtc->base)
> 				continue;
> 			enabled = true;
>@@ -10637,12 +10913,12 @@ check_crtc_state(struct drm_device *dev)
> 		active = dev_priv->display.get_pipe_config(crtc,
> 							   &pipe_config);
> 
>-		/* hw state is inconsistent with the pipe A quirk */
>-		if (crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE)
>+		/* hw state is inconsistent with the pipe quirk */
>+		if ((crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
>+		    (crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
> 			active = crtc->active;
> 
>-		list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>-				    base.head) {
>+		for_each_intel_encoder(dev, encoder) {
> 			enum pipe pipe;
> 			if (encoder->base.crtc != &crtc->base)
> 				continue;
>@@ -10656,7 +10932,7 @@ check_crtc_state(struct drm_device *dev)
> 
> 		if (active &&
> 		    !intel_pipe_config_compare(dev, &crtc->config, &pipe_config)) {
>-			DRM_DEBUG_KMS("pipe state doesn't match!\n");
>+			WARN(1, "pipe state doesn't match!\n");
> 			intel_dump_pipe_config(crtc, &pipe_config,
> 					       "[hw state]");
> 			intel_dump_pipe_config(crtc, &crtc->config,
>@@ -11010,7 +11286,7 @@ static void intel_set_config_restore_state(struct drm_device *dev,
> 	}
> 
> 	count = 0;
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		encoder->new_crtc =
> 			to_intel_crtc(config->save_encoder_crtcs[count++]);
> 	}
>@@ -11169,8 +11445,7 @@ intel_modeset_stage_output_state(struct drm_device *dev,
> 	}
> 
> 	/* Check for any encoders that needs to be disabled. */
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>-			    base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		int num_connectors = 0;
> 		list_for_each_entry(connector,
> 				    &dev->mode_config.connector_list,
>@@ -11203,9 +11478,7 @@ intel_modeset_stage_output_state(struct drm_device *dev,
> 	for_each_intel_crtc(dev, crtc) {
> 		crtc->new_enabled = false;
> 
>-		list_for_each_entry(encoder,
>-				    &dev->mode_config.encoder_list,
>-				    base.head) {
>+		for_each_intel_encoder(dev, encoder) {
> 			if (encoder->new_crtc == crtc) {
> 				crtc->new_enabled = true;
> 				break;
>@@ -11242,7 +11515,7 @@ static void disable_crtc_nofb(struct intel_crtc *crtc)
> 			connector->new_encoder = NULL;
> 	}
> 
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		if (encoder->new_crtc == crtc)
> 			encoder->new_crtc = NULL;
> 	}
>@@ -11305,7 +11578,6 @@ static int intel_crtc_set_config(struct drm_mode_set *set)
> 		ret = intel_set_mode(set->crtc, set->mode,
> 				     set->x, set->y, set->fb);
> 	} else if (config->fb_changed) {
>-		struct drm_i915_private *dev_priv = dev->dev_private;
> 		struct intel_crtc *intel_crtc = to_intel_crtc(set->crtc);
> 
> 		intel_crtc_wait_for_pending_flips(set->crtc);
>@@ -11319,8 +11591,7 @@ static int intel_crtc_set_config(struct drm_mode_set *set)
> 		 */
> 		if (!intel_crtc->primary_enabled && ret == 0) {
> 			WARN_ON(!intel_crtc->active);
>-			intel_enable_primary_hw_plane(dev_priv, intel_crtc->plane,
>-						      intel_crtc->pipe);
>+			intel_enable_primary_hw_plane(set->crtc->primary, set->crtc);
> 		}
> 
> 		/*
>@@ -11473,8 +11744,6 @@ static int
> intel_primary_plane_disable(struct drm_plane *plane)
> {
> 	struct drm_device *dev = plane->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_plane *intel_plane = to_intel_plane(plane);
> 	struct intel_crtc *intel_crtc;
> 
> 	if (!plane->fb)
>@@ -11497,8 +11766,8 @@ intel_primary_plane_disable(struct drm_plane *plane)
> 		goto disable_unpin;
> 
> 	intel_crtc_wait_for_pending_flips(plane->crtc);
>-	intel_disable_primary_hw_plane(dev_priv, intel_plane->plane,
>-				       intel_plane->pipe);
>+	intel_disable_primary_hw_plane(plane, plane->crtc);
>+
> disable_unpin:
> 	mutex_lock(&dev->struct_mutex);
> 	i915_gem_track_fb(intel_fb_obj(plane->fb), NULL,
>@@ -11511,75 +11780,37 @@ disable_unpin:
> }
> 
> static int
>-intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc,
>-			     struct drm_framebuffer *fb, int crtc_x, int crtc_y,
>-			     unsigned int crtc_w, unsigned int crtc_h,
>-			     uint32_t src_x, uint32_t src_y,
>-			     uint32_t src_w, uint32_t src_h)
>+intel_check_primary_plane(struct drm_plane *plane,
>+			  struct intel_plane_state *state)
>+{
>+	struct drm_crtc *crtc = state->crtc;
>+	struct drm_framebuffer *fb = state->fb;
>+	struct drm_rect *dest = &state->dst;
>+	struct drm_rect *src = &state->src;
>+	const struct drm_rect *clip = &state->clip;
>+
>+	return drm_plane_helper_check_update(plane, crtc, fb,
>+					    src, dest, clip,
>+					    DRM_PLANE_HELPER_NO_SCALING,
>+					    DRM_PLANE_HELPER_NO_SCALING,
>+					    false, true, &state->visible);
>+}
>+
>+static int
>+intel_commit_primary_plane(struct drm_plane *plane,
>+			   struct intel_plane_state *state)
> {
>+	struct drm_crtc *crtc = state->crtc;
>+	struct drm_framebuffer *fb = state->fb;
> 	struct drm_device *dev = crtc->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>-	struct intel_plane *intel_plane = to_intel_plane(plane);
> 	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
> 	struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->fb);
>-	struct drm_rect dest = {
>-		/* integer pixels */
>-		.x1 = crtc_x,
>-		.y1 = crtc_y,
>-		.x2 = crtc_x + crtc_w,
>-		.y2 = crtc_y + crtc_h,
>-	};
>-	struct drm_rect src = {
>-		/* 16.16 fixed point */
>-		.x1 = src_x,
>-		.y1 = src_y,
>-		.x2 = src_x + src_w,
>-		.y2 = src_y + src_h,
>-	};
>-	const struct drm_rect clip = {
>-		/* integer pixels */
>-		.x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0,
>-		.y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0,
>-	};
>-	bool visible;
>+	struct intel_plane *intel_plane = to_intel_plane(plane);
>+	struct drm_rect *src = &state->src;
> 	int ret;
> 
>-	ret = drm_plane_helper_check_update(plane, crtc, fb,
>-					    &src, &dest, &clip,
>-					    DRM_PLANE_HELPER_NO_SCALING,
>-					    DRM_PLANE_HELPER_NO_SCALING,
>-					    false, true, &visible);
>-
>-	if (ret)
>-		return ret;
>-
>-	/*
>-	 * If the CRTC isn't enabled, we're just pinning the framebuffer,
>-	 * updating the fb pointer, and returning without touching the
>-	 * hardware.  This allows us to later do a drmModeSetCrtc with fb=-1 to
>-	 * turn on the display with all planes setup as desired.
>-	 */
>-	if (!crtc->enabled) {
>-		mutex_lock(&dev->struct_mutex);
>-
>-		/*
>-		 * If we already called setplane while the crtc was disabled,
>-		 * we may have an fb pinned; unpin it.
>-		 */
>-		if (plane->fb)
>-			intel_unpin_fb_obj(old_obj);
>-
>-		i915_gem_track_fb(old_obj, obj,
>-				  INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe));
>-
>-		/* Pin and return without programming hardware */
>-		ret = intel_pin_and_fence_fb_obj(dev, obj, NULL);
>-		mutex_unlock(&dev->struct_mutex);
>-
>-		return ret;
>-	}
>-
> 	intel_crtc_wait_for_pending_flips(crtc);
> 
> 	/*
>@@ -11588,7 +11819,7 @@ intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc,
> 	 * happens if userspace explicitly disables the plane by passing fb=0
> 	 * because plane->fb still gets set and pinned.
> 	 */
>-	if (!visible) {
>+	if (!state->visible) {
> 		mutex_lock(&dev->struct_mutex);
> 
> 		/*
>@@ -11607,9 +11838,7 @@ intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc,
> 				  INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe));
> 
> 		if (intel_crtc->primary_enabled)
>-			intel_disable_primary_hw_plane(dev_priv,
>-						       intel_plane->plane,
>-						       intel_plane->pipe);
>+			intel_disable_primary_hw_plane(plane, crtc);
> 
> 
> 		if (plane->fb != fb)
>@@ -11618,16 +11847,85 @@ intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc,
> 
> 		mutex_unlock(&dev->struct_mutex);
> 
>-		return 0;
>+	} else {
>+		if (intel_crtc && intel_crtc->active &&
>+		    intel_crtc->primary_enabled) {
>+			/*
>+			 * FBC does not work on some platforms for rotated
>+			 * planes, so disable it when rotation is not 0 and
>+			 * update it when rotation is set back to 0.
>+			 *
>+			 * FIXME: This is redundant with the fbc update done in
>+			 * the primary plane enable function except that that
>+			 * one is done too late. We eventually need to unify
>+			 * this.
>+			 */
>+			if (INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) &&
>+			    dev_priv->fbc.plane == intel_crtc->plane &&
>+			    intel_plane->rotation != BIT(DRM_ROTATE_0)) {
>+				intel_disable_fbc(dev);
>+			}
>+		}
>+		ret = intel_pipe_set_base(crtc, src->x1, src->y1, fb);
>+		if (ret)
>+			return ret;
>+
>+		if (!intel_crtc->primary_enabled)
>+			intel_enable_primary_hw_plane(plane, crtc);
> 	}
> 
>-	ret = intel_pipe_set_base(crtc, src.x1, src.y1, fb);
>+	intel_plane->crtc_x = state->orig_dst.x1;
>+	intel_plane->crtc_y = state->orig_dst.y1;
>+	intel_plane->crtc_w = drm_rect_width(&state->orig_dst);
>+	intel_plane->crtc_h = drm_rect_height(&state->orig_dst);
>+	intel_plane->src_x = state->orig_src.x1;
>+	intel_plane->src_y = state->orig_src.y1;
>+	intel_plane->src_w = drm_rect_width(&state->orig_src);
>+	intel_plane->src_h = drm_rect_height(&state->orig_src);
>+	intel_plane->obj = obj;
>+
>+	return 0;
>+}
>+
>+static int
>+intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc,
>+			     struct drm_framebuffer *fb, int crtc_x, int crtc_y,
>+			     unsigned int crtc_w, unsigned int crtc_h,
>+			     uint32_t src_x, uint32_t src_y,
>+			     uint32_t src_w, uint32_t src_h)
>+{
>+	struct intel_plane_state state;
>+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+	int ret;
>+
>+	state.crtc = crtc;
>+	state.fb = fb;
>+
>+	/* sample coordinates in 16.16 fixed point */
>+	state.src.x1 = src_x;
>+	state.src.x2 = src_x + src_w;
>+	state.src.y1 = src_y;
>+	state.src.y2 = src_y + src_h;
>+
>+	/* integer pixels */
>+	state.dst.x1 = crtc_x;
>+	state.dst.x2 = crtc_x + crtc_w;
>+	state.dst.y1 = crtc_y;
>+	state.dst.y2 = crtc_y + crtc_h;
>+
>+	state.clip.x1 = 0;
>+	state.clip.y1 = 0;
>+	state.clip.x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0;
>+	state.clip.y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0;
>+
>+	state.orig_src = state.src;
>+	state.orig_dst = state.dst;
>+
>+	ret = intel_check_primary_plane(plane, &state);
> 	if (ret)
> 		return ret;
> 
>-	if (!intel_crtc->primary_enabled)
>-		intel_enable_primary_hw_plane(dev_priv, intel_crtc->plane,
>-					      intel_crtc->pipe);
>+	intel_commit_primary_plane(plane, &state);
> 
> 	return 0;
> }
>@@ -11644,6 +11942,7 @@ static const struct drm_plane_funcs intel_primary_plane_funcs = {
> 	.update_plane = intel_primary_plane_setplane,
> 	.disable_plane = intel_primary_plane_disable,
> 	.destroy = intel_plane_destroy,
>+	.set_property = intel_plane_set_property
> };
> 
> static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
>@@ -11661,6 +11960,7 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
> 	primary->max_downscale = 1;
> 	primary->pipe = pipe;
> 	primary->plane = pipe;
>+	primary->rotation = BIT(DRM_ROTATE_0);
> 	if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4)
> 		primary->plane = !pipe;
> 
>@@ -11676,6 +11976,7 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
> 				 &intel_primary_plane_funcs,
> 				 intel_primary_formats, num_formats,
> 				 DRM_PLANE_TYPE_PRIMARY);
>+
> 	return &primary->base;
> }
> 
>@@ -11691,6 +11992,50 @@ intel_cursor_plane_disable(struct drm_plane *plane)
> }
> 
> static int
>+intel_check_cursor_plane(struct drm_plane *plane,
>+			 struct intel_plane_state *state)
>+{
>+	struct drm_crtc *crtc = state->crtc;
>+	struct drm_framebuffer *fb = state->fb;
>+	struct drm_rect *dest = &state->dst;
>+	struct drm_rect *src = &state->src;
>+	const struct drm_rect *clip = &state->clip;
>+
>+	return drm_plane_helper_check_update(plane, crtc, fb,
>+					    src, dest, clip,
>+					    DRM_PLANE_HELPER_NO_SCALING,
>+					    DRM_PLANE_HELPER_NO_SCALING,
>+					    true, true, &state->visible);
>+}
>+
>+static int
>+intel_commit_cursor_plane(struct drm_plane *plane,
>+			  struct intel_plane_state *state)
>+{
>+	struct drm_crtc *crtc = state->crtc;
>+	struct drm_framebuffer *fb = state->fb;
>+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
>+	struct drm_i915_gem_object *obj = intel_fb->obj;
>+	int crtc_w, crtc_h;
>+
>+	crtc->cursor_x = state->orig_dst.x1;
>+	crtc->cursor_y = state->orig_dst.y1;
>+	if (fb != crtc->cursor->fb) {
>+		crtc_w = drm_rect_width(&state->orig_dst);
>+		crtc_h = drm_rect_height(&state->orig_dst);
>+		return intel_crtc_cursor_set_obj(crtc, obj, crtc_w, crtc_h);
>+	} else {
>+		intel_crtc_update_cursor(crtc, state->visible);
>+
>+		intel_frontbuffer_flip(crtc->dev,
>+				       INTEL_FRONTBUFFER_CURSOR(intel_crtc->pipe));
>+
>+		return 0;
>+	}
>+}
>+
>+static int
> intel_cursor_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
> 			  struct drm_framebuffer *fb, int crtc_x, int crtc_y,
> 			  unsigned int crtc_w, unsigned int crtc_h,
>@@ -11698,47 +12043,39 @@ intel_cursor_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
> 			  uint32_t src_w, uint32_t src_h)
> {
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>-	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
>-	struct drm_i915_gem_object *obj = intel_fb->obj;
>-	struct drm_rect dest = {
>-		/* integer pixels */
>-		.x1 = crtc_x,
>-		.y1 = crtc_y,
>-		.x2 = crtc_x + crtc_w,
>-		.y2 = crtc_y + crtc_h,
>-	};
>-	struct drm_rect src = {
>-		/* 16.16 fixed point */
>-		.x1 = src_x,
>-		.y1 = src_y,
>-		.x2 = src_x + src_w,
>-		.y2 = src_y + src_h,
>-	};
>-	const struct drm_rect clip = {
>-		/* integer pixels */
>-		.x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0,
>-		.y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0,
>-	};
>-	bool visible;
>+	struct intel_plane_state state;
> 	int ret;
> 
>-	ret = drm_plane_helper_check_update(plane, crtc, fb,
>-					    &src, &dest, &clip,
>-					    DRM_PLANE_HELPER_NO_SCALING,
>-					    DRM_PLANE_HELPER_NO_SCALING,
>-					    true, true, &visible);
>+	state.crtc = crtc;
>+	state.fb = fb;
>+
>+	/* sample coordinates in 16.16 fixed point */
>+	state.src.x1 = src_x;
>+	state.src.x2 = src_x + src_w;
>+	state.src.y1 = src_y;
>+	state.src.y2 = src_y + src_h;
>+
>+	/* integer pixels */
>+	state.dst.x1 = crtc_x;
>+	state.dst.x2 = crtc_x + crtc_w;
>+	state.dst.y1 = crtc_y;
>+	state.dst.y2 = crtc_y + crtc_h;
>+
>+	state.clip.x1 = 0;
>+	state.clip.y1 = 0;
>+	state.clip.x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0;
>+	state.clip.y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0;
>+
>+	state.orig_src = state.src;
>+	state.orig_dst = state.dst;
>+
>+	ret = intel_check_cursor_plane(plane, &state);
> 	if (ret)
> 		return ret;
> 
>-	crtc->cursor_x = crtc_x;
>-	crtc->cursor_y = crtc_y;
>-	if (fb != crtc->cursor->fb) {
>-		return intel_crtc_cursor_set_obj(crtc, obj, crtc_w, crtc_h);
>-	} else {
>-		intel_crtc_update_cursor(crtc, visible);
>-		return 0;
>-	}
>+	return intel_commit_cursor_plane(plane, &state);
> }
>+
> static const struct drm_plane_funcs intel_cursor_plane_funcs = {
> 	.update_plane = intel_cursor_plane_update,
> 	.disable_plane = intel_cursor_plane_disable,
>@@ -11812,8 +12149,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
> 
> 	intel_crtc->cursor_base = ~0;
> 	intel_crtc->cursor_cntl = ~0;
>-
>-	init_waitqueue_head(&intel_crtc->vbl_wait);
>+	intel_crtc->cursor_size = ~0;
> 
> 	BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
> 	       dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL);
>@@ -11876,8 +12212,7 @@ static int intel_encoder_clones(struct intel_encoder *encoder)
> 	int index_mask = 0;
> 	int entry = 0;
> 
>-	list_for_each_entry(source_encoder,
>-			    &dev->mode_config.encoder_list, base.head) {
>+	for_each_intel_encoder(dev, source_encoder) {
> 		if (encoders_cloneable(encoder, source_encoder))
> 			index_mask |= (1 << entry);
> 
>@@ -12066,7 +12401,7 @@ static void intel_setup_outputs(struct drm_device *dev)
> 
> 	intel_edp_psr_init(dev);
> 
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		encoder->base.possible_crtcs = encoder->crtc_mask;
> 		encoder->base.possible_clones =
> 			intel_encoder_clones(encoder);
>@@ -12332,29 +12667,27 @@ static void intel_init_display(struct drm_device *dev)
> 		dev_priv->display.get_display_clock_speed =
> 			i830_get_display_clock_speed;
> 
>-	if (HAS_PCH_SPLIT(dev)) {
>-		if (IS_GEN5(dev)) {
>-			dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
>-			dev_priv->display.write_eld = ironlake_write_eld;
>-		} else if (IS_GEN6(dev)) {
>-			dev_priv->display.fdi_link_train = gen6_fdi_link_train;
>-			dev_priv->display.write_eld = ironlake_write_eld;
>-			dev_priv->display.modeset_global_resources =
>-				snb_modeset_global_resources;
>-		} else if (IS_IVYBRIDGE(dev)) {
>-			/* FIXME: detect B0+ stepping and use auto training */
>-			dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
>-			dev_priv->display.write_eld = ironlake_write_eld;
>-			dev_priv->display.modeset_global_resources =
>-				ivb_modeset_global_resources;
>-		} else if (IS_HASWELL(dev) || IS_GEN8(dev)) {
>-			dev_priv->display.fdi_link_train = hsw_fdi_link_train;
>-			dev_priv->display.write_eld = haswell_write_eld;
>-			dev_priv->display.modeset_global_resources =
>-				haswell_modeset_global_resources;
>-		}
>-	} else if (IS_G4X(dev)) {
>+	if (IS_G4X(dev)) {
> 		dev_priv->display.write_eld = g4x_write_eld;
>+	} else if (IS_GEN5(dev)) {
>+		dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
>+		dev_priv->display.write_eld = ironlake_write_eld;
>+	} else if (IS_GEN6(dev)) {
>+		dev_priv->display.fdi_link_train = gen6_fdi_link_train;
>+		dev_priv->display.write_eld = ironlake_write_eld;
>+		dev_priv->display.modeset_global_resources =
>+			snb_modeset_global_resources;
>+	} else if (IS_IVYBRIDGE(dev)) {
>+		/* FIXME: detect B0+ stepping and use auto training */
>+		dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
>+		dev_priv->display.write_eld = ironlake_write_eld;
>+		dev_priv->display.modeset_global_resources =
>+			ivb_modeset_global_resources;
>+	} else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
>+		dev_priv->display.fdi_link_train = hsw_fdi_link_train;
>+		dev_priv->display.write_eld = haswell_write_eld;
>+		dev_priv->display.modeset_global_resources =
>+			haswell_modeset_global_resources;
> 	} else if (IS_VALLEYVIEW(dev)) {
> 		dev_priv->display.modeset_global_resources =
> 			valleyview_modeset_global_resources;
>@@ -12388,6 +12721,8 @@ static void intel_init_display(struct drm_device *dev)
> 	}
> 
> 	intel_panel_init_backlight_funcs(dev);
>+
>+	mutex_init(&dev_priv->pps_mutex);
> }
> 
> /*
>@@ -12403,6 +12738,14 @@ static void quirk_pipea_force(struct drm_device *dev)
> 	DRM_INFO("applying pipe a force quirk\n");
> }
> 
>+static void quirk_pipeb_force(struct drm_device *dev)
>+{
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+
>+	dev_priv->quirks |= QUIRK_PIPEB_FORCE;
>+	DRM_INFO("applying pipe b force quirk\n");
>+}
>+
> /*
>  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
>  */
>@@ -12477,6 +12820,12 @@ static struct intel_quirk intel_quirks[] = {
> 	/* ThinkPad T60 needs pipe A force quirk (bug #16494) */
> 	{ 0x2782, 0x17aa, 0x201a, quirk_pipea_force },
> 
>+	/* 830 needs to leave pipe A & dpll A up */
>+	{ 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
>+
>+	/* 830 needs to leave pipe B & dpll B up */
>+	{ 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipeb_force },
>+
> 	/* Lenovo U160 cannot use SSC on LVDS */
> 	{ 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
> 
>@@ -12550,7 +12899,11 @@ static void i915_disable_vga(struct drm_device *dev)
> 	vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
> 	udelay(300);
> 
>-	I915_WRITE(vga_reg, VGA_DISP_DISABLE);
>+	/*
>+	 * Fujitsu-Siemens Lifebook S6010 (830) has problems resuming
>+	 * from S3 without preserving (some of?) the other bits.
>+	 */
>+	I915_WRITE(vga_reg, dev_priv->bios_vgacntr | VGA_DISP_DISABLE);
> 	POSTING_READ(vga_reg);
> }
> 
>@@ -12563,8 +12916,6 @@ void intel_modeset_init_hw(struct drm_device *dev)
> 
> 	intel_init_clock_gating(dev);
> 
>-	intel_reset_dpio(dev);
>-
> 	intel_enable_gt_powersave(dev);
> }
> 
>@@ -12610,7 +12961,10 @@ void intel_modeset_init(struct drm_device *dev)
> 		dev->mode_config.max_height = 8192;
> 	}
> 
>-	if (IS_GEN2(dev)) {
>+	if (IS_845G(dev) || IS_I865G(dev)) {
>+		dev->mode_config.cursor_width = IS_845G(dev) ? 64 : 512;
>+		dev->mode_config.cursor_height = 1023;
>+	} else if (IS_GEN2(dev)) {
> 		dev->mode_config.cursor_width = GEN2_CURSOR_WIDTH;
> 		dev->mode_config.cursor_height = GEN2_CURSOR_HEIGHT;
> 	} else {
>@@ -12624,7 +12978,7 @@ void intel_modeset_init(struct drm_device *dev)
> 		      INTEL_INFO(dev)->num_pipes,
> 		      INTEL_INFO(dev)->num_pipes > 1 ? "s" : "");
> 
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		intel_crtc_init(dev, pipe);
> 		for_each_sprite(pipe, sprite) {
> 			ret = intel_plane_init(dev, pipe, sprite);
>@@ -12635,10 +12989,11 @@ void intel_modeset_init(struct drm_device *dev)
> 	}
> 
> 	intel_init_dpio(dev);
>-	intel_reset_dpio(dev);
> 
> 	intel_shared_dpll_init(dev);
> 
>+	/* save the BIOS value before clobbering it */
>+	dev_priv->bios_vgacntr = I915_READ(i915_vgacntrl_reg(dev));
> 	/* Just disable it once at startup */
> 	i915_disable_vga(dev);
> 	intel_setup_outputs(dev);
>@@ -12730,9 +13085,10 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
> 	I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
> 
> 	/* restore vblank interrupts to correct state */
>-	if (crtc->active)
>+	if (crtc->active) {
>+		update_scanline_offset(crtc);
> 		drm_vblank_on(dev, crtc->pipe);
>-	else
>+	} else
> 		drm_vblank_off(dev, crtc->pipe);
> 
> 	/* We need to sanitize the plane -> pipe mapping first because this will
>@@ -12815,7 +13171,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
> 		}
> 	}
> 
>-	if (crtc->active || IS_VALLEYVIEW(dev) || INTEL_INFO(dev)->gen < 5) {
>+	if (crtc->active || HAS_GMCH_DISPLAY(dev)) {
> 		/*
> 		 * We start out with underrun reporting disabled to avoid races.
> 		 * For correct bookkeeping mark this on active crtcs.
>@@ -12831,8 +13187,6 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
> 		 */
> 		crtc->cpu_fifo_underrun_disabled = true;
> 		crtc->pch_fifo_underrun_disabled = true;
>-
>-		update_scanline_offset(crtc);
> 	}
> }
> 
>@@ -12964,8 +13318,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
> 			intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
> 	}
> 
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>-			    base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		pipe = 0;
> 
> 		if (encoder->get_hw_state(encoder, &pipe)) {
>@@ -13029,12 +13382,11 @@ void intel_modeset_setup_hw_state(struct drm_device *dev,
> 	}
> 
> 	/* HW state is read out, now we need to sanitize this mess. */
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>-			    base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		intel_sanitize_encoder(encoder);
> 	}
> 
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
> 		intel_sanitize_crtc(crtc);
> 		intel_dump_pipe_config(crtc, &crtc->config, "[setup_hw_state]");
>@@ -13062,7 +13414,7 @@ void intel_modeset_setup_hw_state(struct drm_device *dev,
> 		 * We need to use raw interfaces for restoring state to avoid
> 		 * checking (bogus) intermediate states.
> 		 */
>-		for_each_pipe(pipe) {
>+		for_each_pipe(dev_priv, pipe) {
> 			struct drm_crtc *crtc =
> 				dev_priv->pipe_to_crtc_mapping[pipe];
> 
>@@ -13283,7 +13635,7 @@ intel_display_capture_error_state(struct drm_device *dev)
> 	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
> 		error->power_well_driver = I915_READ(HSW_PWR_WELL_DRIVER);
> 
>-	for_each_pipe(i) {
>+	for_each_pipe(dev_priv, i) {
> 		error->pipe[i].power_domain_on =
> 			intel_display_power_enabled_unlocked(dev_priv,
> 							   POWER_DOMAIN_PIPE(i));
>@@ -13347,6 +13699,7 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
> 				struct drm_device *dev,
> 				struct intel_display_error_state *error)
> {
>+	struct drm_i915_private *dev_priv = dev->dev_private;
> 	int i;
> 
> 	if (!error)
>@@ -13356,7 +13709,7 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
> 	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
> 		err_printf(m, "PWR_WELL_CTL2: %08x\n",
> 			   error->power_well_driver);
>-	for_each_pipe(i) {
>+	for_each_pipe(dev_priv, i) {
> 		err_printf(m, "Pipe [%d]:\n", i);
> 		err_printf(m, "  Power: %s\n",
> 			   error->pipe[i].power_domain_on ? "on" : "off");
>@@ -13397,3 +13750,24 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
> 		err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
> 	}
> }
>+
>+void intel_modeset_preclose(struct drm_device *dev, struct drm_file *file)
>+{
>+	struct intel_crtc *crtc;
>+
>+	for_each_intel_crtc(dev, crtc) {
>+		struct intel_unpin_work *work;
>+
>+		spin_lock_irq(&dev->event_lock);
>+
>+		work = crtc->unpin_work;
>+
>+		if (work && work->event &&
>+		    work->event->base.file_priv == file) {
>+			kfree(work->event);
>+			work->event = NULL;
>+		}
>+
>+		spin_unlock_irq(&dev->event_lock);
>+	}
>+}
>diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
>index fdff1d4..0994690 100644
>--- a/drivers/gpu/drm/i915/intel_dp.c
>+++ b/drivers/gpu/drm/i915/intel_dp.c
>@@ -111,7 +111,7 @@ static struct intel_dp *intel_attached_dp(struct drm_connector *connector)
> }
> 
> static void intel_dp_link_down(struct intel_dp *intel_dp);
>-static bool _edp_panel_vdd_on(struct intel_dp *intel_dp);
>+static bool edp_panel_vdd_on(struct intel_dp *intel_dp);
> static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync);
> 
> int
>@@ -290,32 +290,201 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
> 					      struct intel_dp *intel_dp,
> 					      struct edp_power_seq *out);
> 
>+static void pps_lock(struct intel_dp *intel_dp)
>+{
>+	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
>+	struct intel_encoder *encoder = &intel_dig_port->base;
>+	struct drm_device *dev = encoder->base.dev;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	enum intel_display_power_domain power_domain;
>+
>+	/*
>+	 * See vlv_power_sequencer_reset() why we need
>+	 * a power domain reference here.
>+	 */
>+	power_domain = intel_display_port_power_domain(encoder);
>+	intel_display_power_get(dev_priv, power_domain);
>+
>+	mutex_lock(&dev_priv->pps_mutex);
>+}
>+
>+static void pps_unlock(struct intel_dp *intel_dp)
>+{
>+	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
>+	struct intel_encoder *encoder = &intel_dig_port->base;
>+	struct drm_device *dev = encoder->base.dev;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	enum intel_display_power_domain power_domain;
>+
>+	mutex_unlock(&dev_priv->pps_mutex);
>+
>+	power_domain = intel_display_port_power_domain(encoder);
>+	intel_display_power_put(dev_priv, power_domain);
>+}
>+
> static enum pipe
> vlv_power_sequencer_pipe(struct intel_dp *intel_dp)
> {
> 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
>-	struct drm_crtc *crtc = intel_dig_port->base.base.crtc;
> 	struct drm_device *dev = intel_dig_port->base.base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	enum port port = intel_dig_port->port;
>-	enum pipe pipe;
>+	struct intel_encoder *encoder;
>+	unsigned int pipes = (1 << PIPE_A) | (1 << PIPE_B);
>+	struct edp_power_seq power_seq;
>+
>+	lockdep_assert_held(&dev_priv->pps_mutex);
> 
>-	/* modeset should have pipe */
>-	if (crtc)
>-		return to_intel_crtc(crtc)->pipe;
>+	if (intel_dp->pps_pipe != INVALID_PIPE)
>+		return intel_dp->pps_pipe;
>+
>+	/*
>+	 * We don't have power sequencer currently.
>+	 * Pick one that's not used by other ports.
>+	 */
>+	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>+			    base.head) {
>+		struct intel_dp *tmp;
>+
>+		if (encoder->type != INTEL_OUTPUT_EDP)
>+			continue;
>+
>+		tmp = enc_to_intel_dp(&encoder->base);
>+
>+		if (tmp->pps_pipe != INVALID_PIPE)
>+			pipes &= ~(1 << tmp->pps_pipe);
>+	}
>+
>+	/*
>+	 * Didn't find one. This should not happen since there
>+	 * are two power sequencers and up to two eDP ports.
>+	 */
>+	if (WARN_ON(pipes == 0))
>+		return PIPE_A;
>+
>+	intel_dp->pps_pipe = ffs(pipes) - 1;
>+
>+	DRM_DEBUG_KMS("picked pipe %c power sequencer for port %c\n",
>+		      pipe_name(intel_dp->pps_pipe),
>+		      port_name(intel_dig_port->port));
>+
>+	/* init power sequencer on this pipe and port */
>+	intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
>+	intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
>+						      &power_seq);
>+
>+	return intel_dp->pps_pipe;
>+}
>+
>+typedef bool (*vlv_pipe_check)(struct drm_i915_private *dev_priv,
>+			       enum pipe pipe);
>+
>+static bool vlv_pipe_has_pp_on(struct drm_i915_private *dev_priv,
>+			       enum pipe pipe)
>+{
>+	return I915_READ(VLV_PIPE_PP_STATUS(pipe)) & PP_ON;
>+}
>+
>+static bool vlv_pipe_has_vdd_on(struct drm_i915_private *dev_priv,
>+				enum pipe pipe)
>+{
>+	return I915_READ(VLV_PIPE_PP_CONTROL(pipe)) & EDP_FORCE_VDD;
>+}
>+
>+static bool vlv_pipe_any(struct drm_i915_private *dev_priv,
>+			 enum pipe pipe)
>+{
>+	return true;
>+}
>+
>+static enum pipe
>+vlv_initial_pps_pipe(struct drm_i915_private *dev_priv,
>+		     enum port port,
>+		     vlv_pipe_check pipe_check)
>+{
>+	enum pipe pipe;
> 
>-	/* init time, try to find a pipe with this port selected */
> 	for (pipe = PIPE_A; pipe <= PIPE_B; pipe++) {
> 		u32 port_sel = I915_READ(VLV_PIPE_PP_ON_DELAYS(pipe)) &
> 			PANEL_PORT_SELECT_MASK;
>-		if (port_sel == PANEL_PORT_SELECT_DPB_VLV && port == PORT_B)
>-			return pipe;
>-		if (port_sel == PANEL_PORT_SELECT_DPC_VLV && port == PORT_C)
>-			return pipe;
>+
>+		if (port_sel != PANEL_PORT_SELECT_VLV(port))
>+			continue;
>+
>+		if (!pipe_check(dev_priv, pipe))
>+			continue;
>+
>+		return pipe;
> 	}
> 
>-	/* shrug */
>-	return PIPE_A;
>+	return INVALID_PIPE;
>+}
>+
>+static void
>+vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp)
>+{
>+	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
>+	struct drm_device *dev = intel_dig_port->base.base.dev;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct edp_power_seq power_seq;
>+	enum port port = intel_dig_port->port;
>+
>+	lockdep_assert_held(&dev_priv->pps_mutex);
>+
>+	/* try to find a pipe with this port selected */
>+	/* first pick one where the panel is on */
>+	intel_dp->pps_pipe = vlv_initial_pps_pipe(dev_priv, port,
>+						  vlv_pipe_has_pp_on);
>+	/* didn't find one? pick one where vdd is on */
>+	if (intel_dp->pps_pipe == INVALID_PIPE)
>+		intel_dp->pps_pipe = vlv_initial_pps_pipe(dev_priv, port,
>+							  vlv_pipe_has_vdd_on);
>+	/* didn't find one? pick one with just the correct port */
>+	if (intel_dp->pps_pipe == INVALID_PIPE)
>+		intel_dp->pps_pipe = vlv_initial_pps_pipe(dev_priv, port,
>+							  vlv_pipe_any);
>+
>+	/* didn't find one? just let vlv_power_sequencer_pipe() pick one when needed */
>+	if (intel_dp->pps_pipe == INVALID_PIPE) {
>+		DRM_DEBUG_KMS("no initial power sequencer for port %c\n",
>+			      port_name(port));
>+		return;
>+	}
>+
>+	DRM_DEBUG_KMS("initial power sequencer for port %c: pipe %c\n",
>+		      port_name(port), pipe_name(intel_dp->pps_pipe));
>+
>+	intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
>+	intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
>+						      &power_seq);
>+}
>+
>+void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv)
>+{
>+	struct drm_device *dev = dev_priv->dev;
>+	struct intel_encoder *encoder;
>+
>+	if (WARN_ON(!IS_VALLEYVIEW(dev)))
>+		return;
>+
>+	/*
>+	 * We can't grab pps_mutex here due to deadlock with power_domain
>+	 * mutex when power_domain functions are called while holding pps_mutex.
>+	 * That also means that in order to use pps_pipe the code needs to
>+	 * hold both a power domain reference and pps_mutex, and the power domain
>+	 * reference get/put must be done while _not_ holding pps_mutex.
>+	 * pps_{lock,unlock}() do these steps in the correct order, so one
>+	 * should use them always.
>+	 */
>+
>+	list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
>+		struct intel_dp *intel_dp;
>+
>+		if (encoder->type != INTEL_OUTPUT_EDP)
>+			continue;
>+
>+		intel_dp = enc_to_intel_dp(&encoder->base);
>+		intel_dp->pps_pipe = INVALID_PIPE;
>+	}
> }
> 
> static u32 _pp_ctrl_reg(struct intel_dp *intel_dp)
>@@ -349,12 +518,15 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	u32 pp_div;
> 	u32 pp_ctrl_reg, pp_div_reg;
>-	enum pipe pipe = vlv_power_sequencer_pipe(intel_dp);
> 
> 	if (!is_edp(intel_dp) || code != SYS_RESTART)
> 		return 0;
> 
>+	pps_lock(intel_dp);
>+
> 	if (IS_VALLEYVIEW(dev)) {
>+		enum pipe pipe = vlv_power_sequencer_pipe(intel_dp);
>+
> 		pp_ctrl_reg = VLV_PIPE_PP_CONTROL(pipe);
> 		pp_div_reg  = VLV_PIPE_PP_DIVISOR(pipe);
> 		pp_div = I915_READ(pp_div_reg);
>@@ -366,6 +538,8 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
> 		msleep(intel_dp->panel_power_cycle_delay);
> 	}
> 
>+	pps_unlock(intel_dp);
>+
> 	return 0;
> }
> 
>@@ -374,6 +548,8 @@ static bool edp_have_panel_power(struct intel_dp *intel_dp)
> 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 
>+	lockdep_assert_held(&dev_priv->pps_mutex);
>+
> 	return (I915_READ(_pp_stat_reg(intel_dp)) & PP_ON) != 0;
> }
> 
>@@ -381,13 +557,10 @@ static bool edp_have_panel_vdd(struct intel_dp *intel_dp)
> {
> 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
>-	struct intel_encoder *intel_encoder = &intel_dig_port->base;
>-	enum intel_display_power_domain power_domain;
> 
>-	power_domain = intel_display_port_power_domain(intel_encoder);
>-	return intel_display_power_enabled(dev_priv, power_domain) &&
>-	       (I915_READ(_pp_ctrl_reg(intel_dp)) & EDP_FORCE_VDD) != 0;
>+	lockdep_assert_held(&dev_priv->pps_mutex);
>+
>+	return I915_READ(_pp_ctrl_reg(intel_dp)) & EDP_FORCE_VDD;
> }
> 
> static void
>@@ -535,7 +708,15 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
> 	bool has_aux_irq = HAS_AUX_IRQ(dev);
> 	bool vdd;
> 
>-	vdd = _edp_panel_vdd_on(intel_dp);
>+	pps_lock(intel_dp);
>+
>+	/*
>+	 * We will be called with VDD already enabled for dpcd/edid/oui reads.
>+	 * In such cases we want to leave VDD enabled and it's up to upper layers
>+	 * to turn it off. But for eg. i2c-dev access we need to turn it on/off
>+	 * ourselves.
>+	 */
>+	vdd = edp_panel_vdd_on(intel_dp);
> 
> 	/* dp aux is extremely sensitive to irq latency, hence request the
> 	 * lowest possible wakeup latency and so prevent the cpu from going into
>@@ -644,6 +825,8 @@ out:
> 	if (vdd)
> 		edp_panel_vdd_off(intel_dp, false);
> 
>+	pps_unlock(intel_dp);
>+
> 	return ret;
> }
> 
>@@ -828,20 +1011,6 @@ intel_dp_set_clock(struct intel_encoder *encoder,
> 	}
> }
> 
>-static void
>-intel_dp_set_m2_n2(struct intel_crtc *crtc, struct intel_link_m_n *m_n)
>-{
>-	struct drm_device *dev = crtc->base.dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	enum transcoder transcoder = crtc->config.cpu_transcoder;
>-
>-	I915_WRITE(PIPE_DATA_M2(transcoder),
>-		TU_SIZE(m_n->tu) | m_n->gmch_m);
>-	I915_WRITE(PIPE_DATA_N2(transcoder), m_n->gmch_n);
>-	I915_WRITE(PIPE_LINK_M2(transcoder), m_n->link_m);
>-	I915_WRITE(PIPE_LINK_N2(transcoder), m_n->link_n);
>-}
>-
> bool
> intel_dp_compute_config(struct intel_encoder *encoder,
> 			struct intel_crtc_config *pipe_config)
>@@ -867,6 +1036,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
> 		pipe_config->has_pch_encoder = true;
> 
> 	pipe_config->has_dp_encoder = true;
>+	pipe_config->has_drrs = false;
> 	pipe_config->has_audio = intel_dp->has_audio;
> 
> 	if (is_edp(intel_dp) && intel_connector->panel.fixed_mode) {
>@@ -898,23 +1068,15 @@ intel_dp_compute_config(struct intel_encoder *encoder,
> 			bpp = dev_priv->vbt.edp_bpp;
> 		}
> 
>-		if (IS_BROADWELL(dev)) {
>-			/* Yes, it's an ugly hack. */
>-			min_lane_count = max_lane_count;
>-			DRM_DEBUG_KMS("forcing lane count to max (%u) on BDW\n",
>-				      min_lane_count);
>-		} else if (dev_priv->vbt.edp_lanes) {
>-			min_lane_count = min(dev_priv->vbt.edp_lanes,
>-					     max_lane_count);
>-			DRM_DEBUG_KMS("using min %u lanes per VBT\n",
>-				      min_lane_count);
>-		}
>-
>-		if (dev_priv->vbt.edp_rate) {
>-			min_clock = min(dev_priv->vbt.edp_rate >> 3, max_clock);
>-			DRM_DEBUG_KMS("using min %02x link bw per VBT\n",
>-				      bws[min_clock]);
>-		}
>+		/*
>+		 * Use the maximum clock and number of lanes the eDP panel
>+		 * advertizes being capable of. The panels are generally
>+		 * designed to support only a single clock and lane
>+		 * configuration, and typically these values correspond to the
>+		 * native resolution of the panel.
>+		 */
>+		min_lane_count = max_lane_count;
>+		min_clock = max_clock;
> 	}
> 
> 	for (; bpp >= 6*3; bpp -= 2*3) {
>@@ -970,13 +1132,14 @@ found:
> 
> 	if (intel_connector->panel.downclock_mode != NULL &&
> 		intel_dp->drrs_state.type == SEAMLESS_DRRS_SUPPORT) {
>+			pipe_config->has_drrs = true;
> 			intel_link_compute_m_n(bpp, lane_count,
> 				intel_connector->panel.downclock_mode->clock,
> 				pipe_config->port_clock,
> 				&pipe_config->dp_m2_n2);
> 	}
> 
>-	if (HAS_DDI(dev))
>+	if (IS_HASWELL(dev) || IS_BROADWELL(dev))
> 		hsw_dp_set_ddi_pll_sel(pipe_config, intel_dp->link_bw);
> 	else
> 		intel_dp_set_clock(encoder, pipe_config, intel_dp->link_bw);
>@@ -1110,6 +1273,8 @@ static void wait_panel_status(struct intel_dp *intel_dp,
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	u32 pp_stat_reg, pp_ctrl_reg;
> 
>+	lockdep_assert_held(&dev_priv->pps_mutex);
>+
> 	pp_stat_reg = _pp_stat_reg(intel_dp);
> 	pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
> 
>@@ -1173,13 +1338,20 @@ static  u32 ironlake_get_pp_control(struct intel_dp *intel_dp)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	u32 control;
> 
>+	lockdep_assert_held(&dev_priv->pps_mutex);
>+
> 	control = I915_READ(_pp_ctrl_reg(intel_dp));
> 	control &= ~PANEL_UNLOCK_MASK;
> 	control |= PANEL_UNLOCK_REGS;
> 	return control;
> }
> 
>-static bool _edp_panel_vdd_on(struct intel_dp *intel_dp)
>+/*
>+ * Must be paired with edp_panel_vdd_off().
>+ * Must hold pps_mutex around the whole on/off sequence.
>+ * Can be nested with intel_edp_panel_vdd_{on,off}() calls.
>+ */
>+static bool edp_panel_vdd_on(struct intel_dp *intel_dp)
> {
> 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
> 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
>@@ -1190,6 +1362,8 @@ static bool _edp_panel_vdd_on(struct intel_dp *intel_dp)
> 	u32 pp_stat_reg, pp_ctrl_reg;
> 	bool need_to_disable = !intel_dp->want_panel_vdd;
> 
>+	lockdep_assert_held(&dev_priv->pps_mutex);
>+
> 	if (!is_edp(intel_dp))
> 		return false;
> 
>@@ -1227,62 +1401,76 @@ static bool _edp_panel_vdd_on(struct intel_dp *intel_dp)
> 	return need_to_disable;
> }
> 
>+/*
>+ * Must be paired with intel_edp_panel_vdd_off() or
>+ * intel_edp_panel_off().
>+ * Nested calls to these functions are not allowed since
>+ * we drop the lock. Caller must use some higher level
>+ * locking to prevent nested calls from other threads.
>+ */
> void intel_edp_panel_vdd_on(struct intel_dp *intel_dp)
> {
>-	if (is_edp(intel_dp)) {
>-		bool vdd = _edp_panel_vdd_on(intel_dp);
>+	bool vdd;
> 
>-		WARN(!vdd, "eDP VDD already requested on\n");
>-	}
>+	if (!is_edp(intel_dp))
>+		return;
>+
>+	pps_lock(intel_dp);
>+	vdd = edp_panel_vdd_on(intel_dp);
>+	pps_unlock(intel_dp);
>+
>+	WARN(!vdd, "eDP VDD already requested on\n");
> }
> 
> static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp)
> {
> 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct intel_digital_port *intel_dig_port =
>+		dp_to_dig_port(intel_dp);
>+	struct intel_encoder *intel_encoder = &intel_dig_port->base;
>+	enum intel_display_power_domain power_domain;
> 	u32 pp;
> 	u32 pp_stat_reg, pp_ctrl_reg;
> 
>-	WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
>+	lockdep_assert_held(&dev_priv->pps_mutex);
> 
>-	if (!intel_dp->want_panel_vdd && edp_have_panel_vdd(intel_dp)) {
>-		struct intel_digital_port *intel_dig_port =
>-						dp_to_dig_port(intel_dp);
>-		struct intel_encoder *intel_encoder = &intel_dig_port->base;
>-		enum intel_display_power_domain power_domain;
>+	WARN_ON(intel_dp->want_panel_vdd);
> 
>-		DRM_DEBUG_KMS("Turning eDP VDD off\n");
>+	if (!edp_have_panel_vdd(intel_dp))
>+		return;
> 
>-		pp = ironlake_get_pp_control(intel_dp);
>-		pp &= ~EDP_FORCE_VDD;
>+	DRM_DEBUG_KMS("Turning eDP VDD off\n");
> 
>-		pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
>-		pp_stat_reg = _pp_stat_reg(intel_dp);
>+	pp = ironlake_get_pp_control(intel_dp);
>+	pp &= ~EDP_FORCE_VDD;
> 
>-		I915_WRITE(pp_ctrl_reg, pp);
>-		POSTING_READ(pp_ctrl_reg);
>+	pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
>+	pp_stat_reg = _pp_stat_reg(intel_dp);
> 
>-		/* Make sure sequencer is idle before allowing subsequent activity */
>-		DRM_DEBUG_KMS("PP_STATUS: 0x%08x PP_CONTROL: 0x%08x\n",
>-		I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg));
>+	I915_WRITE(pp_ctrl_reg, pp);
>+	POSTING_READ(pp_ctrl_reg);
> 
>-		if ((pp & POWER_TARGET_ON) == 0)
>-			intel_dp->last_power_cycle = jiffies;
>+	/* Make sure sequencer is idle before allowing subsequent activity */
>+	DRM_DEBUG_KMS("PP_STATUS: 0x%08x PP_CONTROL: 0x%08x\n",
>+	I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg));
> 
>-		power_domain = intel_display_port_power_domain(intel_encoder);
>-		intel_display_power_put(dev_priv, power_domain);
>-	}
>+	if ((pp & POWER_TARGET_ON) == 0)
>+		intel_dp->last_power_cycle = jiffies;
>+
>+	power_domain = intel_display_port_power_domain(intel_encoder);
>+	intel_display_power_put(dev_priv, power_domain);
> }
> 
> static void edp_panel_vdd_work(struct work_struct *__work)
> {
> 	struct intel_dp *intel_dp = container_of(to_delayed_work(__work),
> 						 struct intel_dp, panel_vdd_work);
>-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
> 
>-	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
>-	edp_panel_vdd_off_sync(intel_dp);
>-	drm_modeset_unlock(&dev->mode_config.connection_mutex);
>+	pps_lock(intel_dp);
>+	if (!intel_dp->want_panel_vdd)
>+		edp_panel_vdd_off_sync(intel_dp);
>+	pps_unlock(intel_dp);
> }
> 
> static void edp_panel_vdd_schedule_off(struct intel_dp *intel_dp)
>@@ -1298,8 +1486,18 @@ static void edp_panel_vdd_schedule_off(struct intel_dp *intel_dp)
> 	schedule_delayed_work(&intel_dp->panel_vdd_work, delay);
> }
> 
>+/*
>+ * Must be paired with edp_panel_vdd_on().
>+ * Must hold pps_mutex around the whole on/off sequence.
>+ * Can be nested with intel_edp_panel_vdd_{on,off}() calls.
>+ */
> static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
> {
>+	struct drm_i915_private *dev_priv =
>+		intel_dp_to_dev(intel_dp)->dev_private;
>+
>+	lockdep_assert_held(&dev_priv->pps_mutex);
>+
> 	if (!is_edp(intel_dp))
> 		return;
> 
>@@ -1313,6 +1511,22 @@ static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
> 		edp_panel_vdd_schedule_off(intel_dp);
> }
> 
>+/*
>+ * Must be paired with intel_edp_panel_vdd_on().
>+ * Nested calls to these functions are not allowed since
>+ * we drop the lock. Caller must use some higher level
>+ * locking to prevent nested calls from other threads.
>+ */
>+static void intel_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
>+{
>+	if (!is_edp(intel_dp))
>+		return;
>+
>+	pps_lock(intel_dp);
>+	edp_panel_vdd_off(intel_dp, sync);
>+	pps_unlock(intel_dp);
>+}
>+
> void intel_edp_panel_on(struct intel_dp *intel_dp)
> {
> 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
>@@ -1325,9 +1539,11 @@ void intel_edp_panel_on(struct intel_dp *intel_dp)
> 
> 	DRM_DEBUG_KMS("Turn eDP power on\n");
> 
>+	pps_lock(intel_dp);
>+
> 	if (edp_have_panel_power(intel_dp)) {
> 		DRM_DEBUG_KMS("eDP power already on\n");
>-		return;
>+		goto out;
> 	}
> 
> 	wait_panel_power_cycle(intel_dp);
>@@ -1356,6 +1572,9 @@ void intel_edp_panel_on(struct intel_dp *intel_dp)
> 		I915_WRITE(pp_ctrl_reg, pp);
> 		POSTING_READ(pp_ctrl_reg);
> 	}
>+
>+ out:
>+	pps_unlock(intel_dp);
> }
> 
> void intel_edp_panel_off(struct intel_dp *intel_dp)
>@@ -1373,6 +1592,8 @@ void intel_edp_panel_off(struct intel_dp *intel_dp)
> 
> 	DRM_DEBUG_KMS("Turn eDP power off\n");
> 
>+	pps_lock(intel_dp);
>+
> 	WARN(!intel_dp->want_panel_vdd, "Need VDD to turn off panel\n");
> 
> 	pp = ironlake_get_pp_control(intel_dp);
>@@ -1394,9 +1615,12 @@ void intel_edp_panel_off(struct intel_dp *intel_dp)
> 	/* We got a reference when we enabled the VDD. */
> 	power_domain = intel_display_port_power_domain(intel_encoder);
> 	intel_display_power_put(dev_priv, power_domain);
>+
>+	pps_unlock(intel_dp);
> }
> 
>-void intel_edp_backlight_on(struct intel_dp *intel_dp)
>+/* Enable backlight in the panel power control. */
>+static void _intel_edp_backlight_on(struct intel_dp *intel_dp)
> {
> 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
> 	struct drm_device *dev = intel_dig_port->base.base.dev;
>@@ -1404,13 +1628,6 @@ void intel_edp_backlight_on(struct intel_dp *intel_dp)
> 	u32 pp;
> 	u32 pp_ctrl_reg;
> 
>-	if (!is_edp(intel_dp))
>-		return;
>-
>-	DRM_DEBUG_KMS("\n");
>-
>-	intel_panel_enable_backlight(intel_dp->attached_connector);
>-
> 	/*
> 	 * If we enable the backlight right away following a panel power
> 	 * on, we may see slight flicker as the panel syncs with the eDP
>@@ -1418,6 +1635,9 @@ void intel_edp_backlight_on(struct intel_dp *intel_dp)
> 	 * allowing it to appear.
> 	 */
> 	wait_backlight_on(intel_dp);
>+
>+	pps_lock(intel_dp);
>+
> 	pp = ironlake_get_pp_control(intel_dp);
> 	pp |= EDP_BLC_ENABLE;
> 
>@@ -1425,9 +1645,24 @@ void intel_edp_backlight_on(struct intel_dp *intel_dp)
> 
> 	I915_WRITE(pp_ctrl_reg, pp);
> 	POSTING_READ(pp_ctrl_reg);
>+
>+	pps_unlock(intel_dp);
> }
> 
>-void intel_edp_backlight_off(struct intel_dp *intel_dp)
>+/* Enable backlight PWM and backlight PP control. */
>+void intel_edp_backlight_on(struct intel_dp *intel_dp)
>+{
>+	if (!is_edp(intel_dp))
>+		return;
>+
>+	DRM_DEBUG_KMS("\n");
>+
>+	intel_panel_enable_backlight(intel_dp->attached_connector);
>+	_intel_edp_backlight_on(intel_dp);
>+}
>+
>+/* Disable backlight in the panel power control. */
>+static void _intel_edp_backlight_off(struct intel_dp *intel_dp)
> {
> 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>@@ -1437,7 +1672,8 @@ void intel_edp_backlight_off(struct intel_dp *intel_dp)
> 	if (!is_edp(intel_dp))
> 		return;
> 
>-	DRM_DEBUG_KMS("\n");
>+	pps_lock(intel_dp);
>+
> 	pp = ironlake_get_pp_control(intel_dp);
> 	pp &= ~EDP_BLC_ENABLE;
> 
>@@ -1445,13 +1681,51 @@ void intel_edp_backlight_off(struct intel_dp *intel_dp)
> 
> 	I915_WRITE(pp_ctrl_reg, pp);
> 	POSTING_READ(pp_ctrl_reg);
>-	intel_dp->last_backlight_off = jiffies;
> 
>+	pps_unlock(intel_dp);
>+
>+	intel_dp->last_backlight_off = jiffies;
> 	edp_wait_backlight_off(intel_dp);
>+}
> 
>+/* Disable backlight PP control and backlight PWM. */
>+void intel_edp_backlight_off(struct intel_dp *intel_dp)
>+{
>+	if (!is_edp(intel_dp))
>+		return;
>+
>+	DRM_DEBUG_KMS("\n");
>+
>+	_intel_edp_backlight_off(intel_dp);
> 	intel_panel_disable_backlight(intel_dp->attached_connector);
> }
> 
>+/*
>+ * Hook for controlling the panel power control backlight through the bl_power
>+ * sysfs attribute. Take care to handle multiple calls.
>+ */
>+static void intel_edp_backlight_power(struct intel_connector *connector,
>+				      bool enable)
>+{
>+	struct intel_dp *intel_dp = intel_attached_dp(&connector->base);
>+	bool is_enabled;
>+
>+	pps_lock(intel_dp);
>+	is_enabled = ironlake_get_pp_control(intel_dp) & EDP_BLC_ENABLE;
>+	pps_unlock(intel_dp);
>+
>+	if (is_enabled == enable)
>+		return;
>+
>+	DRM_DEBUG_KMS("panel power control backlight %s\n",
>+		      enable ? "enable" : "disable");
>+
>+	if (enable)
>+		_intel_edp_backlight_on(intel_dp);
>+	else
>+		_intel_edp_backlight_off(intel_dp);
>+}
>+
> static void ironlake_edp_pll_on(struct intel_dp *intel_dp)
> {
> 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
>@@ -1515,8 +1789,6 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
> 	if (mode != DRM_MODE_DPMS_ON) {
> 		ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER,
> 					 DP_SET_POWER_D3);
>-		if (ret != 1)
>-			DRM_DEBUG_DRIVER("failed to write sink power state\n");
> 	} else {
> 		/*
> 		 * When turning on, we need to retry for 1ms to give the sink
>@@ -1530,6 +1802,10 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
> 			msleep(1);
> 		}
> 	}
>+
>+	if (ret != 1)
>+		DRM_DEBUG_KMS("failed to %s sink power state\n",
>+			      mode == DRM_MODE_DPMS_ON ? "enable" : "disable");
> }
> 
> static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
>@@ -1576,7 +1852,7 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
> 			return true;
> 		}
> 
>-		for_each_pipe(i) {
>+		for_each_pipe(dev_priv, i) {
> 			trans_dp = I915_READ(TRANS_DP_CTL(i));
> 			if ((trans_dp & TRANS_DP_PORT_SEL_MASK) == trans_sel) {
> 				*pipe = i;
>@@ -2036,7 +2312,6 @@ void intel_edp_psr_init(struct drm_device *dev)
> static void intel_disable_dp(struct intel_encoder *encoder)
> {
> 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
>-	enum port port = dp_to_dig_port(intel_dp)->port;
> 	struct drm_device *dev = encoder->base.dev;
> 
> 	/* Make sure the panel is off before trying to change the mode. But also
>@@ -2046,21 +2321,19 @@ static void intel_disable_dp(struct intel_encoder *encoder)
> 	intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
> 	intel_edp_panel_off(intel_dp);
> 
>-	/* cpu edp my only be disable _after_ the cpu pipe/plane is disabled. */
>-	if (!(port == PORT_A || IS_VALLEYVIEW(dev)))
>+	/* disable the port before the pipe on g4x */
>+	if (INTEL_INFO(dev)->gen < 5)
> 		intel_dp_link_down(intel_dp);
> }
> 
>-static void g4x_post_disable_dp(struct intel_encoder *encoder)
>+static void ilk_post_disable_dp(struct intel_encoder *encoder)
> {
> 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
> 	enum port port = dp_to_dig_port(intel_dp)->port;
> 
>-	if (port != PORT_A)
>-		return;
>-
> 	intel_dp_link_down(intel_dp);
>-	ironlake_edp_pll_off(intel_dp);
>+	if (port == PORT_A)
>+		ironlake_edp_pll_off(intel_dp);
> }
> 
> static void vlv_post_disable_dp(struct intel_encoder *encoder)
>@@ -2103,7 +2376,105 @@ static void chv_post_disable_dp(struct intel_encoder *encoder)
> 	val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
> 	vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val);
> 
>-	mutex_unlock(&dev_priv->dpio_lock);
>+	mutex_unlock(&dev_priv->dpio_lock);
>+}
>+
>+static void
>+_intel_dp_set_link_train(struct intel_dp *intel_dp,
>+			 uint32_t *DP,
>+			 uint8_t dp_train_pat)
>+{
>+	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
>+	struct drm_device *dev = intel_dig_port->base.base.dev;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	enum port port = intel_dig_port->port;
>+
>+	if (HAS_DDI(dev)) {
>+		uint32_t temp = I915_READ(DP_TP_CTL(port));
>+
>+		if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE)
>+			temp |= DP_TP_CTL_SCRAMBLE_DISABLE;
>+		else
>+			temp &= ~DP_TP_CTL_SCRAMBLE_DISABLE;
>+
>+		temp &= ~DP_TP_CTL_LINK_TRAIN_MASK;
>+		switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
>+		case DP_TRAINING_PATTERN_DISABLE:
>+			temp |= DP_TP_CTL_LINK_TRAIN_NORMAL;
>+
>+			break;
>+		case DP_TRAINING_PATTERN_1:
>+			temp |= DP_TP_CTL_LINK_TRAIN_PAT1;
>+			break;
>+		case DP_TRAINING_PATTERN_2:
>+			temp |= DP_TP_CTL_LINK_TRAIN_PAT2;
>+			break;
>+		case DP_TRAINING_PATTERN_3:
>+			temp |= DP_TP_CTL_LINK_TRAIN_PAT3;
>+			break;
>+		}
>+		I915_WRITE(DP_TP_CTL(port), temp);
>+
>+	} else if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || port != PORT_A)) {
>+		*DP &= ~DP_LINK_TRAIN_MASK_CPT;
>+
>+		switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
>+		case DP_TRAINING_PATTERN_DISABLE:
>+			*DP |= DP_LINK_TRAIN_OFF_CPT;
>+			break;
>+		case DP_TRAINING_PATTERN_1:
>+			*DP |= DP_LINK_TRAIN_PAT_1_CPT;
>+			break;
>+		case DP_TRAINING_PATTERN_2:
>+			*DP |= DP_LINK_TRAIN_PAT_2_CPT;
>+			break;
>+		case DP_TRAINING_PATTERN_3:
>+			DRM_ERROR("DP training pattern 3 not supported\n");
>+			*DP |= DP_LINK_TRAIN_PAT_2_CPT;
>+			break;
>+		}
>+
>+	} else {
>+		if (IS_CHERRYVIEW(dev))
>+			*DP &= ~DP_LINK_TRAIN_MASK_CHV;
>+		else
>+			*DP &= ~DP_LINK_TRAIN_MASK;
>+
>+		switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
>+		case DP_TRAINING_PATTERN_DISABLE:
>+			*DP |= DP_LINK_TRAIN_OFF;
>+			break;
>+		case DP_TRAINING_PATTERN_1:
>+			*DP |= DP_LINK_TRAIN_PAT_1;
>+			break;
>+		case DP_TRAINING_PATTERN_2:
>+			*DP |= DP_LINK_TRAIN_PAT_2;
>+			break;
>+		case DP_TRAINING_PATTERN_3:
>+			if (IS_CHERRYVIEW(dev)) {
>+				*DP |= DP_LINK_TRAIN_PAT_3_CHV;
>+			} else {
>+				DRM_ERROR("DP training pattern 3 not supported\n");
>+				*DP |= DP_LINK_TRAIN_PAT_2;
>+			}
>+			break;
>+		}
>+	}
>+}
>+
>+static void intel_dp_enable_port(struct intel_dp *intel_dp)
>+{
>+	struct drm_device *dev = intel_dp_to_dev(intel_dp);
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+
>+	intel_dp->DP |= DP_PORT_EN;
>+
>+	/* enable with pattern 1 (as per spec) */
>+	_intel_dp_set_link_train(intel_dp, &intel_dp->DP,
>+				 DP_TRAINING_PATTERN_1);
>+
>+	I915_WRITE(intel_dp->output_reg, intel_dp->DP);
>+	POSTING_READ(intel_dp->output_reg);
> }
> 
> static void intel_enable_dp(struct intel_encoder *encoder)
>@@ -2116,11 +2487,12 @@ static void intel_enable_dp(struct intel_encoder *encoder)
> 	if (WARN_ON(dp_reg & DP_PORT_EN))
> 		return;
> 
>+	intel_dp_enable_port(intel_dp);
> 	intel_edp_panel_vdd_on(intel_dp);
>+	intel_edp_panel_on(intel_dp);
>+	intel_edp_panel_vdd_off(intel_dp, true);
> 	intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
> 	intel_dp_start_link_train(intel_dp);
>-	intel_edp_panel_on(intel_dp);
>-	edp_panel_vdd_off(intel_dp, true);
> 	intel_dp_complete_link_train(intel_dp);
> 	intel_dp_stop_link_train(intel_dp);
> }
>@@ -2154,6 +2526,78 @@ static void g4x_pre_enable_dp(struct intel_encoder *encoder)
> 	}
> }
> 
>+static void vlv_steal_power_sequencer(struct drm_device *dev,
>+				      enum pipe pipe)
>+{
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct intel_encoder *encoder;
>+
>+	lockdep_assert_held(&dev_priv->pps_mutex);
>+
>+	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>+			    base.head) {
>+		struct intel_dp *intel_dp;
>+		enum port port;
>+
>+		if (encoder->type != INTEL_OUTPUT_EDP)
>+			continue;
>+
>+		intel_dp = enc_to_intel_dp(&encoder->base);
>+		port = dp_to_dig_port(intel_dp)->port;
>+
>+		if (intel_dp->pps_pipe != pipe)
>+			continue;
>+
>+		DRM_DEBUG_KMS("stealing pipe %c power sequencer from port %c\n",
>+			      pipe_name(pipe), port_name(port));
>+
>+		/* make sure vdd is off before we steal it */
>+		edp_panel_vdd_off_sync(intel_dp);
>+
>+		intel_dp->pps_pipe = INVALID_PIPE;
>+	}
>+}
>+
>+static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp)
>+{
>+	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
>+	struct intel_encoder *encoder = &intel_dig_port->base;
>+	struct drm_device *dev = encoder->base.dev;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
>+	struct edp_power_seq power_seq;
>+
>+	lockdep_assert_held(&dev_priv->pps_mutex);
>+
>+	if (intel_dp->pps_pipe == crtc->pipe)
>+		return;
>+
>+	/*
>+	 * If another power sequencer was being used on this
>+	 * port previously make sure to turn off vdd there while
>+	 * we still have control of it.
>+	 */
>+	if (intel_dp->pps_pipe != INVALID_PIPE)
>+		edp_panel_vdd_off_sync(intel_dp);
>+
>+	/*
>+	 * We may be stealing the power
>+	 * sequencer from another port.
>+	 */
>+	vlv_steal_power_sequencer(dev, crtc->pipe);
>+
>+	/* now it's all ours */
>+	intel_dp->pps_pipe = crtc->pipe;
>+
>+	DRM_DEBUG_KMS("initializing pipe %c power sequencer for port %c\n",
>+		      pipe_name(intel_dp->pps_pipe), port_name(intel_dig_port->port));
>+
>+	/* init power sequencer on this pipe and port */
>+	intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
>+	intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
>+						      &power_seq);
>+}
>+
> static void vlv_pre_enable_dp(struct intel_encoder *encoder)
> {
> 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
>@@ -2163,7 +2607,6 @@ static void vlv_pre_enable_dp(struct intel_encoder *encoder)
> 	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
> 	enum dpio_channel port = vlv_dport_to_channel(dport);
> 	int pipe = intel_crtc->pipe;
>-	struct edp_power_seq power_seq;
> 	u32 val;
> 
> 	mutex_lock(&dev_priv->dpio_lock);
>@@ -2182,10 +2625,9 @@ static void vlv_pre_enable_dp(struct intel_encoder *encoder)
> 	mutex_unlock(&dev_priv->dpio_lock);
> 
> 	if (is_edp(intel_dp)) {
>-		/* init power sequencer on this pipe and port */
>-		intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
>-		intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
>-							      &power_seq);
>+		pps_lock(intel_dp);
>+		vlv_init_panel_power_sequencer(intel_dp);
>+		pps_unlock(intel_dp);
> 	}
> 
> 	intel_enable_dp(encoder);
>@@ -2229,7 +2671,6 @@ static void chv_pre_enable_dp(struct intel_encoder *encoder)
> 	struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
> 	struct drm_device *dev = encoder->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct edp_power_seq power_seq;
> 	struct intel_crtc *intel_crtc =
> 		to_intel_crtc(encoder->base.crtc);
> 	enum dpio_channel ch = vlv_dport_to_channel(dport);
>@@ -2275,10 +2716,9 @@ static void chv_pre_enable_dp(struct intel_encoder *encoder)
> 	mutex_unlock(&dev_priv->dpio_lock);
> 
> 	if (is_edp(intel_dp)) {
>-		/* init power sequencer on this pipe and port */
>-		intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
>-		intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
>-							      &power_seq);
>+		pps_lock(intel_dp);
>+		vlv_init_panel_power_sequencer(intel_dp);
>+		pps_unlock(intel_dp);
> 	}
> 
> 	intel_enable_dp(encoder);
>@@ -2297,6 +2737,8 @@ static void chv_dp_pre_pll_enable(struct intel_encoder *encoder)
> 	enum pipe pipe = intel_crtc->pipe;
> 	u32 val;
> 
>+	intel_dp_prepare(encoder);
>+
> 	mutex_lock(&dev_priv->dpio_lock);
> 
> 	/* program left/right clock distribution */
>@@ -2663,8 +3105,8 @@ static uint32_t intel_chv_signal_levels(struct intel_dp *intel_dp)
> 	/* Program swing margin */
> 	for (i = 0; i < 4; i++) {
> 		val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i));
>-		val &= ~DPIO_SWING_MARGIN_MASK;
>-		val |= margin_reg_value << DPIO_SWING_MARGIN_SHIFT;
>+		val &= ~DPIO_SWING_MARGIN000_MASK;
>+		val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT;
> 		vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val);
> 	}
> 
>@@ -2856,29 +3298,29 @@ intel_hsw_signal_levels(uint8_t train_set)
> 					 DP_TRAIN_PRE_EMPHASIS_MASK);
> 	switch (signal_levels) {
> 	case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_0:
>-		return DDI_BUF_EMP_400MV_0DB_HSW;
>+		return DDI_BUF_TRANS_SELECT(0);
> 	case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_3_5:
>-		return DDI_BUF_EMP_400MV_3_5DB_HSW;
>+		return DDI_BUF_TRANS_SELECT(1);
> 	case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_6:
>-		return DDI_BUF_EMP_400MV_6DB_HSW;
>+		return DDI_BUF_TRANS_SELECT(2);
> 	case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_9_5:
>-		return DDI_BUF_EMP_400MV_9_5DB_HSW;
>+		return DDI_BUF_TRANS_SELECT(3);
> 
> 	case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_0:
>-		return DDI_BUF_EMP_600MV_0DB_HSW;
>+		return DDI_BUF_TRANS_SELECT(4);
> 	case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_3_5:
>-		return DDI_BUF_EMP_600MV_3_5DB_HSW;
>+		return DDI_BUF_TRANS_SELECT(5);
> 	case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_6:
>-		return DDI_BUF_EMP_600MV_6DB_HSW;
>+		return DDI_BUF_TRANS_SELECT(6);
> 
> 	case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_0:
>-		return DDI_BUF_EMP_800MV_0DB_HSW;
>+		return DDI_BUF_TRANS_SELECT(7);
> 	case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_3_5:
>-		return DDI_BUF_EMP_800MV_3_5DB_HSW;
>+		return DDI_BUF_TRANS_SELECT(8);
> 	default:
> 		DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:"
> 			      "0x%x\n", signal_levels);
>-		return DDI_BUF_EMP_400MV_0DB_HSW;
>+		return DDI_BUF_TRANS_SELECT(0);
> 	}
> }
> 
>@@ -2925,74 +3367,10 @@ intel_dp_set_link_train(struct intel_dp *intel_dp,
> 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
> 	struct drm_device *dev = intel_dig_port->base.base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	enum port port = intel_dig_port->port;
> 	uint8_t buf[sizeof(intel_dp->train_set) + 1];
> 	int ret, len;
> 
>-	if (HAS_DDI(dev)) {
>-		uint32_t temp = I915_READ(DP_TP_CTL(port));
>-
>-		if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE)
>-			temp |= DP_TP_CTL_SCRAMBLE_DISABLE;
>-		else
>-			temp &= ~DP_TP_CTL_SCRAMBLE_DISABLE;
>-
>-		temp &= ~DP_TP_CTL_LINK_TRAIN_MASK;
>-		switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
>-		case DP_TRAINING_PATTERN_DISABLE:
>-			temp |= DP_TP_CTL_LINK_TRAIN_NORMAL;
>-
>-			break;
>-		case DP_TRAINING_PATTERN_1:
>-			temp |= DP_TP_CTL_LINK_TRAIN_PAT1;
>-			break;
>-		case DP_TRAINING_PATTERN_2:
>-			temp |= DP_TP_CTL_LINK_TRAIN_PAT2;
>-			break;
>-		case DP_TRAINING_PATTERN_3:
>-			temp |= DP_TP_CTL_LINK_TRAIN_PAT3;
>-			break;
>-		}
>-		I915_WRITE(DP_TP_CTL(port), temp);
>-
>-	} else if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || port != PORT_A)) {
>-		*DP &= ~DP_LINK_TRAIN_MASK_CPT;
>-
>-		switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
>-		case DP_TRAINING_PATTERN_DISABLE:
>-			*DP |= DP_LINK_TRAIN_OFF_CPT;
>-			break;
>-		case DP_TRAINING_PATTERN_1:
>-			*DP |= DP_LINK_TRAIN_PAT_1_CPT;
>-			break;
>-		case DP_TRAINING_PATTERN_2:
>-			*DP |= DP_LINK_TRAIN_PAT_2_CPT;
>-			break;
>-		case DP_TRAINING_PATTERN_3:
>-			DRM_ERROR("DP training pattern 3 not supported\n");
>-			*DP |= DP_LINK_TRAIN_PAT_2_CPT;
>-			break;
>-		}
>-
>-	} else {
>-		*DP &= ~DP_LINK_TRAIN_MASK;
>-
>-		switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
>-		case DP_TRAINING_PATTERN_DISABLE:
>-			*DP |= DP_LINK_TRAIN_OFF;
>-			break;
>-		case DP_TRAINING_PATTERN_1:
>-			*DP |= DP_LINK_TRAIN_PAT_1;
>-			break;
>-		case DP_TRAINING_PATTERN_2:
>-			*DP |= DP_LINK_TRAIN_PAT_2;
>-			break;
>-		case DP_TRAINING_PATTERN_3:
>-			DRM_ERROR("DP training pattern 3 not supported\n");
>-			*DP |= DP_LINK_TRAIN_PAT_2;
>-			break;
>-		}
>-	}
>+	_intel_dp_set_link_train(intel_dp, DP, dp_train_pat);
> 
> 	I915_WRITE(intel_dp->output_reg, *DP);
> 	POSTING_READ(intel_dp->output_reg);
>@@ -3276,7 +3654,10 @@ intel_dp_link_down(struct intel_dp *intel_dp)
> 		DP &= ~DP_LINK_TRAIN_MASK_CPT;
> 		I915_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE_CPT);
> 	} else {
>-		DP &= ~DP_LINK_TRAIN_MASK;
>+		if (IS_CHERRYVIEW(dev))
>+			DP &= ~DP_LINK_TRAIN_MASK_CHV;
>+		else
>+			DP &= ~DP_LINK_TRAIN_MASK;
> 		I915_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE);
> 	}
> 	POSTING_READ(intel_dp->output_reg);
>@@ -3322,15 +3703,11 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
> 	struct drm_device *dev = dig_port->base.base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 
>-	char dpcd_hex_dump[sizeof(intel_dp->dpcd) * 3];
>-
> 	if (intel_dp_dpcd_read_wake(&intel_dp->aux, 0x000, intel_dp->dpcd,
> 				    sizeof(intel_dp->dpcd)) < 0)
> 		return false; /* aux transfer failed */
> 
>-	hex_dump_to_buffer(intel_dp->dpcd, sizeof(intel_dp->dpcd),
>-			   32, 1, dpcd_hex_dump, sizeof(dpcd_hex_dump), false);
>-	DRM_DEBUG_KMS("DPCD: %s\n", dpcd_hex_dump);
>+	DRM_DEBUG_KMS("DPCD: %*ph\n", (int) sizeof(intel_dp->dpcd), intel_dp->dpcd);
> 
> 	if (intel_dp->dpcd[DP_DPCD_REV] == 0)
> 		return false; /* DPCD not present */
>@@ -3351,7 +3728,7 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
> 	if (intel_dp->dpcd[DP_DPCD_REV] >= 0x12 &&
> 	    intel_dp->dpcd[DP_MAX_LANE_COUNT] & DP_TPS3_SUPPORTED) {
> 		intel_dp->use_tps3 = true;
>-		DRM_DEBUG_KMS("Displayport TPS3 supported");
>+		DRM_DEBUG_KMS("Displayport TPS3 supported\n");
> 	} else
> 		intel_dp->use_tps3 = false;
> 
>@@ -3388,7 +3765,7 @@ intel_dp_probe_oui(struct intel_dp *intel_dp)
> 		DRM_DEBUG_KMS("Branch OUI: %02hx%02hx%02hx\n",
> 			      buf[0], buf[1], buf[2]);
> 
>-	edp_panel_vdd_off(intel_dp, false);
>+	intel_edp_panel_vdd_off(intel_dp, false);
> }
> 
> static bool
>@@ -3402,7 +3779,7 @@ intel_dp_probe_mst(struct intel_dp *intel_dp)
> 	if (intel_dp->dpcd[DP_DPCD_REV] < 0x12)
> 		return false;
> 
>-	_edp_panel_vdd_on(intel_dp);
>+	intel_edp_panel_vdd_on(intel_dp);
> 	if (intel_dp_dpcd_read_wake(&intel_dp->aux, DP_MSTM_CAP, buf, 1)) {
> 		if (buf[0] & DP_MST_CAP) {
> 			DRM_DEBUG_KMS("Sink is MST capable\n");
>@@ -3412,7 +3789,7 @@ intel_dp_probe_mst(struct intel_dp *intel_dp)
> 			intel_dp->is_mst = false;
> 		}
> 	}
>-	edp_panel_vdd_off(intel_dp, false);
>+	intel_edp_panel_vdd_off(intel_dp, false);
> 
> 	drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, intel_dp->is_mst);
> 	return intel_dp->is_mst;
>@@ -3427,21 +3804,21 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc)
> 	u8 buf[1];
> 
> 	if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, buf) < 0)
>-		return -EAGAIN;
>+		return -EIO;
> 
> 	if (!(buf[0] & DP_TEST_CRC_SUPPORTED))
> 		return -ENOTTY;
> 
> 	if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK,
> 			       DP_TEST_SINK_START) < 0)
>-		return -EAGAIN;
>+		return -EIO;
> 
> 	/* Wait 2 vblanks to be sure we will have the correct CRC value */
> 	intel_wait_for_vblank(dev, intel_crtc->pipe);
> 	intel_wait_for_vblank(dev, intel_crtc->pipe);
> 
> 	if (drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_CRC_R_CR, crc, 6) < 0)
>-		return -EAGAIN;
>+		return -EIO;
> 
> 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK, 0);
> 	return 0;
>@@ -3644,20 +4021,24 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp)
> }
> 
> static enum drm_connector_status
>+edp_detect(struct intel_dp *intel_dp)
>+{
>+	struct drm_device *dev = intel_dp_to_dev(intel_dp);
>+	enum drm_connector_status status;
>+
>+	status = intel_panel_detect(dev);
>+	if (status == connector_status_unknown)
>+		status = connector_status_connected;
>+
>+	return status;
>+}
>+
>+static enum drm_connector_status
> ironlake_dp_detect(struct intel_dp *intel_dp)
> {
> 	struct drm_device *dev = intel_dp_to_dev(intel_dp);
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
>-	enum drm_connector_status status;
>-
>-	/* Can't disconnect eDP, but you can close the lid... */
>-	if (is_edp(intel_dp)) {
>-		status = intel_panel_detect(dev);
>-		if (status == connector_status_unknown)
>-			status = connector_status_connected;
>-		return status;
>-	}
> 
> 	if (!ibx_digital_port_connected(dev_priv, intel_dig_port))
> 		return connector_status_disconnected;
>@@ -3733,9 +4114,9 @@ g4x_dp_detect(struct intel_dp *intel_dp)
> }
> 
> static struct edid *
>-intel_dp_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
>+intel_dp_get_edid(struct intel_dp *intel_dp)
> {
>-	struct intel_connector *intel_connector = to_intel_connector(connector);
>+	struct intel_connector *intel_connector = intel_dp->attached_connector;
> 
> 	/* use cached edid if we have one */
> 	if (intel_connector->edid) {
>@@ -3744,27 +4125,55 @@ intel_dp_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
> 			return NULL;
> 
> 		return drm_edid_duplicate(intel_connector->edid);
>-	}
>+	} else
>+		return drm_get_edid(&intel_connector->base,
>+				    &intel_dp->aux.ddc);
>+}
>+
>+static void
>+intel_dp_set_edid(struct intel_dp *intel_dp)
>+{
>+	struct intel_connector *intel_connector = intel_dp->attached_connector;
>+	struct edid *edid;
> 
>-	return drm_get_edid(connector, adapter);
>+	edid = intel_dp_get_edid(intel_dp);
>+	intel_connector->detect_edid = edid;
>+
>+	if (intel_dp->force_audio != HDMI_AUDIO_AUTO)
>+		intel_dp->has_audio = intel_dp->force_audio == HDMI_AUDIO_ON;
>+	else
>+		intel_dp->has_audio = drm_detect_monitor_audio(edid);
> }
> 
>-static int
>-intel_dp_get_edid_modes(struct drm_connector *connector, struct i2c_adapter *adapter)
>+static void
>+intel_dp_unset_edid(struct intel_dp *intel_dp)
> {
>-	struct intel_connector *intel_connector = to_intel_connector(connector);
>+	struct intel_connector *intel_connector = intel_dp->attached_connector;
> 
>-	/* use cached edid if we have one */
>-	if (intel_connector->edid) {
>-		/* invalid edid */
>-		if (IS_ERR(intel_connector->edid))
>-			return 0;
>+	kfree(intel_connector->detect_edid);
>+	intel_connector->detect_edid = NULL;
> 
>-		return intel_connector_update_modes(connector,
>-						    intel_connector->edid);
>-	}
>+	intel_dp->has_audio = false;
>+}
>+
>+static enum intel_display_power_domain
>+intel_dp_power_get(struct intel_dp *dp)
>+{
>+	struct intel_encoder *encoder = &dp_to_dig_port(dp)->base;
>+	enum intel_display_power_domain power_domain;
>+
>+	power_domain = intel_display_port_power_domain(encoder);
>+	intel_display_power_get(to_i915(encoder->base.dev), power_domain);
>+
>+	return power_domain;
>+}
> 
>-	return intel_ddc_get_modes(connector, adapter);
>+static void
>+intel_dp_power_put(struct intel_dp *dp,
>+		   enum intel_display_power_domain power_domain)
>+{
>+	struct intel_encoder *encoder = &dp_to_dig_port(dp)->base;
>+	intel_display_power_put(to_i915(encoder->base.dev), power_domain);
> }
> 
> static enum drm_connector_status
>@@ -3774,33 +4183,30 @@ intel_dp_detect(struct drm_connector *connector, bool force)
> 	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
> 	struct intel_encoder *intel_encoder = &intel_dig_port->base;
> 	struct drm_device *dev = connector->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
> 	enum drm_connector_status status;
> 	enum intel_display_power_domain power_domain;
>-	struct edid *edid = NULL;
> 	bool ret;
> 
>-	power_domain = intel_display_port_power_domain(intel_encoder);
>-	intel_display_power_get(dev_priv, power_domain);
>-
> 	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
> 		      connector->base.id, connector->name);
>+	intel_dp_unset_edid(intel_dp);
> 
> 	if (intel_dp->is_mst) {
> 		/* MST devices are disconnected from a monitor POV */
> 		if (intel_encoder->type != INTEL_OUTPUT_EDP)
> 			intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
>-		status = connector_status_disconnected;
>-		goto out;
>+		return connector_status_disconnected;
> 	}
> 
>-	intel_dp->has_audio = false;
>+	power_domain = intel_dp_power_get(intel_dp);
> 
>-	if (HAS_PCH_SPLIT(dev))
>+	/* Can't disconnect eDP, but you can close the lid... */
>+	if (is_edp(intel_dp))
>+		status = edp_detect(intel_dp);
>+	else if (HAS_PCH_SPLIT(dev))
> 		status = ironlake_dp_detect(intel_dp);
> 	else
> 		status = g4x_dp_detect(intel_dp);
>-
> 	if (status != connector_status_connected)
> 		goto out;
> 
>@@ -3816,82 +4222,78 @@ intel_dp_detect(struct drm_connector *connector, bool force)
> 		goto out;
> 	}
> 
>-	if (intel_dp->force_audio != HDMI_AUDIO_AUTO) {
>-		intel_dp->has_audio = (intel_dp->force_audio == HDMI_AUDIO_ON);
>-	} else {
>-		edid = intel_dp_get_edid(connector, &intel_dp->aux.ddc);
>-		if (edid) {
>-			intel_dp->has_audio = drm_detect_monitor_audio(edid);
>-			kfree(edid);
>-		}
>-	}
>+	intel_dp_set_edid(intel_dp);
> 
> 	if (intel_encoder->type != INTEL_OUTPUT_EDP)
> 		intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
> 	status = connector_status_connected;
> 
> out:
>-	intel_display_power_put(dev_priv, power_domain);
>+	intel_dp_power_put(intel_dp, power_domain);
> 	return status;
> }
> 
>-static int intel_dp_get_modes(struct drm_connector *connector)
>+static void
>+intel_dp_force(struct drm_connector *connector)
> {
> 	struct intel_dp *intel_dp = intel_attached_dp(connector);
>-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
>-	struct intel_encoder *intel_encoder = &intel_dig_port->base;
>-	struct intel_connector *intel_connector = to_intel_connector(connector);
>-	struct drm_device *dev = connector->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base;
> 	enum intel_display_power_domain power_domain;
>-	int ret;
> 
>-	/* We should parse the EDID data and find out if it has an audio sink
>-	 */
>+	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
>+		      connector->base.id, connector->name);
>+	intel_dp_unset_edid(intel_dp);
> 
>-	power_domain = intel_display_port_power_domain(intel_encoder);
>-	intel_display_power_get(dev_priv, power_domain);
>+	if (connector->status != connector_status_connected)
>+		return;
> 
>-	ret = intel_dp_get_edid_modes(connector, &intel_dp->aux.ddc);
>-	intel_display_power_put(dev_priv, power_domain);
>-	if (ret)
>-		return ret;
>+	power_domain = intel_dp_power_get(intel_dp);
>+
>+	intel_dp_set_edid(intel_dp);
>+
>+	intel_dp_power_put(intel_dp, power_domain);
>+
>+	if (intel_encoder->type != INTEL_OUTPUT_EDP)
>+		intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
>+}
>+
>+static int intel_dp_get_modes(struct drm_connector *connector)
>+{
>+	struct intel_connector *intel_connector = to_intel_connector(connector);
>+	struct edid *edid;
>+
>+	edid = intel_connector->detect_edid;
>+	if (edid) {
>+		int ret = intel_connector_update_modes(connector, edid);
>+		if (ret)
>+			return ret;
>+	}
> 
> 	/* if eDP has no EDID, fall back to fixed mode */
>-	if (is_edp(intel_dp) && intel_connector->panel.fixed_mode) {
>+	if (is_edp(intel_attached_dp(connector)) &&
>+	    intel_connector->panel.fixed_mode) {
> 		struct drm_display_mode *mode;
>-		mode = drm_mode_duplicate(dev,
>+
>+		mode = drm_mode_duplicate(connector->dev,
> 					  intel_connector->panel.fixed_mode);
> 		if (mode) {
> 			drm_mode_probed_add(connector, mode);
> 			return 1;
> 		}
> 	}
>+
> 	return 0;
> }
> 
> static bool
> intel_dp_detect_audio(struct drm_connector *connector)
> {
>-	struct intel_dp *intel_dp = intel_attached_dp(connector);
>-	struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
>-	struct intel_encoder *intel_encoder = &intel_dig_port->base;
>-	struct drm_device *dev = connector->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	enum intel_display_power_domain power_domain;
>-	struct edid *edid;
> 	bool has_audio = false;
>+	struct edid *edid;
> 
>-	power_domain = intel_display_port_power_domain(intel_encoder);
>-	intel_display_power_get(dev_priv, power_domain);
>-
>-	edid = intel_dp_get_edid(connector, &intel_dp->aux.ddc);
>-	if (edid) {
>+	edid = to_intel_connector(connector)->detect_edid;
>+	if (edid)
> 		has_audio = drm_detect_monitor_audio(edid);
>-		kfree(edid);
>-	}
>-
>-	intel_display_power_put(dev_priv, power_domain);
> 
> 	return has_audio;
> }
>@@ -3989,6 +4391,8 @@ intel_dp_connector_destroy(struct drm_connector *connector)
> {
> 	struct intel_connector *intel_connector = to_intel_connector(connector);
> 
>+	kfree(intel_connector->detect_edid);
>+
> 	if (!IS_ERR_OR_NULL(intel_connector->edid))
> 		kfree(intel_connector->edid);
> 
>@@ -4005,16 +4409,20 @@ void intel_dp_encoder_destroy(struct drm_encoder *encoder)
> {
> 	struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
> 	struct intel_dp *intel_dp = &intel_dig_port->dp;
>-	struct drm_device *dev = intel_dp_to_dev(intel_dp);
> 
> 	drm_dp_aux_unregister(&intel_dp->aux);
> 	intel_dp_mst_encoder_cleanup(intel_dig_port);
> 	drm_encoder_cleanup(encoder);
> 	if (is_edp(intel_dp)) {
> 		cancel_delayed_work_sync(&intel_dp->panel_vdd_work);
>-		drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
>+		/*
>+		 * vdd might still be enabled do to the delayed vdd off.
>+		 * Make sure vdd is actually turned off here.
>+		 */
>+		pps_lock(intel_dp);
> 		edp_panel_vdd_off_sync(intel_dp);
>-		drm_modeset_unlock(&dev->mode_config.connection_mutex);
>+		pps_unlock(intel_dp);
>+
> 		if (intel_dp->edp_notifier.notifier_call) {
> 			unregister_reboot_notifier(&intel_dp->edp_notifier);
> 			intel_dp->edp_notifier.notifier_call = NULL;
>@@ -4030,7 +4438,13 @@ static void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder)
> 	if (!is_edp(intel_dp))
> 		return;
> 
>+	/*
>+	 * vdd might still be enabled do to the delayed vdd off.
>+	 * Make sure vdd is actually turned off here.
>+	 */
>+	pps_lock(intel_dp);
> 	edp_panel_vdd_off_sync(intel_dp);
>+	pps_unlock(intel_dp);
> }
> 
> static void intel_dp_encoder_reset(struct drm_encoder *encoder)
>@@ -4041,6 +4455,7 @@ static void intel_dp_encoder_reset(struct drm_encoder *encoder)
> static const struct drm_connector_funcs intel_dp_connector_funcs = {
> 	.dpms = intel_connector_dpms,
> 	.detect = intel_dp_detect,
>+	.force = intel_dp_force,
> 	.fill_modes = drm_helper_probe_single_connector_modes,
> 	.set_property = intel_dp_set_property,
> 	.destroy = intel_dp_connector_destroy,
>@@ -4076,7 +4491,8 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd)
> 	if (intel_dig_port->base.type != INTEL_OUTPUT_EDP)
> 		intel_dig_port->base.type = INTEL_OUTPUT_DISPLAYPORT;
> 
>-	DRM_DEBUG_KMS("got hpd irq on port %d - %s\n", intel_dig_port->port,
>+	DRM_DEBUG_KMS("got hpd irq on port %c - %s\n",
>+		      port_name(intel_dig_port->port),
> 		      long_hpd ? "long" : "short");
> 
> 	power_domain = intel_display_port_power_domain(intel_encoder);
>@@ -4216,6 +4632,8 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev,
> 	u32 pp_on, pp_off, pp_div, pp;
> 	int pp_ctrl_reg, pp_on_reg, pp_off_reg, pp_div_reg;
> 
>+	lockdep_assert_held(&dev_priv->pps_mutex);
>+
> 	if (HAS_PCH_SPLIT(dev)) {
> 		pp_ctrl_reg = PCH_PP_CONTROL;
> 		pp_on_reg = PCH_PP_ON_DELAYS;
>@@ -4315,6 +4733,9 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
> 	u32 pp_on, pp_off, pp_div, port_sel = 0;
> 	int div = HAS_PCH_SPLIT(dev) ? intel_pch_rawclk(dev) : intel_hrawclk(dev);
> 	int pp_on_reg, pp_off_reg, pp_div_reg;
>+	enum port port = dp_to_dig_port(intel_dp)->port;
>+
>+	lockdep_assert_held(&dev_priv->pps_mutex);
> 
> 	if (HAS_PCH_SPLIT(dev)) {
> 		pp_on_reg = PCH_PP_ON_DELAYS;
>@@ -4349,12 +4770,9 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
> 	/* Haswell doesn't have any port selection bits for the panel
> 	 * power sequencer any more. */
> 	if (IS_VALLEYVIEW(dev)) {
>-		if (dp_to_dig_port(intel_dp)->port == PORT_B)
>-			port_sel = PANEL_PORT_SELECT_DPB_VLV;
>-		else
>-			port_sel = PANEL_PORT_SELECT_DPC_VLV;
>+		port_sel = PANEL_PORT_SELECT_VLV(port);
> 	} else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) {
>-		if (dp_to_dig_port(intel_dp)->port == PORT_A)
>+		if (port == PORT_A)
> 			port_sel = PANEL_PORT_SELECT_DPA;
> 		else
> 			port_sel = PANEL_PORT_SELECT_DPD;
>@@ -4438,7 +4856,7 @@ void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate)
> 		val = I915_READ(reg);
> 		if (index > DRRS_HIGH_RR) {
> 			val |= PIPECONF_EDP_RR_MODE_SWITCH;
>-			intel_dp_set_m2_n2(intel_crtc, &config->dp_m2_n2);
>+			intel_dp_set_m_n(intel_crtc);
> 		} else {
> 			val &= ~PIPECONF_EDP_RR_MODE_SWITCH;
> 		}
>@@ -4478,7 +4896,7 @@ intel_dp_drrs_init(struct intel_digital_port *intel_dig_port,
> 	}
> 
> 	if (dev_priv->vbt.drrs_type != SEAMLESS_DRRS_SUPPORT) {
>-		DRM_INFO("VBT doesn't support DRRS\n");
>+		DRM_DEBUG_KMS("VBT doesn't support DRRS\n");
> 		return NULL;
> 	}
> 
>@@ -4486,7 +4904,7 @@ intel_dp_drrs_init(struct intel_digital_port *intel_dig_port,
> 					(dev, fixed_mode, connector);
> 
> 	if (!downclock_mode) {
>-		DRM_INFO("DRRS not supported\n");
>+		DRM_DEBUG_KMS("DRRS not supported\n");
> 		return NULL;
> 	}
> 
>@@ -4497,7 +4915,7 @@ intel_dp_drrs_init(struct intel_digital_port *intel_dig_port,
> 	intel_dp->drrs_state.type = dev_priv->vbt.drrs_type;
> 
> 	intel_dp->drrs_state.refresh_rate_type = DRRS_HIGH_RR;
>-	DRM_INFO("seamless DRRS supported for eDP panel.\n");
>+	DRM_DEBUG_KMS("seamless DRRS supported for eDP panel.\n");
> 	return downclock_mode;
> }
> 
>@@ -4512,8 +4930,11 @@ void intel_edp_panel_vdd_sanitize(struct intel_encoder *intel_encoder)
> 		return;
> 
> 	intel_dp = enc_to_intel_dp(&intel_encoder->base);
>+
>+	pps_lock(intel_dp);
>+
> 	if (!edp_have_panel_vdd(intel_dp))
>-		return;
>+		goto out;
> 	/*
> 	 * The VDD bit needs a power domain reference, so if the bit is
> 	 * already enabled when we boot or resume, grab this reference and
>@@ -4525,6 +4946,8 @@ void intel_edp_panel_vdd_sanitize(struct intel_encoder *intel_encoder)
> 	intel_display_power_get(dev_priv, power_domain);
> 
> 	edp_panel_vdd_schedule_off(intel_dp);
>+ out:
>+	pps_unlock(intel_dp);
> }
> 
> static bool intel_edp_init_connector(struct intel_dp *intel_dp,
>@@ -4552,7 +4975,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
> 	/* Cache DPCD and EDID for edp. */
> 	intel_edp_panel_vdd_on(intel_dp);
> 	has_dpcd = intel_dp_get_dpcd(intel_dp);
>-	edp_panel_vdd_off(intel_dp, false);
>+	intel_edp_panel_vdd_off(intel_dp, false);
> 
> 	if (has_dpcd) {
> 		if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11)
>@@ -4566,7 +4989,9 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
> 	}
> 
> 	/* We now know it's not a ghost, init power sequence regs. */
>+	pps_lock(intel_dp);
> 	intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, power_seq);
>+	pps_unlock(intel_dp);
> 
> 	mutex_lock(&dev->mode_config.mutex);
> 	edid = drm_get_edid(connector, &intel_dp->aux.ddc);
>@@ -4610,6 +5035,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
> 	}
> 
> 	intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode);
>+	intel_connector->panel.backlight_power = intel_edp_backlight_power;
> 	intel_panel_setup_backlight(connector);
> 
> 	return true;
>@@ -4628,6 +5054,8 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
> 	struct edp_power_seq power_seq = { 0 };
> 	int type;
> 
>+	intel_dp->pps_pipe = INVALID_PIPE;
>+
> 	/* intel_dp vfuncs */
> 	if (IS_VALLEYVIEW(dev))
> 		intel_dp->get_aux_clock_divider = vlv_get_aux_clock_divider;
>@@ -4698,8 +5126,15 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
> 	}
> 
> 	if (is_edp(intel_dp)) {
>-		intel_dp_init_panel_power_timestamps(intel_dp);
>-		intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
>+		pps_lock(intel_dp);
>+		if (IS_VALLEYVIEW(dev)) {
>+			vlv_initial_power_sequencer_setup(intel_dp);
>+		} else {
>+			intel_dp_init_panel_power_timestamps(intel_dp);
>+			intel_dp_init_panel_power_sequencer(dev, intel_dp,
>+							    &power_seq);
>+		}
>+		pps_unlock(intel_dp);
> 	}
> 
> 	intel_dp_aux_init(intel_dp, intel_connector);
>@@ -4707,7 +5142,8 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
> 	/* init MST on ports that can support it */
> 	if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
> 		if (port == PORT_B || port == PORT_C || port == PORT_D) {
>-			intel_dp_mst_encoder_init(intel_dig_port, intel_connector->base.base.id);
>+			intel_dp_mst_encoder_init(intel_dig_port,
>+						  intel_connector->base.base.id);
> 		}
> 	}
> 
>@@ -4715,9 +5151,13 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
> 		drm_dp_aux_unregister(&intel_dp->aux);
> 		if (is_edp(intel_dp)) {
> 			cancel_delayed_work_sync(&intel_dp->panel_vdd_work);
>-			drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
>+			/*
>+			 * vdd might still be enabled do to the delayed vdd off.
>+			 * Make sure vdd is actually turned off here.
>+			 */
>+			pps_lock(intel_dp);
> 			edp_panel_vdd_off_sync(intel_dp);
>-			drm_modeset_unlock(&dev->mode_config.connection_mutex);
>+			pps_unlock(intel_dp);
> 		}
> 		drm_connector_unregister(connector);
> 		drm_connector_cleanup(connector);
>@@ -4781,7 +5221,8 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port)
> 	} else {
> 		intel_encoder->pre_enable = g4x_pre_enable_dp;
> 		intel_encoder->enable = g4x_enable_dp;
>-		intel_encoder->post_disable = g4x_post_disable_dp;
>+		if (INTEL_INFO(dev)->gen >= 5)
>+			intel_encoder->post_disable = ilk_post_disable_dp;
> 	}
> 
> 	intel_dig_port->port = port;
>diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
>index b8c8bbd..4ba59c2 100644
>--- a/drivers/gpu/drm/i915/intel_drv.h
>+++ b/drivers/gpu/drm/i915/intel_drv.h
>@@ -25,6 +25,7 @@
> #ifndef __INTEL_DRV_H__
> #define __INTEL_DRV_H__
> 
>+#include <linux/async.h>
> #include <linux/i2c.h>
> #include <linux/hdmi.h>
> #include <drm/i915_drm.h>
>@@ -33,6 +34,7 @@
> #include <drm/drm_crtc_helper.h>
> #include <drm/drm_fb_helper.h>
> #include <drm/drm_dp_mst_helper.h>
>+#include <drm/drm_rect.h>
> 
> /**
>  * _wait_for - magic (register) wait macro
>@@ -179,6 +181,8 @@ struct intel_panel {
> 		bool active_low_pwm;
> 		struct backlight_device *device;
> 	} backlight;
>+
>+	void (*backlight_power)(struct intel_connector *, bool enable);
> };
> 
> struct intel_connector {
>@@ -211,6 +215,7 @@ struct intel_connector {
> 
> 	/* Cached EDID for eDP and LVDS. May hold ERR_PTR for invalid EDID. */
> 	struct edid *edid;
>+	struct edid *detect_edid;
> 
> 	/* since POLL and HPD connectors may use the same HPD line keep the native
> 	   state of connector->polled in case hotplug storm detection changes it */
>@@ -233,6 +238,17 @@ typedef struct dpll {
> 	int	p;
> } intel_clock_t;
> 
>+struct intel_plane_state {
>+	struct drm_crtc *crtc;
>+	struct drm_framebuffer *fb;
>+	struct drm_rect src;
>+	struct drm_rect dst;
>+	struct drm_rect clip;
>+	struct drm_rect orig_src;
>+	struct drm_rect orig_dst;
>+	bool visible;
>+};
>+
> struct intel_plane_config {
> 	bool tiled;
> 	int size;
>@@ -330,6 +346,7 @@ struct intel_crtc_config {
> 
> 	/* m2_n2 for eDP downclock */
> 	struct intel_link_m_n dp_m2_n2;
>+	bool has_drrs;
> 
> 	/*
> 	 * Frequence the dpll for the port should run at. Differs from the
>@@ -377,11 +394,6 @@ struct intel_pipe_wm {
> 	bool sprites_scaled;
> };
> 
>-struct intel_mmio_flip {
>-	u32 seqno;
>-	u32 ring_id;
>-};
>-
> struct intel_crtc {
> 	struct drm_crtc base;
> 	enum pipe pipe;
>@@ -410,6 +422,7 @@ struct intel_crtc {
> 	uint32_t cursor_addr;
> 	int16_t cursor_width, cursor_height;
> 	uint32_t cursor_cntl;
>+	uint32_t cursor_size;
> 	uint32_t cursor_base;
> 
> 	struct intel_plane_config plane_config;
>@@ -430,10 +443,8 @@ struct intel_crtc {
> 		struct intel_pipe_wm active;
> 	} wm;
> 
>-	wait_queue_head_t vbl_wait;
>-
> 	int scanline_offset;
>-	struct intel_mmio_flip mmio_flip;
>+	struct i915_gem_request *mmio_flip;
> };
> 
> struct intel_plane_wm_parameters {
>@@ -455,6 +466,7 @@ struct intel_plane {
> 	unsigned int crtc_w, crtc_h;
> 	uint32_t src_x, src_y;
> 	uint32_t src_w, src_h;
>+	unsigned int rotation;
> 
> 	/* Since we need to change the watermarks before/after
> 	 * enabling/disabling the planes, we need to store the parameters here
>@@ -565,6 +577,12 @@ struct intel_dp {
> 
> 	struct notifier_block edp_notifier;
> 
>+	/*
>+	 * Pipe whose power sequencer is currently locked into
>+	 * this port. Only relevant on VLV/CHV.
>+	 */
>+	enum pipe pps_pipe;
>+
> 	bool use_tps3;
> 	bool can_mst; /* this port supports mst */
> 	bool is_mst;
>@@ -663,6 +681,9 @@ struct intel_unpin_work {
> #define INTEL_FLIP_COMPLETE	2
> 	u32 flip_count;
> 	u32 gtt_offset;
>+	struct i915_gem_request *flip_queued_request;
>+	int flip_queued_vblank;
>+	int flip_ready_vblank;
> 	bool enable_stall_check;
> };
> 
>@@ -780,7 +801,7 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev);
> int intel_pch_rawclk(struct drm_device *dev);
> void intel_mark_busy(struct drm_device *dev);
> void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
>-			     struct intel_engine_cs *ring);
>+			     struct i915_gem_request *rq);
> void intel_frontbuffer_flip_prepare(struct drm_device *dev,
> 				    unsigned frontbuffer_bits);
> void intel_frontbuffer_flip_complete(struct drm_device *dev,
>@@ -827,7 +848,6 @@ int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
> enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
> 					     enum pipe pipe);
> void intel_wait_for_vblank(struct drm_device *dev, int pipe);
>-void intel_wait_for_pipe_off(struct drm_device *dev, int pipe);
> int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp);
> void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
> 			 struct intel_digital_port *dport);
>@@ -839,7 +859,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector,
> 				    struct intel_load_detect_pipe *old);
> int intel_pin_and_fence_fb_obj(struct drm_device *dev,
> 			       struct drm_i915_gem_object *obj,
>-			       struct intel_engine_cs *pipelined);
>+			       struct i915_gem_request *pipelined);
> void intel_unpin_fb_obj(struct drm_i915_gem_object *obj);
> struct drm_framebuffer *
> __intel_framebuffer_create(struct drm_device *dev,
>@@ -848,6 +868,7 @@ __intel_framebuffer_create(struct drm_device *dev,
> void intel_prepare_page_flip(struct drm_device *dev, int plane);
> void intel_finish_page_flip(struct drm_device *dev, int pipe);
> void intel_finish_page_flip_plane(struct drm_device *dev, int plane);
>+void intel_check_page_flip(struct drm_device *dev, int pipe);
> 
> /* shared dpll functions */
> struct intel_shared_dpll *intel_crtc_to_shared_dpll(struct intel_crtc *crtc);
>@@ -882,6 +903,7 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv);
> void hsw_disable_pc8(struct drm_i915_private *dev_priv);
> void intel_dp_get_m_n(struct intel_crtc *crtc,
> 		      struct intel_crtc_config *pipe_config);
>+void intel_dp_set_m_n(struct intel_crtc *crtc);
> int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n);
> void
> ironlake_check_encoder_dotclock(const struct intel_crtc_config *pipe_config,
>@@ -896,7 +918,7 @@ void intel_mode_from_pipe_config(struct drm_display_mode *mode,
> 				 struct intel_crtc_config *pipe_config);
> int intel_format_to_fourcc(int format);
> void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc);
>-
>+void intel_modeset_preclose(struct drm_device *dev, struct drm_file *file);
> 
> /* intel_dp.c */
> void intel_dp_init(struct drm_device *dev, int output_reg, enum port port);
>@@ -935,6 +957,7 @@ void intel_dp_mst_suspend(struct drm_device *dev);
> void intel_dp_mst_resume(struct drm_device *dev);
> int intel_dp_max_link_bw(struct intel_dp *intel_dp);
> void intel_dp_hot_plug(struct intel_encoder *intel_encoder);
>+void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv);
> /* intel_dp_mst.c */
> int intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_id);
> void intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port);
>@@ -949,9 +972,9 @@ void intel_dvo_init(struct drm_device *dev);
> /* legacy fbdev emulation in intel_fbdev.c */
> #ifdef CONFIG_DRM_I915_FBDEV
> extern int intel_fbdev_init(struct drm_device *dev);
>-extern void intel_fbdev_initial_config(struct drm_device *dev);
>+extern void intel_fbdev_initial_config(void *data, async_cookie_t cookie);
> extern void intel_fbdev_fini(struct drm_device *dev);
>-extern void intel_fbdev_set_suspend(struct drm_device *dev, int state);
>+extern void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous);
> extern void intel_fbdev_output_poll_changed(struct drm_device *dev);
> extern void intel_fbdev_restore_mode(struct drm_device *dev);
> #else
>@@ -960,7 +983,7 @@ static inline int intel_fbdev_init(struct drm_device *dev)
> 	return 0;
> }
> 
>-static inline void intel_fbdev_initial_config(struct drm_device *dev)
>+static inline void intel_fbdev_initial_config(void *data, async_cookie_t cookie)
> {
> }
> 
>@@ -968,7 +991,7 @@ static inline void intel_fbdev_fini(struct drm_device *dev)
> {
> }
> 
>-static inline void intel_fbdev_set_suspend(struct drm_device *dev, int state)
>+static inline void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous)
> {
> }
> 
>@@ -1081,6 +1104,7 @@ void intel_runtime_pm_put(struct drm_i915_private *dev_priv);
> void intel_init_runtime_pm(struct drm_i915_private *dev_priv);
> void intel_fini_runtime_pm(struct drm_i915_private *dev_priv);
> void ilk_wm_get_hw_state(struct drm_device *dev);
>+uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_config *pipe_config);
> 
> 
> /* intel_sdvo.c */
>@@ -1091,7 +1115,10 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob);
> int intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane);
> void intel_flush_primary_plane(struct drm_i915_private *dev_priv,
> 			       enum plane plane);
>-void intel_plane_restore(struct drm_plane *plane);
>+int intel_plane_set_property(struct drm_plane *plane,
>+			     struct drm_property *prop,
>+			     uint64_t val);
>+int intel_plane_restore(struct drm_plane *plane);
> void intel_plane_disable(struct drm_plane *plane);
> int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
> 			      struct drm_file *file_priv);
>diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c
>index 670c29a..5bd9e09 100644
>--- a/drivers/gpu/drm/i915/intel_dsi.c
>+++ b/drivers/gpu/drm/i915/intel_dsi.c
>@@ -184,7 +184,7 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder)
> 
> 	/* update the hw state for DPLL */
> 	intel_crtc->config.dpll_hw_state.dpll = DPLL_INTEGRATED_CLOCK_VLV |
>-						DPLL_REFA_CLK_ENABLE_VLV;
>+		DPLL_REFA_CLK_ENABLE_VLV;
> 
> 	tmp = I915_READ(DSPCLK_GATE_D);
> 	tmp |= DPOUNIT_CLOCK_GATE_DISABLE;
>@@ -259,8 +259,8 @@ static void intel_dsi_disable(struct intel_encoder *encoder)
> 	temp = I915_READ(MIPI_CTRL(pipe));
> 	temp &= ~ESCAPE_CLOCK_DIVIDER_MASK;
> 	I915_WRITE(MIPI_CTRL(pipe), temp |
>-			intel_dsi->escape_clk_div <<
>-			ESCAPE_CLOCK_DIVIDER_SHIFT);
>+		   intel_dsi->escape_clk_div <<
>+		   ESCAPE_CLOCK_DIVIDER_SHIFT);
> 
> 	I915_WRITE(MIPI_EOT_DISABLE(pipe), CLOCKSTOP);
> 
>@@ -297,7 +297,7 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
> 	usleep_range(2000, 2500);
> 
> 	if (wait_for(((I915_READ(MIPI_PORT_CTRL(pipe)) & AFE_LATCHOUT)
>-					== 0x00000), 30))
>+		      == 0x00000), 30))
> 		DRM_ERROR("DSI LP not going Low\n");
> 
> 	val = I915_READ(MIPI_PORT_CTRL(pipe));
>@@ -423,9 +423,11 @@ static u16 txclkesc(u32 divider, unsigned int us)
> }
> 
> /* return pixels in terms of txbyteclkhs */
>-static u16 txbyteclkhs(u16 pixels, int bpp, int lane_count)
>+static u16 txbyteclkhs(u16 pixels, int bpp, int lane_count,
>+		       u16 burst_mode_ratio)
> {
>-	return DIV_ROUND_UP(DIV_ROUND_UP(pixels * bpp, 8), lane_count);
>+	return DIV_ROUND_UP(DIV_ROUND_UP(pixels * bpp * burst_mode_ratio,
>+					 8 * 100), lane_count);
> }
> 
> static void set_dsi_timings(struct drm_encoder *encoder,
>@@ -451,10 +453,12 @@ static void set_dsi_timings(struct drm_encoder *encoder,
> 	vbp = mode->vtotal - mode->vsync_end;
> 
> 	/* horizontal values are in terms of high speed byte clock */
>-	hactive = txbyteclkhs(hactive, bpp, lane_count);
>-	hfp = txbyteclkhs(hfp, bpp, lane_count);
>-	hsync = txbyteclkhs(hsync, bpp, lane_count);
>-	hbp = txbyteclkhs(hbp, bpp, lane_count);
>+	hactive = txbyteclkhs(hactive, bpp, lane_count,
>+			      intel_dsi->burst_mode_ratio);
>+	hfp = txbyteclkhs(hfp, bpp, lane_count, intel_dsi->burst_mode_ratio);
>+	hsync = txbyteclkhs(hsync, bpp, lane_count,
>+			    intel_dsi->burst_mode_ratio);
>+	hbp = txbyteclkhs(hbp, bpp, lane_count, intel_dsi->burst_mode_ratio);
> 
> 	I915_WRITE(MIPI_HACTIVE_AREA_COUNT(pipe), hactive);
> 	I915_WRITE(MIPI_HFP_COUNT(pipe), hfp);
>@@ -541,12 +545,14 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder)
> 	    intel_dsi->video_mode_format == VIDEO_MODE_BURST) {
> 		I915_WRITE(MIPI_HS_TX_TIMEOUT(pipe),
> 			   txbyteclkhs(adjusted_mode->htotal, bpp,
>-				       intel_dsi->lane_count) + 1);
>+				       intel_dsi->lane_count,
>+				       intel_dsi->burst_mode_ratio) + 1);
> 	} else {
> 		I915_WRITE(MIPI_HS_TX_TIMEOUT(pipe),
> 			   txbyteclkhs(adjusted_mode->vtotal *
> 				       adjusted_mode->htotal,
>-				       bpp, intel_dsi->lane_count) + 1);
>+				       bpp, intel_dsi->lane_count,
>+				       intel_dsi->burst_mode_ratio) + 1);
> 	}
> 	I915_WRITE(MIPI_LP_RX_TIMEOUT(pipe), intel_dsi->lp_rx_timeout);
> 	I915_WRITE(MIPI_TURN_AROUND_TIMEOUT(pipe), intel_dsi->turn_arnd_val);
>@@ -576,7 +582,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder)
> 	 * XXX: write MIPI_STOP_STATE_STALL?
> 	 */
> 	I915_WRITE(MIPI_HIGH_LOW_SWITCH_COUNT(pipe),
>-						intel_dsi->hs_to_lp_count);
>+		   intel_dsi->hs_to_lp_count);
> 
> 	/* XXX: low power clock equivalence in terms of byte clock. the number
> 	 * of byte clocks occupied in one low power clock. based on txbyteclkhs
>@@ -601,10 +607,10 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder)
> 		 * 64 like 1366 x 768. Enable RANDOM resolution support for such
> 		 * panels by default */
> 		I915_WRITE(MIPI_VIDEO_MODE_FORMAT(pipe),
>-				intel_dsi->video_frmt_cfg_bits |
>-				intel_dsi->video_mode_format |
>-				IP_TG_CONFIG |
>-				RANDOM_DPI_DISPLAY_RESOLUTION);
>+			   intel_dsi->video_frmt_cfg_bits |
>+			   intel_dsi->video_mode_format |
>+			   IP_TG_CONFIG |
>+			   RANDOM_DPI_DISPLAY_RESOLUTION);
> }
> 
> static void intel_dsi_pre_pll_enable(struct intel_encoder *encoder)
>diff --git a/drivers/gpu/drm/i915/intel_dsi.h b/drivers/gpu/drm/i915/intel_dsi.h
>index fd51867..657eb5c 100644
>--- a/drivers/gpu/drm/i915/intel_dsi.h
>+++ b/drivers/gpu/drm/i915/intel_dsi.h
>@@ -116,6 +116,8 @@ struct intel_dsi {
> 	u16 clk_hs_to_lp_count;
> 
> 	u16 init_count;
>+	u32 pclk;
>+	u16 burst_mode_ratio;
> 
> 	/* all delays in ms */
> 	u16 backlight_off_delay;
>diff --git a/drivers/gpu/drm/i915/intel_dsi_cmd.c b/drivers/gpu/drm/i915/intel_dsi_cmd.c
>index 7f1430a..f4767fd 100644
>--- a/drivers/gpu/drm/i915/intel_dsi_cmd.c
>+++ b/drivers/gpu/drm/i915/intel_dsi_cmd.c
>@@ -430,7 +430,7 @@ void wait_for_dsi_fifo_empty(struct intel_dsi *intel_dsi)
> 	u32 mask;
> 
> 	mask = LP_CTRL_FIFO_EMPTY | HS_CTRL_FIFO_EMPTY |
>-					LP_DATA_FIFO_EMPTY | HS_DATA_FIFO_EMPTY;
>+		LP_DATA_FIFO_EMPTY | HS_DATA_FIFO_EMPTY;
> 
> 	if (wait_for((I915_READ(MIPI_GEN_FIFO_STAT(pipe)) & mask) == mask, 100))
> 		DRM_ERROR("DPI FIFOs are not empty\n");
>diff --git a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
>index 47c7584..f6bdd44 100644
>--- a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
>+++ b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
>@@ -271,6 +271,8 @@ static bool generic_init(struct intel_dsi_device *dsi)
> 	u32 ths_prepare_ns, tclk_trail_ns;
> 	u32 tclk_prepare_clkzero, ths_prepare_hszero;
> 	u32 lp_to_hs_switch, hs_to_lp_switch;
>+	u32 pclk, computed_ddr;
>+	u16 burst_mode_ratio;
> 
> 	DRM_DEBUG_KMS("\n");
> 
>@@ -284,8 +286,6 @@ static bool generic_init(struct intel_dsi_device *dsi)
> 	else if (intel_dsi->pixel_format == VID_MODE_FORMAT_RGB565)
> 		bits_per_pixel = 16;
> 
>-	bitrate = (mode->clock * bits_per_pixel) / intel_dsi->lane_count;
>-
> 	intel_dsi->operation_mode = mipi_config->is_cmd_mode;
> 	intel_dsi->video_mode_format = mipi_config->video_transfer_mode;
> 	intel_dsi->escape_clk_div = mipi_config->byte_clk_sel;
>@@ -297,6 +297,40 @@ static bool generic_init(struct intel_dsi_device *dsi)
> 	intel_dsi->video_frmt_cfg_bits =
> 		mipi_config->bta_enabled ? DISABLE_VIDEO_BTA : 0;
> 
>+	pclk = mode->clock;
>+
>+	/* Burst Mode Ratio
>+	 * Target ddr frequency from VBT / non burst ddr freq
>+	 * multiply by 100 to preserve remainder
>+	 */
>+	if (intel_dsi->video_mode_format == VIDEO_MODE_BURST) {
>+		if (mipi_config->target_burst_mode_freq) {
>+			computed_ddr =
>+				(pclk * bits_per_pixel) / intel_dsi->lane_count;
>+
>+			if (mipi_config->target_burst_mode_freq <
>+								computed_ddr) {
>+				DRM_ERROR("Burst mode freq is less than computed\n");
>+				return false;
>+			}
>+
>+			burst_mode_ratio = DIV_ROUND_UP(
>+				mipi_config->target_burst_mode_freq * 100,
>+				computed_ddr);
>+
>+			pclk = DIV_ROUND_UP(pclk * burst_mode_ratio, 100);
>+		} else {
>+			DRM_ERROR("Burst mode target is not set\n");
>+			return false;
>+		}
>+	} else
>+		burst_mode_ratio = 100;
>+
>+	intel_dsi->burst_mode_ratio = burst_mode_ratio;
>+	intel_dsi->pclk = pclk;
>+
>+	bitrate = (pclk * bits_per_pixel) / intel_dsi->lane_count;
>+
> 	switch (intel_dsi->escape_clk_div) {
> 	case 0:
> 		tlpx_ns = 50;
>diff --git a/drivers/gpu/drm/i915/intel_dsi_pll.c b/drivers/gpu/drm/i915/intel_dsi_pll.c
>index d8bb1ea..fa7a6ca 100644
>--- a/drivers/gpu/drm/i915/intel_dsi_pll.c
>+++ b/drivers/gpu/drm/i915/intel_dsi_pll.c
>@@ -134,8 +134,7 @@ static u32 dsi_rr_formula(const struct drm_display_mode *mode,
> #else
> 
> /* Get DSI clock from pixel clock */
>-static u32 dsi_clk_from_pclk(const struct drm_display_mode *mode,
>-			  int pixel_format, int lane_count)
>+static u32 dsi_clk_from_pclk(u32 pclk, int pixel_format, int lane_count)
> {
> 	u32 dsi_clk_khz;
> 	u32 bpp;
>@@ -156,7 +155,7 @@ static u32 dsi_clk_from_pclk(const struct drm_display_mode *mode,
> 
> 	/* DSI data rate = pixel clock * bits per pixel / lane count
> 	   pixel clock is converted from KHz to Hz */
>-	dsi_clk_khz = DIV_ROUND_CLOSEST(mode->clock * bpp, lane_count);
>+	dsi_clk_khz = DIV_ROUND_CLOSEST(pclk * bpp, lane_count);
> 
> 	return dsi_clk_khz;
> }
>@@ -191,7 +190,7 @@ static int dsi_calc_mnp(u32 dsi_clk, struct dsi_mnp *dsi_mnp)
> 	for (m = 62; m <= 92; m++) {
> 		for (p = 2; p <= 6; p++) {
> 			/* Find the optimal m and p divisors
>-			with minimal error +/- the required clock */
>+			   with minimal error +/- the required clock */
> 			calc_dsi_clk = (m * ref_clk) / p;
> 			if (calc_dsi_clk == target_dsi_clk) {
> 				calc_m = m;
>@@ -228,15 +227,13 @@ static int dsi_calc_mnp(u32 dsi_clk, struct dsi_mnp *dsi_mnp)
> static void vlv_configure_dsi_pll(struct intel_encoder *encoder)
> {
> 	struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
>-	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
>-	const struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
> 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
> 	int ret;
> 	struct dsi_mnp dsi_mnp;
> 	u32 dsi_clk;
> 
>-	dsi_clk = dsi_clk_from_pclk(mode, intel_dsi->pixel_format,
>-						intel_dsi->lane_count);
>+	dsi_clk = dsi_clk_from_pclk(intel_dsi->pclk, intel_dsi->pixel_format,
>+				    intel_dsi->lane_count);
> 
> 	ret = dsi_calc_mnp(dsi_clk, &dsi_mnp);
> 	if (ret) {
>@@ -318,8 +315,8 @@ static void assert_bpp_mismatch(int pixel_format, int pipe_bpp)
> 	}
> 
> 	WARN(bpp != pipe_bpp,
>-		"bpp match assertion failure (expected %d, current %d)\n",
>-		bpp, pipe_bpp);
>+	     "bpp match assertion failure (expected %d, current %d)\n",
>+	     bpp, pipe_bpp);
> }
> 
> u32 vlv_get_dsi_pclk(struct intel_encoder *encoder, int pipe_bpp)
>diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
>index 56b47d2..e40e3df 100644
>--- a/drivers/gpu/drm/i915/intel_dvo.c
>+++ b/drivers/gpu/drm/i915/intel_dvo.c
>@@ -85,7 +85,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = {
> 	{
> 	        .type = INTEL_DVO_CHIP_TMDS,
> 		.name = "ns2501",
>-		.dvo_reg = DVOC,
>+		.dvo_reg = DVOB,
> 		.slave_addr = NS2501_ADDR,
> 		.dev_ops = &ns2501_ops,
>        }
>@@ -185,12 +185,13 @@ static void intel_enable_dvo(struct intel_encoder *encoder)
> 	u32 dvo_reg = intel_dvo->dev.dvo_reg;
> 	u32 temp = I915_READ(dvo_reg);
> 
>-	I915_WRITE(dvo_reg, temp | DVO_ENABLE);
>-	I915_READ(dvo_reg);
> 	intel_dvo->dev.dev_ops->mode_set(&intel_dvo->dev,
> 					 &crtc->config.requested_mode,
> 					 &crtc->config.adjusted_mode);
> 
>+	I915_WRITE(dvo_reg, temp | DVO_ENABLE);
>+	I915_READ(dvo_reg);
>+
> 	intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, true);
> }
> 
>@@ -226,10 +227,6 @@ static void intel_dvo_dpms(struct drm_connector *connector, int mode)
> 
> 		intel_crtc_update_dpms(crtc);
> 
>-		intel_dvo->dev.dev_ops->mode_set(&intel_dvo->dev,
>-						 &config->requested_mode,
>-						 &config->adjusted_mode);
>-
> 		intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, true);
> 	} else {
> 		intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, false);
>diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
>index f475414..9b584f3 100644
>--- a/drivers/gpu/drm/i915/intel_fbdev.c
>+++ b/drivers/gpu/drm/i915/intel_fbdev.c
>@@ -24,8 +24,10 @@
>  *     David Airlie
>  */
> 
>+#include <linux/async.h>
> #include <linux/module.h>
> #include <linux/kernel.h>
>+#include <linux/console.h>
> #include <linux/errno.h>
> #include <linux/string.h>
> #include <linux/mm.h>
>@@ -331,24 +333,6 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
> 	int num_connectors_enabled = 0;
> 	int num_connectors_detected = 0;
> 
>-	/*
>-	 * If the user specified any force options, just bail here
>-	 * and use that config.
>-	 */
>-	for (i = 0; i < fb_helper->connector_count; i++) {
>-		struct drm_fb_helper_connector *fb_conn;
>-		struct drm_connector *connector;
>-
>-		fb_conn = fb_helper->connector_info[i];
>-		connector = fb_conn->connector;
>-
>-		if (!enabled[i])
>-			continue;
>-
>-		if (connector->force != DRM_FORCE_UNSPECIFIED)
>-			return false;
>-	}
>-
> 	save_enabled = kcalloc(dev->mode_config.num_connector, sizeof(bool),
> 			       GFP_KERNEL);
> 	if (!save_enabled)
>@@ -374,8 +358,18 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
> 			continue;
> 		}
> 
>+		if (connector->force == DRM_FORCE_OFF) {
>+			DRM_DEBUG_KMS("connector %s is disabled by user, skipping\n",
>+				      connector->name);
>+			enabled[i] = false;
>+			continue;
>+		}
>+
> 		encoder = connector->encoder;
> 		if (!encoder || WARN_ON(!encoder->crtc)) {
>+			if (connector->force > DRM_FORCE_OFF)
>+				goto bail;
>+
> 			DRM_DEBUG_KMS("connector %s has no encoder or crtc, skipping\n",
> 				      connector->name);
> 			enabled[i] = false;
>@@ -394,8 +388,7 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
> 		for (j = 0; j < fb_helper->connector_count; j++) {
> 			if (crtcs[j] == new_crtc) {
> 				DRM_DEBUG_KMS("fallback: cloned configuration\n");
>-				fallback = true;
>-				goto out;
>+				goto bail;
> 			}
> 		}
> 
>@@ -466,8 +459,8 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
> 		fallback = true;
> 	}
> 
>-out:
> 	if (fallback) {
>+bail:
> 		DRM_DEBUG_KMS("Not using firmware configuration\n");
> 		memcpy(enabled, save_enabled, dev->mode_config.num_connector);
> 		kfree(save_enabled);
>@@ -636,6 +629,15 @@ out:
> 	return false;
> }
> 
>+static void intel_fbdev_suspend_worker(struct work_struct *work)
>+{
>+	intel_fbdev_set_suspend(container_of(work,
>+					     struct drm_i915_private,
>+					     fbdev_suspend_work)->dev,
>+				FBINFO_STATE_RUNNING,
>+				true);
>+}
>+
> int intel_fbdev_init(struct drm_device *dev)
> {
> 	struct intel_fbdev *ifbdev;
>@@ -662,14 +664,16 @@ int intel_fbdev_init(struct drm_device *dev)
> 	}
> 
> 	dev_priv->fbdev = ifbdev;
>+	INIT_WORK(&dev_priv->fbdev_suspend_work, intel_fbdev_suspend_worker);
>+
> 	drm_fb_helper_single_add_all_connectors(&ifbdev->helper);
> 
> 	return 0;
> }
> 
>-void intel_fbdev_initial_config(struct drm_device *dev)
>+void intel_fbdev_initial_config(void *data, async_cookie_t cookie)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = data;
> 	struct intel_fbdev *ifbdev = dev_priv->fbdev;
> 
> 	/* Due to peculiar init order wrt to hpd handling this is separate. */
>@@ -682,12 +686,15 @@ void intel_fbdev_fini(struct drm_device *dev)
> 	if (!dev_priv->fbdev)
> 		return;
> 
>+	flush_work(&dev_priv->fbdev_suspend_work);
>+
>+	async_synchronize_full();
> 	intel_fbdev_destroy(dev, dev_priv->fbdev);
> 	kfree(dev_priv->fbdev);
> 	dev_priv->fbdev = NULL;
> }
> 
>-void intel_fbdev_set_suspend(struct drm_device *dev, int state)
>+void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_fbdev *ifbdev = dev_priv->fbdev;
>@@ -698,6 +705,33 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state)
> 
> 	info = ifbdev->helper.fbdev;
> 
>+	if (synchronous) {
>+		/* Flush any pending work to turn the console on, and then
>+		 * wait to turn it off. It must be synchronous as we are
>+		 * about to suspend or unload the driver.
>+		 *
>+		 * Note that from within the work-handler, we cannot flush
>+		 * ourselves, so only flush outstanding work upon suspend!
>+		 */
>+		if (state != FBINFO_STATE_RUNNING)
>+			flush_work(&dev_priv->fbdev_suspend_work);
>+		console_lock();
>+	} else {
>+		/*
>+		 * The console lock can be pretty contented on resume due
>+		 * to all the printk activity.  Try to keep it out of the hot
>+		 * path of resume if possible.
>+		 */
>+		WARN_ON(state != FBINFO_STATE_RUNNING);
>+		if (!console_trylock()) {
>+			/* Don't block our own workqueue as this can
>+			 * be run in parallel with other i915.ko tasks.
>+			 */
>+			schedule_work(&dev_priv->fbdev_suspend_work);
>+			return;
>+		}
>+	}
>+
> 	/* On resume from hibernation: If the object is shmemfs backed, it has
> 	 * been restored from swap. If the object is stolen however, it will be
> 	 * full of whatever garbage was left in there.
>@@ -706,6 +740,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state)
> 		memset_io(info->screen_base, 0, info->screen_size);
> 
> 	fb_set_suspend(info, state);
>+	console_unlock();
> }
> 
> void intel_fbdev_output_poll_changed(struct drm_device *dev)
>diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
>index 5a9de21..1f61303 100644
>--- a/drivers/gpu/drm/i915/intel_hdmi.c
>+++ b/drivers/gpu/drm/i915/intel_hdmi.c
>@@ -732,7 +732,7 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder,
> 	if (tmp & HDMI_MODE_SELECT_HDMI)
> 		pipe_config->has_hdmi_sink = true;
> 
>-	if (tmp & SDVO_AUDIO_ENABLE)
>+	if (tmp & HDMI_MODE_SELECT_HDMI)
> 		pipe_config->has_audio = true;
> 
> 	if (!HAS_PCH_SPLIT(dev) &&
>@@ -869,10 +869,15 @@ static enum drm_mode_status
> intel_hdmi_mode_valid(struct drm_connector *connector,
> 		      struct drm_display_mode *mode)
> {
>-	if (mode->clock > hdmi_portclock_limit(intel_attached_hdmi(connector),
>-					       true))
>+	int clock = mode->clock;
>+
>+	if (mode->flags & DRM_MODE_FLAG_DBLCLK)
>+		clock *= 2;
>+
>+	if (clock > hdmi_portclock_limit(intel_attached_hdmi(connector),
>+					 true))
> 		return MODE_CLOCK_HIGH;
>-	if (mode->clock < 20000)
>+	if (clock < 20000)
> 		return MODE_CLOCK_LOW;
> 
> 	if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
>@@ -890,7 +895,7 @@ static bool hdmi_12bpc_possible(struct intel_crtc *crtc)
> 	if (HAS_GMCH_DISPLAY(dev))
> 		return false;
> 
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		if (encoder->new_crtc != crtc)
> 			continue;
> 
>@@ -926,6 +931,10 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
> 			intel_hdmi->color_range = 0;
> 	}
> 
>+	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK) {
>+		pipe_config->pixel_multiplier = 2;
>+	}
>+
> 	if (intel_hdmi->color_range)
> 		pipe_config->limited_color_range = true;
> 
>@@ -967,104 +976,117 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
> 	return true;
> }
> 
>-static enum drm_connector_status
>-intel_hdmi_detect(struct drm_connector *connector, bool force)
>+static void
>+intel_hdmi_unset_edid(struct drm_connector *connector)
> {
>-	struct drm_device *dev = connector->dev;
> 	struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
>-	struct intel_digital_port *intel_dig_port =
>-		hdmi_to_dig_port(intel_hdmi);
>-	struct intel_encoder *intel_encoder = &intel_dig_port->base;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct edid *edid;
>-	enum intel_display_power_domain power_domain;
>-	enum drm_connector_status status = connector_status_disconnected;
> 
>-	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
>-		      connector->base.id, connector->name);
>+	intel_hdmi->has_hdmi_sink = false;
>+	intel_hdmi->has_audio = false;
>+	intel_hdmi->rgb_quant_range_selectable = false;
>+
>+	kfree(to_intel_connector(connector)->detect_edid);
>+	to_intel_connector(connector)->detect_edid = NULL;
>+}
>+
>+static bool
>+intel_hdmi_set_edid(struct drm_connector *connector)
>+{
>+	struct drm_i915_private *dev_priv = to_i915(connector->dev);
>+	struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
>+	struct intel_encoder *intel_encoder =
>+		&hdmi_to_dig_port(intel_hdmi)->base;
>+	enum intel_display_power_domain power_domain;
>+	struct edid *edid;
>+	bool connected = false;
> 
> 	power_domain = intel_display_port_power_domain(intel_encoder);
> 	intel_display_power_get(dev_priv, power_domain);
> 
>-	intel_hdmi->has_hdmi_sink = false;
>-	intel_hdmi->has_audio = false;
>-	intel_hdmi->rgb_quant_range_selectable = false;
> 	edid = drm_get_edid(connector,
> 			    intel_gmbus_get_adapter(dev_priv,
> 						    intel_hdmi->ddc_bus));
> 
>-	if (edid) {
>-		if (edid->input & DRM_EDID_INPUT_DIGITAL) {
>-			status = connector_status_connected;
>-			if (intel_hdmi->force_audio != HDMI_AUDIO_OFF_DVI)
>-				intel_hdmi->has_hdmi_sink =
>-						drm_detect_hdmi_monitor(edid);
>-			intel_hdmi->has_audio = drm_detect_monitor_audio(edid);
>-			intel_hdmi->rgb_quant_range_selectable =
>-				drm_rgb_quant_range_selectable(edid);
>-		}
>-		kfree(edid);
>-	}
>+	intel_display_power_put(dev_priv, power_domain);
>+
>+	to_intel_connector(connector)->detect_edid = edid;
>+	if (edid && edid->input & DRM_EDID_INPUT_DIGITAL) {
>+		intel_hdmi->rgb_quant_range_selectable =
>+			drm_rgb_quant_range_selectable(edid);
> 
>-	if (status == connector_status_connected) {
>+		intel_hdmi->has_audio = drm_detect_monitor_audio(edid);
> 		if (intel_hdmi->force_audio != HDMI_AUDIO_AUTO)
> 			intel_hdmi->has_audio =
>-				(intel_hdmi->force_audio == HDMI_AUDIO_ON);
>-		intel_encoder->type = INTEL_OUTPUT_HDMI;
>+				intel_hdmi->force_audio == HDMI_AUDIO_ON;
>+
>+		if (intel_hdmi->force_audio != HDMI_AUDIO_OFF_DVI)
>+			intel_hdmi->has_hdmi_sink =
>+				drm_detect_hdmi_monitor(edid);
>+
>+		connected = true;
> 	}
> 
>-	intel_display_power_put(dev_priv, power_domain);
>+	return connected;
>+}
>+
>+static enum drm_connector_status
>+intel_hdmi_detect(struct drm_connector *connector, bool force)
>+{
>+	enum drm_connector_status status;
>+
>+	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
>+		      connector->base.id, connector->name);
>+
>+	intel_hdmi_unset_edid(connector);
>+
>+	if (intel_hdmi_set_edid(connector)) {
>+		struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
>+
>+		hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI;
>+		status = connector_status_connected;
>+	} else
>+		status = connector_status_disconnected;
> 
> 	return status;
> }
> 
>-static int intel_hdmi_get_modes(struct drm_connector *connector)
>+static void
>+intel_hdmi_force(struct drm_connector *connector)
> {
>-	struct intel_encoder *intel_encoder = intel_attached_encoder(connector);
>-	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base);
>-	struct drm_i915_private *dev_priv = connector->dev->dev_private;
>-	enum intel_display_power_domain power_domain;
>-	int ret;
>+	struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
> 
>-	/* We should parse the EDID data and find out if it's an HDMI sink so
>-	 * we can send audio to it.
>-	 */
>+	DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
>+		      connector->base.id, connector->name);
> 
>-	power_domain = intel_display_port_power_domain(intel_encoder);
>-	intel_display_power_get(dev_priv, power_domain);
>+	intel_hdmi_unset_edid(connector);
> 
>-	ret = intel_ddc_get_modes(connector,
>-				   intel_gmbus_get_adapter(dev_priv,
>-							   intel_hdmi->ddc_bus));
>+	if (connector->status != connector_status_connected)
>+		return;
> 
>-	intel_display_power_put(dev_priv, power_domain);
>+	intel_hdmi_set_edid(connector);
>+	hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI;
>+}
> 
>-	return ret;
>+static int intel_hdmi_get_modes(struct drm_connector *connector)
>+{
>+	struct edid *edid;
>+
>+	edid = to_intel_connector(connector)->detect_edid;
>+	if (edid == NULL)
>+		return 0;
>+
>+	return intel_connector_update_modes(connector, edid);
> }
> 
> static bool
> intel_hdmi_detect_audio(struct drm_connector *connector)
> {
>-	struct intel_encoder *intel_encoder = intel_attached_encoder(connector);
>-	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base);
>-	struct drm_i915_private *dev_priv = connector->dev->dev_private;
>-	enum intel_display_power_domain power_domain;
>-	struct edid *edid;
> 	bool has_audio = false;
>+	struct edid *edid;
> 
>-	power_domain = intel_display_port_power_domain(intel_encoder);
>-	intel_display_power_get(dev_priv, power_domain);
>-
>-	edid = drm_get_edid(connector,
>-			    intel_gmbus_get_adapter(dev_priv,
>-						    intel_hdmi->ddc_bus));
>-	if (edid) {
>-		if (edid->input & DRM_EDID_INPUT_DIGITAL)
>-			has_audio = drm_detect_monitor_audio(edid);
>-		kfree(edid);
>-	}
>-
>-	intel_display_power_put(dev_priv, power_domain);
>+	edid = to_intel_connector(connector)->detect_edid;
>+	if (edid && edid->input & DRM_EDID_INPUT_DIGITAL)
>+		has_audio = drm_detect_monitor_audio(edid);
> 
> 	return has_audio;
> }
>@@ -1265,6 +1287,8 @@ static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder)
> 	enum pipe pipe = intel_crtc->pipe;
> 	u32 val;
> 
>+	intel_hdmi_prepare(encoder);
>+
> 	mutex_lock(&dev_priv->dpio_lock);
> 
> 	/* program left/right clock distribution */
>@@ -1434,8 +1458,8 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder)
> 
> 	for (i = 0; i < 4; i++) {
> 		val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i));
>-		val &= ~DPIO_SWING_MARGIN_MASK;
>-		val |= 102 << DPIO_SWING_MARGIN_SHIFT;
>+		val &= ~DPIO_SWING_MARGIN000_MASK;
>+		val |= 102 << DPIO_SWING_MARGIN000_SHIFT;
> 		vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val);
> 	}
> 
>@@ -1482,6 +1506,7 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder)
> 
> static void intel_hdmi_destroy(struct drm_connector *connector)
> {
>+	kfree(to_intel_connector(connector)->detect_edid);
> 	drm_connector_cleanup(connector);
> 	kfree(connector);
> }
>@@ -1489,6 +1514,7 @@ static void intel_hdmi_destroy(struct drm_connector *connector)
> static const struct drm_connector_funcs intel_hdmi_connector_funcs = {
> 	.dpms = intel_connector_dpms,
> 	.detect = intel_hdmi_detect,
>+	.force = intel_hdmi_force,
> 	.fill_modes = drm_helper_probe_single_connector_modes,
> 	.set_property = intel_hdmi_set_property,
> 	.destroy = intel_hdmi_destroy,
>diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
>index fdf4026..a6bd142 100644
>--- a/drivers/gpu/drm/i915/intel_lvds.c
>+++ b/drivers/gpu/drm/i915/intel_lvds.c
>@@ -823,8 +823,7 @@ bool intel_is_dual_link_lvds(struct drm_device *dev)
> 	struct intel_encoder *encoder;
> 	struct intel_lvds_encoder *lvds_encoder;
> 
>-	list_for_each_entry(encoder, &dev->mode_config.encoder_list,
>-			    base.head) {
>+	for_each_intel_encoder(dev, encoder) {
> 		if (encoder->type == INTEL_OUTPUT_LVDS) {
> 			lvds_encoder = to_lvds_encoder(&encoder->base);
> 
>diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
>index d8de1d5..ca52ad2 100644
>--- a/drivers/gpu/drm/i915/intel_opregion.c
>+++ b/drivers/gpu/drm/i915/intel_opregion.c
>@@ -396,16 +396,6 @@ int intel_opregion_notify_adapter(struct drm_device *dev, pci_power_t state)
> 	return -EINVAL;
> }
> 
>-/*
>- * If the vendor backlight interface is not in use and ACPI backlight interface
>- * is broken, do not bother processing backlight change requests from firmware.
>- */
>-static bool should_ignore_backlight_request(void)
>-{
>-	return acpi_video_backlight_support() &&
>-	       !acpi_video_verify_backlight_support();
>-}
>-
> static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>@@ -414,7 +404,11 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
> 
> 	DRM_DEBUG_DRIVER("bclp = 0x%08x\n", bclp);
> 
>-	if (should_ignore_backlight_request()) {
>+	/*
>+	 * If the acpi_video interface is not supposed to be used, don't
>+	 * bother processing backlight level change requests from firmware.
>+	 */
>+	if (!acpi_video_verify_backlight_support()) {
> 		DRM_DEBUG_KMS("opregion backlight request ignored\n");
> 		return 0;
> 	}
>diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
>index dc2f4f2..ae0e577 100644
>--- a/drivers/gpu/drm/i915/intel_overlay.c
>+++ b/drivers/gpu/drm/i915/intel_overlay.c
>@@ -182,7 +182,7 @@ struct intel_overlay {
> 	u32 flip_addr;
> 	struct drm_i915_gem_object *reg_bo;
> 	/* flip handling */
>-	uint32_t last_flip_req;
>+	struct i915_gem_request *flip_request;
> 	void (*flip_tail)(struct intel_overlay *);
> };
> 
>@@ -208,53 +208,86 @@ static void intel_overlay_unmap_regs(struct intel_overlay *overlay,
> 		io_mapping_unmap(regs);
> }
> 
>-static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
>-					 void (*tail)(struct intel_overlay *))
>+/* recover from an interruption due to a signal
>+ * We have to be careful not to repeat work forever an make forward progess. */
>+static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay)
> {
>-	struct drm_device *dev = overlay->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
> 	int ret;
> 
>-	BUG_ON(overlay->last_flip_req);
>-	ret = i915_add_request(ring, &overlay->last_flip_req);
>-	if (ret)
>-		return ret;
>+	if (overlay->flip_request == NULL)
>+		return 0;
> 
>-	overlay->flip_tail = tail;
>-	ret = i915_wait_seqno(ring, overlay->last_flip_req);
>+	ret = i915_request_wait(overlay->flip_request);
> 	if (ret)
> 		return ret;
>-	i915_gem_retire_requests(dev);
> 
>-	overlay->last_flip_req = 0;
>+	i915_request_put(overlay->flip_request);
>+	overlay->flip_request = NULL;
>+
>+	i915_gem_retire_requests(overlay->dev);
>+
>+	if (overlay->flip_tail)
>+		overlay->flip_tail(overlay);
>+
> 	return 0;
> }
> 
>+static int intel_overlay_add_request(struct intel_overlay *overlay,
>+				     struct i915_gem_request *rq,
>+				     void (*tail)(struct intel_overlay *))
>+{
>+	BUG_ON(overlay->flip_request);
>+	overlay->flip_request = rq;
>+	overlay->flip_tail = tail;
>+
>+	return i915_request_commit(rq);
>+}
>+
>+static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
>+					 struct i915_gem_request *rq,
>+					 void (*tail)(struct intel_overlay *))
>+{
>+	intel_overlay_add_request(overlay, rq, tail);
>+	return intel_overlay_recover_from_interrupt(overlay);
>+}
>+
>+static struct i915_gem_request *
>+intel_overlay_alloc_request(struct intel_overlay *overlay)
>+{
>+	struct drm_i915_private *i915 = to_i915(overlay->dev);
>+	return intel_engine_alloc_request(RCS_ENGINE(i915),
>+					  RCS_ENGINE(i915)->default_context);
>+}
>+
> /* overlay needs to be disable in OCMD reg */
> static int intel_overlay_on(struct intel_overlay *overlay)
> {
> 	struct drm_device *dev = overlay->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
>-	int ret;
>+	struct i915_gem_request *rq;
>+	struct intel_ringbuffer *ring;
> 
> 	BUG_ON(overlay->active);
> 	overlay->active = 1;
> 
> 	WARN_ON(IS_I830(dev) && !(dev_priv->quirks & QUIRK_PIPEA_FORCE));
> 
>-	ret = intel_ring_begin(ring, 4);
>-	if (ret)
>-		return ret;
>+	rq = intel_overlay_alloc_request(overlay);
>+	if (IS_ERR(rq))
>+		return PTR_ERR(rq);
>+
>+	ring = intel_ring_begin(rq, 3);
>+	if (IS_ERR(ring)) {
>+		i915_request_put(rq);
>+		return PTR_ERR(ring);
>+	}
> 
> 	intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_ON);
> 	intel_ring_emit(ring, overlay->flip_addr | OFC_UPDATE);
> 	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
>-	intel_ring_emit(ring, MI_NOOP);
> 	intel_ring_advance(ring);
> 
>-	return intel_overlay_do_wait_request(overlay, NULL);
>+	return intel_overlay_do_wait_request(overlay, rq, NULL);
> }
> 
> /* overlay needs to be enabled in OCMD reg */
>@@ -263,10 +296,10 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
> {
> 	struct drm_device *dev = overlay->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
> 	u32 flip_addr = overlay->flip_addr;
>+	struct i915_gem_request *rq;
>+	struct intel_ringbuffer *ring;
> 	u32 tmp;
>-	int ret;
> 
> 	BUG_ON(!overlay->active);
> 
>@@ -278,21 +311,30 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
> 	if (tmp & (1 << 17))
> 		DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp);
> 
>-	ret = intel_ring_begin(ring, 2);
>-	if (ret)
>-		return ret;
>+	rq = intel_overlay_alloc_request(overlay);
>+	if (IS_ERR(rq))
>+		return PTR_ERR(rq);
>+
>+	ring = intel_ring_begin(rq, 2);
>+	if (IS_ERR(ring)) {
>+		i915_request_put(rq);
>+		return PTR_ERR(ring);
>+	}
> 
> 	intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
> 	intel_ring_emit(ring, flip_addr);
> 	intel_ring_advance(ring);
> 
>-	return i915_add_request(ring, &overlay->last_flip_req);
>+	return intel_overlay_add_request(overlay, rq, NULL);
> }
> 
> static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay)
> {
> 	struct drm_i915_gem_object *obj = overlay->old_vid_bo;
> 
>+	i915_gem_track_fb(obj, NULL,
>+			  INTEL_FRONTBUFFER_OVERLAY(overlay->crtc->pipe));
>+
> 	i915_gem_object_ggtt_unpin(obj);
> 	drm_gem_object_unreference(&obj->base);
> 
>@@ -319,10 +361,10 @@ static void intel_overlay_off_tail(struct intel_overlay *overlay)
> static int intel_overlay_off(struct intel_overlay *overlay)
> {
> 	struct drm_device *dev = overlay->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
> 	u32 flip_addr = overlay->flip_addr;
>-	int ret;
>+	struct i915_gem_request *rq;
>+	struct intel_ringbuffer *ring;
>+	int len;
> 
> 	BUG_ON(!overlay->active);
> 
>@@ -332,53 +374,36 @@ static int intel_overlay_off(struct intel_overlay *overlay)
> 	 * of the hw. Do it in both cases */
> 	flip_addr |= OFC_UPDATE;
> 
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret)
>-		return ret;
>+	rq = intel_overlay_alloc_request(overlay);
>+	if (IS_ERR(rq))
>+		return PTR_ERR(rq);
>+
>+	len = 3;
>+	if (!IS_I830(dev))
>+		len += 3;
>+
>+	ring = intel_ring_begin(rq, len);
>+	if (IS_ERR(ring)) {
>+		i915_request_put(rq);
>+		return PTR_ERR(ring);
>+	}
> 
> 	/* wait for overlay to go idle */
> 	intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
> 	intel_ring_emit(ring, flip_addr);
> 	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
>-	/* turn overlay off */
>-	if (IS_I830(dev)) {
>-		/* Workaround: Don't disable the overlay fully, since otherwise
>-		 * it dies on the next OVERLAY_ON cmd. */
>-		intel_ring_emit(ring, MI_NOOP);
>-		intel_ring_emit(ring, MI_NOOP);
>-		intel_ring_emit(ring, MI_NOOP);
>-	} else {
>+	/* turn overlay off
>+	 * Workaround for i830: Don't disable the overlay fully, since
>+	 * otherwise it dies on the next OVERLAY_ON cmd.
>+	 */
>+	if (!IS_I830(dev)) {
> 		intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_OFF);
> 		intel_ring_emit(ring, flip_addr);
> 		intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
> 	}
> 	intel_ring_advance(ring);
> 
>-	return intel_overlay_do_wait_request(overlay, intel_overlay_off_tail);
>-}
>-
>-/* recover from an interruption due to a signal
>- * We have to be careful not to repeat work forever an make forward progess. */
>-static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay)
>-{
>-	struct drm_device *dev = overlay->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
>-	int ret;
>-
>-	if (overlay->last_flip_req == 0)
>-		return 0;
>-
>-	ret = i915_wait_seqno(ring, overlay->last_flip_req);
>-	if (ret)
>-		return ret;
>-	i915_gem_retire_requests(dev);
>-
>-	if (overlay->flip_tail)
>-		overlay->flip_tail(overlay);
>-
>-	overlay->last_flip_req = 0;
>-	return 0;
>+	return intel_overlay_do_wait_request(overlay, rq, intel_overlay_off_tail);
> }
> 
> /* Wait for pending overlay flip and release old frame.
>@@ -387,10 +412,8 @@ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay)
>  */
> static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
> {
>-	struct drm_device *dev = overlay->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
>-	int ret;
>+	struct drm_i915_private *dev_priv = to_i915(overlay->dev);
>+	int ret = 0;
> 
> 	/* Only wait if there is actually an old frame to release to
> 	 * guarantee forward progress.
>@@ -399,27 +422,29 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
> 		return 0;
> 
> 	if (I915_READ(ISR) & I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT) {
>+		struct i915_gem_request *rq;
>+		struct intel_ringbuffer *ring;
>+
>+		rq = intel_overlay_alloc_request(overlay);
>+		if (IS_ERR(rq))
>+			return PTR_ERR(rq);
>+
> 		/* synchronous slowpath */
>-		ret = intel_ring_begin(ring, 2);
>-		if (ret)
>-			return ret;
>+		ring = intel_ring_begin(rq, 1);
>+		if (IS_ERR(ring)) {
>+			i915_request_put(rq);
>+			return PTR_ERR(ring);
>+		}
> 
> 		intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
>-		intel_ring_emit(ring, MI_NOOP);
> 		intel_ring_advance(ring);
> 
>-		ret = intel_overlay_do_wait_request(overlay,
>+		ret = intel_overlay_do_wait_request(overlay, rq,
> 						    intel_overlay_release_old_vid_tail);
>-		if (ret)
>-			return ret;
>-	}
>-
>-	intel_overlay_release_old_vid_tail(overlay);
>-
>+	} else
>+		intel_overlay_release_old_vid_tail(overlay);
> 
>-	i915_gem_track_fb(overlay->old_vid_bo, NULL,
>-			  INTEL_FRONTBUFFER_OVERLAY(overlay->crtc->pipe));
>-	return 0;
>+	return ret;
> }
> 
> struct put_image_params {
>@@ -821,12 +846,7 @@ int intel_overlay_switch_off(struct intel_overlay *overlay)
> 	iowrite32(0, &regs->OCMD);
> 	intel_overlay_unmap_regs(overlay, regs);
> 
>-	ret = intel_overlay_off(overlay);
>-	if (ret != 0)
>-		return ret;
>-
>-	intel_overlay_off_tail(overlay);
>-	return 0;
>+	return intel_overlay_off(overlay);
> }
> 
> static int check_overlay_possible_on_crtc(struct intel_overlay *overlay,
>diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
>index 8e37444..f17ada3 100644
>--- a/drivers/gpu/drm/i915/intel_panel.c
>+++ b/drivers/gpu/drm/i915/intel_panel.c
>@@ -538,14 +538,13 @@ static u32 intel_panel_get_backlight(struct intel_connector *connector)
> 	struct drm_device *dev = connector->base.dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	u32 val;
>-	unsigned long flags;
> 
>-	spin_lock_irqsave(&dev_priv->backlight_lock, flags);
>+	mutex_lock(&dev_priv->backlight_lock);
> 
> 	val = dev_priv->display.get_backlight(connector);
> 	val = intel_panel_compute_brightness(connector, val);
> 
>-	spin_unlock_irqrestore(&dev_priv->backlight_lock, flags);
>+	mutex_unlock(&dev_priv->backlight_lock);
> 
> 	DRM_DEBUG_DRIVER("get backlight PWM = %d\n", val);
> 	return val;
>@@ -629,12 +628,11 @@ static void intel_panel_set_backlight(struct intel_connector *connector,
> 	struct intel_panel *panel = &connector->panel;
> 	enum pipe pipe = intel_get_pipe_from_connector(connector);
> 	u32 hw_level;
>-	unsigned long flags;
> 
> 	if (!panel->backlight.present || pipe == INVALID_PIPE)
> 		return;
> 
>-	spin_lock_irqsave(&dev_priv->backlight_lock, flags);
>+	mutex_lock(&dev_priv->backlight_lock);
> 
> 	WARN_ON(panel->backlight.max == 0);
> 
>@@ -644,7 +642,7 @@ static void intel_panel_set_backlight(struct intel_connector *connector,
> 	if (panel->backlight.enabled)
> 		intel_panel_actually_set_backlight(connector, hw_level);
> 
>-	spin_unlock_irqrestore(&dev_priv->backlight_lock, flags);
>+	mutex_unlock(&dev_priv->backlight_lock);
> }
> 
> /* set backlight brightness to level in range [0..max], assuming hw min is
>@@ -658,12 +656,11 @@ void intel_panel_set_backlight_acpi(struct intel_connector *connector,
> 	struct intel_panel *panel = &connector->panel;
> 	enum pipe pipe = intel_get_pipe_from_connector(connector);
> 	u32 hw_level;
>-	unsigned long flags;
> 
> 	if (!panel->backlight.present || pipe == INVALID_PIPE)
> 		return;
> 
>-	spin_lock_irqsave(&dev_priv->backlight_lock, flags);
>+	mutex_lock(&dev_priv->backlight_lock);
> 
> 	WARN_ON(panel->backlight.max == 0);
> 
>@@ -679,7 +676,7 @@ void intel_panel_set_backlight_acpi(struct intel_connector *connector,
> 	if (panel->backlight.enabled)
> 		intel_panel_actually_set_backlight(connector, hw_level);
> 
>-	spin_unlock_irqrestore(&dev_priv->backlight_lock, flags);
>+	mutex_unlock(&dev_priv->backlight_lock);
> }
> 
> static void pch_disable_backlight(struct intel_connector *connector)
>@@ -733,7 +730,6 @@ void intel_panel_disable_backlight(struct intel_connector *connector)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_panel *panel = &connector->panel;
> 	enum pipe pipe = intel_get_pipe_from_connector(connector);
>-	unsigned long flags;
> 
> 	if (!panel->backlight.present || pipe == INVALID_PIPE)
> 		return;
>@@ -749,12 +745,14 @@ void intel_panel_disable_backlight(struct intel_connector *connector)
> 		return;
> 	}
> 
>-	spin_lock_irqsave(&dev_priv->backlight_lock, flags);
>+	mutex_lock(&dev_priv->backlight_lock);
> 
>+	if (panel->backlight.device)
>+		panel->backlight.device->props.power = FB_BLANK_POWERDOWN;
> 	panel->backlight.enabled = false;
> 	dev_priv->display.disable_backlight(connector);
> 
>-	spin_unlock_irqrestore(&dev_priv->backlight_lock, flags);
>+	mutex_unlock(&dev_priv->backlight_lock);
> }
> 
> static void bdw_enable_backlight(struct intel_connector *connector)
>@@ -935,14 +933,13 @@ void intel_panel_enable_backlight(struct intel_connector *connector)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_panel *panel = &connector->panel;
> 	enum pipe pipe = intel_get_pipe_from_connector(connector);
>-	unsigned long flags;
> 
> 	if (!panel->backlight.present || pipe == INVALID_PIPE)
> 		return;
> 
> 	DRM_DEBUG_KMS("pipe %c\n", pipe_name(pipe));
> 
>-	spin_lock_irqsave(&dev_priv->backlight_lock, flags);
>+	mutex_lock(&dev_priv->backlight_lock);
> 
> 	WARN_ON(panel->backlight.max == 0);
> 
>@@ -957,14 +954,17 @@ void intel_panel_enable_backlight(struct intel_connector *connector)
> 
> 	dev_priv->display.enable_backlight(connector);
> 	panel->backlight.enabled = true;
>+	if (panel->backlight.device)
>+		panel->backlight.device->props.power = FB_BLANK_UNBLANK;
> 
>-	spin_unlock_irqrestore(&dev_priv->backlight_lock, flags);
>+	mutex_unlock(&dev_priv->backlight_lock);
> }
> 
> #if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE)
> static int intel_backlight_device_update_status(struct backlight_device *bd)
> {
> 	struct intel_connector *connector = bl_get_data(bd);
>+	struct intel_panel *panel = &connector->panel;
> 	struct drm_device *dev = connector->base.dev;
> 
> 	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
>@@ -972,6 +972,23 @@ static int intel_backlight_device_update_status(struct backlight_device *bd)
> 		      bd->props.brightness, bd->props.max_brightness);
> 	intel_panel_set_backlight(connector, bd->props.brightness,
> 				  bd->props.max_brightness);
>+
>+	/*
>+	 * Allow flipping bl_power as a sub-state of enabled. Sadly the
>+	 * backlight class device does not make it easy to to differentiate
>+	 * between callbacks for brightness and bl_power, so our backlight_power
>+	 * callback needs to take this into account.
>+	 */
>+	if (panel->backlight.enabled) {
>+		if (panel->backlight_power) {
>+			bool enable = bd->props.power == FB_BLANK_UNBLANK &&
>+				bd->props.brightness != 0;
>+			panel->backlight_power(connector, enable);
>+		}
>+	} else {
>+		bd->props.power = FB_BLANK_POWERDOWN;
>+	}
>+
> 	drm_modeset_unlock(&dev->mode_config.connection_mutex);
> 	return 0;
> }
>@@ -1023,6 +1040,11 @@ static int intel_backlight_device_register(struct intel_connector *connector)
> 					    panel->backlight.level,
> 					    props.max_brightness);
> 
>+	if (panel->backlight.enabled)
>+		props.power = FB_BLANK_UNBLANK;
>+	else
>+		props.power = FB_BLANK_POWERDOWN;
>+
> 	/*
> 	 * Note: using the same name independent of the connector prevents
> 	 * registration of multiple backlight devices in the driver.
>@@ -1203,7 +1225,7 @@ static int vlv_setup_backlight(struct intel_connector *connector)
> 	enum pipe pipe;
> 	u32 ctl, ctl2, val;
> 
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		u32 cur_val = I915_READ(VLV_BLC_PWM_CTL(pipe));
> 
> 		/* Skip if the modulation freq is already set */
>@@ -1240,7 +1262,6 @@ int intel_panel_setup_backlight(struct drm_connector *connector)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	struct intel_connector *intel_connector = to_intel_connector(connector);
> 	struct intel_panel *panel = &intel_connector->panel;
>-	unsigned long flags;
> 	int ret;
> 
> 	if (!dev_priv->vbt.backlight.present) {
>@@ -1253,9 +1274,9 @@ int intel_panel_setup_backlight(struct drm_connector *connector)
> 	}
> 
> 	/* set level and max in panel struct */
>-	spin_lock_irqsave(&dev_priv->backlight_lock, flags);
>+	mutex_lock(&dev_priv->backlight_lock);
> 	ret = dev_priv->display.setup_backlight(intel_connector);
>-	spin_unlock_irqrestore(&dev_priv->backlight_lock, flags);
>+	mutex_unlock(&dev_priv->backlight_lock);
> 
> 	if (ret) {
> 		DRM_DEBUG_KMS("failed to setup backlight for connector %s\n",
>diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
>index 40c1229..490bd69 100644
>--- a/drivers/gpu/drm/i915/intel_pm.c
>+++ b/drivers/gpu/drm/i915/intel_pm.c
>@@ -309,6 +309,9 @@ static void gen7_enable_fbc(struct drm_crtc *crtc)
> 
> 	dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN;
> 
>+	if (dev_priv->fbc.false_color)
>+		dpfc_ctl |= FBC_CTL_FALSE_COLOR;
>+
> 	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
> 
> 	if (IS_IVYBRIDGE(dev)) {
>@@ -336,12 +339,31 @@ bool intel_fbc_enabled(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 
>+	/* If it wasn't never enabled by kernel parameter or platform default
>+	 * we can avoid reading registers so many times in vain
>+	 */
>+	if (!i915.enable_fbc)
>+		return false;
>+
> 	if (!dev_priv->display.fbc_enabled)
> 		return false;
> 
> 	return dev_priv->display.fbc_enabled(dev);
> }
> 
>+void gen8_fbc_sw_flush(struct drm_device *dev, u32 value)
>+{
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+
>+	if (!IS_GEN8(dev))
>+		return;
>+
>+	if (!intel_fbc_enabled(dev))
>+		return;
>+
>+	I915_WRITE(MSG_FBC_REND_STATE, value);
>+}
>+
> static void intel_fbc_work_fn(struct work_struct *__work)
> {
> 	struct intel_fbc_work *work =
>@@ -578,6 +600,12 @@ void intel_update_fbc(struct drm_device *dev)
> 			DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
> 		goto out_disable;
> 	}
>+	if (INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) &&
>+	    to_intel_plane(crtc->primary)->rotation != BIT(DRM_ROTATE_0)) {
>+		if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE))
>+			DRM_DEBUG_KMS("Rotation unsupported, disabling\n");
>+		goto out_disable;
>+	}
> 
> 	/* If the kernel debugger is active, always disable compression */
> 	if (in_dbg_master())
>@@ -853,7 +881,7 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
>  * A value of 5us seems to be a good balance; safe for very low end
>  * platforms but not overly aggressive on lower latency configs.
>  */
>-static const int latency_ns = 5000;
>+static const int pessimal_latency_ns = 5000;
> 
> static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
> {
>@@ -982,13 +1010,20 @@ static const struct intel_watermark_params i915_wm_info = {
> 	.guard_size = 2,
> 	.cacheline_size = I915_FIFO_LINE_SIZE,
> };
>-static const struct intel_watermark_params i830_wm_info = {
>+static const struct intel_watermark_params i830_a_wm_info = {
> 	.fifo_size = I855GM_FIFO_SIZE,
> 	.max_wm = I915_MAX_WM,
> 	.default_wm = 1,
> 	.guard_size = 2,
> 	.cacheline_size = I830_FIFO_LINE_SIZE,
> };
>+static const struct intel_watermark_params i830_bc_wm_info = {
>+	.fifo_size = I855GM_FIFO_SIZE,
>+	.max_wm = I915_MAX_WM/2,
>+	.default_wm = 1,
>+	.guard_size = 2,
>+	.cacheline_size = I830_FIFO_LINE_SIZE,
>+};
> static const struct intel_watermark_params i845_wm_info = {
> 	.fifo_size = I830_FIFO_SIZE,
> 	.max_wm = I915_MAX_WM,
>@@ -1044,6 +1079,17 @@ static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
> 		wm_size = wm->max_wm;
> 	if (wm_size <= 0)
> 		wm_size = wm->default_wm;
>+
>+	/*
>+	 * Bspec seems to indicate that the value shouldn't be lower than
>+	 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
>+	 * Lets go for 8 which is the burst size since certain platforms
>+	 * already use a hardcoded 8 (which is what the spec says should be
>+	 * done).
>+	 */
>+	if (wm_size <= 8)
>+		wm_size = 8;
>+
> 	return wm_size;
> }
> 
>@@ -1268,33 +1314,27 @@ static bool g4x_compute_srwm(struct drm_device *dev,
> 			      display, cursor);
> }
> 
>-static bool vlv_compute_drain_latency(struct drm_device *dev,
>-				     int plane,
>-				     int *plane_prec_mult,
>-				     int *plane_dl,
>-				     int *cursor_prec_mult,
>-				     int *cursor_dl)
>+static bool vlv_compute_drain_latency(struct drm_crtc *crtc,
>+				      int pixel_size,
>+				      int *prec_mult,
>+				      int *drain_latency)
> {
>-	struct drm_crtc *crtc;
>-	int clock, pixel_size;
> 	int entries;
>+	int clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
> 
>-	crtc = intel_get_crtc_for_plane(dev, plane);
>-	if (!intel_crtc_active(crtc))
>+	if (WARN(clock == 0, "Pixel clock is zero!\n"))
> 		return false;
> 
>-	clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
>-	pixel_size = crtc->primary->fb->bits_per_pixel / 8;	/* BPP */
>+	if (WARN(pixel_size == 0, "Pixel size is zero!\n"))
>+		return false;
> 
>-	entries = (clock / 1000) * pixel_size;
>-	*plane_prec_mult = (entries > 128) ?
>-		DRAIN_LATENCY_PRECISION_64 : DRAIN_LATENCY_PRECISION_32;
>-	*plane_dl = (64 * (*plane_prec_mult) * 4) / entries;
>+	entries = DIV_ROUND_UP(clock, 1000) * pixel_size;
>+	*prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_64 :
>+				       DRAIN_LATENCY_PRECISION_32;
>+	*drain_latency = (64 * (*prec_mult) * 4) / entries;
> 
>-	entries = (clock / 1000) * 4;	/* BPP is always 4 for cursor */
>-	*cursor_prec_mult = (entries > 128) ?
>-		DRAIN_LATENCY_PRECISION_64 : DRAIN_LATENCY_PRECISION_32;
>-	*cursor_dl = (64 * (*cursor_prec_mult) * 4) / entries;
>+	if (*drain_latency > DRAIN_LATENCY_MASK)
>+		*drain_latency = DRAIN_LATENCY_MASK;
> 
> 	return true;
> }
>@@ -1307,39 +1347,48 @@ static bool vlv_compute_drain_latency(struct drm_device *dev,
>  * latency value.
>  */
> 
>-static void vlv_update_drain_latency(struct drm_device *dev)
>+static void vlv_update_drain_latency(struct drm_crtc *crtc)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int planea_prec, planea_dl, planeb_prec, planeb_dl;
>-	int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
>-	int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
>-							either 16 or 32 */
>+	struct drm_i915_private *dev_priv = crtc->dev->dev_private;
>+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+	int pixel_size;
>+	int drain_latency;
>+	enum pipe pipe = intel_crtc->pipe;
>+	int plane_prec, prec_mult, plane_dl;
>+
>+	plane_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_PLANE_PRECISION_64 |
>+		   DRAIN_LATENCY_MASK | DDL_CURSOR_PRECISION_64 |
>+		   (DRAIN_LATENCY_MASK << DDL_CURSOR_SHIFT));
> 
>-	/* For plane A, Cursor A */
>-	if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
>-				      &cursor_prec_mult, &cursora_dl)) {
>-		cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
>-			DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_64;
>-		planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
>-			DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_64;
>+	if (!intel_crtc_active(crtc)) {
>+		I915_WRITE(VLV_DDL(pipe), plane_dl);
>+		return;
>+	}
> 
>-		I915_WRITE(VLV_DDL1, cursora_prec |
>-				(cursora_dl << DDL_CURSORA_SHIFT) |
>-				planea_prec | planea_dl);
>+	/* Primary plane Drain Latency */
>+	pixel_size = crtc->primary->fb->bits_per_pixel / 8;	/* BPP */
>+	if (vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) {
>+		plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ?
>+					   DDL_PLANE_PRECISION_64 :
>+					   DDL_PLANE_PRECISION_32;
>+		plane_dl |= plane_prec | drain_latency;
> 	}
> 
>-	/* For plane B, Cursor B */
>-	if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
>-				      &cursor_prec_mult, &cursorb_dl)) {
>-		cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
>-			DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_64;
>-		planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
>-			DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_64;
>+	/* Cursor Drain Latency
>+	 * BPP is always 4 for cursor
>+	 */
>+	pixel_size = 4;
> 
>-		I915_WRITE(VLV_DDL2, cursorb_prec |
>-				(cursorb_dl << DDL_CURSORB_SHIFT) |
>-				planeb_prec | planeb_dl);
>+	/* Program cursor DL only if it is enabled */
>+	if (intel_crtc->cursor_base &&
>+	    vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) {
>+		plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ?
>+					   DDL_CURSOR_PRECISION_64 :
>+					   DDL_CURSOR_PRECISION_32;
>+		plane_dl |= plane_prec | (drain_latency << DDL_CURSOR_SHIFT);
> 	}
>+
>+	I915_WRITE(VLV_DDL(pipe), plane_dl);
> }
> 
> #define single_plane_enabled(mask) is_power_of_2(mask)
>@@ -1355,17 +1404,17 @@ static void valleyview_update_wm(struct drm_crtc *crtc)
> 	unsigned int enabled = 0;
> 	bool cxsr_enabled;
> 
>-	vlv_update_drain_latency(dev);
>+	vlv_update_drain_latency(crtc);
> 
> 	if (g4x_compute_wm0(dev, PIPE_A,
>-			    &valleyview_wm_info, latency_ns,
>-			    &valleyview_cursor_wm_info, latency_ns,
>+			    &valleyview_wm_info, pessimal_latency_ns,
>+			    &valleyview_cursor_wm_info, pessimal_latency_ns,
> 			    &planea_wm, &cursora_wm))
> 		enabled |= 1 << PIPE_A;
> 
> 	if (g4x_compute_wm0(dev, PIPE_B,
>-			    &valleyview_wm_info, latency_ns,
>-			    &valleyview_cursor_wm_info, latency_ns,
>+			    &valleyview_wm_info, pessimal_latency_ns,
>+			    &valleyview_cursor_wm_info, pessimal_latency_ns,
> 			    &planeb_wm, &cursorb_wm))
> 		enabled |= 1 << PIPE_B;
> 
>@@ -1387,7 +1436,8 @@ static void valleyview_update_wm(struct drm_crtc *crtc)
> 		plane_sr = cursor_sr = 0;
> 	}
> 
>-	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
>+	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
>+		      "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
> 		      planea_wm, cursora_wm,
> 		      planeb_wm, cursorb_wm,
> 		      plane_sr, cursor_sr);
>@@ -1396,7 +1446,7 @@ static void valleyview_update_wm(struct drm_crtc *crtc)
> 		   (plane_sr << DSPFW_SR_SHIFT) |
> 		   (cursorb_wm << DSPFW_CURSORB_SHIFT) |
> 		   (planeb_wm << DSPFW_PLANEB_SHIFT) |
>-		   planea_wm);
>+		   (planea_wm << DSPFW_PLANEA_SHIFT));
> 	I915_WRITE(DSPFW2,
> 		   (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
> 		   (cursora_wm << DSPFW_CURSORA_SHIFT));
>@@ -1408,6 +1458,116 @@ static void valleyview_update_wm(struct drm_crtc *crtc)
> 		intel_set_memory_cxsr(dev_priv, true);
> }
> 
>+static void cherryview_update_wm(struct drm_crtc *crtc)
>+{
>+	struct drm_device *dev = crtc->dev;
>+	static const int sr_latency_ns = 12000;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	int planea_wm, planeb_wm, planec_wm;
>+	int cursora_wm, cursorb_wm, cursorc_wm;
>+	int plane_sr, cursor_sr;
>+	int ignore_plane_sr, ignore_cursor_sr;
>+	unsigned int enabled = 0;
>+	bool cxsr_enabled;
>+
>+	vlv_update_drain_latency(crtc);
>+
>+	if (g4x_compute_wm0(dev, PIPE_A,
>+			    &valleyview_wm_info, pessimal_latency_ns,
>+			    &valleyview_cursor_wm_info, pessimal_latency_ns,
>+			    &planea_wm, &cursora_wm))
>+		enabled |= 1 << PIPE_A;
>+
>+	if (g4x_compute_wm0(dev, PIPE_B,
>+			    &valleyview_wm_info, pessimal_latency_ns,
>+			    &valleyview_cursor_wm_info, pessimal_latency_ns,
>+			    &planeb_wm, &cursorb_wm))
>+		enabled |= 1 << PIPE_B;
>+
>+	if (g4x_compute_wm0(dev, PIPE_C,
>+			    &valleyview_wm_info, pessimal_latency_ns,
>+			    &valleyview_cursor_wm_info, pessimal_latency_ns,
>+			    &planec_wm, &cursorc_wm))
>+		enabled |= 1 << PIPE_C;
>+
>+	if (single_plane_enabled(enabled) &&
>+	    g4x_compute_srwm(dev, ffs(enabled) - 1,
>+			     sr_latency_ns,
>+			     &valleyview_wm_info,
>+			     &valleyview_cursor_wm_info,
>+			     &plane_sr, &ignore_cursor_sr) &&
>+	    g4x_compute_srwm(dev, ffs(enabled) - 1,
>+			     2*sr_latency_ns,
>+			     &valleyview_wm_info,
>+			     &valleyview_cursor_wm_info,
>+			     &ignore_plane_sr, &cursor_sr)) {
>+		cxsr_enabled = true;
>+	} else {
>+		cxsr_enabled = false;
>+		intel_set_memory_cxsr(dev_priv, false);
>+		plane_sr = cursor_sr = 0;
>+	}
>+
>+	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
>+		      "B: plane=%d, cursor=%d, C: plane=%d, cursor=%d, "
>+		      "SR: plane=%d, cursor=%d\n",
>+		      planea_wm, cursora_wm,
>+		      planeb_wm, cursorb_wm,
>+		      planec_wm, cursorc_wm,
>+		      plane_sr, cursor_sr);
>+
>+	I915_WRITE(DSPFW1,
>+		   (plane_sr << DSPFW_SR_SHIFT) |
>+		   (cursorb_wm << DSPFW_CURSORB_SHIFT) |
>+		   (planeb_wm << DSPFW_PLANEB_SHIFT) |
>+		   (planea_wm << DSPFW_PLANEA_SHIFT));
>+	I915_WRITE(DSPFW2,
>+		   (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
>+		   (cursora_wm << DSPFW_CURSORA_SHIFT));
>+	I915_WRITE(DSPFW3,
>+		   (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
>+		   (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
>+	I915_WRITE(DSPFW9_CHV,
>+		   (I915_READ(DSPFW9_CHV) & ~(DSPFW_PLANEC_MASK |
>+					      DSPFW_CURSORC_MASK)) |
>+		   (planec_wm << DSPFW_PLANEC_SHIFT) |
>+		   (cursorc_wm << DSPFW_CURSORC_SHIFT));
>+
>+	if (cxsr_enabled)
>+		intel_set_memory_cxsr(dev_priv, true);
>+}
>+
>+static void valleyview_update_sprite_wm(struct drm_plane *plane,
>+					struct drm_crtc *crtc,
>+					uint32_t sprite_width,
>+					uint32_t sprite_height,
>+					int pixel_size,
>+					bool enabled, bool scaled)
>+{
>+	struct drm_device *dev = crtc->dev;
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	int pipe = to_intel_plane(plane)->pipe;
>+	int sprite = to_intel_plane(plane)->plane;
>+	int drain_latency;
>+	int plane_prec;
>+	int sprite_dl;
>+	int prec_mult;
>+
>+	sprite_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_SPRITE_PRECISION_64(sprite) |
>+		    (DRAIN_LATENCY_MASK << DDL_SPRITE_SHIFT(sprite)));
>+
>+	if (enabled && vlv_compute_drain_latency(crtc, pixel_size, &prec_mult,
>+						 &drain_latency)) {
>+		plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ?
>+					   DDL_SPRITE_PRECISION_64(sprite) :
>+					   DDL_SPRITE_PRECISION_32(sprite);
>+		sprite_dl |= plane_prec |
>+			     (drain_latency << DDL_SPRITE_SHIFT(sprite));
>+	}
>+
>+	I915_WRITE(VLV_DDL(pipe), sprite_dl);
>+}
>+
> static void g4x_update_wm(struct drm_crtc *crtc)
> {
> 	struct drm_device *dev = crtc->dev;
>@@ -1419,14 +1579,14 @@ static void g4x_update_wm(struct drm_crtc *crtc)
> 	bool cxsr_enabled;
> 
> 	if (g4x_compute_wm0(dev, PIPE_A,
>-			    &g4x_wm_info, latency_ns,
>-			    &g4x_cursor_wm_info, latency_ns,
>+			    &g4x_wm_info, pessimal_latency_ns,
>+			    &g4x_cursor_wm_info, pessimal_latency_ns,
> 			    &planea_wm, &cursora_wm))
> 		enabled |= 1 << PIPE_A;
> 
> 	if (g4x_compute_wm0(dev, PIPE_B,
>-			    &g4x_wm_info, latency_ns,
>-			    &g4x_cursor_wm_info, latency_ns,
>+			    &g4x_wm_info, pessimal_latency_ns,
>+			    &g4x_cursor_wm_info, pessimal_latency_ns,
> 			    &planeb_wm, &cursorb_wm))
> 		enabled |= 1 << PIPE_B;
> 
>@@ -1443,7 +1603,8 @@ static void g4x_update_wm(struct drm_crtc *crtc)
> 		plane_sr = cursor_sr = 0;
> 	}
> 
>-	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
>+	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
>+		      "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
> 		      planea_wm, cursora_wm,
> 		      planeb_wm, cursorb_wm,
> 		      plane_sr, cursor_sr);
>@@ -1452,7 +1613,7 @@ static void g4x_update_wm(struct drm_crtc *crtc)
> 		   (plane_sr << DSPFW_SR_SHIFT) |
> 		   (cursorb_wm << DSPFW_CURSORB_SHIFT) |
> 		   (planeb_wm << DSPFW_PLANEB_SHIFT) |
>-		   planea_wm);
>+		   (planea_wm << DSPFW_PLANEA_SHIFT));
> 	I915_WRITE(DSPFW2,
> 		   (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
> 		   (cursora_wm << DSPFW_CURSORA_SHIFT));
>@@ -1526,8 +1687,11 @@ static void i965_update_wm(struct drm_crtc *unused_crtc)
> 
> 	/* 965 has limitations... */
> 	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
>-		   (8 << 16) | (8 << 8) | (8 << 0));
>-	I915_WRITE(DSPFW2, (8 << 8) | (8 << 0));
>+		   (8 << DSPFW_CURSORB_SHIFT) |
>+		   (8 << DSPFW_PLANEB_SHIFT) |
>+		   (8 << DSPFW_PLANEA_SHIFT));
>+	I915_WRITE(DSPFW2, (8 << DSPFW_CURSORA_SHIFT) |
>+		   (8 << DSPFW_PLANEC_SHIFT_OLD));
> 	/* update cursor SR watermark */
> 	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
> 
>@@ -1552,7 +1716,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
> 	else if (!IS_GEN2(dev))
> 		wm_info = &i915_wm_info;
> 	else
>-		wm_info = &i830_wm_info;
>+		wm_info = &i830_a_wm_info;
> 
> 	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
> 	crtc = intel_get_crtc_for_plane(dev, 0);
>@@ -1565,10 +1729,16 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
> 		adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
> 		planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
> 					       wm_info, fifo_size, cpp,
>-					       latency_ns);
>+					       pessimal_latency_ns);
> 		enabled = crtc;
>-	} else
>+	} else {
> 		planea_wm = fifo_size - wm_info->guard_size;
>+		if (planea_wm > (long)wm_info->max_wm)
>+			planea_wm = wm_info->max_wm;
>+	}
>+
>+	if (IS_GEN2(dev))
>+		wm_info = &i830_bc_wm_info;
> 
> 	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
> 	crtc = intel_get_crtc_for_plane(dev, 1);
>@@ -1581,13 +1751,16 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
> 		adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
> 		planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
> 					       wm_info, fifo_size, cpp,
>-					       latency_ns);
>+					       pessimal_latency_ns);
> 		if (enabled == NULL)
> 			enabled = crtc;
> 		else
> 			enabled = NULL;
>-	} else
>+	} else {
> 		planeb_wm = fifo_size - wm_info->guard_size;
>+		if (planeb_wm > (long)wm_info->max_wm)
>+			planeb_wm = wm_info->max_wm;
>+	}
> 
> 	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
> 
>@@ -1674,7 +1847,7 @@ static void i845_update_wm(struct drm_crtc *unused_crtc)
> 	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
> 				       &i845_wm_info,
> 				       dev_priv->display.get_fifo_size(dev, 0),
>-				       4, latency_ns);
>+				       4, pessimal_latency_ns);
> 	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
> 	fwater_lo |= (3<<8) | planea_wm;
> 
>@@ -1683,23 +1856,21 @@ static void i845_update_wm(struct drm_crtc *unused_crtc)
> 	I915_WRITE(FW_BLC, fwater_lo);
> }
> 
>-static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
>-				    struct drm_crtc *crtc)
>+uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_config *pipe_config)
> {
>-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 	uint32_t pixel_rate;
> 
>-	pixel_rate = intel_crtc->config.adjusted_mode.crtc_clock;
>+	pixel_rate = pipe_config->adjusted_mode.crtc_clock;
> 
> 	/* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
> 	 * adjust the pixel_rate here. */
> 
>-	if (intel_crtc->config.pch_pfit.enabled) {
>+	if (pipe_config->pch_pfit.enabled) {
> 		uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
>-		uint32_t pfit_size = intel_crtc->config.pch_pfit.size;
>+		uint32_t pfit_size = pipe_config->pch_pfit.size;
> 
>-		pipe_w = intel_crtc->config.pipe_src_w;
>-		pipe_h = intel_crtc->config.pipe_src_h;
>+		pipe_w = pipe_config->pipe_src_w;
>+		pipe_h = pipe_config->pipe_src_h;
> 		pfit_w = (pfit_size >> 16) & 0xFFFF;
> 		pfit_h = pfit_size & 0xFFFF;
> 		if (pipe_w < pfit_w)
>@@ -2121,7 +2292,6 @@ int ilk_wm_max_level(const struct drm_device *dev)
> 	else
> 		return 2;
> }
>-
> static void intel_print_wm_latency(struct drm_device *dev,
> 				   const char *name,
> 				   const uint16_t wm[5])
>@@ -2219,7 +2389,7 @@ static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
> 
> 	p->active = true;
> 	p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal;
>-	p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
>+	p->pixel_rate = ilk_pipe_pixel_rate(&intel_crtc->config);
> 	p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8;
> 	p->cur.bytes_per_pixel = 4;
> 	p->pri.horiz_pixels = intel_crtc->config.pipe_src_w;
>@@ -2527,7 +2697,7 @@ static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
> #define WM_DIRTY_FBC (1 << 24)
> #define WM_DIRTY_DDB (1 << 25)
> 
>-static unsigned int ilk_compute_wm_dirty(struct drm_device *dev,
>+static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
> 					 const struct ilk_wm_values *old,
> 					 const struct ilk_wm_values *new)
> {
>@@ -2535,7 +2705,7 @@ static unsigned int ilk_compute_wm_dirty(struct drm_device *dev,
> 	enum pipe pipe;
> 	int wm_lp;
> 
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
> 			dirty |= WM_DIRTY_LINETIME(pipe);
> 			/* Must disable LP1+ watermarks too */
>@@ -2621,7 +2791,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
> 	unsigned int dirty;
> 	uint32_t val;
> 
>-	dirty = ilk_compute_wm_dirty(dev, previous, results);
>+	dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
> 	if (!dirty)
> 		return;
> 
>@@ -3090,6 +3260,9 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
> {
> 	int new_power;
> 
>+	if (dev_priv->rps.is_bdw_sw_turbo)
>+		return;
>+
> 	new_power = dev_priv->rps.power;
> 	switch (dev_priv->rps.power) {
> 	case LOW_POWER:
>@@ -3297,8 +3470,11 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv)
> 			valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
> 		else if (IS_VALLEYVIEW(dev))
> 			vlv_set_rps_idle(dev_priv);
>-		else
>+		else if (!dev_priv->rps.is_bdw_sw_turbo
>+					|| atomic_read(&dev_priv->rps.sw_turbo.flip_received)){
> 			gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
>+		}
>+
> 		dev_priv->rps.last_adj = 0;
> 	}
> 	mutex_unlock(&dev_priv->rps.hw_lock);
>@@ -3312,8 +3488,11 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv)
> 	if (dev_priv->rps.enabled) {
> 		if (IS_VALLEYVIEW(dev))
> 			valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
>-		else
>+		else if (!dev_priv->rps.is_bdw_sw_turbo
>+					|| atomic_read(&dev_priv->rps.sw_turbo.flip_received)){
> 			gen6_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
>+		}
>+
> 		dev_priv->rps.last_adj = 0;
> 	}
> 	mutex_unlock(&dev_priv->rps.hw_lock);
>@@ -3332,6 +3511,10 @@ void valleyview_set_rps(struct drm_device *dev, u8 val)
> 			 dev_priv->rps.cur_freq,
> 			 vlv_gpu_freq(dev_priv, val), val);
> 
>+	if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
>+		      "Odd GPU freq value\n"))
>+		val &= ~1;
>+
> 	if (val != dev_priv->rps.cur_freq)
> 		vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
> 
>@@ -3344,21 +3527,26 @@ void valleyview_set_rps(struct drm_device *dev, u8 val)
> static void gen8_disable_rps_interrupts(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>+	if (IS_BROADWELL(dev) && dev_priv->rps.is_bdw_sw_turbo){
>+		if (atomic_read(&dev_priv->rps.sw_turbo.flip_received))
>+			del_timer(&dev_priv->rps.sw_turbo.flip_timer);
>+		dev_priv-> rps.is_bdw_sw_turbo = false;
>+	} else {
>+		I915_WRITE(GEN6_PMINTRMSK, ~GEN8_PMINTR_REDIRECT_TO_NON_DISP);
>+		I915_WRITE(GEN8_GT_IER(2), I915_READ(GEN8_GT_IER(2)) &
>+					   ~dev_priv->pm_rps_events);
>+		/* Complete PM interrupt masking here doesn't race with the rps work
>+		 * item again unmasking PM interrupts because that is using a different
>+		 * register (GEN8_GT_IMR(2)) to mask PM interrupts. The only risk is in
>+		 * leaving stale bits in GEN8_GT_IIR(2) and GEN8_GT_IMR(2) which
>+		 * gen8_enable_rps will clean up. */
> 
>-	I915_WRITE(GEN6_PMINTRMSK, ~GEN8_PMINTR_REDIRECT_TO_NON_DISP);
>-	I915_WRITE(GEN8_GT_IER(2), I915_READ(GEN8_GT_IER(2)) &
>-				   ~dev_priv->pm_rps_events);
>-	/* Complete PM interrupt masking here doesn't race with the rps work
>-	 * item again unmasking PM interrupts because that is using a different
>-	 * register (GEN8_GT_IMR(2)) to mask PM interrupts. The only risk is in
>-	 * leaving stale bits in GEN8_GT_IIR(2) and GEN8_GT_IMR(2) which
>-	 * gen8_enable_rps will clean up. */
>-
>-	spin_lock_irq(&dev_priv->irq_lock);
>-	dev_priv->rps.pm_iir = 0;
>-	spin_unlock_irq(&dev_priv->irq_lock);
>+		spin_lock_irq(&dev_priv->irq_lock);
>+		dev_priv->rps.pm_iir = 0;
>+		spin_unlock_irq(&dev_priv->irq_lock);
> 
>-	I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events);
>+		I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events);
>+	}
> }
> 
> static void gen6_disable_rps_interrupts(struct drm_device *dev)
>@@ -3406,8 +3594,14 @@ static void valleyview_disable_rps(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 
>+	/* we're doing forcewake before Disabling RC6,
>+	 * This what the BIOS expects when going into suspend */
>+	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
>+
> 	I915_WRITE(GEN6_RC_CONTROL, 0);
> 
>+	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
>+
> 	gen6_disable_rps_interrupts(dev);
> }
> 
>@@ -3452,9 +3646,11 @@ static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
> 		return enable_rc6 & mask;
> 	}
> 
>-	/* Disable RC6 on Ironlake */
>-	if (INTEL_INFO(dev)->gen == 5)
>+#ifdef CONFIG_INTEL_IOMMU
>+	/* Ironlake + RC6 + VT-d empirically blows up */
>+	if (IS_GEN5(dev) && intel_iommu_gfx_mapped)
> 		return 0;
>+#endif
> 
> 	if (IS_IVYBRIDGE(dev))
> 		return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
>@@ -3510,13 +3706,111 @@ static void parse_rp_state_cap(struct drm_i915_private *dev_priv, u32 rp_state_c
> 		dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
> }
> 
>+static void bdw_sw_calculate_freq(struct drm_device *dev,
>+		struct intel_rps_bdw_cal *c, u32 *cur_time, u32 *c0)
>+{
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+	u64 busy = 0;
>+	u32 busyness_pct = 0;
>+	u32 elapsed_time = 0;
>+	u16 new_freq = 0;
>+
>+	if (!c || !cur_time || !c0)
>+		return;
>+
>+	if (0 == c->last_c0)
>+		goto out;
>+
>+	/* Check Evaluation interval */
>+	elapsed_time = *cur_time - c->last_ts;
>+	if (elapsed_time < c->eval_interval)
>+		return;
>+
>+	mutex_lock(&dev_priv->rps.hw_lock);
>+
>+	/*
>+	 * c0 unit in 32*1.28 usec, elapsed_time unit in 1 usec.
>+	 * Whole busyness_pct calculation should be
>+	 *     busy = ((u64)(*c0 - c->last_c0) << 5 << 7) / 100;
>+	 *     busyness_pct = (u32)(busy * 100 / elapsed_time);
>+	 * The final formula is to simplify CPU calculation
>+	 */
>+	busy = (u64)(*c0 - c->last_c0) << 12;
>+	do_div(busy, elapsed_time);
>+	busyness_pct = (u32)busy;
>+
>+	if (c->is_up && busyness_pct >= c->it_threshold_pct)
>+		new_freq = (u16)dev_priv->rps.cur_freq + 3;
>+	if (!c->is_up && busyness_pct <= c->it_threshold_pct)
>+		new_freq = (u16)dev_priv->rps.cur_freq - 1;
>+
>+	/* Adjust to new frequency busyness and compare with threshold */
>+	if (0 != new_freq) {
>+		if (new_freq > dev_priv->rps.max_freq_softlimit)
>+			new_freq = dev_priv->rps.max_freq_softlimit;
>+		else if (new_freq < dev_priv->rps.min_freq_softlimit)
>+			new_freq = dev_priv->rps.min_freq_softlimit;
>+
>+		gen6_set_rps(dev, new_freq);
>+	}
>+
>+	mutex_unlock(&dev_priv->rps.hw_lock);
>+
>+out:
>+	c->last_c0 = *c0;
>+	c->last_ts = *cur_time;
>+}
>+
>+static void gen8_set_frequency_RP0(struct work_struct *work)
>+{
>+	struct intel_rps_bdw_turbo *p_bdw_turbo =
>+			container_of(work, struct intel_rps_bdw_turbo, work_max_freq);
>+	struct intel_gen6_power_mgmt *p_power_mgmt =
>+			container_of(p_bdw_turbo, struct intel_gen6_power_mgmt, sw_turbo);
>+	struct drm_i915_private *dev_priv =
>+			container_of(p_power_mgmt, struct drm_i915_private, rps);
>+
>+	mutex_lock(&dev_priv->rps.hw_lock);
>+	gen6_set_rps(dev_priv->dev, dev_priv->rps.rp0_freq);
>+	mutex_unlock(&dev_priv->rps.hw_lock);
>+}
>+
>+static void flip_active_timeout_handler(unsigned long var)
>+{
>+	struct drm_i915_private *dev_priv = (struct drm_i915_private *) var;
>+
>+	del_timer(&dev_priv->rps.sw_turbo.flip_timer);
>+	atomic_set(&dev_priv->rps.sw_turbo.flip_received, false);
>+
>+	queue_work(dev_priv->wq, &dev_priv->rps.sw_turbo.work_max_freq);
>+}
>+
>+void bdw_software_turbo(struct drm_device *dev)
>+{
>+	struct drm_i915_private *dev_priv = dev->dev_private;
>+
>+	u32 current_time = I915_READ(TIMESTAMP_CTR); /* unit in usec */
>+	u32 current_c0 = I915_READ(MCHBAR_PCU_C0); /* unit in 32*1.28 usec */
>+
>+	bdw_sw_calculate_freq(dev, &dev_priv->rps.sw_turbo.up,
>+			&current_time, &current_c0);
>+	bdw_sw_calculate_freq(dev, &dev_priv->rps.sw_turbo.down,
>+			&current_time, &current_c0);
>+}
>+
> static void gen8_enable_rps(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	uint32_t rc6_mask = 0, rp_state_cap;
>+	uint32_t threshold_up_pct, threshold_down_pct;
>+	uint32_t ei_up, ei_down; /* up and down evaluation interval */
>+	u32 rp_ctl_flag;
> 	int unused;
> 
>+	/* Use software Turbo for BDW */
>+	dev_priv->rps.is_bdw_sw_turbo = IS_BROADWELL(dev);
>+
> 	/* 1a: Software RC state - RC0 */
> 	I915_WRITE(GEN6_RC_STATE, 0);
> 
>@@ -3534,8 +3828,8 @@ static void gen8_enable_rps(struct drm_device *dev)
> 	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
> 	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
> 	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
>-	for_each_ring(ring, dev_priv, unused)
>-		I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
>+	for_each_engine(engine, dev_priv, unused)
>+		I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
> 	I915_WRITE(GEN6_RC_SLEEP, 0);
> 	if (IS_BROADWELL(dev))
> 		I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
>@@ -3560,35 +3854,74 @@ static void gen8_enable_rps(struct drm_device *dev)
> 		   HSW_FREQUENCY(dev_priv->rps.rp1_freq));
> 	I915_WRITE(GEN6_RC_VIDEO_FREQ,
> 		   HSW_FREQUENCY(dev_priv->rps.rp1_freq));
>-	/* NB: Docs say 1s, and 1000000 - which aren't equivalent */
>-	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
>+	ei_up = 84480; /* 84.48ms */
>+	ei_down = 448000;
>+	threshold_up_pct = 90; /* x percent busy */
>+	threshold_down_pct = 70;
>+
>+	if (dev_priv->rps.is_bdw_sw_turbo) {
>+		dev_priv->rps.sw_turbo.up.it_threshold_pct = threshold_up_pct;
>+		dev_priv->rps.sw_turbo.up.eval_interval = ei_up;
>+		dev_priv->rps.sw_turbo.up.is_up = true;
>+		dev_priv->rps.sw_turbo.up.last_ts = 0;
>+		dev_priv->rps.sw_turbo.up.last_c0 = 0;
>+
>+		dev_priv->rps.sw_turbo.down.it_threshold_pct = threshold_down_pct;
>+		dev_priv->rps.sw_turbo.down.eval_interval = ei_down;
>+		dev_priv->rps.sw_turbo.down.is_up = false;
>+		dev_priv->rps.sw_turbo.down.last_ts = 0;
>+		dev_priv->rps.sw_turbo.down.last_c0 = 0;
>+
>+		/* Start the timer to track if flip comes*/
>+		dev_priv->rps.sw_turbo.timeout = 200*1000; /* in us */
>+
>+		init_timer(&dev_priv->rps.sw_turbo.flip_timer);
>+		dev_priv->rps.sw_turbo.flip_timer.function = flip_active_timeout_handler;
>+		dev_priv->rps.sw_turbo.flip_timer.data  = (unsigned long) dev_priv;
>+		dev_priv->rps.sw_turbo.flip_timer.expires =
>+			usecs_to_jiffies(dev_priv->rps.sw_turbo.timeout) + jiffies;
>+		add_timer(&dev_priv->rps.sw_turbo.flip_timer);
>+		INIT_WORK(&dev_priv->rps.sw_turbo.work_max_freq, gen8_set_frequency_RP0);
>+
>+		atomic_set(&dev_priv->rps.sw_turbo.flip_received, true);
>+	} else {
>+		/* NB: Docs say 1s, and 1000000 - which aren't equivalent
>+		 * 1 second timeout*/
>+		I915_WRITE(GEN6_RP_DOWN_TIMEOUT, FREQ_1_28_US(1000000));
> 
>-	/* Docs recommend 900MHz, and 300 MHz respectively */
>-	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
>-		   dev_priv->rps.max_freq_softlimit << 24 |
>-		   dev_priv->rps.min_freq_softlimit << 16);
>+		/* Docs recommend 900MHz, and 300 MHz respectively */
>+		I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
>+			   dev_priv->rps.max_freq_softlimit << 24 |
>+			   dev_priv->rps.min_freq_softlimit << 16);
> 
>-	I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
>-	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
>-	I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
>-	I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
>+		I915_WRITE(GEN6_RP_UP_THRESHOLD,
>+			FREQ_1_28_US(ei_up * threshold_up_pct / 100));
>+		I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
>+			FREQ_1_28_US(ei_down * threshold_down_pct / 100));
>+		I915_WRITE(GEN6_RP_UP_EI,
>+			FREQ_1_28_US(ei_up));
>+		I915_WRITE(GEN6_RP_DOWN_EI,
>+			FREQ_1_28_US(ei_down));
> 
>-	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
>+		I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
>+	}
> 
> 	/* 5: Enable RPS */
>-	I915_WRITE(GEN6_RP_CONTROL,
>-		   GEN6_RP_MEDIA_TURBO |
>-		   GEN6_RP_MEDIA_HW_NORMAL_MODE |
>-		   GEN6_RP_MEDIA_IS_GFX |
>-		   GEN6_RP_ENABLE |
>-		   GEN6_RP_UP_BUSY_AVG |
>-		   GEN6_RP_DOWN_IDLE_AVG);
>-
>-	/* 6: Ring frequency + overclocking (our driver does this later */
>-
>+	rp_ctl_flag = GEN6_RP_MEDIA_TURBO |
>+					GEN6_RP_MEDIA_HW_NORMAL_MODE |
>+					GEN6_RP_MEDIA_IS_GFX |
>+					GEN6_RP_UP_BUSY_AVG |
>+					GEN6_RP_DOWN_IDLE_AVG;
>+	if (!dev_priv->rps.is_bdw_sw_turbo)
>+		rp_ctl_flag |= GEN6_RP_ENABLE;
>+
>+	I915_WRITE(GEN6_RP_CONTROL, rp_ctl_flag);
>+
>+	/* 6: Ring frequency + overclocking
>+	 * (our driver does this later */
> 	gen6_set_rps(dev, (I915_READ(GEN6_GT_PERF_STATUS) & 0xff00) >> 8);
>-
>-	gen8_enable_rps_interrupts(dev);
>+	if (!dev_priv->rps.is_bdw_sw_turbo)
>+		gen8_enable_rps_interrupts(dev);
> 
> 	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
> }
>@@ -3596,9 +3929,8 @@ static void gen8_enable_rps(struct drm_device *dev)
> static void gen6_enable_rps(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	u32 rp_state_cap;
>-	u32 gt_perf_status;
> 	u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
> 	u32 gtfifodbg;
> 	int rc6_mode;
>@@ -3623,7 +3955,6 @@ static void gen6_enable_rps(struct drm_device *dev)
> 	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
> 
> 	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
>-	gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
> 
> 	parse_rp_state_cap(dev_priv, rp_state_cap);
> 
>@@ -3636,8 +3967,8 @@ static void gen6_enable_rps(struct drm_device *dev)
> 	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
> 	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
> 
>-	for_each_ring(ring, dev_priv, i)
>-		I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
>+	for_each_engine(engine, dev_priv, i)
>+		I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
> 
> 	I915_WRITE(GEN6_RC_SLEEP, 0);
> 	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
>@@ -3965,11 +4296,27 @@ static void valleyview_cleanup_pctx(struct drm_device *dev)
> static void valleyview_init_gt_powersave(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>+	u32 val;
> 
> 	valleyview_setup_pctx(dev);
> 
> 	mutex_lock(&dev_priv->rps.hw_lock);
> 
>+	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
>+	switch ((val >> 6) & 3) {
>+	case 0:
>+	case 1:
>+		dev_priv->mem_freq = 800;
>+		break;
>+	case 2:
>+		dev_priv->mem_freq = 1066;
>+		break;
>+	case 3:
>+		dev_priv->mem_freq = 1333;
>+		break;
>+	}
>+	DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
>+
> 	dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
> 	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
> 	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
>@@ -4004,11 +4351,38 @@ static void valleyview_init_gt_powersave(struct drm_device *dev)
> static void cherryview_init_gt_powersave(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>+	u32 val;
> 
> 	cherryview_setup_pctx(dev);
> 
> 	mutex_lock(&dev_priv->rps.hw_lock);
> 
>+	val = vlv_punit_read(dev_priv, CCK_FUSE_REG);
>+	switch ((val >> 2) & 0x7) {
>+	case 0:
>+	case 1:
>+		dev_priv->rps.cz_freq = 200;
>+		dev_priv->mem_freq = 1600;
>+		break;
>+	case 2:
>+		dev_priv->rps.cz_freq = 267;
>+		dev_priv->mem_freq = 1600;
>+		break;
>+	case 3:
>+		dev_priv->rps.cz_freq = 333;
>+		dev_priv->mem_freq = 2000;
>+		break;
>+	case 4:
>+		dev_priv->rps.cz_freq = 320;
>+		dev_priv->mem_freq = 1600;
>+		break;
>+	case 5:
>+		dev_priv->rps.cz_freq = 400;
>+		dev_priv->mem_freq = 1600;
>+		break;
>+	}
>+	DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
>+
> 	dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
> 	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
> 	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
>@@ -4030,6 +4404,12 @@ static void cherryview_init_gt_powersave(struct drm_device *dev)
> 			 vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq),
> 			 dev_priv->rps.min_freq);
> 
>+	WARN_ONCE((dev_priv->rps.max_freq |
>+		   dev_priv->rps.efficient_freq |
>+		   dev_priv->rps.rp1_freq |
>+		   dev_priv->rps.min_freq) & 1,
>+		  "Odd GPU freq values\n");
>+
> 	/* Preserve min/max settings in case of re-init */
> 	if (dev_priv->rps.max_freq_softlimit == 0)
> 		dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
>@@ -4048,7 +4428,7 @@ static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
> static void cherryview_enable_rps(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	u32 gtfifodbg, val, rc6_mode = 0, pcbr;
> 	int i;
> 
>@@ -4072,8 +4452,8 @@ static void cherryview_enable_rps(struct drm_device *dev)
> 	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
> 	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
> 
>-	for_each_ring(ring, dev_priv, i)
>-		I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
>+	for_each_engine(engine, dev_priv, i)
>+		I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
> 	I915_WRITE(GEN6_RC_SLEEP, 0);
> 
> 	I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
>@@ -4140,7 +4520,7 @@ static void cherryview_enable_rps(struct drm_device *dev)
> static void valleyview_enable_rps(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	u32 gtfifodbg, val, rc6_mode = 0;
> 	int i;
> 
>@@ -4177,8 +4557,8 @@ static void valleyview_enable_rps(struct drm_device *dev)
> 	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
> 	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
> 
>-	for_each_ring(ring, dev_priv, i)
>-		I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
>+	for_each_engine(engine, dev_priv, i)
>+		I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
> 
> 	I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
> 
>@@ -4221,12 +4601,6 @@ void ironlake_teardown_rc6(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 
>-	if (dev_priv->ips.renderctx) {
>-		i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx);
>-		drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
>-		dev_priv->ips.renderctx = NULL;
>-	}
>-
> 	if (dev_priv->ips.pwrctx) {
> 		i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx);
> 		drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
>@@ -4256,11 +4630,6 @@ static int ironlake_setup_rc6(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 
>-	if (dev_priv->ips.renderctx == NULL)
>-		dev_priv->ips.renderctx = intel_alloc_context_page(dev);
>-	if (!dev_priv->ips.renderctx)
>-		return -ENOMEM;
>-
> 	if (dev_priv->ips.pwrctx == NULL)
> 		dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
> 	if (!dev_priv->ips.pwrctx) {
>@@ -4274,9 +4643,6 @@ static int ironlake_setup_rc6(struct drm_device *dev)
> static void ironlake_enable_rc6(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
>-	bool was_interruptible;
>-	int ret;
> 
> 	/* rc6 disabled by default due to repeated reports of hanging during
> 	 * boot and resume.
>@@ -4286,46 +4652,8 @@ static void ironlake_enable_rc6(struct drm_device *dev)
> 
> 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
> 
>-	ret = ironlake_setup_rc6(dev);
>-	if (ret)
>-		return;
>-
>-	was_interruptible = dev_priv->mm.interruptible;
>-	dev_priv->mm.interruptible = false;
>-
>-	/*
>-	 * GPU can automatically power down the render unit if given a page
>-	 * to save state.
>-	 */
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret) {
>-		ironlake_teardown_rc6(dev);
>-		dev_priv->mm.interruptible = was_interruptible;
>-		return;
>-	}
>-
>-	intel_ring_emit(ring, MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
>-	intel_ring_emit(ring, MI_SET_CONTEXT);
>-	intel_ring_emit(ring, i915_gem_obj_ggtt_offset(dev_priv->ips.renderctx) |
>-			MI_MM_SPACE_GTT |
>-			MI_SAVE_EXT_STATE_EN |
>-			MI_RESTORE_EXT_STATE_EN |
>-			MI_RESTORE_INHIBIT);
>-	intel_ring_emit(ring, MI_SUSPEND_FLUSH);
>-	intel_ring_emit(ring, MI_NOOP);
>-	intel_ring_emit(ring, MI_FLUSH);
>-	intel_ring_advance(ring);
>-
>-	/*
>-	 * Wait for the command parser to advance past MI_SET_CONTEXT. The HW
>-	 * does an implicit flush, combined with MI_FLUSH above, it should be
>-	 * safe to assume that renderctx is valid
>-	 */
>-	ret = intel_ring_idle(ring);
>-	dev_priv->mm.interruptible = was_interruptible;
>-	if (ret) {
>+	if (ironlake_setup_rc6(dev)) {
> 		DRM_ERROR("failed to enable ironlake power savings\n");
>-		ironlake_teardown_rc6(dev);
> 		return;
> 	}
> 
>@@ -4784,7 +5112,7 @@ EXPORT_SYMBOL_GPL(i915_gpu_lower);
> bool i915_gpu_busy(void)
> {
> 	struct drm_i915_private *dev_priv;
>-	struct intel_engine_cs *ring;
>+	struct intel_engine_cs *engine;
> 	bool ret = false;
> 	int i;
> 
>@@ -4793,8 +5121,8 @@ bool i915_gpu_busy(void)
> 		goto out_unlock;
> 	dev_priv = i915_mch_dev;
> 
>-	for_each_ring(ring, dev_priv, i)
>-		ret |= !list_empty(&ring->request_list);
>+	for_each_engine(engine, dev_priv, i)
>+		ret |= engine->last_request != NULL;
> 
> out_unlock:
> 	spin_unlock_irq(&mchdev_lock);
>@@ -5015,6 +5343,8 @@ static void intel_gen6_powersave_work(struct work_struct *work)
> 			     rps.delayed_resume_work.work);
> 	struct drm_device *dev = dev_priv->dev;
> 
>+	dev_priv->rps.is_bdw_sw_turbo = false;
>+
> 	mutex_lock(&dev_priv->rps.hw_lock);
> 
> 	if (IS_CHERRYVIEW(dev)) {
>@@ -5088,7 +5418,7 @@ static void g4x_disable_trickle_feed(struct drm_device *dev)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	int pipe;
> 
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		I915_WRITE(DSPCNTR(pipe),
> 			   I915_READ(DSPCNTR(pipe)) |
> 			   DISPPLANE_TRICKLE_FEED_DISABLE);
>@@ -5203,7 +5533,7 @@ static void cpt_init_clock_gating(struct drm_device *dev)
> 	/* The below fixes the weird display corruption, a few pixels shifted
> 	 * downward, on (only) LVDS of some HP laptops with IVY.
> 	 */
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		val = I915_READ(TRANS_CHICKEN2(pipe));
> 		val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
> 		val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
>@@ -5215,7 +5545,7 @@ static void cpt_init_clock_gating(struct drm_device *dev)
> 		I915_WRITE(TRANS_CHICKEN2(pipe), val);
> 	}
> 	/* WADP0ClockGatingDisable */
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		I915_WRITE(TRANS_CHICKEN1(pipe),
> 			   TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
> 	}
>@@ -5383,7 +5713,7 @@ static void lpt_suspend_hw(struct drm_device *dev)
> 	}
> }
> 
>-static void gen8_init_clock_gating(struct drm_device *dev)
>+static void broadwell_init_clock_gating(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 	enum pipe pipe;
>@@ -5395,37 +5725,12 @@ static void gen8_init_clock_gating(struct drm_device *dev)
> 	/* FIXME(BDW): Check all the w/a, some might only apply to
> 	 * pre-production hw. */
> 
>-	/* WaDisablePartialInstShootdown:bdw */
>-	I915_WRITE(GEN8_ROW_CHICKEN,
>-		   _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE));
> 
>-	/* WaDisableThreadStallDopClockGating:bdw */
>-	/* FIXME: Unclear whether we really need this on production bdw. */
>-	I915_WRITE(GEN8_ROW_CHICKEN,
>-		   _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
>-
>-	/*
>-	 * This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for
>-	 * pre-production hardware
>-	 */
>-	I915_WRITE(HALF_SLICE_CHICKEN3,
>-		   _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS));
>-	I915_WRITE(HALF_SLICE_CHICKEN3,
>-		   _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
> 	I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_BWGTLB_DISABLE));
> 
> 	I915_WRITE(_3D_CHICKEN3,
> 		   _MASKED_BIT_ENABLE(_3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(2)));
> 
>-	I915_WRITE(COMMON_SLICE_CHICKEN2,
>-		   _MASKED_BIT_ENABLE(GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE));
>-
>-	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
>-		   _MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE));
>-
>-	/* WaDisableDopClockGating:bdw May not be needed for production */
>-	I915_WRITE(GEN7_ROW_CHICKEN2,
>-		   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
> 
> 	/* WaSwitchSolVfFArbitrationPriority:bdw */
> 	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
>@@ -5435,37 +5740,18 @@ static void gen8_init_clock_gating(struct drm_device *dev)
> 		   I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
> 
> 	/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
>-	for_each_pipe(pipe) {
>+	for_each_pipe(dev_priv, pipe) {
> 		I915_WRITE(CHICKEN_PIPESL_1(pipe),
> 			   I915_READ(CHICKEN_PIPESL_1(pipe)) |
> 			   BDW_DPRS_MASK_VBLANK_SRD);
> 	}
> 
>-	/* Use Force Non-Coherent whenever executing a 3D context. This is a
>-	 * workaround for for a possible hang in the unlikely event a TLB
>-	 * invalidation occurs during a PSD flush.
>-	 */
>-	I915_WRITE(HDC_CHICKEN0,
>-		   I915_READ(HDC_CHICKEN0) |
>-		   _MASKED_BIT_ENABLE(HDC_FORCE_NON_COHERENT));
>-
> 	/* WaVSRefCountFullforceMissDisable:bdw */
> 	/* WaDSRefCountFullforceMissDisable:bdw */
> 	I915_WRITE(GEN7_FF_THREAD_MODE,
> 		   I915_READ(GEN7_FF_THREAD_MODE) &
> 		   ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
> 
>-	/*
>-	 * BSpec recommends 8x4 when MSAA is used,
>-	 * however in practice 16x4 seems fastest.
>-	 *
>-	 * Note that PS/WM thread counts depend on the WIZ hashing
>-	 * disable bit, which we don't touch here, but it's good
>-	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
>-	 */
>-	I915_WRITE(GEN7_GT_MODE,
>-		   GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
>-
> 	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
> 		   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
> 
>@@ -5473,9 +5759,7 @@ static void gen8_init_clock_gating(struct drm_device *dev)
> 	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
> 		   GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
> 
>-	/* Wa4x4STCOptimizationDisable:bdw */
>-	I915_WRITE(CACHE_MODE_1,
>-		   _MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE));
>+	lpt_init_clock_gating(dev);
> }
> 
> static void haswell_init_clock_gating(struct drm_device *dev)
>@@ -5631,24 +5915,6 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
> static void valleyview_init_clock_gating(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	u32 val;
>-
>-	mutex_lock(&dev_priv->rps.hw_lock);
>-	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
>-	mutex_unlock(&dev_priv->rps.hw_lock);
>-	switch ((val >> 6) & 3) {
>-	case 0:
>-	case 1:
>-		dev_priv->mem_freq = 800;
>-		break;
>-	case 2:
>-		dev_priv->mem_freq = 1066;
>-		break;
>-	case 3:
>-		dev_priv->mem_freq = 1333;
>-		break;
>-	}
>-	DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
> 
> 	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
> 
>@@ -5724,48 +5990,11 @@ static void valleyview_init_clock_gating(struct drm_device *dev)
> static void cherryview_init_clock_gating(struct drm_device *dev)
> {
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	u32 val;
>-
>-	mutex_lock(&dev_priv->rps.hw_lock);
>-	val = vlv_punit_read(dev_priv, CCK_FUSE_REG);
>-	mutex_unlock(&dev_priv->rps.hw_lock);
>-	switch ((val >> 2) & 0x7) {
>-	case 0:
>-	case 1:
>-			dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_200;
>-			dev_priv->mem_freq = 1600;
>-			break;
>-	case 2:
>-			dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_267;
>-			dev_priv->mem_freq = 1600;
>-			break;
>-	case 3:
>-			dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_333;
>-			dev_priv->mem_freq = 2000;
>-			break;
>-	case 4:
>-			dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_320;
>-			dev_priv->mem_freq = 1600;
>-			break;
>-	case 5:
>-			dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_400;
>-			dev_priv->mem_freq = 1600;
>-			break;
>-	}
>-	DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
> 
> 	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
> 
> 	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
> 
>-	/* WaDisablePartialInstShootdown:chv */
>-	I915_WRITE(GEN8_ROW_CHICKEN,
>-		   _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE));
>-
>-	/* WaDisableThreadStallDopClockGating:chv */
>-	I915_WRITE(GEN8_ROW_CHICKEN,
>-		   _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
>-
> 	/* WaVSRefCountFullforceMissDisable:chv */
> 	/* WaDSRefCountFullforceMissDisable:chv */
> 	I915_WRITE(GEN7_FF_THREAD_MODE,
>@@ -5784,10 +6013,6 @@ static void cherryview_init_clock_gating(struct drm_device *dev)
> 	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
> 		   GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
> 
>-	/* WaDisableSamplerPowerBypass:chv (pre-production hw) */
>-	I915_WRITE(HALF_SLICE_CHICKEN3,
>-		   _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
>-
> 	/* WaDisableGunitClockGating:chv (pre-production hw) */
> 	I915_WRITE(VLV_GUNIT_CLOCK_GATE, I915_READ(VLV_GUNIT_CLOCK_GATE) |
> 		   GINT_DIS);
>@@ -5797,8 +6022,6 @@ static void cherryview_init_clock_gating(struct drm_device *dev)
> 		   _MASKED_BIT_ENABLE(GEN8_FF_DOP_CLOCK_GATE_DISABLE));
> 
> 	/* WaDisableDopClockGating:chv (pre-production hw) */
>-	I915_WRITE(GEN7_ROW_CHICKEN2,
>-		   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
> 	I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
> 		   GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
> }
>@@ -5883,6 +6106,9 @@ static void gen3_init_clock_gating(struct drm_device *dev)
> 
> 	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
> 	I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
>+
>+	I915_WRITE(MI_ARB_STATE,
>+		   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
> }
> 
> static void i85x_init_clock_gating(struct drm_device *dev)
>@@ -5894,6 +6120,9 @@ static void i85x_init_clock_gating(struct drm_device *dev)
> 	/* interrupts should cause a wake up from C3 */
> 	I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
> 		   _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
>+
>+	I915_WRITE(MEM_MODE,
>+		   _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
> }
> 
> static void i830_init_clock_gating(struct drm_device *dev)
>@@ -5901,6 +6130,10 @@ static void i830_init_clock_gating(struct drm_device *dev)
> 	struct drm_i915_private *dev_priv = dev->dev_private;
> 
> 	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
>+
>+	I915_WRITE(MEM_MODE,
>+		   _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
>+		   _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
> }
> 
> void intel_init_clock_gating(struct drm_device *dev)
>@@ -6203,6 +6436,8 @@ static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
> 	spin_unlock_irq(&dev_priv->irq_lock);
> 
> 	vlv_set_power_well(dev_priv, power_well, false);
>+
>+	vlv_power_sequencer_reset(dev_priv);
> }
> 
> static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
>@@ -6238,12 +6473,11 @@ static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
> static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
> 					    struct i915_power_well *power_well)
> {
>-	struct drm_device *dev = dev_priv->dev;
> 	enum pipe pipe;
> 
> 	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
> 
>-	for_each_pipe(pipe)
>+	for_each_pipe(dev_priv, pipe)
> 		assert_pll_disabled(dev_priv, pipe);
> 
> 	/* Assert common reset */
>@@ -6252,6 +6486,153 @@ static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
> 	vlv_set_power_well(dev_priv, power_well, false);
> }
> 
>+static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
>+					   struct i915_power_well *power_well)
>+{
>+	enum dpio_phy phy;
>+
>+	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC &&
>+		     power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D);
>+
>+	/*
>+	 * Enable the CRI clock source so we can get at the
>+	 * display and the reference clock for VGA
>+	 * hotplug / manual detection.
>+	 */
>+	if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
>+		phy = DPIO_PHY0;
>+		I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) |
>+			   DPLL_REFA_CLK_ENABLE_VLV);
>+		I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) |
>+			   DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
>+	} else {
>+		phy = DPIO_PHY1;
>+		I915_WRITE(DPLL(PIPE_C), I915_READ(DPLL(PIPE_C)) |
>+			   DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
>+	}
>+	udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
>+	vlv_set_power_well(dev_priv, power_well, true);
>+
>+	/* Poll for phypwrgood signal */
>+	if (wait_for(I915_READ(DISPLAY_PHY_STATUS) & PHY_POWERGOOD(phy), 1))
>+		DRM_ERROR("Display PHY %d is not power up\n", phy);
>+
>+	I915_WRITE(DISPLAY_PHY_CONTROL, I915_READ(DISPLAY_PHY_CONTROL) |
>+		   PHY_COM_LANE_RESET_DEASSERT(phy));
>+}
>+
>+static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
>+					    struct i915_power_well *power_well)
>+{
>+	enum dpio_phy phy;
>+
>+	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC &&
>+		     power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D);
>+
>+	if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
>+		phy = DPIO_PHY0;
>+		assert_pll_disabled(dev_priv, PIPE_A);
>+		assert_pll_disabled(dev_priv, PIPE_B);
>+	} else {
>+		phy = DPIO_PHY1;
>+		assert_pll_disabled(dev_priv, PIPE_C);
>+	}
>+
>+	I915_WRITE(DISPLAY_PHY_CONTROL, I915_READ(DISPLAY_PHY_CONTROL) &
>+		   ~PHY_COM_LANE_RESET_DEASSERT(phy));
>+
>+	vlv_set_power_well(dev_priv, power_well, false);
>+}
>+
>+static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
>+					struct i915_power_well *power_well)
>+{
>+	enum pipe pipe = power_well->data;
>+	bool enabled;
>+	u32 state, ctrl;
>+
>+	mutex_lock(&dev_priv->rps.hw_lock);
>+
>+	state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
>+	/*
>+	 * We only ever set the power-on and power-gate states, anything
>+	 * else is unexpected.
>+	 */
>+	WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
>+	enabled = state == DP_SSS_PWR_ON(pipe);
>+
>+	/*
>+	 * A transient state at this point would mean some unexpected party
>+	 * is poking at the power controls too.
>+	 */
>+	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
>+	WARN_ON(ctrl << 16 != state);
>+
>+	mutex_unlock(&dev_priv->rps.hw_lock);
>+
>+	return enabled;
>+}
>+
>+static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
>+				    struct i915_power_well *power_well,
>+				    bool enable)
>+{
>+	enum pipe pipe = power_well->data;
>+	u32 state;
>+	u32 ctrl;
>+
>+	state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
>+
>+	mutex_lock(&dev_priv->rps.hw_lock);
>+
>+#define COND \
>+	((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
>+
>+	if (COND)
>+		goto out;
>+
>+	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
>+	ctrl &= ~DP_SSC_MASK(pipe);
>+	ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
>+	vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
>+
>+	if (wait_for(COND, 100))
>+		DRM_ERROR("timout setting power well state %08x (%08x)\n",
>+			  state,
>+			  vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
>+
>+#undef COND
>+
>+out:
>+	mutex_unlock(&dev_priv->rps.hw_lock);
>+}
>+
>+static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
>+					struct i915_power_well *power_well)
>+{
>+	chv_set_pipe_power_well(dev_priv, power_well, power_well->count > 0);
>+}
>+
>+static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
>+				       struct i915_power_well *power_well)
>+{
>+	WARN_ON_ONCE(power_well->data != PIPE_A &&
>+		     power_well->data != PIPE_B &&
>+		     power_well->data != PIPE_C);
>+
>+	chv_set_pipe_power_well(dev_priv, power_well, true);
>+}
>+
>+static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
>+					struct i915_power_well *power_well)
>+{
>+	WARN_ON_ONCE(power_well->data != PIPE_A &&
>+		     power_well->data != PIPE_B &&
>+		     power_well->data != PIPE_C);
>+
>+	chv_set_pipe_power_well(dev_priv, power_well, false);
>+}
>+
> static void check_power_well_state(struct drm_i915_private *dev_priv,
> 				   struct i915_power_well *power_well)
> {
>@@ -6443,6 +6824,39 @@ EXPORT_SYMBOL_GPL(i915_get_cdclk_freq);
> 	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |	\
> 	BIT(POWER_DOMAIN_INIT))
> 
>+#define CHV_PIPE_A_POWER_DOMAINS (	\
>+	BIT(POWER_DOMAIN_PIPE_A) |	\
>+	BIT(POWER_DOMAIN_INIT))
>+
>+#define CHV_PIPE_B_POWER_DOMAINS (	\
>+	BIT(POWER_DOMAIN_PIPE_B) |	\
>+	BIT(POWER_DOMAIN_INIT))
>+
>+#define CHV_PIPE_C_POWER_DOMAINS (	\
>+	BIT(POWER_DOMAIN_PIPE_C) |	\
>+	BIT(POWER_DOMAIN_INIT))
>+
>+#define CHV_DPIO_CMN_BC_POWER_DOMAINS (		\
>+	BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |	\
>+	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |	\
>+	BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |	\
>+	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |	\
>+	BIT(POWER_DOMAIN_INIT))
>+
>+#define CHV_DPIO_CMN_D_POWER_DOMAINS (		\
>+	BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |	\
>+	BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |	\
>+	BIT(POWER_DOMAIN_INIT))
>+
>+#define CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS (	\
>+	BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |	\
>+	BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |	\
>+	BIT(POWER_DOMAIN_INIT))
>+
>+#define CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS (	\
>+	BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |	\
>+	BIT(POWER_DOMAIN_INIT))
>+
> static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
> 	.sync_hw = i9xx_always_on_power_well_noop,
> 	.enable = i9xx_always_on_power_well_noop,
>@@ -6450,6 +6864,20 @@ static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
> 	.is_enabled = i9xx_always_on_power_well_enabled,
> };
> 
>+static const struct i915_power_well_ops chv_pipe_power_well_ops = {
>+	.sync_hw = chv_pipe_power_well_sync_hw,
>+	.enable = chv_pipe_power_well_enable,
>+	.disable = chv_pipe_power_well_disable,
>+	.is_enabled = chv_pipe_power_well_enabled,
>+};
>+
>+static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
>+	.sync_hw = vlv_power_well_sync_hw,
>+	.enable = chv_dpio_cmn_power_well_enable,
>+	.disable = chv_dpio_cmn_power_well_disable,
>+	.is_enabled = vlv_power_well_enabled,
>+};
>+
> static struct i915_power_well i9xx_always_on_power_well[] = {
> 	{
> 		.name = "always-on",
>@@ -6572,6 +7000,107 @@ static struct i915_power_well vlv_power_wells[] = {
> 	},
> };
> 
>+static struct i915_power_well chv_power_wells[] = {
>+	{
>+		.name = "always-on",
>+		.always_on = 1,
>+		.domains = VLV_ALWAYS_ON_POWER_DOMAINS,
>+		.ops = &i9xx_always_on_power_well_ops,
>+	},
>+#if 0
>+	{
>+		.name = "display",
>+		.domains = VLV_DISPLAY_POWER_DOMAINS,
>+		.data = PUNIT_POWER_WELL_DISP2D,
>+		.ops = &vlv_display_power_well_ops,
>+	},
>+	{
>+		.name = "pipe-a",
>+		.domains = CHV_PIPE_A_POWER_DOMAINS,
>+		.data = PIPE_A,
>+		.ops = &chv_pipe_power_well_ops,
>+	},
>+	{
>+		.name = "pipe-b",
>+		.domains = CHV_PIPE_B_POWER_DOMAINS,
>+		.data = PIPE_B,
>+		.ops = &chv_pipe_power_well_ops,
>+	},
>+	{
>+		.name = "pipe-c",
>+		.domains = CHV_PIPE_C_POWER_DOMAINS,
>+		.data = PIPE_C,
>+		.ops = &chv_pipe_power_well_ops,
>+	},
>+#endif
>+	{
>+		.name = "dpio-common-bc",
>+		/*
>+		 * XXX: cmnreset for one PHY seems to disturb the other.
>+		 * As a workaround keep both powered on at the same
>+		 * time for now.
>+		 */
>+		.domains = CHV_DPIO_CMN_BC_POWER_DOMAINS | CHV_DPIO_CMN_D_POWER_DOMAINS,
>+		.data = PUNIT_POWER_WELL_DPIO_CMN_BC,
>+		.ops = &chv_dpio_cmn_power_well_ops,
>+	},
>+	{
>+		.name = "dpio-common-d",
>+		/*
>+		 * XXX: cmnreset for one PHY seems to disturb the other.
>+		 * As a workaround keep both powered on at the same
>+		 * time for now.
>+		 */
>+		.domains = CHV_DPIO_CMN_BC_POWER_DOMAINS | CHV_DPIO_CMN_D_POWER_DOMAINS,
>+		.data = PUNIT_POWER_WELL_DPIO_CMN_D,
>+		.ops = &chv_dpio_cmn_power_well_ops,
>+	},
>+#if 0
>+	{
>+		.name = "dpio-tx-b-01",
>+		.domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
>+			   VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS,
>+		.ops = &vlv_dpio_power_well_ops,
>+		.data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
>+	},
>+	{
>+		.name = "dpio-tx-b-23",
>+		.domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
>+			   VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS,
>+		.ops = &vlv_dpio_power_well_ops,
>+		.data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
>+	},
>+	{
>+		.name = "dpio-tx-c-01",
>+		.domains = VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
>+			   VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
>+		.ops = &vlv_dpio_power_well_ops,
>+		.data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
>+	},
>+	{
>+		.name = "dpio-tx-c-23",
>+		.domains = VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
>+			   VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
>+		.ops = &vlv_dpio_power_well_ops,
>+		.data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
>+	},
>+	{
>+		.name = "dpio-tx-d-01",
>+		.domains = CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS |
>+			   CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS,
>+		.ops = &vlv_dpio_power_well_ops,
>+		.data = PUNIT_POWER_WELL_DPIO_TX_D_LANES_01,
>+	},
>+	{
>+		.name = "dpio-tx-d-23",
>+		.domains = CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS |
>+			   CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS,
>+		.ops = &vlv_dpio_power_well_ops,
>+		.data = PUNIT_POWER_WELL_DPIO_TX_D_LANES_23,
>+	},
>+#endif
>+};
>+
> static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv,
> 						 enum punit_power_well power_well_id)
> {
>@@ -6608,6 +7137,8 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
> 	} else if (IS_BROADWELL(dev_priv->dev)) {
> 		set_power_wells(power_domains, bdw_power_wells);
> 		hsw_pwr = power_domains;
>+	} else if (IS_CHERRYVIEW(dev_priv->dev)) {
>+		set_power_wells(power_domains, chv_power_wells);
> 	} else if (IS_VALLEYVIEW(dev_priv->dev)) {
> 		set_power_wells(power_domains, vlv_power_wells);
> 	} else {
>@@ -6833,13 +7364,15 @@ void intel_init_pm(struct drm_device *dev)
> 		else if (IS_HASWELL(dev))
> 			dev_priv->display.init_clock_gating = haswell_init_clock_gating;
> 		else if (INTEL_INFO(dev)->gen == 8)
>-			dev_priv->display.init_clock_gating = gen8_init_clock_gating;
>+			dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
> 	} else if (IS_CHERRYVIEW(dev)) {
>-		dev_priv->display.update_wm = valleyview_update_wm;
>+		dev_priv->display.update_wm = cherryview_update_wm;
>+		dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
> 		dev_priv->display.init_clock_gating =
> 			cherryview_init_clock_gating;
> 	} else if (IS_VALLEYVIEW(dev)) {
> 		dev_priv->display.update_wm = valleyview_update_wm;
>+		dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
> 		dev_priv->display.init_clock_gating =
> 			valleyview_init_clock_gating;
> 	} else if (IS_PINEVIEW(dev)) {
>@@ -7025,6 +7558,7 @@ static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
> 		return -1;
> 	}
> 
>+	/* CHV needs even values */
> 	opcode = (DIV_ROUND_CLOSEST((val * 2 * mul), dev_priv->rps.cz_freq) * 2);
> 
> 	return opcode;
>diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
>index 47a126a..24e7ec7 100644
>--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
>+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
>@@ -33,75 +33,39 @@
> #include "i915_trace.h"
> #include "intel_drv.h"
> 
>-/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
>- * but keeps the logic simple. Indeed, the whole purpose of this macro is just
>- * to give some inclination as to some of the magic values used in the various
>- * workarounds!
>- */
>-#define CACHELINE_BYTES 64
>-
>-static inline int __ring_space(int head, int tail, int size)
>-{
>-	int space = head - (tail + I915_RING_FREE_SPACE);
>-	if (space < 0)
>-		space += size;
>-	return space;
>-}
>-
>-static inline int ring_space(struct intel_ringbuffer *ringbuf)
>-{
>-	return __ring_space(ringbuf->head & HEAD_ADDR, ringbuf->tail, ringbuf->size);
>-}
>-
>-static bool intel_ring_stopped(struct intel_engine_cs *ring)
>-{
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>-	return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);
>-}
>-
>-void __intel_ring_advance(struct intel_engine_cs *ring)
>-{
>-	struct intel_ringbuffer *ringbuf = ring->buffer;
>-	ringbuf->tail &= ringbuf->size - 1;
>-	if (intel_ring_stopped(ring))
>-		return;
>-	ring->write_tail(ring, ringbuf->tail);
>-}
>+/* Just userspace ABI convention to limit the wa batch bo to a resonable size */
>+#define I830_BATCH_LIMIT (256*1024)
>+#define I830_TLB_ENTRIES (2)
>+#define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
> 
> static int
>-gen2_render_ring_flush(struct intel_engine_cs *ring,
>-		       u32	invalidate_domains,
>-		       u32	flush_domains)
>+gen2_emit_flush(struct i915_gem_request *rq, u32 flags)
> {
>+	struct intel_ringbuffer *ring;
> 	u32 cmd;
>-	int ret;
> 
> 	cmd = MI_FLUSH;
>-	if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
>+	if ((flags & (I915_FLUSH_CACHES | I915_INVALIDATE_CACHES)) == 0)
> 		cmd |= MI_NO_WRITE_FLUSH;
> 
>-	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
>+	if (flags & I915_INVALIDATE_CACHES)
> 		cmd |= MI_READ_FLUSH;
> 
>-	ret = intel_ring_begin(ring, 2);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 1);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring, cmd);
>-	intel_ring_emit(ring, MI_NOOP);
> 	intel_ring_advance(ring);
> 
> 	return 0;
> }
> 
> static int
>-gen4_render_ring_flush(struct intel_engine_cs *ring,
>-		       u32	invalidate_domains,
>-		       u32	flush_domains)
>+gen4_emit_flush(struct i915_gem_request *rq, u32 flags)
> {
>-	struct drm_device *dev = ring->dev;
>+	struct intel_ringbuffer *ring;
> 	u32 cmd;
>-	int ret;
> 
> 	/*
> 	 * read/write caches:
>@@ -131,22 +95,20 @@ gen4_render_ring_flush(struct intel_engine_cs *ring,
> 	 * are flushed at any MI_FLUSH.
> 	 */
> 
>-	cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
>-	if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
>-		cmd &= ~MI_NO_WRITE_FLUSH;
>-	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
>+	cmd = MI_FLUSH;
>+	if ((flags & (I915_FLUSH_CACHES | I915_INVALIDATE_CACHES)) == 0)
>+		cmd |= MI_NO_WRITE_FLUSH;
>+	if (flags & I915_INVALIDATE_CACHES) {
> 		cmd |= MI_EXE_FLUSH;
>+		if (IS_G4X(rq->i915) || IS_GEN5(rq->i915))
>+			cmd |= MI_INVALIDATE_ISP;
>+	}
> 
>-	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
>-	    (IS_G4X(dev) || IS_GEN5(dev)))
>-		cmd |= MI_INVALIDATE_ISP;
>-
>-	ret = intel_ring_begin(ring, 2);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 1);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring, cmd);
>-	intel_ring_emit(ring, MI_NOOP);
> 	intel_ring_advance(ring);
> 
> 	return 0;
>@@ -190,100 +152,89 @@ gen4_render_ring_flush(struct intel_engine_cs *ring,
>  * really our business.  That leaves only stall at scoreboard.
>  */
> static int
>-intel_emit_post_sync_nonzero_flush(struct intel_engine_cs *ring)
>+gen6_emit_post_sync_nonzero_flush(struct i915_gem_request *rq)
> {
>-	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
>-	int ret;
>-
>+	const u32 scratch = rq->engine->scratch.gtt_offset + 2*CACHELINE_BYTES;
>+	struct intel_ringbuffer *ring;
> 
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 8);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
>-	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
>+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
> 	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
> 			PIPE_CONTROL_STALL_AT_SCOREBOARD);
>-	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
>-	intel_ring_emit(ring, 0); /* low dword */
>-	intel_ring_emit(ring, 0); /* high dword */
>-	intel_ring_emit(ring, MI_NOOP);
>-	intel_ring_advance(ring);
>-
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret)
>-		return ret;
>+	intel_ring_emit(ring, scratch | PIPE_CONTROL_GLOBAL_GTT);
>+	intel_ring_emit(ring, 0);
> 
>-	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
>+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
> 	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
>-	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
>-	intel_ring_emit(ring, 0);
>+	intel_ring_emit(ring, scratch | PIPE_CONTROL_GLOBAL_GTT);
> 	intel_ring_emit(ring, 0);
>-	intel_ring_emit(ring, MI_NOOP);
>-	intel_ring_advance(ring);
> 
>+	intel_ring_advance(ring);
> 	return 0;
> }
> 
> static int
>-gen6_render_ring_flush(struct intel_engine_cs *ring,
>-                         u32 invalidate_domains, u32 flush_domains)
>+gen6_render_emit_flush(struct i915_gem_request *rq, u32 flags)
> {
>-	u32 flags = 0;
>-	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
>+	const u32 scratch = rq->engine->scratch.gtt_offset + 2*CACHELINE_BYTES;
>+	struct intel_ringbuffer *ring;
>+	u32 cmd = 0;
> 	int ret;
> 
>-	/* Force SNB workarounds for PIPE_CONTROL flushes */
>-	ret = intel_emit_post_sync_nonzero_flush(ring);
>-	if (ret)
>-		return ret;
>+	if (flags & I915_FLUSH_CACHES) {
>+		/* Force SNB workarounds for PIPE_CONTROL flushes */
>+		ret = gen6_emit_post_sync_nonzero_flush(rq);
>+		if (ret)
>+			return ret;
> 
>-	/* Just flush everything.  Experiments have shown that reducing the
>-	 * number of bits based on the write domains has little performance
>-	 * impact.
>-	 */
>-	if (flush_domains) {
>-		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
>-		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
>-		/*
>-		 * Ensure that any following seqno writes only happen
>-		 * when the render cache is indeed flushed.
>-		 */
>-		flags |= PIPE_CONTROL_CS_STALL;
>-	}
>-	if (invalidate_domains) {
>-		flags |= PIPE_CONTROL_TLB_INVALIDATE;
>-		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
>+		cmd |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
>+	}
>+	if (flags & I915_INVALIDATE_CACHES) {
>+		cmd |= PIPE_CONTROL_TLB_INVALIDATE;
>+		cmd |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
> 		/*
> 		 * TLB invalidate requires a post-sync write.
> 		 */
>-		flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
>+		cmd |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
> 	}
>+	if (flags & I915_COMMAND_BARRIER)
>+		/*
>+		 * Ensure that any following seqno writes only happen
>+		 * when the render cache is indeed flushed.
>+		 */
>+		cmd |= PIPE_CONTROL_CS_STALL;
> 
>-	ret = intel_ring_begin(ring, 4);
>-	if (ret)
>-		return ret;
>+	if (cmd) {
>+		ring = intel_ring_begin(rq, 4);
>+		if (IS_ERR(ring))
>+			return PTR_ERR(ring);
> 
>-	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
>-	intel_ring_emit(ring, flags);
>-	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
>-	intel_ring_emit(ring, 0);
>-	intel_ring_advance(ring);
>+		intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
>+		intel_ring_emit(ring, cmd);
>+		intel_ring_emit(ring, scratch | PIPE_CONTROL_GLOBAL_GTT);
>+		intel_ring_emit(ring, 0);
>+		intel_ring_advance(ring);
>+	}
> 
> 	return 0;
> }
> 
> static int
>-gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring)
>+gen7_render_ring_cs_stall_wa(struct i915_gem_request *rq)
> {
>-	int ret;
>+	struct intel_ringbuffer *ring;
> 
>-	ret = intel_ring_begin(ring, 4);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 4);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
> 	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
>@@ -295,35 +246,32 @@ gen7_render_ring_cs_stall_wa(struct intel_engine_cs *ring)
> 	return 0;
> }
> 
>-static int gen7_ring_fbc_flush(struct intel_engine_cs *ring, u32 value)
>+static int gen7_ring_fbc_flush(struct i915_gem_request *rq, u32 value)
> {
>-	int ret;
>+	struct intel_ringbuffer *ring;
> 
>-	if (!ring->fbc_dirty)
>-		return 0;
>+	ring = intel_ring_begin(rq, 6);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(rq);
> 
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret)
>-		return ret;
> 	/* WaFbcNukeOn3DBlt:ivb/hsw */
> 	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
> 	intel_ring_emit(ring, MSG_FBC_REND_STATE);
> 	intel_ring_emit(ring, value);
> 	intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) | MI_SRM_LRM_GLOBAL_GTT);
> 	intel_ring_emit(ring, MSG_FBC_REND_STATE);
>-	intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
>+	intel_ring_emit(ring, rq->engine->scratch.gtt_offset + 256);
> 	intel_ring_advance(ring);
> 
>-	ring->fbc_dirty = false;
> 	return 0;
> }
> 
> static int
>-gen7_render_ring_flush(struct intel_engine_cs *ring,
>-		       u32 invalidate_domains, u32 flush_domains)
>+gen7_render_emit_flush(struct i915_gem_request *rq, u32 flags)
> {
>-	u32 flags = 0;
>-	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
>+	const u32 scratch_addr = rq->engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
>+	struct intel_ringbuffer *ring;
>+	u32 cmd = 0;
> 	int ret;
> 
> 	/*
>@@ -334,63 +282,71 @@ gen7_render_ring_flush(struct intel_engine_cs *ring,
> 	 * read-cache invalidate bits set) must have the CS_STALL bit set. We
> 	 * don't try to be clever and just set it unconditionally.
> 	 */
>-	flags |= PIPE_CONTROL_CS_STALL;
>+	cmd |= PIPE_CONTROL_CS_STALL;
> 
> 	/* Just flush everything.  Experiments have shown that reducing the
> 	 * number of bits based on the write domains has little performance
> 	 * impact.
> 	 */
>-	if (flush_domains) {
>-		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
>-		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
>-	}
>-	if (invalidate_domains) {
>-		flags |= PIPE_CONTROL_TLB_INVALIDATE;
>-		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
>+	if (flags & I915_FLUSH_CACHES) {
>+		cmd |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
>+		cmd |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
>+	}
>+	if (flags & I915_INVALIDATE_CACHES) {
>+		cmd |= PIPE_CONTROL_TLB_INVALIDATE;
>+		cmd |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
> 		/*
> 		 * TLB invalidate requires a post-sync write.
> 		 */
>-		flags |= PIPE_CONTROL_QW_WRITE;
>-		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
>+		cmd |= PIPE_CONTROL_QW_WRITE;
>+		cmd |= PIPE_CONTROL_GLOBAL_GTT_IVB;
> 
> 		/* Workaround: we must issue a pipe_control with CS-stall bit
> 		 * set before a pipe_control command that has the state cache
> 		 * invalidate bit set. */
>-		gen7_render_ring_cs_stall_wa(ring);
>+		ret = gen7_render_ring_cs_stall_wa(rq);
>+		if (ret)
>+			return ret;
> 	}
> 
>-	ret = intel_ring_begin(ring, 4);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 4);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
>-	intel_ring_emit(ring, flags);
>+	intel_ring_emit(ring, cmd);
> 	intel_ring_emit(ring, scratch_addr);
> 	intel_ring_emit(ring, 0);
> 	intel_ring_advance(ring);
> 
>-	if (!invalidate_domains && flush_domains)
>-		return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
>+	if (flags & I915_KICK_FBC) {
>+		ret = gen7_ring_fbc_flush(rq, FBC_REND_NUKE);
>+		if (ret)
>+			return ret;
>+	}
> 
> 	return 0;
> }
> 
> static int
>-gen8_emit_pipe_control(struct intel_engine_cs *ring,
>-		       u32 flags, u32 scratch_addr)
>+gen8_emit_pipe_control(struct i915_gem_request *rq,
>+		       u32 cmd, u32 scratch_addr)
> {
>-	int ret;
>+	struct intel_ringbuffer *ring;
> 
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret)
>-		return ret;
>+	if (cmd == 0)
>+		return 0;
>+
>+	ring = intel_ring_begin(rq, 6);
>+	if (IS_ERR(rq))
>+		return PTR_ERR(rq);
> 
> 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
>-	intel_ring_emit(ring, flags);
>+	intel_ring_emit(ring, cmd);
> 	intel_ring_emit(ring, scratch_addr);
> 	intel_ring_emit(ring, 0);
> 	intel_ring_emit(ring, 0);
>@@ -401,31 +357,29 @@ gen8_emit_pipe_control(struct intel_engine_cs *ring,
> }
> 
> static int
>-gen8_render_ring_flush(struct intel_engine_cs *ring,
>-		       u32 invalidate_domains, u32 flush_domains)
>+gen8_render_emit_flush(struct i915_gem_request *rq,
>+		       u32 flags)
> {
>-	u32 flags = 0;
>-	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
>+	const u32 scratch_addr = rq->engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
>+	u32 cmd = 0;
> 	int ret;
> 
>-	flags |= PIPE_CONTROL_CS_STALL;
>-
>-	if (flush_domains) {
>-		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
>-		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
>+	if (flags & I915_FLUSH_CACHES) {
>+		cmd |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
>+		cmd |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
> 	}
>-	if (invalidate_domains) {
>-		flags |= PIPE_CONTROL_TLB_INVALIDATE;
>-		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
>-		flags |= PIPE_CONTROL_QW_WRITE;
>-		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
>+	if (flags & I915_INVALIDATE_CACHES) {
>+		cmd |= PIPE_CONTROL_TLB_INVALIDATE;
>+		cmd |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
>+		cmd |= PIPE_CONTROL_QW_WRITE;
>+		cmd |= PIPE_CONTROL_GLOBAL_GTT_IVB;
> 
> 		/* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
>-		ret = gen8_emit_pipe_control(ring,
>+		ret = gen8_emit_pipe_control(rq,
> 					     PIPE_CONTROL_CS_STALL |
> 					     PIPE_CONTROL_STALL_AT_SCOREBOARD,
> 					     0);
>@@ -433,200 +387,481 @@ gen8_render_ring_flush(struct intel_engine_cs *ring,
> 			return ret;
> 	}
> 
>-	return gen8_emit_pipe_control(ring, flags, scratch_addr);
>+	if (flags & I915_COMMAND_BARRIER)
>+		cmd |= PIPE_CONTROL_CS_STALL;
>+
>+
>+	ret = gen8_emit_pipe_control(rq, cmd, scratch_addr);
>+	if (ret)
>+		return ret;
>+
>+	if (flags & I915_KICK_FBC) {
>+		ret = gen7_ring_fbc_flush(rq, FBC_REND_NUKE);
>+		if (ret)
>+			return ret;
>+	}
>+
>+	return 0;
> }
> 
>-static void ring_write_tail(struct intel_engine_cs *ring,
>+static void ring_write_tail(struct intel_engine_cs *engine,
> 			    u32 value)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>-	I915_WRITE_TAIL(ring, value);
>+	struct drm_i915_private *dev_priv = engine->i915;
>+	I915_WRITE_TAIL(engine, value);
> }
> 
>-u64 intel_ring_get_active_head(struct intel_engine_cs *ring)
>+u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	u64 acthd;
> 
>-	if (INTEL_INFO(ring->dev)->gen >= 8)
>-		acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),
>-					 RING_ACTHD_UDW(ring->mmio_base));
>-	else if (INTEL_INFO(ring->dev)->gen >= 4)
>-		acthd = I915_READ(RING_ACTHD(ring->mmio_base));
>+	if (INTEL_INFO(dev_priv)->gen >= 8)
>+		acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
>+					 RING_ACTHD_UDW(engine->mmio_base));
>+	else if (INTEL_INFO(dev_priv)->gen >= 4)
>+		acthd = I915_READ(RING_ACTHD(engine->mmio_base));
> 	else
> 		acthd = I915_READ(ACTHD);
> 
> 	return acthd;
> }
> 
>-static void ring_setup_phys_status_page(struct intel_engine_cs *ring)
>+static bool engine_stop(struct intel_engine_cs *engine)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>-	u32 addr;
>+	struct drm_i915_private *dev_priv = engine->i915;
>+
>+	if (!IS_GEN2(dev_priv)) {
>+		I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
>+		if (wait_for((I915_READ_MODE(engine) & MODE_IDLE) != 0, 1000)) {
>+			DRM_ERROR("%s : timed out trying to stop ring\n", engine->name);
>+			/* Sometimes we observe that the idle flag is not
>+			 * set even though the ring is empty. So double
>+			 * check before giving up.
>+			 */
>+			if (I915_READ_HEAD(engine) != I915_READ_TAIL(engine))
>+				return false;
>+		}
>+	}
> 
>-	addr = dev_priv->status_page_dmah->busaddr;
>-	if (INTEL_INFO(ring->dev)->gen >= 4)
>-		addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
>-	I915_WRITE(HWS_PGA, addr);
>+	I915_WRITE_CTL(engine, 0);
>+	I915_WRITE_HEAD(engine, 0);
>+	engine->write_tail(engine, 0);
>+
>+	if (!IS_GEN2(dev_priv)) {
>+		(void)I915_READ_CTL(engine);
>+		I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
>+	}
>+
>+	return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0;
>+}
>+
>+static int engine_suspend(struct intel_engine_cs *engine)
>+{
>+	return engine_stop(engine) ? 0 : -EIO;
> }
> 
>-static bool stop_ring(struct intel_engine_cs *ring)
>+static int enable_status_page(struct intel_engine_cs *engine)
> {
>-	struct drm_i915_private *dev_priv = to_i915(ring->dev);
>+	struct drm_i915_private *dev_priv = engine->i915;
>+	u32 mmio, addr;
>+	int ret = 0;
> 
>-	if (!IS_GEN2(ring->dev)) {
>-		I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
>-		if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
>-			DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
>-			return false;
>+	if (!I915_NEED_GFX_HWS(dev_priv)) {
>+		addr = dev_priv->status_page_dmah->busaddr;
>+		if (INTEL_INFO(dev_priv)->gen >= 4)
>+			addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
>+		mmio = HWS_PGA;
>+	} else {
>+		addr = engine->status_page.gfx_addr;
>+		/* The ring status page addresses are no longer next to the rest of
>+		 * the ring registers as of gen7.
>+		 */
>+		if (IS_GEN7(dev_priv)) {
>+			switch (engine->id) {
>+			default:
>+			case RCS:
>+				mmio = RENDER_HWS_PGA_GEN7;
>+				break;
>+			case BCS:
>+				mmio = BLT_HWS_PGA_GEN7;
>+				break;
>+				/*
>+				 * VCS2 actually doesn't exist on Gen7. Only shut up
>+				 * gcc switch check warning
>+				 */
>+			case VCS2:
>+			case VCS:
>+				mmio = BSD_HWS_PGA_GEN7;
>+				break;
>+			case VECS:
>+				mmio = VEBOX_HWS_PGA_GEN7;
>+				break;
>+			}
>+		} else if (IS_GEN6(dev_priv)) {
>+			mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
>+		} else {
>+			/* XXX: gen8 returns to sanity */
>+			mmio = RING_HWS_PGA(engine->mmio_base);
> 		}
> 	}
> 
>-	I915_WRITE_CTL(ring, 0);
>-	I915_WRITE_HEAD(ring, 0);
>-	ring->write_tail(ring, 0);
>+	I915_WRITE(mmio, addr);
>+	POSTING_READ(mmio);
>+
>+	/*
>+	 * Flush the TLB for this page
>+	 *
>+	 * FIXME: These two bits have disappeared on gen8, so a question
>+	 * arises: do we still need this and if so how should we go about
>+	 * invalidating the TLB?
>+	 */
>+	if (INTEL_INFO(dev_priv)->gen >= 6 && INTEL_INFO(dev_priv)->gen < 8) {
>+		u32 reg = RING_INSTPM(engine->mmio_base);
>+
>+		/* ring should be idle before issuing a sync flush*/
>+		WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
> 
>-	if (!IS_GEN2(ring->dev)) {
>-		(void)I915_READ_CTL(ring);
>-		I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
>+		I915_WRITE(reg,
>+			   _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
>+					      INSTPM_SYNC_FLUSH));
>+		if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
>+			     1000)) {
>+			DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
>+				  engine->name);
>+			ret = -EIO;
>+		}
> 	}
> 
>-	return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0;
>+	return ret;
> }
> 
>-static int init_ring_common(struct intel_engine_cs *ring)
>+static struct intel_ringbuffer *
>+engine_get_ring(struct intel_engine_cs *engine,
>+		struct intel_context *ctx)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_ringbuffer *ringbuf = ring->buffer;
>-	struct drm_i915_gem_object *obj = ringbuf->obj;
>+	struct drm_i915_private *dev_priv = engine->i915;
>+	struct intel_ringbuffer *ring;
> 	int ret = 0;
> 
>-	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
>+	ring = engine->legacy_ring;
>+	if (ring)
>+		return ring;
>+
>+	ring = intel_engine_alloc_ring(engine, ctx, 32 * PAGE_SIZE);
>+	if (IS_ERR(ring)) {
>+		DRM_ERROR("Failed to allocate ringbuffer for %s: %ld\n", engine->name, PTR_ERR(ring));
>+		return ERR_CAST(ring);
>+	}
> 
>-	if (!stop_ring(ring)) {
>+	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
>+	if (!engine_stop(engine)) {
> 		/* G45 ring initialization often fails to reset head to zero */
> 		DRM_DEBUG_KMS("%s head not reset to zero "
> 			      "ctl %08x head %08x tail %08x start %08x\n",
>-			      ring->name,
>-			      I915_READ_CTL(ring),
>-			      I915_READ_HEAD(ring),
>-			      I915_READ_TAIL(ring),
>-			      I915_READ_START(ring));
>-
>-		if (!stop_ring(ring)) {
>+			      engine->name,
>+			      I915_READ_CTL(engine),
>+			      I915_READ_HEAD(engine),
>+			      I915_READ_TAIL(engine),
>+			      I915_READ_START(engine));
>+		if (!engine_stop(engine)) {
> 			DRM_ERROR("failed to set %s head to zero "
> 				  "ctl %08x head %08x tail %08x start %08x\n",
>-				  ring->name,
>-				  I915_READ_CTL(ring),
>-				  I915_READ_HEAD(ring),
>-				  I915_READ_TAIL(ring),
>-				  I915_READ_START(ring));
>+				  engine->name,
>+				  I915_READ_CTL(engine),
>+				  I915_READ_HEAD(engine),
>+				  I915_READ_TAIL(engine),
>+				  I915_READ_START(engine));
> 			ret = -EIO;
>-			goto out;
> 		}
> 	}
>+	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
>+
>+	if (ret == 0) {
>+		engine->legacy_ring = ring;
>+	} else {
>+		intel_ring_free(ring);
>+		ring = ERR_PTR(ret);
>+	}
> 
>-	if (I915_NEED_GFX_HWS(dev))
>-		intel_ring_setup_status_page(ring);
>-	else
>-		ring_setup_phys_status_page(ring);
>+	return ring;
>+}
>+
>+static int engine_resume(struct intel_engine_cs *engine)
>+{
>+	struct drm_i915_private *dev_priv = engine->i915;
>+	struct intel_ringbuffer *ring = engine->legacy_ring;
>+	int retry = 3, ret;
>+
>+	if (WARN_ON(ring == NULL))
>+		return -ENODEV;
> 
>+	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
>+
>+	ret = enable_status_page(engine);
>+
>+reset:
> 	/* Enforce ordering by reading HEAD register back */
>-	I915_READ_HEAD(ring);
>+	engine->write_tail(engine, ring->tail);
>+	I915_WRITE_HEAD(engine, ring->head);
>+	(void)I915_READ_HEAD(engine);
> 
> 	/* Initialize the ring. This must happen _after_ we've cleared the ring
> 	 * registers with the above sequence (the readback of the HEAD registers
> 	 * also enforces ordering), otherwise the hw might lose the new ring
> 	 * register values. */
>-	I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
>-	I915_WRITE_CTL(ring,
>-			((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
>-			| RING_VALID);
>-
>-	/* If the head is still not zero, the ring is dead */
>-	if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
>-		     I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
>-		     (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
>+	I915_WRITE_START(engine, i915_gem_obj_ggtt_offset(ring->obj));
>+
>+	/* WaClearRingBufHeadRegAtInit:ctg,elk */
>+	if (I915_READ_HEAD(engine) != ring->head)
>+		DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
>+			  engine->name, I915_READ_HEAD(engine));
>+	I915_WRITE_HEAD(engine, ring->head);
>+	(void)I915_READ_HEAD(engine);
>+
>+	I915_WRITE_CTL(engine,
>+		       ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
>+		       | RING_VALID);
>+
>+	if (wait_for((I915_READ_CTL(engine) & RING_VALID) != 0, 50)) {
>+		if (retry-- && engine_stop(engine))
>+			goto reset;
>+	}
>+
>+	if ((I915_READ_CTL(engine) & RING_VALID) == 0 ||
>+	    I915_READ_START(engine) != i915_gem_obj_ggtt_offset(ring->obj)) {
> 		DRM_ERROR("%s initialization failed "
>-			  "ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
>-			  ring->name,
>-			  I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID,
>-			  I915_READ_HEAD(ring), I915_READ_TAIL(ring),
>-			  I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj));
>+			  "ctl %08x (valid? %d) head %08x [expected %08x], tail %08x [expected %08x], start %08x [expected %08lx]\n",
>+			  engine->name,
>+			  I915_READ_CTL(engine), I915_READ_CTL(engine) & RING_VALID,
>+			  I915_READ_HEAD(engine), ring->head,
>+			  I915_READ_TAIL(engine), ring->tail,
>+			  I915_READ_START(engine), (unsigned long)i915_gem_obj_ggtt_offset(ring->obj));
> 		ret = -EIO;
>-		goto out;
> 	}
> 
>-	if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
>-		i915_kernel_lost_context(ring->dev);
>-	else {
>-		ringbuf->head = I915_READ_HEAD(ring);
>-		ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
>-		ringbuf->space = ring_space(ringbuf);
>-		ringbuf->last_retired_head = -1;
>-	}
>+	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
>+	return ret;
>+}
> 
>-	memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
>+static void engine_put_ring(struct intel_ringbuffer *ring,
>+			    struct intel_context *ctx)
>+{
>+	if (ring->last_context == ctx) {
>+		struct i915_gem_request *rq;
>+		int ret = -EINVAL;
> 
>-out:
>-	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
>+		rq = intel_engine_alloc_request(ring->engine,
>+						ring->engine->default_context);
>+		if (!IS_ERR(rq)) {
>+			ret = i915_request_commit(rq);
>+			i915_request_put(rq);
>+		}
>+		if (WARN_ON(ret))
>+			ring->last_context = ring->engine->default_context;
>+	}
>+}
> 
>-	return ret;
>+static int engine_add_request(struct i915_gem_request *rq)
>+{
>+	rq->engine->write_tail(rq->engine, rq->tail);
>+	list_add_tail(&rq->engine_list, &rq->engine->requests);
>+	return 0;
>+}
>+
>+static bool engine_rq_is_complete(struct i915_gem_request *rq)
>+{
>+	return __i915_seqno_passed(rq->engine->get_seqno(rq->engine),
>+				   rq->seqno);
> }
> 
> static int
>-init_pipe_control(struct intel_engine_cs *ring)
>+init_broken_cs_tlb_wa(struct intel_engine_cs *engine)
> {
>+	struct drm_i915_gem_object *obj;
> 	int ret;
> 
>-	if (ring->scratch.obj)
>-		return 0;
>+	obj = i915_gem_object_create_stolen(engine->i915->dev, I830_WA_SIZE);
>+	if (obj == NULL)
>+		obj = i915_gem_alloc_object(engine->i915->dev, I830_WA_SIZE);
>+	if (obj == NULL) {
>+		DRM_ERROR("Failed to allocate batch bo\n");
>+		return -ENOMEM;
>+	}
>+
>+	ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
>+	if (ret != 0) {
>+		drm_gem_object_unreference(&obj->base);
>+		DRM_ERROR("Failed to ping batch bo\n");
>+		return ret;
>+	}
> 
>-	ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
>-	if (ring->scratch.obj == NULL) {
>+	engine->scratch.obj = obj;
>+	engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
>+	return 0;
>+}
>+
>+static int
>+init_pipe_control(struct intel_engine_cs *engine)
>+{
>+	int ret;
>+
>+	engine->scratch.obj = i915_gem_alloc_object(engine->i915->dev, 4096);
>+	if (engine->scratch.obj == NULL) {
> 		DRM_ERROR("Failed to allocate seqno page\n");
> 		ret = -ENOMEM;
> 		goto err;
> 	}
> 
>-	ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
>+	ret = i915_gem_object_set_cache_level(engine->scratch.obj, I915_CACHE_LLC);
> 	if (ret)
> 		goto err_unref;
> 
>-	ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0);
>+	ret = i915_gem_obj_ggtt_pin(engine->scratch.obj, 4096, 0);
> 	if (ret)
> 		goto err_unref;
> 
>-	ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
>-	ring->scratch.cpu_page = kmap(sg_page(ring->scratch.obj->pages->sgl));
>-	if (ring->scratch.cpu_page == NULL) {
>+	engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(engine->scratch.obj);
>+	engine->scratch.cpu_page = kmap(sg_page(engine->scratch.obj->pages->sgl));
>+	if (engine->scratch.cpu_page == NULL) {
> 		ret = -ENOMEM;
> 		goto err_unpin;
> 	}
> 
> 	DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
>-			 ring->name, ring->scratch.gtt_offset);
>+			 engine->name, engine->scratch.gtt_offset);
> 	return 0;
> 
> err_unpin:
>-	i915_gem_object_ggtt_unpin(ring->scratch.obj);
>+	i915_gem_object_ggtt_unpin(engine->scratch.obj);
> err_unref:
>-	drm_gem_object_unreference(&ring->scratch.obj->base);
>+	drm_gem_object_unreference(&engine->scratch.obj->base);
> err:
>+	engine->scratch.obj = NULL;
> 	return ret;
> }
> 
>-static int init_render_ring(struct intel_engine_cs *ring)
>+static int
>+emit_lri(struct i915_gem_request *rq,
>+	 int num_registers,
>+	 ...)
>+{
>+	struct intel_ringbuffer *ring;
>+	va_list ap;
>+
>+	BUG_ON(num_registers > 60);
>+
>+	ring = intel_ring_begin(rq, 2*num_registers + 1);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
>+
>+	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(num_registers));
>+	va_start(ap, num_registers);
>+	while (num_registers--) {
>+		intel_ring_emit(ring, va_arg(ap, u32));
>+		intel_ring_emit(ring, va_arg(ap, u32));
>+	}
>+	va_end(ap);
>+	intel_ring_advance(ring);
>+
>+	return 0;
>+}
>+
>+static int bdw_render_init_context(struct i915_gem_request *rq)
>+{
>+	int ret;
>+
>+	ret = emit_lri(rq, 8,
>+
>+	/* FIXME: Unclear whether we really need this on production bdw. */
>+	GEN8_ROW_CHICKEN,
>+	/* WaDisablePartialInstShootdown:bdw */
>+	_MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE) |
>+	/* WaDisableThreadStallDopClockGating:bdw */
>+	_MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE),
>+
>+	GEN7_ROW_CHICKEN2,
>+	/* WaDisableDopClockGating:bdw May not be needed for production */
>+	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE),
>+
>+	/*
>+	 * This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for
>+	 * pre-production hardware
>+	 */
>+	HALF_SLICE_CHICKEN3,
>+	_MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS) |
>+	_MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS),
>+
>+	GEN7_HALF_SLICE_CHICKEN1,
>+	_MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE),
>+
>+	COMMON_SLICE_CHICKEN2,
>+	_MASKED_BIT_ENABLE(GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE),
>+
>+	/* Use Force Non-Coherent whenever executing a 3D context. This is a
>+	 * workaround for for a possible hang in the unlikely event a TLB
>+	 * invalidation occurs during a PSD flush.
>+	 */
>+	HDC_CHICKEN0,
>+	_MASKED_BIT_ENABLE(HDC_FORCE_NON_COHERENT),
>+
>+	CACHE_MODE_1,
>+	/* Wa4x4STCOptimizationDisable:bdw */
>+	_MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE),
>+
>+	/*
>+	 * BSpec recommends 8x4 when MSAA is used,
>+	 * however in practice 16x4 seems fastest.
>+	 *
>+	 * Note that PS/WM thread counts depend on the WIZ hashing
>+	 * disable bit, which we don't touch here, but it's good
>+	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
>+	 */
>+	GEN7_GT_MODE,
>+	GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
>+	if (ret)
>+		return ret;
>+
>+	return i915_gem_render_state_init(rq);
>+}
>+
>+static int chv_render_init_context(struct i915_gem_request *rq)
>+{
>+	int ret;
>+
>+	ret = emit_lri(rq, 3,
>+
>+	GEN8_ROW_CHICKEN,
>+	/* WaDisablePartialInstShootdown:chv */
>+	_MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE) |
>+	/* WaDisableThreadStallDopClockGating:chv */
>+	_MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE),
>+
>+	/* WaDisableDopClockGating:chv (pre-production hw) */
>+	GEN7_ROW_CHICKEN2,
>+	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE),
>+
>+	/* WaDisableSamplerPowerBypass:chv (pre-production hw) */
>+	HALF_SLICE_CHICKEN3,
>+	_MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
>+
>+	if (ret)
>+		return ret;
>+
>+	return i915_gem_render_state_init(rq);
>+}
>+
>+static int render_resume(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	int ret = init_ring_common(ring);
>+	struct drm_i915_private *dev_priv = engine->i915;
>+	int ret;
>+
>+	ret = engine_resume(engine);
> 	if (ret)
> 		return ret;
> 
> 	/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
>-	if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
>+	if (INTEL_INFO(dev_priv)->gen >= 4 && INTEL_INFO(dev_priv)->gen < 7)
> 		I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
> 
> 	/* We need to disable the AsyncFlip performance optimisations in order
>@@ -635,28 +870,22 @@ static int init_render_ring(struct intel_engine_cs *ring)
> 	 *
> 	 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
> 	 */
>-	if (INTEL_INFO(dev)->gen >= 6)
>+	if (INTEL_INFO(dev_priv)->gen >= 6)
> 		I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
> 
> 	/* Required for the hardware to program scanline values for waiting */
> 	/* WaEnableFlushTlbInvalidationMode:snb */
>-	if (INTEL_INFO(dev)->gen == 6)
>+	if (INTEL_INFO(dev_priv)->gen == 6)
> 		I915_WRITE(GFX_MODE,
> 			   _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
> 
> 	/* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
>-	if (IS_GEN7(dev))
>+	if (IS_GEN7(dev_priv))
> 		I915_WRITE(GFX_MODE_GEN7,
> 			   _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
> 			   _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
> 
>-	if (INTEL_INFO(dev)->gen >= 5) {
>-		ret = init_pipe_control(ring);
>-		if (ret)
>-			return ret;
>-	}
>-
>-	if (IS_GEN6(dev)) {
>+	if (IS_GEN6(dev_priv)) {
> 		/* From the Sandybridge PRM, volume 1 part 3, page 24:
> 		 * "If this bit is set, STCunit will have LRA as replacement
> 		 *  policy. [...] This bit must be reset.  LRA replacement
>@@ -666,19 +895,40 @@ static int init_render_ring(struct intel_engine_cs *ring)
> 			   _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
> 	}
> 
>-	if (INTEL_INFO(dev)->gen >= 6)
>+	if (INTEL_INFO(dev_priv)->gen >= 6)
> 		I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
> 
>-	if (HAS_L3_DPF(dev))
>-		I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
>+	return 0;
>+}
>+
>+static void cleanup_status_page(struct intel_engine_cs *engine)
>+{
>+	struct drm_i915_gem_object *obj;
> 
>-	return ret;
>+	obj = engine->status_page.obj;
>+	if (obj == NULL)
>+		return;
>+
>+	kunmap(sg_page(obj->pages->sgl));
>+	i915_gem_object_ggtt_unpin(obj);
>+	drm_gem_object_unreference(&obj->base);
>+	engine->status_page.obj = NULL;
>+}
>+
>+static void engine_cleanup(struct intel_engine_cs *engine)
>+{
>+	if (engine->legacy_ring)
>+		intel_ring_free(engine->legacy_ring);
>+
>+	cleanup_status_page(engine);
>+	i915_cmd_parser_fini_engine(engine);
> }
> 
>-static void render_ring_cleanup(struct intel_engine_cs *ring)
>+static void render_cleanup(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
>+
>+	engine_cleanup(engine);
> 
> 	if (dev_priv->semaphore_obj) {
> 		i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);
>@@ -686,163 +936,86 @@ static void render_ring_cleanup(struct intel_engine_cs *ring)
> 		dev_priv->semaphore_obj = NULL;
> 	}
> 
>-	if (ring->scratch.obj == NULL)
>-		return;
>-
>-	if (INTEL_INFO(dev)->gen >= 5) {
>-		kunmap(sg_page(ring->scratch.obj->pages->sgl));
>-		i915_gem_object_ggtt_unpin(ring->scratch.obj);
>+	if (engine->scratch.obj) {
>+		i915_gem_object_ggtt_unpin(engine->scratch.obj);
>+		drm_gem_object_unreference(&engine->scratch.obj->base);
>+		engine->scratch.obj = NULL;
> 	}
>-
>-	drm_gem_object_unreference(&ring->scratch.obj->base);
>-	ring->scratch.obj = NULL;
> }
> 
>-static int gen8_rcs_signal(struct intel_engine_cs *signaller,
>-			   unsigned int num_dwords)
>+static int
>+gen8_rcs_emit_signal(struct i915_gem_request *rq, int id)
> {
>-#define MBOX_UPDATE_DWORDS 8
>-	struct drm_device *dev = signaller->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *waiter;
>-	int i, ret, num_rings;
>+	u64 offset = GEN8_SEMAPHORE_OFFSET(rq->i915, rq->engine->id, id);
>+	struct intel_ringbuffer *ring;
> 
>-	num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
>-	num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
>-#undef MBOX_UPDATE_DWORDS
>+	ring = intel_ring_begin(rq, 8);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
>-	ret = intel_ring_begin(signaller, num_dwords);
>-	if (ret)
>-		return ret;
>-
>-	for_each_ring(waiter, dev_priv, i) {
>-		u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
>-		if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
>-			continue;
>-
>-		intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
>-		intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB |
>-					   PIPE_CONTROL_QW_WRITE |
>-					   PIPE_CONTROL_FLUSH_ENABLE);
>-		intel_ring_emit(signaller, lower_32_bits(gtt_offset));
>-		intel_ring_emit(signaller, upper_32_bits(gtt_offset));
>-		intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
>-		intel_ring_emit(signaller, 0);
>-		intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
>-					   MI_SEMAPHORE_TARGET(waiter->id));
>-		intel_ring_emit(signaller, 0);
>-	}
>-
>-	return 0;
>-}
>-
>-static int gen8_xcs_signal(struct intel_engine_cs *signaller,
>-			   unsigned int num_dwords)
>-{
>-#define MBOX_UPDATE_DWORDS 6
>-	struct drm_device *dev = signaller->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *waiter;
>-	int i, ret, num_rings;
>-
>-	num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
>-	num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
>-#undef MBOX_UPDATE_DWORDS
>-
>-	ret = intel_ring_begin(signaller, num_dwords);
>-	if (ret)
>-		return ret;
>-
>-	for_each_ring(waiter, dev_priv, i) {
>-		u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
>-		if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
>-			continue;
>-
>-		intel_ring_emit(signaller, (MI_FLUSH_DW + 1) |
>-					   MI_FLUSH_DW_OP_STOREDW);
>-		intel_ring_emit(signaller, lower_32_bits(gtt_offset) |
>-					   MI_FLUSH_DW_USE_GTT);
>-		intel_ring_emit(signaller, upper_32_bits(gtt_offset));
>-		intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
>-		intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
>-					   MI_SEMAPHORE_TARGET(waiter->id));
>-		intel_ring_emit(signaller, 0);
>-	}
>+	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
>+	intel_ring_emit(ring,
>+			PIPE_CONTROL_GLOBAL_GTT_IVB |
>+			PIPE_CONTROL_QW_WRITE |
>+			PIPE_CONTROL_FLUSH_ENABLE);
>+	intel_ring_emit(ring, lower_32_bits(offset));
>+	intel_ring_emit(ring, upper_32_bits(offset));
>+	intel_ring_emit(ring, rq->seqno);
>+	intel_ring_emit(ring, 0);
>+	intel_ring_emit(ring,
>+			MI_SEMAPHORE_SIGNAL |
>+			MI_SEMAPHORE_TARGET(id));
>+	intel_ring_emit(ring, 0);
>+	intel_ring_advance(ring);
> 
> 	return 0;
> }
> 
>-static int gen6_signal(struct intel_engine_cs *signaller,
>-		       unsigned int num_dwords)
>+static int
>+gen8_xcs_emit_signal(struct i915_gem_request *rq, int id)
> {
>-	struct drm_device *dev = signaller->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *useless;
>-	int i, ret, num_rings;
>-
>-#define MBOX_UPDATE_DWORDS 3
>-	num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
>-	num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
>-#undef MBOX_UPDATE_DWORDS
>+	u64 offset = GEN8_SEMAPHORE_OFFSET(rq->i915, rq->engine->id, id);
>+	struct intel_ringbuffer *ring;
> 
>-	ret = intel_ring_begin(signaller, num_dwords);
>-	if (ret)
>-		return ret;
>-
>-	for_each_ring(useless, dev_priv, i) {
>-		u32 mbox_reg = signaller->semaphore.mbox.signal[i];
>-		if (mbox_reg != GEN6_NOSYNC) {
>-			intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
>-			intel_ring_emit(signaller, mbox_reg);
>-			intel_ring_emit(signaller, signaller->outstanding_lazy_seqno);
>-		}
>-	}
>+	ring = intel_ring_begin(rq, 6);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
>-	/* If num_dwords was rounded, make sure the tail pointer is correct */
>-	if (num_rings % 2 == 0)
>-		intel_ring_emit(signaller, MI_NOOP);
>+	intel_ring_emit(ring,
>+			MI_FLUSH_DW |
>+			MI_FLUSH_DW_OP_STOREDW |
>+			(4 - 2));
>+	intel_ring_emit(ring,
>+			lower_32_bits(offset) |
>+			MI_FLUSH_DW_USE_GTT);
>+	intel_ring_emit(ring, upper_32_bits(offset));
>+	intel_ring_emit(ring, rq->seqno);
>+	intel_ring_emit(ring,
>+			MI_SEMAPHORE_SIGNAL |
>+			MI_SEMAPHORE_TARGET(id));
>+	intel_ring_emit(ring, 0);
>+	intel_ring_advance(ring);
> 
> 	return 0;
> }
> 
>-/**
>- * gen6_add_request - Update the semaphore mailbox registers
>- * 
>- * @ring - ring that is adding a request
>- * @seqno - return seqno stuck into the ring
>- *
>- * Update the mailbox registers in the *other* rings with the current seqno.
>- * This acts like a signal in the canonical semaphore.
>- */
> static int
>-gen6_add_request(struct intel_engine_cs *ring)
>+gen6_emit_signal(struct i915_gem_request *rq, int id)
> {
>-	int ret;
>-
>-	if (ring->semaphore.signal)
>-		ret = ring->semaphore.signal(ring, 4);
>-	else
>-		ret = intel_ring_begin(ring, 4);
>+	struct intel_ringbuffer *ring;
> 
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 3);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
>-	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
>-	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
>-	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
>-	intel_ring_emit(ring, MI_USER_INTERRUPT);
>-	__intel_ring_advance(ring);
>+	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
>+	intel_ring_emit(ring, rq->engine->semaphore.mbox.signal[id]);
>+	intel_ring_emit(ring, rq->seqno);
>+	intel_ring_advance(ring);
> 
> 	return 0;
> }
> 
>-static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
>-					      u32 seqno)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	return dev_priv->last_seqno < seqno;
>-}
>-
> /**
>  * intel_ring_sync - sync the waiter to the signaller on seqno
>  *
>@@ -852,66 +1025,52 @@ static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
>  */
> 
> static int
>-gen8_ring_sync(struct intel_engine_cs *waiter,
>-	       struct intel_engine_cs *signaller,
>-	       u32 seqno)
>+gen8_emit_wait(struct i915_gem_request *waiter,
>+	       struct i915_gem_request *signaller)
> {
>-	struct drm_i915_private *dev_priv = waiter->dev->dev_private;
>-	int ret;
>+	u64 offset = GEN8_SEMAPHORE_OFFSET(waiter->i915, signaller->engine->id, waiter->engine->id);
>+	struct intel_ringbuffer *ring;
> 
>-	ret = intel_ring_begin(waiter, 4);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(waiter, 4);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
>-	intel_ring_emit(waiter, MI_SEMAPHORE_WAIT |
>-				MI_SEMAPHORE_GLOBAL_GTT |
>-				MI_SEMAPHORE_POLL |
>-				MI_SEMAPHORE_SAD_GTE_SDD);
>-	intel_ring_emit(waiter, seqno);
>-	intel_ring_emit(waiter,
>-			lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
>-	intel_ring_emit(waiter,
>-			upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
>-	intel_ring_advance(waiter);
>+	intel_ring_emit(ring,
>+			MI_SEMAPHORE_WAIT |
>+			MI_SEMAPHORE_GLOBAL_GTT |
>+			MI_SEMAPHORE_POLL |
>+			MI_SEMAPHORE_SAD_GTE_SDD);
>+	intel_ring_emit(ring, signaller->breadcrumb[waiter->engine->id]);
>+	intel_ring_emit(ring, lower_32_bits(offset));
>+	intel_ring_emit(ring, upper_32_bits(offset));
>+	intel_ring_advance(ring);
> 	return 0;
> }
> 
> static int
>-gen6_ring_sync(struct intel_engine_cs *waiter,
>-	       struct intel_engine_cs *signaller,
>-	       u32 seqno)
>+gen6_emit_wait(struct i915_gem_request *waiter,
>+	       struct i915_gem_request *signaller)
> {
> 	u32 dw1 = MI_SEMAPHORE_MBOX |
> 		  MI_SEMAPHORE_COMPARE |
> 		  MI_SEMAPHORE_REGISTER;
>-	u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
>-	int ret;
>+	u32 wait_mbox = signaller->engine->semaphore.mbox.wait[waiter->engine->id];
>+	struct intel_ringbuffer *ring;
>+
>+	WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
>+
>+	ring = intel_ring_begin(waiter, 3);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
>+	intel_ring_emit(ring, dw1 | wait_mbox);
> 	/* Throughout all of the GEM code, seqno passed implies our current
> 	 * seqno is >= the last seqno executed. However for hardware the
> 	 * comparison is strictly greater than.
> 	 */
>-	seqno -= 1;
>-
>-	WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
>-
>-	ret = intel_ring_begin(waiter, 4);
>-	if (ret)
>-		return ret;
>-
>-	/* If seqno wrap happened, omit the wait with no-ops */
>-	if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
>-		intel_ring_emit(waiter, dw1 | wait_mbox);
>-		intel_ring_emit(waiter, seqno);
>-		intel_ring_emit(waiter, 0);
>-		intel_ring_emit(waiter, MI_NOOP);
>-	} else {
>-		intel_ring_emit(waiter, MI_NOOP);
>-		intel_ring_emit(waiter, MI_NOOP);
>-		intel_ring_emit(waiter, MI_NOOP);
>-		intel_ring_emit(waiter, MI_NOOP);
>-	}
>-	intel_ring_advance(waiter);
>+	intel_ring_emit(ring, signaller->breadcrumb[waiter->engine->id] - 1);
>+	intel_ring_emit(ring, 0);
>+	intel_ring_advance(ring);
> 
> 	return 0;
> }
>@@ -926,10 +1085,10 @@ do {									\
> } while (0)
> 
> static int
>-pc_render_add_request(struct intel_engine_cs *ring)
>+gen5_emit_breadcrumb(struct i915_gem_request *rq)
> {
>-	u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
>-	int ret;
>+	u32 scratch_addr = rq->engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
>+	struct intel_ringbuffer *ring;
> 
> 	/* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
> 	 * incoherent with writes to memory, i.e. completely fubar,
>@@ -939,16 +1098,17 @@ pc_render_add_request(struct intel_engine_cs *ring)
> 	 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
> 	 * memory before requesting an interrupt.
> 	 */
>-	ret = intel_ring_begin(ring, 32);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 32);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
> 			PIPE_CONTROL_WRITE_FLUSH |
> 			PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
>-	intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
>-	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
>+	intel_ring_emit(ring, rq->engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
>+	intel_ring_emit(ring, rq->seqno);
> 	intel_ring_emit(ring, 0);
>+
> 	PIPE_CONTROL_FLUSH(ring, scratch_addr);
> 	scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
> 	PIPE_CONTROL_FLUSH(ring, scratch_addr);
>@@ -965,391 +1125,275 @@ pc_render_add_request(struct intel_engine_cs *ring)
> 			PIPE_CONTROL_WRITE_FLUSH |
> 			PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
> 			PIPE_CONTROL_NOTIFY);
>-	intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
>-	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
>+	intel_ring_emit(ring, rq->engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
>+	intel_ring_emit(ring, rq->seqno);
> 	intel_ring_emit(ring, 0);
>-	__intel_ring_advance(ring);
> 
>-	return 0;
>-}
>-
>-static u32
>-gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
>-{
>-	/* Workaround to force correct ordering between irq and seqno writes on
>-	 * ivb (and maybe also on snb) by reading from a CS register (like
>-	 * ACTHD) before reading the status page. */
>-	if (!lazy_coherency) {
>-		struct drm_i915_private *dev_priv = ring->dev->dev_private;
>-		POSTING_READ(RING_ACTHD(ring->mmio_base));
>-	}
>+	intel_ring_advance(ring);
> 
>-	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
>+	return 0;
> }
> 
> static u32
>-ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
>+ring_get_seqno(struct intel_engine_cs *engine)
> {
>-	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
>+	return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
> }
> 
> static void
>-ring_set_seqno(struct intel_engine_cs *ring, u32 seqno)
>+ring_set_seqno(struct intel_engine_cs *engine, u32 seqno)
> {
>-	intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
>+	intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
> }
> 
> static u32
>-pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
>+gen5_render_get_seqno(struct intel_engine_cs *engine)
> {
>-	return ring->scratch.cpu_page[0];
>+	return engine->scratch.cpu_page[0];
> }
> 
> static void
>-pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno)
>+gen5_render_set_seqno(struct intel_engine_cs *engine, u32 seqno)
> {
>-	ring->scratch.cpu_page[0] = seqno;
>+	engine->scratch.cpu_page[0] = seqno;
> }
> 
>-static bool
>-gen5_ring_get_irq(struct intel_engine_cs *ring)
>+static void
>+gen5_irq_get(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *i915 = engine->i915;
> 	unsigned long flags;
> 
>-	if (!dev->irq_enabled)
>-		return false;
>-
>-	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>-	if (ring->irq_refcount++ == 0)
>-		gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
>-
>-	return true;
>+	spin_lock_irqsave(&i915->irq_lock, flags);
>+	if (engine->irq_refcount++ == 0)
>+		gen5_enable_gt_irq(i915, engine->irq_enable_mask);
>+	spin_unlock_irqrestore(&i915->irq_lock, flags);
> }
> 
> static void
>-gen5_ring_put_irq(struct intel_engine_cs *ring)
>+gen5_irq_put(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *i915 = engine->i915;
> 	unsigned long flags;
> 
>-	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>-	if (--ring->irq_refcount == 0)
>-		gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
>-	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
>+	spin_lock_irqsave(&i915->irq_lock, flags);
>+	if (--engine->irq_refcount == 0)
>+		gen5_disable_gt_irq(i915, engine->irq_enable_mask);
>+	spin_unlock_irqrestore(&i915->irq_lock, flags);
> }
> 
>-static bool
>-i9xx_ring_get_irq(struct intel_engine_cs *ring)
>+static void
>+i9xx_irq_get(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	unsigned long flags;
> 
>-	if (!dev->irq_enabled)
>-		return false;
>-
> 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>-	if (ring->irq_refcount++ == 0) {
>-		dev_priv->irq_mask &= ~ring->irq_enable_mask;
>+	if (engine->irq_refcount++ == 0) {
>+		dev_priv->irq_mask &= ~engine->irq_enable_mask;
> 		I915_WRITE(IMR, dev_priv->irq_mask);
> 		POSTING_READ(IMR);
> 	}
> 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
>-
>-	return true;
> }
> 
> static void
>-i9xx_ring_put_irq(struct intel_engine_cs *ring)
>+i9xx_irq_put(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	unsigned long flags;
> 
> 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>-	if (--ring->irq_refcount == 0) {
>-		dev_priv->irq_mask |= ring->irq_enable_mask;
>+	if (--engine->irq_refcount == 0) {
>+		dev_priv->irq_mask |= engine->irq_enable_mask;
> 		I915_WRITE(IMR, dev_priv->irq_mask);
> 		POSTING_READ(IMR);
> 	}
> 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
> }
> 
>-static bool
>-i8xx_ring_get_irq(struct intel_engine_cs *ring)
>+static void
>+i8xx_irq_get(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	unsigned long flags;
> 
>-	if (!dev->irq_enabled)
>-		return false;
>-
> 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>-	if (ring->irq_refcount++ == 0) {
>-		dev_priv->irq_mask &= ~ring->irq_enable_mask;
>+	if (engine->irq_refcount++ == 0) {
>+		dev_priv->irq_mask &= ~engine->irq_enable_mask;
> 		I915_WRITE16(IMR, dev_priv->irq_mask);
> 		POSTING_READ16(IMR);
> 	}
> 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
>-
>-	return true;
> }
> 
> static void
>-i8xx_ring_put_irq(struct intel_engine_cs *ring)
>+i8xx_irq_put(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	unsigned long flags;
> 
> 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>-	if (--ring->irq_refcount == 0) {
>-		dev_priv->irq_mask |= ring->irq_enable_mask;
>+	if (--engine->irq_refcount == 0) {
>+		dev_priv->irq_mask |= engine->irq_enable_mask;
> 		I915_WRITE16(IMR, dev_priv->irq_mask);
> 		POSTING_READ16(IMR);
> 	}
> 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
> }
> 
>-void intel_ring_setup_status_page(struct intel_engine_cs *ring)
>-{
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>-	u32 mmio = 0;
>-
>-	/* The ring status page addresses are no longer next to the rest of
>-	 * the ring registers as of gen7.
>-	 */
>-	if (IS_GEN7(dev)) {
>-		switch (ring->id) {
>-		case RCS:
>-			mmio = RENDER_HWS_PGA_GEN7;
>-			break;
>-		case BCS:
>-			mmio = BLT_HWS_PGA_GEN7;
>-			break;
>-		/*
>-		 * VCS2 actually doesn't exist on Gen7. Only shut up
>-		 * gcc switch check warning
>-		 */
>-		case VCS2:
>-		case VCS:
>-			mmio = BSD_HWS_PGA_GEN7;
>-			break;
>-		case VECS:
>-			mmio = VEBOX_HWS_PGA_GEN7;
>-			break;
>-		}
>-	} else if (IS_GEN6(ring->dev)) {
>-		mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
>-	} else {
>-		/* XXX: gen8 returns to sanity */
>-		mmio = RING_HWS_PGA(ring->mmio_base);
>-	}
>-
>-	I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
>-	POSTING_READ(mmio);
>-
>-	/*
>-	 * Flush the TLB for this page
>-	 *
>-	 * FIXME: These two bits have disappeared on gen8, so a question
>-	 * arises: do we still need this and if so how should we go about
>-	 * invalidating the TLB?
>-	 */
>-	if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
>-		u32 reg = RING_INSTPM(ring->mmio_base);
>-
>-		/* ring should be idle before issuing a sync flush*/
>-		WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
>-
>-		I915_WRITE(reg,
>-			   _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
>-					      INSTPM_SYNC_FLUSH));
>-		if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
>-			     1000))
>-			DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
>-				  ring->name);
>-	}
>-}
>-
> static int
>-bsd_ring_flush(struct intel_engine_cs *ring,
>-	       u32     invalidate_domains,
>-	       u32     flush_domains)
>+bsd_emit_flush(struct i915_gem_request *rq,
>+	       u32 flags)
> {
>-	int ret;
>+	struct intel_ringbuffer *ring;
> 
>-	ret = intel_ring_begin(ring, 2);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 1);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring, MI_FLUSH);
>-	intel_ring_emit(ring, MI_NOOP);
> 	intel_ring_advance(ring);
> 	return 0;
> }
> 
> static int
>-i9xx_add_request(struct intel_engine_cs *ring)
>+i9xx_emit_breadcrumb(struct i915_gem_request *rq)
> {
>-	int ret;
>+	struct intel_ringbuffer *ring;
> 
>-	ret = intel_ring_begin(ring, 4);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 4);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
> 	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
>-	intel_ring_emit(ring, ring->outstanding_lazy_seqno);
>+	intel_ring_emit(ring, rq->seqno);
> 	intel_ring_emit(ring, MI_USER_INTERRUPT);
>-	__intel_ring_advance(ring);
>+	intel_ring_advance(ring);
> 
> 	return 0;
> }
> 
>-static bool
>-gen6_ring_get_irq(struct intel_engine_cs *ring)
>+static void
>+gen6_irq_get(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	unsigned long flags;
> 
>-	if (!dev->irq_enabled)
>-	       return false;
>-
> 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>-	if (ring->irq_refcount++ == 0) {
>-		if (HAS_L3_DPF(dev) && ring->id == RCS)
>-			I915_WRITE_IMR(ring,
>-				       ~(ring->irq_enable_mask |
>-					 GT_PARITY_ERROR(dev)));
>-		else
>-			I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
>-		gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
>+	if (engine->irq_refcount++ == 0) {
>+		I915_WRITE_IMR(engine,
>+			       ~(engine->irq_enable_mask |
>+				 engine->irq_keep_mask));
>+		gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
> 	}
> 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
> 
>-	return true;
>+	/* Keep the device awake to save expensive CPU cycles when
>+	 * reading the registers.
>+	 */
>+	gen6_gt_force_wake_get(dev_priv, engine->power_domains);
> }
> 
> static void
>-gen6_ring_put_irq(struct intel_engine_cs *ring)
>+gen6_irq_barrier(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	/* w/a for lax serialisation of GPU writes with IRQs */
>+	struct drm_i915_private *dev_priv = engine->i915;
>+	(void)I915_READ(RING_ACTHD(engine->mmio_base));
>+}
>+
>+static void
>+gen6_irq_put(struct intel_engine_cs *engine)
>+{
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	unsigned long flags;
> 
>+	gen6_gt_force_wake_put(dev_priv, engine->power_domains);
>+
> 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>-	if (--ring->irq_refcount == 0) {
>-		if (HAS_L3_DPF(dev) && ring->id == RCS)
>-			I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
>-		else
>-			I915_WRITE_IMR(ring, ~0);
>-		gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
>+	if (--engine->irq_refcount == 0) {
>+		I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
>+		gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
> 	}
> 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
> }
> 
>-static bool
>-hsw_vebox_get_irq(struct intel_engine_cs *ring)
>+static void
>+hsw_vebox_irq_get(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	unsigned long flags;
> 
>-	if (!dev->irq_enabled)
>-		return false;
>-
> 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>-	if (ring->irq_refcount++ == 0) {
>-		I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
>-		gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask);
>+	if (engine->irq_refcount++ == 0) {
>+		I915_WRITE_IMR(engine,
>+			       ~(engine->irq_enable_mask |
>+				 engine->irq_keep_mask));
>+		gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask);
> 	}
> 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
> 
>-	return true;
>+	gen6_gt_force_wake_get(dev_priv, engine->power_domains);
> }
> 
> static void
>-hsw_vebox_put_irq(struct intel_engine_cs *ring)
>+hsw_vebox_irq_put(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	unsigned long flags;
> 
>-	if (!dev->irq_enabled)
>-		return;
>+	gen6_gt_force_wake_put(dev_priv, engine->power_domains);
> 
> 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>-	if (--ring->irq_refcount == 0) {
>-		I915_WRITE_IMR(ring, ~0);
>-		gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask);
>+	if (--engine->irq_refcount == 0) {
>+		I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
>+		gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask);
> 	}
> 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
> }
> 
>-static bool
>-gen8_ring_get_irq(struct intel_engine_cs *ring)
>+static void
>+gen8_irq_get(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	unsigned long flags;
> 
>-	if (!dev->irq_enabled)
>-		return false;
>-
> 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>-	if (ring->irq_refcount++ == 0) {
>-		if (HAS_L3_DPF(dev) && ring->id == RCS) {
>-			I915_WRITE_IMR(ring,
>-				       ~(ring->irq_enable_mask |
>-					 GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
>-		} else {
>-			I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
>-		}
>-		POSTING_READ(RING_IMR(ring->mmio_base));
>+	if (engine->irq_refcount++ == 0) {
>+		I915_WRITE_IMR(engine,
>+			       ~(engine->irq_enable_mask |
>+				 engine->irq_keep_mask));
>+		POSTING_READ(RING_IMR(engine->mmio_base));
> 	}
> 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
>-
>-	return true;
> }
> 
> static void
>-gen8_ring_put_irq(struct intel_engine_cs *ring)
>+gen8_irq_put(struct intel_engine_cs *engine)
> {
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 	unsigned long flags;
> 
> 	spin_lock_irqsave(&dev_priv->irq_lock, flags);
>-	if (--ring->irq_refcount == 0) {
>-		if (HAS_L3_DPF(dev) && ring->id == RCS) {
>-			I915_WRITE_IMR(ring,
>-				       ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
>-		} else {
>-			I915_WRITE_IMR(ring, ~0);
>-		}
>-		POSTING_READ(RING_IMR(ring->mmio_base));
>+	if (--engine->irq_refcount == 0) {
>+		I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
>+		POSTING_READ(RING_IMR(engine->mmio_base));
> 	}
> 	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
> }
> 
> static int
>-i965_dispatch_execbuffer(struct intel_engine_cs *ring,
>-			 u64 offset, u32 length,
>-			 unsigned flags)
>+i965_emit_batchbuffer(struct i915_gem_request *rq,
>+		      u64 offset, u32 length,
>+		      unsigned flags)
> {
>-	int ret;
>+	struct intel_ringbuffer *ring;
> 
>-	ret = intel_ring_begin(ring, 2);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 2);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring,
> 			MI_BATCH_BUFFER_START |
>@@ -1361,38 +1405,35 @@ i965_dispatch_execbuffer(struct intel_engine_cs *ring,
> 	return 0;
> }
> 
>-/* Just userspace ABI convention to limit the wa batch bo to a resonable size */
>-#define I830_BATCH_LIMIT (256*1024)
>-#define I830_TLB_ENTRIES (2)
>-#define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
> static int
>-i830_dispatch_execbuffer(struct intel_engine_cs *ring,
>-				u64 offset, u32 len,
>-				unsigned flags)
>-{
>-	u32 cs_offset = ring->scratch.gtt_offset;
>-	int ret;
>-
>-	ret = intel_ring_begin(ring, 6);
>-	if (ret)
>-		return ret;
>-
>-	/* Evict the invalid PTE TLBs */
>-	intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA);
>-	intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096);
>-	intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); /* load each page */
>-	intel_ring_emit(ring, cs_offset);
>-	intel_ring_emit(ring, 0xdeadbeef);
>-	intel_ring_emit(ring, MI_NOOP);
>-	intel_ring_advance(ring);
>+i830_emit_batchbuffer(struct i915_gem_request *rq,
>+		      u64 offset, u32 len,
>+		      unsigned flags)
>+{
>+	u32 cs_offset = rq->engine->scratch.gtt_offset;
>+	struct intel_ringbuffer *ring;
>+
>+	if (rq->batch->vm->dirty) {
>+		ring = intel_ring_begin(rq, 5);
>+		if (IS_ERR(ring))
>+			return PTR_ERR(ring);
>+
>+		/* Evict the invalid PTE TLBs */
>+		intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA);
>+		intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096);
>+		intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4);
>+		intel_ring_emit(ring, cs_offset);
>+		intel_ring_emit(ring, 0xdeadbeef);
>+		intel_ring_advance(ring);
>+	}
> 
> 	if ((flags & I915_DISPATCH_PINNED) == 0) {
> 		if (len > I830_BATCH_LIMIT)
> 			return -ENOSPC;
> 
>-		ret = intel_ring_begin(ring, 6 + 2);
>-		if (ret)
>-			return ret;
>+		ring = intel_ring_begin(rq, 6 + 1);
>+		if (IS_ERR(ring))
>+			return PTR_ERR(ring);
> 
> 		/* Blit the batch (which has now all relocs applied) to the
> 		 * stable batch scratch bo area (so that the CS never
>@@ -1406,36 +1447,34 @@ i830_dispatch_execbuffer(struct intel_engine_cs *ring,
> 		intel_ring_emit(ring, offset);
> 
> 		intel_ring_emit(ring, MI_FLUSH);
>-		intel_ring_emit(ring, MI_NOOP);
> 		intel_ring_advance(ring);
> 
> 		/* ... and execute it. */
> 		offset = cs_offset;
> 	}
> 
>-	ret = intel_ring_begin(ring, 4);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 3);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring, MI_BATCH_BUFFER);
> 	intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
> 	intel_ring_emit(ring, offset + len - 8);
>-	intel_ring_emit(ring, MI_NOOP);
> 	intel_ring_advance(ring);
> 
> 	return 0;
> }
> 
> static int
>-i915_dispatch_execbuffer(struct intel_engine_cs *ring,
>-			 u64 offset, u32 len,
>-			 unsigned flags)
>+i915_emit_batchbuffer(struct i915_gem_request *rq,
>+		      u64 offset, u32 len,
>+		      unsigned flags)
> {
>-	int ret;
>+	struct intel_ringbuffer *ring;
> 
>-	ret = intel_ring_begin(ring, 2);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 2);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
> 	intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
>@@ -1444,485 +1483,221 @@ i915_dispatch_execbuffer(struct intel_engine_cs *ring,
> 	return 0;
> }
> 
>-static void cleanup_status_page(struct intel_engine_cs *ring)
>-{
>-	struct drm_i915_gem_object *obj;
>-
>-	obj = ring->status_page.obj;
>-	if (obj == NULL)
>-		return;
>-
>-	kunmap(sg_page(obj->pages->sgl));
>-	i915_gem_object_ggtt_unpin(obj);
>-	drm_gem_object_unreference(&obj->base);
>-	ring->status_page.obj = NULL;
>-}
>-
>-static int init_status_page(struct intel_engine_cs *ring)
>+static int setup_status_page(struct intel_engine_cs *engine)
> {
> 	struct drm_i915_gem_object *obj;
>+	unsigned flags;
>+	int ret;
> 
>-	if ((obj = ring->status_page.obj) == NULL) {
>-		unsigned flags;
>-		int ret;
>+	obj = i915_gem_alloc_object(engine->i915->dev, 4096);
>+	if (obj == NULL) {
>+		DRM_ERROR("Failed to allocate status page\n");
>+		return -ENOMEM;
>+	}
> 
>-		obj = i915_gem_alloc_object(ring->dev, 4096);
>-		if (obj == NULL) {
>-			DRM_ERROR("Failed to allocate status page\n");
>-			return -ENOMEM;
>-		}
>+	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
>+	if (ret)
>+		goto err_unref;
> 
>-		ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
>-		if (ret)
>-			goto err_unref;
>-
>-		flags = 0;
>-		if (!HAS_LLC(ring->dev))
>-			/* On g33, we cannot place HWS above 256MiB, so
>-			 * restrict its pinning to the low mappable arena.
>-			 * Though this restriction is not documented for
>-			 * gen4, gen5, or byt, they also behave similarly
>-			 * and hang if the HWS is placed at the top of the
>-			 * GTT. To generalise, it appears that all !llc
>-			 * platforms have issues with us placing the HWS
>-			 * above the mappable region (even though we never
>-			 * actualy map it).
>-			 */
>-			flags |= PIN_MAPPABLE;
>-		ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
>-		if (ret) {
>+	flags = 0;
>+	if (!HAS_LLC(engine->i915))
>+		/* On g33, we cannot place HWS above 256MiB, so
>+		 * restrict its pinning to the low mappable arena.
>+		 * Though this restriction is not documented for
>+		 * gen4, gen5, or byt, they also behave similarly
>+		 * and hang if the HWS is placed at the top of the
>+		 * GTT. To generalise, it appears that all !llc
>+		 * platforms have issues with us placing the HWS
>+		 * above the mappable region (even though we never
>+		 * actualy map it).
>+		 */
>+		flags |= PIN_MAPPABLE;
>+	ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
>+	if (ret) {
> err_unref:
>-			drm_gem_object_unreference(&obj->base);
>-			return ret;
>-		}
>-
>-		ring->status_page.obj = obj;
>+		drm_gem_object_unreference(&obj->base);
>+		return ret;
> 	}
> 
>-	ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
>-	ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
>-	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
>+	engine->status_page.obj = obj;
> 
>-	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
>-			ring->name, ring->status_page.gfx_addr);
>+	engine->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
>+	engine->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
>+	memset(engine->status_page.page_addr, 0, PAGE_SIZE);
> 
>+	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
>+			engine->name, engine->status_page.gfx_addr);
> 	return 0;
> }
> 
>-static int init_phys_status_page(struct intel_engine_cs *ring)
>+static int setup_phys_status_page(struct intel_engine_cs *engine)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>+	struct drm_i915_private *i915 = engine->i915;
> 
>-	if (!dev_priv->status_page_dmah) {
>-		dev_priv->status_page_dmah =
>-			drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
>-		if (!dev_priv->status_page_dmah)
>-			return -ENOMEM;
>-	}
>+	i915->status_page_dmah =
>+		drm_pci_alloc(i915->dev, PAGE_SIZE, PAGE_SIZE);
>+	if (!i915->status_page_dmah)
>+		return -ENOMEM;
> 
>-	ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
>-	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
>+	engine->status_page.page_addr = i915->status_page_dmah->vaddr;
>+	memset(engine->status_page.page_addr, 0, PAGE_SIZE);
> 
> 	return 0;
> }
> 
>-static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
>+void intel_ring_free(struct intel_ringbuffer *ring)
> {
>-	if (!ringbuf->obj)
>-		return;
>+	if (ring->obj) {
>+		iounmap(ring->virtual_start);
>+		i915_gem_object_ggtt_unpin(ring->obj);
>+		drm_gem_object_unreference(&ring->obj->base);
>+	}
> 
>-	iounmap(ringbuf->virtual_start);
>-	i915_gem_object_ggtt_unpin(ringbuf->obj);
>-	drm_gem_object_unreference(&ringbuf->obj->base);
>-	ringbuf->obj = NULL;
>+	list_del(&ring->engine_list);
>+	kfree(ring);
> }
> 
>-static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
>-				      struct intel_ringbuffer *ringbuf)
>+struct intel_ringbuffer *
>+intel_engine_alloc_ring(struct intel_engine_cs *engine,
>+			struct intel_context *ctx,
>+			int size)
> {
>-	struct drm_i915_private *dev_priv = to_i915(dev);
>+	struct drm_i915_private *i915 = engine->i915;
>+	struct intel_ringbuffer *ring;
> 	struct drm_i915_gem_object *obj;
> 	int ret;
> 
>-	if (ringbuf->obj)
>-		return 0;
>+	DRM_DEBUG("creating ringbuffer for %s, size %d\n", engine->name, size);
>+
>+	if (WARN_ON(!is_power_of_2(size)))
>+		return ERR_PTR(-EINVAL);
>+
>+	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
>+	if (ring == NULL)
>+		return ERR_PTR(-ENOMEM);
>+
>+	ring->engine = engine;
>+	ring->ctx = ctx;
> 
>-	obj = NULL;
>-	if (!HAS_LLC(dev))
>-		obj = i915_gem_object_create_stolen(dev, ringbuf->size);
>+	obj = i915_gem_object_create_stolen(i915->dev, size);
> 	if (obj == NULL)
>-		obj = i915_gem_alloc_object(dev, ringbuf->size);
>+		obj = i915_gem_alloc_object(i915->dev, size);
> 	if (obj == NULL)
>-		return -ENOMEM;
>+		return ERR_PTR(-ENOMEM);
> 
> 	/* mark ring buffers as read-only from GPU side by default */
> 	obj->gt_ro = 1;
> 
> 	ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
>-	if (ret)
>+	if (ret) {
>+		DRM_ERROR("failed pin ringbuffer into GGTT\n");
> 		goto err_unref;
>+	}
> 
> 	ret = i915_gem_object_set_to_gtt_domain(obj, true);
>-	if (ret)
>+	if (ret) {
>+		DRM_ERROR("failed mark ringbuffer for GTT writes\n");
> 		goto err_unpin;
>+	}
> 
>-	ringbuf->virtual_start =
>-		ioremap_wc(dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj),
>-				ringbuf->size);
>-	if (ringbuf->virtual_start == NULL) {
>+	ring->virtual_start =
>+		ioremap_wc(i915->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj),
>+			   size);
>+	if (ring->virtual_start == NULL) {
>+		DRM_ERROR("failed to map ringbuffer through GTT\n");
> 		ret = -EINVAL;
> 		goto err_unpin;
> 	}
> 
>-	ringbuf->obj = obj;
>-	return 0;
>+	ring->obj = obj;
>+	ring->size = size;
>+
>+	/* Workaround an erratum on the i830 which causes a hang if
>+	 * the TAIL pointer points to within the last 2 cachelines
>+	 * of the buffer.
>+	 */
>+	ring->effective_size = size;
>+	if (IS_I830(i915) || IS_845G(i915))
>+		ring->effective_size -= 2 * CACHELINE_BYTES;
>+
>+	ring->space = intel_ring_space(ring);
>+	ring->retired_head = -1;
>+
>+	INIT_LIST_HEAD(&ring->requests);
>+	INIT_LIST_HEAD(&ring->breadcrumbs);
>+	list_add_tail(&ring->engine_list, &engine->rings);
>+
>+	return ring;
> 
> err_unpin:
> 	i915_gem_object_ggtt_unpin(obj);
> err_unref:
> 	drm_gem_object_unreference(&obj->base);
>-	return ret;
>+	return ERR_PTR(ret);
>+}
>+
>+static void
>+nop_irq_barrier(struct intel_engine_cs *engine)
>+{
> }
> 
>-static int intel_init_ring_buffer(struct drm_device *dev,
>-				  struct intel_engine_cs *ring)
>+static int intel_engine_init(struct intel_engine_cs *engine,
>+			     struct drm_i915_private *i915)
> {
>-	struct intel_ringbuffer *ringbuf = ring->buffer;
> 	int ret;
> 
>-	if (ringbuf == NULL) {
>-		ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
>-		if (!ringbuf)
>-			return -ENOMEM;
>-		ring->buffer = ringbuf;
>-	}
>+	engine->i915 = i915;
> 
>-	ring->dev = dev;
>-	INIT_LIST_HEAD(&ring->active_list);
>-	INIT_LIST_HEAD(&ring->request_list);
>-	ringbuf->size = 32 * PAGE_SIZE;
>-	memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
>+	INIT_LIST_HEAD(&engine->rings);
>+	INIT_LIST_HEAD(&engine->read_list);
>+	INIT_LIST_HEAD(&engine->write_list);
>+	INIT_LIST_HEAD(&engine->requests);
>+	INIT_LIST_HEAD(&engine->pending);
>+	INIT_LIST_HEAD(&engine->submitted);
> 
>-	init_waitqueue_head(&ring->irq_queue);
>+	spin_lock_init(&engine->lock);
>+	spin_lock_init(&engine->irqlock);
> 
>-	if (I915_NEED_GFX_HWS(dev)) {
>-		ret = init_status_page(ring);
>-		if (ret)
>-			goto error;
>-	} else {
>-		BUG_ON(ring->id != RCS);
>-		ret = init_phys_status_page(ring);
>-		if (ret)
>-			goto error;
>-	}
>-
>-	ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
>-	if (ret) {
>-		DRM_ERROR("Failed to allocate ringbuffer %s: %d\n", ring->name, ret);
>-		goto error;
>-	}
>-
>-	/* Workaround an erratum on the i830 which causes a hang if
>-	 * the TAIL pointer points to within the last 2 cachelines
>-	 * of the buffer.
>-	 */
>-	ringbuf->effective_size = ringbuf->size;
>-	if (IS_I830(dev) || IS_845G(dev))
>-		ringbuf->effective_size -= 2 * CACHELINE_BYTES;
>-
>-	ret = i915_cmd_parser_init_ring(ring);
>-	if (ret)
>-		goto error;
>-
>-	ret = ring->init(ring);
>-	if (ret)
>-		goto error;
>-
>-	return 0;
>-
>-error:
>-	kfree(ringbuf);
>-	ring->buffer = NULL;
>-	return ret;
>-}
>-
>-void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
>-{
>-	struct drm_i915_private *dev_priv = to_i915(ring->dev);
>-	struct intel_ringbuffer *ringbuf = ring->buffer;
>-
>-	if (!intel_ring_initialized(ring))
>-		return;
>-
>-	intel_stop_ring_buffer(ring);
>-	WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
>-
>-	intel_destroy_ringbuffer_obj(ringbuf);
>-	ring->preallocated_lazy_request = NULL;
>-	ring->outstanding_lazy_seqno = 0;
>-
>-	if (ring->cleanup)
>-		ring->cleanup(ring);
>-
>-	cleanup_status_page(ring);
>-
>-	i915_cmd_parser_fini_ring(ring);
>-
>-	kfree(ringbuf);
>-	ring->buffer = NULL;
>-}
>-
>-static int intel_ring_wait_request(struct intel_engine_cs *ring, int n)
>-{
>-	struct intel_ringbuffer *ringbuf = ring->buffer;
>-	struct drm_i915_gem_request *request;
>-	u32 seqno = 0;
>-	int ret;
>-
>-	if (ringbuf->last_retired_head != -1) {
>-		ringbuf->head = ringbuf->last_retired_head;
>-		ringbuf->last_retired_head = -1;
>-
>-		ringbuf->space = ring_space(ringbuf);
>-		if (ringbuf->space >= n)
>-			return 0;
>-	}
>-
>-	list_for_each_entry(request, &ring->request_list, list) {
>-		if (__ring_space(request->tail, ringbuf->tail, ringbuf->size) >= n) {
>-			seqno = request->seqno;
>-			break;
>-		}
>-	}
>-
>-	if (seqno == 0)
>-		return -ENOSPC;
>-
>-	ret = i915_wait_seqno(ring, seqno);
>-	if (ret)
>-		return ret;
>-
>-	i915_gem_retire_requests_ring(ring);
>-	ringbuf->head = ringbuf->last_retired_head;
>-	ringbuf->last_retired_head = -1;
>-
>-	ringbuf->space = ring_space(ringbuf);
>-	return 0;
>-}
>-
>-static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
>-{
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_ringbuffer *ringbuf = ring->buffer;
>-	unsigned long end;
>-	int ret;
>-
>-	ret = intel_ring_wait_request(ring, n);
>-	if (ret != -ENOSPC)
>-		return ret;
>-
>-	/* force the tail write in case we have been skipping them */
>-	__intel_ring_advance(ring);
>+	engine->suspend = engine_suspend;
>+	engine->resume = engine_resume;
>+	engine->cleanup = engine_cleanup;
> 
>-	/* With GEM the hangcheck timer should kick us out of the loop,
>-	 * leaving it early runs the risk of corrupting GEM state (due
>-	 * to running on almost untested codepaths). But on resume
>-	 * timers don't work yet, so prevent a complete hang in that
>-	 * case by choosing an insanely large timeout. */
>-	end = jiffies + 60 * HZ;
>+	engine->get_seqno = ring_get_seqno;
>+	engine->set_seqno = ring_set_seqno;
> 
>-	trace_i915_ring_wait_begin(ring);
>-	do {
>-		ringbuf->head = I915_READ_HEAD(ring);
>-		ringbuf->space = ring_space(ringbuf);
>-		if (ringbuf->space >= n) {
>-			ret = 0;
>-			break;
>-		}
>-
>-		if (!drm_core_check_feature(dev, DRIVER_MODESET) &&
>-		    dev->primary->master) {
>-			struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
>-			if (master_priv->sarea_priv)
>-				master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
>-		}
>-
>-		msleep(1);
>-
>-		if (dev_priv->mm.interruptible && signal_pending(current)) {
>-			ret = -ERESTARTSYS;
>-			break;
>-		}
>-
>-		ret = i915_gem_check_wedge(&dev_priv->gpu_error,
>-					   dev_priv->mm.interruptible);
>-		if (ret)
>-			break;
>-
>-		if (time_after(jiffies, end)) {
>-			ret = -EBUSY;
>-			break;
>-		}
>-	} while (1);
>-	trace_i915_ring_wait_end(ring);
>-	return ret;
>-}
>-
>-static int intel_wrap_ring_buffer(struct intel_engine_cs *ring)
>-{
>-	uint32_t __iomem *virt;
>-	struct intel_ringbuffer *ringbuf = ring->buffer;
>-	int rem = ringbuf->size - ringbuf->tail;
>-
>-	if (ringbuf->space < rem) {
>-		int ret = ring_wait_for_space(ring, rem);
>-		if (ret)
>-			return ret;
>-	}
>+	engine->irq_barrier = nop_irq_barrier;
> 
>-	virt = ringbuf->virtual_start + ringbuf->tail;
>-	rem /= 4;
>-	while (rem--)
>-		iowrite32(MI_NOOP, virt++);
>+	engine->get_ring = engine_get_ring;
>+	engine->put_ring = engine_put_ring;
> 
>-	ringbuf->tail = 0;
>-	ringbuf->space = ring_space(ringbuf);
>+	engine->semaphore.wait = NULL;
> 
>-	return 0;
>-}
>-
>-int intel_ring_idle(struct intel_engine_cs *ring)
>-{
>-	u32 seqno;
>-	int ret;
>-
>-	/* We need to add any requests required to flush the objects and ring */
>-	if (ring->outstanding_lazy_seqno) {
>-		ret = i915_add_request(ring, NULL);
>-		if (ret)
>-			return ret;
>-	}
>-
>-	/* Wait upon the last request to be completed */
>-	if (list_empty(&ring->request_list))
>-		return 0;
>-
>-	seqno = list_entry(ring->request_list.prev,
>-			   struct drm_i915_gem_request,
>-			   list)->seqno;
>-
>-	return i915_wait_seqno(ring, seqno);
>-}
>-
>-static int
>-intel_ring_alloc_seqno(struct intel_engine_cs *ring)
>-{
>-	if (ring->outstanding_lazy_seqno)
>-		return 0;
>-
>-	if (ring->preallocated_lazy_request == NULL) {
>-		struct drm_i915_gem_request *request;
>-
>-		request = kmalloc(sizeof(*request), GFP_KERNEL);
>-		if (request == NULL)
>-			return -ENOMEM;
>-
>-		ring->preallocated_lazy_request = request;
>-	}
>-
>-	return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
>-}
>-
>-static int __intel_ring_prepare(struct intel_engine_cs *ring,
>-				int bytes)
>-{
>-	struct intel_ringbuffer *ringbuf = ring->buffer;
>-	int ret;
>+	engine->add_request = engine_add_request;
>+	engine->write_tail = ring_write_tail;
>+	engine->is_complete = engine_rq_is_complete;
> 
>-	if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {
>-		ret = intel_wrap_ring_buffer(ring);
>-		if (unlikely(ret))
>-			return ret;
>-	}
>+	init_waitqueue_head(&engine->irq_queue);
> 
>-	if (unlikely(ringbuf->space < bytes)) {
>-		ret = ring_wait_for_space(ring, bytes);
>-		if (unlikely(ret))
>-			return ret;
>+	if (I915_NEED_GFX_HWS(i915)) {
>+		ret = setup_status_page(engine);
>+	} else {
>+		BUG_ON(engine->id != RCS);
>+		ret = setup_phys_status_page(engine);
> 	}
>-
>-	return 0;
>-}
>-
>-int intel_ring_begin(struct intel_engine_cs *ring,
>-		     int num_dwords)
>-{
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>-	int ret;
>-
>-	ret = i915_gem_check_wedge(&dev_priv->gpu_error,
>-				   dev_priv->mm.interruptible);
> 	if (ret)
> 		return ret;
> 
>-	ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
>+	ret = i915_cmd_parser_init_engine(engine);
> 	if (ret)
> 		return ret;
> 
>-	/* Preallocate the olr before touching the ring */
>-	ret = intel_ring_alloc_seqno(ring);
>-	if (ret)
>-		return ret;
>-
>-	ring->buffer->space -= num_dwords * sizeof(uint32_t);
> 	return 0;
> }
> 
>-/* Align the ring tail to a cacheline boundary */
>-int intel_ring_cacheline_align(struct intel_engine_cs *ring)
>-{
>-	int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
>-	int ret;
>-
>-	if (num_dwords == 0)
>-		return 0;
>-
>-	num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
>-	ret = intel_ring_begin(ring, num_dwords);
>-	if (ret)
>-		return ret;
>-
>-	while (num_dwords--)
>-		intel_ring_emit(ring, MI_NOOP);
>-
>-	intel_ring_advance(ring);
>-
>-	return 0;
>-}
>-
>-void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno)
>-{
>-	struct drm_device *dev = ring->dev;
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-
>-	BUG_ON(ring->outstanding_lazy_seqno);
>-
>-	if (INTEL_INFO(dev)->gen == 6 || INTEL_INFO(dev)->gen == 7) {
>-		I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
>-		I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
>-		if (HAS_VEBOX(dev))
>-			I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
>-	}
>-
>-	ring->set_seqno(ring, seqno);
>-	ring->hangcheck.seqno = seqno;
>-}
>-
>-static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring,
>+static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine,
> 				     u32 value)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>+	struct drm_i915_private *dev_priv = engine->i915;
> 
>        /* Every tail move must follow the sequence below */
> 
>@@ -1942,8 +1717,8 @@ static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring,
> 		DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
> 
> 	/* Now that the ring is fully powered up, update the tail */
>-	I915_WRITE_TAIL(ring, value);
>-	POSTING_READ(RING_TAIL(ring->mmio_base));
>+	I915_WRITE_TAIL(engine, value);
>+	POSTING_READ(RING_TAIL(engine->mmio_base));
> 
> 	/* Let the ring send IDLE messages to the GT again,
> 	 * and so let it sleep to conserve power when idle.
>@@ -1952,79 +1727,77 @@ static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring,
> 		   _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
> }
> 
>-static int gen6_bsd_ring_flush(struct intel_engine_cs *ring,
>-			       u32 invalidate, u32 flush)
>+static int gen6_bsd_emit_flush(struct i915_gem_request *rq,
>+			       u32 flags)
> {
>+	struct intel_ringbuffer *ring;
> 	uint32_t cmd;
>-	int ret;
> 
>-	ret = intel_ring_begin(ring, 4);
>-	if (ret)
>-		return ret;
>-
>-	cmd = MI_FLUSH_DW;
>-	if (INTEL_INFO(ring->dev)->gen >= 8)
>+	cmd = 3;
>+	if (INTEL_INFO(rq->i915)->gen >= 8)
> 		cmd += 1;
>+
>+	ring = intel_ring_begin(rq, cmd);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
>+
> 	/*
> 	 * Bspec vol 1c.5 - video engine command streamer:
> 	 * "If ENABLED, all TLBs will be invalidated once the flush
> 	 * operation is complete. This bit is only valid when the
> 	 * Post-Sync Operation field is a value of 1h or 3h."
> 	 */
>-	if (invalidate & I915_GEM_GPU_DOMAINS)
>-		cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
>-			MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
>+	cmd = MI_FLUSH_DW | (cmd - 2);
>+	if (flags & I915_INVALIDATE_CACHES)
>+		cmd |= (MI_INVALIDATE_TLB |
>+			MI_INVALIDATE_BSD |
>+			MI_FLUSH_DW_STORE_INDEX |
>+			MI_FLUSH_DW_OP_STOREDW);
> 	intel_ring_emit(ring, cmd);
> 	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
>-	if (INTEL_INFO(ring->dev)->gen >= 8) {
>+	if (INTEL_INFO(rq->i915)->gen >= 8)
> 		intel_ring_emit(ring, 0); /* upper addr */
>-		intel_ring_emit(ring, 0); /* value */
>-	} else  {
>-		intel_ring_emit(ring, 0);
>-		intel_ring_emit(ring, MI_NOOP);
>-	}
>+	intel_ring_emit(ring, 0); /* value */
> 	intel_ring_advance(ring);
> 	return 0;
> }
> 
> static int
>-gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
>-			      u64 offset, u32 len,
>-			      unsigned flags)
>+gen8_emit_batchbuffer(struct i915_gem_request *rq,
>+		      u64 offset, u32 len,
>+		      unsigned flags)
> {
>-	struct drm_i915_private *dev_priv = ring->dev->dev_private;
>-	bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL &&
>-		!(flags & I915_DISPATCH_SECURE);
>-	int ret;
>+	bool ppgtt = USES_PPGTT(ring->dev) && !(flags & I915_DISPATCH_SECURE);
>+	struct intel_ringbuffer *ring;
> 
>-	ret = intel_ring_begin(ring, 4);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 3);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	/* FIXME(BDW): Address space and security selectors. */
> 	intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
> 	intel_ring_emit(ring, lower_32_bits(offset));
> 	intel_ring_emit(ring, upper_32_bits(offset));
>-	intel_ring_emit(ring, MI_NOOP);
> 	intel_ring_advance(ring);
> 
> 	return 0;
> }
> 
> static int
>-hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
>-			      u64 offset, u32 len,
>-			      unsigned flags)
>+hsw_emit_batchbuffer(struct i915_gem_request *rq,
>+		     u64 offset, u32 len,
>+		     unsigned flags)
> {
>-	int ret;
>+	struct intel_ringbuffer *ring;
> 
>-	ret = intel_ring_begin(ring, 2);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 2);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring,
>-			MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW |
>-			(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
>+			MI_BATCH_BUFFER_START |
>+			(flags & I915_DISPATCH_SECURE ?
>+			 0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW));
> 	/* bit0-7 is the length on GEN6+ */
> 	intel_ring_emit(ring, offset);
> 	intel_ring_advance(ring);
>@@ -2033,15 +1806,15 @@ hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
> }
> 
> static int
>-gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
>-			      u64 offset, u32 len,
>-			      unsigned flags)
>+gen6_emit_batchbuffer(struct i915_gem_request *rq,
>+		      u64 offset, u32 len,
>+		      unsigned flags)
> {
>-	int ret;
>+	struct intel_ringbuffer *ring;
> 
>-	ret = intel_ring_begin(ring, 2);
>-	if (ret)
>-		return ret;
>+	ring = intel_ring_begin(rq, 2);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
> 
> 	intel_ring_emit(ring,
> 			MI_BATCH_BUFFER_START |
>@@ -2055,60 +1828,102 @@ gen6_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
> 
> /* Blitter support (SandyBridge+) */
> 
>-static int gen6_ring_flush(struct intel_engine_cs *ring,
>-			   u32 invalidate, u32 flush)
>+static int gen6_blt_emit_flush(struct i915_gem_request *rq,
>+			       u32 flags)
> {
>-	struct drm_device *dev = ring->dev;
>+	struct intel_ringbuffer *ring;
> 	uint32_t cmd;
>-	int ret;
>-
>-	ret = intel_ring_begin(ring, 4);
>-	if (ret)
>-		return ret;
> 
>-	cmd = MI_FLUSH_DW;
>-	if (INTEL_INFO(ring->dev)->gen >= 8)
>+	cmd = 3;
>+	if (INTEL_INFO(rq->i915)->gen >= 8)
> 		cmd += 1;
>+
>+	ring = intel_ring_begin(rq, cmd);
>+	if (IS_ERR(ring))
>+		return PTR_ERR(ring);
>+
> 	/*
> 	 * Bspec vol 1c.3 - blitter engine command streamer:
> 	 * "If ENABLED, all TLBs will be invalidated once the flush
> 	 * operation is complete. This bit is only valid when the
> 	 * Post-Sync Operation field is a value of 1h or 3h."
> 	 */
>-	if (invalidate & I915_GEM_DOMAIN_RENDER)
>-		cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
>-			MI_FLUSH_DW_OP_STOREDW;
>+	cmd = MI_FLUSH_DW | (cmd - 2);
>+	if (flags & I915_INVALIDATE_CACHES)
>+		cmd |= (MI_INVALIDATE_TLB |
>+			MI_FLUSH_DW_STORE_INDEX |
>+			MI_FLUSH_DW_OP_STOREDW);
> 	intel_ring_emit(ring, cmd);
> 	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
>-	if (INTEL_INFO(ring->dev)->gen >= 8) {
>+	if (INTEL_INFO(rq->i915)->gen >= 8)
> 		intel_ring_emit(ring, 0); /* upper addr */
>-		intel_ring_emit(ring, 0); /* value */
>-	} else  {
>-		intel_ring_emit(ring, 0);
>-		intel_ring_emit(ring, MI_NOOP);
>-	}
>+	intel_ring_emit(ring, 0); /* value */
> 	intel_ring_advance(ring);
> 
>-	if (IS_GEN7(dev) && !invalidate && flush)
>-		return gen7_ring_fbc_flush(ring, FBC_REND_CACHE_CLEAN);
>+	if (IS_GEN7(rq->i915) && flags & I915_KICK_FBC)
>+		return gen7_ring_fbc_flush(rq, FBC_REND_CACHE_CLEAN);
> 
> 	return 0;
> }
> 
>-int intel_init_render_ring_buffer(struct drm_device *dev)
>+static void gen8_engine_init_semaphore(struct intel_engine_cs *engine)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
>+	if (engine->i915->semaphore_obj == NULL)
>+		return;
>+
>+	engine->semaphore.wait = gen8_emit_wait;
>+	engine->semaphore.signal =
>+		engine->id == RCS ? gen8_rcs_emit_signal : gen8_xcs_emit_signal;
>+}
>+
>+static bool semaphores_enabled(struct drm_i915_private *dev_priv)
>+{
>+	if (INTEL_INFO(dev_priv)->gen < 6)
>+		return false;
>+
>+	if (i915.semaphores >= 0)
>+		return i915.semaphores;
>+
>+	/* Until we get further testing... */
>+	if (IS_GEN8(dev_priv))
>+		return false;
>+
>+#ifdef CONFIG_INTEL_IOMMU
>+	/* Enable semaphores on SNB when IO remapping is off */
>+	if (INTEL_INFO(dev_priv)->gen == 6 && intel_iommu_gfx_mapped)
>+		return false;
>+#endif
>+
>+	return true;
>+}
>+
>+int intel_init_render_engine(struct drm_i915_private *dev_priv)
>+{
>+	struct intel_engine_cs *engine = &dev_priv->engine[RCS];
> 	struct drm_i915_gem_object *obj;
> 	int ret;
> 
>-	ring->name = "render ring";
>-	ring->id = RCS;
>-	ring->mmio_base = RENDER_RING_BASE;
>+	ret = intel_engine_init(engine, dev_priv);
>+	if (ret)
>+		return ret;
>+
>+	engine->name = "render ring";
>+	engine->id = RCS;
>+	engine->power_domains = FORCEWAKE_RENDER;
>+	engine->mmio_base = RENDER_RING_BASE;
>+
>+	engine->init_context = i915_gem_render_state_init;
>+
>+	if (HAS_L3_DPF(dev_priv)) {
>+		if (INTEL_INFO(dev_priv)->gen >= 8)
>+			engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
>+		else
>+			engine->irq_keep_mask |= GT_PARITY_ERROR(dev_priv);
>+	}
> 
>-	if (INTEL_INFO(dev)->gen >= 8) {
>-		if (i915_semaphore_is_enabled(dev)) {
>-			obj = i915_gem_alloc_object(dev, 4096);
>+	if (INTEL_INFO(dev_priv)->gen >= 8) {
>+		if (semaphores_enabled(dev_priv)) {
>+			obj = i915_gem_alloc_object(dev_priv->dev, 4096);
> 			if (obj == NULL) {
> 				DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
> 				i915.semaphores = 0;
>@@ -2123,32 +1938,28 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
> 					dev_priv->semaphore_obj = obj;
> 			}
> 		}
>-		ring->add_request = gen6_add_request;
>-		ring->flush = gen8_render_ring_flush;
>-		ring->irq_get = gen8_ring_get_irq;
>-		ring->irq_put = gen8_ring_put_irq;
>-		ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
>-		ring->get_seqno = gen6_ring_get_seqno;
>-		ring->set_seqno = ring_set_seqno;
>-		if (i915_semaphore_is_enabled(dev)) {
>-			WARN_ON(!dev_priv->semaphore_obj);
>-			ring->semaphore.sync_to = gen8_ring_sync;
>-			ring->semaphore.signal = gen8_rcs_signal;
>-			GEN8_RING_SEMAPHORE_INIT;
>-		}
>-	} else if (INTEL_INFO(dev)->gen >= 6) {
>-		ring->add_request = gen6_add_request;
>-		ring->flush = gen7_render_ring_flush;
>-		if (INTEL_INFO(dev)->gen == 6)
>-			ring->flush = gen6_render_ring_flush;
>-		ring->irq_get = gen6_ring_get_irq;
>-		ring->irq_put = gen6_ring_put_irq;
>-		ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
>-		ring->get_seqno = gen6_ring_get_seqno;
>-		ring->set_seqno = ring_set_seqno;
>-		if (i915_semaphore_is_enabled(dev)) {
>-			ring->semaphore.sync_to = gen6_ring_sync;
>-			ring->semaphore.signal = gen6_signal;
>+		if (IS_CHERRYVIEW(dev_priv))
>+			engine->init_context = chv_render_init_context;
>+		else
>+			engine->init_context = bdw_render_init_context;
>+		engine->emit_breadcrumb = i9xx_emit_breadcrumb;
>+		engine->emit_flush = gen8_render_emit_flush;
>+		engine->irq_get = gen8_irq_get;
>+		engine->irq_put = gen8_irq_put;
>+		engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
>+		gen8_engine_init_semaphore(engine);
>+	} else if (INTEL_INFO(dev_priv)->gen >= 6) {
>+		engine->emit_breadcrumb = i9xx_emit_breadcrumb;
>+		engine->emit_flush = gen7_render_emit_flush;
>+		if (INTEL_INFO(dev_priv)->gen == 6)
>+			engine->emit_flush = gen6_render_emit_flush;
>+		engine->irq_get = gen6_irq_get;
>+		engine->irq_barrier = gen6_irq_barrier;
>+		engine->irq_put = gen6_irq_put;
>+		engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
>+		if (semaphores_enabled(dev_priv)) {
>+			engine->semaphore.wait = gen6_emit_wait;
>+			engine->semaphore.signal = gen6_emit_signal;
> 			/*
> 			 * The current semaphore is only applied on pre-gen8
> 			 * platform.  And there is no VCS2 ring on the pre-gen8
>@@ -2156,312 +1967,206 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
> 			 * initialized as INVALID.  Gen8 will initialize the
> 			 * sema between VCS2 and RCS later.
> 			 */
>-			ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
>-			ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
>-			ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
>-			ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
>-			ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
>-			ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
>-			ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
>-			ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
>-			ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
>-			ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
>+			engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
>+			engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
>+			engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
>+			engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
>+			engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
>+			engine->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
>+			engine->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
>+			engine->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
>+			engine->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
>+			engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
> 		}
>-	} else if (IS_GEN5(dev)) {
>-		ring->add_request = pc_render_add_request;
>-		ring->flush = gen4_render_ring_flush;
>-		ring->get_seqno = pc_render_get_seqno;
>-		ring->set_seqno = pc_render_set_seqno;
>-		ring->irq_get = gen5_ring_get_irq;
>-		ring->irq_put = gen5_ring_put_irq;
>-		ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
>-					GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
>+	} else if (IS_GEN5(dev_priv)) {
>+		engine->emit_breadcrumb = gen5_emit_breadcrumb;
>+		engine->emit_flush = gen4_emit_flush;
>+		engine->get_seqno = gen5_render_get_seqno;
>+		engine->set_seqno = gen5_render_set_seqno;
>+		engine->irq_get = gen5_irq_get;
>+		engine->irq_put = gen5_irq_put;
>+		engine->irq_enable_mask =
>+			GT_RENDER_USER_INTERRUPT |
>+			GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
> 	} else {
>-		ring->add_request = i9xx_add_request;
>-		if (INTEL_INFO(dev)->gen < 4)
>-			ring->flush = gen2_render_ring_flush;
>+		engine->emit_breadcrumb = i9xx_emit_breadcrumb;
>+		if (INTEL_INFO(dev_priv)->gen < 4)
>+			engine->emit_flush = gen2_emit_flush;
> 		else
>-			ring->flush = gen4_render_ring_flush;
>-		ring->get_seqno = ring_get_seqno;
>-		ring->set_seqno = ring_set_seqno;
>-		if (IS_GEN2(dev)) {
>-			ring->irq_get = i8xx_ring_get_irq;
>-			ring->irq_put = i8xx_ring_put_irq;
>+			engine->emit_flush = gen4_emit_flush;
>+		if (IS_GEN2(dev_priv)) {
>+			engine->irq_get = i8xx_irq_get;
>+			engine->irq_put = i8xx_irq_put;
> 		} else {
>-			ring->irq_get = i9xx_ring_get_irq;
>-			ring->irq_put = i9xx_ring_put_irq;
>+			engine->irq_get = i9xx_irq_get;
>+			engine->irq_put = i9xx_irq_put;
> 		}
>-		ring->irq_enable_mask = I915_USER_INTERRUPT;
>-	}
>-	ring->write_tail = ring_write_tail;
>-
>-	if (IS_HASWELL(dev))
>-		ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
>-	else if (IS_GEN8(dev))
>-		ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
>-	else if (INTEL_INFO(dev)->gen >= 6)
>-		ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
>-	else if (INTEL_INFO(dev)->gen >= 4)
>-		ring->dispatch_execbuffer = i965_dispatch_execbuffer;
>-	else if (IS_I830(dev) || IS_845G(dev))
>-		ring->dispatch_execbuffer = i830_dispatch_execbuffer;
>+		engine->irq_enable_mask = I915_USER_INTERRUPT;
>+	}
>+
>+	if (IS_GEN8(dev_priv))
>+		engine->emit_batchbuffer = gen8_emit_batchbuffer;
>+	else if (IS_HASWELL(dev_priv))
>+		engine->emit_batchbuffer = hsw_emit_batchbuffer;
>+	else if (INTEL_INFO(dev_priv)->gen >= 6)
>+		engine->emit_batchbuffer = gen6_emit_batchbuffer;
>+	else if (INTEL_INFO(dev_priv)->gen >= 4)
>+		engine->emit_batchbuffer = i965_emit_batchbuffer;
>+	else if (IS_I830(dev_priv) || IS_845G(dev_priv))
>+		engine->emit_batchbuffer = i830_emit_batchbuffer;
> 	else
>-		ring->dispatch_execbuffer = i915_dispatch_execbuffer;
>-	ring->init = init_render_ring;
>-	ring->cleanup = render_ring_cleanup;
>-
>-	/* Workaround batchbuffer to combat CS tlb bug. */
>-	if (HAS_BROKEN_CS_TLB(dev)) {
>-		obj = i915_gem_alloc_object(dev, I830_WA_SIZE);
>-		if (obj == NULL) {
>-			DRM_ERROR("Failed to allocate batch bo\n");
>-			return -ENOMEM;
>-		}
>+		engine->emit_batchbuffer = i915_emit_batchbuffer;
> 
>-		ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
>-		if (ret != 0) {
>-			drm_gem_object_unreference(&obj->base);
>-			DRM_ERROR("Failed to ping batch bo\n");
>-			return ret;
>-		}
>+	engine->resume = render_resume;
>+	engine->cleanup = render_cleanup;
> 
>-		ring->scratch.obj = obj;
>-		ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
>-	}
>+	if (HAS_BROKEN_CS_TLB(dev_priv))
>+		/* Workaround batchbuffer to combat CS tlb bug. */
>+		ret = init_broken_cs_tlb_wa(engine);
>+	else if (INTEL_INFO(dev_priv)->gen >= 5)
>+		ret = init_pipe_control(engine);
>+	if (ret)
>+		return ret;
> 
>-	return intel_init_ring_buffer(dev, ring);
>+	return 0;
> }
> 
>-int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
>+int intel_init_bsd_engine(struct drm_i915_private *dev_priv)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
>-	struct intel_ringbuffer *ringbuf = ring->buffer;
>+	struct intel_engine_cs *engine = &dev_priv->engine[VCS];
> 	int ret;
> 
>-	if (ringbuf == NULL) {
>-		ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
>-		if (!ringbuf)
>-			return -ENOMEM;
>-		ring->buffer = ringbuf;
>-	}
>-
>-	ring->name = "render ring";
>-	ring->id = RCS;
>-	ring->mmio_base = RENDER_RING_BASE;
>-
>-	if (INTEL_INFO(dev)->gen >= 6) {
>-		/* non-kms not supported on gen6+ */
>-		ret = -ENODEV;
>-		goto err_ringbuf;
>-	}
>-
>-	/* Note: gem is not supported on gen5/ilk without kms (the corresponding
>-	 * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
>-	 * the special gen5 functions. */
>-	ring->add_request = i9xx_add_request;
>-	if (INTEL_INFO(dev)->gen < 4)
>-		ring->flush = gen2_render_ring_flush;
>-	else
>-		ring->flush = gen4_render_ring_flush;
>-	ring->get_seqno = ring_get_seqno;
>-	ring->set_seqno = ring_set_seqno;
>-	if (IS_GEN2(dev)) {
>-		ring->irq_get = i8xx_ring_get_irq;
>-		ring->irq_put = i8xx_ring_put_irq;
>-	} else {
>-		ring->irq_get = i9xx_ring_get_irq;
>-		ring->irq_put = i9xx_ring_put_irq;
>-	}
>-	ring->irq_enable_mask = I915_USER_INTERRUPT;
>-	ring->write_tail = ring_write_tail;
>-	if (INTEL_INFO(dev)->gen >= 4)
>-		ring->dispatch_execbuffer = i965_dispatch_execbuffer;
>-	else if (IS_I830(dev) || IS_845G(dev))
>-		ring->dispatch_execbuffer = i830_dispatch_execbuffer;
>-	else
>-		ring->dispatch_execbuffer = i915_dispatch_execbuffer;
>-	ring->init = init_render_ring;
>-	ring->cleanup = render_ring_cleanup;
>-
>-	ring->dev = dev;
>-	INIT_LIST_HEAD(&ring->active_list);
>-	INIT_LIST_HEAD(&ring->request_list);
>-
>-	ringbuf->size = size;
>-	ringbuf->effective_size = ringbuf->size;
>-	if (IS_I830(ring->dev) || IS_845G(ring->dev))
>-		ringbuf->effective_size -= 2 * CACHELINE_BYTES;
>-
>-	ringbuf->virtual_start = ioremap_wc(start, size);
>-	if (ringbuf->virtual_start == NULL) {
>-		DRM_ERROR("can not ioremap virtual address for"
>-			  " ring buffer\n");
>-		ret = -ENOMEM;
>-		goto err_ringbuf;
>-	}
>-
>-	if (!I915_NEED_GFX_HWS(dev)) {
>-		ret = init_phys_status_page(ring);
>-		if (ret)
>-			goto err_vstart;
>-	}
>-
>-	return 0;
>-
>-err_vstart:
>-	iounmap(ringbuf->virtual_start);
>-err_ringbuf:
>-	kfree(ringbuf);
>-	ring->buffer = NULL;
>-	return ret;
>-}
>+	ret = intel_engine_init(engine, dev_priv);
>+	if (ret)
>+		return ret;
> 
>-int intel_init_bsd_ring_buffer(struct drm_device *dev)
>-{
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[VCS];
>+	engine->name = "bsd ring";
>+	engine->id = VCS;
>+	engine->power_domains = FORCEWAKE_MEDIA;
> 
>-	ring->name = "bsd ring";
>-	ring->id = VCS;
>+	if (INTEL_INFO(dev_priv)->gen >= 6) {
>+		engine->mmio_base = GEN6_BSD_RING_BASE;
> 
>-	ring->write_tail = ring_write_tail;
>-	if (INTEL_INFO(dev)->gen >= 6) {
>-		ring->mmio_base = GEN6_BSD_RING_BASE;
> 		/* gen6 bsd needs a special wa for tail updates */
>-		if (IS_GEN6(dev))
>-			ring->write_tail = gen6_bsd_ring_write_tail;
>-		ring->flush = gen6_bsd_ring_flush;
>-		ring->add_request = gen6_add_request;
>-		ring->get_seqno = gen6_ring_get_seqno;
>-		ring->set_seqno = ring_set_seqno;
>-		if (INTEL_INFO(dev)->gen >= 8) {
>-			ring->irq_enable_mask =
>+		if (IS_GEN6(dev_priv))
>+			engine->write_tail = gen6_bsd_ring_write_tail;
>+		engine->emit_flush = gen6_bsd_emit_flush;
>+		engine->emit_breadcrumb = i9xx_emit_breadcrumb;
>+		if (INTEL_INFO(dev_priv)->gen >= 8) {
>+			engine->irq_enable_mask =
> 				GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
>-			ring->irq_get = gen8_ring_get_irq;
>-			ring->irq_put = gen8_ring_put_irq;
>-			ring->dispatch_execbuffer =
>-				gen8_ring_dispatch_execbuffer;
>-			if (i915_semaphore_is_enabled(dev)) {
>-				ring->semaphore.sync_to = gen8_ring_sync;
>-				ring->semaphore.signal = gen8_xcs_signal;
>-				GEN8_RING_SEMAPHORE_INIT;
>-			}
>+			engine->irq_get = gen8_irq_get;
>+			engine->irq_put = gen8_irq_put;
>+			engine->emit_batchbuffer = gen8_emit_batchbuffer;
>+			gen8_engine_init_semaphore(engine);
> 		} else {
>-			ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
>-			ring->irq_get = gen6_ring_get_irq;
>-			ring->irq_put = gen6_ring_put_irq;
>-			ring->dispatch_execbuffer =
>-				gen6_ring_dispatch_execbuffer;
>-			if (i915_semaphore_is_enabled(dev)) {
>-				ring->semaphore.sync_to = gen6_ring_sync;
>-				ring->semaphore.signal = gen6_signal;
>-				ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
>-				ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
>-				ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
>-				ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
>-				ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
>-				ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
>-				ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
>-				ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
>-				ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
>-				ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
>+			engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
>+			engine->irq_get = gen6_irq_get;
>+			engine->irq_barrier = gen6_irq_barrier;
>+			engine->irq_put = gen6_irq_put;
>+			engine->emit_batchbuffer = gen6_emit_batchbuffer;
>+			if (semaphores_enabled(dev_priv)) {
>+				engine->semaphore.wait = gen6_emit_wait;
>+				engine->semaphore.signal = gen6_emit_signal;
>+				engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
>+				engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
>+				engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
>+				engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
>+				engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
>+				engine->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
>+				engine->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
>+				engine->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
>+				engine->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
>+				engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
> 			}
> 		}
> 	} else {
>-		ring->mmio_base = BSD_RING_BASE;
>-		ring->flush = bsd_ring_flush;
>-		ring->add_request = i9xx_add_request;
>-		ring->get_seqno = ring_get_seqno;
>-		ring->set_seqno = ring_set_seqno;
>-		if (IS_GEN5(dev)) {
>-			ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
>-			ring->irq_get = gen5_ring_get_irq;
>-			ring->irq_put = gen5_ring_put_irq;
>+		engine->mmio_base = BSD_RING_BASE;
>+
>+		engine->emit_flush = bsd_emit_flush;
>+		engine->emit_breadcrumb = i9xx_emit_breadcrumb;
>+		if (IS_GEN5(dev_priv)) {
>+			engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
>+			engine->irq_get = gen5_irq_get;
>+			engine->irq_put = gen5_irq_put;
> 		} else {
>-			ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
>-			ring->irq_get = i9xx_ring_get_irq;
>-			ring->irq_put = i9xx_ring_put_irq;
>+			engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
>+			engine->irq_get = i9xx_irq_get;
>+			engine->irq_put = i9xx_irq_put;
> 		}
>-		ring->dispatch_execbuffer = i965_dispatch_execbuffer;
>+		engine->emit_batchbuffer = i965_emit_batchbuffer;
> 	}
>-	ring->init = init_ring_common;
> 
>-	return intel_init_ring_buffer(dev, ring);
>+	return 0;
> }
> 
> /**
>  * Initialize the second BSD ring for Broadwell GT3.
>  * It is noted that this only exists on Broadwell GT3.
>  */
>-int intel_init_bsd2_ring_buffer(struct drm_device *dev)
>+int intel_init_bsd2_engine(struct drm_i915_private *dev_priv)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
>+	struct intel_engine_cs *engine = &dev_priv->engine[VCS2];
>+	int ret;
> 
>-	if ((INTEL_INFO(dev)->gen != 8)) {
>+	if ((INTEL_INFO(dev_priv)->gen != 8)) {
> 		DRM_ERROR("No dual-BSD ring on non-BDW machine\n");
> 		return -EINVAL;
> 	}
> 
>-	ring->name = "bsd2 ring";
>-	ring->id = VCS2;
>+	ret = intel_engine_init(engine, dev_priv);
>+	if (ret)
>+		return ret;
>+
>+	engine->name = "bsd2 ring";
>+	engine->id = VCS2;
>+	engine->power_domains = FORCEWAKE_MEDIA;
>+	engine->mmio_base = GEN8_BSD2_RING_BASE;
> 
>-	ring->write_tail = ring_write_tail;
>-	ring->mmio_base = GEN8_BSD2_RING_BASE;
>-	ring->flush = gen6_bsd_ring_flush;
>-	ring->add_request = gen6_add_request;
>-	ring->get_seqno = gen6_ring_get_seqno;
>-	ring->set_seqno = ring_set_seqno;
>-	ring->irq_enable_mask =
>+	engine->emit_flush = gen6_bsd_emit_flush;
>+	engine->emit_breadcrumb = i9xx_emit_breadcrumb;
>+	engine->emit_batchbuffer = gen8_emit_batchbuffer;
>+	engine->irq_enable_mask =
> 			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
>-	ring->irq_get = gen8_ring_get_irq;
>-	ring->irq_put = gen8_ring_put_irq;
>-	ring->dispatch_execbuffer =
>-			gen8_ring_dispatch_execbuffer;
>-	if (i915_semaphore_is_enabled(dev)) {
>-		ring->semaphore.sync_to = gen8_ring_sync;
>-		ring->semaphore.signal = gen8_xcs_signal;
>-		GEN8_RING_SEMAPHORE_INIT;
>-	}
>-	ring->init = init_ring_common;
>+	engine->irq_get = gen8_irq_get;
>+	engine->irq_put = gen8_irq_put;
>+	gen8_engine_init_semaphore(engine);
> 
>-	return intel_init_ring_buffer(dev, ring);
>+	return 0;
> }
> 
>-int intel_init_blt_ring_buffer(struct drm_device *dev)
>+int intel_init_blt_engine(struct drm_i915_private *dev_priv)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[BCS];
>+	struct intel_engine_cs *engine = &dev_priv->engine[BCS];
>+	int ret;
> 
>-	ring->name = "blitter ring";
>-	ring->id = BCS;
>+	ret = intel_engine_init(engine, dev_priv);
>+	if (ret)
>+		return ret;
> 
>-	ring->mmio_base = BLT_RING_BASE;
>-	ring->write_tail = ring_write_tail;
>-	ring->flush = gen6_ring_flush;
>-	ring->add_request = gen6_add_request;
>-	ring->get_seqno = gen6_ring_get_seqno;
>-	ring->set_seqno = ring_set_seqno;
>-	if (INTEL_INFO(dev)->gen >= 8) {
>-		ring->irq_enable_mask =
>+	engine->name = "blitter ring";
>+	engine->id = BCS;
>+	engine->power_domains = FORCEWAKE_MEDIA;
>+	engine->mmio_base = BLT_RING_BASE;
>+
>+	engine->emit_flush = gen6_blt_emit_flush;
>+	engine->emit_breadcrumb = i9xx_emit_breadcrumb;
>+	if (INTEL_INFO(dev_priv)->gen >= 8) {
>+		engine->irq_enable_mask =
> 			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
>-		ring->irq_get = gen8_ring_get_irq;
>-		ring->irq_put = gen8_ring_put_irq;
>-		ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
>-		if (i915_semaphore_is_enabled(dev)) {
>-			ring->semaphore.sync_to = gen8_ring_sync;
>-			ring->semaphore.signal = gen8_xcs_signal;
>-			GEN8_RING_SEMAPHORE_INIT;
>-		}
>+		engine->irq_get = gen8_irq_get;
>+		engine->irq_put = gen8_irq_put;
>+		engine->emit_batchbuffer = gen8_emit_batchbuffer;
>+		gen8_engine_init_semaphore(engine);
> 	} else {
>-		ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
>-		ring->irq_get = gen6_ring_get_irq;
>-		ring->irq_put = gen6_ring_put_irq;
>-		ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
>-		if (i915_semaphore_is_enabled(dev)) {
>-			ring->semaphore.signal = gen6_signal;
>-			ring->semaphore.sync_to = gen6_ring_sync;
>+		engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
>+		engine->irq_get = gen6_irq_get;
>+		engine->irq_barrier = gen6_irq_barrier;
>+		engine->irq_put = gen6_irq_put;
>+		engine->emit_batchbuffer = gen6_emit_batchbuffer;
>+		if (semaphores_enabled(dev_priv)) {
>+			engine->semaphore.signal = gen6_emit_signal;
>+			engine->semaphore.wait = gen6_emit_wait;
> 			/*
> 			 * The current semaphore is only applied on pre-gen8
> 			 * platform.  And there is no VCS2 ring on the pre-gen8
>@@ -2469,124 +2174,510 @@ int intel_init_blt_ring_buffer(struct drm_device *dev)
> 			 * initialized as INVALID.  Gen8 will initialize the
> 			 * sema between BCS and VCS2 later.
> 			 */
>-			ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
>-			ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
>-			ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
>-			ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
>-			ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
>-			ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
>-			ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
>-			ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
>-			ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
>-			ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
>+			engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
>+			engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
>+			engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
>+			engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
>+			engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
>+			engine->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
>+			engine->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
>+			engine->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
>+			engine->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
>+			engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
> 		}
> 	}
>-	ring->init = init_ring_common;
> 
>-	return intel_init_ring_buffer(dev, ring);
>+	return 0;
> }
> 
>-int intel_init_vebox_ring_buffer(struct drm_device *dev)
>+int intel_init_vebox_engine(struct drm_i915_private *dev_priv)
> {
>-	struct drm_i915_private *dev_priv = dev->dev_private;
>-	struct intel_engine_cs *ring = &dev_priv->ring[VECS];
>+	struct intel_engine_cs *engine = &dev_priv->engine[VECS];
>+	int ret;
>+
>+	ret = intel_engine_init(engine, dev_priv);
>+	if (ret)
>+		return ret;
> 
>-	ring->name = "video enhancement ring";
>-	ring->id = VECS;
>+	engine->name = "video enhancement ring";
>+	engine->id = VECS;
>+	engine->power_domains = FORCEWAKE_MEDIA;
>+	engine->mmio_base = VEBOX_RING_BASE;
> 
>-	ring->mmio_base = VEBOX_RING_BASE;
>-	ring->write_tail = ring_write_tail;
>-	ring->flush = gen6_ring_flush;
>-	ring->add_request = gen6_add_request;
>-	ring->get_seqno = gen6_ring_get_seqno;
>-	ring->set_seqno = ring_set_seqno;
>+	engine->emit_flush = gen6_blt_emit_flush;
>+	engine->emit_breadcrumb = i9xx_emit_breadcrumb;
> 
>-	if (INTEL_INFO(dev)->gen >= 8) {
>-		ring->irq_enable_mask =
>+	if (INTEL_INFO(dev_priv)->gen >= 8) {
>+		engine->irq_enable_mask =
> 			GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
>-		ring->irq_get = gen8_ring_get_irq;
>-		ring->irq_put = gen8_ring_put_irq;
>-		ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
>-		if (i915_semaphore_is_enabled(dev)) {
>-			ring->semaphore.sync_to = gen8_ring_sync;
>-			ring->semaphore.signal = gen8_xcs_signal;
>-			GEN8_RING_SEMAPHORE_INIT;
>-		}
>+		engine->irq_get = gen8_irq_get;
>+		engine->irq_put = gen8_irq_put;
>+		engine->emit_batchbuffer = gen8_emit_batchbuffer;
>+		gen8_engine_init_semaphore(engine);
> 	} else {
>-		ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
>-		ring->irq_get = hsw_vebox_get_irq;
>-		ring->irq_put = hsw_vebox_put_irq;
>-		ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
>-		if (i915_semaphore_is_enabled(dev)) {
>-			ring->semaphore.sync_to = gen6_ring_sync;
>-			ring->semaphore.signal = gen6_signal;
>-			ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
>-			ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
>-			ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
>-			ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
>-			ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
>-			ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
>-			ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
>-			ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
>-			ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
>-			ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
>+		engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
>+		engine->irq_get = hsw_vebox_irq_get;
>+		engine->irq_barrier = gen6_irq_barrier;
>+		engine->irq_put = hsw_vebox_irq_put;
>+		engine->emit_batchbuffer = gen6_emit_batchbuffer;
>+		if (semaphores_enabled(dev_priv)) {
>+			engine->semaphore.wait = gen6_emit_wait;
>+			engine->semaphore.signal = gen6_emit_signal;
>+			engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
>+			engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
>+			engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
>+			engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
>+			engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
>+			engine->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
>+			engine->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
>+			engine->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
>+			engine->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
>+			engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
> 		}
> 	}
>-	ring->init = init_ring_common;
> 
>-	return intel_init_ring_buffer(dev, ring);
>+	return 0;
> }
> 
> int
>-intel_ring_flush_all_caches(struct intel_engine_cs *ring)
>+intel_engine_flush(struct intel_engine_cs *engine,
>+		   struct intel_context *ctx)
> {
>+	struct i915_gem_request *rq;
> 	int ret;
> 
>-	if (!ring->gpu_caches_dirty)
>+	rq = intel_engine_alloc_request(engine, ctx);
>+	if (IS_ERR(rq))
>+		return PTR_ERR(rq);
>+
>+	ret = i915_request_emit_breadcrumb(rq);
>+	if (ret == 0)
>+		ret = i915_request_commit(rq);
>+	i915_request_put(rq);
>+
>+	return ret;
>+}
>+
>+int intel_engine_sync(struct intel_engine_cs *engine)
>+{
>+	/* Wait upon the last request to be completed */
>+	if (engine->last_request == NULL)
> 		return 0;
> 
>-	ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
>+	return i915_request_wait(engine->last_request);
>+}
>+
>+static u32
>+next_seqno(struct drm_i915_private *i915)
>+{
>+	/* reserve 0 for non-seqno */
>+	if (++i915->next_seqno == 0)
>+		++i915->next_seqno;
>+	return i915->next_seqno;
>+}
>+
>+struct i915_gem_request *
>+intel_engine_alloc_request(struct intel_engine_cs *engine,
>+			   struct intel_context *ctx)
>+{
>+	struct intel_ringbuffer *ring;
>+	struct i915_gem_request *rq;
>+	int ret, n;
>+
>+	ring = ctx->ring[engine->id].ring;
>+	if (ring == NULL) {
>+		ring = engine->get_ring(engine, ctx);
>+		if (IS_ERR(ring))
>+			return ERR_CAST(ring);
>+
>+		ctx->ring[engine->id].ring = ring;
>+	}
>+
>+	rq = kzalloc(sizeof(*rq), GFP_KERNEL);
>+	if (rq == NULL)
>+		return ERR_PTR(-ENOMEM);
>+
>+	kref_init(&rq->kref);
>+	INIT_LIST_HEAD(&rq->vmas);
>+	INIT_LIST_HEAD(&rq->breadcrumb_link);
>+
>+	rq->i915 = engine->i915;
>+	rq->ring = ring;
>+	rq->engine = engine;
>+
>+	rq->reset_counter = atomic_read(&rq->i915->gpu_error.reset_counter);
>+	if (rq->reset_counter & (I915_RESET_IN_PROGRESS_FLAG | I915_WEDGED)) {
>+		ret = rq->reset_counter & I915_WEDGED ? -EIO : -EAGAIN;
>+		goto err;
>+	}
>+
>+	rq->seqno = next_seqno(rq->i915);
>+	memcpy(rq->semaphore, engine->semaphore.sync, sizeof(rq->semaphore));
>+	for (n = 0; n < ARRAY_SIZE(rq->semaphore); n++)
>+		if (__i915_seqno_passed(rq->semaphore[n], rq->seqno))
>+			rq->semaphore[n] = 0;
>+	rq->head = ring->tail;
>+	rq->outstanding = true;
>+	rq->pending_flush = ring->pending_flush;
>+
>+	rq->ctx = ctx;
>+	i915_gem_context_reference(rq->ctx);
>+
>+	ret = i915_request_switch_context(rq);
> 	if (ret)
>-		return ret;
>+		goto err_ctx;
>+
>+	return rq;
>+
>+err_ctx:
>+	i915_gem_context_unreference(ctx);
>+err:
>+	kfree(rq);
>+	return ERR_PTR(ret);
>+}
>+
>+struct i915_gem_request *
>+intel_engine_seqno_to_request(struct intel_engine_cs *engine,
>+			      u32 seqno)
>+{
>+	struct i915_gem_request *rq;
>+
>+	list_for_each_entry(rq, &engine->requests, engine_list) {
>+		if (rq->seqno == seqno)
>+			return rq;
>+
>+		if (__i915_seqno_passed(rq->seqno, seqno))
>+			break;
>+	}
>+
>+	return NULL;
>+}
>+
>+void intel_engine_cleanup(struct intel_engine_cs *engine)
>+{
>+	WARN_ON(engine->last_request);
>+
>+	if (engine->cleanup)
>+		engine->cleanup(engine);
>+}
>+
>+static void intel_engine_clear_rings(struct intel_engine_cs *engine)
>+{
>+	struct intel_ringbuffer *ring;
>+
>+	list_for_each_entry(ring, &engine->rings, engine_list) {
>+		if (ring->retired_head != -1) {
>+			ring->head = ring->retired_head;
>+			ring->retired_head = -1;
>+
>+			ring->space = intel_ring_space(ring);
>+		}
>+
>+		if (ring->last_context != NULL) {
>+			struct drm_i915_gem_object *obj;
>+
>+			obj = ring->last_context->ring[engine->id].state;
>+			if (obj)
>+				i915_gem_object_ggtt_unpin(obj);
>+
>+			ring->last_context = NULL;
>+		}
>+	}
>+}
>+
>+int intel_engine_suspend(struct intel_engine_cs *engine)
>+{
>+	struct drm_i915_private *dev_priv = engine->i915;
>+	int ret = 0;
>+
>+	if (WARN_ON(!intel_engine_initialized(engine)))
>+		return 0;
>+
>+	I915_WRITE_IMR(engine, ~0);
>+
>+	if (engine->suspend)
>+		ret = engine->suspend(engine);
>+
>+	intel_engine_clear_rings(engine);
>+
>+	return ret;
>+}
>+
>+int intel_engine_resume(struct intel_engine_cs *engine)
>+{
>+	struct drm_i915_private *dev_priv = engine->i915;
>+	int ret = 0;
>+
>+	if (WARN_ON(!intel_engine_initialized(engine)))
>+		return 0;
>+
>+	if (engine->resume)
>+		ret = engine->resume(engine);
>+
>+	I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
>+	return ret;
>+}
>+
>+int intel_engine_retire(struct intel_engine_cs *engine,
>+			u32 seqno)
>+{
>+	int count;
> 
>-	trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);
>+	if (engine->retire)
>+		engine->retire(engine, seqno);
>+
>+	count = 0;
>+	while (!list_empty(&engine->requests)) {
>+		struct i915_gem_request *rq;
>+
>+		rq = list_first_entry(&engine->requests,
>+				      struct i915_gem_request,
>+				      engine_list);
>+
>+		if (!__i915_seqno_passed(seqno, rq->seqno))
>+			break;
>+
>+		i915_request_retire(rq);
>+		count++;
>+	}
>+
>+	if (unlikely(engine->trace_irq_seqno &&
>+		     __i915_seqno_passed(seqno, engine->trace_irq_seqno))) {
>+		engine->irq_put(engine);
>+		engine->trace_irq_seqno = 0;
>+	}
>+
>+	return count;
>+}
>+
>+static struct i915_gem_request *
>+find_active_batch(struct list_head *list)
>+{
>+	struct i915_gem_request *rq, *last = NULL;
>+
>+	list_for_each_entry(rq, list, engine_list) {
>+		if (rq->batch == NULL)
>+			continue;
>+
>+		if (!__i915_request_complete__wa(rq))
>+			return rq;
>+
>+		last = rq;
>+	}
>+
>+	return last;
>+}
>+
>+static bool context_is_banned(const struct intel_context *ctx,
>+			      unsigned long now)
>+{
>+	const struct i915_ctx_hang_stats *hs = &ctx->hang_stats;
>+
>+	if (hs->banned)
>+		return true;
>+
>+	if (hs->ban_period_seconds == 0)
>+		return false;
>+
>+	if (now - hs->guilty_ts <= hs->ban_period_seconds) {
>+		if (!i915_gem_context_is_default(ctx)) {
>+			DRM_DEBUG("context hanging too fast, banning!\n");
>+			return true;
>+		} else if (i915_stop_ring_allow_ban(ctx->i915)) {
>+			if (i915_stop_ring_allow_warn(ctx->i915))
>+				DRM_ERROR("gpu hanging too fast, banning!\n");
>+			return true;
>+		}
>+	}
>+
>+	return false;
>+}
>+
>+static void
>+intel_engine_hangstats(struct intel_engine_cs *engine)
>+{
>+	struct i915_ctx_hang_stats *hs;
>+	struct i915_gem_request *rq;
>+
>+	rq = find_active_batch(&engine->requests);
>+	if (rq == NULL)
>+		return;
>+
>+	hs = &rq->ctx->hang_stats;
>+	if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
>+		unsigned long now = get_seconds();
>+		hs->banned = context_is_banned(rq->ctx, now);
>+		hs->guilty_ts = now;
>+		hs->batch_active++;
>+	} else
>+		hs->batch_pending++;
>+
>+	list_for_each_entry_continue(rq, &engine->requests, engine_list) {
>+		if (rq->batch == NULL)
>+			continue;
>+
>+		if (__i915_request_complete__wa(rq))
>+			continue;
>+
>+		rq->ctx->hang_stats.batch_pending++;
>+	}
>+}
>+
>+void intel_engine_reset(struct intel_engine_cs *engine)
>+{
>+	if (WARN_ON(!intel_engine_initialized(engine)))
>+		return;
>+
>+	if (engine->reset)
>+		engine->reset(engine);
>+
>+	memset(&engine->hangcheck, 0, sizeof(engine->hangcheck));
>+	intel_engine_hangstats(engine);
>+
>+	intel_engine_retire(engine, engine->i915->next_seqno);
>+	intel_engine_clear_rings(engine);
>+}
>+
>+static int ring_wait(struct intel_ringbuffer *ring, int n)
>+{
>+	int ret;
>+
>+	trace_intel_ringbuffer_wait(ring, n);
>+
>+	do {
>+		struct i915_gem_request *rq;
>+
>+		i915_gem_retire_requests__engine(ring->engine);
>+		if (ring->retired_head != -1) {
>+			ring->head = ring->retired_head;
>+			ring->retired_head = -1;
>+
>+			ring->space = intel_ring_space(ring);
>+			if (ring->space >= n)
>+				return 0;
>+		}
>+
>+		list_for_each_entry(rq, &ring->breadcrumbs, breadcrumb_link)
>+			if (__intel_ring_space(rq->tail, ring->tail,
>+					       ring->size, I915_RING_RSVD) >= n)
>+				break;
>+
>+		if (WARN_ON(&rq->breadcrumb_link == &ring->breadcrumbs))
>+			return -EDEADLK;
>+
>+		ret = i915_request_wait(rq);
>+	} while (ret == 0);
>+
>+	return ret;
>+}
>+
>+static int ring_wrap(struct intel_ringbuffer *ring, int bytes)
>+{
>+	uint32_t __iomem *virt;
>+	int rem;
>+
>+	rem = ring->size - ring->tail;
>+	if (unlikely(ring->space < rem)) {
>+		rem = ring_wait(ring, rem);
>+		if (rem)
>+			return rem;
>+	}
>+
>+	trace_intel_ringbuffer_wrap(ring, rem);
>+
>+	virt = ring->virtual_start + ring->tail;
>+	rem = ring->size - ring->tail;
>+
>+	ring->space -= rem;
>+	ring->tail = 0;
>+
>+	rem /= 4;
>+	while (rem--)
>+		iowrite32(MI_NOOP, virt++);
> 
>-	ring->gpu_caches_dirty = false;
> 	return 0;
> }
> 
>-int
>-intel_ring_invalidate_all_caches(struct intel_engine_cs *ring)
>+static int __intel_ring_prepare(struct intel_ringbuffer *ring,
>+				int bytes)
> {
>-	uint32_t flush_domains;
> 	int ret;
> 
>-	flush_domains = 0;
>-	if (ring->gpu_caches_dirty)
>-		flush_domains = I915_GEM_GPU_DOMAINS;
>+	trace_intel_ringbuffer_begin(ring, bytes);
> 
>-	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
>-	if (ret)
>-		return ret;
>+	if (unlikely(ring->tail + bytes > ring->effective_size)) {
>+		ret = ring_wrap(ring, bytes);
>+		if (unlikely(ret))
>+			return ret;
>+	}
> 
>-	trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
>+	if (unlikely(ring->space < bytes)) {
>+		ret = ring_wait(ring, bytes);
>+		if (unlikely(ret))
>+			return ret;
>+	}
> 
>-	ring->gpu_caches_dirty = false;
> 	return 0;
> }
> 
>-void
>-intel_stop_ring_buffer(struct intel_engine_cs *ring)
>+struct intel_ringbuffer *
>+intel_ring_begin(struct i915_gem_request *rq,
>+		 int num_dwords)
> {
>+	struct intel_ringbuffer *ring = rq->ring;
> 	int ret;
> 
>-	if (!intel_ring_initialized(ring))
>-		return;
>+	/* TAIL updates must be aligned to a qword, so make sure we
>+	 * reserve space for any implicit padding required for this
>+	 * command.
>+	 */
>+	ret = __intel_ring_prepare(ring,
>+				   ALIGN(num_dwords, 2) * sizeof(uint32_t));
>+	if (ret)
>+		return ERR_PTR(ret);
>+
>+	ring->space -= num_dwords * sizeof(uint32_t);
>+
>+	return ring;
>+}
>+
>+/* Align the ring tail to a cacheline boundary */
>+int intel_ring_cacheline_align(struct i915_gem_request *rq)
>+{
>+	struct intel_ringbuffer *ring;
>+	int tail, num_dwords;
>+
>+	do {
>+		tail = rq->ring->tail;
>+		num_dwords = (tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
>+		if (num_dwords == 0)
>+			return 0;
>+
>+		num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
>+		ring = intel_ring_begin(rq, num_dwords);
>+		if (IS_ERR(ring))
>+			return PTR_ERR(ring);
>+	} while (tail != rq->ring->tail);
>+
>+	while (num_dwords--)
>+		intel_ring_emit(ring, MI_NOOP);
>+
>+	intel_ring_advance(ring);
>+
>+	return 0;
>+}
>+
>+struct i915_gem_request *
>+intel_engine_find_active_batch(struct intel_engine_cs *engine)
>+{
>+	struct i915_gem_request *rq;
>+	unsigned long flags;
> 
>-	ret = intel_ring_idle(ring);
>-	if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
>-		DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
>-			  ring->name, ret);
>+	spin_lock_irqsave(&engine->irqlock, flags);
>+	rq = find_active_batch(&engine->submitted);
>+	spin_unlock_irqrestore(&engine->irqlock, flags);
>+	if (rq)
>+		return rq;
> 
>-	stop_ring(ring);
>+	return find_active_batch(&engine->requests);
> }
>diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
>index 70525d0..b61e08c 100644
>--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
>+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
>@@ -5,6 +5,13 @@
> 
> #define I915_CMD_HASH_ORDER 9
> 
>+/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
>+ * but keeps the logic simple. Indeed, the whole purpose of this macro is just
>+ * to give some inclination as to some of the magic values used in the various
>+ * workarounds!
>+ */
>+#define CACHELINE_BYTES 64
>+
> /*
>  * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use"
>  * Gen3 BSpec "vol1c Memory Interface Functions" / 2.3.4.5 "Ring Buffer Use"
>@@ -13,61 +20,48 @@
>  * "If the Ring Buffer Head Pointer and the Tail Pointer are on the same
>  * cacheline, the Head Pointer must not be greater than the Tail
>  * Pointer."
>+ *
>+ * To also accommodate errata on 830/845 which makes the last pair of
>+ * cachelines in the ringbuffer unavailable, reduce the available space
>+ * further.
>  */
>-#define I915_RING_FREE_SPACE 64
>+#define I915_RING_RSVD (2*CACHELINE_BYTES)
> 
>-struct  intel_hw_status_page {
>+struct intel_hw_status_page {
> 	u32		*page_addr;
> 	unsigned int	gfx_addr;
> 	struct		drm_i915_gem_object *obj;
> };
> 
>-#define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base))
>-#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val)
>+#define I915_READ_TAIL(engine) I915_READ(RING_TAIL((engine)->mmio_base))
>+#define I915_WRITE_TAIL(engine, val) I915_WRITE(RING_TAIL((engine)->mmio_base), val)
> 
>-#define I915_READ_START(ring) I915_READ(RING_START((ring)->mmio_base))
>-#define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val)
>+#define I915_READ_START(engine) I915_READ(RING_START((engine)->mmio_base))
>+#define I915_WRITE_START(engine, val) I915_WRITE(RING_START((engine)->mmio_base), val)
> 
>-#define I915_READ_HEAD(ring)  I915_READ(RING_HEAD((ring)->mmio_base))
>-#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val)
>+#define I915_READ_HEAD(engine)  I915_READ(RING_HEAD((engine)->mmio_base))
>+#define I915_WRITE_HEAD(engine, val) I915_WRITE(RING_HEAD((engine)->mmio_base), val)
> 
>-#define I915_READ_CTL(ring) I915_READ(RING_CTL((ring)->mmio_base))
>-#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val)
>+#define I915_READ_CTL(engine) I915_READ(RING_CTL((engine)->mmio_base))
>+#define I915_WRITE_CTL(engine, val) I915_WRITE(RING_CTL((engine)->mmio_base), val)
> 
>-#define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base))
>-#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val)
>+#define I915_READ_IMR(engine) I915_READ(RING_IMR((engine)->mmio_base))
>+#define I915_WRITE_IMR(engine, val) I915_WRITE(RING_IMR((engine)->mmio_base), val)
> 
>-#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base))
>-#define I915_WRITE_MODE(ring, val) I915_WRITE(RING_MI_MODE((ring)->mmio_base), val)
>+#define I915_READ_MODE(engine) I915_READ(RING_MI_MODE((engine)->mmio_base))
>+#define I915_WRITE_MODE(engine, val) I915_WRITE(RING_MI_MODE((engine)->mmio_base), val)
> 
> /* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
>  * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
>  */
> #define i915_semaphore_seqno_size sizeof(uint64_t)
>-#define GEN8_SIGNAL_OFFSET(__ring, to)			     \
>-	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
>-	((__ring)->id * I915_NUM_RINGS * i915_semaphore_seqno_size) +	\
>-	(i915_semaphore_seqno_size * (to)))
>-
>-#define GEN8_WAIT_OFFSET(__ring, from)			     \
>-	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
>-	((from) * I915_NUM_RINGS * i915_semaphore_seqno_size) + \
>-	(i915_semaphore_seqno_size * (__ring)->id))
>-
>-#define GEN8_RING_SEMAPHORE_INIT do { \
>-	if (!dev_priv->semaphore_obj) { \
>-		break; \
>-	} \
>-	ring->semaphore.signal_ggtt[RCS] = GEN8_SIGNAL_OFFSET(ring, RCS); \
>-	ring->semaphore.signal_ggtt[VCS] = GEN8_SIGNAL_OFFSET(ring, VCS); \
>-	ring->semaphore.signal_ggtt[BCS] = GEN8_SIGNAL_OFFSET(ring, BCS); \
>-	ring->semaphore.signal_ggtt[VECS] = GEN8_SIGNAL_OFFSET(ring, VECS); \
>-	ring->semaphore.signal_ggtt[VCS2] = GEN8_SIGNAL_OFFSET(ring, VCS2); \
>-	ring->semaphore.signal_ggtt[ring->id] = MI_SEMAPHORE_SYNC_INVALID; \
>-	} while(0)
>-
>-enum intel_ring_hangcheck_action {
>+#define GEN8_SEMAPHORE_OFFSET(__dp, __from, __to)			     \
>+	(i915_gem_obj_ggtt_offset((__dp)->semaphore_obj) + \
>+	 ((__from) * I915_NUM_ENGINES + (__to)) * i915_semaphore_seqno_size)
>+
>+enum intel_engine_hangcheck_action {
> 	HANGCHECK_IDLE = 0,
>+	HANGCHECK_IDLE_WAITERS,
> 	HANGCHECK_WAIT,
> 	HANGCHECK_ACTIVE,
> 	HANGCHECK_ACTIVE_LOOP,
>@@ -77,38 +71,61 @@ enum intel_ring_hangcheck_action {
> 
> #define HANGCHECK_SCORE_RING_HUNG 31
> 
>-struct intel_ring_hangcheck {
>+struct intel_engine_hangcheck {
> 	u64 acthd;
> 	u64 max_acthd;
> 	u32 seqno;
>+	u32 interrupts;
> 	int score;
>-	enum intel_ring_hangcheck_action action;
>+	enum intel_engine_hangcheck_action action;
> 	int deadlock;
> };
> 
>+struct i915_gem_request;
>+struct intel_context;
>+struct intel_engine_cs;
>+
> struct intel_ringbuffer {
>+	struct intel_context *last_context;
>+
>+	struct intel_engine_cs *engine;
>+	struct intel_context *ctx;
>+	struct list_head engine_list;
>+
> 	struct drm_i915_gem_object *obj;
> 	void __iomem *virtual_start;
> 
>-	u32 head;
>-	u32 tail;
>+	/**
>+	 * List of breadcrumbs associated with GPU requests currently
>+	 * outstanding.
>+	 */
>+	struct list_head requests;
>+	struct list_head breadcrumbs;
>+
>+	int head;
>+	int tail;
> 	int space;
>+
> 	int size;
> 	int effective_size;
> 
> 	/** We track the position of the requests in the ring buffer, and
>-	 * when each is retired we increment last_retired_head as the GPU
>+	 * when each is retired we increment retired_head as the GPU
> 	 * must have finished processing the request and so we know we
> 	 * can advance the ringbuffer up to that position.
> 	 *
>-	 * last_retired_head is set to -1 after the value is consumed so
>+	 * retired_head is set to -1 after the value is consumed so
> 	 * we can detect new retirements.
> 	 */
>-	u32 last_retired_head;
>+	int retired_head;
>+	int breadcrumb_tail;
>+
>+	unsigned pending_flush:4;
> };
> 
>-struct  intel_engine_cs {
>-	const char	*name;
>+struct intel_engine_cs {
>+	struct drm_i915_private *i915;
>+	const char *name;
> 	enum intel_ring_id {
> 		RCS = 0x0,
> 		VCS,
>@@ -116,44 +133,82 @@ struct  intel_engine_cs {
> 		VECS,
> 		VCS2
> 	} id;
>-#define I915_NUM_RINGS 5
>+#define I915_NUM_ENGINES 5
>+#define I915_NUM_ENGINE_BITS 4
> #define LAST_USER_RING (VECS + 1)
>-	u32		mmio_base;
>-	struct		drm_device *dev;
>-	struct intel_ringbuffer *buffer;
>+	u32 mmio_base;
>+	u32 power_domains;
>+
>+	/* protects requests against hangcheck */
>+	spinlock_t lock;
>+	/* protects exlists: pending + submitted */
>+	spinlock_t irqlock;
>+
>+	atomic_t interrupts;
>+	u32 breadcrumb[I915_NUM_ENGINES];
>+	u16 tag, next_tag;
>+
>+	struct list_head rings;
>+	struct list_head requests;
>+	struct list_head pending, submitted;
>+	struct i915_gem_request *last_request;
> 
> 	struct intel_hw_status_page status_page;
> 
>-	unsigned irq_refcount; /* protected by dev_priv->irq_lock */
>-	u32		irq_enable_mask;	/* bitmask to enable ring interrupt */
>-	u32		trace_irq_seqno;
>-	bool __must_check (*irq_get)(struct intel_engine_cs *ring);
>-	void		(*irq_put)(struct intel_engine_cs *ring);
>+	struct intel_ringbuffer *legacy_ring;
> 
>-	int		(*init)(struct intel_engine_cs *ring);
>+	unsigned irq_refcount; /* protected by i915->irq_lock */
>+	u32		irq_enable_mask; /* bitmask to enable ring interrupt */
>+	u32             irq_keep_mask; /* never mask these interrupts */
>+	u32		trace_irq_seqno;
>+	void		(*irq_get)(struct intel_engine_cs *engine);
>+	void		(*irq_barrier)(struct intel_engine_cs *engine);
>+	void		(*irq_put)(struct intel_engine_cs *engine);
>+
>+	struct intel_ringbuffer *
>+			(*get_ring)(struct intel_engine_cs *engine,
>+				    struct intel_context *ctx);
>+	void		(*put_ring)(struct intel_ringbuffer *ring,
>+				    struct intel_context *ctx);
>+
>+	void		(*retire)(struct intel_engine_cs *engine,
>+				  u32 seqno);
>+	void		(*reset)(struct intel_engine_cs *engine);
>+	int		(*suspend)(struct intel_engine_cs *engine);
>+	int		(*resume)(struct intel_engine_cs *engine);
>+	void		(*cleanup)(struct intel_engine_cs *engine);
> 
>-	void		(*write_tail)(struct intel_engine_cs *ring,
>-				      u32 value);
>-	int __must_check (*flush)(struct intel_engine_cs *ring,
>-				  u32	invalidate_domains,
>-				  u32	flush_domains);
>-	int		(*add_request)(struct intel_engine_cs *ring);
> 	/* Some chipsets are not quite as coherent as advertised and need
> 	 * an expensive kick to force a true read of the up-to-date seqno.
> 	 * However, the up-to-date seqno is not always required and the last
> 	 * seen value is good enough. Note that the seqno will always be
> 	 * monotonic, even if not coherent.
> 	 */
>-	u32		(*get_seqno)(struct intel_engine_cs *ring,
>-				     bool lazy_coherency);
>-	void		(*set_seqno)(struct intel_engine_cs *ring,
>+	u32		(*get_seqno)(struct intel_engine_cs *engine);
>+	void		(*set_seqno)(struct intel_engine_cs *engine,
> 				     u32 seqno);
>-	int		(*dispatch_execbuffer)(struct intel_engine_cs *ring,
>-					       u64 offset, u32 length,
>-					       unsigned flags);
>+
>+	int		(*init_context)(struct i915_gem_request *rq);
>+
>+	int __must_check (*emit_flush)(struct i915_gem_request *rq,
>+				       u32 domains);
>+#define I915_COMMAND_BARRIER 0x1
>+#define I915_FLUSH_CACHES 0x2
>+#define I915_INVALIDATE_CACHES 0x4
>+#define I915_KICK_FBC 0x8
>+	int __must_check (*emit_batchbuffer)(struct i915_gem_request *rq,
>+					     u64 offset, u32 length,
>+					     unsigned flags);
> #define I915_DISPATCH_SECURE 0x1
> #define I915_DISPATCH_PINNED 0x2
>-	void		(*cleanup)(struct intel_engine_cs *ring);
>+	int __must_check (*emit_breadcrumb)(struct i915_gem_request *rq);
>+
>+	int __must_check (*add_request)(struct i915_gem_request *rq);
>+	void		(*write_tail)(struct intel_engine_cs *engine,
>+				      u32 value);
>+
>+	bool (*is_complete)(struct i915_gem_request *rq);
>+
> 
> 	/* GEN8 signal/wait table - never trust comments!
> 	 *	  signal to	signal to    signal to   signal to      signal to
>@@ -193,27 +248,25 @@ struct  intel_engine_cs {
> 	 *  ie. transpose of f(x, y)
> 	 */
> 	struct {
>-		u32	sync_seqno[I915_NUM_RINGS-1];
>-
>-		union {
>-			struct {
>-				/* our mbox written by others */
>-				u32		wait[I915_NUM_RINGS];
>-				/* mboxes this ring signals to */
>-				u32		signal[I915_NUM_RINGS];
>-			} mbox;
>-			u64		signal_ggtt[I915_NUM_RINGS];
>-		};
>-
>-		/* AKA wait() */
>-		int	(*sync_to)(struct intel_engine_cs *ring,
>-				   struct intel_engine_cs *to,
>-				   u32 seqno);
>-		int	(*signal)(struct intel_engine_cs *signaller,
>-				  /* num_dwords needed by caller */
>-				  unsigned int num_dwords);
>+		struct {
>+			/* our mbox written by others */
>+			u32		wait[I915_NUM_ENGINES];
>+			/* mboxes this ring signals to */
>+			u32		signal[I915_NUM_ENGINES];
>+		} mbox;
>+
>+		int	(*wait)(struct i915_gem_request *waiter,
>+				struct i915_gem_request *signaller);
>+		int	(*signal)(struct i915_gem_request *rq, int id);
>+
>+		u32 sync[I915_NUM_ENGINES];
> 	} semaphore;
> 
>+	/* Execlists */
>+	bool execlists_enabled;
>+	u32 execlists_submitted;
>+	u8 next_context_status_buffer;
>+
> 	/**
> 	 * List of objects currently involved in rendering from the
> 	 * ringbuffer.
>@@ -224,28 +277,13 @@ struct  intel_engine_cs {
> 	 *
> 	 * A reference is held on the buffer while on this list.
> 	 */
>-	struct list_head active_list;
>-
>-	/**
>-	 * List of breadcrumbs associated with GPU requests currently
>-	 * outstanding.
>-	 */
>-	struct list_head request_list;
>-
>-	/**
>-	 * Do we have some not yet emitted requests outstanding?
>-	 */
>-	struct drm_i915_gem_request *preallocated_lazy_request;
>-	u32 outstanding_lazy_seqno;
>-	bool gpu_caches_dirty;
>-	bool fbc_dirty;
>+	struct list_head read_list, write_list, fence_list;
> 
> 	wait_queue_head_t irq_queue;
> 
> 	struct intel_context *default_context;
>-	struct intel_context *last_context;
> 
>-	struct intel_ring_hangcheck hangcheck;
>+	struct intel_engine_hangcheck hangcheck;
> 
> 	struct {
> 		struct drm_i915_gem_object *obj;
>@@ -288,52 +326,31 @@ struct  intel_engine_cs {
> };
> 
> static inline bool
>-intel_ring_initialized(struct intel_engine_cs *ring)
>+intel_engine_initialized(struct intel_engine_cs *engine)
> {
>-	return ring->buffer && ring->buffer->obj;
>+	return engine->default_context;
> }
> 
> static inline unsigned
>-intel_ring_flag(struct intel_engine_cs *ring)
>+intel_engine_flag(struct intel_engine_cs *engine)
> {
>-	return 1 << ring->id;
>-}
>-
>-static inline u32
>-intel_ring_sync_index(struct intel_engine_cs *ring,
>-		      struct intel_engine_cs *other)
>-{
>-	int idx;
>-
>-	/*
>-	 * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2;
>-	 * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs;
>-	 * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs;
>-	 * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs;
>-	 * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
>-	 */
>-
>-	idx = (other - ring) - 1;
>-	if (idx < 0)
>-		idx += I915_NUM_RINGS;
>-
>-	return idx;
>+	return 1 << engine->id;
> }
> 
> static inline u32
>-intel_read_status_page(struct intel_engine_cs *ring,
>+intel_read_status_page(struct intel_engine_cs *engine,
> 		       int reg)
> {
> 	/* Ensure that the compiler doesn't optimize away the load. */
> 	barrier();
>-	return ring->status_page.page_addr[reg];
>+	return engine->status_page.page_addr[reg];
> }
> 
> static inline void
>-intel_write_status_page(struct intel_engine_cs *ring,
>+intel_write_status_page(struct intel_engine_cs *engine,
> 			int reg, u32 value)
> {
>-	ring->status_page.page_addr[reg] = value;
>+	engine->status_page.page_addr[reg] = value;
> }
> 
> /**
>@@ -355,57 +372,77 @@ intel_write_status_page(struct intel_engine_cs *ring,
> #define I915_GEM_HWS_SCRATCH_INDEX	0x30
> #define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
> 
>-void intel_stop_ring_buffer(struct intel_engine_cs *ring);
>-void intel_cleanup_ring_buffer(struct intel_engine_cs *ring);
>+struct intel_ringbuffer *
>+intel_engine_alloc_ring(struct intel_engine_cs *engine,
>+			struct intel_context *ctx,
>+			int size);
>+void intel_ring_free(struct intel_ringbuffer *ring);
> 
>-int __must_check intel_ring_begin(struct intel_engine_cs *ring, int n);
>-int __must_check intel_ring_cacheline_align(struct intel_engine_cs *ring);
>-static inline void intel_ring_emit(struct intel_engine_cs *ring,
>+struct intel_ringbuffer *__must_check
>+intel_ring_begin(struct i915_gem_request *rq, int n);
>+int __must_check intel_ring_cacheline_align(struct i915_gem_request *rq);
>+static inline void intel_ring_emit(struct intel_ringbuffer *ring,
> 				   u32 data)
> {
>-	struct intel_ringbuffer *ringbuf = ring->buffer;
>-	iowrite32(data, ringbuf->virtual_start + ringbuf->tail);
>-	ringbuf->tail += 4;
>+	iowrite32(data, ring->virtual_start + ring->tail);
>+	ring->tail += 4;
> }
>-static inline void intel_ring_advance(struct intel_engine_cs *ring)
>+static inline void intel_ring_advance(struct intel_ringbuffer *ring)
> {
>-	struct intel_ringbuffer *ringbuf = ring->buffer;
>-	ringbuf->tail &= ringbuf->size - 1;
>+	ring->tail &= ring->size - 1;
> }
>-void __intel_ring_advance(struct intel_engine_cs *ring);
> 
>-int __must_check intel_ring_idle(struct intel_engine_cs *ring);
>-void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno);
>-int intel_ring_flush_all_caches(struct intel_engine_cs *ring);
>-int intel_ring_invalidate_all_caches(struct intel_engine_cs *ring);
>-
>-int intel_init_render_ring_buffer(struct drm_device *dev);
>-int intel_init_bsd_ring_buffer(struct drm_device *dev);
>-int intel_init_bsd2_ring_buffer(struct drm_device *dev);
>-int intel_init_blt_ring_buffer(struct drm_device *dev);
>-int intel_init_vebox_ring_buffer(struct drm_device *dev);
>-
>-u64 intel_ring_get_active_head(struct intel_engine_cs *ring);
>-void intel_ring_setup_status_page(struct intel_engine_cs *ring);
>-
>-static inline u32 intel_ring_get_tail(struct intel_ringbuffer *ringbuf)
>+static inline int __intel_ring_space(int head, int tail, int size, int rsvd)
> {
>-	return ringbuf->tail;
>+	int space = head - (tail + 8);
>+	if (space < 0)
>+		space += size;
>+	return space - rsvd;
> }
> 
>-static inline u32 intel_ring_get_seqno(struct intel_engine_cs *ring)
>+static inline int intel_ring_space(struct intel_ringbuffer *ring)
> {
>-	BUG_ON(ring->outstanding_lazy_seqno == 0);
>-	return ring->outstanding_lazy_seqno;
>+	return __intel_ring_space(ring->head, ring->tail,
>+				  ring->size, I915_RING_RSVD);
> }
> 
>-static inline void i915_trace_irq_get(struct intel_engine_cs *ring, u32 seqno)
>+
>+struct i915_gem_request * __must_check __attribute__((nonnull))
>+intel_engine_alloc_request(struct intel_engine_cs *engine,
>+			   struct intel_context *ctx);
>+
>+struct i915_gem_request *
>+intel_engine_find_active_batch(struct intel_engine_cs *engine);
>+
>+struct i915_gem_request *
>+intel_engine_seqno_to_request(struct intel_engine_cs *engine,
>+			      u32 seqno);
>+
>+int intel_init_render_engine(struct drm_i915_private *i915);
>+int intel_init_bsd_engine(struct drm_i915_private *i915);
>+int intel_init_bsd2_engine(struct drm_i915_private *i915);
>+int intel_init_blt_engine(struct drm_i915_private *i915);
>+int intel_init_vebox_engine(struct drm_i915_private *i915);
>+
>+int __must_check intel_engine_sync(struct intel_engine_cs *engine);
>+int __must_check intel_engine_flush(struct intel_engine_cs *engine,
>+				    struct intel_context *ctx);
>+
>+int intel_engine_retire(struct intel_engine_cs *engine, u32 seqno);
>+void intel_engine_reset(struct intel_engine_cs *engine);
>+int intel_engine_suspend(struct intel_engine_cs *engine);
>+int intel_engine_resume(struct intel_engine_cs *engine);
>+void intel_engine_cleanup(struct intel_engine_cs *engine);
>+
>+
>+u64 intel_engine_get_active_head(struct intel_engine_cs *engine);
>+
>+static inline void i915_trace_irq_get(struct intel_engine_cs *engine, u32 seqno)
> {
>-	if (ring->trace_irq_seqno == 0 && ring->irq_get(ring))
>-		ring->trace_irq_seqno = seqno;
>-}
>+	if (engine->trace_irq_seqno == 0)
>+		engine->irq_get(engine);
> 
>-/* DRI warts */
>-int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size);
>+	engine->trace_irq_seqno = seqno;
>+}
> 
> #endif /* _INTEL_RINGBUFFER_H_ */
>diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
>index 168c665..07059b4 100644
>--- a/drivers/gpu/drm/i915/intel_sprite.c
>+++ b/drivers/gpu/drm/i915/intel_sprite.c
>@@ -37,6 +37,12 @@
> #include <drm/i915_drm.h>
> #include "i915_drv.h"
> 
>+/* Backport! From include/drm/drmP.h in later kernels */
>+static inline wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
>+{
>+        return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
>+}
>+
> static int usecs_to_scanlines(const struct drm_display_mode *mode, int usecs)
> {
> 	/* paranoia */
>@@ -53,6 +59,7 @@ static bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl
> 	enum pipe pipe = crtc->pipe;
> 	long timeout = msecs_to_jiffies_timeout(1);
> 	int scanline, min, max, vblank_start;
>+	wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
> 	DEFINE_WAIT(wait);
> 
> 	WARN_ON(!drm_modeset_is_locked(&crtc->base.mutex));
>@@ -81,7 +88,7 @@ static bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl
> 		 * other CPUs can see the task state update by the time we
> 		 * read the scanline.
> 		 */
>-		prepare_to_wait(&crtc->vbl_wait, &wait, TASK_UNINTERRUPTIBLE);
>+		prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
> 
> 		scanline = intel_get_crtc_scanline(crtc);
> 		if (scanline < min || scanline > max)
>@@ -100,7 +107,7 @@ static bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl
> 		local_irq_disable();
> 	}
> 
>-	finish_wait(&crtc->vbl_wait, &wait);
>+	finish_wait(wq, &wait);
> 
> 	drm_vblank_put(dev, pipe);
> 
>@@ -163,6 +170,7 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc,
> 	sprctl &= ~SP_PIXFORMAT_MASK;
> 	sprctl &= ~SP_YUV_BYTE_ORDER_MASK;
> 	sprctl &= ~SP_TILED;
>+	sprctl &= ~SP_ROTATE_180;
> 
> 	switch (fb->pixel_format) {
> 	case DRM_FORMAT_YUYV:
>@@ -235,6 +243,14 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc,
> 							fb->pitches[0]);
> 	linear_offset -= sprsurf_offset;
> 
>+	if (intel_plane->rotation == BIT(DRM_ROTATE_180)) {
>+		sprctl |= SP_ROTATE_180;
>+
>+		x += src_w;
>+		y += src_h;
>+		linear_offset += src_h * fb->pitches[0] + src_w * pixel_size;
>+	}
>+
> 	atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count);
> 
> 	intel_update_primary_plane(intel_crtc);
>@@ -364,6 +380,7 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
> 	sprctl &= ~SPRITE_RGB_ORDER_RGBX;
> 	sprctl &= ~SPRITE_YUV_BYTE_ORDER_MASK;
> 	sprctl &= ~SPRITE_TILED;
>+	sprctl &= ~SPRITE_ROTATE_180;
> 
> 	switch (fb->pixel_format) {
> 	case DRM_FORMAT_XBGR8888:
>@@ -426,6 +443,18 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
> 					       pixel_size, fb->pitches[0]);
> 	linear_offset -= sprsurf_offset;
> 
>+	if (intel_plane->rotation == BIT(DRM_ROTATE_180)) {
>+		sprctl |= SPRITE_ROTATE_180;
>+
>+		/* HSW and BDW does this automagically in hardware */
>+		if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
>+			x += src_w;
>+			y += src_h;
>+			linear_offset += src_h * fb->pitches[0] +
>+				src_w * pixel_size;
>+		}
>+	}
>+
> 	atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count);
> 
> 	intel_update_primary_plane(intel_crtc);
>@@ -571,6 +600,7 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
> 	dvscntr &= ~DVS_RGB_ORDER_XBGR;
> 	dvscntr &= ~DVS_YUV_BYTE_ORDER_MASK;
> 	dvscntr &= ~DVS_TILED;
>+	dvscntr &= ~DVS_ROTATE_180;
> 
> 	switch (fb->pixel_format) {
> 	case DRM_FORMAT_XBGR8888:
>@@ -628,6 +658,14 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
> 					       pixel_size, fb->pitches[0]);
> 	linear_offset -= dvssurf_offset;
> 
>+	if (intel_plane->rotation == BIT(DRM_ROTATE_180)) {
>+		dvscntr |= DVS_ROTATE_180;
>+
>+		x += src_w;
>+		y += src_h;
>+		linear_offset += src_h * fb->pitches[0] + src_w * pixel_size;
>+	}
>+
> 	atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count);
> 
> 	intel_update_primary_plane(intel_crtc);
>@@ -813,57 +851,24 @@ static bool colorkey_enabled(struct intel_plane *intel_plane)
> }
> 
> static int
>-intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
>-		   struct drm_framebuffer *fb, int crtc_x, int crtc_y,
>-		   unsigned int crtc_w, unsigned int crtc_h,
>-		   uint32_t src_x, uint32_t src_y,
>-		   uint32_t src_w, uint32_t src_h)
>+intel_check_sprite_plane(struct drm_plane *plane,
>+			 struct intel_plane_state *state)
> {
>-	struct drm_device *dev = plane->dev;
>-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+	struct intel_crtc *intel_crtc = to_intel_crtc(state->crtc);
> 	struct intel_plane *intel_plane = to_intel_plane(plane);
>-	enum pipe pipe = intel_crtc->pipe;
>+	struct drm_framebuffer *fb = state->fb;
> 	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
> 	struct drm_i915_gem_object *obj = intel_fb->obj;
>-	struct drm_i915_gem_object *old_obj = intel_plane->obj;
>-	int ret;
>-	bool primary_enabled;
>-	bool visible;
>+	int crtc_x, crtc_y;
>+	unsigned int crtc_w, crtc_h;
>+	uint32_t src_x, src_y, src_w, src_h;
>+	struct drm_rect *src = &state->src;
>+	struct drm_rect *dst = &state->dst;
>+	struct drm_rect *orig_src = &state->orig_src;
>+	const struct drm_rect *clip = &state->clip;
> 	int hscale, vscale;
> 	int max_scale, min_scale;
> 	int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
>-	struct drm_rect src = {
>-		/* sample coordinates in 16.16 fixed point */
>-		.x1 = src_x,
>-		.x2 = src_x + src_w,
>-		.y1 = src_y,
>-		.y2 = src_y + src_h,
>-	};
>-	struct drm_rect dst = {
>-		/* integer pixels */
>-		.x1 = crtc_x,
>-		.x2 = crtc_x + crtc_w,
>-		.y1 = crtc_y,
>-		.y2 = crtc_y + crtc_h,
>-	};
>-	const struct drm_rect clip = {
>-		.x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0,
>-		.y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0,
>-	};
>-	const struct {
>-		int crtc_x, crtc_y;
>-		unsigned int crtc_w, crtc_h;
>-		uint32_t src_x, src_y, src_w, src_h;
>-	} orig = {
>-		.crtc_x = crtc_x,
>-		.crtc_y = crtc_y,
>-		.crtc_w = crtc_w,
>-		.crtc_h = crtc_h,
>-		.src_x = src_x,
>-		.src_y = src_y,
>-		.src_w = src_w,
>-		.src_h = src_h,
>-	};
> 
> 	/* Don't modify another pipe's plane */
> 	if (intel_plane->pipe != intel_crtc->pipe) {
>@@ -895,49 +900,55 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
> 	max_scale = intel_plane->max_downscale << 16;
> 	min_scale = intel_plane->can_scale ? 1 : (1 << 16);
> 
>-	hscale = drm_rect_calc_hscale_relaxed(&src, &dst, min_scale, max_scale);
>+	drm_rect_rotate(src, fb->width << 16, fb->height << 16,
>+			intel_plane->rotation);
>+
>+	hscale = drm_rect_calc_hscale_relaxed(src, dst, min_scale, max_scale);
> 	BUG_ON(hscale < 0);
> 
>-	vscale = drm_rect_calc_vscale_relaxed(&src, &dst, min_scale, max_scale);
>+	vscale = drm_rect_calc_vscale_relaxed(src, dst, min_scale, max_scale);
> 	BUG_ON(vscale < 0);
> 
>-	visible = drm_rect_clip_scaled(&src, &dst, &clip, hscale, vscale);
>+	state->visible =  drm_rect_clip_scaled(src, dst, clip, hscale, vscale);
> 
>-	crtc_x = dst.x1;
>-	crtc_y = dst.y1;
>-	crtc_w = drm_rect_width(&dst);
>-	crtc_h = drm_rect_height(&dst);
>+	crtc_x = dst->x1;
>+	crtc_y = dst->y1;
>+	crtc_w = drm_rect_width(dst);
>+	crtc_h = drm_rect_height(dst);
> 
>-	if (visible) {
>+	if (state->visible) {
> 		/* check again in case clipping clamped the results */
>-		hscale = drm_rect_calc_hscale(&src, &dst, min_scale, max_scale);
>+		hscale = drm_rect_calc_hscale(src, dst, min_scale, max_scale);
> 		if (hscale < 0) {
> 			DRM_DEBUG_KMS("Horizontal scaling factor out of limits\n");
>-			drm_rect_debug_print(&src, true);
>-			drm_rect_debug_print(&dst, false);
>+			drm_rect_debug_print(src, true);
>+			drm_rect_debug_print(dst, false);
> 
> 			return hscale;
> 		}
> 
>-		vscale = drm_rect_calc_vscale(&src, &dst, min_scale, max_scale);
>+		vscale = drm_rect_calc_vscale(src, dst, min_scale, max_scale);
> 		if (vscale < 0) {
> 			DRM_DEBUG_KMS("Vertical scaling factor out of limits\n");
>-			drm_rect_debug_print(&src, true);
>-			drm_rect_debug_print(&dst, false);
>+			drm_rect_debug_print(src, true);
>+			drm_rect_debug_print(dst, false);
> 
> 			return vscale;
> 		}
> 
> 		/* Make the source viewport size an exact multiple of the scaling factors. */
>-		drm_rect_adjust_size(&src,
>-				     drm_rect_width(&dst) * hscale - drm_rect_width(&src),
>-				     drm_rect_height(&dst) * vscale - drm_rect_height(&src));
>+		drm_rect_adjust_size(src,
>+				     drm_rect_width(dst) * hscale - drm_rect_width(src),
>+				     drm_rect_height(dst) * vscale - drm_rect_height(src));
>+
>+		drm_rect_rotate_inv(src, fb->width << 16, fb->height << 16,
>+				    intel_plane->rotation);
> 
> 		/* sanity check to make sure the src viewport wasn't enlarged */
>-		WARN_ON(src.x1 < (int) src_x ||
>-			src.y1 < (int) src_y ||
>-			src.x2 > (int) (src_x + src_w) ||
>-			src.y2 > (int) (src_y + src_h));
>+		WARN_ON(src->x1 < (int) orig_src->x1 ||
>+			src->y1 < (int) orig_src->y1 ||
>+			src->x2 > (int) orig_src->x2 ||
>+			src->y2 > (int) orig_src->y2);
> 
> 		/*
> 		 * Hardware doesn't handle subpixel coordinates.
>@@ -945,10 +956,10 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
> 		 * increase the source viewport size, because that could
> 		 * push the downscaling factor out of bounds.
> 		 */
>-		src_x = src.x1 >> 16;
>-		src_w = drm_rect_width(&src) >> 16;
>-		src_y = src.y1 >> 16;
>-		src_h = drm_rect_height(&src) >> 16;
>+		src_x = src->x1 >> 16;
>+		src_w = drm_rect_width(src) >> 16;
>+		src_y = src->y1 >> 16;
>+		src_h = drm_rect_height(src) >> 16;
> 
> 		if (format_is_yuv(fb->pixel_format)) {
> 			src_x &= ~1;
>@@ -962,12 +973,12 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
> 				crtc_w &= ~1;
> 
> 			if (crtc_w == 0)
>-				visible = false;
>+				state->visible = false;
> 		}
> 	}
> 
> 	/* Check size restrictions when scaling */
>-	if (visible && (src_w != crtc_w || src_h != crtc_h)) {
>+	if (state->visible && (src_w != crtc_w || src_h != crtc_h)) {
> 		unsigned int width_bytes;
> 
> 		WARN_ON(!intel_plane->can_scale);
>@@ -975,12 +986,13 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
> 		/* FIXME interlacing min height is 6 */
> 
> 		if (crtc_w < 3 || crtc_h < 3)
>-			visible = false;
>+			state->visible = false;
> 
> 		if (src_w < 3 || src_h < 3)
>-			visible = false;
>+			state->visible = false;
> 
>-		width_bytes = ((src_x * pixel_size) & 63) + src_w * pixel_size;
>+		width_bytes = ((src_x * pixel_size) & 63) +
>+					src_w * pixel_size;
> 
> 		if (src_w > 2048 || src_h > 2048 ||
> 		    width_bytes > 4096 || fb->pitches[0] > 4096) {
>@@ -989,42 +1001,76 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
> 		}
> 	}
> 
>-	dst.x1 = crtc_x;
>-	dst.x2 = crtc_x + crtc_w;
>-	dst.y1 = crtc_y;
>-	dst.y2 = crtc_y + crtc_h;
>+	if (state->visible) {
>+		src->x1 = src_x;
>+		src->x2 = src_x + src_w;
>+		src->y1 = src_y;
>+		src->y2 = src_y + src_h;
>+	}
>+
>+	dst->x1 = crtc_x;
>+	dst->x2 = crtc_x + crtc_w;
>+	dst->y1 = crtc_y;
>+	dst->y2 = crtc_y + crtc_h;
>+
>+	return 0;
>+}
>+
>+static int
>+intel_commit_sprite_plane(struct drm_plane *plane,
>+			  struct intel_plane_state *state)
>+{
>+	struct drm_device *dev = plane->dev;
>+	struct drm_crtc *crtc = state->crtc;
>+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+	struct intel_plane *intel_plane = to_intel_plane(plane);
>+	enum pipe pipe = intel_crtc->pipe;
>+	struct drm_framebuffer *fb = state->fb;
>+	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
>+	struct drm_i915_gem_object *obj = intel_fb->obj;
>+	struct drm_i915_gem_object *old_obj = intel_plane->obj;
>+	int crtc_x, crtc_y;
>+	unsigned int crtc_w, crtc_h;
>+	uint32_t src_x, src_y, src_w, src_h;
>+	struct drm_rect *dst = &state->dst;
>+	const struct drm_rect *clip = &state->clip;
>+	bool primary_enabled;
>+	int ret;
> 
> 	/*
> 	 * If the sprite is completely covering the primary plane,
> 	 * we can disable the primary and save power.
> 	 */
>-	primary_enabled = !drm_rect_equals(&dst, &clip) || colorkey_enabled(intel_plane);
>-	WARN_ON(!primary_enabled && !visible && intel_crtc->active);
>-
>-	mutex_lock(&dev->struct_mutex);
>+	primary_enabled = !drm_rect_equals(dst, clip) || colorkey_enabled(intel_plane);
>+	WARN_ON(!primary_enabled && !state->visible && intel_crtc->active);
> 
>-	/* Note that this will apply the VT-d workaround for scanouts,
>-	 * which is more restrictive than required for sprites. (The
>-	 * primary plane requires 256KiB alignment with 64 PTE padding,
>-	 * the sprite planes only require 128KiB alignment and 32 PTE padding.
>-	 */
>-	ret = intel_pin_and_fence_fb_obj(dev, obj, NULL);
> 
>-	i915_gem_track_fb(old_obj, obj,
>-			  INTEL_FRONTBUFFER_SPRITE(pipe));
>-	mutex_unlock(&dev->struct_mutex);
>+	if (old_obj != obj) {
>+		mutex_lock(&dev->struct_mutex);
> 
>-	if (ret)
>-		return ret;
>+		/* Note that this will apply the VT-d workaround for scanouts,
>+		 * which is more restrictive than required for sprites. (The
>+		 * primary plane requires 256KiB alignment with 64 PTE padding,
>+		 * the sprite planes only require 128KiB alignment and 32 PTE
>+		 * padding.
>+		 */
>+		ret = intel_pin_and_fence_fb_obj(dev, obj, NULL);
>+		if (ret == 0)
>+			i915_gem_track_fb(old_obj, obj,
>+					  INTEL_FRONTBUFFER_SPRITE(pipe));
>+		mutex_unlock(&dev->struct_mutex);
>+		if (ret)
>+			return ret;
>+	}
> 
>-	intel_plane->crtc_x = orig.crtc_x;
>-	intel_plane->crtc_y = orig.crtc_y;
>-	intel_plane->crtc_w = orig.crtc_w;
>-	intel_plane->crtc_h = orig.crtc_h;
>-	intel_plane->src_x = orig.src_x;
>-	intel_plane->src_y = orig.src_y;
>-	intel_plane->src_w = orig.src_w;
>-	intel_plane->src_h = orig.src_h;
>+	intel_plane->crtc_x = state->orig_dst.x1;
>+	intel_plane->crtc_y = state->orig_dst.y1;
>+	intel_plane->crtc_w = drm_rect_width(&state->orig_dst);
>+	intel_plane->crtc_h = drm_rect_height(&state->orig_dst);
>+	intel_plane->src_x = state->orig_src.x1;
>+	intel_plane->src_y = state->orig_src.y1;
>+	intel_plane->src_w = drm_rect_width(&state->orig_src);
>+	intel_plane->src_h = drm_rect_height(&state->orig_src);
> 	intel_plane->obj = obj;
> 
> 	if (intel_crtc->active) {
>@@ -1038,12 +1084,22 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
> 		if (primary_was_enabled && !primary_enabled)
> 			intel_pre_disable_primary(crtc);
> 
>-		if (visible)
>+		if (state->visible) {
>+			crtc_x = state->dst.x1;
>+			crtc_y = state->dst.y1;
>+			crtc_w = drm_rect_width(&state->dst);
>+			crtc_h = drm_rect_height(&state->dst);
>+			src_x = state->src.x1;
>+			src_y = state->src.y1;
>+			src_w = drm_rect_width(&state->src);
>+			src_h = drm_rect_height(&state->src);
> 			intel_plane->update_plane(plane, crtc, fb, obj,
> 						  crtc_x, crtc_y, crtc_w, crtc_h,
> 						  src_x, src_y, src_w, src_h);
>-		else
>+		} else {
> 			intel_plane->disable_plane(plane, crtc);
>+		}
>+
> 
> 		intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_SPRITE(pipe));
> 
>@@ -1052,14 +1108,15 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
> 	}
> 
> 	/* Unpin old obj after new one is active to avoid ugliness */
>-	if (old_obj) {
>+	if (old_obj && old_obj != obj) {
>+
> 		/*
> 		 * It's fairly common to simply update the position of
> 		 * an existing object.  In that case, we don't need to
> 		 * wait for vblank to avoid ugliness, we only need to
> 		 * do the pin & ref bookkeeping.
> 		 */
>-		if (old_obj != obj && intel_crtc->active)
>+		if (intel_crtc->active)
> 			intel_wait_for_vblank(dev, intel_crtc->pipe);
> 
> 		mutex_lock(&dev->struct_mutex);
>@@ -1071,6 +1128,46 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
> }
> 
> static int
>+intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
>+		   struct drm_framebuffer *fb, int crtc_x, int crtc_y,
>+		   unsigned int crtc_w, unsigned int crtc_h,
>+		   uint32_t src_x, uint32_t src_y,
>+		   uint32_t src_w, uint32_t src_h)
>+{
>+	struct intel_plane_state state;
>+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
>+	int ret;
>+
>+	state.crtc = crtc;
>+	state.fb = fb;
>+
>+	/* sample coordinates in 16.16 fixed point */
>+	state.src.x1 = src_x;
>+	state.src.x2 = src_x + src_w;
>+	state.src.y1 = src_y;
>+	state.src.y2 = src_y + src_h;
>+
>+	/* integer pixels */
>+	state.dst.x1 = crtc_x;
>+	state.dst.x2 = crtc_x + crtc_w;
>+	state.dst.y1 = crtc_y;
>+	state.dst.y2 = crtc_y + crtc_h;
>+
>+	state.clip.x1 = 0;
>+	state.clip.y1 = 0;
>+	state.clip.x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0;
>+	state.clip.y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0;
>+	state.orig_src = state.src;
>+	state.orig_dst = state.dst;
>+
>+	ret = intel_check_sprite_plane(plane, &state);
>+	if (ret)
>+		return ret;
>+
>+	return intel_commit_sprite_plane(plane, &state);
>+}
>+
>+static int
> intel_disable_plane(struct drm_plane *plane)
> {
> 	struct drm_device *dev = plane->dev;
>@@ -1180,18 +1277,27 @@ out_unlock:
> 	return ret;
> }
> 
>-void intel_plane_restore(struct drm_plane *plane)
>+int intel_plane_set_property(struct drm_plane *plane,
>+			     struct drm_property *prop,
>+			     uint64_t val)
>+{
>+	int ret = -ENOENT;
>+
>+	return ret;
>+}
>+
>+int intel_plane_restore(struct drm_plane *plane)
> {
> 	struct intel_plane *intel_plane = to_intel_plane(plane);
> 
> 	if (!plane->crtc || !plane->fb)
>-		return;
>+		return 0;
> 
>-	intel_update_plane(plane, plane->crtc, plane->fb,
>-			   intel_plane->crtc_x, intel_plane->crtc_y,
>-			   intel_plane->crtc_w, intel_plane->crtc_h,
>-			   intel_plane->src_x, intel_plane->src_y,
>-			   intel_plane->src_w, intel_plane->src_h);
>+	return plane->funcs->update_plane(plane, plane->crtc, plane->fb,
>+				  intel_plane->crtc_x, intel_plane->crtc_y,
>+				  intel_plane->crtc_w, intel_plane->crtc_h,
>+				  intel_plane->src_x, intel_plane->src_y,
>+				  intel_plane->src_w, intel_plane->src_h);
> }
> 
> void intel_plane_disable(struct drm_plane *plane)
>@@ -1206,6 +1312,7 @@ static const struct drm_plane_funcs intel_plane_funcs = {
> 	.update_plane = intel_update_plane,
> 	.disable_plane = intel_disable_plane,
> 	.destroy = intel_destroy_plane,
>+	.set_property = intel_plane_set_property,
> };
> 
> static uint32_t ilk_plane_formats[] = {
>@@ -1310,13 +1417,17 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
> 
> 	intel_plane->pipe = pipe;
> 	intel_plane->plane = plane;
>+	intel_plane->rotation = BIT(DRM_ROTATE_0);
> 	possible_crtcs = (1 << pipe);
>-	ret = drm_plane_init(dev, &intel_plane->base, possible_crtcs,
>-			     &intel_plane_funcs,
>-			     plane_formats, num_plane_formats,
>-			     false);
>-	if (ret)
>+	ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs,
>+				       &intel_plane_funcs,
>+				       plane_formats, num_plane_formats,
>+				       DRM_PLANE_TYPE_OVERLAY);
>+	if (ret) {
> 		kfree(intel_plane);
>+		goto out;
>+	}
> 
>+ out:
> 	return ret;
> }
>diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
>index c14341c..6f5f59b 100644
>--- a/drivers/gpu/drm/i915/intel_tv.c
>+++ b/drivers/gpu/drm/i915/intel_tv.c
>@@ -1182,18 +1182,17 @@ intel_tv_detect_type(struct intel_tv *intel_tv,
> 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
> 	struct drm_device *dev = encoder->dev;
> 	struct drm_i915_private *dev_priv = dev->dev_private;
>-	unsigned long irqflags;
> 	u32 tv_ctl, save_tv_ctl;
> 	u32 tv_dac, save_tv_dac;
> 	int type;
> 
> 	/* Disable TV interrupts around load detect or we'll recurse */
> 	if (connector->polled & DRM_CONNECTOR_POLL_HPD) {
>-		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+		spin_lock_irq(&dev_priv->irq_lock);
> 		i915_disable_pipestat(dev_priv, 0,
> 				      PIPE_HOTPLUG_INTERRUPT_STATUS |
> 				      PIPE_HOTPLUG_TV_INTERRUPT_STATUS);
>-		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+		spin_unlock_irq(&dev_priv->irq_lock);
> 	}
> 
> 	save_tv_dac = tv_dac = I915_READ(TV_DAC);
>@@ -1266,11 +1265,11 @@ intel_tv_detect_type(struct intel_tv *intel_tv,
> 
> 	/* Restore interrupt config */
> 	if (connector->polled & DRM_CONNECTOR_POLL_HPD) {
>-		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
>+		spin_lock_irq(&dev_priv->irq_lock);
> 		i915_enable_pipestat(dev_priv, 0,
> 				     PIPE_HOTPLUG_INTERRUPT_STATUS |
> 				     PIPE_HOTPLUG_TV_INTERRUPT_STATUS);
>-		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
>+		spin_unlock_irq(&dev_priv->irq_lock);
> 	}
> 
> 	return type;
>diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
>index e81bc3b..0e99852 100644
>--- a/drivers/gpu/drm/i915/intel_uncore.c
>+++ b/drivers/gpu/drm/i915/intel_uncore.c
>@@ -101,7 +101,7 @@ static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
> {
> 	u32 forcewake_ack;
> 
>-	if (IS_HASWELL(dev_priv->dev) || IS_GEN8(dev_priv->dev))
>+	if (IS_HASWELL(dev_priv->dev) || IS_BROADWELL(dev_priv->dev))
> 		forcewake_ack = FORCEWAKE_ACK_HSW;
> 	else
> 		forcewake_ack = FORCEWAKE_MT_ACK;
>@@ -194,13 +194,15 @@ static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
> static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
> 						int fw_engine)
> {
>+	/*
>+	 * WaRsDontPollForAckOnClearingFWBits:vlv
>+	 * Hardware clears ack bits lazily (only when all ack
>+	 * bits become 0) so don't poll for individiual ack
>+	 * bits to be clear here like on other platforms.
>+	 */
>+
> 	/* Check for Render Engine */
> 	if (FORCEWAKE_RENDER & fw_engine) {
>-		if (wait_for_atomic((__raw_i915_read32(dev_priv,
>-						FORCEWAKE_ACK_VLV) &
>-						FORCEWAKE_KERNEL) == 0,
>-					FORCEWAKE_ACK_TIMEOUT_MS))
>-			DRM_ERROR("Timed out: Render forcewake old ack to clear.\n");
> 
> 		__raw_i915_write32(dev_priv, FORCEWAKE_VLV,
> 				   _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
>@@ -214,11 +216,6 @@ static void __vlv_force_wake_get(struct drm_i915_private *dev_priv,
> 
> 	/* Check for Media Engine */
> 	if (FORCEWAKE_MEDIA & fw_engine) {
>-		if (wait_for_atomic((__raw_i915_read32(dev_priv,
>-						FORCEWAKE_ACK_MEDIA_VLV) &
>-						FORCEWAKE_KERNEL) == 0,
>-					FORCEWAKE_ACK_TIMEOUT_MS))
>-			DRM_ERROR("Timed out: Media forcewake old ack to clear.\n");
> 
> 		__raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV,
> 				   _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
>@@ -334,7 +331,7 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
> 	else if (IS_GEN6(dev) || IS_GEN7(dev))
> 		__gen6_gt_force_wake_reset(dev_priv);
> 
>-	if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_GEN8(dev))
>+	if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev))
> 		__gen7_gt_force_wake_mt_reset(dev_priv);
> 
> 	if (restore) { /* If reset with a user forcewake, try to restore */
>@@ -838,7 +835,7 @@ void intel_uncore_init(struct drm_device *dev)
> 	if (IS_VALLEYVIEW(dev)) {
> 		dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;
> 		dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;
>-	} else if (IS_HASWELL(dev) || IS_GEN8(dev)) {
>+	} else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
> 		dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get;
> 		dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put;
> 	} else if (IS_IVYBRIDGE(dev)) {
>-- 
>1.9.3
>

Actions: View | Splinter Review

Attachments on bug 83677: 105992 | 106501 | 106508 | 106519 | 106669 | 107665 | 108029 | 108310 | 108459 | 108976 | 109009 | 109018 | 109019 | 109165 | 109331 | 109459 | 110698 | 110699

Use of freedesktop.org services, including Bugzilla, is subject to our Code of Conduct. How we collect and use information is described in our Privacy Policy.