diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 3b0fe9f1f3c9..0b26ab99657a 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -102,6 +102,12 @@ static void chv_prepare_pll(struct intel_crtc *crtc,
 static void intel_begin_crtc_commit(struct drm_crtc *crtc);
 static void intel_finish_crtc_commit(struct drm_crtc *crtc);
 
+static void intel_fb_unreference(struct drm_framebuffer *fb)
+{
+	BUG_ON(mutex_is_locked(&fb->dev->struct_mutex));
+	drm_framebuffer_unreference(fb);
+}
+
 static struct intel_encoder *intel_find_encoder(struct intel_connector *connector, int pipe)
 {
 	if (!connector->mst_port)
@@ -2426,7 +2432,7 @@ update_state_fb(struct drm_plane *plane)
 		return;
 
 	if (plane->state->fb)
-		drm_framebuffer_unreference(plane->state->fb);
+		intel_fb_unreference(plane->state->fb);
 	plane->state->fb = plane->fb;
 	if (plane->state->fb)
 		drm_framebuffer_reference(plane->state->fb);
@@ -8790,7 +8796,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector,
 
 		if (old->release_fb) {
 			drm_framebuffer_unregister_private(old->release_fb);
-			drm_framebuffer_unreference(old->release_fb);
+			intel_fb_unreference(old->release_fb);
 		}
 
 		return;
@@ -9107,7 +9113,6 @@ static void intel_unpin_work_fn(struct work_struct *__work)
 	mutex_lock(&dev->struct_mutex);
 	intel_unpin_fb_obj(intel_fb_obj(work->old_fb));
 	drm_gem_object_unreference(&work->pending_flip_obj->base);
-	drm_framebuffer_unreference(work->old_fb);
 
 	intel_fbc_update(dev);
 
@@ -9116,6 +9121,7 @@ static void intel_unpin_work_fn(struct work_struct *__work)
 	mutex_unlock(&dev->struct_mutex);
 
 	intel_frontbuffer_flip_complete(dev, INTEL_FRONTBUFFER_PRIMARY(pipe));
+	intel_fb_unreference(work->old_fb);
 
 	BUG_ON(atomic_read(&to_intel_crtc(work->crtc)->unpin_work_count) == 0);
 	atomic_dec(&to_intel_crtc(work->crtc)->unpin_work_count);
@@ -9857,9 +9863,9 @@ cleanup_pending:
 	atomic_dec(&intel_crtc->unpin_work_count);
 	crtc->primary->fb = old_fb;
 	update_state_fb(crtc->primary);
-	drm_framebuffer_unreference(work->old_fb);
 	drm_gem_object_unreference(&obj->base);
 	mutex_unlock(&dev->struct_mutex);
+	intel_fb_unreference(work->old_fb);
 
 cleanup:
 	spin_lock_irq(&dev->event_lock);
@@ -13671,6 +13677,8 @@ void intel_modeset_gem_init(struct drm_device *dev)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_crtc *c;
 	struct drm_i915_gem_object *obj;
+	struct drm_framebuffer *unused[I915_NUM_PIPES];
+	int n_unused = 0;
 
 	mutex_lock(&dev->struct_mutex);
 	intel_init_gt_powersave(dev);
@@ -13706,13 +13714,16 @@ void intel_modeset_gem_init(struct drm_device *dev)
 					       NULL)) {
 			DRM_ERROR("failed to pin boot fb on pipe %d\n",
 				  to_intel_crtc(c)->pipe);
-			drm_framebuffer_unreference(c->primary->fb);
+			unused[n_unused++] = c->primary->fb;
 			c->primary->fb = NULL;
 			update_state_fb(c->primary);
 		}
 	}
 	mutex_unlock(&dev->struct_mutex);
 
+	while (n_unused--)
+		intel_fb_unreference(unused[n_unused]);
+
 	intel_backlight_register(dev);
 }