diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c
index 7780567aa669..1634352285dc 100644
--- a/drivers/gpu/drm/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/drm_dp_mst_topology.c
@@ -1713,7 +1713,7 @@ static struct drm_dp_mst_branch *drm_dp_get_last_connected_port_and_mstb(struct
 static int drm_dp_payload_send_msg(struct drm_dp_mst_topology_mgr *mgr,
 				   struct drm_dp_mst_port *port,
 				   int id,
-				   int pbn)
+				   int pbn, bool destroy)
 {
 	struct drm_dp_sideband_msg_tx *txmsg;
 	struct drm_dp_mst_branch *mstb;
@@ -1721,9 +1721,11 @@ static int drm_dp_payload_send_msg(struct drm_dp_mst_topology_mgr *mgr,
 	u8 sinks[DRM_DP_MAX_SDP_STREAMS];
 	int i;
 
-	port = drm_dp_get_validated_port_ref(mgr, port);
-	if (!port)
-		return -EINVAL;
+	if (!destroy) {
+		port = drm_dp_get_validated_port_ref(mgr, port);
+		if (!port)
+			return -EINVAL;
+	}
 
 	port_num = port->port_num;
 	mstb = drm_dp_get_validated_mstb_ref(mgr, port->parent);
@@ -1762,7 +1764,8 @@ static int drm_dp_payload_send_msg(struct drm_dp_mst_topology_mgr *mgr,
 	kfree(txmsg);
 fail_put:
 	drm_dp_put_mst_branch_device(mstb);
-	drm_dp_put_port(port);
+	if (!destroy)
+		drm_dp_put_port(port);
 	return ret;
 }

@@ -1821,7 +1824,7 @@ static int drm_dp_create_payload_step2(struct drm_dp_mst_topology_mgr *mgr,
 				       struct drm_dp_payload *payload)
 {
 	int ret;
-	ret = drm_dp_payload_send_msg(mgr, port, id, port->vcpi.pbn);
+	ret = drm_dp_payload_send_msg(mgr, port, id, port->vcpi.pbn, false);
 	if (ret < 0)
 		return ret;
 	payload->payload_state = DP_PAYLOAD_REMOTE;
@@ -1836,7 +1839,7 @@ static int drm_dp_destroy_payload_step1(struct drm_dp_mst_topology_mgr *mgr,
 	DRM_DEBUG_KMS("\n");
 	/* its okay for these to fail */
 	if (port) {
-		drm_dp_payload_send_msg(mgr, port, id, 0);
+		drm_dp_payload_send_msg(mgr, port, id, 0, true);
 	}
 
 	drm_dp_dpcd_write_payload(mgr, id, payload);


@@ -1877,8 +1880,8 @@ int drm_dp_update_payload_part1(struct drm_dp_mst_topology_mgr *mgr)
 		/* solve the current payloads - compare to the hw ones
 		   - update the hw view */
 		req_payload.start_slot = cur_slots;
-		if (mgr->proposed_vcpis[i]) {
-			port = container_of(mgr->proposed_vcpis[i], struct drm_dp_mst_port, vcpi);
+		port = container_of(mgr->proposed_vcpis[i], struct drm_dp_mst_port, vcpi);
+		if (mgr->proposed_vcpis[i] && mgr->proposed_vcpis[i]->num_slots) {
 			port = drm_dp_get_validated_port_ref(mgr, port);
 			if (!port) {
 				mutex_unlock(&mgr->payload_lock);
@@ -1887,7 +1890,6 @@ int drm_dp_update_payload_part1(struct drm_dp_mst_topology_mgr *mgr)
 			req_payload.num_slots = mgr->proposed_vcpis[i]->num_slots;
 			req_payload.vcpi = mgr->proposed_vcpis[i]->vcpi;
 		} else {
-			port = NULL;
 			req_payload.num_slots = 0;
 		}
 
@@ -1912,7 +1914,7 @@ int drm_dp_update_payload_part1(struct drm_dp_mst_topology_mgr *mgr)
 		}
 		cur_slots += req_payload.num_slots;
 
-		if (port)
+		if (mgr->proposed_vcpis[i] && mgr->proposed_vcpis[i]->num_slots)
 			drm_dp_put_port(port);
 	}