From 9ec84e083311d89f18636440f6a905ac1e946d4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michel=20D=C3=A4nzer?= <michel.daenzer@amd.com>
Date: Fri, 19 Oct 2018 11:02:41 +0200
Subject: [PATCH xf86-video-amdgpu] Detect and fix up non-premultiplied cursor
 data
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Michel Dänzer <michel.daenzer@amd.com>
---
 src/drmmode_display.c | 89 ++++++++++++++++++++++++++++---------------
 1 file changed, 58 insertions(+), 31 deletions(-)

diff --git a/src/drmmode_display.c b/src/drmmode_display.c
index 9065e0fa2..78625c72e 100644
--- a/src/drmmode_display.c
+++ b/src/drmmode_display.c
@@ -1500,49 +1500,63 @@ drmmode_cursor_src_offset(Rotation rotation, int width, int height,
 
 #endif
 
-static uint32_t
-drmmode_cursor_gamma_passthrough(xf86CrtcPtr crtc, uint32_t argb)
-{
-	uint32_t alpha = argb >> 24;
-
-	if (!alpha)
-		return 0;
-
-	return argb;
-}
-
-static uint32_t
-drmmode_cursor_gamma(xf86CrtcPtr crtc, uint32_t argb)
+static Bool
+drmmode_cursor_pixel(xf86CrtcPtr crtc, uint32_t *argb, Bool premultiplied,
+		     Bool apply_gamma)
 {
-	uint32_t alpha = argb >> 24;
+	uint32_t alpha = *argb >> 24;
 	uint32_t rgb[3];
 	int i;
 
-	if (!alpha)
-		return 0;
+	if (premultiplied) {
+		if ((alpha | alpha << 8 | alpha << 16 | alpha << 24) < *argb)
+			/* Doesn't look like premultiplied alpha */
+			return FALSE;
+
+		if (!apply_gamma)
+			return TRUE;
+	}
 
-	/* Un-premultiply alpha */
+	if (!alpha) {
+		*argb = 0;
+		return TRUE;
+	}
+
+	/* Extract RGB */
 	for (i = 0; i < 3; i++)
-		rgb[i] = ((argb >> (i * 8)) & 0xff) * 0xff / alpha;
+		rgb[i] = (*argb >> (i * 8)) & 0xff;
+
+	if (premultiplied) {
+		/* Un-premultiply alpha */
+		for (i = 0; i < 3; i++)
+			rgb[i] = rgb[i] * 0xff / alpha;
+	}
+
+	if (apply_gamma) {
+		rgb[0] = crtc->gamma_blue[rgb[0]] >> 8;
+		rgb[1] = crtc->gamma_green[rgb[1]] >> 8;
+		rgb[2] = crtc->gamma_red[rgb[2]] >> 8;
+	}
 
-	/* Apply gamma correction and pre-multiply alpha */
-	rgb[0] = (crtc->gamma_blue[rgb[0]] >> 8) * alpha / 0xff;
-	rgb[1] = (crtc->gamma_green[rgb[1]] >> 8) * alpha / 0xff;
-	rgb[2] = (crtc->gamma_red[rgb[2]] >> 8) * alpha / 0xff;
+	/* Premultiply alpha */
+	for (i = 0; i < 3; i++)
+		rgb[i] = rgb[i] * alpha / 0xff;
 
-	return alpha << 24 | rgb[2] << 16 | rgb[1] << 8 | rgb[0];
+	*argb = alpha << 24 | rgb[2] << 16 | rgb[1] << 8 | rgb[0];
+	return TRUE;
 }
 
 static void drmmode_do_load_cursor_argb(xf86CrtcPtr crtc, CARD32 *image, uint32_t *ptr)
 {
 	ScrnInfoPtr pScrn = crtc->scrn;
 	AMDGPUInfoPtr info = AMDGPUPTR(pScrn);
-	uint32_t (*cursor_gamma)(xf86CrtcPtr crtc, uint32_t argb) =
-		drmmode_cursor_gamma;
+	Bool premultiplied = TRUE;
+	Bool apply_gamma = TRUE;
+	uint32_t argb;
 
 	if ((crtc->scrn->depth != 24 && crtc->scrn->depth != 32) ||
 	    drmmode_cm_enabled(&info->drmmode))
-		cursor_gamma = drmmode_cursor_gamma_passthrough;
+		apply_gamma = FALSE;
 
 #if XF86_CRTC_VERSION < 7
 	if (crtc->driverIsPerformingTransform) {
@@ -1550,15 +1564,20 @@ static void drmmode_do_load_cursor_argb(xf86CrtcPtr crtc, CARD32 *image, uint32_
 		int dstx, dsty;
 		int srcoffset;
 
+retry_transform:
 		for (dsty = 0; dsty < cursor_h; dsty++) {
 			for (dstx = 0; dstx < cursor_w; dstx++) {
 				srcoffset = drmmode_cursor_src_offset(crtc->rotation,
 								      cursor_w,
 								      cursor_h,
 								      dstx, dsty);
-
-				ptr[dsty * info->cursor_w + dstx] =
-					cpu_to_le32(cursor_gamma(crtc, image[srcoffset]));
+				argb = image[srcoffset];
+				if (!drmmode_cursor_pixel(crtc, &argb, premultiplied,
+							  apply_gamma)) {
+					premultiplied = FALSE;
+					goto retry_transform;
+				}
+				ptr[dsty * info->cursor_w + dstx] = cpu_to_le32(argb);
 			}
 		}
 	} else
@@ -1567,8 +1586,16 @@ static void drmmode_do_load_cursor_argb(xf86CrtcPtr crtc, CARD32 *image, uint32_
 		uint32_t cursor_size = info->cursor_w * info->cursor_h;
 		int i;
 
-		for (i = 0; i < cursor_size; i++)
-			ptr[i] = cpu_to_le32(cursor_gamma(crtc, image[i]));
+retry:
+		for (i = 0; i < cursor_size; i++) {
+			argb = image[i];
+			if (!drmmode_cursor_pixel(crtc, &argb, premultiplied,
+						  apply_gamma)) {
+				premultiplied = FALSE;
+				goto retry;
+			}
+			ptr[i] = cpu_to_le32(argb);
+		}
 	}
 }
 
-- 
2.19.1