Attachment 3170 Details for Bug 4248 – Fix crash on cmap alloc failure and fix failure with 32 bits visual

[patch] Fix crash on cmap alloc failure and fix failure with 32 bits visual

cmap.diff (text/plain), 2.80 KB, created by Benjamin Herrenschmidt on 2005-09-04 21:21:34 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Benjamin Herrenschmidt

Created: 2005-09-04 21:21:34 UTC

Size: 2.80 KB

patch

obsolete

>diff -urN xc-HEAD.orig/programs/Xserver/hw/xfree86/common/xf86cmap.c xc-HEAD/programs/Xserver/hw/xfree86/common/xf86cmap.c
>--- xc-HEAD.orig/programs/Xserver/hw/xfree86/common/xf86cmap.c	2005-08-17 11:55:35.000000000 +1000
>+++ xc-HEAD/programs/Xserver/hw/xfree86/common/xf86cmap.c	2005-09-05 14:14:52.000000000 +1000
>@@ -255,10 +255,10 @@
>     int numColors;
>     LOCO *colors;
> 
>-    if((1 << pmap->pVisual->nplanes) > pScreenPriv->maxColors)
>+    if((1ull << pmap->pVisual->nplanes) > pScreenPriv->maxColors)
> 	numColors = pmap->pVisual->ColormapEntries;
>     else 
>-	numColors = 1 << pmap->pVisual->nplanes; 
>+	numColors = 1ull << pmap->pVisual->nplanes; 
> 
>     if(!(colors = xalloc(numColors * sizeof(LOCO))))
> 	return FALSE;
>@@ -295,6 +295,7 @@
>     Bool ret = FALSE;
> 
>     pScreen->CreateColormap = pScreenPriv->CreateColormap;
>+    pmap->devPrivates[CMapColormapIndex].ptr = NULL;
>     if((*pScreen->CreateColormap)(pmap)) { 
> 	if(CMapAllocateColormapPrivate(pmap)) 
> 	   ret = TRUE;
>@@ -371,7 +372,7 @@
> 	   (CMapColormapPtr) pmap->devPrivates[CMapColormapIndex].ptr;
> 	int i;
> 
>-        if((1 << pVisual->nplanes) > pScreenPriv->maxColors) {
>+        if((1ull << pVisual->nplanes) > pScreenPriv->maxColors) {
> 	    int index;
> 
> 	    num = 0;
>@@ -442,7 +443,7 @@
> 
>     if(!(pScreenPriv->flags & CMAP_PALETTED_TRUECOLOR) &&
> 	(pmap->pVisual->class == TrueColor) &&
>-	((1 << pmap->pVisual->nplanes) > pScreenPriv->maxColors))
>+	((1ull << pmap->pVisual->nplanes) > pScreenPriv->maxColors))
> 		return;
> 
>     if(LOAD_PALETTE(pmap, index))
>@@ -572,7 +573,7 @@
> 	}
> 	break;
>     case TrueColor:
>-	if((1 << pVisual->nplanes) > pScreenPriv->maxColors) {
>+	if((1ull << pVisual->nplanes) > pScreenPriv->maxColors) {
> 	    for(i = 0; i <= reds; i++) 
> 		colors[i].red   = gamma[i * maxValue / reds].red;
> 	    for(i = 0; i <= greens; i++) 
>@@ -615,7 +616,7 @@
> 	}
> 	break;
>     case DirectColor:
>-	if((1 << pVisual->nplanes) > pScreenPriv->maxColors) {
>+	if((1ull << pVisual->nplanes) > pScreenPriv->maxColors) {
> 	    for(i = 0; i < defs; i++) { 
> 		index = indices[i];
> 		if(index <= reds)
>@@ -932,7 +933,7 @@
> 
> 	if(!(pScreenPriv->flags & CMAP_PALETTED_TRUECOLOR) &&
> 	    (pMap->pVisual->class == TrueColor) &&
>-	    ((1 << pMap->pVisual->nplanes) > pScreenPriv->maxColors)) {
>+	    ((1ull << pMap->pVisual->nplanes) > pScreenPriv->maxColors)) {
> 
> 	    /* if the current map doesn't have a palette look
> 		for another map to change the gamma on. */
>@@ -1019,7 +1020,7 @@
> 
>         if(!(pScreenPriv->flags & CMAP_PALETTED_TRUECOLOR) &&
>             (pMap->pVisual->class == TrueColor) &&
>-            ((1 << pMap->pVisual->nplanes) > pScreenPriv->maxColors)) {
>+            ((1ull << pMap->pVisual->nplanes) > pScreenPriv->maxColors)) {
> 
>             /* if the current map doesn't have a palette look
>                 for another map to change the gamma on. */

Actions: View | Splinter Review

Attachments on bug 4248: 3170 | 3171 | 3172

Use of freedesktop.org services, including Bugzilla, is subject to our Code of Conduct. How we collect and use information is described in our Privacy Policy.