An 32 bit X11 application running on 64 bit RHEL 7 and use libX11-1.6.3-2.el7.i686 rpm X11 library. This application is facing problem while creating its own color map. 

This X11 application is getting BadValue error while calling XAllocColorCells function when I run the application from VNC. But same code is working properly on Citrix client. Application tries to create a new color map by calling XCreateColormap.

From the gdb debugging I can see two major difference in Xlibrary call behavior between VNC and Citrix clients.

1> XDisplayCells return values are different . on VNC XDisplayCells returns 64 but on Citrix client it return 256.

2> On VNC the XCreateColormap call is returning a bigger color map ID (compared with Citrix client) which maybe out of range and so bad value error occur.
For VNC XCreateColormap return 37748737 but for Citrix it return 8388609.


Below is the high level code of the color map creation…




void createColormap(Display *display)
{
int i, n, snr, coffset, ncolors;
Display *dnr;
XColor dcolors[256];
Window win;
Colormap colormap0, colormap1;
Visual *vis;

win = DefaultRootWindow(display);
ncolors = XDisplayCells(display, DefaultScreen(display)); // Return 64 on VNC , and 256 for Citrix.
dnr = display;
snr = XDefaultScreen(display);
colormap0 = DefaultColormap(dnr,snr);
colormap = colormap0;

if(ncolors > 256) {
return;
}
if(ncolors > 2) {

vis = XDefaultVisual(dnr, XDefaultScreen(dnr));
if(vis == NULL) {
return;
}

//----------------------------------------- portion between this dotted lines do not execute on VNC as ncolors =64 and thus coffset =0 -------------------------------------------------------- //
if(ncolors > 16) {
if(ncolors > 128) {
coffset = 32;
} else {
coffset = 0;
}
} else {
coffset = 0;
}
if(coffset > ncolors) {
coffset = 0;
}
n = ncolors - coffset;

for(i=0; i dcolors[i].pixel = i;
XQueryColor(dnr, colormap0, &(dcolors[i]));
dcolors[i].flags = DoRed | DoGreen | DoBlue;
}
//----------------------------------------- portion between this dotted lines do not execute on VNC as ncolors =64 and thus coffset =0 -------------------------------------------------------- //


colormap1 = XCreateColormap(dnr, win, vis, AllocNone); // This line execute on both. Returns 37748737 for VNC but 8388609 Citrix.
if(colormap1 == colormap) {
return;
}

//============================ portion between this dotted lines do not execute on VNC as ncolors < 256 and coffset < 128 ========================== //
if(ncolors >= 256 && coffset < 128) {
int r, g, b;
for(r=0; r < 4; r++) {
for(g=0; g < 4; g++) {
for(b=0; b < 4; b++) {
dcolors[coffset].pixel = coffset;
dcolors[coffset].flags = DoRed | DoGreen | DoBlue;
dcolors[coffset].red = (short) (r * 21845);
dcolors[coffset].green = (short) (g * 21845);
dcolors[coffset].blue = (short) (b * 21845);
coffset++;
if(coffset >= 256) break;
}
if(coffset >= 256) break;
}
if(coffset >= 256) break;
}
}
//============================ portion between this dotted lines do not execute on VNC as ncolors < 256 and coffset < 128 ========================== //

unsigned long plane_masks_return[1];
unsigned int nplanes= 0;
unsigned long pixels_return[256];
unsigned int npixels = coffset;
int status;
status = XAllocColorCells(dnr, colormap1, True, plane_masks_return, nplanes, pixels_return, npixels); //this call throw BadValue error.
if(status == 0) {
return;
} else {
XStoreColors(dnr, colormap1, &(dcolors[0]), coffset);
colormap = colormap1;
}
}
}