Index: atimach64.c
===================================================================
RCS file: /cvs/xorg/xc/programs/Xserver/hw/xfree86/drivers/ati/atimach64.c,v
retrieving revision 1.6
diff -u -3 -p -r1.6 atimach64.c
--- atimach64.c	11 Jul 2005 02:29:44 -0000	1.6
+++ atimach64.c	3 Apr 2006 22:44:14 -0000
@@ -181,6 +181,7 @@ ATIMach64PreInit
 
     if (pATI->Chip >= ATI_CHIP_264VTB)
     {
+        pATIHW->mem_buf_cntl = inr(MEM_BUF_CNTL) | INVALIDATE_RB_CACHE;
         pATIHW->mem_cntl = (pATI->LockData.mem_cntl &
             ~(CTL_MEM_LOWER_APER_ENDIAN | CTL_MEM_UPPER_APER_ENDIAN)) |
             SetBits(CTL_MEM_APER_BYTE_ENDIAN, CTL_MEM_LOWER_APER_ENDIAN);
@@ -422,6 +423,7 @@ ATIMach64Save
 
     if (pATI->Chip >= ATI_CHIP_264VTB)
     {
+        pATIHW->mem_buf_cntl = inr(MEM_BUF_CNTL) | INVALIDATE_RB_CACHE;
         pATIHW->mem_cntl = inr(MEM_CNTL);
         pATIHW->mpp_config = inr(MPP_CONFIG);
         pATIHW->mpp_strobe_seq = inr(MPP_STROBE_SEQ);
@@ -1055,6 +1057,7 @@ ATIMach64Set
 
         if (pATI->Chip >= ATI_CHIP_264VTB)
         {
+            outr(MEM_BUF_CNTL, pATIHW->mem_buf_cntl);
             outr(MEM_CNTL, pATIHW->mem_cntl);
             outr(MPP_CONFIG, pATIHW->mpp_config);
             outr(MPP_STROBE_SEQ, pATIHW->mpp_strobe_seq);
Index: atimach64accel.c
===================================================================
RCS file: /cvs/xorg/xc/programs/Xserver/hw/xfree86/drivers/ati/atimach64accel.c,v
retrieving revision 1.4
diff -u -3 -p -r1.4 atimach64accel.c
--- atimach64accel.c	11 Jul 2005 02:29:44 -0000	1.4
+++ atimach64accel.c	3 Apr 2006 22:44:14 -0000
@@ -386,7 +386,110 @@ ATIMach64Sync
     if (pATI->pXAAInfo)
       pATI->pXAAInfo->NeedToSync = FALSE;
 
-    pATI = *(volatile ATIPtr *)pATI->pMemory;
+     if (pATI->Chip >= ATI_CHIP_264VTB)
+      {
+          /*
+          * Flush the read-back cache (by turning on INVALIDATE_RB_CACHE),
+          * otherwise the host might get stale data when reading through the
+          * aperture.
+           */
+         outr(MEM_BUF_CNTL, pATI->NewHW.mem_buf_cntl);
+     }
+  
+     if (!pATI->OptionMMIOCache || !pATI->OptionTestMMIOCache)
+         return;
+  
+     /*
+      * For debugging purposes, attempt to verify that each cached register
+      * should actually be cached.
+      */
+     TestRegisterCaching(SRC_CNTL);
+  
+     TestRegisterCaching(HOST_CNTL);
+  
+     TestRegisterCaching(PAT_REG0);
+     TestRegisterCaching(PAT_REG1);
+     TestRegisterCaching(PAT_CNTL);
+  
+     if (RegisterIsCached(SC_LEFT_RIGHT) &&      /* Special case */
+         (CacheSlot(SC_LEFT_RIGHT) !=
+          (SetWord(inm(SC_RIGHT), 1) | SetWord(inm(SC_LEFT), 0))))
+     {
+         UncacheRegister(SC_LEFT_RIGHT);
+         xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
+             "SC_LEFT_RIGHT write cache disabled!\n");
+     }
+  
+     if (RegisterIsCached(SC_TOP_BOTTOM) &&      /* Special case */
+         (CacheSlot(SC_TOP_BOTTOM) !=
+          (SetWord(inm(SC_BOTTOM), 1) | SetWord(inm(SC_TOP), 0))))
+     {
+         UncacheRegister(SC_TOP_BOTTOM);
+         xf86DrvMsg(pScreenInfo->scrnIndex, X_WARNING,
+             "SC_TOP_BOTTOM write cache disabled!\n");
+     }
+  
+     TestRegisterCaching(DP_BKGD_CLR);
+     TestRegisterCaching(DP_FRGD_CLR);
+     TestRegisterCaching(DP_WRITE_MASK);
+     TestRegisterCaching(DP_MIX);
+  
+     TestRegisterCaching(CLR_CMP_CLR);
+     TestRegisterCaching(CLR_CMP_MSK);
+     TestRegisterCaching(CLR_CMP_CNTL);
+  
+     if (!pATI->Block1Base)
+         return;
+  
+     TestRegisterCaching(OVERLAY_Y_X_START);
+     TestRegisterCaching(OVERLAY_Y_X_END);
+  
+     TestRegisterCaching(OVERLAY_GRAPHICS_KEY_CLR);
+     TestRegisterCaching(OVERLAY_GRAPHICS_KEY_MSK);
+  
+     TestRegisterCaching(OVERLAY_KEY_CNTL);
+  
+     TestRegisterCaching(OVERLAY_SCALE_INC);
+     TestRegisterCaching(OVERLAY_SCALE_CNTL);
+ 
+     TestRegisterCaching(SCALER_HEIGHT_WIDTH);
+ 
+     TestRegisterCaching(SCALER_TEST);
+ 
+     TestRegisterCaching(VIDEO_FORMAT);
+ 
+     if (pATI->Chip < ATI_CHIP_264VTB)
+     {
+         TestRegisterCaching(BUF0_OFFSET);
+         TestRegisterCaching(BUF0_PITCH);
+         TestRegisterCaching(BUF1_OFFSET);
+         TestRegisterCaching(BUF1_PITCH);
+ 
+         return;
+      }
+  
+     TestRegisterCaching(SCALER_BUF0_OFFSET);
+     TestRegisterCaching(SCALER_BUF1_OFFSET);
+     TestRegisterCaching(SCALER_BUF_PITCH);
+ 
+     TestRegisterCaching(OVERLAY_EXCLUSIVE_HORZ);
+     TestRegisterCaching(OVERLAY_EXCLUSIVE_VERT);
+ 
+     if (pATI->Chip < ATI_CHIP_264GTPRO)
+         return;
+ 
+     TestRegisterCaching(SCALER_COLOUR_CNTL);
+ 
+     TestRegisterCaching(SCALER_H_COEFF0);
+     TestRegisterCaching(SCALER_H_COEFF1);
+     TestRegisterCaching(SCALER_H_COEFF2);
+     TestRegisterCaching(SCALER_H_COEFF3);
+     TestRegisterCaching(SCALER_H_COEFF4);
+ 
+     TestRegisterCaching(SCALER_BUF0_OFFSET_U);
+     TestRegisterCaching(SCALER_BUF0_OFFSET_V);
+     TestRegisterCaching(SCALER_BUF1_OFFSET_U);
+     TestRegisterCaching(SCALER_BUF1_OFFSET_V);
 }
 
 /*
@@ -497,6 +600,18 @@ ATIMach64SubsequentScreenToScreenCopy
     outf(SRC_WIDTH1, w);
     outf(DST_Y_X, SetWord(xDst, 1) | SetWord(yDst, 0));
     outf(DST_HEIGHT_WIDTH, SetWord(w, 1) | SetWord(h, 0));
+
+    /*
+     * On VTB's and later, the engine will randomly not wait for a copy
+     * operation to commit its results to video memory before starting the next
+     * one.  The probability of such occurrences increases with GUI_WB_FLUSH
+     * (or GUI_WB_FLUSH_P) setting, bitsPerPixel and/or CRTC clock.  This
+     * would point to some kind of video memory bandwidth problem were it noti
+     * for the fact that the problem occurs less often (but still occurs) when
+     * copying larger rectangles.
+     */
+    if ((pATI->Chip >= ATI_CHIP_264VTB) && !pATI->OptionDevel)
+        ATIMach64Sync(pScreenInfo);
 }
 
 /*
Index: atiregs.h
===================================================================
RCS file: /cvs/xorg/xc/programs/Xserver/hw/xfree86/drivers/ati/atiregs.h,v
retrieving revision 1.4
diff -u -3 -p -r1.4 atiregs.h
--- atiregs.h	28 Aug 2005 18:10:34 -0000	1.4
+++ atiregs.h	3 Apr 2006 22:44:15 -0000
@@ -692,6 +692,19 @@
 /*	?				0xf8000000ul */
 #define TIMER_CONFIG		BlockIOTag(0x0au)	/* VTB/GTB/LT */
 #define MEM_BUF_CNTL		BlockIOTag(0x0bu)	/* VTB/GTB/LT */
+#define Z_WB_FLUSH			0x00000007ul
+#define Z_WB_FLUSH_P			0x0000000ful	/* GTPro */
+#define VID_WB_FLUSH_P			0x000000f0ul	/* GTPro */
+#define VID_WB_FLUSH_MSB		0x00000100ul
+#define GUI_WB_FLUSH_P			0x00001f00ul	/* GTPro */
+#define HST_WB_FLUSH_P			0x0000e000ul	/* GTPro */
+#define SCL_MIN_BURST_LEN		0x001f0000ul
+#define SCL_THRESH			0x003f0000ul	/* GTPro */
+/*	?				0x00400000ul */
+#define INVALIDATE_RB_CACHE		0x00800000ul
+#define HST_WB_FLUSH			0x03000000ul
+#define VID_WB_FLUSH			0x1c000000ul
+#define GUI_WB_FLUSH			0xe0000000ul
 #define SHARED_CNTL		BlockIOTag(0x0cu)	/* VTB/GTB/LT */
 #define SHARED_MEM_CONFIG	BlockIOTag(0x0du)	/* VTB/GTB/LT */
 #define MEM_ADDR_CONFIG		BlockIOTag(0x0du)	/* GTPro */
Index: atistruct.h
===================================================================
RCS file: /cvs/xorg/xc/programs/Xserver/hw/xfree86/drivers/ati/atistruct.h,v
retrieving revision 1.5
diff -u -3 -p -r1.5 atistruct.h
--- atistruct.h	31 Jul 2005 17:19:27 -0000	1.5
+++ atistruct.h	3 Apr 2006 22:44:15 -0000
@@ -110,7 +110,7 @@ typedef struct _ATIHWRec
     /* Mach64 CPIO registers */
     CARD32 crtc_h_total_disp, crtc_h_sync_strt_wid,
            crtc_v_total_disp, crtc_v_sync_strt_wid,
-           crtc_off_pitch, crtc_gen_cntl, dsp_config, dsp_on_off,
+           crtc_off_pitch, crtc_gen_cntl, dsp_config, dsp_on_off, mem_buf_cntl,
            ovr_clr, ovr_wid_left_right, ovr_wid_top_bottom,
            cur_clr0, cur_clr1, cur_offset,
            cur_horz_vert_posn, cur_horz_vert_off,