Enhance PSTokenizer::getToken 

1. Reserve space for terminating 0 by decrementing size in advance (rather than always having to 
account for it)
2. Utilize new method consumeChar() to indicate we've used the char returned from lookChar().
3. Made placement of calls to consumeChar() consistent inside parsing code.

Add new method PSTokenizer::consumeChar()
This method is more efficient than getChar() for consuming character returned by lookChar().

Enhance PSTokenizer::getChar()
Changed to reduce writes to charBuffer.

*** PSTokenizer.h.~1.1.1.1.~    Thu Mar  3 11:46:01 2005
--- PSTokenizer.h       Wed Dec 20 08:37:09 2006
***************
*** 29,34 ****
--- 29,35 ----
  private:
  
    int lookChar();
+   void consumeChar();
    int getChar();
  
    int (*getCharFunc)(void *);

*** PSTokenizer.cc.~1.1.1.1.~   Thu Mar  3 11:46:03 2005
--- PSTokenizer.cc      Wed Dec 20 08:32:02 2006
***************
*** 55,61 ****
    int c;
    int i;
  
!   // skip whitespace and comments
    comment = gFalse;
    while (1) {
      if ((c = getChar()) == EOF) {
--- 55,61 ----
    int c;
    int i;
  
!   // skip leading whitespace and comments
    comment = gFalse;
    while (1) {
      if ((c = getChar()) == EOF) {
***************
*** 74,89 ****
      }
    }
  
    // read a token
    i = 0;
    buf[i++] = c;
    if (c == '(') {
      backslash = gFalse;
      while ((c = lookChar()) != EOF) {
!       if (i < size - 1) {
        buf[i++] = c;
        }
-       getChar();
        if (c == '\\') {
        backslash = gTrue;
        } else if (!backslash && c == ')') {
--- 74,93 ----
      }
    }
  
+   // Reserve room for terminating '\0'
+   size--;
+ 
    // read a token
    i = 0;
    buf[i++] = c;
+ 
    if (c == '(') {
      backslash = gFalse;
      while ((c = lookChar()) != EOF) {
!       consumeChar();
!       if (i < size) {
        buf[i++] = c;
        }
        if (c == '\\') {
        backslash = gTrue;
        } else if (!backslash && c == ')') {
***************
*** 94,101 ****
      }
    } else if (c == '<') {
      while ((c = lookChar()) != EOF) {
!       getChar();
!       if (i < size - 1) {
        buf[i++] = c;
        }
        if (c == '>') {
--- 98,105 ----
      }
    } else if (c == '<') {
      while ((c = lookChar()) != EOF) {
!       consumeChar();
!       if (i < size) {
        buf[i++] = c;
        }
        if (c == '>') {
***************
*** 104,116 ****
      }
    } else if (c != '[' && c != ']') {
      while ((c = lookChar()) != EOF && !specialChars[c]) {
!       getChar();
!       if (i < size - 1) {
        buf[i++] = c;
        }
      }
    }
    buf[i] = '\0';
    *length = i;
  
    return gTrue;
--- 108,124 ----
      }
    } else if (c != '[' && c != ']') {
      while ((c = lookChar()) != EOF && !specialChars[c]) {
!       consumeChar();
!       if (i < size) {
        buf[i++] = c;
        }
      }
    }
+ 
+   // Zero terminate token string
    buf[i] = '\0';
+ 
+   // Return length of token
    *length = i;
  
    return gTrue;
***************
*** 123,135 ****
    return charBuf;
  }
  
  int PSTokenizer::getChar() {
!   int c;
  
!   if (charBuf < 0) {
!     charBuf = (*getCharFunc)(data);
    }
!   c = charBuf;
!   charBuf = -1;
    return c;
  }
--- 131,148 ----
    return charBuf;
  }
  
+ void PSTokenizer::consumeChar() {
+   charBuf = -1;
+ }
+ 
  int PSTokenizer::getChar() {
!   int c = charBuf;
  
!   if (c < 0) {
!     c = (*getCharFunc)(data);
!   } else {
!     charBuf = -1;
    }
! 
    return c;
  }