Merge pull request #45 from alliedmodders/methodmaps-inline

Introduce 4-token lookahead buffer.

sourcepawn/compiler/sc.h

@@ -306,6 +306,15 @@ typedef struct {
   char *str;
 } token_t;
 
+// The method name buffer is larger since we can include our parent class's
+// name, a "." to separate it, and a "~" for constructors.
+#define METHOD_NAMEMAX sNAMEMAX * 2 + 2
+
+typedef struct {
+  token_t tok;
+  char name[METHOD_NAMEMAX + 1];
+} token_ident_t;
+
 /* macros for code generation */
 #define opcodes(n)      ((n)*sizeof(cell))      /* opcode size */
 #define opargs(n)       ((n)*sizeof(cell))      /* size of typical argument */
@@ -595,6 +604,8 @@ SC_FUNC int matchtoken(int token);
 SC_FUNC int tokeninfo(cell *val,char **str);
 SC_FUNC int needtoken(int token);
 SC_FUNC int expecttoken(int id, token_t *tok);
+SC_FUNC int matchsymbol(token_ident_t *ident);
+SC_FUNC int needsymbol(token_ident_t *ident);
 SC_FUNC void litadd(cell value);
 SC_FUNC void litinsert(cell value,int pos);
 SC_FUNC int alphanum(char c);

sourcepawn/compiler/sc1.c

@@ -78,6 +78,12 @@ int pc_functag = 0;
 int pc_tag_string = 0;
 int pc_tag_void = 0;
 
+typedef struct funcstub_setup_s {
+  const char *name;
+  int return_tag;
+  int this_tag;
+} funcstub_setup_t;
+
 static void resetglobals(void);
 static void initglobals(void);
 static char *get_extension(char *filename);
@@ -110,7 +116,7 @@ static cell initvector(int ident,int tag,cell size,int fillzero,
 static cell init(int ident,int *tag,int *errorfound);
 static int getstates(const char *funcname);
 static void attachstatelist(symbol *sym, int state_id);
-static symbol *funcstub(int fnative);
+static symbol *funcstub(int fnative, const funcstub_setup_t *setup);
 static int newfunc(char *firstname,int firsttag,int fpublic,int fstatic,int stock);
 static int declargs(symbol *sym,int chkshadow);
 static void doarg(char *name,int ident,int offset,int tags[],int numtags,
@@ -1564,10 +1570,10 @@ static void parse(void)
       } /* if */
       break;
     case tNATIVE:
-      funcstub(TRUE);           /* create a dummy function */
+      funcstub(TRUE, NULL);     /* create a dummy function */
       break;
     case tFORWARD:
-      funcstub(FALSE);
+      funcstub(FALSE, NULL);
       break;
     case '}':
       error(54);                /* unmatched closing brace */
@@ -3346,16 +3352,57 @@ void define_constructor(methodmap_t *map, methodmap_method_t *method)
   sym->target = method->target;
 }
 
+// Current lexer position is, we've parsed "public", an optional "native", and
+// a type expression.
+//
+// This returns true if there is a method bind, i.e. "() = Y".
+static int match_method_bind()
+{
+  // The grammar here is a little complicated. We must differentiate
+  // between two different rules:
+  //   public X() = Y;
+  //   public X() { ...
+  //
+  // If we parse up to '=', then it becomes harder to call newfunc() later,
+  // since ideally we'd like to back up to the '('. To work around this we
+  // use a hacked in lexer API to push older tokens back into the token
+  // stream.
+  token_t tok;
+  if (lextok(&tok) != '(') {
+    lexpush();
+    return FALSE;
+  }
+
+  if (!matchtoken(')')) {
+    for (int i = 0; i < 2; i++)
+      lexpush();
+    return FALSE;
+  }
+
+  if (!matchtoken('=')) {
+    for (int i = 0; i < 3; i++)
+      lexpush();
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
 methodmap_method_t *parse_method(methodmap_t *map)
 {
+  int is_ctor = 0;
   int is_dtor = 0;
   int is_bind = 0;
   int is_native = 0;
   const char *spectype = layout_spec_name(map->spec);
 
-  // We keep a wider buffer since we do name munging.
+  // This stores the name of the method (for destructors, we add a ~).
-  char ident[sNAMEMAX * 3 + 1] = "<unknown>";
+  token_ident_t ident;
-  char bindname[sNAMEMAX * 3 + 1] = "<unknown>";
+  strcpy(ident.name, "<unknown>");
+
+  // For binding syntax, like X() = Y, this stores the right-hand name.
+  token_ident_t bindsource;
+  strcpy(bindsource.name, "<unknown>");
 
   needtoken(tPUBLIC);
 
@@ -3363,80 +3410,100 @@ methodmap_method_t *parse_method(methodmap_t *map)
   declinfo_t decl;
   if (matchtoken('~')) {
     // We got something like "public ~Blah = X"
-    is_bind = 1;
+    is_bind = TRUE;
-    is_dtor = 1;
+    is_dtor = TRUE;
-    if (needtoken(tSYMBOL)) {
+    if (!needsymbol(&ident))
-      tokeninfo(&tok.val, &tok.str);
+      return NULL;
-      strcpy(ident, tok.str);
+    if (!needtoken('('))
-    }
+      return NULL;
-    needtoken('(');
+    if (!needtoken(')'))
-    needtoken(')');
+      return NULL;
-    needtoken('=');
+    if (!needtoken('='))
-    if (!expecttoken(tSYMBOL, &tok))
+      return NULL;
+    if (!needsymbol(&bindsource))
       return NULL;
-    strcpy(bindname, tok.str);
   } else {
+    int got_symbol;
+
     is_native = matchtoken(tNATIVE);
+    got_symbol = matchsymbol(&ident);
 
-    if (is_native) {
+    if (!is_native && got_symbol) {
-      // If we have a native, we should always get a type expression next.
+      // We didn't see "native", but we saw a symbol. Match for '() =' which
-      parse_decl(&decl, NULL, 0);
+      // would indicate a method bind.
-    } else {
+      is_bind = match_method_bind();
-      // Parsing "public Clone =" and "public Handle Clone = " requires two tokens
-      // of lookahead. By the time we see the '=' in the first example, we'd have
-      // expected a function name, but it's too late to back up - _lexpush is only
-      // one token deep.
-      //
-      // If we see a symbol and a '=', we know it's a simple binding. Otherwise,
-      // we have to take the token we got from lextok() and ask parse_decl() to
-      // start parsing it as a type expression.
-      int is_symbol = (lextok(&tok) == tSYMBOL);
-      if (is_symbol) {
-        // Save the string because matchtoken() will overwrite the token buffer.
-        // Note we also have to repoint tok so parse_decl will point at our
-        // local copy.
-        strcpy(ident, tok.str);
-        tok.str = ident;
 
-        if (matchtoken('(')) {
+      if (is_bind) {
-          needtoken(')');
+        // If we saw "X() =", then grab the right-hand name.
-          needtoken('=');
+        if (!needsymbol(&bindsource))
-
+          return NULL;
-          // Grab the name we're binding to.
-          is_bind = 1;
-          if (!expecttoken(tSYMBOL, &tok))
-            return NULL;
-          strcpy(bindname, tok.str);
-        }
       }
+    }
 
-      if (!is_bind) {
+    if (!is_bind) {
-        // We didn't find an '=', so proceed with a normal function signature.
+      // All we know at this point is that we do NOT have a method bind. Keep
-        parse_decl(&decl, &tok, 0);
+      // pattern matching for an inline constructor, destructor, or method.
+      if (!got_symbol) {
+        // We never saw an initial symbol, so it should be a destructor. If we
+        // don't see a '~', the current token (which is not a symbol) will fail
+        // the needsymbol() check, and we'll bail out.
         is_dtor = matchtoken('~');
+        if (!needsymbol(&ident))
+          return NULL;
+      } else if (matchtoken('(')) {
+        // There's no type expression. this is probably a constructor.
+        is_ctor = TRUE;
+      } else {
+        // Parse for type expression, priming it with the token we predicted
+        // would be an identifier.
+        if (!parse_decl(&decl, &ident.tok, 0))
+          return NULL;
 
-        if (lextok(&tok) != tSYMBOL) {
+        // Now, we should get an identifier.
-          // Error, and if EOF, return. The lexpush is so we don't accidentally
+        if (!needsymbol(&ident))
-          // skip over a terminator or something, since we scan to the end of the
+          return NULL;
-          // line.
-          lexpush();
-          error(111);
-          if (tok.id == 0)
-            return NULL;
-        }
 
-        strcpy(ident, tok.str);
+        // If the identifier is a constructor, error, since the user specified
-      } // if (tok == symbol && matchtoken('='))
+        // a type.
-    } // if (is_native)
+        if (strcmp(ident.name, map->name) == 0)
+          error(99, "constructor");
+      }
+    } else {
+      is_ctor = (strcmp(ident.name, map->name) == 0);
+    }
   } // if (matchtoken('~'))
+
+  // Do some preliminary verification of ctor/dtor names.
+  if (is_dtor) {
+    if (strcmp(ident.name, map->name) != 0)
+      error(114, "destructor", spectype, map->name);
+  } else if (is_ctor) {
+    if (strcmp(ident.name, map->name) != 0)
+      error(114, "constructor", spectype, map->name);
+  }
   
   symbol *target = NULL;
   if (is_bind) {
-    target = findglb(bindname, sGLOBAL);
+    target = findglb(bindsource.name, sGLOBAL);
     if (!target)
-      error(17, bindname);
+      error(17, bindsource.name);
     else if (target->ident != iFUNCTN) 
       error(10);
+    // if (decl.usage & uCONST)
+    //   error(112, map->name);
+
+    // funcstub_setup_t setup;
+    // if (is_dtor)
+    //   setup.return_tag = -1;
+    // else if (is_ctor)
+    //   setup.return_tag = map->tag;
+    // else
+    //   setup.return_tag = pc_addtag(decl.tag);
+
+    // setup.this_tag = map->tag;
+
+    // if (is_native)
+    //   target = funcstub(TRUE, &setup);
   } else {
     error(10);
   }
@@ -3444,11 +3511,8 @@ methodmap_method_t *parse_method(methodmap_t *map)
   if (!target)
     return NULL;
 
+  // Verify destructor targets.
   if (is_dtor) {
-    // Make sure the dtor has the right name.
-    if (strcmp(ident, map->name) != 0)
-      error(114, spectype, map->name);
-
     if (!(target->usage & uNATIVE)) {
       // Must be a native.
       error(118);
@@ -3457,7 +3521,7 @@ methodmap_method_t *parse_method(methodmap_t *map)
 
     if (target->tag != 0 && target->tag != pc_tag_void) {
       // Cannot return a value.
-      error(99);
+      error(99, "destructor");
       return NULL;
     }
 
@@ -3467,30 +3531,33 @@ methodmap_method_t *parse_method(methodmap_t *map)
       return NULL;
     }
 
-    // Make sure the final name includes the ~.
+    // Make sure the final name has "~" in it.
-    strcpy(ident, "~");
+    strcpy(ident.name, "~");
-    strcat(ident, map->name);
+    strcat(ident.name, map->name);
+  }
+
+  // Verify constructor targets.
+  if (is_ctor) {
+    if (target->tag != map->tag)
+      error(112, map->name);
   }
 
   // Check that a method with this name doesn't already exist.
   for (size_t i = 0; i < map->nummethods; i++) {
-    if (strcmp(map->methods[i]->name, ident) == 0) {
+    if (strcmp(map->methods[i]->name, ident.name) == 0) {
-      error(103, ident, spectype);
+      error(103, ident.name, spectype);
       return NULL;
     }
   }
 
   methodmap_method_t *method = (methodmap_method_t *)calloc(1, sizeof(methodmap_method_t));
-  strcpy(method->name, ident);
+  strcpy(method->name, ident.name);
   method->target = target;
   if (is_dtor)
     map->dtor = method;
 
-  // If the symbol is a constructor, we bypass the initial argument checks,
+  // If the symbol is a constructor, we bypass the initial argument checks.
-  // and instead require that it returns something with the same tag.
+  if (is_ctor) {
-  if (strcmp(ident, map->name) == 0) {
-    if (target->tag != map->tag)
-      error(112, map->name);
     define_constructor(map, method);
     return method;
   }
@@ -4477,7 +4544,7 @@ SC_FUNC char *funcdisplayname(char *dest,char *funcname)
   return dest;
 }
 
-static symbol *funcstub(int fnative)
+static symbol *funcstub(int fnative, const funcstub_setup_t *setup)
 {
   int tok,tag,fpublic;
   char *str;
@@ -4494,25 +4561,35 @@ static symbol *funcstub(int fnative)
   litidx=0;                     /* clear the literal pool */
   assert(loctab.next==NULL);    /* local symbol table should be empty */
 
-  tag=pc_addtag(NULL);			/* get the tag of the return value */
+  // Either use an explicit return tag, or find a new one.
+  if (!setup || setup->return_tag == 0)
+    tag = pc_addtag(NULL);
+  else if (setup->return_tag == -1)
+    tag = 0;
+  else
+    tag = setup->return_tag;
+
   numdim=0;
-  while (matchtoken('[')) {
+  if (!setup) {
-    /* the function returns an array, get this tag for the index and the array
+    // Method functions can't return arrays, since it's broken anyway.
-     * dimensions
+    while (matchtoken('[')) {
-     */
+      /* the function returns an array, get this tag for the index and the array
-    if (numdim == sDIMEN_MAX) {
+       * dimensions
-      error(53);                /* exceeding maximum number of dimensions */
+       */
-      return NULL;
+      if (numdim == sDIMEN_MAX) {
-    } /* if */
+        error(53);                /* exceeding maximum number of dimensions */
-    size=needsub(&idxtag[numdim],NULL); /* get size; size==0 for "var[]" */
+        return NULL;
-    if (size==0)
+      } /* if */
-      error(9);                 /* invalid array size */
+      size=needsub(&idxtag[numdim],NULL); /* get size; size==0 for "var[]" */
-    #if INT_MAX < LONG_MAX
+      if (size==0)
-      if (size > INT_MAX)
+        error(9);                 /* invalid array size */
-        error(125);             /* overflow, exceeding capacity */
+      #if INT_MAX < LONG_MAX
-    #endif
+        if (size > INT_MAX)
-    dim[numdim++]=(int)size;
+          error(125);             /* overflow, exceeding capacity */
-  } /* while */
+      #endif
+      dim[numdim++]=(int)size;
+    } /* while */
+  }
 
   if (tag == pc_tag_string && numdim && dim[numdim-1])
     dim[numdim-1] = (size + sizeof(cell)-1) / sizeof(cell);
@@ -4591,7 +4668,10 @@ static symbol *funcstub(int fnative)
       } /* if */
     } /* if */
   } /* if */
-  needtoken(tTERM);
+
+  // Don't assume inline if we're being setup.
+  if (!setup)
+    needtoken(tTERM);
 
   /* attach the array to the function symbol */
   if (numdim>0) {

sourcepawn/compiler/sc2.c

@@ -1,3 +1,4 @@
+// vim: set ts=8 sts=2 sw=2 tw=99 et:
 /*  Pawn compiler - File input, preprocessing and lexical analysis functions
  *
  *  Copyright (c) ITB CompuPhase, 1997-2006
@@ -28,6 +29,7 @@
 #include <math.h>
 #include "lstring.h"
 #include "sc.h"
+#include "tokenbuffer.h"
 #if defined LINUX || defined __FreeBSD__ || defined __OpenBSD__
   #include <sclinux.h>
 #endif
@@ -1900,24 +1902,41 @@ static const unsigned char *packedstring(const unsigned char *lptr,int flags)
  *  Global references: lptr          (altered)
  *                     fline         (referred to only)
  *                     litidx        (referred to only)
- *                     _lextok, _lexval, _lexstr
  *                     _pushed
  */
 
-static int _pushed;
-static int _lextok;
-static cell _lexval;
-static char _lexstr[sLINEMAX+1];
 static int _lexnewline;
 
+// lex() is called recursively, which messes up the lookahead buffer. To get
+// around this we use two separate token buffers.
+token_buffer_t sNormalBuffer;
+token_buffer_t sPreprocessBuffer;
+token_buffer_t *sTokenBuffer;
+
+static full_token_t *current_token()
+{
+  return &sTokenBuffer->tokens[sTokenBuffer->cursor];
+}
+
+static full_token_t *last_token()
+{
+  assert(sTokenBuffer->depth > 0);
+  int cursor = sTokenBuffer->cursor + 1;
+  if (cursor == MAX_TOKEN_DEPTH)
+    cursor = 0;
+  return &sTokenBuffer->tokens[cursor];
+}
+
 SC_FUNC void lexinit(void)
 {
   stkidx=0;             /* index for pushstk() and popstk() */
   iflevel=0;            /* preprocessor: nesting of "#if" is currently 0 */
   skiplevel=0;          /* preprocessor: not currently skipping */
   icomment=0;           /* currently not in a multiline comment */
-  _pushed=FALSE;        /* no token pushed back into lex */
   _lexnewline=FALSE;
+  memset(&sNormalBuffer, 0, sizeof(sNormalBuffer));
+  memset(&sPreprocessBuffer, 0, sizeof(sPreprocessBuffer));
+  sTokenBuffer = &sNormalBuffer;
 }
 
 char *sc_tokens[] = {
@@ -1936,24 +1955,49 @@ char *sc_tokens[] = {
          "-label-", "-string-"
        };
 
+static full_token_t *next_token_ptr()
+{
+  assert(sTokenBuffer->depth == 0);
+  sTokenBuffer->num_tokens++;
+  sTokenBuffer->cursor++;
+  if (sTokenBuffer->cursor == MAX_TOKEN_DEPTH)
+    sTokenBuffer->cursor = 0;
+
+  return current_token();
+}
+
+static void preprocess_in_lex()
+{
+  sTokenBuffer = &sPreprocessBuffer;
+  preprocess();
+  sTokenBuffer = &sNormalBuffer;
+}
+
 SC_FUNC int lex(cell *lexvalue,char **lexsym)
 {
   int i,toolong,newline;
   char **tokptr;
   const unsigned char *starttoken;
 
-  if (_pushed) {
+  if (sTokenBuffer->depth > 0) {
-    _pushed=FALSE;      /* reset "_pushed" flag */
+    sTokenBuffer->depth--;
-    *lexvalue=_lexval;
+    sTokenBuffer->cursor++;
-    *lexsym=_lexstr;
+    if (sTokenBuffer->cursor == MAX_TOKEN_DEPTH)
-    return _lextok;
+      sTokenBuffer->cursor = 0;
-  } /* if */
+    *lexvalue = current_token()->value;
+    *lexsym = current_token()->str;
+    return current_token()->id;
+  }
+
+  full_token_t *tok = next_token_ptr();
+  tok->id = 0;
+  tok->value = 0;
+  tok->str[0] = '\0';
+  tok->len = 0;
+
+  *lexvalue = tok->value;
+  *lexsym = tok->str;
 
-  _lextok=0;            /* preset all values */
-  _lexval=0;
-  _lexstr[0]='\0';
-  *lexvalue=_lexval;
-  *lexsym=_lexstr;
   _lexnewline=FALSE;
   if (!freading)
     return 0;
@@ -1961,11 +2005,11 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
   newline= (lptr==pline);       /* does lptr point to start of line buffer */
   while (*lptr<=' ') {          /* delete leading white space */
     if (*lptr=='\0') {
-      preprocess();             /* preprocess resets "lptr" */
+      preprocess_in_lex();
       if (!freading)
         return 0;
       if (lptr==term_expr)      /* special sequence to terminate a pending expression */
-        return (_lextok=tENDEXPR);
+        return (tok->id = tENDEXPR);
       _lexnewline=TRUE;         /* set this after preprocess(), because
                                  * preprocess() calls lex() recursively */
       newline=TRUE;
@@ -1986,63 +2030,66 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
   tokptr=sc_tokens;
   while (i<=tMIDDLE) {  /* match multi-character operators */
     if (*lptr==**tokptr && match(*tokptr,FALSE)) {
-      _lextok=i;
+      tok->id = i;
       if (pc_docexpr)   /* optionally concatenate to documentation string */
         insert_autolist(*tokptr);
-      return _lextok;
+      return tok->id;
     } /* if */
     i+=1;
     tokptr+=1;
   } /* while */
   while (i<=tLAST) {    /* match reserved words and compiler directives */
     if (*lptr==**tokptr && match(*tokptr,TRUE)) {
-      _lextok=i;
+      tok->id = i;
       errorset(sRESET,0); /* reset error flag (clear the "panic mode")*/
       if (pc_docexpr)   /* optionally concatenate to documentation string */
         insert_autolist(*tokptr);
-      return _lextok;
+      return tok->id;
     } /* if */
     i+=1;
     tokptr+=1;
   } /* while */
 
   starttoken=lptr;      /* save start pointer (for concatenating to documentation string) */
-  if ((i=number(&_lexval,lptr))!=0) {   /* number */
+  if ((i=number(&tok->value, lptr))!=0) {   /* number */
-    _lextok=tNUMBER;
+    tok->id = tNUMBER;
-    *lexvalue=_lexval;
+    *lexvalue = tok->value;
     lptr+=i;
-  } else if ((i=ftoi(&_lexval,lptr))!=0) {
+  } else if ((i=ftoi(&tok->value, lptr))!=0) {
-    _lextok=tRATIONAL;
+    tok->id = tRATIONAL;
-    *lexvalue=_lexval;
+    *lexvalue = tok->value;
     lptr+=i;
   } else if (alpha(*lptr)) {            /* symbol or label */
     /*  Note: only sNAMEMAX characters are significant. The compiler
      *        generates a warning if a symbol exceeds this length.
      */
-    _lextok=tSYMBOL;
+    tok->id = tSYMBOL;
     i=0;
     toolong=0;
     while (alphanum(*lptr)){
-      _lexstr[i]=*lptr;
+      tok->str[i]=*lptr;
       lptr+=1;
       if (i<sNAMEMAX)
         i+=1;
       else
         toolong=1;
     } /* while */
-    _lexstr[i]='\0';
+    tok->str[i]='\0';
-    if (toolong)
+    tok->len = i;
-      error(200,_lexstr,sNAMEMAX);  /* symbol too long, truncated to sNAMEMAX chars */
+    if (toolong) {
-    if (_lexstr[0]==PUBLIC_CHAR && _lexstr[1]=='\0') {
+      /* symbol too long, truncated to sNAMEMAX chars */
-      _lextok=PUBLIC_CHAR;  /* '@' all alone is not a symbol, it is an operator */
+      error(200, tok->str, sNAMEMAX);  
-    } else if (_lexstr[0]=='_' && _lexstr[1]=='\0') {
+    }
-      _lextok='_';      /* '_' by itself is not a symbol, it is a placeholder */
+    if (tok->str[0]==PUBLIC_CHAR && tok->str[1]=='\0') {
+      tok->id = PUBLIC_CHAR;  /* '@' all alone is not a symbol, it is an operator */
+    } else if (tok->str[0]=='_' && tok->str[1]=='\0') {
+      tok->id = '_';      /* '_' by itself is not a symbol, it is a placeholder */
     } /* if */
-    if (*lptr==':' && *(lptr+1)!=':' && _lextok!=PUBLIC_CHAR) {
+    if (*lptr==':' && *(lptr+1)!=':' && tok->id != PUBLIC_CHAR) {
       if (sc_allowtags) {
-        _lextok=tLABEL; /* it wasn't a normal symbol, it was a label/tagname */
+        tok->id = tLABEL; /* it wasn't a normal symbol, it was a label/tagname */
         lptr+=1;        /* skip colon */
-      } else if (find_constval(&tagname_tab,_lexstr,0)!=NULL) {
+      } else if (find_constval(&tagname_tab,tok->str,0)!=NULL) {
         /* this looks like a tag override (because a tag with this name
          * exists), but tags are not allowed right now, so it is probably an
          * error
@@ -2061,9 +2108,9 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
   {                                     
     int stringflags,segmentflags;
     char *cat;
-    _lextok=tSTRING;
+    tok->id = tSTRING;
-    *lexvalue=_lexval=litidx;
+    *lexvalue = tok->value = litidx;
-    _lexstr[0]='\0';
+    tok->str[0]='\0';
     stringflags=-1;       /* to mark the first segment */
     for ( ;; ) {
       if(*lptr=='!')
@@ -2082,9 +2129,9 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
         stringflags=segmentflags;
       else if (stringflags!=segmentflags)
         error(238);       /* mixing packed/unpacked/raw strings in concatenation */
-      cat=strchr(_lexstr,'\0');
+      cat=strchr(tok->str,'\0');
       assert(cat!=NULL);
-      while (*lptr!='\"' && *lptr!='\0' && (cat-_lexstr)<sLINEMAX) {
+      while (*lptr!='\"' && *lptr!='\0' && (cat-tok->str)<sLINEMAX) {
         if (*lptr!='\a') {  /* ignore '\a' (which was inserted at a line concatenation) */
           *cat++=*lptr;
 					if (*lptr==sc_ctrlchar && *(lptr+1)!='\0')
@@ -2093,6 +2140,7 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
         lptr++;
       } /* while */
       *cat='\0';          /* terminate string */
+      tok->len = (size_t)(cat - tok->str);
       if (*lptr=='\"')
         lptr+=1;          /* skip final quote */
       else
@@ -2103,7 +2151,7 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
       /* there is an ellipses, go on parsing (this time with full preprocessing) */
       while (*lptr<=' ') {
         if (*lptr=='\0') {
-          preprocess();           /* preprocess resets "lptr" */
+          preprocess_in_lex();
           assert(freading && lptr!=term_expr);
         } else {
           lptr++;
@@ -2113,7 +2161,7 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
       lptr+=3;
       while (*lptr<=' ') {
         if (*lptr=='\0') {
-          preprocess();           /* preprocess resets "lptr" */
+          preprocess_in_lex();
           assert(freading && lptr!=term_expr);
         } else {
           lptr++;
@@ -2135,23 +2183,23 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
     if (sc_packstr)
       stringflags ^= ISPACKED;    /* invert packed/unpacked parameters */
     if ((stringflags & ISPACKED)!=0)
-      packedstring((unsigned char *)_lexstr,stringflags);
+      packedstring((unsigned char *)tok->str,stringflags);
     else
-      unpackedstring((unsigned char *)_lexstr,stringflags);
+      unpackedstring((unsigned char *)tok->str,stringflags);
   } else if (*lptr=='\'') {             /* character literal */
     lptr+=1;            /* skip quote */
-    _lextok=tNUMBER;
+    tok->id = tNUMBER;
-    *lexvalue=_lexval=litchar(&lptr,UTF8MODE);
+    *lexvalue = tok->value = litchar(&lptr,UTF8MODE);
     if (*lptr=='\'')
       lptr+=1;          /* skip final quote */
     else
       error(27);        /* invalid character constant (must be one character) */
   } else if (*lptr==';') {      /* semicolumn resets "error" flag */
-    _lextok=';';
+    tok->id = ';';
     lptr+=1;
     errorset(sRESET,0); /* reset error flag (clear the "panic mode")*/
   } else {
-    _lextok=*lptr;      /* if every match fails, return the character */
+    tok->id = *lptr;    /* if every match fails, return the character */
     lptr+=1;            /* increase the "lptr" pointer */
   } /* if */
 
@@ -2163,7 +2211,7 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
       free(docstr);
     } /* if */
   } /* if */
-  return _lextok;
+  return tok->id;
 }
 
 /*  lexpush
@@ -2180,8 +2228,14 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
  */
 SC_FUNC void lexpush(void)
 {
-  assert(_pushed==FALSE);
+  assert(sTokenBuffer->depth < MAX_TOKEN_DEPTH);
-  _pushed=TRUE;
+  assert(sTokenBuffer->depth < 1);
+  sTokenBuffer->depth++;
+  if (sTokenBuffer->cursor == 0)
+    sTokenBuffer->cursor = MAX_TOKEN_DEPTH - 1;
+  else
+    sTokenBuffer->cursor--;
+  assert(sTokenBuffer->depth <= sTokenBuffer->num_tokens);
 }
 
 /*  lexclr
@@ -2192,7 +2246,7 @@ SC_FUNC void lexpush(void)
  */
 SC_FUNC void lexclr(int clreol)
 {
-  _pushed=FALSE;
+  sTokenBuffer->depth = 0;
   if (clreol) {
     lptr=(unsigned char*)strchr((char*)pline,'\0');
     assert(lptr!=NULL);
@@ -2243,10 +2297,10 @@ SC_FUNC int tokeninfo(cell *val,char **str)
   /* if the token was pushed back, tokeninfo() returns the token and
    * parameters of the *next* token, not of the *current* token.
    */
-  assert(!_pushed);
+  assert(sTokenBuffer->depth == 0);
-  *val=_lexval;
+  *val = current_token()->value;
-  *str=_lexstr;
+  *str = current_token()->str;
-  return _lextok;
+  return current_token()->id;
 }
 
 /*  needtoken
@@ -2255,8 +2309,6 @@ SC_FUNC int tokeninfo(cell *val,char **str)
  *  it isn't there (and returns 0/FALSE in that case). Like function matchtoken(),
  *  this function returns 1 for "token found" and 2 for "statement termination
  *  token" found; see function matchtoken() for details.
- *
- *  Global references: _lextok;
  */
 SC_FUNC int needtoken(int token)
 {
@@ -2267,17 +2319,17 @@ SC_FUNC int needtoken(int token)
     return t;
   } else {
     /* token already pushed back */
-    assert(_pushed);
+    assert(sTokenBuffer->depth > 0);
     if (token<256)
       sprintf(s1,"%c",(char)token);        /* single character token */
     else
       strcpy(s1,sc_tokens[token-tFIRST]);  /* multi-character symbol */
     if (!freading)
       strcpy(s2,"-end of file-");
-    else if (_lextok<256)
+    else if (current_token()->id < 256)
-      sprintf(s2,"%c",(char)_lextok);
+      sprintf(s2,"%c",(char)current_token()->id);
     else
-      strcpy(s2,sc_tokens[_lextok-tFIRST]);
+      strcpy(s2, sc_tokens[current_token()->id - tFIRST]);
     error(1,s1,s2);     /* expected ..., but found ... */
     return FALSE;
   } /* if */
@@ -3057,3 +3109,23 @@ SC_FUNC int expecttoken(int id, token_t *tok)
   }
   return FALSE;
 }
+
+SC_FUNC int matchsymbol(token_ident_t *ident)
+{
+  if (lextok(&ident->tok) != tSYMBOL) {
+    lexpush();
+    return FALSE;
+  }
+  strcpy(ident->name, ident->tok.str);
+  ident->tok.str = ident->name;
+  return TRUE;
+}
+
+SC_FUNC int needsymbol(token_ident_t *ident)
+{
+  if (!expecttoken(tSYMBOL, &ident->tok))
+    return FALSE;
+  strcpy(ident->name, ident->tok.str);
+  ident->tok.str = ident->name;
+  return TRUE;
+}

sourcepawn/compiler/sc5.scp

@@ -142,7 +142,7 @@ static char *errmsg[] = {
 /*096*/  "could not find member \"%s\" in struct \"%s\"\n",
 /*097*/  "symbol \"%s\" does not have a matching type\n",
 /*098*/  "type \"%s\" should be \"%s\" in new-style declarations\n",
-/*099*/  "destructors cannot return values\n",
+/*099*/  "%s should not have an explicit return type\n",
 /*100*/  "function prototypes do not match\n",
 /*101*/  "specify either all dimensions or only the last dimension\n",
 /*102*/  "cannot find %s %s\n",
@@ -157,7 +157,7 @@ static char *errmsg[] = {
 /*111*/  "expected identifier - did you forget a type?\n",
 /*112*/  "constructor function must return tag %s\n",
 /*113*/  "cannot define constructor for \"%s\"; already exists as a %s\n",
-/*114*/  "destructor must have the same name as %s \"%s\"\n",
+/*114*/  "%s must have the same name as %s \"%s\"\n",
 /*115*/  "cannot use delete, %s %s has no destructor\n",
 /*116*/  "no methodmap or class was found for %s\n",
 /*117*/  "no destructor was found for %s %s\n",

sourcepawn/compiler/sctracker.h

@@ -79,10 +79,6 @@ typedef enum LayoutSpec_t
   Layout_Class
 } LayoutSpec;
 
-// The method name buffer is larger since we can include our parent class's
-// name, a "." to separate it, and a "~" for constructors.
-#define METHOD_NAMEMAX sNAMEMAX * 2 + 2
-
 typedef struct methodmap_method_s
 {
   char name[METHOD_NAMEMAX + 1];

sourcepawn/compiler/tests/fail-delete-no-dtor.txt

@@ -1 +1 @@
-methodmap Handle has no destructor; delete cannot be used
+cannot use delete, methodmap Handle has no destructor

sourcepawn/compiler/tests/fail-dtor-returns-value.txt

@@ -1 +1 @@
-destructors cannot return values
+destructor should not have an explicit return type

sourcepawn/compiler/tokenbuffer.h

@@ -0,0 +1,28 @@
+// vim: set ts=8 sts=2 sw=2 tw=99 et:
+#ifndef _sourcepawn_compiler_token_stream_h_
+#define _sourcepawn_compiler_token_stream_h_
+
+typedef struct {
+  int id;
+  int value;
+  char str[sLINEMAX + 1];
+  size_t len;
+} full_token_t;
+
+#define MAX_TOKEN_DEPTH 4
+
+typedef struct {
+  // Total number of tokens parsed.
+  int num_tokens;
+
+  // Number of tokens that we've rewound back to.
+  int depth;
+
+  // Most recently fetched token.
+  int cursor;
+
+  // Circular token buffer.
+  full_token_t tokens[MAX_TOKEN_DEPTH];
+} token_buffer_t;
+
+#endif // _sourcepawn_compiler_token_stream_h_