From 10a511a35ef345f4ca81f9574268f9dd03affd77 Mon Sep 17 00:00:00 2001
From: David Anderson <dvander@alliedmods.net>
Date: Sat, 21 Jun 2014 23:35:55 -0700
Subject: [PATCH] Introduce 4-token lookahead buffer.
---
sourcepawn/compiler/sc.h | 11 +
sourcepawn/compiler/sc1.c | 278 +++++++++++-------
sourcepawn/compiler/sc2.c | 208 ++++++++-----
sourcepawn/compiler/sc5.scp | 4 +-
sourcepawn/compiler/sctracker.h | 4 -
.../compiler/tests/fail-delete-no-dtor.txt | 2 +-
.../tests/fail-dtor-returns-value.txt | 2 +-
sourcepawn/compiler/tokenbuffer.h | 28 ++
8 files changed, 362 insertions(+), 175 deletions(-)
create mode 100644 sourcepawn/compiler/tokenbuffer.h
diff --git a/sourcepawn/compiler/sc.h b/sourcepawn/compiler/sc.h
index a226aa0d..a3ccfc09 100644
--- a/sourcepawn/compiler/sc.h
+++ b/sourcepawn/compiler/sc.h
@@ -306,6 +306,15 @@ typedef struct {
char *str;
} token_t;
+// The method name buffer is larger since we can include our parent class's
+// name, a "." to separate it, and a "~" for constructors.
+#define METHOD_NAMEMAX sNAMEMAX * 2 + 2
+
+typedef struct {
+ token_t tok;
+ char name[METHOD_NAMEMAX + 1];
+} token_ident_t;
+
/* macros for code generation */
#define opcodes(n) ((n)*sizeof(cell)) /* opcode size */
#define opargs(n) ((n)*sizeof(cell)) /* size of typical argument */
@@ -595,6 +604,8 @@ SC_FUNC int matchtoken(int token);
SC_FUNC int tokeninfo(cell *val,char **str);
SC_FUNC int needtoken(int token);
SC_FUNC int expecttoken(int id, token_t *tok);
+SC_FUNC int matchsymbol(token_ident_t *ident);
+SC_FUNC int needsymbol(token_ident_t *ident);
SC_FUNC void litadd(cell value);
SC_FUNC void litinsert(cell value,int pos);
SC_FUNC int alphanum(char c);
diff --git a/sourcepawn/compiler/sc1.c b/sourcepawn/compiler/sc1.c
index 1ad1f8aa..ee9a0c06 100644
--- a/sourcepawn/compiler/sc1.c
+++ b/sourcepawn/compiler/sc1.c
@@ -78,6 +78,12 @@ int pc_functag = 0;
int pc_tag_string = 0;
int pc_tag_void = 0;
+typedef struct funcstub_setup_s {
+ const char *name;
+ int return_tag;
+ int this_tag;
+} funcstub_setup_t;
+
static void resetglobals(void);
static void initglobals(void);
static char *get_extension(char *filename);
@@ -110,7 +116,7 @@ static cell initvector(int ident,int tag,cell size,int fillzero,
static cell init(int ident,int *tag,int *errorfound);
static int getstates(const char *funcname);
static void attachstatelist(symbol *sym, int state_id);
-static symbol *funcstub(int fnative);
+static symbol *funcstub(int fnative, const funcstub_setup_t *setup);
static int newfunc(char *firstname,int firsttag,int fpublic,int fstatic,int stock);
static int declargs(symbol *sym,int chkshadow);
static void doarg(char *name,int ident,int offset,int tags[],int numtags,
@@ -1564,10 +1570,10 @@ static void parse(void)
} /* if */
break;
case tNATIVE:
- funcstub(TRUE); /* create a dummy function */
+ funcstub(TRUE, NULL); /* create a dummy function */
break;
case tFORWARD:
- funcstub(FALSE);
+ funcstub(FALSE, NULL);
break;
case '}':
error(54); /* unmatched closing brace */
@@ -3346,16 +3352,57 @@ void define_constructor(methodmap_t *map, methodmap_method_t *method)
sym->target = method->target;
}
+// Current lexer position is, we've parsed "public", an optional "native", and
+// a type expression.
+//
+// This returns true if there is a method bind, i.e. "() = Y".
+static int match_method_bind()
+{
+ // The grammar here is a little complicated. We must differentiate
+ // between two different rules:
+ // public X() = Y;
+ // public X() { ...
+ //
+ // If we parse up to '=', then it becomes harder to call newfunc() later,
+ // since ideally we'd like to back up to the '('. To work around this we
+ // use a hacked in lexer API to push older tokens back into the token
+ // stream.
+ token_t tok;
+ if (lextok(&tok) != '(') {
+ lexpush();
+ return FALSE;
+ }
+
+ if (!matchtoken(')')) {
+ for (int i = 0; i < 2; i++)
+ lexpush();
+ return FALSE;
+ }
+
+ if (!matchtoken('=')) {
+ for (int i = 0; i < 3; i++)
+ lexpush();
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
methodmap_method_t *parse_method(methodmap_t *map)
{
+ int is_ctor = 0;
int is_dtor = 0;
int is_bind = 0;
int is_native = 0;
const char *spectype = layout_spec_name(map->spec);
- // We keep a wider buffer since we do name munging.
- char ident[sNAMEMAX * 3 + 1] = "<unknown>";
- char bindname[sNAMEMAX * 3 + 1] = "<unknown>";
+ // This stores the name of the method (for destructors, we add a ~).
+ token_ident_t ident;
+ strcpy(ident.name, "<unknown>");
+
+ // For binding syntax, like X() = Y, this stores the right-hand name.
+ token_ident_t bindsource;
+ strcpy(bindsource.name, "<unknown>");
needtoken(tPUBLIC);
@@ -3363,80 +3410,100 @@ methodmap_method_t *parse_method(methodmap_t *map)
declinfo_t decl;
if (matchtoken('~')) {
// We got something like "public ~Blah = X"
- is_bind = 1;
- is_dtor = 1;
- if (needtoken(tSYMBOL)) {
- tokeninfo(&tok.val, &tok.str);
- strcpy(ident, tok.str);
- }
- needtoken('(');
- needtoken(')');
- needtoken('=');
- if (!expecttoken(tSYMBOL, &tok))
+ is_bind = TRUE;
+ is_dtor = TRUE;
+ if (!needsymbol(&ident))
+ return NULL;
+ if (!needtoken('('))
+ return NULL;
+ if (!needtoken(')'))
+ return NULL;
+ if (!needtoken('='))
+ return NULL;
+ if (!needsymbol(&bindsource))
return NULL;
- strcpy(bindname, tok.str);
} else {
+ int got_symbol;
+
is_native = matchtoken(tNATIVE);
+ got_symbol = matchsymbol(&ident);
- if (is_native) {
- // If we have a native, we should always get a type expression next.
- parse_decl(&decl, NULL, 0);
- } else {
- // Parsing "public Clone =" and "public Handle Clone = " requires two tokens
- // of lookahead. By the time we see the '=' in the first example, we'd have
- // expected a function name, but it's too late to back up - _lexpush is only
- // one token deep.
- //
- // If we see a symbol and a '=', we know it's a simple binding. Otherwise,
- // we have to take the token we got from lextok() and ask parse_decl() to
- // start parsing it as a type expression.
- int is_symbol = (lextok(&tok) == tSYMBOL);
- if (is_symbol) {
- // Save the string because matchtoken() will overwrite the token buffer.
- // Note we also have to repoint tok so parse_decl will point at our
- // local copy.
- strcpy(ident, tok.str);
- tok.str = ident;
+ if (!is_native && got_symbol) {
+ // We didn't see "native", but we saw a symbol. Match for '() =' which
+ // would indicate a method bind.
+ is_bind = match_method_bind();
- if (matchtoken('(')) {
- needtoken(')');
- needtoken('=');
-
- // Grab the name we're binding to.
- is_bind = 1;
- if (!expecttoken(tSYMBOL, &tok))
- return NULL;
- strcpy(bindname, tok.str);
- }
+ if (is_bind) {
+ // If we saw "X() =", then grab the right-hand name.
+ if (!needsymbol(&bindsource))
+ return NULL;
}
+ }
- if (!is_bind) {
- // We didn't find an '=', so proceed with a normal function signature.
- parse_decl(&decl, &tok, 0);
+ if (!is_bind) {
+ // All we know at this point is that we do NOT have a method bind. Keep
+ // pattern matching for an inline constructor, destructor, or method.
+ if (!got_symbol) {
+ // We never saw an initial symbol, so it should be a destructor. If we
+ // don't see a '~', the current token (which is not a symbol) will fail
+ // the needsymbol() check, and we'll bail out.
is_dtor = matchtoken('~');
+ if (!needsymbol(&ident))
+ return NULL;
+ } else if (matchtoken('(')) {
+ // There's no type expression. this is probably a constructor.
+ is_ctor = TRUE;
+ } else {
+ // Parse for type expression, priming it with the token we predicted
+ // would be an identifier.
+ if (!parse_decl(&decl, &ident.tok, 0))
+ return NULL;
- if (lextok(&tok) != tSYMBOL) {
- // Error, and if EOF, return. The lexpush is so we don't accidentally
- // skip over a terminator or something, since we scan to the end of the
- // line.
- lexpush();
- error(111);
- if (tok.id == 0)
- return NULL;
- }
+ // Now, we should get an identifier.
+ if (!needsymbol(&ident))
+ return NULL;
- strcpy(ident, tok.str);
- } // if (tok == symbol && matchtoken('='))
- } // if (is_native)
+ // If the identifier is a constructor, error, since the user specified
+ // a type.
+ if (strcmp(ident.name, map->name) == 0)
+ error(99, "constructor");
+ }
+ } else {
+ is_ctor = (strcmp(ident.name, map->name) == 0);
+ }
} // if (matchtoken('~'))
+
+ // Do some preliminary verification of ctor/dtor names.
+ if (is_dtor) {
+ if (strcmp(ident.name, map->name) != 0)
+ error(114, "destructor", spectype, map->name);
+ } else if (is_ctor) {
+ if (strcmp(ident.name, map->name) != 0)
+ error(114, "constructor", spectype, map->name);
+ }
symbol *target = NULL;
if (is_bind) {
- target = findglb(bindname, sGLOBAL);
+ target = findglb(bindsource.name, sGLOBAL);
if (!target)
- error(17, bindname);
+ error(17, bindsource.name);
else if (target->ident != iFUNCTN)
error(10);
+ // if (decl.usage & uCONST)
+ // error(112, map->name);
+
+ // funcstub_setup_t setup;
+ // if (is_dtor)
+ // setup.return_tag = -1;
+ // else if (is_ctor)
+ // setup.return_tag = map->tag;
+ // else
+ // setup.return_tag = pc_addtag(decl.tag);
+
+ // setup.this_tag = map->tag;
+
+ // if (is_native)
+ // target = funcstub(TRUE, &setup);
} else {
error(10);
}
@@ -3444,11 +3511,8 @@ methodmap_method_t *parse_method(methodmap_t *map)
if (!target)
return NULL;
+ // Verify destructor targets.
if (is_dtor) {
- // Make sure the dtor has the right name.
- if (strcmp(ident, map->name) != 0)
- error(114, spectype, map->name);
-
if (!(target->usage & uNATIVE)) {
// Must be a native.
error(118);
@@ -3457,7 +3521,7 @@ methodmap_method_t *parse_method(methodmap_t *map)
if (target->tag != 0 && target->tag != pc_tag_void) {
// Cannot return a value.
- error(99);
+ error(99, "destructor");
return NULL;
}
@@ -3467,30 +3531,33 @@ methodmap_method_t *parse_method(methodmap_t *map)
return NULL;
}
- // Make sure the final name includes the ~.
- strcpy(ident, "~");
- strcat(ident, map->name);
+ // Make sure the final name has "~" in it.
+ strcpy(ident.name, "~");
+ strcat(ident.name, map->name);
+ }
+
+ // Verify constructor targets.
+ if (is_ctor) {
+ if (target->tag != map->tag)
+ error(112, map->name);
}
// Check that a method with this name doesn't already exist.
for (size_t i = 0; i < map->nummethods; i++) {
- if (strcmp(map->methods[i]->name, ident) == 0) {
- error(103, ident, spectype);
+ if (strcmp(map->methods[i]->name, ident.name) == 0) {
+ error(103, ident.name, spectype);
return NULL;
}
}
methodmap_method_t *method = (methodmap_method_t *)calloc(1, sizeof(methodmap_method_t));
- strcpy(method->name, ident);
+ strcpy(method->name, ident.name);
method->target = target;
if (is_dtor)
map->dtor = method;
- // If the symbol is a constructor, we bypass the initial argument checks,
- // and instead require that it returns something with the same tag.
- if (strcmp(ident, map->name) == 0) {
- if (target->tag != map->tag)
- error(112, map->name);
+ // If the symbol is a constructor, we bypass the initial argument checks.
+ if (is_ctor) {
define_constructor(map, method);
return method;
}
@@ -4477,7 +4544,7 @@ SC_FUNC char *funcdisplayname(char *dest,char *funcname)
return dest;
}
-static symbol *funcstub(int fnative)
+static symbol *funcstub(int fnative, const funcstub_setup_t *setup)
{
int tok,tag,fpublic;
char *str;
@@ -4494,25 +4561,35 @@ static symbol *funcstub(int fnative)
litidx=0; /* clear the literal pool */
assert(loctab.next==NULL); /* local symbol table should be empty */
- tag=pc_addtag(NULL); /* get the tag of the return value */
+ // Either use an explicit return tag, or find a new one.
+ if (!setup || setup->return_tag == 0)
+ tag = pc_addtag(NULL);
+ else if (setup->return_tag == -1)
+ tag = 0;
+ else
+ tag = setup->return_tag;
+
numdim=0;
- while (matchtoken('[')) {
- /* the function returns an array, get this tag for the index and the array
- * dimensions
- */
- if (numdim == sDIMEN_MAX) {
- error(53); /* exceeding maximum number of dimensions */
- return NULL;
- } /* if */
- size=needsub(&idxtag[numdim],NULL); /* get size; size==0 for "var[]" */
- if (size==0)
- error(9); /* invalid array size */
- #if INT_MAX < LONG_MAX
- if (size > INT_MAX)
- error(125); /* overflow, exceeding capacity */
- #endif
- dim[numdim++]=(int)size;
- } /* while */
+ if (!setup) {
+ // Method functions can't return arrays, since it's broken anyway.
+ while (matchtoken('[')) {
+ /* the function returns an array, get this tag for the index and the array
+ * dimensions
+ */
+ if (numdim == sDIMEN_MAX) {
+ error(53); /* exceeding maximum number of dimensions */
+ return NULL;
+ } /* if */
+ size=needsub(&idxtag[numdim],NULL); /* get size; size==0 for "var[]" */
+ if (size==0)
+ error(9); /* invalid array size */
+ #if INT_MAX < LONG_MAX
+ if (size > INT_MAX)
+ error(125); /* overflow, exceeding capacity */
+ #endif
+ dim[numdim++]=(int)size;
+ } /* while */
+ }
if (tag == pc_tag_string && numdim && dim[numdim-1])
dim[numdim-1] = (size + sizeof(cell)-1) / sizeof(cell);
@@ -4591,7 +4668,10 @@ static symbol *funcstub(int fnative)
} /* if */
} /* if */
} /* if */
- needtoken(tTERM);
+
+ // Don't assume inline if we're being setup.
+ if (!setup)
+ needtoken(tTERM);
/* attach the array to the function symbol */
if (numdim>0) {
diff --git a/sourcepawn/compiler/sc2.c b/sourcepawn/compiler/sc2.c
index aea32437..9c924834 100644
--- a/sourcepawn/compiler/sc2.c
+++ b/sourcepawn/compiler/sc2.c
@@ -1,3 +1,4 @@
+// vim: set ts=8 sts=2 sw=2 tw=99 et:
/* Pawn compiler - File input, preprocessing and lexical analysis functions
*
* Copyright (c) ITB CompuPhase, 1997-2006
@@ -28,6 +29,7 @@
#include <math.h>
#include "lstring.h"
#include "sc.h"
+#include "tokenbuffer.h"
#if defined LINUX || defined __FreeBSD__ || defined __OpenBSD__
#include <sclinux.h>
#endif
@@ -1900,24 +1902,41 @@ static const unsigned char *packedstring(const unsigned char *lptr,int flags)
* Global references: lptr (altered)
* fline (referred to only)
* litidx (referred to only)
- * _lextok, _lexval, _lexstr
* _pushed
*/
-static int _pushed;
-static int _lextok;
-static cell _lexval;
-static char _lexstr[sLINEMAX+1];
static int _lexnewline;
+// lex() is called recursively, which messes up the lookahead buffer. To get
+// around this we use two separate token buffers.
+token_buffer_t sNormalBuffer;
+token_buffer_t sPreprocessBuffer;
+token_buffer_t *sTokenBuffer;
+
+static full_token_t *current_token()
+{
+ return &sTokenBuffer->tokens[sTokenBuffer->cursor];
+}
+
+static full_token_t *last_token()
+{
+ assert(sTokenBuffer->depth > 0);
+ int cursor = sTokenBuffer->cursor + 1;
+ if (cursor == MAX_TOKEN_DEPTH)
+ cursor = 0;
+ return &sTokenBuffer->tokens[cursor];
+}
+
SC_FUNC void lexinit(void)
{
stkidx=0; /* index for pushstk() and popstk() */
iflevel=0; /* preprocessor: nesting of "#if" is currently 0 */
skiplevel=0; /* preprocessor: not currently skipping */
icomment=0; /* currently not in a multiline comment */
- _pushed=FALSE; /* no token pushed back into lex */
_lexnewline=FALSE;
+ memset(&sNormalBuffer, 0, sizeof(sNormalBuffer));
+ memset(&sPreprocessBuffer, 0, sizeof(sPreprocessBuffer));
+ sTokenBuffer = &sNormalBuffer;
}
char *sc_tokens[] = {
@@ -1936,24 +1955,49 @@ char *sc_tokens[] = {
"-label-", "-string-"
};
+static full_token_t *next_token_ptr()
+{
+ assert(sTokenBuffer->depth == 0);
+ sTokenBuffer->num_tokens++;
+ sTokenBuffer->cursor++;
+ if (sTokenBuffer->cursor == MAX_TOKEN_DEPTH)
+ sTokenBuffer->cursor = 0;
+
+ return current_token();
+}
+
+static void preprocess_in_lex()
+{
+ sTokenBuffer = &sPreprocessBuffer;
+ preprocess();
+ sTokenBuffer = &sNormalBuffer;
+}
+
SC_FUNC int lex(cell *lexvalue,char **lexsym)
{
int i,toolong,newline;
char **tokptr;
const unsigned char *starttoken;
- if (_pushed) {
- _pushed=FALSE; /* reset "_pushed" flag */
- *lexvalue=_lexval;
- *lexsym=_lexstr;
- return _lextok;
- } /* if */
+ if (sTokenBuffer->depth > 0) {
+ sTokenBuffer->depth--;
+ sTokenBuffer->cursor++;
+ if (sTokenBuffer->cursor == MAX_TOKEN_DEPTH)
+ sTokenBuffer->cursor = 0;
+ *lexvalue = current_token()->value;
+ *lexsym = current_token()->str;
+ return current_token()->id;
+ }
+
+ full_token_t *tok = next_token_ptr();
+ tok->id = 0;
+ tok->value = 0;
+ tok->str[0] = '\0';
+ tok->len = 0;
+
+ *lexvalue = tok->value;
+ *lexsym = tok->str;
- _lextok=0; /* preset all values */
- _lexval=0;
- _lexstr[0]='\0';
- *lexvalue=_lexval;
- *lexsym=_lexstr;
_lexnewline=FALSE;
if (!freading)
return 0;
@@ -1961,11 +2005,11 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
newline= (lptr==pline); /* does lptr point to start of line buffer */
while (*lptr<=' ') { /* delete leading white space */
if (*lptr=='\0') {
- preprocess(); /* preprocess resets "lptr" */
+ preprocess_in_lex();
if (!freading)
return 0;
if (lptr==term_expr) /* special sequence to terminate a pending expression */
- return (_lextok=tENDEXPR);
+ return (tok->id = tENDEXPR);
_lexnewline=TRUE; /* set this after preprocess(), because
* preprocess() calls lex() recursively */
newline=TRUE;
@@ -1986,63 +2030,66 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
tokptr=sc_tokens;
while (i<=tMIDDLE) { /* match multi-character operators */
if (*lptr==**tokptr && match(*tokptr,FALSE)) {
- _lextok=i;
+ tok->id = i;
if (pc_docexpr) /* optionally concatenate to documentation string */
insert_autolist(*tokptr);
- return _lextok;
+ return tok->id;
} /* if */
i+=1;
tokptr+=1;
} /* while */
while (i<=tLAST) { /* match reserved words and compiler directives */
if (*lptr==**tokptr && match(*tokptr,TRUE)) {
- _lextok=i;
+ tok->id = i;
errorset(sRESET,0); /* reset error flag (clear the "panic mode")*/
if (pc_docexpr) /* optionally concatenate to documentation string */
insert_autolist(*tokptr);
- return _lextok;
+ return tok->id;
} /* if */
i+=1;
tokptr+=1;
} /* while */
starttoken=lptr; /* save start pointer (for concatenating to documentation string) */
- if ((i=number(&_lexval,lptr))!=0) { /* number */
- _lextok=tNUMBER;
- *lexvalue=_lexval;
+ if ((i=number(&tok->value, lptr))!=0) { /* number */
+ tok->id = tNUMBER;
+ *lexvalue = tok->value;
lptr+=i;
- } else if ((i=ftoi(&_lexval,lptr))!=0) {
- _lextok=tRATIONAL;
- *lexvalue=_lexval;
+ } else if ((i=ftoi(&tok->value, lptr))!=0) {
+ tok->id = tRATIONAL;
+ *lexvalue = tok->value;
lptr+=i;
} else if (alpha(*lptr)) { /* symbol or label */
/* Note: only sNAMEMAX characters are significant. The compiler
* generates a warning if a symbol exceeds this length.
*/
- _lextok=tSYMBOL;
+ tok->id = tSYMBOL;
i=0;
toolong=0;
while (alphanum(*lptr)){
- _lexstr[i]=*lptr;
+ tok->str[i]=*lptr;
lptr+=1;
if (i<sNAMEMAX)
i+=1;
else
toolong=1;
} /* while */
- _lexstr[i]='\0';
- if (toolong)
- error(200,_lexstr,sNAMEMAX); /* symbol too long, truncated to sNAMEMAX chars */
- if (_lexstr[0]==PUBLIC_CHAR && _lexstr[1]=='\0') {
- _lextok=PUBLIC_CHAR; /* '@' all alone is not a symbol, it is an operator */
- } else if (_lexstr[0]=='_' && _lexstr[1]=='\0') {
- _lextok='_'; /* '_' by itself is not a symbol, it is a placeholder */
+ tok->str[i]='\0';
+ tok->len = i;
+ if (toolong) {
+ /* symbol too long, truncated to sNAMEMAX chars */
+ error(200, tok->str, sNAMEMAX);
+ }
+ if (tok->str[0]==PUBLIC_CHAR && tok->str[1]=='\0') {
+ tok->id = PUBLIC_CHAR; /* '@' all alone is not a symbol, it is an operator */
+ } else if (tok->str[0]=='_' && tok->str[1]=='\0') {
+ tok->id = '_'; /* '_' by itself is not a symbol, it is a placeholder */
} /* if */
- if (*lptr==':' && *(lptr+1)!=':' && _lextok!=PUBLIC_CHAR) {
+ if (*lptr==':' && *(lptr+1)!=':' && tok->id != PUBLIC_CHAR) {
if (sc_allowtags) {
- _lextok=tLABEL; /* it wasn't a normal symbol, it was a label/tagname */
+ tok->id = tLABEL; /* it wasn't a normal symbol, it was a label/tagname */
lptr+=1; /* skip colon */
- } else if (find_constval(&tagname_tab,_lexstr,0)!=NULL) {
+ } else if (find_constval(&tagname_tab,tok->str,0)!=NULL) {
/* this looks like a tag override (because a tag with this name
* exists), but tags are not allowed right now, so it is probably an
* error
@@ -2061,9 +2108,9 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
{
int stringflags,segmentflags;
char *cat;
- _lextok=tSTRING;
- *lexvalue=_lexval=litidx;
- _lexstr[0]='\0';
+ tok->id = tSTRING;
+ *lexvalue = tok->value = litidx;
+ tok->str[0]='\0';
stringflags=-1; /* to mark the first segment */
for ( ;; ) {
if(*lptr=='!')
@@ -2082,9 +2129,9 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
stringflags=segmentflags;
else if (stringflags!=segmentflags)
error(238); /* mixing packed/unpacked/raw strings in concatenation */
- cat=strchr(_lexstr,'\0');
+ cat=strchr(tok->str,'\0');
assert(cat!=NULL);
- while (*lptr!='\"' && *lptr!='\0' && (cat-_lexstr)<sLINEMAX) {
+ while (*lptr!='\"' && *lptr!='\0' && (cat-tok->str)<sLINEMAX) {
if (*lptr!='\a') { /* ignore '\a' (which was inserted at a line concatenation) */
*cat++=*lptr;
if (*lptr==sc_ctrlchar && *(lptr+1)!='\0')
@@ -2093,6 +2140,7 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
lptr++;
} /* while */
*cat='\0'; /* terminate string */
+ tok->len = (size_t)(cat - tok->str);
if (*lptr=='\"')
lptr+=1; /* skip final quote */
else
@@ -2103,7 +2151,7 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
/* there is an ellipses, go on parsing (this time with full preprocessing) */
while (*lptr<=' ') {
if (*lptr=='\0') {
- preprocess(); /* preprocess resets "lptr" */
+ preprocess_in_lex();
assert(freading && lptr!=term_expr);
} else {
lptr++;
@@ -2113,7 +2161,7 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
lptr+=3;
while (*lptr<=' ') {
if (*lptr=='\0') {
- preprocess(); /* preprocess resets "lptr" */
+ preprocess_in_lex();
assert(freading && lptr!=term_expr);
} else {
lptr++;
@@ -2135,23 +2183,23 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
if (sc_packstr)
stringflags ^= ISPACKED; /* invert packed/unpacked parameters */
if ((stringflags & ISPACKED)!=0)
- packedstring((unsigned char *)_lexstr,stringflags);
+ packedstring((unsigned char *)tok->str,stringflags);
else
- unpackedstring((unsigned char *)_lexstr,stringflags);
+ unpackedstring((unsigned char *)tok->str,stringflags);
} else if (*lptr=='\'') { /* character literal */
lptr+=1; /* skip quote */
- _lextok=tNUMBER;
- *lexvalue=_lexval=litchar(&lptr,UTF8MODE);
+ tok->id = tNUMBER;
+ *lexvalue = tok->value = litchar(&lptr,UTF8MODE);
if (*lptr=='\'')
lptr+=1; /* skip final quote */
else
error(27); /* invalid character constant (must be one character) */
} else if (*lptr==';') { /* semicolumn resets "error" flag */
- _lextok=';';
+ tok->id = ';';
lptr+=1;
errorset(sRESET,0); /* reset error flag (clear the "panic mode")*/
} else {
- _lextok=*lptr; /* if every match fails, return the character */
+ tok->id = *lptr; /* if every match fails, return the character */
lptr+=1; /* increase the "lptr" pointer */
} /* if */
@@ -2163,7 +2211,7 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
free(docstr);
} /* if */
} /* if */
- return _lextok;
+ return tok->id;
}
/* lexpush
@@ -2180,8 +2228,14 @@ SC_FUNC int lex(cell *lexvalue,char **lexsym)
*/
SC_FUNC void lexpush(void)
{
- assert(_pushed==FALSE);
- _pushed=TRUE;
+ assert(sTokenBuffer->depth < MAX_TOKEN_DEPTH);
+ assert(sTokenBuffer->depth < 1);
+ sTokenBuffer->depth++;
+ if (sTokenBuffer->cursor == 0)
+ sTokenBuffer->cursor = MAX_TOKEN_DEPTH - 1;
+ else
+ sTokenBuffer->cursor--;
+ assert(sTokenBuffer->depth <= sTokenBuffer->num_tokens);
}
/* lexclr
@@ -2192,7 +2246,7 @@ SC_FUNC void lexpush(void)
*/
SC_FUNC void lexclr(int clreol)
{
- _pushed=FALSE;
+ sTokenBuffer->depth = 0;
if (clreol) {
lptr=(unsigned char*)strchr((char*)pline,'\0');
assert(lptr!=NULL);
@@ -2243,10 +2297,10 @@ SC_FUNC int tokeninfo(cell *val,char **str)
/* if the token was pushed back, tokeninfo() returns the token and
* parameters of the *next* token, not of the *current* token.
*/
- assert(!_pushed);
- *val=_lexval;
- *str=_lexstr;
- return _lextok;
+ assert(sTokenBuffer->depth == 0);
+ *val = current_token()->value;
+ *str = current_token()->str;
+ return current_token()->id;
}
/* needtoken
@@ -2255,8 +2309,6 @@ SC_FUNC int tokeninfo(cell *val,char **str)
* it isn't there (and returns 0/FALSE in that case). Like function matchtoken(),
* this function returns 1 for "token found" and 2 for "statement termination
* token" found; see function matchtoken() for details.
- *
- * Global references: _lextok;
*/
SC_FUNC int needtoken(int token)
{
@@ -2267,17 +2319,17 @@ SC_FUNC int needtoken(int token)
return t;
} else {
/* token already pushed back */
- assert(_pushed);
+ assert(sTokenBuffer->depth > 0);
if (token<256)
sprintf(s1,"%c",(char)token); /* single character token */
else
strcpy(s1,sc_tokens[token-tFIRST]); /* multi-character symbol */
if (!freading)
strcpy(s2,"-end of file-");
- else if (_lextok<256)
- sprintf(s2,"%c",(char)_lextok);
+ else if (current_token()->id < 256)
+ sprintf(s2,"%c",(char)current_token()->id);
else
- strcpy(s2,sc_tokens[_lextok-tFIRST]);
+ strcpy(s2, sc_tokens[current_token()->id - tFIRST]);
error(1,s1,s2); /* expected ..., but found ... */
return FALSE;
} /* if */
@@ -3057,3 +3109,23 @@ SC_FUNC int expecttoken(int id, token_t *tok)
}
return FALSE;
}
+
+SC_FUNC int matchsymbol(token_ident_t *ident)
+{
+ if (lextok(&ident->tok) != tSYMBOL) {
+ lexpush();
+ return FALSE;
+ }
+ strcpy(ident->name, ident->tok.str);
+ ident->tok.str = ident->name;
+ return TRUE;
+}
+
+SC_FUNC int needsymbol(token_ident_t *ident)
+{
+ if (!expecttoken(tSYMBOL, &ident->tok))
+ return FALSE;
+ strcpy(ident->name, ident->tok.str);
+ ident->tok.str = ident->name;
+ return TRUE;
+}
diff --git a/sourcepawn/compiler/sc5.scp b/sourcepawn/compiler/sc5.scp
index 977f1367..07a266fd 100644
--- a/sourcepawn/compiler/sc5.scp
+++ b/sourcepawn/compiler/sc5.scp
@@ -142,7 +142,7 @@ static char *errmsg[] = {
/*096*/ "could not find member \"%s\" in struct \"%s\"\n",
/*097*/ "symbol \"%s\" does not have a matching type\n",
/*098*/ "type \"%s\" should be \"%s\" in new-style declarations\n",
-/*099*/ "destructors cannot return values\n",
+/*099*/ "%s should not have an explicit return type\n",
/*100*/ "function prototypes do not match\n",
/*101*/ "specify either all dimensions or only the last dimension\n",
/*102*/ "cannot find %s %s\n",
@@ -157,7 +157,7 @@ static char *errmsg[] = {
/*111*/ "expected identifier - did you forget a type?\n",
/*112*/ "constructor function must return tag %s\n",
/*113*/ "cannot define constructor for \"%s\"; already exists as a %s\n",
-/*114*/ "destructor must have the same name as %s \"%s\"\n",
+/*114*/ "%s must have the same name as %s \"%s\"\n",
/*115*/ "cannot use delete, %s %s has no destructor\n",
/*116*/ "no methodmap or class was found for %s\n",
/*117*/ "no destructor was found for %s %s\n",
diff --git a/sourcepawn/compiler/sctracker.h b/sourcepawn/compiler/sctracker.h
index 15e35f90..3e8ef711 100644
--- a/sourcepawn/compiler/sctracker.h
+++ b/sourcepawn/compiler/sctracker.h
@@ -79,10 +79,6 @@ typedef enum LayoutSpec_t
Layout_Class
} LayoutSpec;
-// The method name buffer is larger since we can include our parent class's
-// name, a "." to separate it, and a "~" for constructors.
-#define METHOD_NAMEMAX sNAMEMAX * 2 + 2
-
typedef struct methodmap_method_s
{
char name[METHOD_NAMEMAX + 1];
diff --git a/sourcepawn/compiler/tests/fail-delete-no-dtor.txt b/sourcepawn/compiler/tests/fail-delete-no-dtor.txt
index d04d6699..f8623d99 100644
--- a/sourcepawn/compiler/tests/fail-delete-no-dtor.txt
+++ b/sourcepawn/compiler/tests/fail-delete-no-dtor.txt
@@ -1 +1 @@
-methodmap Handle has no destructor; delete cannot be used
+cannot use delete, methodmap Handle has no destructor
diff --git a/sourcepawn/compiler/tests/fail-dtor-returns-value.txt b/sourcepawn/compiler/tests/fail-dtor-returns-value.txt
index 425de1c0..3825f3f5 100644
--- a/sourcepawn/compiler/tests/fail-dtor-returns-value.txt
+++ b/sourcepawn/compiler/tests/fail-dtor-returns-value.txt
@@ -1 +1 @@
-destructors cannot return values
+destructor should not have an explicit return type
diff --git a/sourcepawn/compiler/tokenbuffer.h b/sourcepawn/compiler/tokenbuffer.h
new file mode 100644
index 00000000..6ef58fcc
--- /dev/null
+++ b/sourcepawn/compiler/tokenbuffer.h
@@ -0,0 +1,28 @@
+// vim: set ts=8 sts=2 sw=2 tw=99 et:
+#ifndef _sourcepawn_compiler_token_stream_h_
+#define _sourcepawn_compiler_token_stream_h_
+
+typedef struct {
+ int id;
+ int value;
+ char str[sLINEMAX + 1];
+ size_t len;
+} full_token_t;
+
+#define MAX_TOKEN_DEPTH 4
+
+typedef struct {
+ // Total number of tokens parsed.
+ int num_tokens;
+
+ // Number of tokens that we've rewound back to.
+ int depth;
+
+ // Most recently fetched token.
+ int cursor;
+
+ // Circular token buffer.
+ full_token_t tokens[MAX_TOKEN_DEPTH];
+} token_buffer_t;
+
+#endif // _sourcepawn_compiler_token_stream_h_