af05dbbcdf
--HG-- extra : convert_revision : svn%3A39bc706e-5318-0410-9160-8a85361fbb7c/trunk%402088
1343 lines
41 KiB
C#
1343 lines
41 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Text;
|
|
|
|
namespace incparser
|
|
{
|
|
enum LexToken
|
|
{
|
|
TOKEN_NONE,
|
|
TOKEN_INCLUDE,
|
|
TOKEN_DEFINE,
|
|
TOKEN_CONST,
|
|
TOKEN_STRUCT,
|
|
TOKEN_FUNCTAG,
|
|
TOKEN_FUNCENUM,
|
|
TOKEN_ENUM,
|
|
TOKEN_NATIVE,
|
|
TOKEN_STOCK,
|
|
TOKEN_PUBLIC,
|
|
TOKEN_FORWARD,
|
|
TOKEN_QUOTCHAR,
|
|
TOKEN_OPERATOR,
|
|
TOKEN_CHARACTER,
|
|
/*TOKEN_STRING,*/
|
|
TOKEN_LABEL,
|
|
TOKEN_SYMBOL,
|
|
TOKEN_DOCBLOCK,
|
|
TOKEN_EOF,
|
|
}
|
|
|
|
class ParseException : System.Exception
|
|
{
|
|
public ParseException(string message) : base(message)
|
|
{
|
|
}
|
|
};
|
|
|
|
class KeywordToken
|
|
{
|
|
public KeywordToken(string k, LexToken t)
|
|
{
|
|
keyword = k;
|
|
token = t;
|
|
}
|
|
public string keyword;
|
|
public LexToken token;
|
|
};
|
|
|
|
class Tokenizer
|
|
{
|
|
private static bool s_initialized = false;
|
|
private static int s_position = 0;
|
|
private static KeywordToken [] s_tokens = null;
|
|
public static KeywordToken[] Tokens = null;
|
|
|
|
public static void Initialize()
|
|
{
|
|
if (s_initialized)
|
|
{
|
|
return;
|
|
}
|
|
s_tokens = new KeywordToken[12];
|
|
s_initialized = true;
|
|
s_position = 0;
|
|
|
|
AddToken("#include", LexToken.TOKEN_INCLUDE);
|
|
AddToken("#define", LexToken.TOKEN_DEFINE);
|
|
AddToken("native", LexToken.TOKEN_NATIVE);
|
|
AddToken("functag", LexToken.TOKEN_FUNCTAG);
|
|
AddToken("funcenum", LexToken.TOKEN_FUNCENUM);
|
|
AddToken("enum", LexToken.TOKEN_ENUM);
|
|
AddToken("stock", LexToken.TOKEN_STOCK);
|
|
AddToken("public", LexToken.TOKEN_PUBLIC);
|
|
AddToken("forward", LexToken.TOKEN_FORWARD);
|
|
AddToken("const", LexToken.TOKEN_CONST);
|
|
AddToken("struct", LexToken.TOKEN_STRUCT);
|
|
AddToken("operator", LexToken.TOKEN_OPERATOR);
|
|
|
|
Tokens = s_tokens;
|
|
}
|
|
|
|
private static void AddToken(string lex, LexToken tok)
|
|
{
|
|
s_tokens[s_position] = new KeywordToken(lex, tok);
|
|
s_position++;
|
|
}
|
|
};
|
|
|
|
class IncParser
|
|
{
|
|
private string FileName;
|
|
private StringBuilder Contents;
|
|
private bool LexPushed;
|
|
private LexToken _LastToken;
|
|
private string _LexString;
|
|
private char _LexChar;
|
|
private uint LineNo;
|
|
|
|
public string LexString
|
|
{
|
|
get
|
|
{
|
|
return _LexString;
|
|
}
|
|
}
|
|
|
|
public char LexChar
|
|
{
|
|
get
|
|
{
|
|
return _LexChar;
|
|
}
|
|
}
|
|
|
|
public uint GetLineNumber()
|
|
{
|
|
return LineNo;
|
|
}
|
|
|
|
public IncParser(string file)
|
|
{
|
|
FileName = file;
|
|
|
|
/* Clear out lexer stuff */
|
|
LexPushed = false;
|
|
_LastToken = LexToken.TOKEN_NONE;
|
|
_LexString = null;
|
|
_LexChar = '\0';
|
|
LineNo = 1;
|
|
|
|
/* Initialize; this can throw an exception */
|
|
Initialize();
|
|
}
|
|
|
|
public void Parse(ParseWriter w)
|
|
{
|
|
w.BeginSection(System.IO.Path.GetFileName(FileName));
|
|
LexToken tok = LexToken.TOKEN_NONE;
|
|
while ((tok = lex()) != LexToken.TOKEN_EOF)
|
|
{
|
|
switch (tok)
|
|
{
|
|
case LexToken.TOKEN_DOCBLOCK:
|
|
{
|
|
w.WritePair("doc", LexString);
|
|
break;
|
|
}
|
|
case LexToken.TOKEN_DEFINE:
|
|
{
|
|
PARSE_Define(w);
|
|
break;
|
|
}
|
|
case LexToken.TOKEN_ENUM:
|
|
{
|
|
PARSE_Enum(w);
|
|
break;
|
|
}
|
|
case LexToken.TOKEN_FORWARD:
|
|
{
|
|
PARSE_Function(tok, w);
|
|
break;
|
|
}
|
|
case LexToken.TOKEN_NATIVE:
|
|
{
|
|
PARSE_Function(tok, w);
|
|
break;
|
|
}
|
|
case LexToken.TOKEN_STOCK:
|
|
{
|
|
PARSE_Function(tok, w);
|
|
break;
|
|
}
|
|
case LexToken.TOKEN_FUNCTAG:
|
|
{
|
|
PARSE_FuncTag(w);
|
|
break;
|
|
}
|
|
case LexToken.TOKEN_FUNCENUM:
|
|
{
|
|
PARSE_FuncEnum(w);
|
|
break;
|
|
}
|
|
case LexToken.TOKEN_INCLUDE:
|
|
{
|
|
PARSE_Include(w);
|
|
break;
|
|
}
|
|
case LexToken.TOKEN_STRUCT:
|
|
{
|
|
PARSE_Struct(w);
|
|
break;
|
|
}
|
|
case LexToken.TOKEN_PUBLIC:
|
|
{
|
|
PARSE_Public(w);
|
|
break;
|
|
}
|
|
default:
|
|
{
|
|
if (_LexString == "static" || _LexString == "new")
|
|
{
|
|
PARSE_Public(w);
|
|
break;
|
|
}
|
|
throw new ParseException("Unrecognized token: " + tok + " " + _LexString);
|
|
}
|
|
}
|
|
}
|
|
w.EndSection();
|
|
}
|
|
|
|
private void PARSE_Public(ParseWriter w)
|
|
{
|
|
MatchToken(LexToken.TOKEN_CONST);
|
|
|
|
/* Eat up an optional tag */
|
|
MatchToken(LexToken.TOKEN_LABEL);
|
|
|
|
/* Eat up a name */
|
|
NeedToken(LexToken.TOKEN_SYMBOL);
|
|
|
|
while (MatchChar('['))
|
|
{
|
|
DiscardUntilChar(']');
|
|
NeedChar(']');
|
|
}
|
|
|
|
if (MatchChar('='))
|
|
{
|
|
ignore_block();
|
|
}
|
|
else if (MatchChar('(') && MatchChar(')'))
|
|
{
|
|
ignore_block();
|
|
return;
|
|
}
|
|
|
|
NeedChar(';');
|
|
}
|
|
|
|
private void PARSE_Struct(ParseWriter w)
|
|
{
|
|
/* For now, we completely ignore these (they're not written to the output) */
|
|
NeedToken(LexToken.TOKEN_SYMBOL);
|
|
w.structList.Add(LexString);
|
|
NeedChar('{');
|
|
|
|
bool need_closebrace = true;
|
|
bool more_entries = true;
|
|
do
|
|
{
|
|
if (MatchChar('}'))
|
|
{
|
|
need_closebrace = false;
|
|
break;
|
|
}
|
|
MatchToken(LexToken.TOKEN_CONST);
|
|
MatchToken(LexToken.TOKEN_LABEL);
|
|
NeedToken(LexToken.TOKEN_SYMBOL);
|
|
while (MatchChar('['))
|
|
{
|
|
DiscardUntilChar(']');
|
|
NeedChar(']');
|
|
}
|
|
more_entries = MatchChar(',');
|
|
MatchToken(LexToken.TOKEN_DOCBLOCK);
|
|
if (!more_entries)
|
|
{
|
|
break;
|
|
}
|
|
} while (true);
|
|
if (need_closebrace)
|
|
{
|
|
NeedChar('}');
|
|
}
|
|
NeedChar(';');
|
|
}
|
|
|
|
private void PARSE_Include(ParseWriter w)
|
|
{
|
|
/* read until the end of the line */
|
|
int index = Contents.ToString().IndexOf('\n');
|
|
string value;
|
|
if (index == -1)
|
|
{
|
|
value = Contents.ToString();
|
|
Contents.Remove(0, value.Length);
|
|
}
|
|
else if (index == 0)
|
|
{
|
|
value = "";
|
|
}
|
|
else
|
|
{
|
|
value = Contents.ToString().Substring(0, index);
|
|
Contents.Remove(0, index);
|
|
}
|
|
|
|
/* Strip whitespace */
|
|
value = value.Trim('\r', '\t', ' ');
|
|
|
|
/* Write */
|
|
w.BeginSection("include");
|
|
w.WritePair("name", value);
|
|
w.EndSection();
|
|
}
|
|
|
|
private void PARSE_FuncEnum(ParseWriter w)
|
|
{
|
|
w.BeginSection("funcenum");
|
|
|
|
/* Get the functag name */
|
|
NeedToken(LexToken.TOKEN_SYMBOL);
|
|
w.WritePair("name", LexString);
|
|
w.funcenumList.Add(LexString);
|
|
|
|
NeedChar('{');
|
|
|
|
bool need_closebrace = true;
|
|
do
|
|
{
|
|
/* Shortcut out? */
|
|
if (MatchChar('}'))
|
|
{
|
|
need_closebrace = false;
|
|
break;
|
|
}
|
|
|
|
/* Start function section */
|
|
w.BeginSection("function");
|
|
|
|
if (MatchToken(LexToken.TOKEN_DOCBLOCK))
|
|
{
|
|
w.WritePair("doc", LexString);
|
|
}
|
|
|
|
/* Get the return tag */
|
|
if (MatchToken(LexToken.TOKEN_LABEL))
|
|
{
|
|
w.WritePair("return", LexString);
|
|
}
|
|
|
|
/* Get function type */
|
|
if (MatchToken(LexToken.TOKEN_PUBLIC))
|
|
{
|
|
w.WritePair("type", "public");
|
|
}
|
|
else if (MatchToken(LexToken.TOKEN_STOCK))
|
|
{
|
|
w.WritePair("type", "stock");
|
|
}
|
|
|
|
/* Parse the parameters */
|
|
ParseParameters(w);
|
|
|
|
/* End the section */
|
|
w.EndSection();
|
|
|
|
if (!MatchChar(','))
|
|
{
|
|
break;
|
|
}
|
|
} while (true);
|
|
|
|
if (need_closebrace)
|
|
{
|
|
NeedChar('}');
|
|
}
|
|
NeedChar(';');
|
|
|
|
w.EndSection();
|
|
}
|
|
|
|
private void PARSE_FuncTag(ParseWriter w)
|
|
{
|
|
w.BeginSection("functag");
|
|
|
|
/* Get the functag name */
|
|
NeedToken(LexToken.TOKEN_SYMBOL);
|
|
w.WritePair("name", LexString);
|
|
w.functagList.Add(LexString);
|
|
|
|
if (MatchToken(LexToken.TOKEN_LABEL))
|
|
{
|
|
w.WritePair("return", LexString);
|
|
}
|
|
|
|
if (MatchToken(LexToken.TOKEN_PUBLIC))
|
|
{
|
|
w.WritePair("type", "public");
|
|
}
|
|
else if (MatchToken(LexToken.TOKEN_STOCK))
|
|
{
|
|
w.WritePair("type", "stock");
|
|
}
|
|
|
|
ParseParameters(w);
|
|
NeedChar(';');
|
|
|
|
w.EndSection();
|
|
}
|
|
|
|
private void PARSE_Function(LexToken tok, ParseWriter w)
|
|
{
|
|
string tag="", name;
|
|
|
|
/* Get the return value */
|
|
if (MatchToken(LexToken.TOKEN_LABEL))
|
|
{
|
|
tag = LexString;
|
|
}
|
|
|
|
if (MatchToken(LexToken.TOKEN_OPERATOR))
|
|
{
|
|
/* Check if we're some sort of invalid function */
|
|
DiscardUntilCharOrComment('(', false);
|
|
ParseParameters(null);
|
|
if (tok == LexToken.TOKEN_STOCK)
|
|
{
|
|
ignore_block();
|
|
}
|
|
else if (tok == LexToken.TOKEN_NATIVE || tok == LexToken.TOKEN_FORWARD)
|
|
{
|
|
if (MatchChar('='))
|
|
{
|
|
DiscardUntilCharOrComment(';', false);
|
|
}
|
|
NeedChar(';');
|
|
}
|
|
return;
|
|
}
|
|
|
|
/* Get the name */
|
|
NeedToken(LexToken.TOKEN_SYMBOL);
|
|
name = LexString;
|
|
|
|
PARSE_Function(tok, w, tag, name);
|
|
}
|
|
|
|
private void PARSE_Function(LexToken tok, ParseWriter w, string tag, string name)
|
|
{
|
|
if (tok == LexToken.TOKEN_FORWARD)
|
|
{
|
|
w.BeginSection("forward");
|
|
w.forwardList.Add(name);
|
|
}
|
|
else if (tok == LexToken.TOKEN_NATIVE)
|
|
{
|
|
w.BeginSection("native");
|
|
w.nativeList.Add(name);
|
|
}
|
|
else if (tok == LexToken.TOKEN_STOCK)
|
|
{
|
|
w.BeginSection("stock");
|
|
w.stockList.Add(name);
|
|
}
|
|
|
|
w.WritePair("name", name);
|
|
|
|
if (tag.Length > 0)
|
|
{
|
|
w.WritePair("return", tag);
|
|
}
|
|
|
|
ParseParameters(w);
|
|
|
|
if (tok == LexToken.TOKEN_STOCK)
|
|
{
|
|
ignore_block();
|
|
}
|
|
else
|
|
{
|
|
/* Make sure there's a semicolon */
|
|
if (MatchChar('='))
|
|
{
|
|
DiscardUntilCharOrComment(';', false);
|
|
}
|
|
NeedChar(';');
|
|
}
|
|
|
|
w.EndSection();
|
|
}
|
|
|
|
private void ParseParameters(ParseWriter w)
|
|
{
|
|
NeedChar('(');
|
|
string extra;
|
|
do
|
|
{
|
|
if (MatchChar(')'))
|
|
{
|
|
return;
|
|
}
|
|
|
|
/* Start the parameter */
|
|
if (w != null)
|
|
{
|
|
w.BeginSection("parameter");
|
|
}
|
|
|
|
/* Check for a 'const' token */
|
|
if (MatchToken(LexToken.TOKEN_CONST) && w != null)
|
|
{
|
|
w.WritePair("const", "true");
|
|
}
|
|
|
|
/* Check if it's by reference */
|
|
if (MatchChar('&') && w != null)
|
|
{
|
|
w.WritePair("byref", "true");
|
|
}
|
|
|
|
/* Check for a tag */
|
|
if (MatchToken(LexToken.TOKEN_LABEL) && w != null)
|
|
{
|
|
w.WritePair("tag", LexString);
|
|
}
|
|
|
|
/* Get the symbol and write it */
|
|
if (MatchString(0, "...") && w != null)
|
|
{
|
|
w.WritePair("name", "...");
|
|
}
|
|
else
|
|
{
|
|
NeedToken(LexToken.TOKEN_SYMBOL);
|
|
if (w != null)
|
|
{
|
|
w.WritePair("name", LexString);
|
|
}
|
|
}
|
|
|
|
/* Check if we have a default value yet */
|
|
bool default_value = false;
|
|
if (MatchChar('='))
|
|
{
|
|
default_value = true;
|
|
}
|
|
else if (MatchChar('['))
|
|
{
|
|
extra = "";
|
|
string temp;
|
|
do
|
|
{
|
|
extra += "[";
|
|
temp = ConsumeUntilChar(']');
|
|
if (temp != null)
|
|
{
|
|
extra += temp;
|
|
}
|
|
extra += "]";
|
|
NeedChar(']');
|
|
} while (MatchChar('['));
|
|
w.WritePair("decoration", extra);
|
|
}
|
|
|
|
/* If we have a default value, get it */
|
|
if (default_value || MatchChar('='))
|
|
{
|
|
if ((extra = ConsumeParamDefValue()) == null)
|
|
{
|
|
throw new ParseException("Expected a default value; found none");
|
|
}
|
|
if (w != null)
|
|
{
|
|
w.WritePair("defval", extra);
|
|
}
|
|
}
|
|
|
|
/* End the parameter */
|
|
if (w != null)
|
|
{
|
|
w.EndSection();
|
|
}
|
|
|
|
if (!MatchChar(','))
|
|
{
|
|
break;
|
|
}
|
|
} while (true);
|
|
|
|
NeedChar(')');
|
|
}
|
|
|
|
private void PARSE_Enum(ParseWriter w)
|
|
{
|
|
string name = "";
|
|
if (MatchToken(LexToken.TOKEN_SYMBOL))
|
|
{
|
|
name = LexString;
|
|
}
|
|
|
|
w.BeginSection("enum");
|
|
w.WritePair("name", name);
|
|
w.BeginSection("properties");
|
|
|
|
w.enumList.Add(name);
|
|
|
|
NeedChar('{');
|
|
bool more_entries = true;
|
|
bool need_closebrace = true;
|
|
|
|
if (MatchToken(LexToken.TOKEN_DOCBLOCK))
|
|
{
|
|
w.WritePair("doc", LexString);
|
|
}
|
|
|
|
do
|
|
{
|
|
if (MatchChar('}'))
|
|
{
|
|
need_closebrace = false;
|
|
break;
|
|
}
|
|
NeedToken(LexToken.TOKEN_SYMBOL);
|
|
name = LexString;
|
|
|
|
if (MatchChar('='))
|
|
{
|
|
DiscardUntilCharOrComment(",\n}", true);
|
|
}
|
|
more_entries = MatchChar(',');
|
|
w.WritePair("name", name);
|
|
w.enumTypeList.Add(name);
|
|
if (MatchToken(LexToken.TOKEN_DOCBLOCK))
|
|
{
|
|
w.WritePair("doc", LexString);
|
|
}
|
|
} while (more_entries);
|
|
if (need_closebrace)
|
|
{
|
|
NeedChar('}');
|
|
}
|
|
NeedChar(';');
|
|
w.EndSection();
|
|
w.EndSection();
|
|
}
|
|
|
|
private void PARSE_Define(ParseWriter w)
|
|
{
|
|
/* first, get the symbol name */
|
|
NeedToken(LexToken.TOKEN_SYMBOL);
|
|
string name = LexString;
|
|
string value;
|
|
|
|
/* read until we hit the end of a line OR a comment */
|
|
int endpoint = -1;
|
|
for (int i = 0; i < Contents.Length; i++)
|
|
{
|
|
if (Contents[i] == '\n')
|
|
{
|
|
endpoint = i;
|
|
break;
|
|
}
|
|
else if (CheckString(i, "/**"))
|
|
{
|
|
endpoint = i;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (endpoint < 1)
|
|
{
|
|
value = "";
|
|
}
|
|
else
|
|
{
|
|
value = Contents.ToString().Substring(0, endpoint);
|
|
Contents.Remove(0, endpoint);
|
|
}
|
|
|
|
/* Write */
|
|
w.defineList.Add(name);
|
|
|
|
w.BeginSection("define");
|
|
w.WritePair("name", name);
|
|
w.WritePair("linetext", value);
|
|
w.EndSection();
|
|
}
|
|
|
|
/* Reads up to a character and discards all text before it.
|
|
*/
|
|
private void DiscardUntilChar(char c)
|
|
{
|
|
int eol = Contents.ToString().IndexOf(c);
|
|
|
|
if (eol < 1)
|
|
{
|
|
return;
|
|
}
|
|
|
|
Contents.Remove(0, eol);
|
|
}
|
|
|
|
/* Same as DiscardUntilChar, except the discarded
|
|
* text is returned first.
|
|
*/
|
|
private string ConsumeUntilChar(char c)
|
|
{
|
|
int eol = Contents.ToString().IndexOf(c);
|
|
|
|
if (eol < 1)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
string rest = Contents.ToString().Substring(0, eol);
|
|
Contents.Remove(0, eol);
|
|
return rest;
|
|
}
|
|
|
|
/**
|
|
* Returns a parameter's default value.
|
|
*/
|
|
private string ConsumeParamDefValue()
|
|
{
|
|
int eol = FindEndOfExprLine(0);
|
|
if (eol < 1)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
string rest = Contents.ToString().Substring(0, eol);
|
|
Contents.Remove(0, eol);
|
|
return rest;
|
|
}
|
|
|
|
/**
|
|
* Discards all data up to a certain character, ignoring
|
|
* the contents, unless specified to stop at a doc block comment.
|
|
*/
|
|
private void DiscardUntilCharOrComment(char c, bool stop_at_comment)
|
|
{
|
|
for (int i = 0; i < Contents.Length; i++)
|
|
{
|
|
if (Contents[i] == '\n')
|
|
{
|
|
LineNo++;
|
|
}
|
|
else if (Contents[i] == c)
|
|
{
|
|
if (i > 0)
|
|
{
|
|
Contents.Remove(0, i);
|
|
}
|
|
return;
|
|
}
|
|
else if (stop_at_comment && CheckString(i, "/**"))
|
|
{
|
|
if (i > 0)
|
|
{
|
|
Contents.Remove(0, i);
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Discards all data up to a certain character, ignoring
|
|
* the contents, unless specified to stop at a doc block comment.
|
|
*/
|
|
private void DiscardUntilCharOrComment(string s, bool stop_at_comment)
|
|
{
|
|
for (int i = 0; i < Contents.Length; i++)
|
|
{
|
|
if (Contents[i] == '\n')
|
|
{
|
|
LineNo++;
|
|
}
|
|
else if (s.Contains(Contents[i].ToString()))
|
|
{
|
|
if (i > 0)
|
|
{
|
|
Contents.Remove(0, i);
|
|
}
|
|
return;
|
|
}
|
|
else if (stop_at_comment && CheckString(i, "/**"))
|
|
{
|
|
if (i > 0)
|
|
{
|
|
Contents.Remove(0, i);
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Finds the end of an expression
|
|
*/
|
|
private int FindEndOfExprLine(int start)
|
|
{
|
|
for (int i = start; i < Contents.Length; i++)
|
|
{
|
|
if (Contents[i] == '{')
|
|
{
|
|
if ((i = FindEndOfConsumptionLine('}', start+1)) == -1)
|
|
{
|
|
throw new ParseException("Expected token: } (found none)");
|
|
}
|
|
}
|
|
else if (Contents[i] == ')' || Contents[i] == ',')
|
|
{
|
|
return i;
|
|
}
|
|
}
|
|
|
|
return -1;
|
|
}
|
|
|
|
private int FindEndOfConsumptionLine(char c, int start)
|
|
{
|
|
for (int i = start; i < Contents.Length; i++)
|
|
{
|
|
if (Contents[i] == '{')
|
|
{
|
|
if ((i = FindEndOfConsumptionLine('}', i + 1)) == -1)
|
|
{
|
|
throw new ParseException("Expected token: } (found none)");
|
|
}
|
|
}
|
|
else if (Contents[i] == c)
|
|
{
|
|
return i;
|
|
}
|
|
}
|
|
|
|
return -1;
|
|
}
|
|
|
|
private void Initialize()
|
|
{
|
|
/* Open file, read it, and close it */
|
|
StreamReader sr = File.OpenText(FileName);
|
|
Contents = new StringBuilder(sr.ReadToEnd());
|
|
sr.Close();
|
|
|
|
/* Strip comments (may throw an exception) */
|
|
preprocess();
|
|
|
|
/* Lastly, initialize the tokenizer */
|
|
Tokenizer.Initialize();
|
|
}
|
|
|
|
/* Consumes a token iff it matches */
|
|
private bool MatchToken(LexToken token)
|
|
{
|
|
LexToken tok = lex();
|
|
if (tok != token)
|
|
{
|
|
lexpush();
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/* Consumes a token and aborts the parser if there is no match */
|
|
private void NeedToken(LexToken token)
|
|
{
|
|
if (!MatchToken(token))
|
|
{
|
|
throw new ParseException("Expected token: " + token);
|
|
}
|
|
}
|
|
|
|
/* Consumes a character iff it matches */
|
|
private bool MatchChar(char c)
|
|
{
|
|
if (lex() != LexToken.TOKEN_CHARACTER)
|
|
{
|
|
lexpush();
|
|
return false;
|
|
}
|
|
if (_LexChar != c)
|
|
{
|
|
lexpush();
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/* Consumes a character or aborts the parser if there is no match */
|
|
private void NeedChar(char c)
|
|
{
|
|
if (!MatchChar(c))
|
|
{
|
|
throw new ParseException("Expected character: " + c);
|
|
}
|
|
}
|
|
|
|
/* Saves the lexer state so we don't reparse unnecessarily */
|
|
private void lexpush()
|
|
{
|
|
if (LexPushed)
|
|
{
|
|
throw new ParseException("Lexer cannot be pushed twice");
|
|
}
|
|
LexPushed = true;
|
|
}
|
|
|
|
/* Attempts to match a string against the next stream tokens.
|
|
* On success, the token stream is advanced.
|
|
*/
|
|
private bool MatchString(int startPos, string str)
|
|
{
|
|
if (str.Length > Contents.Length - startPos)
|
|
{
|
|
return false;
|
|
}
|
|
for (int i = 0; i < str.Length; i++)
|
|
{
|
|
if (str[i] != Contents[startPos + i])
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
Contents.Remove(startPos, str.Length);
|
|
return true;
|
|
}
|
|
|
|
/* Attempts to match a string against the next stream tokens */
|
|
private bool CheckString(int startPos, string str)
|
|
{
|
|
if (str.Length > Contents.Length - startPos)
|
|
{
|
|
return false;
|
|
}
|
|
for (int i = 0; i < str.Length; i++)
|
|
{
|
|
if (str[i] != Contents[startPos + i])
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/* Ignores a block starting and ending with { and } respectively */
|
|
private void ignore_block()
|
|
{
|
|
/* Hrm, this is tricky. First, expect an open brace. */
|
|
NeedChar('{');
|
|
/* Consume the entire function by keeping track of braces */
|
|
int numBraces = 1;
|
|
bool in_string = false;
|
|
bool found_end = false;
|
|
for (int i = 0; i < Contents.Length; i++)
|
|
{
|
|
if (Contents[i] == '\n')
|
|
{
|
|
LineNo++;
|
|
}
|
|
if (!in_string)
|
|
{
|
|
if (Contents[i] == '"')
|
|
{
|
|
in_string = true;
|
|
}
|
|
else if (Contents[i] == '{')
|
|
{
|
|
numBraces++;
|
|
}
|
|
else if (Contents[i] == '}')
|
|
{
|
|
if (numBraces == 0)
|
|
{
|
|
throw new ParseException("Unable to find a matching open brace");
|
|
}
|
|
if (--numBraces == 0)
|
|
{
|
|
Contents.Remove(0, i + 1);
|
|
found_end = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
/* It is possible to be in a string and at the first character */
|
|
if (Contents[i] == '"' && Contents[i - 1] != '\\')
|
|
{
|
|
in_string = false;
|
|
}
|
|
}
|
|
}
|
|
if (!found_end)
|
|
{
|
|
throw new ParseException("Unable to find a matching close brace");
|
|
}
|
|
}
|
|
|
|
/* Strips all non-doc C and C++ style comments. as well as unused pragmas */
|
|
private void preprocess()
|
|
{
|
|
if (Contents.Length == 1)
|
|
{
|
|
return;
|
|
}
|
|
|
|
bool in_comment = false;
|
|
bool is_ml = false;
|
|
bool is_doc = false;
|
|
bool is_newline = true;
|
|
|
|
for (int i=0; i<Contents.Length; i++)
|
|
{
|
|
if (Contents[i] == '\n')
|
|
{
|
|
is_newline = true;
|
|
}
|
|
if (!in_comment)
|
|
{
|
|
if (Contents[i] == '/')
|
|
{
|
|
if (Contents[i+1] == '/')
|
|
{
|
|
in_comment = true;
|
|
is_ml = false;
|
|
is_doc = false;
|
|
is_newline = false;
|
|
} else if (Contents[i+1] == '*') {
|
|
/* Detect whether this is a doc comment or not */
|
|
if (Contents.Length - i >= 3
|
|
&& Contents[i+2] == '*')
|
|
{
|
|
is_doc = true;
|
|
} else {
|
|
is_doc = false;
|
|
}
|
|
in_comment = true;
|
|
is_ml = true;
|
|
}
|
|
if (in_comment && !is_doc)
|
|
{
|
|
Contents[i] = ' ';
|
|
Contents[i+1] = ' '; /* Just for safety */
|
|
}
|
|
}
|
|
else if (is_newline)
|
|
{
|
|
if (Contents[i] == '#')
|
|
{
|
|
if (!CheckString(i, "#define")
|
|
&& !CheckString(i, "#include"))
|
|
{
|
|
/* Ignore! */
|
|
Contents[i] = ' ';
|
|
in_comment = true;
|
|
is_doc = false;
|
|
is_ml = false;
|
|
}
|
|
is_newline = false;
|
|
}
|
|
else if (!Char.IsWhiteSpace(Contents[i]))
|
|
{
|
|
is_newline = false;
|
|
}
|
|
}
|
|
} else {
|
|
if (is_ml)
|
|
{
|
|
if (Contents[i] == '*' && Contents[i+1] == '/')
|
|
{
|
|
if (!is_doc)
|
|
{
|
|
Contents[i] = ' ';
|
|
Contents[i+1] = ' ';
|
|
}
|
|
in_comment = false;
|
|
i++; /* Don't look at the next character */
|
|
}
|
|
else if (!is_doc && Contents[i] != '\n')
|
|
{
|
|
/* Erase comment */
|
|
Contents[i] = ' ';
|
|
}
|
|
} else {
|
|
if (is_newline)
|
|
{
|
|
in_comment = false;
|
|
}
|
|
else
|
|
{
|
|
/* Erase comment */
|
|
Contents[i] = ' ';
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (in_comment)
|
|
{
|
|
throw new ParseException("Multi-line comment has no terminator");
|
|
}
|
|
}
|
|
|
|
/* Strips whitespace */
|
|
private void trim_whitespace_pre()
|
|
{
|
|
int spaces = 0;
|
|
while (spaces < Contents.Length
|
|
&& Contents[spaces] != '\0'
|
|
&& Char.IsWhiteSpace(Contents[spaces]))
|
|
{
|
|
if (Contents[spaces] == '\n')
|
|
{
|
|
LineNo++;
|
|
}
|
|
spaces++;
|
|
}
|
|
if (spaces > 0)
|
|
{
|
|
Contents.Remove(0, spaces);
|
|
}
|
|
}
|
|
|
|
private void preprocess_line()
|
|
{
|
|
trim_whitespace_pre();
|
|
}
|
|
|
|
private bool IsSymbolChar(char c, bool first)
|
|
{
|
|
bool valid = (first ? Char.IsLetter(c) : Char.IsLetterOrDigit(c));
|
|
return (valid || (c == '_'));
|
|
}
|
|
|
|
private bool IsEscapedChar(char c)
|
|
{
|
|
return (c == 'n'
|
|
|| c == 'r'
|
|
|| c == 't'
|
|
|| c == 'v'
|
|
|| c == '\\'
|
|
|| c == '"');
|
|
}
|
|
|
|
private bool IsNotKeywordChar(char c)
|
|
{
|
|
return (Char.IsWhiteSpace(c)
|
|
|| (c == '-' || c == '+')
|
|
|| (c == '/' || c == '*')
|
|
|| (c == '(' || c == ')')
|
|
|| (c == '=' || c == '%')
|
|
|| (c == '>' || c == '<')
|
|
|| (c == '!' || c == '~')
|
|
);
|
|
}
|
|
|
|
/* Consumes one lexical token and gathers information about it */
|
|
private LexToken lex()
|
|
{
|
|
if (LexPushed)
|
|
{
|
|
LexPushed = false;
|
|
return _LastToken;
|
|
}
|
|
|
|
/* Number of chars we will be deleting from the input */
|
|
int stripchars = 0;
|
|
|
|
/* Clear our state */
|
|
_LastToken = LexToken.TOKEN_NONE;
|
|
|
|
/* Remove stuff we don't want from the line */
|
|
preprocess_line();
|
|
|
|
if (Contents.Length < 1)
|
|
{
|
|
_LastToken = LexToken.TOKEN_EOF;
|
|
return _LastToken;
|
|
}
|
|
|
|
/* Get the token list */
|
|
KeywordToken[] tokens = Tokenizer.Tokens;
|
|
for (int i = 0; i < tokens.Length; i++)
|
|
{
|
|
if (!CheckString(0, tokens[i].keyword))
|
|
{
|
|
continue;
|
|
}
|
|
int len = tokens[i].keyword.Length;
|
|
/* Now check to see what the next token is */
|
|
if (Contents.Length == len
|
|
|| IsNotKeywordChar(Contents[len]))
|
|
{
|
|
/* We have a token match! */
|
|
_LastToken = tokens[i].token;
|
|
_LexString = null;
|
|
_LexChar = '\0';
|
|
stripchars = len;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (_LastToken == LexToken.TOKEN_NONE)
|
|
{
|
|
/* See if we can try to read a symbol */
|
|
if (IsSymbolChar(Contents[0], true))
|
|
{
|
|
int characters = 1;
|
|
while (Contents[characters] != '\0'
|
|
&& IsSymbolChar(Contents[characters], false))
|
|
{
|
|
characters++;
|
|
}
|
|
stripchars = characters;
|
|
/* We're done! See what's next.. */
|
|
if (Contents[characters] == ':')
|
|
{
|
|
_LastToken = LexToken.TOKEN_LABEL;
|
|
stripchars++;
|
|
}
|
|
else
|
|
{
|
|
_LastToken = LexToken.TOKEN_SYMBOL;
|
|
}
|
|
_LexString = Contents.ToString().Substring(0, characters);
|
|
_LexChar = _LexString[0];
|
|
}/*
|
|
else if (Contents[0] == '"')
|
|
{
|
|
bool reached_end = false;
|
|
StringBuilder builder = new StringBuilder();
|
|
for (int i = 1; i < Contents.Length; i++)
|
|
{
|
|
if (i < Contents.Length - 1
|
|
&& Contents[i] == '\\'
|
|
&& IsEscapedChar(Contents[i+1]))
|
|
{
|
|
i++;
|
|
}
|
|
else if (Contents[i] == '"')
|
|
{
|
|
reached_end = true;
|
|
break;
|
|
}
|
|
builder.Append(Contents[i]);
|
|
}
|
|
if (!reached_end)
|
|
{
|
|
throw new ParseException("Expected end of string; none found");
|
|
}
|
|
_LexString = builder.ToString();
|
|
_LexChar = _LexString.Length > 0 ? _LexString[0] : '\0';
|
|
}*/
|
|
else if (Contents[0] == '\'')
|
|
{
|
|
char c = '\0';
|
|
if (Contents.Length < 3)
|
|
{
|
|
throw new ParseException("Expected end of character; none found");
|
|
}
|
|
else if (Contents[1] == '\\')
|
|
{
|
|
if (Contents.Length < 4 || !IsEscapedChar(Contents[2]))
|
|
{
|
|
throw new ParseException("Expected end of character; none found");
|
|
}
|
|
if (Contents[2] == 'r')
|
|
{
|
|
c = '\r';
|
|
}
|
|
else if (Contents[2] == 'n')
|
|
{
|
|
c = '\n';
|
|
}
|
|
else if (Contents[2] == 't')
|
|
{
|
|
c = '\t';
|
|
}
|
|
else if (Contents[2] == 'v')
|
|
{
|
|
c = '\v';
|
|
}
|
|
else if (Contents[2] == '\\')
|
|
{
|
|
c = '\\';
|
|
}
|
|
else if (Contents[2] == '"')
|
|
{
|
|
c = '"';
|
|
}
|
|
stripchars = 4;
|
|
}
|
|
else
|
|
{
|
|
c = Contents[1];
|
|
stripchars = 3;
|
|
}
|
|
_LexString = c.ToString();
|
|
_LexChar = c;
|
|
_LastToken = LexToken.TOKEN_QUOTCHAR;
|
|
}
|
|
else if (CheckString(0, "/**"))
|
|
{
|
|
int endpoint = 0;
|
|
for (int i=3; i<Contents.Length-1; i++)
|
|
{
|
|
if (Contents[i] == '\n')
|
|
{
|
|
LineNo++;
|
|
}
|
|
else if (Contents[i] == '*' && Contents[i+1] == '/')
|
|
{
|
|
endpoint = i+1;
|
|
break;
|
|
}
|
|
}
|
|
if (endpoint == 0)
|
|
{
|
|
throw new ParseException("Expected end of multi-line comment; found end of file");
|
|
}
|
|
/* We have to factor one more character in because we're zero based */
|
|
endpoint++;
|
|
_LastToken = LexToken.TOKEN_DOCBLOCK;
|
|
_LexString = Contents.ToString().Substring(0, endpoint);
|
|
_LexChar = _LexString[0];
|
|
stripchars = endpoint;
|
|
}
|
|
else
|
|
{
|
|
_LastToken = LexToken.TOKEN_CHARACTER;
|
|
_LexString = Contents[0].ToString();
|
|
_LexChar = Contents[0];
|
|
stripchars = 1;
|
|
}
|
|
}
|
|
|
|
/* Strip N chars */
|
|
if (stripchars > 0)
|
|
{
|
|
Contents.Remove(0, stripchars);
|
|
}
|
|
|
|
return _LastToken;
|
|
}
|
|
}
|
|
}
|