btparser/cparser/main.cpp

638 lines
20 KiB
C++
Raw Normal View History

#include <windows.h>
#include <stdio.h>
#include <string>
#include <stdint.h>
#include <unordered_map>
2016-06-05 07:01:28 +08:00
#include <functional>
2016-06-05 09:20:55 +08:00
#include <vector>
#include <tuple>
#include "filehelper.h"
#include "stringutils.h"
2016-06-05 07:01:28 +08:00
#include "testfiles.h"
#define MAKE_OP_TRIPLE(ch1, ch2, ch3) (ch3 << 16 | ch2 << 8 | ch1)
#define MAKE_OP_DOUBLE(ch1, ch2) (ch2 << 8 | ch1)
#define MAKE_OP_SINGLE(ch1) (ch1)
2016-06-05 20:55:46 +08:00
#define DEFAULT_STRING_BUFFER 65536
using namespace std;
2016-06-05 07:01:28 +08:00
struct Lexer
{
explicit Lexer()
{
SetupTokenMaps();
2016-06-05 07:01:28 +08:00
}
2016-06-05 07:01:28 +08:00
enum Token
{
//status tokens
tok_eof = -10000,
tok_error,
2016-06-05 07:01:28 +08:00
//keywords
2016-06-05 08:41:40 +08:00
#define DEF_KEYWORD(keyword) tok_##keyword,
#include "keywords.h"
#undef DEF_KEYWORD
2016-06-05 07:01:28 +08:00
//others
tok_identifier, //[a-zA-Z_][a-zA-Z0-9_]
2016-06-05 08:41:40 +08:00
tok_number, //(0x[0-9a-fA-F]+)|([0-9]+)
tok_stringlit, //"([^\\"]|\\([\\"'?abfnrtv0]|x[0-9a-fA-f]{2}))*"
tok_charlit, //'([^\\]|\\([\\"'?abfnrtv0]|x[0-9a-fA-f]{2}))'
//operators
#define DEF_OP_TRIPLE(enumval, ch1, ch2, ch3) tok_##enumval,
#define DEF_OP_DOUBLE(enumval, ch1, ch2) tok_##enumval,
#define DEF_OP_SINGLE(enumval, ch1) tok_##enumval,
#include "operators.h"
#undef DEF_OP_TRIPLE
#undef DEF_OP_DOUBLE
#undef DEF_OP_SINGLE
2016-06-05 07:01:28 +08:00
};
2016-06-05 09:20:55 +08:00
vector<uint8_t> Input;
2016-06-05 08:41:40 +08:00
size_t Index = 0;
string Error;
2016-06-05 09:20:55 +08:00
vector<String> Warnings;
2016-06-05 08:41:40 +08:00
//lexer state
string IdentifierStr;
uint64_t NumberVal = 0;
2016-06-05 21:46:44 +08:00
bool IsHexNumberVal = false;
2016-06-05 08:41:40 +08:00
string StringLit;
2016-06-05 20:55:46 +08:00
string NumStr;
char CharLit = '\0';
2016-06-05 08:41:40 +08:00
int LastChar = ' ';
int CurLine = 0;
2016-06-05 21:46:44 +08:00
int LineIndex = 0;
2016-06-05 08:41:40 +08:00
2016-06-05 20:55:46 +08:00
static void clearReserve(string & str, size_t reserve = DEFAULT_STRING_BUFFER)
{
str.clear();
str.reserve(reserve);
}
static void appendCh(string & str, char ch)
{
str.resize(str.size() + 1);
str[str.size() - 1] = ch;
}
2016-06-05 08:41:40 +08:00
void ResetLexerState()
{
Input.clear();
2016-06-05 20:55:46 +08:00
Input.reserve(1024 * 1024);
2016-06-05 08:41:40 +08:00
Index = 0;
Error.clear();
2016-06-05 20:55:46 +08:00
Warnings.clear();
clearReserve(IdentifierStr);
2016-06-05 08:41:40 +08:00
NumberVal = 0;
2016-06-05 21:46:44 +08:00
IsHexNumberVal = false;
2016-06-05 20:55:46 +08:00
clearReserve(StringLit);
clearReserve(NumStr, 16);
CharLit = '\0';
2016-06-05 08:41:40 +08:00
LastChar = ' ';
CurLine = 0;
2016-06-05 21:46:44 +08:00
LineIndex = 0;
2016-06-05 08:41:40 +08:00
}
2016-06-05 07:01:28 +08:00
unordered_map<string, Token> KeywordMap;
unordered_map<Token, string> ReverseTokenMap;
unordered_map<int, Token> OpTripleMap;
unordered_map<int, Token> OpDoubleMap;
unordered_map<int, Token> OpSingleMap;
2016-06-05 07:01:28 +08:00
void SetupTokenMaps()
{
//setup keyword map
2016-06-05 08:41:40 +08:00
#define DEF_KEYWORD(keyword) KeywordMap[#keyword] = tok_##keyword;
#include "keywords.h"
#undef DEF_KEYWORD
//setup token maps
#define DEF_OP_TRIPLE(enumval, ch1, ch2, ch3) OpTripleMap[MAKE_OP_TRIPLE(ch1, ch2, ch3)] = tok_##enumval;
#define DEF_OP_DOUBLE(enumval, ch1, ch2) OpDoubleMap[MAKE_OP_DOUBLE(ch1, ch2)] = tok_##enumval;
#define DEF_OP_SINGLE(enumval, ch1) OpSingleMap[MAKE_OP_SINGLE(ch1)] = tok_##enumval;
#include "operators.h"
#undef DEF_OP_TRIPLE
#undef DEF_OP_DOUBLE
#undef DEF_OP_SINGLE
//setup reverse token maps
#define DEF_KEYWORD(keyword) ReverseTokenMap[tok_##keyword] = #keyword;
2016-06-05 08:41:40 +08:00
#include "keywords.h"
#undef DEF_KEYWORD
#define DEF_OP_TRIPLE(enumval, ch1, ch2, ch3) ReverseTokenMap[tok_##enumval] = string({ch1, ch2, ch3});
#define DEF_OP_DOUBLE(enumval, ch1, ch2) ReverseTokenMap[tok_##enumval] = string({ch1, ch2});
#define DEF_OP_SINGLE(enumval, ch1) ReverseTokenMap[tok_##enumval] = string({ch1});
#include "operators.h"
#undef DEF_OP_TRIPLE
#undef DEF_OP_DOUBLE
#undef DEF_OP_SINGLE
2016-06-05 07:01:28 +08:00
}
Token ReportError(const String & error)
{
Error = error;
return tok_error;
}
2016-06-05 09:20:55 +08:00
void ReportWarning(const String & warning)
{
Warnings.push_back(warning);
}
2016-06-05 07:01:28 +08:00
String TokString(int tok)
{
switch (Token(tok))
{
2016-06-05 07:01:28 +08:00
case tok_eof: return "tok_eof";
2016-06-05 21:46:44 +08:00
case tok_error: return StringUtils::sprintf("error(line %d, col %d, \"%s\")", CurLine + 1, LineIndex, Error.c_str());
case tok_identifier: return IdentifierStr.c_str();
case tok_number: return StringUtils::sprintf(IsHexNumberVal ? "0x%llX" : "%llu", NumberVal);
case tok_stringlit: return StringUtils::sprintf("\"%s\"", StringUtils::Escape(StringLit).c_str());
case tok_charlit:
{
String s;
s = CharLit;
2016-06-05 21:46:44 +08:00
return StringUtils::sprintf("'%s'", StringUtils::Escape(s).c_str());
}
2016-06-05 07:01:28 +08:00
default:
2016-06-05 08:41:40 +08:00
{
auto found = ReverseTokenMap.find(Token(tok));
if (found != ReverseTokenMap.end())
2016-06-05 08:41:40 +08:00
return found->second;
2016-06-05 07:01:28 +08:00
return "<INVALID TOKEN>";
}
2016-06-05 08:41:40 +08:00
}
}
2016-06-05 07:01:28 +08:00
int PeekChar(int distance = 0)
{
2016-06-05 09:20:55 +08:00
if (Index + distance >= Input.size())
2016-06-05 07:01:28 +08:00
return EOF;
2016-06-05 09:20:55 +08:00
auto ch = Input[Index + distance];
if (ch == '\0')
{
ReportWarning(StringUtils::sprintf("\\0 character in file data"));
return PeekChar(distance + 1);
}
return ch;
2016-06-05 07:01:28 +08:00
}
2016-06-05 07:01:28 +08:00
int ReadChar()
{
2016-06-05 09:20:55 +08:00
if (Index == Input.size())
2016-06-05 07:01:28 +08:00
return EOF;
2016-06-05 09:20:55 +08:00
auto ch = Input[Index++];
2016-06-05 21:46:44 +08:00
LineIndex++;
2016-06-05 09:20:55 +08:00
if (ch == '\0')
{
ReportWarning(StringUtils::sprintf("\\0 character in file data"));
return ReadChar();
}
return ch;
2016-06-05 07:01:28 +08:00
}
2016-06-05 08:41:40 +08:00
bool CheckString(const string & expected)
{
for (size_t i = 0; i < expected.size(); i++)
{
auto ch = PeekChar(i);
if (ch == EOF)
return false;
if (ch != uint8_t(expected[i]))
return false;
}
Index += expected.size();
return true;
}
2016-06-05 20:55:46 +08:00
int NextChar()
2016-06-05 08:41:40 +08:00
{
2016-06-05 20:55:46 +08:00
return LastChar = ReadChar();
}
2016-06-05 21:46:44 +08:00
void SignalNextLine()
{
CurLine++;
LineIndex = 0;
}
2016-06-05 20:55:46 +08:00
static const char* ConvertNumber(const char* str, uint64_t & result, int radix)
{
errno = 0;
char* end;
result = strtoull(str, &end, radix);
if (!result && end == str)
return "not a number";
if (result == ULLONG_MAX && errno)
return "does not fit";
if (*end)
return "str not completely consumed";
return nullptr;
2016-06-05 08:41:40 +08:00
}
2016-06-05 07:01:28 +08:00
int GetToken()
{
2016-06-05 07:01:28 +08:00
//skip whitespace
while (isspace(LastChar))
{
if (LastChar == '\n')
2016-06-05 21:46:44 +08:00
SignalNextLine();
NextChar();
}
//skip \\[\r\n]
if (LastChar == '\\' && (PeekChar() == '\r' || PeekChar() == '\n'))
{
2016-06-05 08:41:40 +08:00
NextChar();
return GetToken();
}
//character literal
if (LastChar == '\'')
{
string charLit;
while (true)
{
NextChar();
if (LastChar == EOF) //end of file
return ReportError("unexpected end of file in character literal (1)");
if (LastChar == '\r' || LastChar == '\n')
return ReportError("unexpected newline in character literal (1)");
if (LastChar == '\'') //end of character literal
{
2016-06-05 21:46:44 +08:00
if (charLit.length() != 1)
return ReportError(StringUtils::sprintf("invalid character literal '%s'", charLit.c_str()));
CharLit = charLit[0];
NextChar();
return tok_charlit;
}
if (LastChar == '\\') //escape sequence
{
NextChar();
if (LastChar == EOF)
return ReportError("unexpected end of file in character literal (2)");
if (LastChar == '\r' || LastChar == '\n')
return ReportError("unexpected newline in character literal (2)");
if (LastChar == '\'' || LastChar == '\"' || LastChar == '?' || LastChar == '\\')
LastChar = LastChar;
else if (LastChar == 'a')
LastChar = '\a';
else if (LastChar == 'b')
LastChar = '\b';
else if (LastChar == 'f')
LastChar = '\f';
else if (LastChar == 'n')
LastChar = '\n';
else if (LastChar == 'r')
LastChar = '\r';
else if (LastChar == 't')
LastChar = '\t';
else if (LastChar == 'v')
LastChar = '\v';
else if (LastChar == '0')
LastChar = '\0';
else if (LastChar == 'x') //\xHH
{
auto ch1 = NextChar();
auto ch2 = NextChar();
if (isxdigit(ch1) && isxdigit(ch2))
{
char byteStr[3] = "";
byteStr[0] = ch1;
byteStr[1] = ch2;
2016-06-05 20:55:46 +08:00
uint64_t hexData;
auto error = ConvertNumber(byteStr, hexData, 16);
if (error)
return ReportError(StringUtils::sprintf("ConvertNumber failed (%s) for hex sequence \"\\x%c%c\" in character literal", error, ch1, ch2));
LastChar = hexData & 0xFF;
}
else
return ReportError(StringUtils::sprintf("invalid hex sequence \"\\x%c%c\" in character literal", ch1, ch2));
}
else
return ReportError(StringUtils::sprintf("invalid escape sequence \"\\%c\" in character literal", LastChar));
}
charLit += LastChar;
}
}
2016-06-05 08:41:40 +08:00
//string literal
if (LastChar == '\"')
{
StringLit.clear();
while (true)
{
NextChar();
if (LastChar == EOF) //end of file
2016-06-05 09:20:55 +08:00
return ReportError("unexpected end of file in string literal (1)");
if (LastChar == '\r' || LastChar == '\n')
return ReportError("unexpected newline in string literal (1)");
2016-06-05 08:41:40 +08:00
if (LastChar == '\"') //end of string literal
{
NextChar();
return tok_stringlit;
}
if (LastChar == '\\') //escape sequence
{
NextChar();
if (LastChar == EOF)
2016-06-05 09:20:55 +08:00
return ReportError("unexpected end of file in string literal (2)");
if (LastChar == '\r' || LastChar == '\n')
return ReportError("unexpected newline in string literal (2)");
2016-06-05 09:20:55 +08:00
if (LastChar == '\'' || LastChar == '\"' || LastChar == '?' || LastChar == '\\')
2016-06-05 08:41:40 +08:00
LastChar = LastChar;
else if (LastChar == 'a')
LastChar = '\a';
else if (LastChar == 'b')
LastChar = '\b';
else if (LastChar == 'f')
LastChar = '\f';
else if (LastChar == 'n')
LastChar = '\n';
else if (LastChar == 'r')
LastChar = '\r';
else if (LastChar == 't')
LastChar = '\t';
else if (LastChar == 'v')
LastChar = '\v';
else if (LastChar == '0')
LastChar = '\0';
2016-06-05 09:20:55 +08:00
else if (LastChar == 'x') //\xHH
{
auto ch1 = NextChar();
auto ch2 = NextChar();
if (isxdigit(ch1) && isxdigit(ch2))
{
char byteStr[3] = "";
byteStr[0] = ch1;
byteStr[1] = ch2;
2016-06-05 20:55:46 +08:00
uint64_t hexData;
auto error = ConvertNumber(byteStr, hexData, 16);
if (error)
return ReportError(StringUtils::sprintf("ConvertNumber failed (%s) for hex sequence \"\\x%c%c\" in string literal", error, ch1, ch2));
LastChar = hexData & 0xFF;
2016-06-05 09:20:55 +08:00
}
else
return ReportError(StringUtils::sprintf("invalid hex sequence \"\\x%c%c\" in string literal", ch1, ch2));
}
2016-06-05 08:41:40 +08:00
else
return ReportError(StringUtils::sprintf("invalid escape sequence \"\\%c\" in string literal", LastChar));
}
2016-06-05 20:55:46 +08:00
appendCh(StringLit, LastChar);
2016-06-05 08:41:40 +08:00
}
}
2016-06-05 07:01:28 +08:00
//identifier/keyword
if (isalpha(LastChar) || LastChar == '_') //[a-zA-Z_]
{
IdentifierStr = LastChar;
2016-06-05 08:41:40 +08:00
NextChar();
2016-06-05 07:01:28 +08:00
while (isalnum(LastChar) || LastChar == '_') //[0-9a-zA-Z_]
{
2016-06-05 20:55:46 +08:00
appendCh(IdentifierStr, LastChar);
2016-06-05 08:41:40 +08:00
NextChar();
2016-06-05 07:01:28 +08:00
}
2016-06-05 07:01:28 +08:00
//keywords
auto found = KeywordMap.find(IdentifierStr);
if (found != KeywordMap.end())
return found->second;
2016-06-05 07:01:28 +08:00
return tok_identifier;
}
2016-06-05 07:01:28 +08:00
//hex numbers
if (LastChar == '0' && PeekChar() == 'x') //0x
{
2016-06-05 21:46:44 +08:00
NextChar(); //consume the 'x'
2016-06-05 20:55:46 +08:00
NumStr.clear();
2016-06-05 07:01:28 +08:00
2016-06-05 08:41:40 +08:00
while (isxdigit(NextChar())) //[0-9a-fA-F]*
2016-06-05 20:55:46 +08:00
appendCh(NumStr, LastChar);
if (!NumStr.length()) //check for error condition
return ReportError("no hex digits after \"0x\" prefix");
2016-06-05 20:55:46 +08:00
auto error = ConvertNumber(NumStr.c_str(), NumberVal, 16);
if (error)
return ReportError(StringUtils::sprintf("ConvertNumber failed (%s) on hexadecimal number", error));
2016-06-05 21:46:44 +08:00
IsHexNumberVal = true;
return tok_number;
}
2016-06-05 07:01:28 +08:00
if (isdigit(LastChar)) //[0-9]
{
2016-06-05 07:01:28 +08:00
NumStr = LastChar;
2016-06-05 08:41:40 +08:00
while (isdigit(NextChar())) //[0-9]*
2016-06-05 07:01:28 +08:00
NumStr += LastChar;
2016-06-05 20:55:46 +08:00
auto error = ConvertNumber(NumStr.c_str(), NumberVal, 10);
if (error)
return ReportError(StringUtils::sprintf("ConvertNumber failed (%s) on decimal number", error));
2016-06-05 21:46:44 +08:00
IsHexNumberVal = false;
2016-06-05 07:01:28 +08:00
return tok_number;
}
2016-06-05 07:01:28 +08:00
//comments
if (LastChar == '/' && PeekChar() == '/') //line comment
{
do
{
if (LastChar == '\n')
2016-06-05 21:46:44 +08:00
SignalNextLine();
NextChar();
} while (!(LastChar == EOF || LastChar == '\n'));
return GetToken(); //interpret the next line
2016-06-05 07:01:28 +08:00
}
if (LastChar == '/' && PeekChar() == '*') //block comment
2016-06-05 07:01:28 +08:00
{
do
{
if (LastChar == '\n')
2016-06-05 21:46:44 +08:00
SignalNextLine();
NextChar();
} while (!(LastChar == EOF || LastChar == '*' && PeekChar() == '/'));
if (LastChar == EOF) //unexpected end of file
2016-06-05 21:46:44 +08:00
{
LineIndex++;
return ReportError("unexpected end of file in block comment");
2016-06-05 21:46:44 +08:00
}
2016-06-05 20:55:46 +08:00
NextChar();
NextChar();
return GetToken(); //get the next non-comment token
}
//operators
auto opFound = OpTripleMap.find(MAKE_OP_TRIPLE(LastChar, PeekChar(), PeekChar(1)));
if (opFound != OpTripleMap.end())
{
2016-06-05 20:55:46 +08:00
NextChar();
NextChar();
NextChar();
return opFound->second;
}
opFound = OpDoubleMap.find(MAKE_OP_DOUBLE(LastChar, PeekChar()));
if (opFound != OpDoubleMap.end())
{
2016-06-05 20:55:46 +08:00
NextChar();
NextChar();
return opFound->second;
}
opFound = OpSingleMap.find(MAKE_OP_SINGLE(LastChar));
if (opFound != OpSingleMap.end())
{
2016-06-05 20:55:46 +08:00
NextChar();
return opFound->second;
}
2016-06-05 07:01:28 +08:00
//end of file
if (LastChar == EOF)
return tok_eof;
//unknown character
2016-06-05 20:55:46 +08:00
return ReportError(StringUtils::sprintf("unexpected character \'%c\'", LastChar));
}
2016-06-05 07:01:28 +08:00
bool ReadInputFile(const string & filename)
{
2016-06-05 08:41:40 +08:00
ResetLexerState();
2016-06-05 09:20:55 +08:00
return FileHelper::ReadAllData(filename, Input);
2016-06-05 07:01:28 +08:00
}
2016-06-05 20:55:46 +08:00
bool TestLex(function<void(const string & line)> lexEnum, bool output = true)
2016-06-05 07:01:28 +08:00
{
auto line = 0;
2016-06-05 20:55:46 +08:00
if (output)
lexEnum("1: ");
2016-06-05 07:01:28 +08:00
int tok;
2016-06-05 20:55:46 +08:00
string toks;
clearReserve(toks);
char newlineText[128] = "";
2016-06-05 07:01:28 +08:00
do
{
tok = GetToken();
2016-06-05 20:55:46 +08:00
if (!output)
continue;
toks.clear();
while (line < CurLine)
{
line++;
2016-06-05 20:55:46 +08:00
sprintf_s(newlineText, "\n%d: ", line + 1);
toks.append(newlineText);
}
2016-06-05 20:55:46 +08:00
toks.append(TokString(tok));
appendCh(toks, ' ');
lexEnum(toks);
2016-06-05 07:01:28 +08:00
} while (tok != tok_eof && tok != tok_error);
2016-06-05 08:41:40 +08:00
if (tok != tok_error && tok != tok_eof)
tok = ReportError("lexer did not finish at the end of the file");
2016-06-05 09:20:55 +08:00
for (const auto & warning : Warnings)
2016-06-05 20:55:46 +08:00
if (output)
lexEnum("\nwarning: " + warning);
2016-06-05 08:41:40 +08:00
return tok != tok_error;
2016-06-05 07:01:28 +08:00
}
};
2016-06-05 20:55:46 +08:00
bool TestLexer(Lexer & lexer, const string & filename)
{
2016-06-05 07:01:28 +08:00
if (!lexer.ReadInputFile("tests\\" + filename))
{
printf("failed to read \"%s\"\n", filename.c_str());
return false;
}
string actual;
2016-06-05 20:55:46 +08:00
Lexer::clearReserve(actual);
auto success = lexer.TestLex([&](const string & line)
2016-06-05 08:41:40 +08:00
{
2016-06-05 20:55:46 +08:00
actual.append(line);
});
2016-06-05 08:41:40 +08:00
string expected;
2016-06-05 20:55:46 +08:00
if (FileHelper::ReadAllText("tests\\exp_lex\\" + filename, expected))
2016-06-05 07:01:28 +08:00
{
2016-06-05 20:55:46 +08:00
if (expected == actual)
{
printf("lexer test for \"%s\" success!\n", filename.c_str());
return true;
}
2016-06-05 07:01:28 +08:00
}
2016-06-05 20:55:46 +08:00
if (success)
return true;
printf("lexer test for \"%s\" failed...\n", filename.c_str());
2016-06-05 07:01:28 +08:00
FileHelper::WriteAllText("expected.out", expected);
FileHelper::WriteAllText("actual.out", actual);
return false;
}
2016-06-05 20:55:46 +08:00
bool DebugLexer(Lexer & lexer, const string & filename, bool output)
{
2016-06-05 07:01:28 +08:00
if (!lexer.ReadInputFile("tests\\" + filename))
{
2016-06-05 07:01:28 +08:00
printf("failed to read \"%s\"\n", filename.c_str());
return false;
}
2016-06-05 20:55:46 +08:00
auto success = lexer.TestLex([](const string & line)
2016-06-05 07:01:28 +08:00
{
printf("%s", line.c_str());
2016-06-05 20:55:46 +08:00
}, output);
if (output)
puts("");
return success;
}
2016-06-05 20:55:46 +08:00
void GenerateExpected(Lexer & lexer, const string & filename)
2016-06-05 19:00:55 +08:00
{
if (!lexer.ReadInputFile("tests\\" + filename))
{
printf("failed to read \"%s\"\n", filename.c_str());
return;
}
string actual;
2016-06-05 20:55:46 +08:00
Lexer::clearReserve(actual);
lexer.TestLex([&](const string & line)
2016-06-05 19:00:55 +08:00
{
2016-06-05 20:55:46 +08:00
actual.append(line);
});
FileHelper::WriteAllText("tests\\exp_lex\\" + filename, actual);
}
void GenerateExpectedTests()
{
Lexer lexer;
for (auto file : testFiles)
GenerateExpected(lexer, file);
}
void RunLexerTests()
{
Lexer lexer;
for (auto file : testFiles)
TestLexer(lexer, file);
}
void DebugLexerTests(bool output = true)
{
Lexer lexer;
for (auto file : testFiles)
DebugLexer(lexer, file, output);
2016-06-05 19:00:55 +08:00
}
int main()
{
2016-06-05 20:55:46 +08:00
//GenerateExpectedTests();
auto ticks = GetTickCount();
//Lexer lexer;
//DebugLexer(lexer, "AndroidManifestTemplate.bt", false);
2016-06-05 21:03:38 +08:00
RunLexerTests();
2016-06-05 20:55:46 +08:00
printf("finished in %ums\n", GetTickCount() - ticks);
system("pause");
return 0;
}