support for more keywords

cparser/cparser.vcxproj

@@ -16,7 +16,11 @@
     <ClCompile Include="stringutils.cpp" />
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="dynamicmem.h" />
     <ClInclude Include="filehelper.h" />
+    <ClInclude Include="handle.h" />
+    <ClInclude Include="stringutils.h" />
+    <ClInclude Include="testfiles.h" />
   </ItemGroup>
   <PropertyGroup Label="Globals">
     <ProjectGuid>{B0411C78-2F06-49E0-8DE9-5C52A466F5DE}</ProjectGuid>

cparser/cparser.vcxproj.filters

@@ -29,5 +29,17 @@
     <ClInclude Include="filehelper.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="testfiles.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="dynamicmem.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="handle.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="stringutils.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

cparser/main.cpp

@@ -3,127 +3,158 @@
 #include <string>
 #include <stdint.h>
 #include <unordered_map>
+#include <functional>
 #include "filehelper.h"
 #include "stringutils.h"
+#include "testfiles.h"
 
 using namespace std;
 
-string Input;
+struct Lexer
-string ConsumedInput;
-size_t Index = 0;
-string Error;
-
-string IdentifierStr = "";
-uint64_t NumberVal = 0;
-
-int LastChar = ' ';
-
-enum Token
 {
-    //status tokens
+    explicit Lexer()
-    tok_eof = -10000,
-    tok_error,
-
-    //keywords
-    tok_typedef, //"typedef"
-    tok_struct, //"struct"
-    tok_char, //"char"
-    tok_unsigned, //"unsigned"
-    tok_int, //"int"
-    tok_sizeof, //"sizeof"
-    tok_WORD, //"WORD"
-    tok_DWORD, //"DWORD"
-
-    //others
-    tok_identifier, //[a-zA-Z][a-zA-Z0-9]
-    tok_number //(0x[0-9a-fA-F]+)|([0-9]+)
-};
-
-unordered_map<string, Token> KeywordMap;
-
-void setup()
-{
-    KeywordMap["typedef"] = tok_typedef;
-    KeywordMap["struct"] = tok_struct;
-    KeywordMap["char"] = tok_char;
-    KeywordMap["unsigned"] = tok_unsigned;
-    KeywordMap["int"] = tok_int;
-    KeywordMap["sizeof"] = tok_sizeof;
-    KeywordMap["WORD"] = tok_WORD;
-    KeywordMap["DWORD"] = tok_DWORD;
-}
-
-Token ReportError(const String & error)
-{
-    Error = error;
-    return tok_error;
-}
-
-String tokString(int tok)
-{
-    switch (Token(tok))
     {
-    case tok_eof: return "tok_eof";
+        SetupKeywordMap();
-    case tok_error: return StringUtils::sprintf("tok_error \"%s\"", Error.c_str());
-    case tok_identifier: return StringUtils::sprintf("tok_identifier \"%s\"", IdentifierStr.c_str());
-    case tok_number: return StringUtils::sprintf("tok_number %llu (0x%llX)", NumberVal, NumberVal);
-    default:
-        for (const auto & itr : KeywordMap)
-        {
-            if (tok == itr.second)
-                return "tok_" + itr.first;
-        }
-        if (tok > 0 && tok < 265)
-        {
-            String s;
-            s = tok;
-            return s;
-        }
-        return "<INVALID TOKEN>";
     }
-}
 
-int readChar()
+    string Input;
-{
+    string ConsumedInput;
-    if (Index == Input.length())
+    size_t Index = 0;
-        return EOF;
+    string Error;
-    ConsumedInput += Input[Index];
-    return uint8_t(Input[Index++]); //do not sign-extend to support UTF-8
-}
 
-int getToken()
+    string IdentifierStr = "";
-{
+    uint64_t NumberVal = 0;
-    //skip whitespace
-    while (isspace(LastChar))
-        LastChar = readChar();
 
-    //identifier/keyword
+    int LastChar = ' ';
-    if (isalpha(LastChar)) //[a-zA-Z]
+
+    enum Token
     {
-        IdentifierStr = LastChar;
+        //status tokens
-        while (isalnum(LastChar = readChar())) //[0-9a-zA-Z]
+        tok_eof = -10000,
-            IdentifierStr += LastChar;
+        tok_error,
 
         //keywords
-        auto found = KeywordMap.find(IdentifierStr);
+        tok_typedef, //"typedef"
-        if (found != KeywordMap.end())
+        tok_struct, //"struct"
-            return found->second;
+        tok_char, //"char"
+        tok_unsigned, //"unsigned"
+        tok_int, //"int"
+        tok_sizeof, //"sizeof"
+        tok_BYTE, //"BYTE"
+        tok_WORD, //"WORD"
+        tok_DWORD, //"DWORD"
+        tok_ushort, //"ushort"
+        tok_uint, //"uint"
+        tok_byte, //"byte"
+        tok_double, //"double"
+        tok_string, //"string"
+        tok_return, //"return"
+        tok_enum, //"enum"
 
-        return tok_identifier;
+        //others
+        tok_identifier, //[a-zA-Z_][a-zA-Z0-9_]
+        tok_number //(0x[0-9a-fA-F]+)|([0-9]+)
+    };
+
+    unordered_map<string, Token> KeywordMap;
+
+    void SetupKeywordMap()
+    {
+        KeywordMap["typedef"] = tok_typedef;
+        KeywordMap["struct"] = tok_struct;
+        KeywordMap["char"] = tok_char;
+        KeywordMap["unsigned"] = tok_unsigned;
+        KeywordMap["int"] = tok_int;
+        KeywordMap["sizeof"] = tok_sizeof;
+        KeywordMap["BYTE"] = tok_BYTE;
+        KeywordMap["WORD"] = tok_WORD;
+        KeywordMap["DWORD"] = tok_DWORD;
+        KeywordMap["byte"] = tok_byte;
+        KeywordMap["ushort"] = tok_ushort;
+        KeywordMap["uint"] = tok_uint;
+        KeywordMap["double"] = tok_double;
+        KeywordMap["string"] = tok_string;
+        KeywordMap["return"] = tok_return;
+        KeywordMap["enum"] = tok_enum;
     }
 
-    //(hex) numbers
+    Token ReportError(const String & error)
-    if (isdigit(LastChar)) //[0-9]
     {
-        string NumStr;
+        Error = error;
-        NumStr = LastChar;
+        return tok_error;
-        LastChar = readChar(); //this might not be a digit
+    }
 
-        //hexadecimal numbers
+    String TokString(int tok)
-        if (NumStr[0] == '0' && LastChar == 'x') //0x
+    {
+        switch (Token(tok))
         {
-            NumStr = "";
+        case tok_eof: return "tok_eof";
-            while (isxdigit(LastChar = readChar())) //[0-9a-fA-F]*
+        case tok_error: return StringUtils::sprintf("tok_error \"%s\"", Error.c_str());
+        case tok_identifier: return StringUtils::sprintf("tok_identifier \"%s\"", IdentifierStr.c_str());
+        case tok_number: return StringUtils::sprintf("tok_number %llu (0x%llX)", NumberVal, NumberVal);
+        default:
+            for (const auto & itr : KeywordMap)
+            {
+                if (tok == itr.second)
+                    return "tok_" + itr.first;
+            }
+            if (tok > 0 && tok < 265)
+            {
+                String s;
+                s = tok;
+                return s;
+            }
+            return "<INVALID TOKEN>";
+        }
+    }
+
+    int PeekChar(int distance = 0)
+    {
+        if (Index + distance >= Input.length())
+            return EOF;
+        return Input[Index + distance];
+    }
+
+    int ReadChar()
+    {
+        if (Index == Input.length())
+            return EOF;
+        ConsumedInput += Input[Index];
+        return uint8_t(Input[Index++]); //do not sign-extend to support UTF-8
+    }
+
+    int GetToken()
+    {
+        //skip whitespace
+        while (isspace(LastChar))
+            LastChar = ReadChar();
+
+        //identifier/keyword
+        if (isalpha(LastChar) || LastChar == '_') //[a-zA-Z_]
+        {
+            IdentifierStr = LastChar;
+            LastChar = ReadChar();
+            while (isalnum(LastChar) || LastChar == '_') //[0-9a-zA-Z_]
+            {
+                IdentifierStr += LastChar;
+                LastChar = ReadChar();
+            }
+
+            //keywords
+            auto found = KeywordMap.find(IdentifierStr);
+            if (found != KeywordMap.end())
+                return found->second;
+
+            return tok_identifier;
+        }
+
+        //hex numbers
+        if (LastChar == '0' && PeekChar() == 'x') //0x
+        {
+            string NumStr;
+            ReadChar(); //consume the 'x'
+
+            while (isxdigit(LastChar = ReadChar())) //[0-9a-fA-F]*
                 NumStr += LastChar;
 
             if (!NumStr.length()) //check for error condition
@@ -133,78 +164,118 @@ int getToken()
                 return ReportError("sscanf_s failed on hexadecimal number");
             return tok_number;
         }
-
+        if (isdigit(LastChar)) //[0-9]
-        //decimal numbers
-        while (isdigit(LastChar)) //[0-9]*
         {
-            NumStr += LastChar;
+            string NumStr;
-            LastChar = readChar();
+            NumStr = LastChar;
+
+            while (isdigit(LastChar = ReadChar())) //[0-9]*
+                NumStr += LastChar;
+
+            if (sscanf_s(NumStr.c_str(), "%llu", &NumberVal) != 1)
+                return ReportError("sscanf_s failed on decimal number");
+            return tok_number;
         }
 
-        if (sscanf_s(NumStr.c_str(), "%llu", &NumberVal) != 1)
+        //comments
-            return ReportError("sscanf_s failed on decimal number");
+        if (LastChar == '/' && PeekChar() == '/') //line comment
-        return tok_number;
-    }
-
-    //comments
-    if (LastChar == '/')
-    {
-        LastChar = readChar();
-
-        //line comment
-        if (LastChar == '/')
         {
             do
             {
-                LastChar = readChar();
+                LastChar = ReadChar();
             } while (LastChar != EOF && LastChar != '\n');
 
             if (LastChar == '\n')
-                return getToken(); //interpret the next line
+                return GetToken(); //interpret the next line
         }
-        else
+        else if (LastChar == '/' && PeekChar() == '*') //block comment
-            return ReportError("invalid comment");
+        {
+            //TODO: implement this
+        }
+
+        //end of file
+        if (LastChar == EOF)
+            return tok_eof;
+
+        //unknown character
+        auto ThisChar = LastChar;
+        LastChar = ReadChar();
+        return ThisChar;
     }
 
-    //end of file
+    bool ReadInputFile(const string & filename)
-    if (LastChar == EOF)
-        return tok_eof;
-
-    //unknown character
-    auto ThisChar = LastChar;
-    LastChar = readChar();
-    return ThisChar;
-}
-
-bool ReadInputFile(const char* filename)
-{
-    return FileHelper::ReadAllText(filename, Input);
-}
-
-void testLex()
-{
-    int tok;
-    do
     {
-        tok = getToken();
+        return FileHelper::ReadAllText(filename, Input);
-        puts(tokString(tok).c_str());
-    } while (tok != tok_eof && tok != tok_error);
-}
-
-void test()
-{
-    if (!ReadInputFile("test.bt"))
-    {
-        puts("failed to read input file");
-        return;
     }
-    setup();
+
-    testLex();
+    void TestLex(function<void(const string & line)> lexEnum)
+    {
+        int tok;
+        do
+        {
+            tok = GetToken();
+            lexEnum(TokString(tok));
+        } while (tok != tok_eof && tok != tok_error);
+    }
+};
+
+bool TestLexer(const string & filename)
+{
+    Lexer lexer;
+    if (!lexer.ReadInputFile("tests\\" + filename))
+    {
+        printf("failed to read \"%s\"\n", filename.c_str());
+        return false;
+    }
+    string expected;
+    if (!FileHelper::ReadAllText(filename + ".lextest", expected)) //don't fail tests that we didn't specify yet
+        return true;
+    StringUtils::ReplaceAll(expected, "\r\n", "\n");
+    expected = StringUtils::Trim(expected);
+    string actual;
+    lexer.TestLex([&](const string & line)
+    {
+        actual += line + "\n";
+    });
+    actual = StringUtils::Trim(actual);
+    if (expected == actual)
+    {
+        printf("lexer test for \"%s\" success!\n", filename.c_str());
+        return true;
+    }
+    printf("lexer test for \"%s\" failed\n", filename.c_str());
+    FileHelper::WriteAllText("expected.out", expected);
+    FileHelper::WriteAllText("actual.out", actual);
+    return false;
+}
+
+void RunLexerTests()
+{
+    for (auto file : testFiles)
+        TestLexer(file);
+}
+
+bool DebugLexer(const string & filename)
+{
+    printf("Debugging \"%s\"\n", filename.c_str());
+    Lexer lexer;
+    if (!lexer.ReadInputFile("tests\\" + filename))
+    {
+        printf("failed to read \"%s\"\n", filename.c_str());
+        return false;
+    }
+    lexer.TestLex([](const string & line)
+    {
+        puts(line.c_str());
+    });
+    puts("");
+    return true;
 }
 
 int main()
 {
-    test();
+    DebugLexer(testFiles[1]);
+    RunLexerTests();
     system("pause");
     return 0;
 }

cparser/testfiles.h

@@ -0,0 +1,95 @@
+static const char* testFiles[] =
+{
+    "test.bt",
+    "CDATemplate.bt",
+    "NetflowVersion5.bt",
+    "SHXTemplate.bt",
+    "WinhexPosTemplate.bt",
+    "Mifare1kTemplate.bt",
+    "PALTemplate.bt",
+    "GocleverTemplate.bt",
+    "OGGTemplate.bt",
+    "STLTemplate.bt",
+    "SinclairMicrodriveImage.bt",
+    "RDBTemplate.bt",
+    "DBFTemplate.bt",
+    "Mifare4kTemplate.bt",
+    "GPTTemplate.bt",
+    "SSPTemplate.bt",
+    "SHPTemplate.bt",
+    "SRecTemplate.bt",
+    "FLVTemplate.bt",
+    "LUKSTemplate.bt",
+    "PCXTemplate.bt",
+    "UTMPTemplate.bt",
+    "ElTorito.bt",
+    "DMPTemplate.bt",
+    "OscarItemTemplate.bt",
+    "EOTTemplate.bt",
+    "ISOTemplate.bt",
+    "CLASSTemplate2.bt",
+    "EVSBTemplate.bt",
+    "BMPTemplate.bt",
+    "TGATemplate.bt",
+    "TOCTemplate.bt",
+    "CABTemplate.bt",
+    "RIFFTemplate.bt",
+    "AndroidManifestTemplate.bt",
+    "InspectorWithMP4DateTime.bt",
+    "FAT16Template.bt",
+    "PNGTemplate.bt",
+    "ICOTemplate.bt",
+    "RegistryPolicyFileTemplate.bt",
+    "VHDTemplate.bt",
+    "ISOBMFTemplate.bt",
+    "PCAPTemplate.bt",
+    "AVITemplate.bt",
+    "ZIPTemplate.bt",
+    "CRXTemplate.bt",
+    "MIDITemplate.bt",
+    "GZipTemplate.bt",
+    "GIFTemplate.bt",
+    "InspectorDates.bt",
+    "WAVTemplate.bt",
+    "RegistryHive.bt",
+    "EMFTemplate.bt",
+    "ROMFS.bt",
+    "OrCad3.20a_SCH.bt",
+    "MP4Template.bt",
+    "CLASSTemplate.bt",
+    "WMFTemplate.bt",
+    "LNKTemplate.bt",
+    "OrCAD3.20a_LIB.bt",
+    "PSFTemplate.bt",
+    "RARTemplate.bt",
+    "PYCTemplate.bt",
+    "EXETemplate.bt",
+    "PNG12Template.bt",
+    "TacxTemplate.bt",
+    "MFTRecord.bt",
+    "MP3Template.bt",
+    "MBRTemplate.bt",
+    "WAVTemplateAdv.bt",
+    "PDFTemplate.bt",
+    "EXETemplate2.bt",
+    "RESTemplate.bt",
+    "ZIPTemplateAdv.bt",
+    "SF2Template.bt",
+    "MOBITemplate.bt",
+    "MBRTemplateFAT.bt",
+    "exFATTemplate.bt",
+    "ELFTemplate.new.bt",
+    "ELFTemplate.bt",
+    "MachOTemplate.bt",
+    "PETemplate.bt",
+    "EDIDTemplate.bt",
+    "GeoTIFTemplate.bt",
+    "CLASSTemplate3.bt",
+    "CAPTemplate.bt",
+    "TIFTemplate.bt",
+    "TTFTemplate.bt",
+    "JPGTemplate.bt",
+    "DEXTemplate.bt",
+    "DEXTemplate.new.bt",
+    "SWFTemplate.bt",
+};

cparser/tests/test.bt.lextest

@@ -0,0 +1,53 @@
+tok_struct
+tok_identifier "DBZ"
+{
+tok_struct
+tok_identifier "HEADER"
+{
+tok_char
+tok_identifier "magic"
+[
+tok_number 4 (0x4)
+]
+;
+tok_unsigned
+tok_int
+tok_identifier "size"
+;
+tok_unsigned
+tok_int
+tok_identifier "dataStart"
+;
+tok_unsigned
+tok_int
+tok_identifier "numEntries"
+;
+}
+tok_identifier "header"
+;
+tok_char
+tok_identifier "empty"
+[
+tok_identifier "header"
+.
+tok_identifier "size"
+-
+tok_sizeof
+(
+tok_identifier "HEADER"
+)
+]
+;
+tok_unsigned
+tok_int
+tok_identifier "entryOffsets"
+[
+tok_identifier "header"
+.
+tok_identifier "numEntries"
+]
+;
+}
+tok_identifier "dbz"
+;
+tok_eof