DBG+GUI: having fun with some control flow analysis

src/dbg/instruction.cpp

@@ -35,6 +35,7 @@
 #include "threading.h"
 #include "mnemonichelp.h"
 #include "error.h"
+#include "recursiveanalysis.h"
 
 static bool bRefinit = false;
 static int maxFindResults = 5000;
@@ -2153,6 +2154,23 @@ CMDRESULT cbInstrExanalyse(int argc, char* argv[])
     return STATUS_CONTINUE;
 }
 
+CMDRESULT cbInstrAnalrecur(int argc, char* argv[])
+{
+    if(argc < 2)
+        return STATUS_ERROR;
+    duint entry;
+    if(!valfromstring(argv[1], &entry, false))
+        return STATUS_ERROR;
+    duint size;
+    auto base = MemFindBaseAddr(entry, &size);
+    if(!base)
+        return STATUS_ERROR;
+    RecursiveAnalysis analysis(base, size, entry, 0);
+    analysis.Analyse();
+    analysis.SetMarkers();
+    return STATUS_CONTINUE;
+}
+
 CMDRESULT cbInstrVirtualmod(int argc, char* argv[])
 {
     if(argc < 3)
@@ -2494,3 +2512,36 @@ CMDRESULT cbHandleClose(int argc, char* argv[])
     dprintf("Handle %" fext "X closed!\n", handle);
     return STATUS_CONTINUE;
 }
+
+CMDRESULT cbInstrBriefcheck(int argc, char* argv[])
+{
+    if(argc < 2)
+        return STATUS_ERROR;
+    duint addr;
+    if(!valfromstring(argv[1], &addr, false))
+        return STATUS_ERROR;
+    duint size;
+    auto base = DbgMemFindBaseAddr(addr, &size);
+    if(!base)
+        return STATUS_ERROR;
+    Memory<unsigned char*> buffer(size + 16);
+    DbgMemRead(base, buffer(), size);
+    Capstone cp;
+    std::unordered_set<String> reported;
+    for(duint i = 0; i < size;)
+    {
+        if(!cp.Disassemble(base + i, buffer() + i, 16))
+        {
+            i++;
+            continue;
+        }
+        i += cp.Size();
+        auto mnem = StringUtils::ToLower(cp.MnemonicId());
+        auto brief = MnemonicHelp::getBriefDescription(mnem.c_str());
+        if(brief.length() || reported.count(mnem))
+            continue;
+        reported.insert(mnem);
+        dprintf(fhex ": %s\n", cp.Address(), mnem.c_str());
+    }
+    return STATUS_CONTINUE;
+}

src/dbg/instruction.h

@@ -72,6 +72,7 @@ CMDRESULT cbInstrLog(int argc, char* argv[]);
 CMDRESULT cbInstrCapstone(int argc, char* argv[]);
 CMDRESULT cbInstrAnalyseNukem(int argc, char* argv[]);
 CMDRESULT cbInstrAnalyse(int argc, char* argv[]);
+CMDRESULT cbInstrAnalrecur(int argc, char* argv[]);
 CMDRESULT cbInstrVisualize(int argc, char* argv[]);
 CMDRESULT cbInstrMeminfo(int argc, char* argv[]);
 CMDRESULT cbInstrCfanalyse(int argc, char* argv[]);
@@ -86,5 +87,6 @@ CMDRESULT cbGetPrivilegeState(int argc, char* argv[]);
 CMDRESULT cbEnablePrivilege(int argc, char* argv[]);
 CMDRESULT cbDisablePrivilege(int argc, char* argv[]);
 CMDRESULT cbHandleClose(int argc, char* argv[]);
+CMDRESULT cbInstrBriefcheck(int argc, char* argv[]);
 
 #endif // _INSTRUCTION_H

src/dbg/recursiveanalysis.cpp

@@ -0,0 +1,104 @@
+#include "recursiveanalysis.h"
+#include <queue>
+#include "console.h"
+#include "filehelper.h"
+#include "function.h"
+
+RecursiveAnalysis::RecursiveAnalysis(duint base, duint size, duint entryPoint, duint maxDepth, bool dump)
+    : Analysis(base, size),
+      mEntryPoint(entryPoint),
+      mMaxDepth(maxDepth),
+      mDump(dump)
+{
+}
+
+void RecursiveAnalysis::Analyse()
+{
+    //TODO: implement queue to analyze multiple functions
+    analyzeFunction(mEntryPoint);
+}
+
+void RecursiveAnalysis::SetMarkers()
+{
+    if(mDump)
+        for(const auto & function : mFunctions)
+            FileHelper::WriteAllText(StringUtils::sprintf("cfgraph_" fhex ".dot", function.entryPoint), function.ToDot());
+
+    for(const auto & function : mFunctions)
+    {
+        duint start = ~0;
+        duint end = 0;
+        duint icount = 0;
+        for(const auto & node : function.nodes)
+        {
+            icount += node.second.icount;
+            start = min(node.second.start, start);
+            end = max(node.second.end, end);
+        }
+        if(!FunctionAdd(start, end, false, icount))
+        {
+            FunctionDelete(start);
+            FunctionDelete(end);
+            FunctionAdd(start, end, false, icount);
+        }
+    }
+    GuiUpdateAllViews();
+}
+
+void RecursiveAnalysis::analyzeFunction(duint entryPoint)
+{
+    //BFS through the disassembly starting at entryPoint
+    CFGraph graph(entryPoint);
+    UintSet visited;
+    std::queue<duint> queue;
+    queue.push(graph.entryPoint);
+    while(!queue.empty())
+    {
+        auto start = queue.front();
+        queue.pop();
+        if(visited.count(start) || !inRange(start))  //already visited or out of range
+            continue;
+        visited.insert(start);
+
+        CFNode node(graph.entryPoint, start, start);
+        while(true)
+        {
+            node.icount++;
+            if(!mCp.Disassemble(node.end, translateAddr(node.end)))
+            {
+                node.end++;
+                continue;
+            }
+            if(mCp.InGroup(CS_GRP_JUMP) || mCp.IsLoop())  //jump
+            {
+                //set the branch destinations
+                node.brtrue = mCp.BranchDestination();
+                if(mCp.GetId() != X86_INS_JMP)  //unconditional jumps dont have a brfalse
+                    node.brfalse = node.end + mCp.Size();
+
+                //add node to the function graph
+                graph.AddNode(node);
+
+                //enqueue branch destinations
+                if(node.brtrue)
+                    queue.push(node.brtrue);
+                if(node.brfalse)
+                    queue.push(node.brfalse);
+
+                break;
+            }
+            if(mCp.InGroup(CS_GRP_CALL))  //call
+            {
+                //TODO: add this to a queue to be analyzed later
+            }
+            if(mCp.InGroup(CS_GRP_RET))  //return
+            {
+                node.terminal = true;
+                graph.AddNode(node);
+                break;
+            }
+            node.end += mCp.Size();
+        }
+    }
+    mFunctions.push_back(graph);
+}

src/dbg/recursiveanalysis.h

@@ -0,0 +1,134 @@
+#pragma once
+
+#include "analysis.h"
+
+class RecursiveAnalysis : public Analysis
+{
+public:
+    explicit RecursiveAnalysis(duint base, duint size, duint entryPoint, duint maxDepth, bool dump = false);
+    void Analyse() override;
+    void SetMarkers() override;
+
+    using UintSet = std::unordered_set<duint>;
+
+    template<class T>
+    using UintMap = std::unordered_map<duint, T>;
+
+    struct CFNode
+    {
+        duint parentGraph; //function of which this node is a part
+        duint start; //start of the block
+        duint end; //end of the block (inclusive)
+        duint brtrue; //destination if condition is true
+        duint brfalse; //destination if condition is false
+        duint icount; //number of instructions in node
+        bool terminal; //node is a RET
+
+        explicit CFNode(duint parentGraph, duint start, duint end)
+            : parentGraph(parentGraph),
+              start(start),
+              end(end),
+              brtrue(0),
+              brfalse(0),
+              icount(0),
+              terminal(false)
+        {
+        }
+
+        explicit CFNode()
+            : CFNode(0, 0, 0)
+        {
+        }
+
+        String ToString() const
+        {
+            return StringUtils::sprintf("start: " fhex "\nend: " fhex "\nfunction: " fhex, start, end, parentGraph);
+        }
+    };
+
+    struct CFGraph
+    {
+        duint entryPoint; //graph entry point
+        UintMap<CFNode> nodes; //CFNode.start -> CFNode
+        UintMap<UintSet> parents; //CFNode.start -> parents
+
+        explicit CFGraph(duint entryPoint)
+            : entryPoint(entryPoint)
+        {
+        }
+
+        void AddNode(const CFNode & node)
+        {
+            nodes[node.start] = node;
+            AddParent(node.start, node.brtrue);
+            AddParent(node.start, node.brfalse);
+        }
+
+        void AddParent(duint child, duint parent)
+        {
+            if(!child || !parent)
+                return;
+            auto found = parents.find(child);
+            if(found == parents.end())
+            {
+                UintSet p;
+                p.insert(parent);
+                parents[child] = p;
+            }
+            else
+                found->second.insert(parent);
+        }
+
+        const char* GetNodeColor(const CFNode & node) const
+        {
+            if(node.terminal)
+                return "firebrick";
+            if(node.start == entryPoint)
+                return "forestgreen";
+            return "lightblue";
+        }
+
+        String ToDot() const
+        {
+            String result = "digraph CFGraph {\n";
+            for(const auto & node : nodes)
+                result += StringUtils::sprintf("    n" fhex "[label=\"%s\" style=filled fillcolor=%s shape=box]\n",
+                                               node.second.start,
+                                               node.second.ToString().c_str(),
+                                               GetNodeColor(node.second));
+            result += "\n";
+            for(const auto & node : nodes)
+            {
+                if(node.second.brtrue)
+                    result += StringUtils::sprintf("    n" fhex "-> n" fhex " [color=green]\n",
+                                                   node.second.start,
+                                                   node.second.brtrue);
+                if(node.second.brfalse)
+                    result += StringUtils::sprintf("    n" fhex "-> n" fhex " [color=red]\n",
+                                                   node.second.start,
+                                                   node.second.brfalse);
+            }
+            result += "\n";
+
+            for(const auto & parent : parents)
+            {
+                for(const auto & node : parent.second)
+                    result += StringUtils::sprintf("    n" fhex "-> n" fhex " [style=dotted color=grey]\n",
+                                                   node,
+                                                   parent.first);
+            }
+            result += "}";
+            return result;
+        }
+    };
+
+protected:
+    duint mEntryPoint;
+    std::vector<CFGraph> mFunctions;
+
+private:
+    duint mMaxDepth;
+    bool mDump;
+
+    void analyzeFunction(duint entryPoint);
+};

src/dbg/x64_dbg.cpp

@@ -272,38 +272,8 @@ static void registercommands()
     dbgcmdnew("EnablePrivilege", cbEnablePrivilege, true); //enable priv
     dbgcmdnew("DisablePrivilege", cbDisablePrivilege, true); //disable priv
     dbgcmdnew("handleclose", cbHandleClose, true); //close remote handle
-    dbgcmdnew("briefcheck", [](int argc, char* argv[])
-    {
-        if(argc < 2)
-            return STATUS_ERROR;
-        duint addr;
-        if(!valfromstring(argv[1], &addr, false))
-            return STATUS_ERROR;
-        duint size;
-        auto base = DbgMemFindBaseAddr(addr, &size);
-        if(!base)
-            return STATUS_ERROR;
-        Memory<unsigned char*> buffer(size + 16);
-        DbgMemRead(base, buffer(), size);
-        Capstone cp;
-        std::unordered_set<String> reported;
-        for(duint i = 0; i < size;)
-        {
-            if(!cp.Disassemble(base + i, buffer() + i, 16))
-            {
-                i++;
-                continue;
-            }
-            i += cp.Size();
-            auto mnem = StringUtils::ToLower(cp.MnemonicId());
-            auto brief = MnemonicHelp::getBriefDescription(mnem.c_str());
-            if(brief.length() || reported.count(mnem))
-                continue;
-            reported.insert(mnem);
-            dprintf(fhex ": %s\n", cp.Address(), mnem.c_str());
-        }
-        return STATUS_CONTINUE;
-    }, true);
+    dbgcmdnew("briefcheck", cbInstrBriefcheck, true); //check if mnemonic briefs are missing
+    dbgcmdnew("analrecur\1analr", cbInstrAnalrecur, true); //analyze a single function
 }
 
 static bool cbCommandProvider(char* cmd, int maxlen)

src/dbg/x64_dbg_dbg.vcxproj

@@ -64,6 +64,7 @@
     <ClCompile Include="patches.cpp" />
     <ClCompile Include="patternfind.cpp" />
     <ClCompile Include="plugin_loader.cpp" />
+    <ClCompile Include="recursiveanalysis.cpp" />
     <ClCompile Include="reference.cpp" />
     <ClCompile Include="simplescript.cpp" />
     <ClCompile Include="stackinfo.cpp" />
@@ -154,6 +155,7 @@
     <ClInclude Include="patches.h" />
     <ClInclude Include="patternfind.h" />
     <ClInclude Include="plugin_loader.h" />
+    <ClInclude Include="recursiveanalysis.h" />
     <ClInclude Include="reference.h" />
     <ClInclude Include="serializablemap.h" />
     <ClInclude Include="tcpconnections.h" />

src/dbg/x64_dbg_dbg.vcxproj.filters

@@ -314,6 +314,9 @@
     <ClCompile Include="_scriptapi_argument.cpp">
       <Filter>Source Files\Interfaces/Exports\_scriptapi</Filter>
     </ClCompile>
+    <ClCompile Include="recursiveanalysis.cpp">
+      <Filter>Source Files\Analysis</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="x64_dbg.h">
@@ -691,5 +694,8 @@
     <ClInclude Include="_scriptapi_argument.h">
       <Filter>Header Files\Interfaces/Exports\_scriptapi</Filter>
     </ClInclude>
+    <ClInclude Include="recursiveanalysis.h">
+      <Filter>Header Files\Analysis</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

src/gui/Src/Gui/CPUDisassembly.cpp

@@ -403,6 +403,7 @@ void CPUDisassembly::setupRightClickContextMenu()
             toggleFunctionAction->setText(tr("Delete function"));
         return true;
     });
+    mMenuBuilder->addAction(makeAction(QIcon(":/icons/images/analyzesinglefunction.png"), tr("Analyze single function"), SLOT(analyzeSingleFunctionSlot())));
     mMenuBuilder->addAction(makeShortcutAction(QIcon(":/icons/images/compile.png"), tr("Assemble"), SLOT(assembleSlot()), "ActionAssemble"));
     removeAction(mMenuBuilder->addAction(makeShortcutAction(QIcon(":/icons/images/patch.png"), tr("Patches"), SLOT(showPatchesSlot()), "ViewPatches"))); //prevent conflicting shortcut with the MainWindow
     mMenuBuilder->addAction(makeShortcutAction(QIcon(":/icons/images/yara.png"), tr("&Yara..."), SLOT(yaraSlot()), "ActionYara"));
@@ -1433,3 +1434,8 @@ void CPUDisassembly::mnemonicHelpSlot()
     DbgCmdExecDirect(QString("mnemonichelp %1").arg(disasm.instruction).toUtf8().constData());
     emit displayLogWidget();
 }
+
+void CPUDisassembly::analyzeSingleFunctionSlot()
+{
+    DbgCmdExec(QString("analr %1").arg(ToHexString(rvaToVa(getInitialSelection()))).toUtf8().constData());
+}

src/gui/Src/Gui/CPUDisassembly.h

@@ -93,6 +93,7 @@ public slots:
     void displayWarningSlot(QString title, QString text);
     void labelHelpSlot();
     void editSoftBpActionSlot();
+    void analyzeSingleFunctionSlot();
 
 protected:
     void paintEvent(QPaintEvent* event);

src/gui/Src/ThirdPartyLibs/float128/float128.cpp

@@ -1715,6 +1715,7 @@ dd_real pown(dd_real const & a, int n)
 
     case 2:
         s = sqr(a);
+        break;
 
     default:                            /* Use binary exponentiation */
     {

src/gui/resource.qrc

@@ -96,5 +96,6 @@
         <file>images/helpmnemonic.png</file>
         <file>images/handles.png</file>
         <file>images/dump.png</file>
+        <file>images/analyzesinglefunction.png</file>
     </qresource>
 </RCC>