DBG: improved RecursiveAnalysis

src/dbg/analysis/recursiveanalysis.cpp

@@ -55,7 +55,7 @@ void RecursiveAnalysis::SetMarkers()
 
 void RecursiveAnalysis::analyzeFunction(duint entryPoint)
 {
-    //BFS through the disassembly starting at entryPoint
+    //first pass: BFS through the disassembly starting at entryPoint
     CFGraph graph(entryPoint);
     UintSet visited;
     std::queue<duint> queue;
@@ -128,5 +128,36 @@ void RecursiveAnalysis::analyzeFunction(duint entryPoint)
             node.end += mCp.Size();
         }
     }
+    //second pass: split overlapping blocks introduced by backedges
+    for(auto & nodeIt : graph.nodes)
+    {
+        auto & node = nodeIt.second;
+        duint addr = node.start;
+        duint icount = 0;
+        while(addr < node.end)
+        {
+            icount++;
+            auto size = mCp.Disassemble(addr, translateAddr(addr)) ? mCp.Size() : 1;
+            if(graph.nodes.count(addr + size))
+            {
+                node.end = addr;
+                node.split = true;
+                node.brtrue = addr + size;
+                node.brfalse = 0;
+                node.terminal = false;
+                node.icount = icount;
+                break;
+            }
+            addr += size;
+        }
+    }
+    //third pass: correct the parents
+    graph.parents.clear();
+    for(const auto & nodeIt : graph.nodes)
+    {
+        const auto & node = nodeIt.second;
+        graph.AddParent(node.start, node.brtrue);
+        graph.AddParent(node.start, node.brfalse);
+    }
     mFunctions.push_back(graph);
 }

src/dbg/analysis/recursiveanalysis.h

@@ -23,6 +23,7 @@ public:
         duint brfalse; //destination if condition is false
         duint icount; //number of instructions in node
         bool terminal; //node is a RET
+        bool split; //node is a split (brtrue points to the next node)
 
         explicit CFNode(duint parentGraph, duint start, duint end)
             : parentGraph(parentGraph),
@@ -31,7 +32,8 @@ public:
               brtrue(0),
               brfalse(0),
               icount(0),
-              terminal(false)
+              terminal(false),
+              split(false)
         {
         }
 
@@ -42,7 +44,7 @@ public:
 
         String ToString() const
         {
-            return StringUtils::sprintf("start: " fhex "\nend: " fhex "\nfunction: " fhex, start, end, parentGraph);
+            return StringUtils::sprintf("start: " fhex ", %" fext "d\nend: " fhex "\nfunction: " fhex, start, icount, end, parentGraph);
         }
     };
 
@@ -100,9 +102,10 @@ public:
             for(const auto & node : nodes)
             {
                 if(node.second.brtrue)
-                    result += StringUtils::sprintf("    n" fhex "-> n" fhex " [color=green]\n",
+                    result += StringUtils::sprintf("    n" fhex "-> n" fhex " [color=%s]\n",
                                                    node.second.start,
-                                                   node.second.brtrue);
+                                                   node.second.brtrue,
+                                                   node.second.split ? "black" : "green");
                 if(node.second.brfalse)
                     result += StringUtils::sprintf("    n" fhex "-> n" fhex " [color=red]\n",
                                                    node.second.start,