Trace into/over beyond/into trace record (#756)

* replace map with hash map

* Fix the issue with trace record saving

* Trace into/over beyond/into trace record

* Trace into/over beyond/into trace record

* Trace into/over beyond/into trace record

and fix the issue that when a breakpoint is hit during tracing, trace would not be available.

* Trace into/over beyond/into trace record

* Trace into/over beyond/into trace record

* Trace record now sets the entire module

instead of a single page

* Trace into/over beyond/into trace record

* Trace into/over beyond/into trace record

* Trace into/over beyond/into trace record

src/dbg/TraceRecord.cpp

@@ -300,7 +300,6 @@ void TraceRecordManager::saveToDb(JSON root)
     }
     if(json_array_size(jsonTraceRecords))
         json_object_set_new(root, "tracerecord", jsonTraceRecords);
-    json_decref(jsonTraceRecords);
 }
 
 void TraceRecordManager::loadFromDb(JSON root)

src/dbg/TraceRecord.h

@@ -76,7 +76,7 @@ private:
     };
 
     //Key := page base, value := trace record raw data
-    std::map<duint, TraceRecordPage> TraceRecord;
+    std::unordered_map<duint, TraceRecordPage> TraceRecord;
     std::vector<std::string> ModuleNames;
     unsigned int getModuleIndex(std::string moduleName);
     unsigned int instructionCounter;

src/dbg/debugger.cpp

@@ -586,6 +586,7 @@ static void cbGenericBreakpoint(BP_TYPE bptype, void* ExceptionAddress = nullptr
     }
     if(breakCondition)  //break the debugger
     {
+        dbgcleartracecondition();
         SetForegroundWindow(GuiGetWindowHandle());
         bSkipExceptions = false;
     }
@@ -796,12 +797,13 @@ void cbStep()
 
 static void cbRtrFinalStep()
 {
+    dbgcleartracecondition();
     hActiveThread = ThreadGetHandle(((DEBUG_EVENT*)GetDebugData())->dwThreadId);
     GuiSetDebugState(paused);
     duint CIP = GetContextDataEx(hActiveThread, UE_CIP);
-    DebugUpdateGui(CIP, true);
     // Trace record
     _dbg_dbgtraceexecute(CIP);
+    DebugUpdateGui(CIP, true);
     //lock
     lock(WAITID_RUN);
     SetForegroundWindow(GuiGetWindowHandle());
@@ -812,18 +814,12 @@ static void cbRtrFinalStep()
     wait(WAITID_RUN);
 }
 
-static unsigned char getCIPch()
+void cbRtrStep()
 {
     unsigned char ch = 0x90;
     duint cip = GetContextDataEx(hActiveThread, UE_CIP);
     MemRead(cip, &ch, 1);
-    return ch;
-}
-
-void cbRtrStep()
-{
-    unsigned int cipch = getCIPch();
-    if(cipch == 0xC3 || cipch == 0xC2)
+    if(ch == 0xC3 || ch == 0xC2)
         cbRtrFinalStep();
     else
         StepOver((void*)cbRtrStep);
@@ -853,6 +849,94 @@ void cbTICNDStep()
     }
 }
 
+void cbTIBTStep()
+{
+    // Trace record
+    duint CIP = GetContextDataEx(hActiveThread, UE_CIP);
+    if(!traceCondition)
+    {
+        _dbg_dbgtraceexecute(CIP);
+        dprintf("Bad tracing state.\n");
+        cbRtrFinalStep();
+        return;
+    }
+    if((TraceRecord.getTraceRecordType(CIP) != TraceRecordManager::TraceRecordNone && TraceRecord.getHitCount(CIP) == 0) || !traceCondition->ContinueTrace())
+    {
+        _dbg_dbgtraceexecute(CIP);
+        auto steps = dbgcleartracecondition();
+        dprintf("Trace finished after %" fext "u steps!\n", steps);
+        cbRtrFinalStep();
+        return;
+    }
+    StepInto((void*)cbTIBTStep);
+}
+
+void cbTOBTStep()
+{
+    // Trace record
+    duint CIP = GetContextDataEx(hActiveThread, UE_CIP);
+    if(!traceCondition)
+    {
+        _dbg_dbgtraceexecute(CIP);
+        dprintf("Bad tracing state.\n");
+        cbRtrFinalStep();
+        return;
+    }
+    if((TraceRecord.getTraceRecordType(CIP) != TraceRecordManager::TraceRecordNone && TraceRecord.getHitCount(CIP) == 0) || !traceCondition->ContinueTrace())
+    {
+        _dbg_dbgtraceexecute(CIP);
+        auto steps = dbgcleartracecondition();
+        dprintf("Trace finished after %" fext "u steps!\n", steps);
+        cbRtrFinalStep();
+        return;
+    }
+    StepOver((void*)cbTOBTStep);
+}
+
+void cbTIITStep()
+{
+    // Trace record
+    duint CIP = GetContextDataEx(hActiveThread, UE_CIP);
+    if(!traceCondition)
+    {
+        _dbg_dbgtraceexecute(CIP);
+        dprintf("Bad tracing state.\n");
+        cbRtrFinalStep();
+        return;
+    }
+    if((TraceRecord.getTraceRecordType(CIP) != TraceRecordManager::TraceRecordNone && TraceRecord.getHitCount(CIP) != 0) || !traceCondition->ContinueTrace())
+    {
+        _dbg_dbgtraceexecute(CIP);
+        auto steps = dbgcleartracecondition();
+        dprintf("Trace finished after %" fext "u steps!\n", steps);
+        cbRtrFinalStep();
+        return;
+    }
+    StepInto((void*)cbTIITStep);
+}
+
+void cbTOITStep()
+{
+    // Trace record
+    duint CIP = GetContextDataEx(hActiveThread, UE_CIP);
+    if(!traceCondition)
+    {
+        _dbg_dbgtraceexecute(CIP);
+        dprintf("Bad tracing state.\n");
+        cbRtrFinalStep();
+        return;
+    }
+    if((TraceRecord.getTraceRecordType(CIP) != TraceRecordManager::TraceRecordNone && TraceRecord.getHitCount(CIP) != 0) || !traceCondition->ContinueTrace())
+    {
+        _dbg_dbgtraceexecute(CIP);
+        auto steps = dbgcleartracecondition();
+        dprintf("Trace finished after %" fext "u steps!\n", steps);
+        cbRtrFinalStep();
+        return;
+    }
+    StepOver((void*)cbTOITStep);
+}
+
 static void cbCreateProcess(CREATE_PROCESS_DEBUG_INFO* CreateProcessInfo)
 {
     void* base = CreateProcessInfo->lpBaseOfImage;

src/dbg/debugger.h

@@ -110,6 +110,10 @@ bool cbDeleteAllMemoryBreakpoints(const BREAKPOINT* bp);
 bool cbDeleteAllHardwareBreakpoints(const BREAKPOINT* bp);
 void cbTOCNDStep();
 void cbTICNDStep();
+void cbTIBTStep();
+void cbTOBTStep();
+void cbTIITStep();
+void cbTOITStep();
 DWORD WINAPI threadAttachLoop(void* lpParameter);
 void cbDetach();
 bool cbSetModuleBreakpoints(const BREAKPOINT* bp);

src/dbg/debugger_commands.cpp

@@ -1255,7 +1255,7 @@ CMDRESULT cbDebugeRtr(int argc, char* argv[])
     return cbDebugRtr(argc, argv);
 }
 
-CMDRESULT cbDebugTocnd(int argc, char* argv[])
+static CMDRESULT cbDebugConditionalTrace(void* callBack, bool stepOver, int argc, char* argv[])
 {
     if(argc < 2)
     {
@@ -1275,34 +1275,66 @@ CMDRESULT cbDebugTocnd(int argc, char* argv[])
         dprintf("Invalid expression \"%s\"\n", argv[1]);
         return STATUS_ERROR;
     }
-    StepOver((void*)cbTOCNDStep);
+    if(stepOver)
+        StepOver(callBack);
+    else
+        StepInto(callBack);
     cbDebugRun(argc, argv);
     return STATUS_CONTINUE;
 }
 
+CMDRESULT cbDebugTocnd(int argc, char* argv[])
+{
+    return cbDebugConditionalTrace((void*)cbTOCNDStep, true, argc, argv);
+}
+
 CMDRESULT cbDebugTicnd(int argc, char* argv[])
 {
-    if(argc < 2)
+    return cbDebugConditionalTrace((void*)cbTICNDStep, false, argc, argv);
+}
+
+CMDRESULT cbDebugTibt(int argc, char* argv[])
+{
+    if(argc == 1)
     {
-        dputs("Not enough arguments");
-        return STATUS_ERROR;
+        char* new_argv[] = { "tibt", "0" };
+        return cbDebugConditionalTrace((void*)cbTIBTStep, false, 2, new_argv);
     }
-    if(dbgtraceactive())
+    else
+        return cbDebugConditionalTrace((void*)cbTIBTStep, false, argc, argv);
+}
+
+CMDRESULT cbDebugTobt(int argc, char* argv[])
+{
+    if(argc == 1)
     {
-        dputs("Trace already active");
-        return STATUS_ERROR;
+        char* new_argv[] = { "tobt", "0" };
+        return cbDebugConditionalTrace((void*)cbTOBTStep, true, 2, new_argv);
     }
-    duint maxCount = 50000;
-    if(argc > 2 && !valfromstring(argv[2], &maxCount, false))
-        return STATUS_ERROR;
-    if(!dbgsettracecondition(argv[1], maxCount))
+    else
+        return cbDebugConditionalTrace((void*)cbTOBTStep, true, argc, argv);
+}
+
+CMDRESULT cbDebugTiit(int argc, char* argv[])
+{
+    if(argc == 1)
     {
-        dprintf("Invalid expression \"%s\"\n", argv[1]);
-        return STATUS_ERROR;
+        char* new_argv[] = { "tiit", "0" };
+        return cbDebugConditionalTrace((void*)cbTIITStep, false, 2, new_argv);
     }
-    StepInto((void*)cbTICNDStep);
-    cbDebugRun(argc, argv);
-    return STATUS_CONTINUE;
+    else
+        return cbDebugConditionalTrace((void*)cbTIITStep, false, argc, argv);
+}
+
+CMDRESULT cbDebugToit(int argc, char* argv[])
+{
+    if(argc == 1)
+    {
+        char* new_argv[] = { "toit", "0" };
+        return cbDebugConditionalTrace((void*)cbTOITStep, true, 2, new_argv);
+    }
+    else
+        return cbDebugConditionalTrace((void*)cbTOITStep, true, argc, argv);
 }
 
 CMDRESULT cbDebugAlloc(int argc, char* argv[])

src/dbg/debugger_commands.h

@@ -94,6 +94,10 @@ CMDRESULT cbDebugGetPageRights(int argc, char* argv[]);
 CMDRESULT cbDebugSetPageRights(int argc, char* argv[]);
 CMDRESULT cbDebugSkip(int argc, char* argv[]);
 CMDRESULT cbDebugSetfreezestack(int argc, char* argv[]);
+CMDRESULT cbDebugTibt(int argc, char* argv[]);
+CMDRESULT cbDebugTobt(int argc, char* argv[]);
+CMDRESULT cbDebugTiit(int argc, char* argv[]);
+CMDRESULT cbDebugToit(int argc, char* argv[]);
 
 //misc
 void showcommandlineerror(cmdline_error_t* cmdline_error);

src/dbg/x64_dbg.cpp

@@ -89,8 +89,12 @@ static void registercommands()
     dbgcmdnew("eSingleStep\1esstep\1esst", cbDebugeSingleStep, true); //SingleStep arg1:count + skip first chance exceptions
     dbgcmdnew("StepOut\1rtr", cbDebugRtr, true); //rtr
     dbgcmdnew("eStepOut\1ertr", cbDebugeRtr, true); //rtr + skip first chance exceptions
-    dbgcmdnew("tocnd", cbDebugTocnd, true); //tocnd
-    dbgcmdnew("ticnd", cbDebugTicnd, true); //ticnd
+    dbgcmdnew("TraceOverConditional\1tocnd", cbDebugTocnd, true); //tocnd
+    dbgcmdnew("TraceIntoConditional\1ticnd", cbDebugTicnd, true); //ticnd
+    dbgcmdnew("TraceIntoBeyondTraceRecord\1tibt", cbDebugTibt, true); //Trace into beyond trace record
+    dbgcmdnew("TraceOverBeyondTraceRecord\1tobt", cbDebugTobt, true); //Trace over beyond trace record
+    dbgcmdnew("TraceIntoIntoTraceRecord\1tiit", cbDebugTiit, true); //Trace into into trace record
+    dbgcmdnew("TraceOverIntoTraceRecord\1toit", cbDebugToit, true); //Trace over into trace record
 
     dbgcmdnew("DebugContinue\1con", cbDebugContinue, true); //set continue status
 

src/gui/Src/Gui/CPUDisassembly.cpp

@@ -1409,26 +1409,58 @@ void CPUDisassembly::editSoftBpActionSlot()
 
 void CPUDisassembly::ActionTraceRecordBitSlot()
 {
-    if(!(DbgFunctions()->SetTraceRecordType(rvaToVa(getInitialSelection()), TRACERECORDTYPE::TraceRecordBitExec)))
-        GuiAddLogMessage("Failed to set trace record.\n");
+    duint base = mMemPage->getBase();
+    duint size = mMemPage->getSize();
+    for(duint i = base; i < base + size; i += 4096)
+    {
+        if(!(DbgFunctions()->SetTraceRecordType(i, TRACERECORDTYPE::TraceRecordBitExec)))
+        {
+            GuiAddLogMessage(tr("Failed to set trace record.\n").toUtf8().constData());
+            break;
+        }
+    }
 }
 
 void CPUDisassembly::ActionTraceRecordByteSlot()
 {
-    if(!(DbgFunctions()->SetTraceRecordType(rvaToVa(getInitialSelection()), TRACERECORDTYPE::TraceRecordByteWithExecTypeAndCounter)))
-        GuiAddLogMessage("Failed to set trace record.\n");
+    duint base = mMemPage->getBase();
+    duint size = mMemPage->getSize();
+    for(duint i = base; i < base + size; i += 4096)
+    {
+        if(!(DbgFunctions()->SetTraceRecordType(i, TRACERECORDTYPE::TraceRecordByteWithExecTypeAndCounter)))
+        {
+            GuiAddLogMessage(tr("Failed to set trace record.\n").toUtf8().constData());
+            break;
+        }
+    }
 }
 
 void CPUDisassembly::ActionTraceRecordWordSlot()
 {
-    if(!(DbgFunctions()->SetTraceRecordType(rvaToVa(getInitialSelection()), TRACERECORDTYPE::TraceRecordWordWithExecTypeAndCounter)))
-        GuiAddLogMessage("Failed to set trace record.\n");
+    duint base = mMemPage->getBase();
+    duint size = mMemPage->getSize();
+    for(duint i = base; i < base + size; i += 4096)
+    {
+        if(!(DbgFunctions()->SetTraceRecordType(i, TRACERECORDTYPE::TraceRecordWordWithExecTypeAndCounter)))
+        {
+            GuiAddLogMessage(tr("Failed to set trace record.\n").toUtf8().constData());
+            break;
+        }
+    }
 }
 
 void CPUDisassembly::ActionTraceRecordDisableSlot()
 {
-    if(!(DbgFunctions()->SetTraceRecordType(rvaToVa(getInitialSelection()), TRACERECORDTYPE::TraceRecordNone)))
-        GuiAddLogMessage("Failed to set trace record.\n");
+    duint base = mMemPage->getBase();
+    duint size = mMemPage->getSize();
+    for(duint i = base; i < base + size; i += 4096)
+    {
+        if(!(DbgFunctions()->SetTraceRecordType(i, TRACERECORDTYPE::TraceRecordNone)))
+        {
+            GuiAddLogMessage(tr("Failed to set trace record.\n").toUtf8().constData());
+            break;
+        }
+    }
 }
 
 void CPUDisassembly::mnemonicBriefSlot()

src/gui/Src/Gui/MainWindow.cpp

@@ -234,6 +234,14 @@ MainWindow::MainWindow(QWidget* parent)
     connect(ui->actioneRtr, SIGNAL(triggered()), this, SLOT(execeRtr()));
     connect(ui->actionTicnd, SIGNAL(triggered()), this, SLOT(execTicnd()));
     connect(ui->actionTocnd, SIGNAL(triggered()), this, SLOT(execTocnd()));
+    connect(ui->actionTRBit, SIGNAL(triggered()), this, SLOT(execTRBit()));
+    connect(ui->actionTRByte, SIGNAL(triggered()), this, SLOT(execTRByte()));
+    connect(ui->actionTRWord, SIGNAL(triggered()), this, SLOT(execTRWord()));
+    connect(ui->actionTRNone, SIGNAL(triggered()), this, SLOT(execTRNone()));
+    connect(ui->actionTRTIBT, SIGNAL(triggered()), this, SLOT(execTRTIBT()));
+    connect(ui->actionTRTOBT, SIGNAL(triggered()), this, SLOT(execTRTOBT()));
+    connect(ui->actionTRTIIT, SIGNAL(triggered()), this, SLOT(execTRTIIT()));
+    connect(ui->actionTRTOIT, SIGNAL(triggered()), this, SLOT(execTRTOIT()));
     connect(ui->actionSkipNextInstruction, SIGNAL(triggered()), this, SLOT(execSkip()));
     connect(ui->actionScript, SIGNAL(triggered()), this, SLOT(displayScriptWidget()));
     connect(ui->actionRunSelection, SIGNAL(triggered()), this, SLOT(runSelection()));
@@ -593,6 +601,46 @@ void MainWindow::execRtr()
     DbgCmdExec("rtr");
 }
 
+void MainWindow::execTRBit()
+{
+    mCpuWidget->getDisasmWidget()->ActionTraceRecordBitSlot();
+}
+
+void MainWindow::execTRByte()
+{
+    mCpuWidget->getDisasmWidget()->ActionTraceRecordByteSlot();
+}
+
+void MainWindow::execTRWord()
+{
+    mCpuWidget->getDisasmWidget()->ActionTraceRecordWordSlot();
+}
+
+void MainWindow::execTRNone()
+{
+    mCpuWidget->getDisasmWidget()->ActionTraceRecordDisableSlot();
+}
+
+void MainWindow::execTRTIBT()
+{
+    DbgCmdExec("tibt");
+}
+
+void MainWindow::execTRTOBT()
+{
+    DbgCmdExec("tobt");
+}
+
+void MainWindow::execTRTIIT()
+{
+    DbgCmdExec("tiit");
+}
+
+void MainWindow::execTRTOIT()
+{
+    DbgCmdExec("toit");
+}
+
 void MainWindow::execTicnd()
 {
     if(!DbgIsDebugging())

src/gui/Src/Gui/MainWindow.h

@@ -75,6 +75,14 @@ public slots:
     void execeRun();
     void execeRtr();
     void execSkip();
+    void execTRBit();
+    void execTRByte();
+    void execTRWord();
+    void execTRNone();
+    void execTRTIBT();
+    void execTRTOBT();
+    void execTRTIIT();
+    void execTRTOIT();
     void displayCpuWidget();
     void displaySymbolWidget();
     void displaySourceViewWidget();

src/gui/Src/Gui/MainWindow.ui

@@ -75,6 +75,20 @@
     <property name="title">
      <string>&amp;Debug</string>
     </property>
+    <widget class="QMenu" name="menuTrace_record">
+     <property name="title">
+      <string>Trace record</string>
+     </property>
+     <addaction name="actionTRBit"/>
+     <addaction name="actionTRByte"/>
+     <addaction name="actionTRWord"/>
+     <addaction name="actionTRNone"/>
+     <addaction name="separator"/>
+     <addaction name="actionTRTIBT"/>
+     <addaction name="actionTRTOBT"/>
+     <addaction name="actionTRTIIT"/>
+     <addaction name="actionTRTOIT"/>
+    </widget>
     <addaction name="actionRun"/>
     <addaction name="actioneRun"/>
     <addaction name="actionRunSelection"/>
@@ -93,6 +107,8 @@
     <addaction name="actioneRtr"/>
     <addaction name="actionSkipNextInstruction"/>
     <addaction name="separator"/>
+    <addaction name="menuTrace_record"/>
+    <addaction name="separator"/>
     <addaction name="actionCommand"/>
     <addaction name="separator"/>
     <addaction name="actionHideDebugger"/>
@@ -785,6 +801,46 @@
     <string>Trace into until condition</string>
    </property>
   </action>
+  <action name="actionTRBit">
+   <property name="text">
+    <string>Bit</string>
+   </property>
+  </action>
+  <action name="actionTRByte">
+   <property name="text">
+    <string>Byte</string>
+   </property>
+  </action>
+  <action name="actionTRWord">
+   <property name="text">
+    <string>Word</string>
+   </property>
+  </action>
+  <action name="actionTRTIBT">
+   <property name="text">
+    <string>Trace into beyond trace record</string>
+   </property>
+  </action>
+  <action name="actionTRTOBT">
+   <property name="text">
+    <string>Trace over beyond trace record</string>
+   </property>
+  </action>
+  <action name="actionTRTIIT">
+   <property name="text">
+    <string>Trace into into trace record</string>
+   </property>
+  </action>
+  <action name="actionTRTOIT">
+   <property name="text">
+    <string>Trace over into trace record</string>
+   </property>
+  </action>
+  <action name="actionTRNone">
+   <property name="text">
+    <string>None</string>
+   </property>
+  </action>
  </widget>
  <layoutdefault spacing="6" margin="11"/>
  <resources>