From 5156c1ca6b46b6555ab41a89d12462bb35374343 Mon Sep 17 00:00:00 2001 From: Nukem Date: Fri, 27 Nov 2015 23:54:20 -0500 Subject: [PATCH] DBG: LinearPass: more random analysis changes --- src/dbg/BasicBlock.h | 29 +++++++++-------- src/dbg/FunctionPass.cpp | 67 +++++++++++++++++++++++++++++++++------- src/dbg/LinearPass.cpp | 48 +++++++++++++++------------- 3 files changed, 98 insertions(+), 46 deletions(-) diff --git a/src/dbg/BasicBlock.h b/src/dbg/BasicBlock.h index b973fcf9..71fc6391 100644 --- a/src/dbg/BasicBlock.h +++ b/src/dbg/BasicBlock.h @@ -4,26 +4,29 @@ enum BasicBlockFlags : duint { - BASIC_BLOCK_FLAG_NONE = 0, // No flag + BASIC_BLOCK_FLAG_NONE = 0, // No flag - BASIC_BLOCK_FLAG_FUNCTION = (1 << 1), // Scanned; also part of a known function - BASIC_BLOCK_FLAG_CUTOFF = (1 << 3), // Ends prematurely because of another JMP to location - BASIC_BLOCK_FLAG_DELETE = (1 << 4), // Delete element at the next possible time + BASIC_BLOCK_FLAG_FUNCTION = (1 << 1), // Scanned; also part of a known function + BASIC_BLOCK_FLAG_CUTOFF = (1 << 3), // Ends prematurely because of another JMP to location + BASIC_BLOCK_FLAG_DELETE = (1 << 4), // Delete element at the next possible time - BASIC_BLOCK_FLAG_CALL = (1 << 5), // The block ends with a call - BASIC_BLOCK_FLAG_RET = (1 << 6), // The block ends with a retn - BASIC_BLOCK_FLAG_ABSJMP = (1 << 7), // Branch is absolute - BASIC_BLOCK_FLAG_INDIRECT = (1 << 8), // This block ends with an indirect branch - BASIC_BLOCK_FLAG_INDIRPTR = (1 << 9), // This block ends with an indirect branch; pointer known + BASIC_BLOCK_FLAG_CALL = (1 << 5), // The block ends with a call + BASIC_BLOCK_FLAG_RET = (1 << 6), // The block ends with a retn + BASIC_BLOCK_FLAG_ABSJMP = (1 << 7), // Branch is absolute + BASIC_BLOCK_FLAG_INDIRECT = (1 << 8), // This block ends with an indirect branch; memory or register + BASIC_BLOCK_FLAG_INDIRPTR = (1 << 9), // This block ends with an indirect branch; pointer known - BASIC_BLOCK_FLAG_PREPAD = (1 << 10), // Block ends because there was padding afterwards - BASIC_BLOCK_FLAG_PAD = (1 << 11), // Block is only a series of padding instructions + BASIC_BLOCK_FLAG_CALL_TARGET = (1 << 10), // Block is pointed to by a call instruction + //BASIC_BLOCK_FLAG_JMP_TARGET = (1 << 11), + + BASIC_BLOCK_FLAG_PREPAD = (1 << 12), // Block ends because there was padding afterwards + BASIC_BLOCK_FLAG_PAD = (1 << 13), // Block is only a series of padding instructions }; struct BasicBlock { duint VirtualStart; // Inclusive - duint VirtualEnd; // Exclusive + duint VirtualEnd; // Inclusive duint Flags; duint Target; @@ -58,7 +61,7 @@ struct BasicBlock struct FunctionDef { duint VirtualStart; // Inclusive - duint VirtualEnd; // Exclusive + duint VirtualEnd; // Inclusive duint BBlockStart; // Index of first basic block duint BBlockEnd; // Index of last basic block diff --git a/src/dbg/FunctionPass.cpp b/src/dbg/FunctionPass.cpp index 30a85334..ff77dd6b 100644 --- a/src/dbg/FunctionPass.cpp +++ b/src/dbg/FunctionPass.cpp @@ -103,7 +103,7 @@ bool FunctionPass::Analyse() FunctionClear(); for(auto & func : funcs) { - FunctionAdd(func.VirtualStart, func.VirtualEnd - 1, true, func.InstrCount); + FunctionAdd(func.VirtualStart, func.VirtualEnd, true, func.InstrCount); } GuiUpdateAllViews(); @@ -280,6 +280,7 @@ bool FunctionPass::ResolveKnownFunctionEnd(FunctionDef* Function) return true; } +#include "console.h" bool FunctionPass::ResolveFunctionEnd(FunctionDef* Function, BasicBlock* LastBlock) { ASSERT_TRUE(Function->VirtualStart != 0); @@ -301,6 +302,12 @@ bool FunctionPass::ResolveFunctionEnd(FunctionDef* Function, BasicBlock* LastBlo // Loop forever until the end is found for(; (duint)block <= (duint)LastBlock; block++) { + if (block->GetFlag(BASIC_BLOCK_FLAG_CALL_TARGET) && block->VirtualStart != Function->VirtualStart) + { + block--; + break; + } + // Block is now in use block->SetFlag(BASIC_BLOCK_FLAG_FUNCTION); @@ -308,19 +315,57 @@ bool FunctionPass::ResolveFunctionEnd(FunctionDef* Function, BasicBlock* LastBlo Function->InstrCount += block->InstrCount; // Calculate max from just linear instructions - maximumAddr = max(maximumAddr, block->VirtualEnd - 1); + maximumAddr = max(maximumAddr, block->VirtualEnd); // Find maximum jump target if(!block->GetFlag(BASIC_BLOCK_FLAG_CALL) && !block->GetFlag(BASIC_BLOCK_FLAG_INDIRECT)) { - if(block->Target != 0) + if (block->Target != 0 && block->Target >= maximumAddr) { // Here's a problem: Compilers add tail-call elimination with a jump. - // Solve this by creating a maximum jump limit: +/- 1024 bytes from the end. - // - // abs(block->VirtualEnd - block->Target) -- unsigned - if(min(block->VirtualEnd - block->Target, block->Target - block->VirtualEnd) <= 1024) - maximumAddr = max(maximumAddr, block->Target); + // Solve this by creating a maximum jump limit. + auto targetBlock = FindBBlockInRange(block->Target); + + // If (target block found) and (target block is not called) + if (targetBlock && !targetBlock->GetFlag(BASIC_BLOCK_FLAG_CALL_TARGET)) + { + duint blockEnd = targetBlock->VirtualEnd; + + // + // Edge case when a compiler emits: + // + // pop ebp + // jmp some_func + // int3 + // int3 + // some_func: + // push ebp + // + // Where INT3 will align "some_func" to 4, 8, 12, or 16. + // INT3 padding is also optional (if the jump fits perfectly). + // + if (true/*block->GetFlag(BASIC_BLOCK_FLAG_ABSJMP)*/) + { + + { + // Check if padding is aligned to 4 + auto nextBlock = block + 1; + + if ((duint)nextBlock <= (duint)LastBlock) + { + if (nextBlock->GetFlag(BASIC_BLOCK_FLAG_PAD)) + { + // If this block is aligned to 4 bytes at the end + if ((nextBlock->VirtualEnd + 1) % 4 == 0) + blockEnd = block->VirtualEnd; + } + } + } + } + + // Now calculate the maximum end address, taking into account the jump destination + maximumAddr = max(maximumAddr, blockEnd); + } } } @@ -328,7 +373,7 @@ bool FunctionPass::ResolveFunctionEnd(FunctionDef* Function, BasicBlock* LastBlo ASSERT_TRUE(maximumAddr >= block->VirtualStart); // Does this node contain the maximum address? - if(maximumAddr >= block->VirtualStart && maximumAddr < block->VirtualEnd) + if(maximumAddr >= block->VirtualStart && maximumAddr <= block->VirtualEnd) { // It does! There's 4 possibilities next: // @@ -347,9 +392,7 @@ bool FunctionPass::ResolveFunctionEnd(FunctionDef* Function, BasicBlock* LastBlo if(block->GetFlag(BASIC_BLOCK_FLAG_ABSJMP)) { // 2. - // - // abs(block->VirtualEnd - block->Target) -- unsigned - if(min(block->VirtualEnd - block->Target, block->Target - block->VirtualEnd) > 128) + if (block->VirtualEnd == maximumAddr) break; // 3. diff --git a/src/dbg/LinearPass.cpp b/src/dbg/LinearPass.cpp index 40636431..08af636c 100644 --- a/src/dbg/LinearPass.cpp +++ b/src/dbg/LinearPass.cpp @@ -288,30 +288,36 @@ void LinearPass::AnalysisOverlapWorker(duint Start, duint End, BBlockArray* Inse { removal = FindBBlockInRange(curr->Target); - // If the target does not equal the block start... - if(removal && curr->Target != removal->VirtualStart) + if (removal) { - // Mark for deletion - removal->SetFlag(BASIC_BLOCK_FLAG_DELETE); + if (curr->GetFlag(BASIC_BLOCK_FLAG_CALL)) + removal->SetFlag(BASIC_BLOCK_FLAG_CALL_TARGET); - // Block part 1 - BasicBlock block1; - block1.VirtualStart = removal->VirtualStart; - block1.VirtualEnd = curr->Target; - block1.Target = 0; - block1.Flags = BASIC_BLOCK_FLAG_CUTOFF; // Attributes of the top half - block1.InstrCount = removal->InstrCount; + // If the target does not equal the block start... + if (curr->Target != removal->VirtualStart) + { + // Mark for deletion + removal->SetFlag(BASIC_BLOCK_FLAG_DELETE); - // Block part 2 - BasicBlock block2; - block2.VirtualStart = curr->Target; - block2.VirtualEnd = removal->VirtualEnd; - block2.Target = removal->Target; - block2.Flags = removal->Flags; // Attributes of the bottom half (taken from original block) - block2.InstrCount = removal->InstrCount; + // Block part 1 + BasicBlock block1; + block1.VirtualStart = removal->VirtualStart; + block1.VirtualEnd = curr->Target; + block1.Target = 0; + block1.Flags = BASIC_BLOCK_FLAG_CUTOFF; // Attributes of the top half + block1.InstrCount = removal->InstrCount; - Insertions->push_back(block1); - Insertions->push_back(block2); + // Block part 2 + BasicBlock block2; + block2.VirtualStart = curr->Target; + block2.VirtualEnd = removal->VirtualEnd; + block2.Target = removal->Target; + block2.Flags = removal->Flags; // Attributes of the bottom half (taken from original block) + block2.InstrCount = removal->InstrCount; + + Insertions->push_back(block1); + Insertions->push_back(block2); + } } } } @@ -319,7 +325,7 @@ void LinearPass::AnalysisOverlapWorker(duint Start, duint End, BBlockArray* Inse BasicBlock* LinearPass::CreateBlockWorker(std::vector* Blocks, duint Start, duint End, bool Call, bool Jmp, bool Ret, bool Pad) { - BasicBlock block { Start, End, 0, 0, 0 }; + BasicBlock block { Start, End - 1, 0, 0, 0 }; // Check for calls if(Call)