Download GenericDX.7z
Surprisingly I haven't found any existing information about this type of hooking and decided to implement it myself. The whole idea I believe belongs to Indy(Clerk), but there wasn't any code back then.
The point of this approach is to destroy some pointer value then catch generated exception and trace code up to target function. Full tracing can add significant performance hit, so trace engine skips function calls by invalidating return address upon entering each new function and waiting for exception on function return.
Pros:
- No code or page protection modifications. Even pointers in the heap can be used
- Harder to detect than other methods
- Hooking point may belong to virtually any function involved in target function execution path
Cons:
- Hard to implement
- Each hook must be carefully analyzed beforehand
- Can add noticeable performance hit
Also my implementation has multithreading issues and I still haven't found stable way to distinguish different hook tracing paths in exception handler.
Example:
I took Text3D.exe sample from directx SDK to demonstrate how it all works. Target functions are BeginScene and EndScene.
Application execution flow:
DXUTRender3DEnvironment9->OnFrameRender->BeginScene...EndScene.
Although it is possible to hook functions directly from OnFrameRender, I've chosen to destroy pointer to OnFrameMove for the sake of demonstration.
DXUTRender3DEnvironment9 fragment:
The point of this approach is to destroy some pointer value then catch generated exception and trace code up to target function. Full tracing can add significant performance hit, so trace engine skips function calls by invalidating return address upon entering each new function and waiting for exception on function return.
Pros:
- No code or page protection modifications. Even pointers in the heap can be used
- Harder to detect than other methods
- Hooking point may belong to virtually any function involved in target function execution path
Cons:
- Hard to implement
- Each hook must be carefully analyzed beforehand
- Can add noticeable performance hit
Also my implementation has multithreading issues and I still haven't found stable way to distinguish different hook tracing paths in exception handler.
Example:
I took Text3D.exe sample from directx SDK to demonstrate how it all works. Target functions are BeginScene and EndScene.
Application execution flow:
DXUTRender3DEnvironment9->OnFrameRender->BeginScene...EndScene.
Although it is possible to hook functions directly from OnFrameRender, I've chosen to destroy pointer to OnFrameMove for the sake of demonstration.
DXUTRender3DEnvironment9 fragment:
PHP Code:
v15 = GetDXUTState();
v40 = DXUTState::GetFrameMoveFunc(v15);
if ( !v40
|| (v16 = GetDXUTState(),
v17 = DXUTState::GetFrameMoveFuncUserContext(v16),
((void (__stdcall *)(_DWORD, _DWORD, _DWORD, int))v40)(LODWORD(v37), HIDWORD(v37), LODWORD(v39), v17), // OnFrameMove
(v1 = DXUTGetD3D9Device()) != 0) )
{
v18 = GetDXUTState();
if ( !(unsigned __int8)DXUTState::GetRenderingPaused(v18) )
{
v19 = GetDXUTState();
v40 = DXUTState::GetD3D9FrameRenderFunc(v19);
if ( v40 )
{
v20 = GetDXUTState();
v21 = DXUTState::GetD3D9FrameRenderFuncUserContext(v20);
((void (__stdcall *)(int, _DWORD, _DWORD, _DWORD, int))v40)(v1, LODWORD(v37), HIDWORD(v37), LODWORD(v39), v21); // OnFrameRender 1. Return from OnFrameMove
2. Single-step and skip function calls until OnFrameRender is invoked.
3. Single-step and skip function calls until BeginScene and EndScene are invoked.
Now to the code.
Hook installation (dllmain.cpp):
PHP Code:
#include "HookMgr.h"
using namespace hook;
#ifdef _M_AMD64
#define STORAGE_OFST 0x70EF8
#define HOOK_OFST 0x420//0x468
#define FUNC_PTR 0x47B58
#else
#define STORAGE_OFST 0x67F8C
#define HOOK_OFST 0x370//0x394
#define FUNC_PTR 0x45F73
#endif
typedef HRESULT( __stdcall* fnEndScene )(IDirect3DDevice9*); fnEndScene pEndScene = nullptr; fnEndScene pBeginScene = nullptr;
HRESULT __stdcall hkEndScene( IDirect3DDevice9* arg )
{
OutputDebugStringA( "hkEndScene called\n" );
return pEndScene( arg );
}
HRESULT __stdcall hkBeginScene( IDirect3DDevice9* arg )
{
OutputDebugStringA( "hkBeginScene called\n" );
return pBeginScene( arg );
}
void Hook()
{
uintptr_t hMain = (uintptr_t)GetModuleHandleW( NULL );
uint8_t* pDxStorage = *(uint8_t**)(hMain + STORAGE_OFST);
uintptr_t* pDxDevice = *(uintptr_t**)(pDxStorage + sizeof(void*));
pEndScene = *(fnEndScene*)(*pDxDevice + 0x2A * sizeof(void*));
pBeginScene = *(fnEndScene*)(*pDxDevice + 0x29 * sizeof(void*));
HookContext::vecState path{ TS_StepOut, { TS_StepInto, hMain + FUNC_PTR }, TS_Step };
HookMgr::Instance().ApplyHook( pEndScene, &hkEndScene, pDxStorage + HOOK_OFST, path );
HookMgr::Instance().ApplyHook( pBeginScene, &hkBeginScene, pDxStorage + HOOK_OFST );
}
void Unhook()
{
HookMgr::Instance().RemoveHook( pEndScene );
HookMgr::Instance().RemoveHook( pBeginScene );
Sleep( 100 );
}
BOOL APIENTRY DllMain( HMODULE /*hModule*/, DWORD ul_reason_for_call, LPVOID /*lpReserved*/ )
{
switch (ul_reason_for_call)
{
case DLL_PROCESS_ATTACH:
Hook();
break;
case DLL_PROCESS_DETACH:
Unhook();
break;
}
return TRUE;
} Hookmgr.h:
PHP Code:
#pragma once
#include "WinHeaders.h"
#include <stdint.h>
#include <vector>
#include <map>
#include <unordered_map>
namespace hook { enum TraceState {
TS_Start, // Initial state. Internal use only
TS_Step, // Do single-step
TS_StepOut, // Break on function return
TS_StepInto, // Step into specific function
TS_WaitReturn, // Wait for break-on-return };
struct PathNode {
TraceState action;
uintptr_t arg;
PathNode( TraceState _action, uintptr_t _arg = 0 )
: action( _action )
, arg( _arg ) { }
};
/// <summary>
/// Hook-related data
/// </summary> struct HookContext {
typedef std::unordered_map<uintptr_t, std::pair<uintptr_t, bool>> mapHooks;
typedef std::vector<PathNode> vecState;
uintptr_t lastIP = 0; // Previous EIP/RIP value
uintptr_t lastSP = 0; // Previous ESP/RSP value
uintptr_t targetPtr = 0; // Address causing exception
uintptr_t origPtrVal = 0; // Original pointer value
uintptr_t checkIP = 0; // Address of instruction that checks target pointer
uintptr_t breakValue = 0; // Value used to generate exception
uintptr_t stateIdx = 0; // Current state index in state vector
TraceState state = TS_Start; // Current tracing state
vecState tracePath; // Function trace path
mapHooks hooks; // List of hooks associated with current pointer
/// <summary>
/// Reset tracing state
/// </summary>
void reset()
{
state = TS_Start;
lastIP = lastSP = 0;
stateIdx = 0;
// Mark hooks as non-called
for (auto& item : hooks)
item.second.second = false;
}
};
class HookMgr {
public:
typedef std::map<uintptr_t, HookContext> mapContext;
typedef std::vector <std::pair<uintptr_t, uintptr_t>> vecStackFrames;
public:
~HookMgr();
static HookMgr& Instance();
/// <summary>
/// Setup hook
/// </summary>
/// <param name="targetFunc">Target function to be hooked</param>
/// <param name="hookFunc">New function</param>
/// <param name="ptrAddress">Address of pointer to destroy</param>
/// <param name="tracePath">Function tracing path</param>
/// <param name="checkIP">Optional. Address of instruction that checks target pointer</param>
/// <returns>true on success, false if already hooked</returns>
bool ApplyHook( void* targetFunc,
void* hookFunc,
void* ptrAddress,
const HookContext::vecState& tracePath = HookContext::vecState(),
void* checkIP = 0 );
/// <summary>
/// Remove existing hook
/// </summary>
/// <param name="targetFunc">Target function ptr</param>
/// <returns>true on success, false if not found</returns>
bool RemoveHook( void* targetFunc );
private:
//
// Singleton
//
HookMgr();
HookMgr( const HookMgr& ) = delete;
HookMgr& operator =( const HookMgr& ) = delete;
//
// Exception handlers
//
static LONG __stdcall VecHandler( PEXCEPTION_POINTERS ExceptionInfo );
LONG VecHandlerP( PEXCEPTION_POINTERS ExceptionInfo );
/// <summary>
/// Capture stack frames
/// </summary>
/// <param name="ip">Current instruction pointer</param>
/// <param name="sp">Current stack pointer</param>
/// <param name="results">Found frames.</param>
/// <param name="depth">Frame depth limit</param>
/// <returns>Number of found frames</returns>
size_t StackBacktrace( uintptr_t ip, uintptr_t sp, vecStackFrames& results, uintptr_t depth = 10 );
/// <summary>
/// Setup exception upon function return
/// </summary>
/// <param name="ExceptionInfo">The exception information</param>
inline void BreakOnReturn( uintptr_t sp );
/// <summary>
/// Check if last instruction caused branching
/// </summary>
/// <param name="ctx">Current hook info</param>
/// <param name="ip">Instruction pointer</param>
/// <param name="sp">Stack pointer</param>
/// <returns>True if branching has occurred</returns>
bool CheckBranching( const HookContext& ctx, uintptr_t ip, uintptr_t sp );
/// <summary>
/// Handle branching
/// </summary>
/// <param name="ctx">Current hook context</param>
/// <param name="exptContex">Thread context</param>
void HandleBranch( HookContext& ctx, PCONTEXT exptContex );
/// <summary>
/// Restore original pointer value
/// </summary>
/// <param name="ctx">The CTX.</param>
/// <param name="ExceptionInfo">The exception information</param>
/// <returns>true on success, false if no invalid register was found</returns>
bool RestorePtr( const HookContext& ctx, PEXCEPTION_POINTERS ExceptionInfo );
private:
PVOID _pExptHandler = nullptr; // Exception handler
mapContext _contexts; // Hook contexts
uintptr_t _breakPtr = 0x2000; // Exception pointer generator
uintptr_t _lastCtx = 0;
};
} PHP Code:
#define BEA_USE_STDCALL
#include "BeaEngine/headers/BeaEngine.h"
#include <algorithm>
#define HIGHEST_BIT_SET (1LL << (sizeof(void*) * 8 - 1))
#define HIGHEST_BIT_UNSET ~HIGHEST_BIT_SET
#define SingleStep 0x100
// Architecture-specific
#ifdef _M_AMD64
#define Nip Rip
#define Nsp Rsp
#define Ndi Rdi
#define Nax Rax
#define ADDR_MASK 0xFFFFFFFFFFFFF000
#else
#define Nip Eip
#define Nsp Esp
#define Ndi Edi
#define Nax Eax
#define ADDR_MASK 0xFFFFF000
#endif
namespace hook {
HookMgr::HookMgr()
{
}
HookMgr::~HookMgr()
{
if (_pExptHandler != nullptr)
RemoveVectoredExceptionHandler( _pExptHandler );
}
HookMgr& HookMgr::Instance()
{
static HookMgr instance;
return instance;
}
/// <summary>
/// Setup hook
/// </summary>
/// <param name="targetFunc">Target function to be hooked</param>
/// <param name="hookFunc">New function</param>
/// <param name="ptrAddress">Address of pointer to destroy</param>
/// <param name="tracePath">Function tracing path</param>
/// <param name="checkIP">Optional. Address of instruction that checks target pointer</param>
/// <returns>true on success, false if already hooked</returns> bool HookMgr::ApplyHook( void* targetFunc,
void* hookFunc,
void* ptrAddress,
const HookContext::vecState& tracePath /*= HookContext::vecState()*/,
void* chekcIP /*= 0*/)
{
// Pointer already present
if (_contexts.count( (uintptr_t)ptrAddress ))
{
HookContext& ctx = _contexts[(uintptr_t)ptrAddress];
// Already hooked
if (ctx.hooks.count( (uintptr_t)targetFunc ))
return false;
else
ctx.hooks.emplace( std::make_pair( (uintptr_t)targetFunc, std::make_pair( (uintptr_t)hookFunc, false ) ) );
}
// Create new context
else
{
HookContext ctx;
// Setup context
ctx.targetPtr = (uintptr_t)ptrAddress;
ctx.checkIP = (uintptr_t)chekcIP;
ctx.origPtrVal = *(uintptr_t*)ptrAddress;
ctx.breakValue = _breakPtr;
ctx.tracePath = tracePath;
ctx.hooks.emplace( std::make_pair( (uintptr_t)targetFunc, std::make_pair( (uintptr_t)hookFunc, false ) ) );
_contexts.emplace( std::make_pair( (uintptr_t)ptrAddress, std::move( ctx ) ) );
if (_pExptHandler == nullptr)
_pExptHandler = AddVectoredExceptionHandler( 0, &HookMgr::VecHandler );
// Setup exception
*(uintptr_t*)ptrAddress = ctx.breakValue;
_breakPtr += 0x10;
}
return true;
}
/// <summary>
/// Remove existing hook
/// </summary>
/// <param name="targetFunc">Target function ptr</param>
/// <returns>true on success, false if not found</returns> bool HookMgr::RemoveHook( void* targetFunc )
{
auto findfn = [targetFunc]( const mapContext::value_type& val ) {
return val.second.hooks.count( (uintptr_t)targetFunc );
};
// Get context containing target function
auto iter = std::find_if( _contexts.begin(), _contexts.end(), findfn );
if (iter != _contexts.end())
{
auto& ctx = iter->second;
// Remove function from list
ctx.hooks.erase( (uintptr_t)targetFunc );
if (ctx.hooks.empty())
{
// Remove hook itself
*(uintptr_t*)ctx.targetPtr = ctx.origPtrVal;
_contexts.erase( iter );
}
// Before handler is removed we need to make sure no thread will
// generate exception again
Sleep( 10 );
// Remove exception handler
if (_contexts.empty() && _pExptHandler != nullptr)
{
RemoveVectoredExceptionHandler( _pExptHandler );
_pExptHandler = nullptr;
}
return true;
}
return false;
}
/// <summary>
/// Exception handler
/// </summary>
/// <param name="ExceptionInfo">The exception information</param>
/// <returns>Handling status</returns> LONG __stdcall HookMgr::VecHandler( PEXCEPTION_POINTERS ExceptionInfo )
{
return Instance().VecHandlerP( ExceptionInfo );
}
/// <summary>
/// Exception handler
/// </summary>
/// <param name="ExceptionInfo">The exception information</param>
/// <returns>Handling status</returns> LONG HookMgr::VecHandlerP( PEXCEPTION_POINTERS ExceptionInfo )
{
auto exptContex = ExceptionInfo->ContextRecord;
auto exptRecord = ExceptionInfo->ExceptionRecord;
auto exptCode = exptRecord->ExceptionCode;
// TODO: Somehow determine current hook context
HookContext* ctx = &_contexts.begin()->second;
// Check if exception should be handled
if (exptCode != EXCEPTION_SINGLE_STEP && exptCode != EXCEPTION_ACCESS_VIOLATION)
{
return EXCEPTION_CONTINUE_SEARCH;
}
else if (exptCode == EXCEPTION_ACCESS_VIOLATION && (ctx->state == TS_Step || ctx->state == TS_StepInto))
{
if ((exptRecord->ExceptionInformation[1] & ADDR_MASK) != (ctx->breakValue & ADDR_MASK))
return EXCEPTION_CONTINUE_SEARCH;
// Pointer accessed by non-target address
if (ctx->checkIP != 0 && exptContex->Nip != ctx->checkIP)
{
exptContex->EFlags |= SingleStep;
RestorePtr( *ctx, ExceptionInfo );
return EXCEPTION_CONTINUE_EXECUTION;
}
}
switch (ctx->state)
{
// Start of tracing
case TS_Start:
{
ctx->state = ctx->tracePath[ctx->stateIdx].action;
RestorePtr( *ctx, ExceptionInfo );
return VecHandlerP( ExceptionInfo );
}
break;
// Single step
case TS_Step:
{
// Function call occurred. Queue break on return.
if (CheckBranching( *ctx, exptContex->Nip, exptContex->Nsp ))
{
// Target function reached
if (ctx->hooks.count( exptContex->Nip ))
{
HandleBranch( *ctx, exptContex );
return EXCEPTION_CONTINUE_EXECUTION;
}
else
{
ctx->state = TS_WaitReturn;
BreakOnReturn( exptContex->Nsp );
}
}
// Step further
else
exptContex->EFlags |= SingleStep;
}
break;
// Step out from function
case TS_StepOut:
{
// Get current stack frame
vecStackFrames frames;
StackBacktrace( exptContex->Nip, exptContex->Nsp, frames, 1 );
if (frames.size() > 1)
{
ctx->stateIdx++;
ctx->state = TS_WaitReturn;
BreakOnReturn( frames.back().first );
}
}
break;
// Step into specific function
case TS_StepInto:
{
// Check if step into path function has occurred
if (CheckBranching( *ctx, exptContex->Nip, exptContex->Nsp ))
{
if (exptContex->Nip == ctx->tracePath[ctx->stateIdx].arg)
{
ctx->stateIdx++;
ctx->state = ctx->tracePath[ctx->stateIdx].action;
}
}
exptContex->EFlags |= SingleStep;
}
break;
// Break on 'ret' instruction
case TS_WaitReturn:
{
exptContex->Nip &= HIGHEST_BIT_UNSET;
// Restore stack for x64
#ifdef _M_AMD64
*(uintptr_t*)exptContex->Nsp &= HIGHEST_BIT_UNSET;
#endif // _M_AMD64
// Continue stepping
exptContex->EFlags |= SingleStep;
ctx->state = ctx->tracePath[ctx->stateIdx].action;
}
break;
default:
break;
}
ctx->lastIP = exptContex->Nip;
ctx->lastSP = exptContex->Nsp;
return EXCEPTION_CONTINUE_EXECUTION;
}
/// <summary>
/// Check if last instruction caused branching
/// </summary>
/// <param name="ctx">Current hook info</param>
/// <param name="ip">Instruction pointer</param>
/// <param name="sp">Stack pointer</param>
/// <returns>True if branching has occurred</returns> bool HookMgr::CheckBranching( const HookContext& ctx, uintptr_t ip, uintptr_t sp )
{
// Not yet initialized
if (ctx.lastIP == 0 || ctx.lastSP == 0)
return false;
// Difference in instruction pointer more than possible 'call' length
// Stack pointer changed
if (ip - ctx.lastIP >= 8 && sp != ctx.lastSP)
{
DISASM info = { 0 };
info.EIP = ctx.lastIP;
#ifdef _M_AMD64
info.Archi = 64;
#endif
// Double-check call instruction using disasm
if (Disasm( &info ) > 0 && info.Instruction.BranchType == CallType)
return true;
}
return false;
}
/// <summary>
/// Handle branching
/// </summary>
/// <param name="ctx">Current hook context</param>
/// <param name="exptContex">Thread context</param> void HookMgr::HandleBranch( HookContext& ctx, PCONTEXT exptContex )
{
// Mark this hook as called
ctx.hooks[exptContex->Nip].second = true;
// Reset tracing state if all hooks were called
auto iter = std::find_if( ctx.hooks.begin(), ctx.hooks.end(),
[]( const decltype(ctx.hooks)::value_type& val ){ return (val.second.second == false); } );
// Break after hook execution
if (iter != ctx.hooks.end())
{
ctx.state = TS_WaitReturn;
BreakOnReturn( exptContex->Nsp );
}
// Reset state
else
ctx.reset();
// Reroute to hook function
exptContex->Nip = ctx.hooks[exptContex->Nip].first;
}
/// <summary>
/// Setup exception upon function return
/// </summary>
/// <param name="ExceptionInfo">The exception information</param> inline void HookMgr::BreakOnReturn( uintptr_t sp )
{
*(DWORD_PTR*)sp |= HIGHEST_BIT_SET;
}
/// <summary>
/// Restore original pointer value
/// </summary>
/// <param name="ctx">The CTX.</param>
/// <param name="ExceptionInfo">The exception information</param>
/// <returns>true on success, false if no invalid register was found</returns> bool HookMgr::RestorePtr( const HookContext& ctx, PEXCEPTION_POINTERS ExceptionInfo )
{
bool found = false;
auto expCtx = ExceptionInfo->ContextRecord;
// Exception on execute
if (ExceptionInfo->ExceptionRecord->ExceptionInformation[0] == 8)
{
expCtx->Nip = ctx.origPtrVal;
return true;
}
// Exception on read/write
#ifdef _M_AMD64
for (DWORD_PTR* pRegVal = &expCtx->Rax; pRegVal <= &expCtx->R15; pRegVal++) #else
for (DWORD_PTR* pRegVal = &expCtx->Ndi; pRegVal <= &expCtx->Nax; pRegVal++) #endif
{
// Compare high address parts
if ((*pRegVal & ADDR_MASK) == (ExceptionInfo->ExceptionRecord->ExceptionInformation[1] & ADDR_MASK))
{
*pRegVal = ctx.origPtrVal;
found = true;
}
}
return found;
}
/// <summary>
/// Capture stack frames
/// </summary>
/// <param name="ip">Current instruction pointer</param>
/// <param name="sp">Current stack pointer</param>
/// <param name="results">Found frames.</param>
/// <param name="depth">Frame depth limit</param>
/// <returns>Number of found frames</returns> size_t HookMgr::StackBacktrace( uintptr_t ip, uintptr_t sp, vecStackFrames& results, uintptr_t depth /*= 10 */ )
{
SYSTEM_INFO sysinfo = { 0 };
uintptr_t stack_base = (uintptr_t)((PNT_TIB)NtCurrentTeb())->StackBase;
GetNativeSystemInfo( &sysinfo );
// Store exception address
results.emplace_back( std::make_pair( 0, ip ) );
// Walk stack
for (uintptr_t stackPtr = sp; stackPtr < stack_base && results.size() <= depth; stackPtr += sizeof(void*))
{
uintptr_t stack_val = *(uintptr_t*)stackPtr;
MEMORY_BASIC_INFORMATION meminfo = { 0 };
// Decode value
uintptr_t original = stack_val & HIGHEST_BIT_UNSET;
// Invalid value
if ( original < (uintptr_t)sysinfo.lpMinimumApplicationAddress ||
original > (uintptr_t)sysinfo.lpMaximumApplicationAddress)
{
continue;
}
// Check if memory is executable
if (VirtualQuery( (LPVOID)original, &meminfo, sizeof(meminfo) ) != sizeof(meminfo))
continue;
if ( meminfo.Protect != PAGE_EXECUTE_READ &&
meminfo.Protect != PAGE_EXECUTE_WRITECOPY &&
meminfo.Protect != PAGE_EXECUTE_READWRITE)
{
continue;
}
// Detect 'call' instruction
for (uintptr_t j = 1; j < 8; j++)
{
DISASM info = { 0 };
info.EIP = original - j;
#ifdef _M_AMD64
info.Archi = 64;
#endif
if (Disasm( &info ) > 0 && info.Instruction.BranchType == CallType)
{
results.emplace_back( std::make_pair( stackPtr, stack_val ) );
break;
}
}
}
return results.size();
}
} The code probably has some bugs and is subject to improve.
