SyscallCallback.c File Reference

Implementation of the functions related to the callback for Syscall. More...

#include "pch.h"

Functions
BOOLEAN	SyscallCallbackInitialize ()
	Initialize the syscall callback.
BOOLEAN	SyscallCallbackIsInitialized ()
	Check whether the syscall callback is initialized.
BOOLEAN	SyscallCallbackUninitialize ()
	Uninitialize the syscall callback.
BOOLEAN	SyscallCallbackStoreProcessInformation (UINT32 ProcessId, UINT32 ThreadId, UINT64 Context, SYSCALL_CALLBACK_CONTEXT_PARAMS *Params)
	This function makes sure to unset the RFLAGS.TF on next trigger of #DB on the target process/thread.
BOOLEAN	SyscallCallbackSetTrapFlagAfterSyscall (GUEST_REGS *Regs, UINT32 ProcessId, UINT32 ThreadId, UINT64 Context, SYSCALL_CALLBACK_CONTEXT_PARAMS *Params)
	Set the trap flag in the guest after a syscall.
BOOLEAN	SyscallCallbackCheckAndHandleAfterSyscallTrapFlags (VIRTUAL_MACHINE_STATE *VCpu, UINT32 ProcessId, UINT32 ThreadId)
	Handle the trap flags as the result of interception of the return of the system-call.
VOID	SyscallCallbackHandleSystemCallHook (VIRTUAL_MACHINE_STATE *VCpu)
	Handle the system call hook callback.

Detailed Description

Implementation of the functions related to the callback for Syscall.

Author

Sina Karvandi (sina@.nosp@m.hype.nosp@m.rdbg..nosp@m.org)

jtaw5649

Version

0.14

Date

2025-06-07

Copyright

This project is released under the GNU Public License v3.

Function Documentation

SyscallCallbackCheckAndHandleAfterSyscallTrapFlags()

BOOLEAN SyscallCallbackCheckAndHandleAfterSyscallTrapFlags	(	VIRTUAL_MACHINE_STATE *	VCpu,
		UINT32	ProcessId,
		UINT32	ThreadId )

Handle the trap flags as the result of interception of the return of the system-call.

Parameters

VCpu	The virtual processor's state
ProcessId	The process id of the thread
ThreadId	The thread id of the thread

Returns

BOOLEAN

{
    RFLAGS                                      Rflags = {0};
    UINT32                                      Index;
    UINT64                                      Context = NULL64_ZERO;
    SYSCALL_CALLBACK_CONTEXT_PARAMS             Params;
    SYSCALL_CALLBACK_PROCESS_THREAD_INFORMATION ProcThrdInfo = {0};
    BOOLEAN                                     Result;
    BOOLEAN                                     ResultToReturn;
 
    //
    // Read the trap flag
    //
    Rflags.AsUInt = HvGetRflags();
 
    if (!Rflags.TrapFlag)
    {
        //
        // The trap flag is not set, so we don't need to do anything
        //
        return FALSE;
    }
 
    //
    // Form the process id and thread id into a 64-bit value
    //
    ProcThrdInfo.Fields.ProcessId = ProcessId;
    ProcThrdInfo.Fields.ThreadId  = ThreadId;
 
    //
    // Make sure, nobody is in the middle of modifying the list
    //
    SpinlockLock(&SyscallCallbackModeTrapListLock);
 
    //
    // *** Search the list of processes/threads for the current process's trap flag state ***
    //
    Result = BinarySearchPerformSearchItem((UINT64 *)&g_SyscallCallbackTrapFlagState->ThreadInformation[0],
                                           g_SyscallCallbackTrapFlagState->NumberOfItems,
                                           &Index,
                                           ProcThrdInfo.asUInt);
 
    //
    // Check whether this thread is expected to have trap flag
    // by the syscall callback or not
    //
    if (Result)
    {
        //
        // Read the context of the caller
        //
        Context = g_SyscallCallbackTrapFlagState->Context[Index];
 
        //
        // Read the (optional) parameters of the caller
        //
        memcpy(&Params, &g_SyscallCallbackTrapFlagState->Params[Index], sizeof(SYSCALL_CALLBACK_CONTEXT_PARAMS));
 
        //
        // Clear the trap flag from the RFLAGS register
        //
        HvSetRflagTrapFlag(FALSE);
 
        //
        // Remove the thread/process from the list of processes/threads
        //
        InsertionSortDeleteItem((UINT64 *)&g_SyscallCallbackTrapFlagState->ThreadInformation[0],
                                &g_SyscallCallbackTrapFlagState->NumberOfItems,
                                Index);
 
        //
        // Handled by the syscall callback
        //
        ResultToReturn = TRUE;
 
        goto ReturnResult;
    }
    else
    {
        //
        // Not related to the syscall callback
        //
        ResultToReturn = FALSE;
 
        goto ReturnResult;
    }
 
ReturnResult:
 
    //
    // Unlock the list modification lock
    //
    SpinlockUnlock(&SyscallCallbackModeTrapListLock);
 
    //
    // Call the callback function to handle the trap flag if its needed
    // Note that we call it here so we already unlocked the list lock
    // to optimize the performance (avoid holding the lock for a long time)
    //
    if (ResultToReturn)
    {
        TransparentCallbackHandleAfterSyscall(VCpu->Regs, ProcessId, ThreadId, Context, &Params);
    }
 
    return ResultToReturn;
}

SyscallCallbackHandleSystemCallHook()

VOID SyscallCallbackHandleSystemCallHook

(

VIRTUAL_MACHINE_STATE *

VCpu

)

Handle the system call hook callback.

Parameters

VCpu	The virtual processor's state

Returns

VOID

{
    TransparentHandleSystemCallHook(VCpu->Regs);
}

SyscallCallbackInitialize()

BOOLEAN SyscallCallbackInitialize

(

)

Initialize the syscall callback.

Returns

BOOLEAN

{
    MSR Msr = {0};
 
    //
    // Check whether the syscall callback was already initialized or not
    //
    if (!g_SyscallCallbackStatus)
    {
        //
        // Insert EPT memory page hook for Windows system call handler, KiSystemCall64()
        //
        Msr.Flags = CpuReadMsr(IA32_LSTAR);
 
        //
        // We set the hook at the address of the system call handler + 3
        // because we don't want to hook the first 3 bytes of the system call handler
        // which is SWAPGS instruction
        //
        g_SystemCallHookAddress = (PVOID)(Msr.Flags + 3);
 
        //
        // Apply the hook from vmx non-root mode
        //
        if (!ConfigureEptHook(g_SystemCallHookAddress, (UINT32)(ULONG_PTR)PsGetCurrentProcessId()))
        {
            // LogInfo("Error while inserting EPT page hook for Windows system call handler at address 0x%p+3", Msr.Flags);
 
            return FALSE;
        }
 
        //
        // Allocate buffer for the syscall callback trap flag state
        //
        g_SyscallCallbackTrapFlagState = (SYSCALL_CALLBACK_TRAP_FLAG_STATE *)PlatformMemAllocateZeroedNonPagedPool(sizeof(SYSCALL_CALLBACK_TRAP_FLAG_STATE));
 
        //
        // Intercept trap flags #DBs and #BPs for the syscall callback
        //
        BroadcastEnableDbAndBpExitingAllCores();
 
        //
        // Enable the syscall callback
        //
        g_SyscallCallbackStatus = TRUE;
 
        //
        // Successfully enabled the syscall callback
        //
        return TRUE;
    }
    else
    {
        return FALSE;
    }
}

SyscallCallbackIsInitialized()

BOOLEAN SyscallCallbackIsInitialized

(

)

Check whether the syscall callback is initialized.

Returns

BOOLEAN

{
    return g_SyscallCallbackStatus;
}

SyscallCallbackSetTrapFlagAfterSyscall()

BOOLEAN SyscallCallbackSetTrapFlagAfterSyscall	(	GUEST_REGS *	Regs,
		UINT32	ProcessId,
		UINT32	ThreadId,
		UINT64	Context,
		SYSCALL_CALLBACK_CONTEXT_PARAMS *	Params )

Set the trap flag in the guest after a syscall.

Parameters

Regs	The virtual processor's state of registers
ProcessId	The process id of the thread
ThreadId	The thread id of the thread
Context	The context of the caller
Params	The (optional) parameters of the caller

Returns

BOOLEAN

{
    //
    // Do not add anything to the list if the syscall callback is not enabled (or disabled by the user)
    //
    if (!g_SyscallCallbackStatus)
    {
        //
        // syscall callback is not enabled
        //
        return FALSE;
    }
 
    //
    // Insert the thread/process into the list of processes/threads
    //
    if (!SyscallCallbackStoreProcessInformation(ProcessId, ThreadId, Context, Params))
    {
        //
        // Failed to store the process/thread information
        //
        return FALSE;
    }
 
    //
    // *** Successfully stored the process/thread information ***
    //
 
    //
    // Set the trap flag to TRUE because we want to intercept the thread again
    // once it returns to the user-mode (SYSRET) instruction
    //
    // Here the RFLAGS is in the R11 register (See Intel manual about the SYSCALL register)
    //
    Regs->r11 |= X86_FLAGS_TF;
 
    //
    // Create log message for the syscall
    //
    // LogInfo("Syscall callback set trap flag for process: %x, thread: %x\n", ProcessId, ThreadId);
 
    return TRUE;
}

SyscallCallbackStoreProcessInformation()

BOOLEAN SyscallCallbackStoreProcessInformation	(	UINT32	ProcessId,
		UINT32	ThreadId,
		UINT64	Context,
		SYSCALL_CALLBACK_CONTEXT_PARAMS *	Params )

This function makes sure to unset the RFLAGS.TF on next trigger of #DB on the target process/thread.

Parameters

ProcessId
ThreadId
Context
Params

Returns

BOOLEAN

{
    UINT32                                      Index;
    BOOLEAN                                     Result;
    BOOLEAN                                     SuccessfullyStored;
    SYSCALL_CALLBACK_PROCESS_THREAD_INFORMATION ProcThrdInfo = {0};
 
    //
    // Form the process id and thread id into a 64-bit value
    //
    ProcThrdInfo.Fields.ProcessId = ProcessId;
    ProcThrdInfo.Fields.ThreadId  = ThreadId;
 
    //
    // Make sure, nobody is in the middle of modifying the list
    //
    SpinlockLock(&SyscallCallbackModeTrapListLock);
 
    //
    // *** Search the list of processes/threads for the current process's trap flag state ***
    //
    Result = BinarySearchPerformSearchItem((UINT64 *)&g_SyscallCallbackTrapFlagState->ThreadInformation[0],
                                           g_SyscallCallbackTrapFlagState->NumberOfItems,
                                           &Index,
                                           ProcThrdInfo.asUInt);
 
    if (Result)
    {
        //
        // It means that we already find this entry in the stored list
        // so, just imply that the addition was successful (no need for extra addition)
        //
        SuccessfullyStored = TRUE;
        goto Return;
    }
    else
    {
        //
        // Insert the thread into the list as the item is not already present
        //
        SuccessfullyStored = InsertionSortInsertItem((UINT64 *)&g_SyscallCallbackTrapFlagState->ThreadInformation[0],
                                                     &g_SyscallCallbackTrapFlagState->NumberOfItems,
                                                     MAXIMUM_NUMBER_OF_THREAD_INFORMATION_FOR_SYSCALL_CALLBACK_TRAPS,
                                                     &Index,
                                                     ProcThrdInfo.asUInt);
 
        if (SuccessfullyStored)
        {
            //
            // Successfully inserted the thread/process into the list
            // Now let's store the context of the caller along with parameters
            //
            g_SyscallCallbackTrapFlagState->Context[Index] = Context;
            memcpy(&g_SyscallCallbackTrapFlagState->Params[Index], Params, sizeof(SYSCALL_CALLBACK_CONTEXT_PARAMS));
        }
 
        goto Return;
    }
 
Return:
    //
    // Unlock the list modification lock
    //
    SpinlockUnlock(&SyscallCallbackModeTrapListLock);
 
    return SuccessfullyStored;
}

SyscallCallbackUninitialize()

BOOLEAN SyscallCallbackUninitialize

(

)

Uninitialize the syscall callback.

Returns

BOOLEAN

{
    if (g_SyscallCallbackStatus)
    {
        //
        // Unset the EPT hook from the syscall entry before disabling state.
        //
        if (!ConfigureEptHookUnHookSingleAddress((UINT64)g_SystemCallHookAddress, (UINT64)NULL, (UINT32)(ULONG_PTR)PsGetCurrentProcessId()))
        {
            LogInfo("Error while removing the EPT hook from windows syscall handler at address 0x%p", g_SystemCallHookAddress);
 
            return FALSE;
        }
 
        //
        // Unset the trap flags #DBs and #BPs for the syscall callback
        //
        BroadcastDisableDbAndBpExitingAllCores();
 
        //
        // Free the buffer for the syscall callback trap flag state
        //
        PlatformMemFreePool(g_SyscallCallbackTrapFlagState);
 
        g_SyscallCallbackTrapFlagState = NULL;
        g_SystemCallHookAddress        = NULL;
        g_SyscallCallbackStatus        = FALSE;
 
        return TRUE;
    }
    else
    {
        return FALSE;
    }
}