Add support for custom calling convention passing arguments in registers

If the compiler decided to pass an argument in a register on an internal function instead of pushing it on the stack to save time, allow us to specify the register the parameter is going to be in.

DHookAddParam received another parameter to set the register.

DynamicHooks/convention.h

@@ -67,9 +67,11 @@ typedef struct DataTypeSized_s {
 	{
 		type = DATA_TYPE_POINTER;
 		size = 0;
+		custom_register = None;
 	}
 	DataType_t type;
 	size_t size;
+	Register_t custom_register;
 } DataTypeSized_t;
 
 
@@ -184,6 +186,11 @@ public:
 	virtual int GetArgStackSize() = 0;
 	virtual void** GetStackArgumentPtr(CRegisters* pRegisters) = 0;
 
+	/*
+	Returns the number of bytes that the buffer to store all the arguments in that are passed in a register.
+	*/
+	virtual int GetArgRegisterSize() = 0;
+
 	/*
 	Returns a pointer to the argument at the given index.
 

DynamicHooks/conventions/x86MsCdecl.cpp

@@ -78,6 +78,16 @@ ke::Vector<Register_t> x86MsCdecl::GetRegisters()
 		}
 	}
 
+	// Save all the custom calling convention registers as well.
+	for (unsigned int i = 0; i < m_vecArgTypes.length(); i++)
+	{
+		if (m_vecArgTypes[i].custom_register == None)
+			continue;
+
+		// TODO: Make sure the list is unique? Set?
+		registers.append(m_vecArgTypes[i].custom_register);
+	}
+
 	return registers;
 }
 
@@ -92,7 +102,8 @@ int x86MsCdecl::GetArgStackSize()
 
 	for (unsigned int i = 0; i < m_vecArgTypes.length(); i++)
 	{
-		iArgStackSize += m_vecArgTypes[i].size;
+		if (m_vecArgTypes[i].custom_register == None)
+			iArgStackSize += m_vecArgTypes[i].size;
 	}
 
 	return iArgStackSize;
@@ -103,12 +114,36 @@ void** x86MsCdecl::GetStackArgumentPtr(CRegisters* pRegisters)
 	return (void **)(pRegisters->m_esp->GetValue<unsigned long>() + 4);
 }
 
+int x86MsCdecl::GetArgRegisterSize()
+{
+	int iArgRegisterSize = 0;
+
+	for (unsigned int i = 0; i < m_vecArgTypes.length(); i++)
+	{
+		if (m_vecArgTypes[i].custom_register != None)
+			iArgRegisterSize += m_vecArgTypes[i].size;
+	}
+
+	return iArgRegisterSize;
+}
+
 void* x86MsCdecl::GetArgumentPtr(int iIndex, CRegisters* pRegisters)
 {
+	// Check if this argument was passed in a register.
+	if (m_vecArgTypes[iIndex].custom_register != None)
+	{
+		CRegister *pRegister = pRegisters->GetRegister(m_vecArgTypes[iIndex].custom_register);
+		if (!pRegister)
+			return NULL;
+
+		return pRegister->m_pAddress;
+	}
+
 	int iOffset = 4;
 	for(int i=0; i < iIndex; i++)
 	{
-		iOffset += m_vecArgTypes[i].size;
+		if (m_vecArgTypes[i].custom_register == None)
+			iOffset += m_vecArgTypes[i].size;
 	}
 
 	return (void *) (pRegisters->m_esp->GetValue<unsigned long>() + iOffset);

DynamicHooks/conventions/x86MsCdecl.h

@@ -70,6 +70,7 @@ public:
 	virtual int GetPopSize();
 	virtual int GetArgStackSize();
 	virtual void** GetStackArgumentPtr(CRegisters* pRegisters);
+	virtual int GetArgRegisterSize();
 	
 	virtual void* GetArgumentPtr(int iIndex, CRegisters* pRegisters);
 	virtual void ArgumentPtrChanged(int iIndex, CRegisters* pRegisters, void* pArgumentPtr);

DynamicHooks/conventions/x86MsStdcall.cpp

@@ -78,6 +78,16 @@ ke::Vector<Register_t> x86MsStdcall::GetRegisters()
 		}
 	}
 
+	// Save all the custom calling convention registers as well.
+	for (unsigned int i = 0; i < m_vecArgTypes.length(); i++)
+	{
+		if (m_vecArgTypes[i].custom_register == None)
+			continue;
+
+		// TODO: Make sure the list is unique? Set?
+		registers.append(m_vecArgTypes[i].custom_register);
+	}
+
 	return registers;
 }
 
@@ -87,7 +97,8 @@ int x86MsStdcall::GetPopSize()
 
 	for(unsigned int i=0; i < m_vecArgTypes.length(); i++)
 	{
-		iPopSize += m_vecArgTypes[i].size;
+		if (m_vecArgTypes[i].custom_register == None)
+			iPopSize += m_vecArgTypes[i].size;
 	}
 
 	return iPopSize;
@@ -99,7 +110,8 @@ int x86MsStdcall::GetArgStackSize()
 
 	for (unsigned int i = 0; i < m_vecArgTypes.length(); i++)
 	{
-		iArgStackSize += m_vecArgTypes[i].size;
+		if (m_vecArgTypes[i].custom_register == None)
+			iArgStackSize += m_vecArgTypes[i].size;
 	}
 
 	return iArgStackSize;
@@ -110,15 +122,39 @@ void** x86MsStdcall::GetStackArgumentPtr(CRegisters* pRegisters)
 	return (void **)(pRegisters->m_esp->GetValue<unsigned long>() + 4);
 }
 
-void* x86MsStdcall::GetArgumentPtr(int iIndex, CRegisters* pRegisters)
+int x86MsStdcall::GetArgRegisterSize()
 {
-	int iOffset = 4;
-	for(int i=0; i < iIndex; i++)
+	int iArgRegisterSize = 0;
+
+	for (unsigned int i = 0; i < m_vecArgTypes.length(); i++)
 	{
-		iOffset += m_vecArgTypes[i].size;
+		if (m_vecArgTypes[i].custom_register != None)
+			iArgRegisterSize += m_vecArgTypes[i].size;
 	}
 
-	return (void *) (pRegisters->m_esp->GetValue<unsigned long>() + iOffset);
+	return iArgRegisterSize;
+}
+
+void* x86MsStdcall::GetArgumentPtr(int iIndex, CRegisters* pRegisters)
+{
+	// Check if this argument was passed in a register.
+	if (m_vecArgTypes[iIndex].custom_register != None)
+	{
+		CRegister *pRegister = pRegisters->GetRegister(m_vecArgTypes[iIndex].custom_register);
+		if (!pRegister)
+			return NULL;
+
+		return pRegister->m_pAddress;
+	}
+
+	int iOffset = 4;
+	for (int i = 0; i < iIndex; i++)
+	{
+		if (m_vecArgTypes[i].custom_register == None)
+			iOffset += m_vecArgTypes[i].size;
+	}
+
+	return (void *)(pRegisters->m_esp->GetValue<unsigned long>() + iOffset);
 }
 
 void x86MsStdcall::ArgumentPtrChanged(int iIndex, CRegisters* pRegisters, void* pArgumentPtr)

DynamicHooks/conventions/x86MsStdcall.h

@@ -70,6 +70,7 @@ public:
 	virtual int GetPopSize();
 	virtual int GetArgStackSize();
 	virtual void** GetStackArgumentPtr(CRegisters* pRegisters);
+	virtual int GetArgRegisterSize();
 	
 	virtual void* GetArgumentPtr(int iIndex, CRegisters* pRegisters);
 	virtual void ArgumentPtrChanged(int iIndex, CRegisters* pRegisters, void* pArgumentPtr);

DynamicHooks/conventions/x86MsThiscall.cpp

@@ -66,6 +66,7 @@ ke::Vector<Register_t> x86MsThiscall::GetRegisters()
 	ke::Vector<Register_t> registers;
 	
 	registers.append(ESP);
+	// TODO: Allow custom this register.
 	registers.append(ECX);
 
 	if (m_returnType.type == DATA_TYPE_FLOAT || m_returnType.type == DATA_TYPE_DOUBLE)
@@ -81,6 +82,16 @@ ke::Vector<Register_t> x86MsThiscall::GetRegisters()
 		}
 	}
 
+	// Save all the custom calling convention registers as well.
+	for (unsigned int i = 0; i < m_vecArgTypes.length(); i++)
+	{
+		if (m_vecArgTypes[i].custom_register == None)
+			continue;
+
+		// TODO: Make sure the list is unique? Set?
+		registers.append(m_vecArgTypes[i].custom_register);
+	}
+
 	return registers;
 }
 
@@ -93,7 +104,9 @@ int x86MsThiscall::GetPopSize()
 
 	for(unsigned int i=0; i < m_vecArgTypes.length(); i++)
 	{
-		iPopSize += m_vecArgTypes[i].size;
+		// Only pop arguments that are actually on the stack.
+		if (m_vecArgTypes[i].custom_register == None)
+			iPopSize += m_vecArgTypes[i].size;
 	}
 
 	return iPopSize;
@@ -105,7 +118,8 @@ int x86MsThiscall::GetArgStackSize()
 
 	for (unsigned int i = 0; i < m_vecArgTypes.length(); i++)
 	{
-		iArgStackSize += m_vecArgTypes[i].size;
+		if (m_vecArgTypes[i].custom_register == None)
+			iArgStackSize += m_vecArgTypes[i].size;
 	}
 
 	return iArgStackSize;
@@ -116,17 +130,42 @@ void** x86MsThiscall::GetStackArgumentPtr(CRegisters* pRegisters)
 	return (void **)(pRegisters->m_esp->GetValue<unsigned long>() + 4);
 }
 
+int x86MsThiscall::GetArgRegisterSize()
+{
+	int iArgRegisterSize = 0;
+
+	for (unsigned int i = 0; i < m_vecArgTypes.length(); i++)
+	{
+		if (m_vecArgTypes[i].custom_register != None)
+			iArgRegisterSize += m_vecArgTypes[i].size;
+	}
+
+	return iArgRegisterSize;
+}
+
 void* x86MsThiscall::GetArgumentPtr(int iIndex, CRegisters* pRegisters)
 {
 	if (iIndex == 0)
 	{
+		// TODO: Allow custom this register.
 		return pRegisters->m_ecx->m_pAddress;
 	}
 
+	// Check if this argument was passed in a register.
+	if (m_vecArgTypes[iIndex-1].custom_register != None)
+	{
+		CRegister *pRegister = pRegisters->GetRegister(m_vecArgTypes[iIndex-1].custom_register);
+		if (!pRegister)
+			return NULL;
+
+		return pRegister->m_pAddress;
+	}
+
 	int iOffset = 4;
 	for(int i=0; i < iIndex-1; i++)
 	{
-		iOffset += m_vecArgTypes[i].size;
+		if (m_vecArgTypes[i].custom_register == None)
+			iOffset += m_vecArgTypes[i].size;
 	}
 
 	return (void *) (pRegisters->m_esp->GetValue<unsigned long>() + iOffset);

DynamicHooks/conventions/x86MsThiscall.h

@@ -69,8 +69,9 @@ public:
 
 	virtual ke::Vector<Register_t> GetRegisters();
 	virtual int GetPopSize();
-	virtual int x86MsThiscall::GetArgStackSize();
+	virtual int GetArgStackSize();
 	virtual void** GetStackArgumentPtr(CRegisters* pRegisters);
+	virtual int GetArgRegisterSize();
 	
 	virtual void* GetArgumentPtr(int iIndex, CRegisters* pRegisters);
 	virtual void ArgumentPtrChanged(int iIndex, CRegisters* pRegisters, void* pArgumentPtr);

DynamicHooks/registers.cpp

@@ -378,4 +378,128 @@ void CRegisters::DeleteRegister(CRegister* pRegister)
 	{
 		delete pRegister;
 	}
+}
+
+CRegister* CRegisters::GetRegister(Register_t reg)
+{
+	switch (reg)
+	{
+	case AL:
+		return m_al;
+	case CL:
+		return m_cl;
+	case DL:
+		return m_dl;
+	case BL:
+		return m_bl;
+	case AH:
+		return m_ah;
+	case CH:
+		return m_ch;
+	case DH:
+		return m_dh;
+	case BH:
+		return m_bh;
+
+	case AX:
+		return m_ax;
+	case CX:
+		return m_cx;
+	case DX:
+		return m_dx;
+	case BX:
+		return m_bx;
+	case SP:
+		return m_sp;
+	case BP:
+		return m_bp;
+	case SI:
+		return m_si;
+	case DI:
+		return m_di;
+
+	case EAX:
+		return m_eax;
+	case ECX:
+		return m_ecx;
+	case EDX:
+		return m_edx;
+	case EBX:
+		return m_ebx;
+	case ESP:
+		return m_esp;
+	case EBP:
+		return m_ebp;
+	case ESI:
+		return m_esi;
+	case EDI:
+		return m_edi;
+
+	case MM0:
+		return m_mm0;
+	case MM1:
+		return m_mm1;
+	case MM2:
+		return m_mm2;
+	case MM3:
+		return m_mm3;
+	case MM4:
+		return m_mm4;
+	case MM5:
+		return m_mm5;
+	case MM6:
+		return m_mm6;
+	case MM7:
+		return m_mm7;
+
+	case XMM0:
+		return m_xmm0;
+	case XMM1:
+		return m_xmm1;
+	case XMM2:
+		return m_xmm2;
+	case XMM3:
+		return m_xmm3;
+	case XMM4:
+		return m_xmm4;
+	case XMM5:
+		return m_xmm5;
+	case XMM6:
+		return m_xmm6;
+	case XMM7:
+		return m_xmm7;
+
+	case CS:
+		return m_cs;
+	case SS:
+		return m_ss;
+	case DS:
+		return m_ds;
+	case ES:
+		return m_es;
+	case FS:
+		return m_fs;
+	case GS:
+		return m_gs;
+
+	case ST0:
+		return m_st0;
+	case ST1:
+		return m_st1;
+	case ST2:
+		return m_st2;
+	case ST3:
+		return m_st3;
+	case ST4:
+		return m_st4;
+	case ST5:
+		return m_st5;
+	case ST6:
+		return m_st6;
+	case ST7:
+		return m_st7;
+
+	default:
+		return NULL;
+	}
 }

DynamicHooks/registers.h

@@ -43,6 +43,9 @@
 // ============================================================================
 enum Register_t
 {
+	// No register at all.
+	None,
+
 	// ========================================================================
 	// >> 8-bit General purpose registers
 	// ========================================================================
@@ -263,6 +266,8 @@ public:
 	CRegisters(ke::Vector<Register_t> registers);
 	~CRegisters();
 
+	CRegister* GetRegister(Register_t reg);
+
 private:
 	CRegister* CreateRegister(ke::Vector<Register_t>& registers, Register_t reg, int iSize);
 	void DeleteRegister(CRegister* pRegister);