From 250bbf5955bbe7420810f0276d5d76c64d3108e2 Mon Sep 17 00:00:00 2001
From: peace-maker <peace-maker@wcfan.de>
Date: Tue, 31 May 2022 14:58:36 +0200
Subject: [PATCH] DHooks: Error on argument passflags for detours (#1773)

The passflags are only supported by SourceHook for virtual hooks and are ignored for detours with DynamicDetours. This caused confusion, so throw an error when trying to set e.g. the DHookPass_ByRef flag on detour arguments.
---
 extensions/dhooks/natives.cpp    | 16 ++++++++++++++++
 extensions/dhooks/signatures.cpp | 14 ++++++++++++++
 extensions/dhooks/vhook.h        |  8 ++++++++
 3 files changed, 38 insertions(+)

diff --git a/extensions/dhooks/natives.cpp b/extensions/dhooks/natives.cpp
index e1ace0b4..8757e831 100644
--- a/extensions/dhooks/natives.cpp
+++ b/extensions/dhooks/natives.cpp
@@ -256,6 +256,15 @@ cell_t Native_SetFromConf(IPluginContext *pContext, const cell_t *params)
 	setup->funcAddr = addr;
 	setup->offset = offset;
 
+	if (addr == nullptr)
+	{
+		setup->hookMethod = Virtual;
+	}
+	else
+	{
+		setup->hookMethod = Detour;
+	}
+
 	return 1;
 }
 
@@ -282,6 +291,13 @@ cell_t Native_AddParam(IPluginContext *pContext, const cell_t *params)
 		info.flags = PASSFLAG_BYVAL;
 	}
 
+	// DynamicDetours doesn't expose the passflags concept like SourceHook.
+	// See if we're trying to set some invalid flags on detour arguments.
+	if(setup->hookMethod == Detour && (info.flags & ~PASSFLAG_BYVAL) > 0)
+	{
+		return pContext->ThrowNativeError("Pass flags are only supported for virtual hooks.");
+	}
+
 	if (params[0] >= 5)
 	{
 		PluginRegister custom_register = (PluginRegister)params[5];
diff --git a/extensions/dhooks/signatures.cpp b/extensions/dhooks/signatures.cpp
index e5ddd8b9..45b673cc 100644
--- a/extensions/dhooks/signatures.cpp
+++ b/extensions/dhooks/signatures.cpp
@@ -376,6 +376,20 @@ SMCResult SignatureGameConfig::ReadSMC_LeavingSection(const SMCStates *states)
 			return SMCResult_HaltFail;
 		}
 
+		if (!g_CurrentSignature->offset.length())
+		{
+			// DynamicDetours doesn't expose the passflags concept like SourceHook.
+			// See if we're trying to set some invalid flags on detour arguments.
+			for (auto &arg : g_CurrentSignature->args)
+			{
+				if ((arg.info.flags & ~PASSFLAG_BYVAL) > 0)
+				{
+					smutils->LogError(myself, "Function \"%s\" uses unsupported pass flags in argument \"%s\". Flags are only supported for virtual hooks: line: %i col: %i", g_CurrentFunctionName.c_str(), arg.name.c_str(), states->line, states->col);
+					return SMCResult_HaltFail;
+				}
+			}
+		}
+
 		// Save this function signature in our cache.
 		signatures_.insert(g_CurrentFunctionName.c_str(), g_CurrentSignature);
 		g_CurrentFunctionName = "";
diff --git a/extensions/dhooks/vhook.h b/extensions/dhooks/vhook.h
index 3340049c..6146d1ad 100644
--- a/extensions/dhooks/vhook.h
+++ b/extensions/dhooks/vhook.h
@@ -225,6 +225,11 @@ public:
 	DHooksInfo *dg;
 };
 
+enum HookMethod {
+	Virtual,
+	Detour
+};
+
 class HookSetup
 {
 public:
@@ -238,6 +243,7 @@ public:
 		this->offset = offset;
 		this->funcAddr = nullptr;
 		this->callback = callback;
+		this->hookMethod = Virtual;
 	};
 	HookSetup(ReturnType returnType, unsigned int returnFlag, CallingConvention callConv, ThisPointerType thisType, void *funcAddr)
 	{
@@ -249,6 +255,7 @@ public:
 		this->offset = -1;
 		this->funcAddr = funcAddr;
 		this->callback = nullptr;
+		this->hookMethod = Detour;
 	};
 	~HookSetup(){};
 
@@ -266,6 +273,7 @@ public:
 	int offset;
 	void *funcAddr;
 	IPluginFunction *callback;
+	HookMethod hookMethod;
 };
 
 class DHooksManager