Move the code cache into Environment, and out of knight/shared.

sourcepawn/jit/AMBuilder

@@ -9,7 +9,6 @@ Includes = [
   os.path.join(builder.sourcePath, 'public', 'amtl'),
   os.path.join(builder.sourcePath, 'public', 'jit'),
   os.path.join(builder.sourcePath, 'public', 'jit', 'x86'),
-  os.path.join(builder.sourcePath, 'knight', 'shared'),
 
   # The include path for SP v2 stuff.
   os.path.join(builder.sourcePath, 'sourcepawn', 'include'),
@@ -31,6 +30,7 @@ def setup(binary):
 library = setup(builder.compiler.StaticLibrary('sourcepawn'))
 library.sources += [
   'api.cpp',
+  'code-allocator.cpp',
   'plugin-runtime.cpp',
   'compiled-function.cpp',
   'debug-trace.cpp',
@@ -54,7 +54,6 @@ library.sources += [
   'zlib/uncompr.c',
   'zlib/zutil.c',
   'md5/md5.cpp',
-  '../../knight/shared/KeCodeAllocator.cpp',
   '../../public/jit/x86/assembler-x86.cpp',
 ]
 libsourcepawn = builder.Add(library).binary

sourcepawn/jit/api.cpp

@@ -13,7 +13,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
-#include <KeCodeAllocator.h>
 #include "x86/jit_x86.h"
 #include "environment.h"
 #include "api.h"
@@ -74,7 +73,7 @@ SourcePawnEngine::ExecAlloc(size_t size)
 void *
 SourcePawnEngine::AllocatePageMemory(size_t size)
 {
-  return g_Jit.AllocCode(size);
+  return Environment::get()->AllocateCode(size);
 }
 
 void
@@ -92,7 +91,7 @@ SourcePawnEngine::SetReadWrite(void *ptr)
 void
 SourcePawnEngine::FreePageMemory(void *ptr)
 {
-  g_Jit.FreeCode(ptr);
+  Environment::get()->FreeCode(ptr);
 }
 
 void

sourcepawn/jit/code-allocator.cpp

@@ -0,0 +1,419 @@
+#include <assert.h>
+#include <string.h>
+#include <am-utility.h>
+
+#if defined(WIN32)
+#include <windows.h>
+#else
+#include <unistd.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#endif
+
+#include "code-allocator.h"
+
+#define ALIGNMENT	16
+
+using namespace Knight;
+
+struct KeFreedCode;
+
+/**
+ * Defines a region of memory that is made of pages. 
+ */
+struct KeCodeRegion
+{
+	KeCodeRegion *next;
+	unsigned char *block_start;
+	unsigned char *block_pos;
+	KeFreedCode *free_list;
+	size_t total_size;
+	size_t end_free;
+	size_t total_free;
+};
+
+/**
+ * Defines freed code.  We keep the size here because 
+ * when we touch the linked list we don't want to dirty pages.
+ */
+struct KeFreedCode
+{
+	KeCodeRegion *region;
+	unsigned char *block_start;
+	size_t size;
+	KeFreedCode *next;
+};
+
+struct KeSecret
+{
+	KeCodeRegion *region;
+	size_t size;
+};
+
+class Knight::KeCodeCache
+{
+public:
+	/**
+	 * First region that is live for use.
+	 */
+	KeCodeRegion *first_live;
+
+	/**
+	 * First region that is full but has free entries.
+	 */
+	KeCodeRegion *first_partial;
+
+	/**
+	 * First region that is full.
+	 */
+	KeCodeRegion *first_full;
+
+	/**
+	 * Page granularity and size.
+	 */
+	unsigned int page_size;
+	unsigned int page_granularity;
+
+	/**
+	 * This isn't actually for code, this is the node cache.
+	 */
+	KeCodeRegion *node_cache;
+	KeFreedCode *free_node_list;
+};
+
+KeCodeCache *Knight::KE_CreateCodeCache()
+{
+	KeCodeCache *cache;
+
+	cache = new KeCodeCache;
+
+	memset(cache, 0, sizeof(KeCodeCache));
+
+#if defined(WIN32)
+	SYSTEM_INFO info;
+
+	GetSystemInfo(&info);
+	cache->page_size = info.dwPageSize;
+	cache->page_granularity = info.dwAllocationGranularity;
+#else
+	cache->page_size = cache->page_granularity = sysconf(_SC_PAGESIZE);
+#endif
+
+	return cache;
+}
+
+inline size_t MinAllocSize()
+{
+	size_t size;
+
+	size = sizeof(KeSecret);
+	size += ALIGNMENT;
+	size -= size % ALIGNMENT;
+
+	return size;
+}
+
+inline size_t ke_GetAllocSize(size_t size)
+{
+	size += sizeof(KeSecret);
+	size += ALIGNMENT;
+	size -= size % ALIGNMENT;
+
+	return size;
+}
+
+void *ke_AllocInRegion(KeCodeCache *cache,
+					   KeCodeRegion **prev,
+					   KeCodeRegion *region,
+					   unsigned char *ptr,
+					   size_t alloc_size,
+					   bool is_live)
+{
+	KeSecret *secret;
+
+	/* Squirrel some info in the alloc. */
+	secret = (KeSecret *)ptr;
+	secret->region = region;
+	secret->size = alloc_size;
+	ptr += sizeof(KeSecret);
+
+	region->total_free -= alloc_size;
+
+	/* Check if we can't use the fast-path anymore. */
+	if ((is_live && region->end_free < MinAllocSize())
+		|| (!is_live && region->total_free < MinAllocSize()))
+	{
+		KeCodeRegion **start;
+
+		*prev = region->next;
+
+		/* Select the appropriate arena. */
+		if (is_live)
+		{
+			if (region->total_free < MinAllocSize())
+			{
+				start = &cache->first_full;
+			}
+			else
+			{
+				start = &cache->first_partial;
+			}
+		}
+		else
+		{
+			start = &cache->first_full;
+		}
+		
+		region->next = *start;
+		*start = region;
+	}
+
+	return ptr;
+}
+
+void *ke_AllocFromLive(KeCodeCache *cache, size_t size)
+{
+	void *ptr;
+	size_t alloc_size;
+	KeCodeRegion *region, **prev;
+
+	region = cache->first_live;
+	prev = &cache->first_live;
+	alloc_size = ke_GetAllocSize(size);
+
+	while (region != NULL)
+	{
+		if (region->end_free >= alloc_size)
+		{
+			/* Yay! We can do a simple alloc here. */
+			ptr = ke_AllocInRegion(cache, prev, region, region->block_pos, alloc_size, true);
+
+			/* Update our counters. */
+			region->block_pos += alloc_size;
+			region->end_free -= alloc_size;
+
+			return ptr;
+		}
+		prev = &region->next;
+		region = region->next;
+	}
+
+	return NULL;
+}
+
+void *ke_AllocFromPartial(KeCodeCache *cache, size_t size)
+{
+	void *ptr;
+	size_t alloc_size;
+	KeCodeRegion *region, **prev;
+
+	region = cache->first_partial;
+	prev = &cache->first_partial;
+	alloc_size = ke_GetAllocSize(size);
+
+	while (region != NULL)
+	{
+		if (region->total_free >= alloc_size)
+		{
+			KeFreedCode *node, **last;
+
+			assert(region->free_list != NULL);
+
+			last = &region->free_list;
+			node = region->free_list;
+			while (node != NULL)
+			{
+				if (node->size >= alloc_size)
+				{
+					/* Use this node */
+					ptr = ke_AllocInRegion(cache, prev, region, node->block_start, alloc_size, false);
+
+					region->total_free -= node->size;
+					*last = node->next;
+
+					/* Make sure bookkeepping is correct. */
+					assert((region->free_list == NULL && region->total_free == 0)
+						   || (region->free_list != NULL && region->total_free != 0));
+
+					/* Link us back into the free node list. */
+					node->next = cache->free_node_list;
+					cache->free_node_list = node->next;
+
+					return ptr;
+				}
+				last = &node->next;
+				node = node->next;
+			}
+		}
+		prev = &region->next;
+		region = region->next;
+	}
+
+	return NULL;
+}
+
+KeCodeRegion *ke_AddRegionForSize(KeCodeCache *cache, size_t size)
+{
+	KeCodeRegion *region;
+
+	region = new KeCodeRegion;
+
+	size = ke_GetAllocSize(size);
+	size += cache->page_granularity * 2;
+	size -= size % cache->page_granularity;
+
+#if defined(WIN32)
+	region->block_start = (unsigned char *)VirtualAlloc(NULL, size, MEM_COMMIT|MEM_RESERVE, PAGE_EXECUTE_READWRITE);
+#else
+	region->block_start = (unsigned char *)mmap(NULL, size, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANON, -1, 0);
+	region->block_start = (region->block_start == MAP_FAILED) ? NULL : region->block_start;
+#endif
+
+	if (region->block_start == NULL)
+	{
+		delete region;
+		return NULL;
+	}
+
+	region->block_pos = region->block_start;
+	region->end_free = region->total_free = region->total_size = size;
+	region->next = cache->first_live;
+	cache->first_live = region;
+	region->free_list = NULL;
+
+	return region;
+}
+
+void *Knight::KE_AllocCode(KeCodeCache *cache, size_t size)
+{
+	void *ptr;
+
+	/* Check live easy-adds */
+	if (cache->first_live != NULL)
+	{
+		if ((ptr = ke_AllocFromLive(cache, size)) != NULL)
+		{
+			return ptr;
+		}
+	}
+	
+	/* Try looking in the free lists */
+	if (cache->first_partial != NULL)
+	{
+		if ((ptr = ke_AllocFromPartial(cache, size)) != NULL)
+		{
+			return ptr;
+		}
+	}
+
+	/* Create a new region */
+	if (ke_AddRegionForSize(cache, size) == NULL)
+	{
+		return NULL;
+	}
+
+	return ke_AllocFromLive(cache, size);
+}
+
+KeFreedCode *ke_GetFreeNode(KeCodeCache *cache)
+{
+	KeFreedCode *ret;
+
+	if (cache->free_node_list != NULL)
+	{
+		ret = cache->free_node_list;
+		cache->free_node_list = ret->next;
+
+		return ret;
+	}
+
+	/* See if the current free node region has space. */
+	if (cache->node_cache != NULL
+		&& cache->node_cache->end_free >= sizeof(KeFreedCode))
+	{
+		ret = (KeFreedCode *)cache->node_cache->block_pos;
+		cache->node_cache->block_pos += sizeof(KeFreedCode);
+		cache->node_cache->total_free -= sizeof(KeFreedCode);
+		cache->node_cache->end_free -= sizeof(KeFreedCode);
+
+		return ret;
+	}
+
+	/* Otherwise, we need to alloc a new region. */
+	KeCodeRegion *region = new KeCodeRegion;
+	
+	region->block_start = new unsigned char[cache->page_size / sizeof(KeFreedCode)];
+	region->block_pos = region->block_start + sizeof(KeFreedCode);
+	region->total_size = cache->page_size / sizeof(KeFreedCode);
+	region->total_free = region->end_free = (region->total_size - sizeof(KeFreedCode));
+	region->free_list = NULL;
+	region->next = cache->node_cache;
+	cache->node_cache = region;
+
+	return (KeFreedCode *)region->block_start;
+}
+
+void Knight::KE_FreeCode(KeCodeCache *cache, void *code)
+{
+	KeSecret *secret;
+	KeFreedCode *node;
+	unsigned char *ptr;
+	KeCodeRegion *region;
+
+	ptr = (unsigned char *)code;
+	secret = (KeSecret *)(ptr - sizeof(KeSecret));
+	region = secret->region;
+	node = ke_GetFreeNode(cache);
+	node->block_start = (unsigned char *)code;
+	node->next = region->free_list;
+	region->free_list = node;
+	node->region = region;
+	node->size = secret->size;
+}
+
+KeCodeRegion *ke_DestroyRegion(KeCodeRegion *region)
+{
+	KeCodeRegion *next;
+
+	next = region->next;
+
+#if defined(WIN32)
+	VirtualFree(region->block_start, 0, MEM_RELEASE);
+#else
+	munmap(region->block_start, region->total_size);
+#endif
+
+	delete region;
+
+	return next;
+}
+
+void ke_DestroyRegionChain(KeCodeRegion *first)
+{
+	while (first != NULL)
+	{
+		first = ke_DestroyRegion(first);
+	}
+}
+
+void Knight::KE_DestroyCodeCache(KeCodeCache *cache)
+{
+	/* Destroy every region and call it a day. */
+	ke_DestroyRegionChain(cache->first_full);
+	ke_DestroyRegionChain(cache->first_live);
+	ke_DestroyRegionChain(cache->first_partial);
+
+	/* We use normal malloc for node cache regions */
+	KeCodeRegion *region, *next;
+	
+	region = cache->node_cache;
+	while (region != NULL)
+	{
+		next = region->next;
+		delete [] region->block_start;
+		delete region;
+		region = next;
+	}
+	
+	delete cache;
+}

sourcepawn/jit/code-allocator.h

@@ -0,0 +1,47 @@
+#ifndef _INCLUDE_KNIGHT_KE_CODE_ALLOCATOR_H_
+#define _INCLUDE_KNIGHT_KE_CODE_ALLOCATOR_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+namespace Knight
+{
+	class KeCodeCache;
+
+	/**
+	 * @brief Creates a new code cache/allocator.
+	 *
+	 * @return				New code cache allocator.
+	 */
+	extern KeCodeCache *KE_CreateCodeCache();
+
+	/**
+	 * @brief Destroys a code cache allocator.
+	 *
+	 * @param cache			Code cache object.
+	 */
+	extern void KE_DestroyCodeCache(KeCodeCache *cache);
+
+	/**
+	 * @brief Allocates code memory that is readable, writable, 
+	 * and executable.
+	 *
+	 * The address returned wlil be aligned, minimally, on a 16-byte 
+	 * boundary.
+	 *
+	 * @param cache			Code cache object.
+	 * @param size			Amount of memory needed.
+	 * @return				Address pointing to the memory.
+	 */
+	extern void *KE_AllocCode(KeCodeCache *cache, size_t size);
+
+	/**
+	 * @brief Frees code memory.
+	 *
+	 * @param cache			Code cache object.
+	 * @param code			Address of code memory.
+	 */
+	extern void KE_FreeCode(KeCodeCache *cache, void *code);
+}
+
+#endif //_INCLUDE_KNIGHT_KE_CODE_ALLOCATOR_H_

sourcepawn/jit/compiled-function.cpp

@@ -11,7 +11,9 @@
 // SourcePawn. If not, see http://www.gnu.org/licenses/.
 //
 #include "compiled-function.h"
-#include "x86/jit_x86.h"
+#include "environment.h"
+
+using namespace sp;
 
 CompiledFunction::CompiledFunction(void *entry_addr, cell_t pcode_offs, FixedArray<LoopEdge> *edges)
   : entry_(entry_addr),
@@ -22,5 +24,5 @@ CompiledFunction::CompiledFunction(void *entry_addr, cell_t pcode_offs, FixedArr
 
 CompiledFunction::~CompiledFunction()
 {
-  g_Jit.FreeCode(entry_);
+  Environment::get()->FreeCode(entry_);
 }

sourcepawn/jit/environment.cpp

@@ -26,7 +26,8 @@ Environment::Environment()
  : debugger_(nullptr),
    profiler_(nullptr),
    jit_enabled_(true),
-   profiling_enabled_(false)
+   profiling_enabled_(false),
+   code_pool_(nullptr)
 {
 }
 
@@ -41,14 +42,14 @@ Environment::New()
   if (sEnvironment)
     return nullptr;
 
-  Environment *env = new Environment();
-  if (!env->Initialize()) {
-    delete env;
+  sEnvironment = new Environment();
+  if (!sEnvironment->Initialize()) {
+    delete sEnvironment;
+    sEnvironment = nullptr;
     return nullptr;
   }
 
-  sEnvironment = env;
-  return env;
+  return sEnvironment;
 }
 
 Environment *
@@ -64,6 +65,10 @@ Environment::Initialize()
   api_v2_ = new SourcePawnEngine2();
   watchdog_timer_ = new WatchdogTimer();
 
+  if ((code_pool_ = Knight::KE_CreateCodeCache()) == nullptr)
+    return false;
+
+  // Safe to initialize JIT now that we have the code cache.
   if (!g_Jit.InitializeJIT())
     return false;
 
@@ -75,6 +80,10 @@ Environment::Shutdown()
 {
   watchdog_timer_->Shutdown();
   g_Jit.ShutdownJIT();
+  Knight::KE_DestroyCodeCache(code_pool_);
+
+  assert(sEnvironment == this);
+  sEnvironment = nullptr;
 }
 
 void
@@ -160,3 +169,15 @@ Environment::ReportError(PluginRuntime *runtime, int err, const char *errstr, ce
 
   debugger_->OnContextExecuteError(runtime->GetDefaultContext(), &trace);
 }
+
+void *
+Environment::AllocateCode(size_t size)
+{
+  return Knight::KE_AllocCode(code_pool_, size);
+}
+
+void
+Environment::FreeCode(void *code)
+{
+  Knight::KE_FreeCode(code_pool_, code);
+}

sourcepawn/jit/environment.h

@@ -15,6 +15,7 @@
 
 #include <sp_vm_api.h>
 #include <am-utility.h> // Replace with am-cxx later.
+#include "code-allocator.h"
 
 class PluginRuntime;
 
@@ -54,6 +55,10 @@ class Environment : public ISourcePawnEnvironment
   const char *GetErrorString(int err);
   void ReportError(PluginRuntime *runtime, int err, const char *errstr, cell_t rp_start);
 
+  // Allocate and free executable memory.
+  void *AllocateCode(size_t size);
+  void FreeCode(void *code);
+
   // Helpers.
   void SetProfiler(IProfilingTool *profiler) {
     profiler_ = profiler;
@@ -96,6 +101,8 @@ class Environment : public ISourcePawnEnvironment
   IProfilingTool *profiler_;
   bool jit_enabled_;
   bool profiling_enabled_;
+
+  Knight::KeCodeCache *code_pool_;
 };
 
 class EnterProfileScope

sourcepawn/jit/x86/jit_x86.cpp

@@ -40,7 +40,6 @@
 #include "environment.h"
 
 using namespace sp;
-using namespace Knight;
 
 #if defined USE_UNGEN_OPCODES
 #include "ungen_opcodes.h"
@@ -49,7 +48,6 @@ using namespace Knight;
 #define __ masm.
 
 JITX86 g_Jit;
-KeCodeCache *g_pCodeCache = NULL;
 
 static inline uint8_t *
 LinkCode(AssemblerX86 &masm)
@@ -57,7 +55,7 @@ LinkCode(AssemblerX86 &masm)
   if (masm.outOfMemory())
     return NULL;
 
-  void *code = Knight::KE_AllocCode(g_pCodeCache, masm.length());
+  void *code = Environment::get()->AllocateCode(masm.length());
   if (!code)
     return NULL;
 
@@ -1901,8 +1899,6 @@ JITX86::JITX86()
 bool
 JITX86::InitializeJIT()
 {
-  g_pCodeCache = KE_CreateCodeCache();
-
   m_pJitEntry = GenerateEntry(&m_pJitReturn, &m_pJitTimeout);
   if (!m_pJitEntry)
     return false;
@@ -1913,7 +1909,6 @@ JITX86::InitializeJIT()
   if (!code)
     return false;
   MacroAssemblerX86::RunFeatureDetection(code);
-  KE_FreeCode(g_pCodeCache, code);
 
   return true;
 }
@@ -1921,7 +1916,6 @@ JITX86::InitializeJIT()
 void
 JITX86::ShutdownJIT()
 {
-  KE_DestroyCodeCache(g_pCodeCache);
 }
 
 CompiledFunction *
@@ -1981,7 +1975,7 @@ JITX86::CreateFakeNative(SPVM_FAKENATIVE_FUNC callback, void *pData)
 void
 JITX86::DestroyFakeNative(SPVM_NATIVE_FUNC func)
 {
-  KE_FreeCode(g_pCodeCache, (void *)func);
+  Environment::get()->FreeCode((void *)func);
 }
 
 ICompilation *
@@ -2046,18 +2040,6 @@ JITX86::InvokeFunction(PluginRuntime *runtime, CompiledFunction *fn, cell_t *res
   return err;
 }
 
-void *
-JITX86::AllocCode(size_t size)
-{
-  return Knight::KE_AllocCode(g_pCodeCache, size);
-}
-
-void
-JITX86::FreeCode(void *code)
-{
-  KE_FreeCode(g_pCodeCache, code);
-}
-
 void
 JITX86::RegisterRuntime(PluginRuntime *rt)
 {

sourcepawn/jit/x86/jit_x86.h

@@ -19,7 +19,6 @@
 
 #include <sp_vm_types.h>
 #include <sp_vm_api.h>
-#include <KeCodeAllocator.h>
 #include <macro-assembler-x86.h>
 #include <am-vector.h>
 #include "jit_shared.h"
@@ -173,9 +172,6 @@ class JITX86
   ExternalAddress GetUniversalReturn() {
     return ExternalAddress(m_pJitReturn);
   }
-  void *AllocCode(size_t size);
-  void FreeCode(void *code);
-
   uintptr_t FrameId() const {
     return frame_id_;
   }
@@ -203,7 +199,6 @@ const Register dat = esi;
 const Register tmp = ecx;
 const Register frm = ebx;
 
-extern Knight::KeCodeCache *g_pCodeCache;
 extern JITX86 g_Jit;
 
 #endif //_INCLUDE_SOURCEPAWN_JIT_X86_H_