Browse Source

Dynamic loading of compile time functions

* Removed Square.cpp test file
* Added header and source headings and footers
* Added compile errors for OS-specific files
* Dynamic loading of recently compiled functions now works. Next, using them!
ModuleSystem
Macoy Madson 10 months ago
parent
commit
1a4bb41734
11 changed files with 125 additions and 80 deletions
  1. +2
    -1
      BuildAndRunTests.sh
  2. +23
    -30
      src/DynamicLoader.cpp
  3. +5
    -4
      src/DynamicLoader.hpp
  4. +56
    -14
      src/Evaluator.cpp
  5. +1
    -4
      src/Jamfile
  6. +4
    -1
      src/OutputPreambles.cpp
  7. +3
    -1
      src/OutputPreambles.hpp
  8. +5
    -1
      src/RunProcess.cpp
  9. +0
    -12
      src/Square.cpp
  10. +22
    -10
      src/Writer.cpp
  11. +4
    -2
      src/Writer.hpp

+ 2
- 1
BuildAndRunTests.sh View File

@ -1,5 +1,6 @@
#!/bin/sh
jam -j4 && ./cakelisp test/Basic.cake
jam -j4 && ./cakelisp test/Dependencies.cake
# jam -j4 && ./src/dependencyTest
# jam -j4 && ./src/runProcessTest
# jam -j4 && ./src/dynamicLoadTest

+ 23
- 30
src/DynamicLoader.cpp View File

@ -4,64 +4,57 @@
#ifdef UNIX
#include <dlfcn.h>
#else
#error "Platform support needed for dynamic loading"
#endif
extern "C"
DynamicLibHandle loadDynamicLibrary(const char* libraryPath)
{
float hostSquareFunc(float numToSquare)
{
return numToSquare * numToSquare;
}
}
void* libHandle = nullptr;
int main()
{
#ifdef UNIX
void* libHandle;
// RTLD_LAZY: Don't look up symbols the shared library needs until it encounters them
// Note that this requires linking with -Wl,-rpath,. in order to turn up relative path .so files
libHandle = dlopen("src/libSquare.so", RTLD_LAZY);
libHandle = dlopen(libraryPath, RTLD_LAZY);
if (!libHandle)
{
fprintf(stderr, "DynamicLoader Error:\n%s\n", dlerror());
return 1;
return nullptr;
}
#endif
// Clear any existing error before running dlsym
char* error = dlerror();
if (error != nullptr)
return libHandle;
}
void* getSymbolFromDynamicLibrary(DynamicLibHandle library, const char* symbolName)
{
if (!library)
{
fprintf(stderr, "DynamicLoader Error:\n%s\n", error);
return 1;
fprintf(stderr, "DynamicLoader Error: Received empty library handle\n");
return nullptr;
}
printf("About to load\n");
float (*square)(float) = (float (*)(float))dlsym(libHandle, "square");
printf("Done load\n");
error = dlerror();
#ifdef UNIX
// Clear any existing error before running dlsym
char* error = dlerror();
if (error != nullptr)
{
fprintf(stderr, "DynamicLoader Error:\n%s\n", error);
return 1;
return nullptr;
}
printf("About to call\n");
printf("%f\n", (*square)(2.f));
printf("Done call\n");
dlclose(libHandle);
void* symbol = dlsym(library, symbolName);
error = dlerror();
if (error != nullptr)
{
fprintf(stderr, "DynamicLoader Error:\n%s\n", error);
return 1;
return nullptr;
}
return 0;
return symbol;
#else
return 1;
return nullptr;
#endif
}

+ 5
- 4
src/DynamicLoader.hpp View File

@ -1,6 +1,7 @@
#pragma once
extern "C"
{
float hostSquareFunc(float numToSquare);
}
typedef void* DynamicLibHandle;
DynamicLibHandle loadDynamicLibrary(const char* libraryPath);
void* getSymbolFromDynamicLibrary(DynamicLibHandle library, const char* symbolName);

+ 56
- 14
src/Evaluator.cpp View File

@ -1,13 +1,14 @@
#include "Evaluator.hpp"
#include "Converters.hpp"
#include "DynamicLoader.hpp"
#include "GeneratorHelpers.hpp"
#include "Generators.hpp"
#include "OutputPreambles.hpp"
#include "RunProcess.hpp"
#include "Tokenizer.hpp"
#include "Utilities.hpp"
#include "Writer.hpp"
#include "OutputPreambles.hpp"
// TODO: safe version of strcat
#include <stdio.h>
@ -452,7 +453,8 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment)
BuildStage_None,
BuildStage_Compiling,
BuildStage_Linking,
BuildStage_Loading
BuildStage_Loading,
BuildStage_Finished
};
// Note: environment.definitions can be resized/rehashed during evaluation, which invalidates
@ -460,10 +462,12 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment)
// references on resize. This will need to change if the data structure changes
struct BuildObject
{
int buildId;
int status;
BuildStage stage;
ObjectDefinition* definition;
int buildId = -1;
int status = -1;
BuildStage stage = BuildStage_None;
std::string artifactsFilePath;
std::string dynamicLibraryPath;
ObjectDefinition* definition = nullptr;
};
std::vector<BuildObject> definitionsToBuild;
for (ObjectDefinitionPair& definitionPair : environment.definitions)
@ -510,8 +514,10 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment)
if (canBuild)
{
definitionsToBuild.push_back(
{getNextFreeBuildId(environment), -1, BuildStage_None, &definition});
BuildObject objectToBuild = {};
objectToBuild.buildId = getNextFreeBuildId(environment);
objectToBuild.definition = &definition;
definitionsToBuild.push_back(objectToBuild);
}
}
}
@ -520,6 +526,7 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment)
// TODO: Limit number spawned at once?
// TODO: Combine sure-thing builds into batches (ones where we know all references)
// TODO: Instead of creating files, pipe straight to compiler?
// TODO: Make pipeline able to start e.g. linker while other objects are still compiling
// NOTE: definitionsToBuild must not be resized from when runProcess() is called until
// waitForAllProcessesClosed(), else the status pointer could be invalidated
for (BuildObject& buildObject : definitionsToBuild)
@ -530,13 +537,15 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment)
char sourceOutputName[MAX_PATH_LENGTH] = {0};
// Writer will append the appropriate file extension
PrintfBuffer(sourceOutputName, "CakelispCompileTime_%d", buildObject.buildId);
buildObject.artifactsFilePath = sourceOutputName;
// Output definition to a file our compiler will be happy with
// TODO: Make these come from the top
NameStyleSettings nameSettings;
WriterFormatSettings formatSettings;
WriterOutputSettings outputSettings;
outputSettings.sourcePreamble = macroPreamble;
outputSettings.sourceHeading = macroSourceHeading;
outputSettings.sourceFooter = macroSourceFooter;
outputSettings.sourceCakelispFilename = sourceOutputName;
if (!writeGeneratorOutput(*definition->output, nameSettings, formatSettings,
outputSettings))
@ -552,7 +561,7 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment)
PrintBuffer(fileToExec, "/usr/bin/clang++");
// TODO: Get file extension from output (once .c vs .cpp is implemented)
PrintfBuffer(sourceOutputName, "CakelispCompileTime_%d.cpp", buildObject.buildId);
PrintfBuffer(sourceOutputName, "%s.cpp", buildObject.artifactsFilePath.c_str());
// TODO: Get arguments all the way from the top
// If not null terminated, the call will fail
@ -573,6 +582,9 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment)
// Linking
for (BuildObject& buildObject : definitionsToBuild)
{
if (buildObject.stage != BuildStage_Compiling)
continue;
if (buildObject.status != 0)
{
ErrorAtTokenf(*buildObject.definition->name,
@ -589,9 +601,10 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment)
// TODO Store this on the build object
char buildObjectName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(buildObjectName, "CakelispCompileTime_%d.o", buildObject.buildId);
PrintfBuffer(buildObjectName, "%s.o", buildObject.artifactsFilePath.c_str());
char dynamicLibraryOut[MAX_PATH_LENGTH] = {0};
PrintfBuffer(dynamicLibraryOut, "libCakelispCompileTime_%d.so", buildObject.buildId);
PrintfBuffer(dynamicLibraryOut, "lib%s.so", buildObject.artifactsFilePath.c_str());
buildObject.dynamicLibraryPath = dynamicLibraryOut;
char* arguments[] = {linkerExecutable, strdup("-shared"), strdup("-o"),
dynamicLibraryOut, buildObjectName, nullptr};
@ -609,15 +622,44 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment)
for (BuildObject& buildObject : definitionsToBuild)
{
if (buildObject.stage == BuildStage_Linking && buildObject.status != 0)
if (buildObject.stage != BuildStage_Linking)
continue;
if (buildObject.status != 0)
{
ErrorAtToken(*buildObject.definition->name, "Failed to link definition");
continue;
}
buildObject.stage = BuildStage_Loading;
printf("Linked %s successfully\n", buildObject.definition->name->contents.c_str());
buildObject.stage = BuildStage_Loading;
DynamicLibHandle builtLib = loadDynamicLibrary(buildObject.dynamicLibraryPath.c_str());
if (!builtLib)
{
ErrorAtToken(*buildObject.definition->name, "Failed to load compile-time library");
continue;
}
// We need to do name conversion to be compatible with C naming
// TODO: Make these come from the top
NameStyleSettings nameSettings;
char symbolNameBuffer[MAX_NAME_LENGTH] = {0};
lispNameStyleToCNameStyle(nameSettings.functionNameMode,
buildObject.definition->name->contents.c_str(), symbolNameBuffer,
sizeof(symbolNameBuffer));
void* compileTimeFunction = getSymbolFromDynamicLibrary(builtLib, symbolNameBuffer);
if (!compileTimeFunction)
{
ErrorAtToken(*buildObject.definition->name, "Failed to find symbol in loaded library");
continue;
}
buildObject.stage = BuildStage_Finished;
printf("Successfully built and loaded %s\n",
buildObject.definition->name->contents.c_str());
}
return 0;


+ 1
- 4
src/Jamfile View File

@ -8,14 +8,11 @@ Generators.cpp
GeneratorHelpers.cpp
RunProcess.cpp
OutputPreambles.cpp
DynamicLoader.cpp
;
MakeLocate cakelisp : ../cakelisp ;
Main Square$(SUFSHR) : Square.cpp ;
Main dynamicLoadTest : DynamicLoader.cpp ;
Main dependencyTest : TestRefResolver.cpp
Utilities.cpp ;
# C++ on src/TestRefResolver.o = g++ ;


+ 4
- 1
src/OutputPreambles.cpp View File

@ -1,3 +1,6 @@
#include "OutputPreambles.hpp"
const char* macroPreamble = "#include \"Evaluator.hpp\"\n";
// Must use extern "C" for dynamic symbols, because otherwise name mangling makes things hard
const char* macroSourceHeading = "#include \"Evaluator.hpp\"\n#include \"EvaluatorEnums.hpp\"\nextern \"C\"\n{\n";
// Close extern "C" block
const char* macroSourceFooter = "}\n";

+ 3
- 1
src/OutputPreambles.hpp View File

@ -1,3 +1,5 @@
#pragma once
extern const char* macroPreamble;
// All generated macro source files automatically have this
extern const char* macroSourceHeading;
extern const char* macroSourceFooter;

+ 5
- 1
src/RunProcess.cpp View File

@ -8,6 +8,8 @@
#include <sys/types.h> // pid
#include <sys/wait.h> // waitpid
#include <unistd.h> // exec, fork
#else
#error Platform support needed for running subprocesses
#endif
#include "Utilities.hpp"
@ -51,7 +53,7 @@ void subprocessReceiveStdOut(const char* processOutputBuffer)
int runProcess(const RunProcessArguments& arguments, int* statusOut)
{
#ifdef UNIX
printf("Compiling file with command:\n");
printf("RunProcess command: ");
for (char** arg = arguments.arguments; *arg != nullptr; ++arg)
{
printf("%s ", *arg);
@ -125,7 +127,9 @@ void waitForAllProcessesClosed(SubprocessOnOutputFunc onOutput)
for (Subprocess& process : s_subprocesses)
{
#ifdef UNIX
waitpid(process.processId, process.statusOut, 0);
#endif
}
s_subprocesses.clear();


+ 0
- 12
src/Square.cpp View File

@ -1,12 +0,0 @@
#include "DynamicLoader.hpp"
// clang++ -shared -o libSquare.so Square.o
extern "C"
{
float square(float numToSquare)
{
// return numToSquare * numToSquare;
return hostSquareFunc(numToSquare);
}
}

+ 22
- 10
src/Writer.cpp View File

@ -239,11 +239,11 @@ bool moveFile(const char* srcFilename, const char* destFilename)
}
// TODO This sucks
// TODO Lots of params
bool writeIfContentsNewer(const NameStyleSettings& nameSettings,
const WriterFormatSettings& formatSettings,
const WriterOutputSettings& outputSettings,
const char* preamble,
const std::vector<StringOutput>& outputOperations,
const WriterOutputSettings& outputSettings, const char* heading,
const char* footer, const std::vector<StringOutput>& outputOperations,
const char* outputFilename)
{
// Write to a temporary file
@ -253,11 +253,11 @@ bool writeIfContentsNewer(const NameStyleSettings& nameSettings,
// TODO: If this fails to open, Writer_Writef just won't write to the file, it'll print
outputState.fileOut = fileOpen(tempFilename);
if (preamble)
if (heading)
{
Writer_Writef(outputState, preamble);
Writer_Writef(outputState, heading);
// TODO: Put this in Writer_Writef
for (const char* c = preamble; *c != '\0'; ++c)
for (const char* c = heading; *c != '\0'; ++c)
{
if (*c == '\n')
++outputState.currentLine;
@ -275,6 +275,18 @@ bool writeIfContentsNewer(const NameStyleSettings& nameSettings,
writeStringOutput(nameSettings, formatSettings, operation, outputState);
}
if (footer)
{
Writer_Writef(outputState, footer);
// TODO: Put this in Writer_Writef
for (const char* c = footer; *c != '\0'; ++c)
{
if (*c == '\n')
++outputState.currentLine;
}
}
if (outputState.fileOut)
{
fclose(outputState.fileOut);
@ -350,8 +362,8 @@ bool writeGeneratorOutput(const GeneratorOutput& generatedOutput,
char sourceOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(sourceOutputName, "%s.cpp", outputSettings.sourceCakelispFilename);
if (!writeIfContentsNewer(nameSettings, formatSettings, outputSettings,
outputSettings.sourcePreamble, generatedOutput.source,
sourceOutputName))
outputSettings.sourceHeading, outputSettings.sourceFooter,
generatedOutput.source, sourceOutputName))
return false;
}
@ -360,8 +372,8 @@ bool writeGeneratorOutput(const GeneratorOutput& generatedOutput,
char headerOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(headerOutputName, "%s.hpp", outputSettings.sourceCakelispFilename);
if (!writeIfContentsNewer(nameSettings, formatSettings, outputSettings,
outputSettings.headerPreamble, generatedOutput.header,
headerOutputName))
outputSettings.headerHeading, outputSettings.headerFooter,
generatedOutput.header, headerOutputName))
return false;
}


+ 4
- 2
src/Writer.hpp View File

@ -20,8 +20,10 @@ struct WriterFormatSettings
struct WriterOutputSettings
{
const char* sourceCakelispFilename;
const char* sourcePreamble;
const char* headerPreamble;
const char* sourceHeading;
const char* sourceFooter;
const char* headerHeading;
const char* headerFooter;
};
const char* importLanguageToString(ImportLanguage type);


Loading…
Cancel
Save