Browse Source

Modules now build, and can override commands

I'm not super happy with how "dirty" it is, but I'm happy to at least get
something in so I can refine it in the future.
GeneralDependencyResolver
Macoy Madson 1 month ago
parent
commit
666d0ada5b
13 changed files with 308 additions and 112 deletions
  1. +7
    -5
      BuildAndRunTests.sh
  2. +10
    -0
      runtime/HotReloading.cake
  3. +1
    -6
      runtime/Macros.cake
  4. +6
    -64
      src/Evaluator.cpp
  5. +3
    -14
      src/Evaluator.hpp
  6. +0
    -12
      src/EvaluatorEnums.hpp
  7. +71
    -0
      src/Generators.cpp
  8. +85
    -1
      src/ModuleManager.cpp
  9. +14
    -8
      src/ModuleManager.hpp
  10. +9
    -0
      src/ModuleManagerEnums.hpp
  11. +56
    -2
      src/RunProcess.cpp
  12. +33
    -0
      src/RunProcess.hpp
  13. +13
    -0
      src/RunProcessEnums.hpp

+ 7
- 5
BuildAndRunTests.sh View File

@@ -1,9 +1,11 @@
#!/bin/sh

jam -j4 && ./bin/cakelisp test/Macros.cake
jam -j4 && ./bin/cakelisp test/Dependencies.cake
jam -j4 && ./bin/cakelisp test/Basic.cake
# jam -j4 && ./bin/cakelisp test/Macros.cake
# jam -j4 && ./bin/cakelisp test/Dependencies.cake
# jam -j4 && ./bin/cakelisp test/Basic.cake

# Build Cakelisp itself, generate the runtime, then build the runtime
jam -j4 && ./bin/cakelisp --enable-hot-reloading runtime/TestMain.cake runtime/TextAdventure.cake \
&& cd runtime && jam -j4
# jam -j4 && ./bin/cakelisp --enable-hot-reloading runtime/TestMain.cake runtime/TextAdventure.cake \
# && cd runtime && jam -j4

jam -j4 && ./bin/cakelisp --enable-hot-reloading runtime/TestMain.cake runtime/TextAdventure.cake

+ 10
- 0
runtime/HotReloading.cake View File

@@ -1,3 +1,13 @@
(set-cakelisp-option enable-hot-reloading true)

(set-module-option build-time-compiler "/usr/bin/clang++")
;; Include cakelisp source for DynamicLoader.hpp
(set-module-option build-time-compile-arguments
"-g" "-c" 'source-input "-o" 'object-output "-fPIC" "-Isrc/")
(set-module-option build-time-linker "/usr/bin/clang++")
(set-module-option build-time-link-arguments
"-shared" "-o" 'library-output 'object-input)

(import &comptime-only "Macros.cake")
(c-import "<unordered_map>" "<vector>")
(c-import "DynamicLoader.hpp")


+ 1
- 6
runtime/Macros.cake View File

@@ -1,9 +1,4 @@
(set-cakelisp-option compile-time-compiler "/usr/bin/clang++")
(set-cakelisp-option compile-time-compile-arguments
"-g" "-c" 'source-input "-o" 'object-output 'cakelisp-headers-include "-fPIC")
(set-cakelisp-option compile-time-linker "/usr/bin/clang++")
(set-cakelisp-option compile-time-link-arguments
"-shared" "-o" 'library-output 'object-input)
(skip-build)

;; TODO: This should be builtin to macros and generators
(defmacro destructure-arguments ()


+ 6
- 64
src/Evaluator.cpp View File

@@ -478,65 +478,7 @@ void PropagateRequiredToReferences(EvaluatorEnvironment& environment)
} while (numRequiresStatusChanged);
}

struct ProcessCommandInput
{
ProcessCommandArgumentType type;
const char* value;
};

void PrintProcessArguments(const char** processArguments)
{
for (const char* argument = processArguments[0]; argument; ++argument)
printf("%s ", argument);
printf("\n");
}

// The array will need to be deleted, but the array members will not
const char** MakeProcessArgumentsFromCommand(ProcessCommand& command,
const ProcessCommandInput* inputs, int numInputs)
{
int numArguments = command.arguments.size();
// +1 for file to execute
int numFinalArguments = numArguments + 1;
// +1 again for the null terminator
const char** newArguments = (const char**)calloc(sizeof(const char*), numFinalArguments + 1);
for (int i = 0; i < numFinalArguments; ++i)
newArguments[i] = nullptr;
newArguments[numFinalArguments] = nullptr;
newArguments[0] = command.fileToExecute.c_str();

for (int i = 0; i < numArguments; ++i)
{
int finalArgumentIndex = i + 1;
ProcessCommandArgument& argument = command.arguments[i];

if (argument.type == ProcessCommandArgumentType_String)
newArguments[finalArgumentIndex] = argument.contents.c_str();
else
{
bool found = false;
for (int input = 0; input < numInputs; ++input)
{
if (inputs[input].type == argument.type)
{
newArguments[finalArgumentIndex] = inputs[input].value;
found = true;
break;
}
}
if (!found)
{
printf("error: command missing input\n");
free(newArguments);
return nullptr;
}
}
}

return newArguments;
}

void OnCompileProcessOutput(const char* output)
static void OnCompileProcessOutput(const char* output)
{
// TODO C/C++ error to Cakelisp token mapper
}
@@ -577,7 +519,6 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
// TODO: Make pipeline able to start e.g. linker while other objects are still compiling
// NOTE: definitionsToBuild must not be resized from when runProcess() is called until
// waitForAllProcessesClosed(), else the status pointer could be invalidated
const int maxProcessesSpawned = 8;
int currentNumProcessesSpawned = 0;
for (BuildObject& buildObject : definitionsToBuild)
{
@@ -681,9 +622,7 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
// TODO: Abort building if cannot invoke compiler
continue;
}
// const char* arguments[] = {fileToExec, "-g", "-c", sourceOutputName,
// "-o", buildObjectName, headerInclude, "-fPIC", nullptr};
// TODO: Get rid of intermediate structure

RunProcessArguments compileArguments = {};
compileArguments.fileToExecute = environment.compileTimeBuildCommand.fileToExecute.c_str();
compileArguments.arguments = buildArguments;
@@ -696,10 +635,11 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,

free(buildArguments);

// TODO: Move this to other processes as well
// TODO This could be made smarter by allowing more spawning right when a process closes,
// instead of starting in waves
++currentNumProcessesSpawned;
if (currentNumProcessesSpawned >= maxProcessesSpawned)
if (currentNumProcessesSpawned >= maxProcessesRecommendedSpawned)
{
waitForAllProcessesClosed(OnCompileProcessOutput);
currentNumProcessesSpawned = 0;
@@ -708,6 +648,7 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,

// The result of the builds will go straight to our definitionsToBuild
waitForAllProcessesClosed(OnCompileProcessOutput);
currentNumProcessesSpawned = 0;

// Linking
for (BuildObject& buildObject : definitionsToBuild)
@@ -752,6 +693,7 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,

// The result of the linking will go straight to our definitionsToBuild
waitForAllProcessesClosed(OnCompileProcessOutput);
currentNumProcessesSpawned = 0;

for (BuildObject& buildObject : definitionsToBuild)
{


+ 3
- 14
src/Evaluator.hpp View File

@@ -1,17 +1,18 @@
#pragma once

#include "EvaluatorEnums.hpp"
#include "RunProcess.hpp"

#include <string>
#include <vector>
// TODO: Replace with fast hash table
#include <unordered_map>

struct NameStyleSettings;
struct Token;
struct GeneratorOutput;
struct ModuleManager;
struct Module;
struct NameStyleSettings;
struct Token;

// Rather than needing to allocate and edit a buffer eventually equal to the size of the final
// output, store output operations instead. This also facilitates source <-> generated mapping data
@@ -169,18 +170,6 @@ typedef std::pair<const std::string, ObjectDefinition> ObjectDefinitionPair;
typedef std::unordered_map<std::string, ObjectReferencePool> ObjectReferencePoolMap;
typedef std::pair<const std::string, ObjectReferencePool> ObjectReferencePoolPair;

struct ProcessCommandArgument
{
ProcessCommandArgumentType type;
std::string contents;
};

struct ProcessCommand
{
std::string fileToExecute;
std::vector<ProcessCommandArgument> arguments;
};

// Unlike context, which can't be changed, environment can be changed.
// Use care when modifying the environment. Only add things once you know things have succeeded.
// Keep in mind that calling functions which can change the environment may invalidate your pointers


+ 0
- 12
src/EvaluatorEnums.hpp View File

@@ -71,15 +71,3 @@ enum ObjectReferenceGuessState
GuessState_WaitingForLoad,
GuessState_Resolved,
};

enum ProcessCommandArgumentType
{
ProcessCommandArgumentType_String,

ProcessCommandArgumentType_SourceInput,
ProcessCommandArgumentType_ObjectOutput,
ProcessCommandArgumentType_CakelispHeadersInclude,

ProcessCommandArgumentType_ObjectInput,
ProcessCommandArgumentType_DynamicLibraryOutput,
};

+ 71
- 0
src/Generators.cpp View File

@@ -188,6 +188,74 @@ bool SetCakelispOption(EvaluatorEnvironment& environment, const EvaluatorContext
return false;
}

bool SetModuleOption(EvaluatorEnvironment& environment, const EvaluatorContext& context,
const std::vector<Token>& tokens, int startTokenIndex, GeneratorOutput& output)
{
if (!context.module)
{
ErrorAtToken(tokens[startTokenIndex], "modules not supported (internal code error?)");
return false;
}

int endInvocationIndex = FindCloseParenTokenIndex(tokens, startTokenIndex);

int optionNameIndex =
getExpectedArgument("expected option name", tokens, startTokenIndex, 1, endInvocationIndex);
if (optionNameIndex == -1)
return false;

// TODO: Copy-pasted
struct ProcessCommandOptions
{
const char* optionName;
ProcessCommand* command;
ProcessCommandOptionFunc handler;
};
ProcessCommandOptions commandOptions[] = {
{"compile-time-compiler", &context.module->compileTimeBuildCommand,
SetProcessCommandFileToExec},
{"compile-time-compile-arguments", &context.module->compileTimeBuildCommand,
SetProcessCommandArguments},
{"compile-time-linker", &context.module->compileTimeLinkCommand,
SetProcessCommandFileToExec},
{"compile-time-link-arguments", &context.module->compileTimeLinkCommand,
SetProcessCommandArguments},
{"build-time-compiler", &context.module->buildTimeBuildCommand,
SetProcessCommandFileToExec},
{"build-time-compile-arguments", &context.module->buildTimeBuildCommand,
SetProcessCommandArguments},
{"build-time-linker", &context.module->buildTimeLinkCommand, SetProcessCommandFileToExec},
{"build-time-link-arguments", &context.module->buildTimeLinkCommand,
SetProcessCommandArguments},
};

for (unsigned int i = 0; i < ArraySize(commandOptions); ++i)
{
if (tokens[optionNameIndex].contents.compare(commandOptions[i].optionName) == 0)
{
return commandOptions[i].handler(environment, tokens, startTokenIndex,
commandOptions[i].command);
}
}

return true;
}

bool SkipBuildGenerator(EvaluatorEnvironment& environment, const EvaluatorContext& context,
const std::vector<Token>& tokens, int startTokenIndex,
GeneratorOutput& output)
{
if (context.module)
context.module->skipBuild = true;
else
{
ErrorAtToken(tokens[startTokenIndex], "building not supported (internal code error?)");
return false;
}

return true;
}

enum ImportState
{
WithDefinitions,
@@ -1898,6 +1966,9 @@ void importFundamentalGenerators(EvaluatorEnvironment& environment)

// Cakelisp options
environment.generators["set-cakelisp-option"] = SetCakelispOption;
environment.generators["set-module-option"] = SetModuleOption;

environment.generators["skip-build"] = SkipBuildGenerator;

// Dispatches based on invocation name
const char* cStatementKeywords[] = {


+ 85
- 1
src/ModuleManager.cpp View File

@@ -268,10 +268,22 @@ bool moduleManagerWriteGeneratedOutput(ModuleManager& manager)
return true;
}

static void OnCompileProcessOutput(const char* output)
{
// TODO C/C++ error to Cakelisp token mapper
}

bool moduleManagerBuild(ModuleManager& manager)
{
for (Module* module : manager.modules)
int currentNumProcessesSpawned = 0;

int numModules = manager.modules.size();
std::vector<int> buildStatusCodes(numModules);

for (int moduleIndex = 0; moduleIndex < numModules; ++moduleIndex)
{
Module* module = manager.modules[moduleIndex];

printf("Build module %s\n", module->sourceOutputName.c_str());
for (ModuleDependency& dependency : module->dependencies)
{
@@ -282,6 +294,78 @@ bool moduleManagerBuild(ModuleManager& manager)
if (dependency.type == ModuleDependency_Cakelisp)
continue;
}

// TODO: Importing needs to set this on the module, not the dependency...
if (module->skipBuild)
continue;

// TODO: Lots of overlap between this and compile-time building
char buildFilename[MAX_NAME_LENGTH] = {0};
getFilenameFromPath(module->sourceOutputName.c_str(), buildFilename, sizeof(buildFilename));
if (!buildFilename[0])
return false;

char buildObjectName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(buildObjectName, "%s/%s.o", cakelispWorkingDir, buildFilename);
if (!fileIsMoreRecentlyModified(module->sourceOutputName.c_str(), buildObjectName))
{
if (log.buildProcess)
printf("Skipping compiling %s (using cached object)\n",
module->sourceOutputName.c_str());
}
else
{
ProcessCommand& buildCommand = (!module->buildTimeBuildCommand.fileToExecute.empty() &&
!module->buildTimeBuildCommand.arguments.empty()) ?
module->buildTimeBuildCommand :
manager.environment.buildTimeBuildCommand;

ProcessCommandInput buildTimeInputs[] = {
{ProcessCommandArgumentType_SourceInput, module->sourceOutputName.c_str()},
{ProcessCommandArgumentType_ObjectOutput, buildObjectName}};
const char** buildArguments = MakeProcessArgumentsFromCommand(
buildCommand, buildTimeInputs, ArraySize(buildTimeInputs));
if (!buildArguments)
{
printf("error: failed to construct build arguments\n");
return false;
}
RunProcessArguments compileArguments = {};
compileArguments.fileToExecute = buildCommand.fileToExecute.c_str();
compileArguments.arguments = buildArguments;
// PrintProcessArguments(buildArguments);
if (runProcess(compileArguments, &buildStatusCodes[moduleIndex]) != 0)
{
printf("error: failed to invoke compiler\n");
free(buildArguments);
return false;
}

free(buildArguments);

// TODO This could be made smarter by allowing more spawning right when a process
// closes, instead of starting in waves
++currentNumProcessesSpawned;
if (currentNumProcessesSpawned >= maxProcessesRecommendedSpawned)
{
waitForAllProcessesClosed(OnCompileProcessOutput);
currentNumProcessesSpawned = 0;
}
}
}

waitForAllProcessesClosed(OnCompileProcessOutput);
currentNumProcessesSpawned = 0;

for (int moduleIndex = 0; moduleIndex < numModules; ++moduleIndex)
{
// Module* module = manager.modules[moduleIndex];
int buildResult = buildStatusCodes[moduleIndex];
if (buildResult != 0)
{
printf("error: failed to build file\n");
return false;
}
}

return true;


+ 14
- 8
src/ModuleManager.hpp View File

@@ -2,15 +2,11 @@

#include <vector>

#include "Tokenizer.hpp"
#include "Evaluator.hpp"
#include "ModuleManagerEnums.hpp"

enum ModuleDependencyType
{
ModuleDependency_Cakelisp,
ModuleDependency_Library,
ModuleDependency_CFile
};
#include "Evaluator.hpp"
#include "RunProcess.hpp"
#include "Tokenizer.hpp"

struct ModuleDependency
{
@@ -26,7 +22,17 @@ struct Module
GeneratorOutput* generatedOutput;
std::string sourceOutputName;
std::string headerOutputName;

// Build system
std::vector<ModuleDependency> dependencies;
bool skipBuild;

// These make sense to overload if you want a compile-time dependency
ProcessCommand compileTimeBuildCommand;
ProcessCommand compileTimeLinkCommand;

ProcessCommand buildTimeBuildCommand;
ProcessCommand buildTimeLinkCommand;
};

struct ModuleManager


+ 9
- 0
src/ModuleManagerEnums.hpp View File

@@ -0,0 +1,9 @@
#pragma once


enum ModuleDependencyType
{
ModuleDependency_Cakelisp,
ModuleDependency_Library,
ModuleDependency_CFile
};

+ 56
- 2
src/RunProcess.cpp View File

@@ -142,9 +142,9 @@ int runProcess(const RunProcessArguments& arguments, int* statusOut)
s_subprocesses.push_back({statusOut, pid, pipeFileDescriptors[PipeRead], command});
}

return 1;
#endif
return 0;
#endif
return 1;
}

void waitForAllProcessesClosed(SubprocessOnOutputFunc onOutput)
@@ -179,3 +179,57 @@ void waitForAllProcessesClosed(SubprocessOnOutputFunc onOutput)

s_subprocesses.clear();
}

void PrintProcessArguments(const char** processArguments)
{
for (const char** argument = processArguments; *argument; ++argument)
printf("%s ", *argument);
printf("\n");
}

// The array will need to be deleted, but the array members will not
const char** MakeProcessArgumentsFromCommand(ProcessCommand& command,
const ProcessCommandInput* inputs, int numInputs)
{
int numArguments = command.arguments.size();
// +1 for file to execute
int numFinalArguments = numArguments + 1;
// +1 again for the null terminator
const char** newArguments = (const char**)calloc(sizeof(const char*), numFinalArguments + 1);
for (int i = 0; i < numFinalArguments; ++i)
newArguments[i] = nullptr;
newArguments[numFinalArguments] = nullptr;
newArguments[0] = command.fileToExecute.c_str();

for (int i = 0; i < numArguments; ++i)
{
int finalArgumentIndex = i + 1;
ProcessCommandArgument& argument = command.arguments[i];

if (argument.type == ProcessCommandArgumentType_String)
newArguments[finalArgumentIndex] = argument.contents.c_str();
else
{
bool found = false;
for (int input = 0; input < numInputs; ++input)
{
if (inputs[input].type == argument.type)
{
newArguments[finalArgumentIndex] = inputs[input].value;
found = true;
break;
}
}
if (!found)
{
printf("error: command missing input\n");
free(newArguments);
return nullptr;
}
}
}

return newArguments;
}

const int maxProcessesRecommendedSpawned = 8;

+ 33
- 0
src/RunProcess.hpp View File

@@ -1,5 +1,10 @@
#pragma once

#include "RunProcessEnums.hpp"

#include <string>
#include <vector>

struct RunProcessArguments
{
const char* fileToExecute;
@@ -11,3 +16,31 @@ int runProcess(const RunProcessArguments& arguments, int* statusOut);
typedef void (*SubprocessOnOutputFunc)(const char* subprocessOutput);

void waitForAllProcessesClosed(SubprocessOnOutputFunc onOutput);

//
// Helpers for programmatically constructing arguments
//

struct ProcessCommandArgument
{
ProcessCommandArgumentType type;
std::string contents;
};

struct ProcessCommand
{
std::string fileToExecute;
std::vector<ProcessCommandArgument> arguments;
};

struct ProcessCommandInput
{
ProcessCommandArgumentType type;
const char* value;
};

void PrintProcessArguments(const char** processArguments);
const char** MakeProcessArgumentsFromCommand(ProcessCommand& command,
const ProcessCommandInput* inputs, int numInputs);

extern const int maxProcessesRecommendedSpawned;

+ 13
- 0
src/RunProcessEnums.hpp View File

@@ -0,0 +1,13 @@
#pragma once

enum ProcessCommandArgumentType
{
ProcessCommandArgumentType_String,

ProcessCommandArgumentType_SourceInput,
ProcessCommandArgumentType_ObjectOutput,
ProcessCommandArgumentType_CakelispHeadersInclude,

ProcessCommandArgumentType_ObjectInput,
ProcessCommandArgumentType_DynamicLibraryOutput,
};

Loading…
Cancel
Save