Browse Source

WIP Configurable process arguments for building

GeneralDependencyResolver
Macoy Madson 1 month ago
parent
commit
eed2ad9aff
11 changed files with 298 additions and 36 deletions
  1. +1
    -0
      doc/Debugging.org
  2. +7
    -0
      runtime/Macros.cake
  3. +84
    -10
      src/Evaluator.cpp
  4. +19
    -0
      src/Evaluator.hpp
  5. +8
    -0
      src/EvaluatorEnums.hpp
  6. +74
    -1
      src/Generators.cpp
  7. +6
    -0
      src/Main.cpp
  8. +74
    -17
      src/ModuleManager.cpp
  9. +19
    -1
      src/ModuleManager.hpp
  10. +2
    -7
      src/Writer.cpp
  11. +4
    -0
      src/Writer.hpp

+ 1
- 0
doc/Debugging.org View File

@@ -8,6 +8,7 @@ The following command may be run in order to tell GDB where the ~.so~ files you

#+BEGIN_SRC sh
set solib-search-path ~/Development/code/repositories/cakelisp/
set cwd ~/Development/code/repositories/cakelisp/
#+END_SRC

(adjust that path to where you installed cakelisp, of course).


+ 7
- 0
runtime/Macros.cake View File

@@ -1,3 +1,10 @@
(set-cakelisp-option compile-time-compiler "/usr/bin/clang++")
(set-cakelisp-option compile-time-build-arguments
"-g" "-c" 'sourceInput "-o" 'objectOutput 'cakelispHeadersInclude "-fPIC")
(set-cakelisp-option compile-time-linker "/usr/bin/clang++")
(set-cakelisp-option compile-time-link-arguments
"-shared" "-o" 'libraryOutput 'objectInput)

;; TODO: This should be builtin to macros and generators
(defmacro destructure-arguments ()
(var end-invocation-index int (FindCloseParenTokenIndex tokens startTokenIndex))


+ 84
- 10
src/Evaluator.cpp View File

@@ -478,6 +478,62 @@ void PropagateRequiredToReferences(EvaluatorEnvironment& environment)
} while (numRequiresStatusChanged);
}

struct ProcessCommandInput
{
ProcessCommandArgumentType type;
const char* value;
};

void PrintProcessArguments(const char** processArguments)
{
for (const char* argument = processArguments[0]; argument; ++argument)
printf("%s ", argument);
printf("\n");
}

// The array will need to be deleted, but the array members will not
const char** MakeProcessArgumentsFromCommand(ProcessCommand& command,
const ProcessCommandInput* inputs, int numInputs)
{
// +1 for file to execute
int numArguments = command.arguments.size() + 1;
// +1 again for the null terminator
const char** newArguments = (const char**)calloc(sizeof(const char*), numArguments + 1);
for (int i = 0; i < numArguments; ++i)
newArguments[i] = nullptr;
newArguments[numArguments] = nullptr;
newArguments[0] = command.fileToExecute.c_str();

for (int i = 1; i < numArguments - 1; ++i)
{
ProcessCommandArgument& argument = command.arguments[i];

if (argument.type == ProcessCommandArgumentType_String)
newArguments[i] = argument.contents.c_str();
else
{
bool found = false;
for (int input = 0; input < numInputs; ++input)
{
if (inputs[input].type == argument.type)
{
newArguments[i] = inputs[input].value;
found = true;
break;
}
}
if (!found)
{
printf("error: command missing input\n");
free(newArguments);
return nullptr;
}
}
}

return newArguments;
}

void OnCompileProcessOutput(const char* output)
{
// TODO C/C++ error to Cakelisp token mapper
@@ -557,6 +613,14 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
outputSettings.sourceFooter = macroSourceFooter;
}
outputSettings.sourceCakelispFilename = fileOutputName;
{
char writerSourceOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(writerSourceOutputName, "%s.cpp", fileOutputName);
char writerHeaderOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(writerHeaderOutputName, "%s.hpp", fileOutputName);
outputSettings.sourceOutputName = writerSourceOutputName;
outputSettings.headerOutputName = writerHeaderOutputName;
}
// Use the separate output prepared specifically for this compile-time object
if (!writeGeneratorOutput(*definition->output, nameSettings, formatSettings,
outputSettings))
@@ -568,9 +632,6 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,

buildObject.stage = BuildStage_Compiling;

char fileToExec[MAX_PATH_LENGTH] = {0};
PrintBuffer(fileToExec, "/usr/bin/clang++");

// The evaluator is written in C++, so all generators and macros need to support the C++
// features used (e.g. their signatures have std::vector<>)
char sourceOutputName[MAX_PATH_LENGTH] = {0};
@@ -607,19 +668,32 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
PrintfBuffer(headerInclude, "-I%s", environment.cakelispSrcDir.c_str());
}

// TODO: Get arguments all the way from the top
// If not null terminated, the call will fail
const char* arguments[] = {fileToExec, "-g", "-c", sourceOutputName, "-o",
buildObjectName, headerInclude, "-fPIC", nullptr};
ProcessCommandInput compileTimeInputs[] = {
{ProcessCommandArgumentType_SourceInput, sourceOutputName},
{ProcessCommandArgumentType_ObjectOutput, buildObjectName},
{ProcessCommandArgumentType_CakelispHeadersInclude, headerInclude}};
const char** buildArguments = MakeProcessArgumentsFromCommand(
environment.compileTimeBuildCommand, compileTimeInputs, ArraySize(compileTimeInputs));
if (!buildArguments)
{
// TODO: Abort building if cannot invoke compiler
continue;
}
// const char* arguments[] = {fileToExec, "-g", "-c", sourceOutputName,
// "-o", buildObjectName, headerInclude, "-fPIC", nullptr};
// TODO: Get rid of intermediate structure
RunProcessArguments compileArguments = {};
compileArguments.fileToExecute = fileToExec;
compileArguments.arguments = arguments;
compileArguments.fileToExecute = environment.compileTimeBuildCommand.fileToExecute.c_str();
compileArguments.arguments = buildArguments;
if (runProcess(compileArguments, &buildObject.status) != 0)
{
// TODO: Abort building if cannot invoke compiler
// TODO: Abort building if cannot invoke compiler?
// free(buildArguments);
// return 0;
}

free(buildArguments);

// TODO This could be made smarter by allowing more spawning right when a process closes,
// instead of starting in waves
++currentNumProcessesSpawned;


+ 19
- 0
src/Evaluator.hpp View File

@@ -11,6 +11,7 @@ struct NameStyleSettings;
struct Token;
struct GeneratorOutput;
struct ModuleManager;
struct Module;

// Rather than needing to allocate and edit a buffer eventually equal to the size of the final
// output, store output operations instead. This also facilitates source <-> generated mapping data
@@ -82,6 +83,7 @@ struct EvaluatorContext
bool isRequired;
// Associate all unknown references with this definition
const Token* definitionName;
Module* module;
};

struct EvaluatorEnvironment;
@@ -167,6 +169,18 @@ typedef std::pair<const std::string, ObjectDefinition> ObjectDefinitionPair;
typedef std::unordered_map<std::string, ObjectReferencePool> ObjectReferencePoolMap;
typedef std::pair<const std::string, ObjectReferencePool> ObjectReferencePoolPair;

struct ProcessCommandArgument
{
ProcessCommandArgumentType type;
std::string contents;
};

struct ProcessCommand
{
std::string fileToExecute;
std::vector<ProcessCommandArgument> arguments;
};

// Unlike context, which can't be changed, environment can be changed.
// Use care when modifying the environment. Only add things once you know things have succeeded.
// Keep in mind that calling functions which can change the environment may invalidate your pointers
@@ -200,6 +214,11 @@ struct EvaluatorEnvironment
// Added as a search directory for compile time code execution
std::string cakelispSrcDir;

ProcessCommand compileTimeBuildCommand;
ProcessCommand compileTimeLinkCommand;
ProcessCommand buildTimeBuildCommand;
ProcessCommand buildTimeLinkCommand;

// Will NOT clean up macroExpansions! Use environmentDestroyInvalidateTokens()
~EvaluatorEnvironment();
};


+ 8
- 0
src/EvaluatorEnums.hpp View File

@@ -71,3 +71,11 @@ enum ObjectReferenceGuessState
GuessState_WaitingForLoad,
GuessState_Resolved,
};

enum ProcessCommandArgumentType
{
ProcessCommandArgumentType_String,
ProcessCommandArgumentType_SourceInput,
ProcessCommandArgumentType_ObjectOutput,
ProcessCommandArgumentType_CakelispHeadersInclude,
};

+ 74
- 1
src/Generators.cpp View File

@@ -9,6 +9,35 @@

#include <string.h>

typedef bool (*ProcessCommandOptionFunc)(EvaluatorEnvironment& environment,
const std::vector<Token>& tokens, int startTokenIndex,
ProcessCommand* command);

bool SetProcessCommandFileToExec(EvaluatorEnvironment& environment,
const std::vector<Token>& tokens, int startTokenIndex,
ProcessCommand* command)
{
int endInvocationIndex = FindCloseParenTokenIndex(tokens, startTokenIndex);
int argumentIndex = getExpectedArgument("expected path to compiler", tokens, startTokenIndex, 2,
endInvocationIndex);
if (argumentIndex == -1)
return false;

const Token& argumentToken = tokens[argumentIndex];

if (!ExpectTokenType("file to execute", argumentToken, TokenType_String))
return false;

command->fileToExecute = argumentToken.contents;
return true;
}

bool SetProcessCommandArguments(EvaluatorEnvironment& environment, const std::vector<Token>& tokens,
int startTokenIndex, ProcessCommand* command)
{
return true;
}

bool SetCakelispOption(EvaluatorEnvironment& environment, const EvaluatorContext& context,
const std::vector<Token>& tokens, int startTokenIndex,
GeneratorOutput& output)
@@ -47,7 +76,7 @@ bool SetCakelispOption(EvaluatorEnvironment& environment, const EvaluatorContext
}

// This needs to be defined early, else things will only be partially supported
if (tokens[optionNameIndex].contents.compare("enable-hot-reloading") == 0)
else if (tokens[optionNameIndex].contents.compare("enable-hot-reloading") == 0)
{
int enableStateIndex =
getExpectedArgument("expected path", tokens, startTokenIndex, 2, endInvocationIndex);
@@ -72,6 +101,37 @@ bool SetCakelispOption(EvaluatorEnvironment& environment, const EvaluatorContext
return true;
}

struct ProcessCommandOptions
{
const char* optionName;
ProcessCommand* command;
ProcessCommandOptionFunc handler;
};
ProcessCommandOptions commandOptions[] = {
{"compile-time-compiler", &environment.compileTimeBuildCommand,
SetProcessCommandFileToExec},
{"compile-time-compile-arguments", &environment.compileTimeBuildCommand,
SetProcessCommandArguments},
{"compile-time-linker", &environment.compileTimeLinkCommand, SetProcessCommandFileToExec},
{"compile-time-link-arguments", &environment.compileTimeLinkCommand,
SetProcessCommandArguments},
{"build-time-compiler", &environment.buildTimeBuildCommand, SetProcessCommandFileToExec},
{"build-time-compile-arguments", &environment.buildTimeBuildCommand,
SetProcessCommandArguments},
{"build-time-linker", &environment.buildTimeLinkCommand, SetProcessCommandFileToExec},
{"build-time-link-arguments", &environment.buildTimeLinkCommand,
SetProcessCommandArguments},
};

for (unsigned int i = 0; i < ArraySize(commandOptions); ++i)
{
if (tokens[optionNameIndex].contents.compare(commandOptions[i].optionName))
{
return commandOptions[i].handler(environment, tokens, startTokenIndex,
commandOptions[i].command);
}
}

ErrorAtToken(tokens[optionNameIndex], "unrecognized option");
return false;
}
@@ -144,6 +204,19 @@ bool ImportGenerator(EvaluatorEnvironment& environment, const EvaluatorContext&
}
else
{
if (context.module)
{
ModuleDependency newCakelispDependency = {};
newCakelispDependency.type = ModuleDependency_Cakelisp;
newCakelispDependency.name = currentToken.contents;
context.module->dependencies.push_back(newCakelispDependency);
}
else
{
NoteAtToken(currentToken,
"module cannot track dependency (potential internal error)");
}

char relativePathBuffer[MAX_PATH_LENGTH] = {0};
getDirectoryFromPath(currentToken.source, relativePathBuffer,
sizeof(relativePathBuffer));


+ 6
- 0
src/Main.cpp View File

@@ -148,6 +148,12 @@ int main(int numArguments, char* arguments[])

printf("Successfully generated files\n");

if (!moduleManagerBuild(moduleManager))
{
moduleManagerDestroy(moduleManager);
return 1;
}

moduleManagerDestroy(moduleManager);
return 0;
}

+ 74
- 17
src/ModuleManager.cpp View File

@@ -3,6 +3,7 @@
#include "Converters.hpp"
#include "DynamicLoader.hpp"
#include "Evaluator.hpp"
#include "EvaluatorEnums.hpp"
#include "FileUtilities.hpp"
#include "Generators.hpp"
#include "Logging.hpp"
@@ -37,6 +38,32 @@ void moduleManagerInitialize(ModuleManager& manager)

manager.environment.moduleManager = &manager;

// Command defaults
{
manager.environment.compileTimeBuildCommand.fileToExecute = "/usr/bin/clang++";
manager.environment.compileTimeBuildCommand.arguments = {
{ProcessCommandArgumentType_String, "-g"},
{ProcessCommandArgumentType_String, "-c"},
{ProcessCommandArgumentType_SourceInput, EmptyString},
{ProcessCommandArgumentType_String, "-o"},
{ProcessCommandArgumentType_ObjectOutput, EmptyString},
{ProcessCommandArgumentType_CakelispHeadersInclude, EmptyString},
{ProcessCommandArgumentType_String, "-fPIC"}};

manager.environment.compileTimeLinkCommand.fileToExecute = "/usr/bin/clang++";

manager.environment.buildTimeBuildCommand.fileToExecute = "/usr/bin/clang++";
manager.environment.buildTimeBuildCommand.arguments = {
{ProcessCommandArgumentType_String, "-g"},
{ProcessCommandArgumentType_String, "-c"},
{ProcessCommandArgumentType_SourceInput, EmptyString},
{ProcessCommandArgumentType_String, "-o"},
{ProcessCommandArgumentType_ObjectOutput, EmptyString},
{ProcessCommandArgumentType_String, "-fPIC"}};

manager.environment.buildTimeLinkCommand.fileToExecute = "/usr/bin/clang++";
}

makeDirectory(cakelispWorkingDir);
printf("Using cache at %s\n", cakelispWorkingDir);
}
@@ -44,11 +71,12 @@ void moduleManagerInitialize(ModuleManager& manager)
void moduleManagerDestroy(ModuleManager& manager)
{
environmentDestroyInvalidateTokens(manager.environment);
for (Module& module : manager.modules)
for (Module* module : manager.modules)
{
delete module.tokens;
delete module.generatedOutput;
free((void*)module.filename);
delete module->tokens;
delete module->generatedOutput;
free((void*)module->filename);
delete module;
}
manager.modules.clear();
closeAllDynamicLibraries();
@@ -145,27 +173,28 @@ bool moduleLoadTokenizeValidate(const char* filename, const std::vector<Token>**

bool moduleManagerAddEvaluateFile(ModuleManager& manager, const char* filename)
{
for (Module& module : manager.modules)
for (Module* module : manager.modules)
{
if (strcmp(module.filename, filename) == 0)
if (strcmp(module->filename, filename) == 0)
{
printf("Already loaded %s\n", filename);
return true;
}
}

Module newModule = {};
Module* newModule = new Module();
// We need to keep this memory around for the lifetime of the token, regardless of relocation
newModule.filename = strdup(filename);
newModule->filename = strdup(filename);
// This stage cleans up after itself if it fails
if (!moduleLoadTokenizeValidate(newModule.filename, &newModule.tokens))
if (!moduleLoadTokenizeValidate(newModule->filename, &newModule->tokens))
return false;

newModule.generatedOutput = new GeneratorOutput;
newModule->generatedOutput = new GeneratorOutput;

manager.modules.push_back(newModule);

EvaluatorContext moduleContext = {};
moduleContext.module = newModule;
moduleContext.scope = EvaluatorScope_Module;
moduleContext.definitionName = &manager.globalPseudoInvocationName;
// Module always requires all its functions
@@ -175,14 +204,14 @@ bool moduleManagerAddEvaluateFile(ModuleManager& manager, const char* filename)
StringOutput bodyDelimiterTemplate = {};
bodyDelimiterTemplate.modifiers = StringOutMod_NewlineAfter;
int numErrors = EvaluateGenerateAll_Recursive(
manager.environment, moduleContext, *newModule.tokens,
/*startTokenIndex=*/0, &bodyDelimiterTemplate, *newModule.generatedOutput);
manager.environment, moduleContext, *newModule->tokens,
/*startTokenIndex=*/0, &bodyDelimiterTemplate, *newModule->generatedOutput);
// After this point, the module may have references to its tokens in the environmment, so we
// cannot destroy it until we're done evaluating everything
if (numErrors)
return false;

printf("Loaded %s\n", newModule.filename);
printf("Loaded %s\n", newModule->filename);
return true;
}

@@ -196,14 +225,14 @@ bool moduleManagerWriteGeneratedOutput(ModuleManager& manager)
NameStyleSettings nameSettings;
WriterFormatSettings formatSettings;

for (Module& module : manager.modules)
for (Module* module : manager.modules)
{
WriterOutputSettings outputSettings;
outputSettings.sourceCakelispFilename = module.filename;
outputSettings.sourceCakelispFilename = module->filename;

// TODO: hpp to h support
char relativeIncludeBuffer[MAX_PATH_LENGTH];
getFilenameFromPath(module.filename, relativeIncludeBuffer, sizeof(relativeIncludeBuffer));
getFilenameFromPath(module->filename, relativeIncludeBuffer, sizeof(relativeIncludeBuffer));
char sourceHeadingBuffer[1024] = {0};
PrintfBuffer(sourceHeadingBuffer, "#include \"%s.hpp\"\n%s", relativeIncludeBuffer,
generatedSourceHeading ? generatedSourceHeading : "");
@@ -212,10 +241,38 @@ bool moduleManagerWriteGeneratedOutput(ModuleManager& manager)
outputSettings.headerHeading = generatedHeaderHeading;
outputSettings.headerFooter = generatedHeaderFooter;

if (!writeGeneratorOutput(*module.generatedOutput, nameSettings, formatSettings,
char sourceOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(sourceOutputName, "%s.cpp", outputSettings.sourceCakelispFilename);
char headerOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(headerOutputName, "%s.hpp", outputSettings.sourceCakelispFilename);
module->sourceOutputName = sourceOutputName;
module->headerOutputName = headerOutputName;
outputSettings.sourceOutputName = module->sourceOutputName.c_str();
outputSettings.headerOutputName = module->headerOutputName.c_str();

if (!writeGeneratorOutput(*module->generatedOutput, nameSettings, formatSettings,
outputSettings))
return false;
}

return true;
}

bool moduleManagerBuild(ModuleManager& manager)
{
for (Module* module : manager.modules)
{
printf("Build module %s\n", module->sourceOutputName.c_str());
for (ModuleDependency& dependency : module->dependencies)
{
printf("\tRequires %s\n", dependency.name.c_str());

// Cakelisp files are built at the module manager level, so we need not concern
// ourselves with them
if (dependency.type == ModuleDependency_Cakelisp)
continue;
}
}

return true;
}

+ 19
- 1
src/ModuleManager.hpp View File

@@ -5,12 +5,28 @@
#include "Tokenizer.hpp"
#include "Evaluator.hpp"

enum ModuleDependencyType
{
ModuleDependency_Cakelisp,
ModuleDependency_Library,
ModuleDependency_CFile
};

struct ModuleDependency
{
ModuleDependencyType type;
std::string name;
};

// A module is typically associated with a single file. Keywords like local mean in-module only
struct Module
{
const char* filename;
const std::vector<Token>* tokens;
GeneratorOutput* generatedOutput;
std::string sourceOutputName;
std::string headerOutputName;
std::vector<ModuleDependency> dependencies;
};

struct ModuleManager
@@ -18,7 +34,8 @@ struct ModuleManager
// Shared environment across all modules
EvaluatorEnvironment environment;
Token globalPseudoInvocationName;
std::vector<Module> modules;
// Pointer only so things cannot move around
std::vector<Module*> modules;
};

void moduleManagerInitialize(ModuleManager& manager);
@@ -28,3 +45,4 @@ bool moduleLoadTokenizeValidate(const char* filename, const std::vector<Token>**
bool moduleManagerAddEvaluateFile(ModuleManager& manager, const char* filename);
bool moduleManagerEvaluateResolveReferences(ModuleManager& manager);
bool moduleManagerWriteGeneratedOutput(ModuleManager& manager);
bool moduleManagerBuild(ModuleManager& manager);

+ 2
- 7
src/Writer.cpp View File

@@ -371,11 +371,6 @@ void writeOutputFollowSplices_Recursive(const NameStyleSettings& nameSettings,
bool writeOutputs(const NameStyleSettings& nameSettings, const WriterFormatSettings& formatSettings,
const WriterOutputSettings& outputSettings, const GeneratorOutput& outputToWrite)
{
char sourceOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(sourceOutputName, "%s.cpp", outputSettings.sourceCakelispFilename);
char headerOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(headerOutputName, "%s.hpp", outputSettings.sourceCakelispFilename);

struct
{
bool isHeader;
@@ -388,7 +383,7 @@ bool writeOutputs(const NameStyleSettings& nameSettings, const WriterFormatSetti
StringOutputState stateBeforeOutputWrite;
char tempFilename[MAX_PATH_LENGTH];
} outputs[] = {{/*isHeader=*/false,
sourceOutputName,
outputSettings.sourceOutputName,
outputSettings.sourceHeading,
outputSettings.sourceFooter,
{},
@@ -396,7 +391,7 @@ bool writeOutputs(const NameStyleSettings& nameSettings, const WriterFormatSetti
{0}},
{
/*isHeader=*/true,
headerOutputName,
outputSettings.headerOutputName,
outputSettings.headerHeading,
outputSettings.headerFooter,
{},


+ 4
- 0
src/Writer.hpp View File

@@ -20,6 +20,10 @@ struct WriterFormatSettings
struct WriterOutputSettings
{
const char* sourceCakelispFilename;

const char* sourceOutputName;
const char* headerOutputName;

const char* sourceHeading;
const char* sourceFooter;
const char* headerHeading;


Loading…
Cancel
Save