3 Commits

Author SHA1 Message Date
  Macoy Madson 588e39756b Process command arguments can now be changed 1 month ago
  Macoy Madson eed2ad9aff WIP Configurable process arguments for building 1 month ago
  Macoy Madson 87a9b71a30 Moved comptime function building into function 1 month ago
13 changed files with 585 additions and 227 deletions
Split View
  1. +3
    -3
      BuildAndRunTests.sh
  2. +1
    -0
      doc/Debugging.org
  3. +7
    -0
      runtime/Macros.cake
  4. +289
    -197
      src/Evaluator.cpp
  5. +19
    -0
      src/Evaluator.hpp
  6. +12
    -0
      src/EvaluatorEnums.hpp
  7. +126
    -1
      src/Generators.cpp
  8. +6
    -0
      src/Main.cpp
  9. +84
    -17
      src/ModuleManager.cpp
  10. +19
    -1
      src/ModuleManager.hpp
  11. +13
    -1
      src/RunProcess.cpp
  12. +2
    -7
      src/Writer.cpp
  13. +4
    -0
      src/Writer.hpp

+ 3
- 3
BuildAndRunTests.sh View File

@@ -1,8 +1,8 @@
#!/bin/sh

# jam -j4 && ./bin/cakelisp test/Macros.cake
# jam -j4 && ./bin/cakelisp test/Dependencies.cake
# jam -j4 && ./bin/cakelisp test/Basic.cake
jam -j4 && ./bin/cakelisp test/Macros.cake
jam -j4 && ./bin/cakelisp test/Dependencies.cake
jam -j4 && ./bin/cakelisp test/Basic.cake

# Build Cakelisp itself, generate the runtime, then build the runtime
jam -j4 && ./bin/cakelisp --enable-hot-reloading runtime/TestMain.cake runtime/TextAdventure.cake \


+ 1
- 0
doc/Debugging.org View File

@@ -8,6 +8,7 @@ The following command may be run in order to tell GDB where the ~.so~ files you

#+BEGIN_SRC sh
set solib-search-path ~/Development/code/repositories/cakelisp/
set cwd ~/Development/code/repositories/cakelisp/
#+END_SRC

(adjust that path to where you installed cakelisp, of course).


+ 7
- 0
runtime/Macros.cake View File

@@ -1,3 +1,10 @@
(set-cakelisp-option compile-time-compiler "/usr/bin/clang++")
(set-cakelisp-option compile-time-compile-arguments
"-g" "-c" 'source-input "-o" 'object-output 'cakelisp-headers-include "-fPIC")
(set-cakelisp-option compile-time-linker "/usr/bin/clang++")
(set-cakelisp-option compile-time-link-arguments
"-shared" "-o" 'library-output 'object-input)

;; TODO: This should be builtin to macros and generators
(defmacro destructure-arguments ()
(var end-invocation-index int (FindCloseParenTokenIndex tokens startTokenIndex))


+ 289
- 197
src/Evaluator.cpp View File

@@ -478,196 +478,99 @@ void PropagateRequiredToReferences(EvaluatorEnvironment& environment)
} while (numRequiresStatusChanged);
}

void OnCompileProcessOutput(const char* output)
struct ProcessCommandInput
{
// TODO C/C++ error to Cakelisp token mapper
}
ProcessCommandArgumentType type;
const char* value;
};

int BuildEvaluateReferences(EvaluatorEnvironment& environment, int& numErrorsOut)
void PrintProcessArguments(const char** processArguments)
{
int numReferencesResolved = 0;

enum BuildStage
{
BuildStage_None,
BuildStage_Compiling,
BuildStage_Linking,
BuildStage_Loading,
BuildStage_ResolvingReferences,
BuildStage_Finished
};

// Note: environment.definitions can be resized/rehashed during evaluation, which invalidates
// iterators. For now, I will rely on the fact that std::unordered_map does not invalidate
// references on resize. This will need to change if the data structure changes
struct BuildObject
{
int buildId = -1;
int status = -1;
BuildStage stage = BuildStage_None;
std::string artifactsName;
std::string dynamicLibraryPath;
std::string buildObjectName;
ObjectDefinition* definition = nullptr;
};

std::vector<BuildObject> definitionsToBuild;

// We must copy references in case environment.definitions is modified, which would invalidate
// iterators, but not references
std::vector<ObjectDefinition*> definitionsToCheck;
definitionsToCheck.reserve(environment.definitions.size());
for (ObjectDefinitionPair& definitionPair : environment.definitions)
{
ObjectDefinition& definition = definitionPair.second;
// Does it need to be built?
if (!definition.isRequired)
continue;

// Is it already in the environment?
if (definition.isLoaded)
continue;

definitionsToCheck.push_back(&definition);
}
for (const char* argument = processArguments[0]; argument; ++argument)
printf("%s ", argument);
printf("\n");
}

for (ObjectDefinition* definitionPointer : definitionsToCheck)
// The array will need to be deleted, but the array members will not
const char** MakeProcessArgumentsFromCommand(ProcessCommand& command,
const ProcessCommandInput* inputs, int numInputs)
{
int numArguments = command.arguments.size();
// +1 for file to execute
int numFinalArguments = numArguments + 1;
// +1 again for the null terminator
const char** newArguments = (const char**)calloc(sizeof(const char*), numFinalArguments + 1);
for (int i = 0; i < numFinalArguments; ++i)
newArguments[i] = nullptr;
newArguments[numFinalArguments] = nullptr;
newArguments[0] = command.fileToExecute.c_str();

for (int i = 0; i < numArguments; ++i)
{
ObjectDefinition& definition = *definitionPointer;
const char* defName = definition.name->contents.c_str();
int finalArgumentIndex = i + 1;
ProcessCommandArgument& argument = command.arguments[i];

if (log.buildReasons)
printf("Checking to build %s\n", defName);

// Can it be built in the current environment?
bool canBuild = true;
bool hasRelevantChangeOccurred = false;
bool hasGuessedRefs = false;
// If there were new guesses, we will do another pass over this definition's references in
// case new references turned up
bool guessMaybeDirtiedReferences = false;
do
if (argument.type == ProcessCommandArgumentType_String)
newArguments[finalArgumentIndex] = argument.contents.c_str();
else
{
guessMaybeDirtiedReferences = false;

// Copy pointers to refs in case of iterator invalidation
std::vector<ObjectReferenceStatus*> referencesToCheck;
referencesToCheck.reserve(definition.references.size());
for (ObjectReferenceStatusPair& referencePair : definition.references)
{
referencesToCheck.push_back(&referencePair.second);
}
for (ObjectReferenceStatus* referencePointer : referencesToCheck)
bool found = false;
for (int input = 0; input < numInputs; ++input)
{
ObjectReferenceStatus& referenceStatus = *referencePointer;

// TODO: (Performance) Add shortcut in reference
ObjectDefinitionMap::iterator findIt =
environment.definitions.find(referenceStatus.name->contents);
if (findIt != environment.definitions.end())
{
if (isCompileTimeObject(findIt->second.type))
{
bool refCompileTimeCodeLoaded = findIt->second.isLoaded;
if (refCompileTimeCodeLoaded)
{
// The reference is ready to go. Built objects immediately resolve
// references. We will react to it if the last thing we did was guess
// incorrectly that this was a C call
if (referenceStatus.guessState != GuessState_Resolved)
{
if (log.buildReasons)
printf("\tRequired code has been loaded\n");

hasRelevantChangeOccurred = true;
}

referenceStatus.guessState = GuessState_Resolved;
}
else
{
// If we know we are missing a compile time function, we won't try to
// guess
if (log.buildReasons)
printf("\tCannot build until %s is loaded\n",
referenceStatus.name->contents.c_str());

referenceStatus.guessState = GuessState_WaitingForLoad;
canBuild = false;
}
}
else if (findIt->second.type == ObjectType_Function &&
referenceStatus.guessState != GuessState_Resolved)
{
// A known Cakelisp function call
for (int i = 0; i < (int)referenceStatus.references.size(); ++i)
{
ObjectReference& reference = referenceStatus.references[i];
// Run function invocation on it
// TODO: Make invocation generator know it is a Cakelisp function
bool result = FunctionInvocationGenerator(
environment, reference.context, *reference.tokens,
reference.startIndex, *reference.spliceOutput);
// Our guess didn't even evaluate
if (!result)
canBuild = false;
}

referenceStatus.guessState = GuessState_Resolved;
}
// TODO: Building references to non-comptime functions at comptime
}
else
if (inputs[input].type == argument.type)
{
if (referenceStatus.guessState == GuessState_None)
{
if (log.buildReasons)
printf("\tCannot build until %s is guessed. Guessing now\n",
referenceStatus.name->contents.c_str());

// Find all the times the definition makes this reference
// We must use indices because the call to FunctionInvocationGenerator can
// add new references to the list
for (int i = 0; i < (int)referenceStatus.references.size(); ++i)
{
ObjectReference& reference = referenceStatus.references[i];
// Run function invocation on it
bool result = FunctionInvocationGenerator(
environment, reference.context, *reference.tokens,
reference.startIndex, *reference.spliceOutput);
// Our guess didn't even evaluate
if (!result)
canBuild = false;
}

referenceStatus.guessState = GuessState_Guessed;
hasRelevantChangeOccurred = true;
hasGuessedRefs = true;
guessMaybeDirtiedReferences = true;
}
else if (referenceStatus.guessState == GuessState_Guessed)
{
// It has been guessed, and still isn't in definitions
hasGuessedRefs = true;
}
newArguments[finalArgumentIndex] = inputs[input].value;
found = true;
break;
}
}
} while (guessMaybeDirtiedReferences);

// hasRelevantChangeOccurred being false suppresses rebuilding compile-time functions which
// still have the same missing references. Note that only compile time objects can be built.
// We put normal functions through the guessing system too because they need their functions
// resolved as well. It's a bit dirty but not too bad
if (canBuild && (!hasGuessedRefs || hasRelevantChangeOccurred) &&
isCompileTimeObject(definition.type))
{
BuildObject objectToBuild = {};
objectToBuild.buildId = getNextFreeBuildId(environment);
objectToBuild.definition = &definition;
definitionsToBuild.push_back(objectToBuild);
if (!found)
{
printf("error: command missing input\n");
free(newArguments);
return nullptr;
}
}
}

return newArguments;
}

void OnCompileProcessOutput(const char* output)
{
// TODO C/C++ error to Cakelisp token mapper
}

enum BuildStage
{
BuildStage_None,
BuildStage_Compiling,
BuildStage_Linking,
BuildStage_Loading,
BuildStage_ResolvingReferences,
BuildStage_Finished
};

// Note: environment.definitions can be resized/rehashed during evaluation, which invalidates
// iterators. For now, I will rely on the fact that std::unordered_map does not invalidate
// references on resize. This will need to change if the data structure changes
struct BuildObject
{
int buildId = -1;
int status = -1;
BuildStage stage = BuildStage_None;
std::string artifactsName;
std::string dynamicLibraryPath;
std::string buildObjectName;
ObjectDefinition* definition = nullptr;
};

int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
std::vector<BuildObject>& definitionsToBuild,
int& numErrorsOut)
{
int numReferencesResolved = 0;

// Spin up as many compile processes as necessary
// TODO: Combine sure-thing builds into batches (ones where we know all references)
// TODO: Instead of creating files, pipe straight to compiler?
@@ -712,6 +615,14 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment, int& numErrorsOut
outputSettings.sourceFooter = macroSourceFooter;
}
outputSettings.sourceCakelispFilename = fileOutputName;
{
char writerSourceOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(writerSourceOutputName, "%s.cpp", fileOutputName);
char writerHeaderOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(writerHeaderOutputName, "%s.hpp", fileOutputName);
outputSettings.sourceOutputName = writerSourceOutputName;
outputSettings.headerOutputName = writerHeaderOutputName;
}
// Use the separate output prepared specifically for this compile-time object
if (!writeGeneratorOutput(*definition->output, nameSettings, formatSettings,
outputSettings))
@@ -723,9 +634,6 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment, int& numErrorsOut

buildObject.stage = BuildStage_Compiling;

char fileToExec[MAX_PATH_LENGTH] = {0};
PrintBuffer(fileToExec, "/usr/bin/clang++");

// The evaluator is written in C++, so all generators and macros need to support the C++
// features used (e.g. their signatures have std::vector<>)
char sourceOutputName[MAX_PATH_LENGTH] = {0};
@@ -762,19 +670,32 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment, int& numErrorsOut
PrintfBuffer(headerInclude, "-I%s", environment.cakelispSrcDir.c_str());
}

// TODO: Get arguments all the way from the top
// If not null terminated, the call will fail
const char* arguments[] = {fileToExec, "-g", "-c", sourceOutputName, "-o",
buildObjectName, headerInclude, "-fPIC", nullptr};
ProcessCommandInput compileTimeInputs[] = {
{ProcessCommandArgumentType_SourceInput, sourceOutputName},
{ProcessCommandArgumentType_ObjectOutput, buildObjectName},
{ProcessCommandArgumentType_CakelispHeadersInclude, headerInclude}};
const char** buildArguments = MakeProcessArgumentsFromCommand(
environment.compileTimeBuildCommand, compileTimeInputs, ArraySize(compileTimeInputs));
if (!buildArguments)
{
// TODO: Abort building if cannot invoke compiler
continue;
}
// const char* arguments[] = {fileToExec, "-g", "-c", sourceOutputName,
// "-o", buildObjectName, headerInclude, "-fPIC", nullptr};
// TODO: Get rid of intermediate structure
RunProcessArguments compileArguments = {};
compileArguments.fileToExecute = fileToExec;
compileArguments.arguments = arguments;
compileArguments.fileToExecute = environment.compileTimeBuildCommand.fileToExecute.c_str();
compileArguments.arguments = buildArguments;
if (runProcess(compileArguments, &buildObject.status) != 0)
{
// TODO: Abort building if cannot invoke compiler
// TODO: Abort building if cannot invoke compiler?
// free(buildArguments);
// return 0;
}

free(buildArguments);

// TODO This could be made smarter by allowing more spawning right when a process closes,
// instead of starting in waves
++currentNumProcessesSpawned;
@@ -807,22 +728,26 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment, int& numErrorsOut
if (log.buildProcess)
printf("Compiled %s successfully\n", buildObject.definition->name->contents.c_str());

char linkerExecutable[MAX_PATH_LENGTH] = {0};
PrintBuffer(linkerExecutable, "/usr/bin/clang++");

const char* arguments[] = {linkerExecutable,
"-shared",
"-o",
buildObject.dynamicLibraryPath.c_str(),
buildObject.buildObjectName.c_str(),
nullptr};
ProcessCommandInput linkTimeInputs[] = {
{ProcessCommandArgumentType_DynamicLibraryOutput,
buildObject.dynamicLibraryPath.c_str()},
{ProcessCommandArgumentType_ObjectInput, buildObject.buildObjectName.c_str()}};
const char** linkArgumentList = MakeProcessArgumentsFromCommand(
environment.compileTimeLinkCommand, linkTimeInputs, ArraySize(linkTimeInputs));
if (!linkArgumentList)
{
// TODO: Abort building if cannot invoke compiler
continue;
}
RunProcessArguments linkArguments = {};
linkArguments.fileToExecute = linkerExecutable;
linkArguments.arguments = arguments;
linkArguments.fileToExecute = environment.compileTimeLinkCommand.fileToExecute.c_str();
linkArguments.arguments = linkArgumentList;
if (runProcess(linkArguments, &buildObject.status) != 0)
{
// TODO: Abort if linker failed?
// free(linkArgumentList);
}
free(linkArgumentList);
}

// The result of the linking will go straight to our definitionsToBuild
@@ -933,6 +858,173 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment, int& numErrorsOut
return numReferencesResolved;
}

int BuildEvaluateReferences(EvaluatorEnvironment& environment, int& numErrorsOut)
{
int numReferencesResolved = 0;

std::vector<BuildObject> definitionsToBuild;

// We must copy references in case environment.definitions is modified, which would invalidate
// iterators, but not references
std::vector<ObjectDefinition*> definitionsToCheck;
definitionsToCheck.reserve(environment.definitions.size());
for (ObjectDefinitionPair& definitionPair : environment.definitions)
{
ObjectDefinition& definition = definitionPair.second;
// Does it need to be built?
if (!definition.isRequired)
continue;

// Is it already in the environment?
if (definition.isLoaded)
continue;

definitionsToCheck.push_back(&definition);
}

for (ObjectDefinition* definitionPointer : definitionsToCheck)
{
ObjectDefinition& definition = *definitionPointer;
const char* defName = definition.name->contents.c_str();

if (log.buildReasons)
printf("Checking to build %s\n", defName);

// Can it be built in the current environment?
bool canBuild = true;
bool hasRelevantChangeOccurred = false;
bool hasGuessedRefs = false;
// If there were new guesses, we will do another pass over this definition's references in
// case new references turned up
bool guessMaybeDirtiedReferences = false;
do
{
guessMaybeDirtiedReferences = false;

// Copy pointers to refs in case of iterator invalidation
std::vector<ObjectReferenceStatus*> referencesToCheck;
referencesToCheck.reserve(definition.references.size());
for (ObjectReferenceStatusPair& referencePair : definition.references)
{
referencesToCheck.push_back(&referencePair.second);
}
for (ObjectReferenceStatus* referencePointer : referencesToCheck)
{
ObjectReferenceStatus& referenceStatus = *referencePointer;

// TODO: (Performance) Add shortcut in reference
ObjectDefinitionMap::iterator findIt =
environment.definitions.find(referenceStatus.name->contents);
if (findIt != environment.definitions.end())
{
if (isCompileTimeObject(findIt->second.type))
{
bool refCompileTimeCodeLoaded = findIt->second.isLoaded;
if (refCompileTimeCodeLoaded)
{
// The reference is ready to go. Built objects immediately resolve
// references. We will react to it if the last thing we did was guess
// incorrectly that this was a C call
if (referenceStatus.guessState != GuessState_Resolved)
{
if (log.buildReasons)
printf("\tRequired code has been loaded\n");

hasRelevantChangeOccurred = true;
}

referenceStatus.guessState = GuessState_Resolved;
}
else
{
// If we know we are missing a compile time function, we won't try to
// guess
if (log.buildReasons)
printf("\tCannot build until %s is loaded\n",
referenceStatus.name->contents.c_str());

referenceStatus.guessState = GuessState_WaitingForLoad;
canBuild = false;
}
}
else if (findIt->second.type == ObjectType_Function &&
referenceStatus.guessState != GuessState_Resolved)
{
// A known Cakelisp function call
for (int i = 0; i < (int)referenceStatus.references.size(); ++i)
{
ObjectReference& reference = referenceStatus.references[i];
// Run function invocation on it
// TODO: Make invocation generator know it is a Cakelisp function
bool result = FunctionInvocationGenerator(
environment, reference.context, *reference.tokens,
reference.startIndex, *reference.spliceOutput);
// Our guess didn't even evaluate
if (!result)
canBuild = false;
}

referenceStatus.guessState = GuessState_Resolved;
}
// TODO: Building references to non-comptime functions at comptime
}
else
{
if (referenceStatus.guessState == GuessState_None)
{
if (log.buildReasons)
printf("\tCannot build until %s is guessed. Guessing now\n",
referenceStatus.name->contents.c_str());

// Find all the times the definition makes this reference
// We must use indices because the call to FunctionInvocationGenerator can
// add new references to the list
for (int i = 0; i < (int)referenceStatus.references.size(); ++i)
{
ObjectReference& reference = referenceStatus.references[i];
// Run function invocation on it
bool result = FunctionInvocationGenerator(
environment, reference.context, *reference.tokens,
reference.startIndex, *reference.spliceOutput);
// Our guess didn't even evaluate
if (!result)
canBuild = false;
}

referenceStatus.guessState = GuessState_Guessed;
hasRelevantChangeOccurred = true;
hasGuessedRefs = true;
guessMaybeDirtiedReferences = true;
}
else if (referenceStatus.guessState == GuessState_Guessed)
{
// It has been guessed, and still isn't in definitions
hasGuessedRefs = true;
}
}
}
} while (guessMaybeDirtiedReferences);

// hasRelevantChangeOccurred being false suppresses rebuilding compile-time functions which
// still have the same missing references. Note that only compile time objects can be built.
// We put normal functions through the guessing system too because they need their functions
// resolved as well. It's a bit dirty but not too bad
if (canBuild && (!hasGuessedRefs || hasRelevantChangeOccurred) &&
isCompileTimeObject(definition.type))
{
BuildObject objectToBuild = {};
objectToBuild.buildId = getNextFreeBuildId(environment);
objectToBuild.definition = &definition;
definitionsToBuild.push_back(objectToBuild);
}
}

numReferencesResolved +=
BuildExecuteCompileTimeFunctions(environment, definitionsToBuild, numErrorsOut);

return numReferencesResolved;
}

bool EvaluateResolveReferences(EvaluatorEnvironment& environment)
{
// Print state


+ 19
- 0
src/Evaluator.hpp View File

@@ -11,6 +11,7 @@ struct NameStyleSettings;
struct Token;
struct GeneratorOutput;
struct ModuleManager;
struct Module;

// Rather than needing to allocate and edit a buffer eventually equal to the size of the final
// output, store output operations instead. This also facilitates source <-> generated mapping data
@@ -82,6 +83,7 @@ struct EvaluatorContext
bool isRequired;
// Associate all unknown references with this definition
const Token* definitionName;
Module* module;
};

struct EvaluatorEnvironment;
@@ -167,6 +169,18 @@ typedef std::pair<const std::string, ObjectDefinition> ObjectDefinitionPair;
typedef std::unordered_map<std::string, ObjectReferencePool> ObjectReferencePoolMap;
typedef std::pair<const std::string, ObjectReferencePool> ObjectReferencePoolPair;

struct ProcessCommandArgument
{
ProcessCommandArgumentType type;
std::string contents;
};

struct ProcessCommand
{
std::string fileToExecute;
std::vector<ProcessCommandArgument> arguments;
};

// Unlike context, which can't be changed, environment can be changed.
// Use care when modifying the environment. Only add things once you know things have succeeded.
// Keep in mind that calling functions which can change the environment may invalidate your pointers
@@ -200,6 +214,11 @@ struct EvaluatorEnvironment
// Added as a search directory for compile time code execution
std::string cakelispSrcDir;

ProcessCommand compileTimeBuildCommand;
ProcessCommand compileTimeLinkCommand;
ProcessCommand buildTimeBuildCommand;
ProcessCommand buildTimeLinkCommand;

// Will NOT clean up macroExpansions! Use environmentDestroyInvalidateTokens()
~EvaluatorEnvironment();
};


+ 12
- 0
src/EvaluatorEnums.hpp View File

@@ -71,3 +71,15 @@ enum ObjectReferenceGuessState
GuessState_WaitingForLoad,
GuessState_Resolved,
};

enum ProcessCommandArgumentType
{
ProcessCommandArgumentType_String,

ProcessCommandArgumentType_SourceInput,
ProcessCommandArgumentType_ObjectOutput,
ProcessCommandArgumentType_CakelispHeadersInclude,

ProcessCommandArgumentType_ObjectInput,
ProcessCommandArgumentType_DynamicLibraryOutput,
};

+ 126
- 1
src/Generators.cpp View File

@@ -9,6 +9,87 @@

#include <string.h>

typedef bool (*ProcessCommandOptionFunc)(EvaluatorEnvironment& environment,
const std::vector<Token>& tokens, int startTokenIndex,
ProcessCommand* command);

bool SetProcessCommandFileToExec(EvaluatorEnvironment& environment,
const std::vector<Token>& tokens, int startTokenIndex,
ProcessCommand* command)
{
int endInvocationIndex = FindCloseParenTokenIndex(tokens, startTokenIndex);
int argumentIndex = getExpectedArgument("expected path to compiler", tokens, startTokenIndex, 2,
endInvocationIndex);
if (argumentIndex == -1)
return false;

const Token& argumentToken = tokens[argumentIndex];

if (!ExpectTokenType("file to execute", argumentToken, TokenType_String))
return false;

command->fileToExecute = argumentToken.contents;
return true;
}

bool SetProcessCommandArguments(EvaluatorEnvironment& environment, const std::vector<Token>& tokens,
int startTokenIndex, ProcessCommand* command)
{
command->arguments.clear();

int endInvocationIndex = FindCloseParenTokenIndex(tokens, startTokenIndex);
int startArgsIndex = getArgument(tokens, startTokenIndex, 2, endInvocationIndex);
// No args is weird, but we'll allow it
if (startArgsIndex == -1)
return true;

for (int argumentIndex = startArgsIndex; argumentIndex < endInvocationIndex; ++argumentIndex)
{
const Token& argumentToken = tokens[argumentIndex];
if (argumentToken.type == TokenType_String)
{
command->arguments.push_back(
{ProcessCommandArgumentType_String, argumentToken.contents});
}
else if (argumentToken.type == TokenType_Symbol)
{
struct
{
const char* symbolName;
ProcessCommandArgumentType type;
} symbolsToCommandTypes[] = {
{"'source-input", ProcessCommandArgumentType_SourceInput},
{"'object-output", ProcessCommandArgumentType_ObjectOutput},
{"'cakelisp-headers-include", ProcessCommandArgumentType_CakelispHeadersInclude},
{"'object-input", ProcessCommandArgumentType_ObjectInput},
{"'library-output", ProcessCommandArgumentType_DynamicLibraryOutput},
};
bool found = false;
for (unsigned int i = 0; i < ArraySize(symbolsToCommandTypes); ++i)
{
if (argumentToken.contents.compare(symbolsToCommandTypes[i].symbolName) == 0)
{
command->arguments.push_back({symbolsToCommandTypes[i].type, EmptyString});
found = true;
break;
}
}
if (!found)
{
ErrorAtToken(argumentToken, "unrecognized symbol");
return false;
}
}
else
{
ErrorAtTokenf(argumentToken, "expected string argument or symbol, got %s",
tokenTypeToString(argumentToken.type));
return false;
}
}
return true;
}

bool SetCakelispOption(EvaluatorEnvironment& environment, const EvaluatorContext& context,
const std::vector<Token>& tokens, int startTokenIndex,
GeneratorOutput& output)
@@ -47,7 +128,7 @@ bool SetCakelispOption(EvaluatorEnvironment& environment, const EvaluatorContext
}

// This needs to be defined early, else things will only be partially supported
if (tokens[optionNameIndex].contents.compare("enable-hot-reloading") == 0)
else if (tokens[optionNameIndex].contents.compare("enable-hot-reloading") == 0)
{
int enableStateIndex =
getExpectedArgument("expected path", tokens, startTokenIndex, 2, endInvocationIndex);
@@ -72,6 +153,37 @@ bool SetCakelispOption(EvaluatorEnvironment& environment, const EvaluatorContext
return true;
}

struct ProcessCommandOptions
{
const char* optionName;
ProcessCommand* command;
ProcessCommandOptionFunc handler;
};
ProcessCommandOptions commandOptions[] = {
{"compile-time-compiler", &environment.compileTimeBuildCommand,
SetProcessCommandFileToExec},
{"compile-time-compile-arguments", &environment.compileTimeBuildCommand,
SetProcessCommandArguments},
{"compile-time-linker", &environment.compileTimeLinkCommand, SetProcessCommandFileToExec},
{"compile-time-link-arguments", &environment.compileTimeLinkCommand,
SetProcessCommandArguments},
{"build-time-compiler", &environment.buildTimeBuildCommand, SetProcessCommandFileToExec},
{"build-time-compile-arguments", &environment.buildTimeBuildCommand,
SetProcessCommandArguments},
{"build-time-linker", &environment.buildTimeLinkCommand, SetProcessCommandFileToExec},
{"build-time-link-arguments", &environment.buildTimeLinkCommand,
SetProcessCommandArguments},
};

for (unsigned int i = 0; i < ArraySize(commandOptions); ++i)
{
if (tokens[optionNameIndex].contents.compare(commandOptions[i].optionName) == 0)
{
return commandOptions[i].handler(environment, tokens, startTokenIndex,
commandOptions[i].command);
}
}

ErrorAtToken(tokens[optionNameIndex], "unrecognized option");
return false;
}
@@ -144,6 +256,19 @@ bool ImportGenerator(EvaluatorEnvironment& environment, const EvaluatorContext&
}
else
{
if (context.module)
{
ModuleDependency newCakelispDependency = {};
newCakelispDependency.type = ModuleDependency_Cakelisp;
newCakelispDependency.name = currentToken.contents;
context.module->dependencies.push_back(newCakelispDependency);
}
else
{
NoteAtToken(currentToken,
"module cannot track dependency (potential internal error)");
}

char relativePathBuffer[MAX_PATH_LENGTH] = {0};
getDirectoryFromPath(currentToken.source, relativePathBuffer,
sizeof(relativePathBuffer));


+ 6
- 0
src/Main.cpp View File

@@ -148,6 +148,12 @@ int main(int numArguments, char* arguments[])

printf("Successfully generated files\n");

if (!moduleManagerBuild(moduleManager))
{
moduleManagerDestroy(moduleManager);
return 1;
}

moduleManagerDestroy(moduleManager);
return 0;
}

+ 84
- 17
src/ModuleManager.cpp View File

@@ -3,6 +3,7 @@
#include "Converters.hpp"
#include "DynamicLoader.hpp"
#include "Evaluator.hpp"
#include "EvaluatorEnums.hpp"
#include "FileUtilities.hpp"
#include "Generators.hpp"
#include "Logging.hpp"
@@ -37,6 +38,42 @@ void moduleManagerInitialize(ModuleManager& manager)

manager.environment.moduleManager = &manager;

// Command defaults
{
manager.environment.compileTimeBuildCommand.fileToExecute = "/usr/bin/clang++";
manager.environment.compileTimeBuildCommand.arguments = {
{ProcessCommandArgumentType_String, "-g"},
{ProcessCommandArgumentType_String, "-c"},
{ProcessCommandArgumentType_SourceInput, EmptyString},
{ProcessCommandArgumentType_String, "-o"},
{ProcessCommandArgumentType_ObjectOutput, EmptyString},
{ProcessCommandArgumentType_CakelispHeadersInclude, EmptyString},
{ProcessCommandArgumentType_String, "-fPIC"}};

manager.environment.compileTimeLinkCommand.fileToExecute = "/usr/bin/clang++";
manager.environment.compileTimeLinkCommand.arguments = {
{ProcessCommandArgumentType_String, "-shared"},
{ProcessCommandArgumentType_String, "-o"},
{ProcessCommandArgumentType_DynamicLibraryOutput, EmptyString},
{ProcessCommandArgumentType_ObjectInput, EmptyString}};

manager.environment.buildTimeBuildCommand.fileToExecute = "/usr/bin/clang++";
manager.environment.buildTimeBuildCommand.arguments = {
{ProcessCommandArgumentType_String, "-g"},
{ProcessCommandArgumentType_String, "-c"},
{ProcessCommandArgumentType_SourceInput, EmptyString},
{ProcessCommandArgumentType_String, "-o"},
{ProcessCommandArgumentType_ObjectOutput, EmptyString},
{ProcessCommandArgumentType_String, "-fPIC"}};

manager.environment.buildTimeLinkCommand.fileToExecute = "/usr/bin/clang++";
}

// TODO: Add defaults for Windows
#ifdef WINDOWS
#error Set sensible defaults for compile time build command
#endif

makeDirectory(cakelispWorkingDir);
printf("Using cache at %s\n", cakelispWorkingDir);
}
@@ -44,11 +81,12 @@ void moduleManagerInitialize(ModuleManager& manager)
void moduleManagerDestroy(ModuleManager& manager)
{
environmentDestroyInvalidateTokens(manager.environment);
for (Module& module : manager.modules)
for (Module* module : manager.modules)
{
delete module.tokens;
delete module.generatedOutput;
free((void*)module.filename);
delete module->tokens;
delete module->generatedOutput;
free((void*)module->filename);
delete module;
}
manager.modules.clear();
closeAllDynamicLibraries();
@@ -145,27 +183,28 @@ bool moduleLoadTokenizeValidate(const char* filename, const std::vector<Token>**

bool moduleManagerAddEvaluateFile(ModuleManager& manager, const char* filename)
{
for (Module& module : manager.modules)
for (Module* module : manager.modules)
{
if (strcmp(module.filename, filename) == 0)
if (strcmp(module->filename, filename) == 0)
{
printf("Already loaded %s\n", filename);
return true;
}
}

Module newModule = {};
Module* newModule = new Module();
// We need to keep this memory around for the lifetime of the token, regardless of relocation
newModule.filename = strdup(filename);
newModule->filename = strdup(filename);
// This stage cleans up after itself if it fails
if (!moduleLoadTokenizeValidate(newModule.filename, &newModule.tokens))
if (!moduleLoadTokenizeValidate(newModule->filename, &newModule->tokens))
return false;

newModule.generatedOutput = new GeneratorOutput;
newModule->generatedOutput = new GeneratorOutput;

manager.modules.push_back(newModule);

EvaluatorContext moduleContext = {};
moduleContext.module = newModule;
moduleContext.scope = EvaluatorScope_Module;
moduleContext.definitionName = &manager.globalPseudoInvocationName;
// Module always requires all its functions
@@ -175,14 +214,14 @@ bool moduleManagerAddEvaluateFile(ModuleManager& manager, const char* filename)
StringOutput bodyDelimiterTemplate = {};
bodyDelimiterTemplate.modifiers = StringOutMod_NewlineAfter;
int numErrors = EvaluateGenerateAll_Recursive(
manager.environment, moduleContext, *newModule.tokens,
/*startTokenIndex=*/0, &bodyDelimiterTemplate, *newModule.generatedOutput);
manager.environment, moduleContext, *newModule->tokens,
/*startTokenIndex=*/0, &bodyDelimiterTemplate, *newModule->generatedOutput);
// After this point, the module may have references to its tokens in the environmment, so we
// cannot destroy it until we're done evaluating everything
if (numErrors)
return false;

printf("Loaded %s\n", newModule.filename);
printf("Loaded %s\n", newModule->filename);
return true;
}

@@ -196,14 +235,14 @@ bool moduleManagerWriteGeneratedOutput(ModuleManager& manager)
NameStyleSettings nameSettings;
WriterFormatSettings formatSettings;

for (Module& module : manager.modules)
for (Module* module : manager.modules)
{
WriterOutputSettings outputSettings;
outputSettings.sourceCakelispFilename = module.filename;
outputSettings.sourceCakelispFilename = module->filename;

// TODO: hpp to h support
char relativeIncludeBuffer[MAX_PATH_LENGTH];
getFilenameFromPath(module.filename, relativeIncludeBuffer, sizeof(relativeIncludeBuffer));
getFilenameFromPath(module->filename, relativeIncludeBuffer, sizeof(relativeIncludeBuffer));
char sourceHeadingBuffer[1024] = {0};
PrintfBuffer(sourceHeadingBuffer, "#include \"%s.hpp\"\n%s", relativeIncludeBuffer,
generatedSourceHeading ? generatedSourceHeading : "");
@@ -212,10 +251,38 @@ bool moduleManagerWriteGeneratedOutput(ModuleManager& manager)
outputSettings.headerHeading = generatedHeaderHeading;
outputSettings.headerFooter = generatedHeaderFooter;

if (!writeGeneratorOutput(*module.generatedOutput, nameSettings, formatSettings,
char sourceOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(sourceOutputName, "%s.cpp", outputSettings.sourceCakelispFilename);
char headerOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(headerOutputName, "%s.hpp", outputSettings.sourceCakelispFilename);
module->sourceOutputName = sourceOutputName;
module->headerOutputName = headerOutputName;
outputSettings.sourceOutputName = module->sourceOutputName.c_str();
outputSettings.headerOutputName = module->headerOutputName.c_str();

if (!writeGeneratorOutput(*module->generatedOutput, nameSettings, formatSettings,
outputSettings))
return false;
}

return true;
}

bool moduleManagerBuild(ModuleManager& manager)
{
for (Module* module : manager.modules)
{
printf("Build module %s\n", module->sourceOutputName.c_str());
for (ModuleDependency& dependency : module->dependencies)
{
printf("\tRequires %s\n", dependency.name.c_str());

// Cakelisp files are built at the module manager level, so we need not concern
// ourselves with them
if (dependency.type == ModuleDependency_Cakelisp)
continue;
}
}

return true;
}

+ 19
- 1
src/ModuleManager.hpp View File

@@ -5,12 +5,28 @@
#include "Tokenizer.hpp"
#include "Evaluator.hpp"

enum ModuleDependencyType
{
ModuleDependency_Cakelisp,
ModuleDependency_Library,
ModuleDependency_CFile
};

struct ModuleDependency
{
ModuleDependencyType type;
std::string name;
};

// A module is typically associated with a single file. Keywords like local mean in-module only
struct Module
{
const char* filename;
const std::vector<Token>* tokens;
GeneratorOutput* generatedOutput;
std::string sourceOutputName;
std::string headerOutputName;
std::vector<ModuleDependency> dependencies;
};

struct ModuleManager
@@ -18,7 +34,8 @@ struct ModuleManager
// Shared environment across all modules
EvaluatorEnvironment environment;
Token globalPseudoInvocationName;
std::vector<Module> modules;
// Pointer only so things cannot move around
std::vector<Module*> modules;
};

void moduleManagerInitialize(ModuleManager& manager);
@@ -28,3 +45,4 @@ bool moduleLoadTokenizeValidate(const char* filename, const std::vector<Token>**
bool moduleManagerAddEvaluateFile(ModuleManager& manager, const char* filename);
bool moduleManagerEvaluateResolveReferences(ModuleManager& manager);
bool moduleManagerWriteGeneratedOutput(ModuleManager& manager);
bool moduleManagerBuild(ModuleManager& manager);

+ 13
- 1
src/RunProcess.cpp View File

@@ -26,6 +26,7 @@ struct Subprocess
int* statusOut;
ProcessId processId;
int pipeReadFileDescriptor;
std::string command;
};

static std::vector<Subprocess> s_subprocesses;
@@ -131,7 +132,14 @@ int runProcess(const RunProcessArguments& arguments, int* statusOut)
if (log.processes)
printf("Created child process %d\n", pid);

s_subprocesses.push_back({statusOut, pid, pipeFileDescriptors[PipeRead]});
std::string command = "";
for (const char** arg = arguments.arguments; *arg != nullptr; ++arg)
{
command.append(*arg);
command.append(" ");
}

s_subprocesses.push_back({statusOut, pid, pipeFileDescriptors[PipeRead], command});
}

return 1;
@@ -162,6 +170,10 @@ void waitForAllProcessesClosed(SubprocessOnOutputFunc onOutput)
close(process.pipeReadFileDescriptor);

waitpid(process.processId, process.statusOut, 0);

// It's pretty useful to see the command which resulted in failure
if (*process.statusOut != 0)
printf("%s\n", process.command.c_str());
#endif
}



+ 2
- 7
src/Writer.cpp View File

@@ -371,11 +371,6 @@ void writeOutputFollowSplices_Recursive(const NameStyleSettings& nameSettings,
bool writeOutputs(const NameStyleSettings& nameSettings, const WriterFormatSettings& formatSettings,
const WriterOutputSettings& outputSettings, const GeneratorOutput& outputToWrite)
{
char sourceOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(sourceOutputName, "%s.cpp", outputSettings.sourceCakelispFilename);
char headerOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(headerOutputName, "%s.hpp", outputSettings.sourceCakelispFilename);

struct
{
bool isHeader;
@@ -388,7 +383,7 @@ bool writeOutputs(const NameStyleSettings& nameSettings, const WriterFormatSetti
StringOutputState stateBeforeOutputWrite;
char tempFilename[MAX_PATH_LENGTH];
} outputs[] = {{/*isHeader=*/false,
sourceOutputName,
outputSettings.sourceOutputName,
outputSettings.sourceHeading,
outputSettings.sourceFooter,
{},
@@ -396,7 +391,7 @@ bool writeOutputs(const NameStyleSettings& nameSettings, const WriterFormatSetti
{0}},
{
/*isHeader=*/true,
headerOutputName,
outputSettings.headerOutputName,
outputSettings.headerHeading,
outputSettings.headerFooter,
{},


+ 4
- 0
src/Writer.hpp View File

@@ -20,6 +20,10 @@ struct WriterFormatSettings
struct WriterOutputSettings
{
const char* sourceCakelispFilename;

const char* sourceOutputName;
const char* headerOutputName;

const char* sourceHeading;
const char* sourceFooter;
const char* headerHeading;


Loading…
Cancel
Save