Browse Source

Most stages of compile time pipeline in

* Compile-time functions are now compiled and linked
* Added preamble feature for generated file boilerplate
* RunProcess is now useful
ModuleSystem
Macoy Madson 10 months ago
parent
commit
8ec2b19a82
12 changed files with 254 additions and 52 deletions
  1. +4
    -1
      .gitignore
  2. +132
    -6
      src/Evaluator.cpp
  3. +9
    -0
      src/Evaluator.hpp
  4. +5
    -3
      src/Generators.cpp
  5. +1
    -0
      src/Jamfile
  6. +26
    -24
      src/Main.cpp
  7. +3
    -0
      src/OutputPreambles.cpp
  8. +3
    -0
      src/OutputPreambles.hpp
  9. +44
    -12
      src/RunProcess.cpp
  10. +5
    -2
      src/RunProcess.hpp
  11. +20
    -4
      src/Writer.cpp
  12. +2
    -0
      src/Writer.hpp

+ 4
- 1
.gitignore View File

@ -33,6 +33,9 @@ TAGS
cakelisp
src/dynamicLoadTest
src/runProcessTest
src/dependencyTest
*.cake.cpp
*.cake.hpp
*.cake.hpp
*CakelispCompileTime*

+ 132
- 6
src/Evaluator.cpp View File

@ -1,11 +1,13 @@
#include "Evaluator.hpp"
#include "Converters.hpp"
#include "Generators.hpp"
#include "GeneratorHelpers.hpp"
#include "Generators.hpp"
#include "RunProcess.hpp"
#include "Tokenizer.hpp"
#include "Utilities.hpp"
#include "Writer.hpp"
#include "OutputPreambles.hpp"
// TODO: safe version of strcat
#include <stdio.h>
@ -90,6 +92,11 @@ void addObjectReference(EvaluatorEnvironment& environment, EvaluatorContext cont
}
}
int getNextFreeBuildId(EvaluatorEnvironment& environment)
{
return ++environment.nextFreeBuildId;
}
//
// Evaluator
//
@ -419,8 +426,7 @@ void PropagateRequiredToReferences(EvaluatorEnvironment& environment)
{
ObjectDefinitionMap::iterator findIt =
environment.definitions.find(referenceStatus.name->contents);
if (findIt != environment.definitions.end() &&
!findIt->second.isRequired)
if (findIt != environment.definitions.end() && !findIt->second.isRequired)
{
printf("\t Infecting %s with required due to %s\n",
referenceStatus.name->contents.c_str(),
@ -435,12 +441,31 @@ void PropagateRequiredToReferences(EvaluatorEnvironment& environment)
} while (numRequiresStatusChanged);
}
void OnCompileProcessOutput(const char* output)
{
}
int BuildEvaluateReferences(EvaluatorEnvironment& environment)
{
enum BuildStage
{
BuildStage_None,
BuildStage_Compiling,
BuildStage_Linking,
BuildStage_Loading
};
// Note: environment.definitions can be resized/rehashed during evaluation, which invalidates
// iterators. For now, I will rely on the fact that std::unordered_map does not invalidate
// references on resize. This will need to change if the data structure changes
std::vector<ObjectDefinition*> definitionsToBuild;
struct BuildObject
{
int buildId;
int status;
BuildStage stage;
ObjectDefinition* definition;
};
std::vector<BuildObject> definitionsToBuild;
for (ObjectDefinitionPair& definitionPair : environment.definitions)
{
ObjectDefinition& definition = definitionPair.second;
@ -485,15 +510,116 @@ int BuildEvaluateReferences(EvaluatorEnvironment& environment)
if (canBuild)
{
definitionsToBuild.push_back(&definition);
definitionsToBuild.push_back(
{getNextFreeBuildId(environment), -1, BuildStage_None, &definition});
}
}
}
for (ObjectDefinition* definition : definitionsToBuild)
// Spin up as many compile processes as necessary
// TODO: Limit number spawned at once?
// TODO: Combine sure-thing builds into batches (ones where we know all references)
// TODO: Instead of creating files, pipe straight to compiler?
// NOTE: definitionsToBuild must not be resized from when runProcess() is called until
// waitForAllProcessesClosed(), else the status pointer could be invalidated
for (BuildObject& buildObject : definitionsToBuild)
{
ObjectDefinition* definition = buildObject.definition;
printf("Build %s\n", definition->name->contents.c_str());
char sourceOutputName[MAX_PATH_LENGTH] = {0};
// Writer will append the appropriate file extension
PrintfBuffer(sourceOutputName, "CakelispCompileTime_%d", buildObject.buildId);
// Output definition to a file our compiler will be happy with
// TODO: Make these come from the top
NameStyleSettings nameSettings;
WriterFormatSettings formatSettings;
WriterOutputSettings outputSettings;
outputSettings.sourcePreamble = macroPreamble;
outputSettings.sourceCakelispFilename = sourceOutputName;
if (!writeGeneratorOutput(*definition->output, nameSettings, formatSettings,
outputSettings))
{
ErrorAtToken(*buildObject.definition->name,
"Failed to write to compile-time source file");
continue;
}
buildObject.stage = BuildStage_Compiling;
char fileToExec[MAX_PATH_LENGTH] = {0};
PrintBuffer(fileToExec, "/usr/bin/clang++");
// TODO: Get file extension from output (once .c vs .cpp is implemented)
PrintfBuffer(sourceOutputName, "CakelispCompileTime_%d.cpp", buildObject.buildId);
// TODO: Get arguments all the way from the top
// If not null terminated, the call will fail
char* arguments[] = {fileToExec, strdup("-c"), sourceOutputName, strdup("-Isrc/"), nullptr};
RunProcessArguments compileArguments = {};
compileArguments.fileToExecute = fileToExec;
compileArguments.arguments = arguments;
if (runProcess(compileArguments, &buildObject.status) != 0)
{
// TODO: Abort building if cannot invoke compiler
// return 0;
}
}
// The result of the builds will go straight to our definitionsToBuild
waitForAllProcessesClosed(OnCompileProcessOutput);
// Linking
for (BuildObject& buildObject : definitionsToBuild)
{
if (buildObject.status != 0)
{
ErrorAtTokenf(*buildObject.definition->name,
"Failed to compile definition with status %d", buildObject.status);
continue;
}
buildObject.stage = BuildStage_Linking;
printf("Compiled %s successfully\n", buildObject.definition->name->contents.c_str());
char linkerExecutable[MAX_PATH_LENGTH] = {0};
PrintBuffer(linkerExecutable, "/usr/bin/clang++");
// TODO Store this on the build object
char buildObjectName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(buildObjectName, "CakelispCompileTime_%d.o", buildObject.buildId);
char dynamicLibraryOut[MAX_PATH_LENGTH] = {0};
PrintfBuffer(dynamicLibraryOut, "libCakelispCompileTime_%d.so", buildObject.buildId);
char* arguments[] = {linkerExecutable, strdup("-shared"), strdup("-o"),
dynamicLibraryOut, buildObjectName, nullptr};
RunProcessArguments linkArguments = {};
linkArguments.fileToExecute = linkerExecutable;
linkArguments.arguments = arguments;
if (runProcess(linkArguments, &buildObject.status) != 0)
{
// TODO: Abort if linker failed?
}
}
// The result of the linking will go straight to our definitionsToBuild
waitForAllProcessesClosed(OnCompileProcessOutput);
for (BuildObject& buildObject : definitionsToBuild)
{
if (buildObject.stage == BuildStage_Linking && buildObject.status != 0)
{
ErrorAtToken(*buildObject.definition->name, "Failed to link definition");
continue;
}
printf("Linked %s successfully\n", buildObject.definition->name->contents.c_str());
buildObject.stage = BuildStage_Loading;
}
return 0;
}


+ 9
- 0
src/Evaluator.hpp View File

@ -142,6 +142,12 @@ struct ObjectDefinition
// order for the objects to be built. Required-ness spreads from the top level module scope
bool isRequired;
ObjectReferenceStatusMap references;
// Used only for compile-time functions
// Note that these don't need headers or metadata because they are found via dynamic linking.
// GeneratorOutput is (somewhat wastefully) used in order to make the API consistent for
// compile-time vs. runtime code generation
GeneratorOutput* output;
};
struct ObjectReference
@ -187,6 +193,9 @@ struct EvaluatorEnvironment
ObjectDefinitionMap definitions;
ObjectReferenceMap references;
// Used to ensure unique filenames for compile-time artifacts
int nextFreeBuildId;
// Will NOT clean up macroExpansions! Use environmentDestroyInvalidateTokens()
~EvaluatorEnvironment();
};


+ 5
- 3
src/Generators.cpp View File

@ -863,18 +863,20 @@ bool DefMacroGenerator(EvaluatorEnvironment& environment, const EvaluatorContext
if (!ExpectTokenType("defmacro", argsStart, TokenType_OpenParen))
return false;
// Will be cleaned up when the environment is destroyed
GeneratorOutput* compTimeOutput = new GeneratorOutput;
ObjectDefinition newMacroDef = {};
newMacroDef.name= &nameToken;
newMacroDef.name = &nameToken;
newMacroDef.type = ObjectType_CompileTimeFunction;
// Let the reference required propagation step handle this
// TODO: This can also just be a quick lookup to see whether the reference already exists
newMacroDef.isRequired = false;
newMacroDef.output = compTimeOutput;
if (!addObjectDefinition(environment, newMacroDef))
return false;
CompileTimeFunctionDefiniton newFunction = {};
// Will be cleaned up when the environment is destroyed
GeneratorOutput* compTimeOutput = new GeneratorOutput;
newFunction.output = compTimeOutput;
newFunction.startInvocation = &tokens[startTokenIndex];
newFunction.name = &nameToken;


+ 1
- 0
src/Jamfile View File

@ -7,6 +7,7 @@ Writer.cpp
Generators.cpp
GeneratorHelpers.cpp
RunProcess.cpp
OutputPreambles.cpp
;
MakeLocate cakelisp : ../cakelisp ;


+ 26
- 24
src/Main.cpp View File

@ -111,33 +111,35 @@ int main(int argc, char* argv[])
fclose(file);
// TODO Move
if (false)
{
char sourceOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(sourceOutputName, "%s.cpp", filename);
char fileToExec[MAX_PATH_LENGTH] = {0};
PrintBuffer(fileToExec, "/usr/bin/clang++");
// PrintBuffer(arguments.fileToExecute, "/usr/bin/ls");
// char arg0[64] = {0};
// PrintBuffer(arg0, "--version");
// If not null terminated, the call will fail
// char* arguments[] = {arguments.fileToExecute, strdup("--version"), nullptr};
char* arguments[] = {fileToExec, strdup("-c"), sourceOutputName, nullptr};
CompilationArguments compilationArguments = {};
compilationArguments.fileToExecute = fileToExec;
compilationArguments.arguments = arguments;
if (compileFile(compilationArguments) != 0)
{
delete tokens;
return 1;
}
}
// if (false)
// {
// char sourceOutputName[MAX_PATH_LENGTH] = {0};
// PrintfBuffer(sourceOutputName, "%s.cpp", filename);
// char fileToExec[MAX_PATH_LENGTH] = {0};
// PrintBuffer(fileToExec, "/usr/bin/clang++");
// // PrintBuffer(arguments.fileToExecute, "/usr/bin/ls");
// // char arg0[64] = {0};
// // PrintBuffer(arg0, "--version");
// // If not null terminated, the call will fail
// // char* arguments[] = {arguments.fileToExecute, strdup("--version"), nullptr};
// char* arguments[] = {fileToExec, strdup("-c"), sourceOutputName, nullptr};
// CompilationArguments compilationArguments = {};
// compilationArguments.fileToExecute = fileToExec;
// compilationArguments.arguments = arguments;
// int status = -1;
// if (compileFile(compilationArguments, &status) != 0)
// {
// delete tokens;
// return 1;
// }
// // wa
// }
printf("\nParsing and code generation:\n");
EvaluatorEnvironment environment;
EvaluatorEnvironment environment = {};
importFundamentalGenerators(environment);
// TODO Remove test macro
environment.macros["square"] = SquareMacro;


+ 3
- 0
src/OutputPreambles.cpp View File

@ -0,0 +1,3 @@
#include "OutputPreambles.hpp"
const char* macroPreamble = "#include \"Evaluator.hpp\"\n";

+ 3
- 0
src/OutputPreambles.hpp View File

@ -0,0 +1,3 @@
#pragma once
extern const char* macroPreamble;

+ 44
- 12
src/RunProcess.cpp View File

@ -1,18 +1,33 @@
#include "RunProcess.hpp"
#include <stdio.h>
#include <vector>
#ifdef UNIX
#include <string.h>
#include <sys/types.h> // pid
#include <sys/wait.h> // waitpid
#include <unistd.h> // exec, fork
#include <string.h>
#endif
#include "Utilities.hpp"
#ifdef UNIX
typedef pid_t ProcessId;
#else
typedef int ProcessId;
#endif
struct Subprocess
{
int* statusOut;
ProcessId processId;
};
static std::vector<Subprocess> s_subprocesses;
// Never returns, if success
void systemExecuteCompile(const CompilationArguments& arguments)
void systemExecute(const RunProcessArguments& arguments)
{
#ifdef UNIX
// pid_t pid;
@ -33,7 +48,7 @@ void subprocessReceiveStdOut(const char* processOutputBuffer)
// printf("%s", processOutputBuffer);
// }
int compileFile(const CompilationArguments& arguments)
int runProcess(const RunProcessArguments& arguments, int* statusOut)
{
#ifdef UNIX
printf("Compiling file with command:\n");
@ -69,7 +84,7 @@ int compileFile(const CompilationArguments& arguments)
}
// Only write
close(pipeFileDescriptors[PipeRead]);
systemExecuteCompile(arguments);
systemExecute(arguments);
// A failed child should not flush parent files
_exit(EXIT_FAILURE); /* */
}
@ -84,17 +99,34 @@ int compileFile(const CompilationArguments& arguments)
// Only read
close(pipeFileDescriptors[PipeWrite]);
printf("Created child process %d\n", pid);
char processOutputBuffer[1024];
while (fgets(processOutputBuffer, sizeof(processOutputBuffer), stdin))
{
subprocessReceiveStdOut(processOutputBuffer);
}
int status;
waitpid(pid, &status, 0);
return status;
s_subprocesses.push_back({statusOut, pid});
}
return 1;
#endif
return 0;
}
void waitForAllProcessesClosed(SubprocessOnOutputFunc onOutput)
{
if (s_subprocesses.empty())
{
printf("No subprocesses to wait for\n");
return;
}
// TODO: Don't merge all subprocesses to stdin, keep them separate to prevent multi-line errors
// from being split
char processOutputBuffer[1024];
while (fgets(processOutputBuffer, sizeof(processOutputBuffer), stdin))
{
subprocessReceiveStdOut(processOutputBuffer);
onOutput(processOutputBuffer);
}
for (Subprocess& process : s_subprocesses)
{
waitpid(process.processId, process.statusOut, 0);
}
s_subprocesses.clear();
}

+ 5
- 2
src/RunProcess.hpp View File

@ -1,9 +1,12 @@
#pragma once
struct CompilationArguments
struct RunProcessArguments
{
const char* fileToExecute;
char** arguments;
};
int compileFile(const CompilationArguments& arguments);
typedef void (*SubprocessOnOutputFunc)(const char* subprocessOutput);
int runProcess(const RunProcessArguments& arguments, int* statusOut);
void waitForAllProcessesClosed(SubprocessOnOutputFunc onOutput);

+ 20
- 4
src/Writer.cpp View File

@ -107,7 +107,7 @@ static void printIndentation(const WriterFormatSettings& formatSettings, StringO
}
}
static void printStringOutput(const NameStyleSettings& nameSettings,
static void writeStringOutput(const NameStyleSettings& nameSettings,
const WriterFormatSettings& formatSettings,
const StringOutput& outputOperation, StringOutputState& state)
{
@ -242,6 +242,7 @@ bool moveFile(const char* srcFilename, const char* destFilename)
bool writeIfContentsNewer(const NameStyleSettings& nameSettings,
const WriterFormatSettings& formatSettings,
const WriterOutputSettings& outputSettings,
const char* preamble,
const std::vector<StringOutput>& outputOperations,
const char* outputFilename)
{
@ -249,7 +250,20 @@ bool writeIfContentsNewer(const NameStyleSettings& nameSettings,
StringOutputState outputState = {};
char tempFilename[MAX_PATH_LENGTH] = {0};
PrintfBuffer(tempFilename, "%s.temp", outputFilename);
// TODO: If this fails to open, Writer_Writef just won't write to the file, it'll print
outputState.fileOut = fileOpen(tempFilename);
if (preamble)
{
Writer_Writef(outputState, preamble);
// TODO: Put this in Writer_Writef
for (const char* c = preamble; *c != '\0'; ++c)
{
if (*c == '\n')
++outputState.currentLine;
}
}
for (const StringOutput& operation : outputOperations)
{
// Debug print mapping
@ -259,7 +273,7 @@ bool writeIfContentsNewer(const NameStyleSettings& nameSettings,
outputState.currentLine + 1);
}
printStringOutput(nameSettings, formatSettings, operation, outputState);
writeStringOutput(nameSettings, formatSettings, operation, outputState);
}
if (outputState.fileOut)
{
@ -336,7 +350,8 @@ bool writeGeneratorOutput(const GeneratorOutput& generatedOutput,
char sourceOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(sourceOutputName, "%s.cpp", outputSettings.sourceCakelispFilename);
if (!writeIfContentsNewer(nameSettings, formatSettings, outputSettings,
generatedOutput.source, sourceOutputName))
outputSettings.sourcePreamble, generatedOutput.source,
sourceOutputName))
return false;
}
@ -345,7 +360,8 @@ bool writeGeneratorOutput(const GeneratorOutput& generatedOutput,
char headerOutputName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(headerOutputName, "%s.hpp", outputSettings.sourceCakelispFilename);
if (!writeIfContentsNewer(nameSettings, formatSettings, outputSettings,
generatedOutput.header, headerOutputName))
outputSettings.headerPreamble, generatedOutput.header,
headerOutputName))
return false;
}


+ 2
- 0
src/Writer.hpp View File

@ -20,6 +20,8 @@ struct WriterFormatSettings
struct WriterOutputSettings
{
const char* sourceCakelispFilename;
const char* sourcePreamble;
const char* headerPreamble;
};
const char* importLanguageToString(ImportLanguage type);


Loading…
Cancel
Save