Browse Source

Fixed Windows bugs due to comptime, cache file

* My compile-time functions change broke Windows because Windows
requires you link the import library in order to call functions in
other DLLs. I added ImportLibraryPath and ImportLibraries as link
inputs. These may be merged with the other libraries inputs, once I
determine how best to do that (shouldn't be too hard)
* Fixed bug where additional references were being created to
compile-time functions for no reason, and causing crashes. Context now
holds which reference is being resolved
* Relocate .pdb files to facilitate parallel compilation. I was
encountering an error with multiple processes writing to the same PDB,
vc140.pdb. Now, each object will explicitly specify its pdb location
* Delete pdbs before building. This only needs to happen when
  /DEBUG:FASTLINK is specified
* Moved Build argument converter into Build.cpp for shared use. This
may be how I solve the very painful argument conversion stuff. Much
cleanup is needed to make this sustainable
* Do not write Cache file if nothing to write. This prevents empty
file error when tokenizing cache
* Added CAKELISP_API to functions I was using in tests
* Use Build_RunTests.bat for a more exhaustive testing
* Changed how compile-time functions are declared in order to
facilitate DLL importing/exporting. I'm not certain the C linkage is
necessary yet
* Improved empty file error to print file name
* test/BuildHelpers.cake runs Cakelisp itself as a test executable, to
make it easier to test cross-platform
precompiled-headers
Macoy Madson 4 months ago
parent
commit
89117825d9
16 changed files with 399 additions and 130 deletions
  1. +1
    -1
      Build.bat
  2. +84
    -0
      Build_RunTests.bat
  3. +2
    -0
      runtime/Config_Windows.cake
  4. +73
    -3
      src/Build.cpp
  5. +22
    -2
      src/Build.hpp
  6. +85
    -20
      src/Evaluator.cpp
  7. +25
    -13
      src/Evaluator.hpp
  8. +10
    -9
      src/GeneratorHelpers.hpp
  9. +20
    -12
      src/Generators.cpp
  10. +36
    -49
      src/ModuleManager.cpp
  11. +7
    -1
      src/RunProcess.cpp
  12. +6
    -5
      src/RunProcess.hpp
  13. +3
    -0
      src/RunProcessEnums.hpp
  14. +5
    -4
      src/Tokenizer.hpp
  15. +11
    -9
      test/BuildHelpers.cake
  16. +9
    -2
      test/RunTests.cake

+ 1
- 1
Build.bat View File

@ -67,7 +67,7 @@ CL.exe src/Tokenizer.cpp ^
:fail
goto end
:success
:success
goto end
:end


+ 84
- 0
Build_RunTests.bat View File

@ -0,0 +1,84 @@
echo off
rem Set environment variables. The user may need to adjust this path
rem See https://docs.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-160#developer_command_file_locations
if exist "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" (
call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
) else (
echo This script builds using MSVC.
echo You must download and install MSVC before it will work. Download it here:
echo https://visualstudio.microsoft.com/downloads/
echo Select workloads for C++ projects. Ensure you install the C++ developer tools.
echo If you're still seeing this, you may need to edit Build.bat to your vcvars path
echo Please see the following link:
echo https://docs.microsoft.com/en-us/cpp/build/building-on-the-command-line?view=msvc-160
goto fail
)
if not exist "bin" (
mkdir bin
)
if not exist "bin\cakelisp_bootstrap.exe" (
goto manualBuild
) else (
goto bootstrapBuild
)
:manualBuild
CL.exe src/Tokenizer.cpp ^
src/Evaluator.cpp ^
src/Utilities.cpp ^
src/FileUtilities.cpp ^
src/Converters.cpp ^
src/Writer.cpp ^
src/Generators.cpp ^
src/GeneratorHelpers.cpp ^
src/RunProcess.cpp ^
src/OutputPreambles.cpp ^
src/DynamicLoader.cpp ^
src/ModuleManager.cpp ^
src/Logging.cpp ^
src/Build.cpp ^
src/Main.cpp ^
/EHsc /MP /DWINDOWS /DCAKELISP_EXPORTING ^
/Fe"bin\cakelisp_bootstrap" /Zi /Fd"bin\cakelisp_bootstrap.pdb" /DEBUG:FASTLINK
echo %ERRORLEVEL%
@if %ERRORLEVEL% == 0 (
echo Success building
rem Clean up working directory
del *.obj
goto bootstrapBuild
) else (
echo Error while building
goto fail
)
:bootstrapBuild
"bin\cakelisp_bootstrap.exe" Bootstrap_MSVC.cake
@if %ERRORLEVEL% == 0 (
echo Success! Use bin\cakelisp.exe to build your programs
goto build_user
) else (
echo Error while bootstrapping cakelisp
goto fail
)
:build_user
"bin\cakelisp.exe" --verbose-processes --execute runtime/Config_Windows.cake test/RunTests.cake
@if %ERRORLEVEL% == 0 (
echo Success!
goto success
) else (
echo Error while building user program
goto fail
)
:fail
goto end
:success
goto end
:end
echo Done

+ 2
- 0
runtime/Config_Windows.cake View File

@ -0,0 +1,2 @@
(skip-build)
(comptime-define-symbol 'Windows)

+ 73
- 3
src/Build.cpp View File

@ -15,12 +15,19 @@
#ifdef WINDOWS
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
// TODO: These should change based on compiler, i.e. you should be able to build things via mingw on
// Windows, using the Unix extensions
const char* compilerObjectExtension = "obj";
const char* linkerDynamicLibraryPrefix = "";
const char* compilerDebugSymbolsExtension = "pdb";
const char* compilerImportLibraryExtension = "lib";
const char* linkerDynamicLibraryPrefix = ""; // Not applicable
const char* linkerDynamicLibraryExtension = "dll";
const char* defaultExecutableName = "output.exe";
#else
const char* compilerObjectExtension = "o";
const char* compilerDebugSymbolsExtension = ""; // Not applicable
const char* compilerImportLibraryExtension = ""; // Not applicable
const char* linkerDynamicLibraryPrefix = "lib";
const char* linkerDynamicLibraryExtension = "so";
const char* defaultExecutableName = "a.out";
@ -46,6 +53,33 @@ void makeObjectOutputArgument(char* buffer, int bufferSize, const char* objectNa
#endif
}
void makeImportLibraryPathArgument(char* buffer, int bufferSize, const char* path,
const char* buildExecutable)
{
if (StrCompareIgnoreCase(buildExecutable, "link.exe") == 0)
{
SafeSnprintf(buffer, bufferSize, "/LIBPATH:%s", path);
}
// else if (StrCompareIgnoreCase(buildExecutable, "link.exe") == 0)
// {
// SafeSnprintf(buffer, bufferSize, "/OUT:\"%s\"", path);
// }
else
{
SafeSnprintf(buffer, bufferSize, "%s", path);
}
}
void makeDebugSymbolsOutputArgument(char* buffer, int bufferSize, const char* debugSymbolsName)
{
// TODO: Make this a setting rather than a define
#ifdef WINDOWS
SafeSnprintf(buffer, bufferSize, "/Fd\"%s\"", debugSymbolsName);
// #else // No repositioning of debug symbols necessary
// SafeSnprintf(buffer, bufferSize, "%s", debugSymbolsName);
#endif
}
void makeDynamicLibraryOutputArgument(char* buffer, int bufferSize, const char* libraryName,
const char* buildExecutable)
{
@ -154,6 +188,38 @@ void makeLinkerArgument(char* buffer, int bufferSize, const char* argument,
}
}
void convertBuildArguments(BuildArgumentConverter* argumentsToConvert, int numArgumentsToConvert,
const char* buildExecutable)
{
for (int typeIndex = 0; typeIndex < numArgumentsToConvert; ++typeIndex)
{
int numStrings = (int)argumentsToConvert[typeIndex].stringsIn->size();
argumentsToConvert[typeIndex].argumentsOutMemory.resize(numStrings);
argumentsToConvert[typeIndex].argumentsOut->resize(numStrings);
int currentString = 0;
for (const std::string& stringIn : *argumentsToConvert[typeIndex].stringsIn)
{
if (argumentsToConvert[typeIndex].argumentConversionFunc)
{
// Give some extra padding for prefixes
char argumentOut[MAX_PATH_LENGTH + 15] = {0};
argumentsToConvert[typeIndex].argumentConversionFunc(
argumentOut, sizeof(argumentOut), stringIn.c_str(), buildExecutable);
argumentsToConvert[typeIndex].argumentsOutMemory[currentString] = argumentOut;
}
else
argumentsToConvert[typeIndex].argumentsOutMemory[currentString] = stringIn;
++currentString;
}
for (int stringIndex = 0; stringIndex < numStrings; ++stringIndex)
(*argumentsToConvert[typeIndex].argumentsOut)[stringIndex] =
argumentsToConvert[typeIndex].argumentsOutMemory[stringIndex].c_str();
}
}
bool resolveExecutablePath(const char* fileToExecute, char* resolvedPathOut,
int resolvedPathOutSize)
{
@ -207,8 +273,6 @@ bool resolveExecutablePath(const char* fileToExecute, char* resolvedPathOut,
if (variablesFound)
{
// SafeSnprintf(resolvedPathOut, resolvedPathOutSize, "%sHost%s/%s/%s", vcInstallDir,
// vcHostArchitecture,
SafeSnprintf(resolvedPathOut, resolvedPathOutSize, "%sbin\\Host%s\\%s\\%s",
vcInstallDir, vcHostArchitecture, vcTargetArchitecture, fileToExecute);
@ -268,6 +332,12 @@ void buildWriteCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCom
outputTokens.push_back(closeParen);
}
if (outputTokens.empty())
{
Log("no tokens to write to cache file");
return;
}
FILE* file = fileOpen(outputFilename, "w");
if (!file)
{


+ 22
- 2
src/Build.hpp View File

@ -4,17 +4,37 @@
#include <string>
#include <vector>
#include "Exporting.hpp"
#include "FileTypes.hpp"
extern const char* compilerObjectExtension;
extern const char* compilerDebugSymbolsExtension;
extern const char* compilerImportLibraryExtension;
extern const char* linkerDynamicLibraryPrefix;
extern const char* linkerDynamicLibraryExtension;
extern const char* defaultExecutableName;
struct BuildArgumentConverter
{
std::vector<std::string>* stringsIn;
// Use C++ to manage our string memory, pointed to by argumentsOut
std::vector<std::string> argumentsOutMemory;
std::vector<const char*>* argumentsOut;
void (*argumentConversionFunc)(char* buffer, int bufferSize, const char* stringIn,
const char* executableName);
};
void convertBuildArguments(BuildArgumentConverter* argumentsToConvert, int numArgumentsToConvert,
const char* buildExecutable);
void makeIncludeArgument(char* buffer, int bufferSize, const char* searchDir);
// On Windows, extra formatting is required to output objects
void makeObjectOutputArgument(char* buffer, int bufferSize, const char* objectName);
void makeDebugSymbolsOutputArgument(char* buffer, int bufferSize, const char* debugSymbolsName);
void makeImportLibraryPathArgument(char* buffer, int bufferSize, const char* path,
const char* buildExecutable);
void makeDynamicLibraryOutputArgument(char* buffer, int bufferSize, const char* libraryName,
const char* buildExecutable);
void makeExecutableOutputArgument(char* buffer, int bufferSize, const char* executableName,
@ -30,8 +50,8 @@ void makeLinkerArgument(char* buffer, int bufferSize, const char* argument,
// On Windows, extra work is done to find the compiler and linker executables. This function handles
// looking up those environment variables to determine which executable to use
bool resolveExecutablePath(const char* fileToExecute, char* resolvedPathOut,
int resolvedPathOutSize);
CAKELISP_API bool resolveExecutablePath(const char* fileToExecute, char* resolvedPathOut,
int resolvedPathOutSize);
typedef std::unordered_map<std::string, FileModifyTime> HeaderModificationTimeTable;


+ 85
- 20
src/Evaluator.cpp View File

@ -383,19 +383,22 @@ bool HandleInvocation_Recursive(EvaluatorEnvironment& environment, const Evaluat
if (findIt->second.type == ObjectType_CompileTimeFunction &&
findCompileTimeFunction(environment, invocationName.contents.c_str()))
{
ObjectReference newReference = {};
newReference.type = ObjectReferenceResolutionType_AlreadyLoaded;
newReference.tokens = &tokens;
newReference.startIndex = invocationStartIndex;
newReference.context = context;
const ObjectReferenceStatus* referenceStatus =
addObjectReference(environment, invocationName, newReference);
if (!referenceStatus)
if (context.resolvingReference != &invocationName)
{
ErrorAtToken(tokens[invocationStartIndex],
"failed to create reference status (internal error)");
return false;
ObjectReference newReference = {};
newReference.type = ObjectReferenceResolutionType_AlreadyLoaded;
newReference.tokens = &tokens;
newReference.startIndex = invocationStartIndex;
newReference.context = context;
const ObjectReferenceStatus* referenceStatus =
addObjectReference(environment, invocationName, newReference);
if (!referenceStatus)
{
ErrorAtToken(tokens[invocationStartIndex],
"failed to create reference status (internal error)");
return false;
}
}
return FunctionInvocationGenerator(environment, context, tokens, invocationStartIndex,
output);
@ -737,6 +740,7 @@ struct ComptimeBuildObject
std::string artifactsName;
std::string dynamicLibraryPath;
std::string buildObjectName;
std::vector<std::string> importLibraries;
ObjectDefinition* definition = nullptr;
};
@ -798,6 +802,11 @@ static int ReevaluateResolveReferences(EvaluatorEnvironment& environment,
NoteAtToken((*referenceValidPreEval->tokens)[referenceValidPreEval->startIndex],
"resolving reference");
// Make sure we don't create additional references for this same reference, as we are
// resolving it
referenceValidPreEval->context.resolvingReference =
&(*referenceValidPreEval->tokens)[referenceValidPreEval->startIndex + 1];
// Evaluate from that reference
int result = EvaluateGenerate_Recursive(
environment, referenceValidPreEval->context, *referenceValidPreEval->tokens,
@ -889,8 +898,8 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
outputSettings.heading = &header;
outputSettings.footer = &footer;
// Automatically include referenced compile-time function headers
bool foundHeaders = true;
// Add referenced compile-time function headers and import libraries
bool foundRequiredComptimeFunctions = true;
for (ObjectReferenceStatusPair& reference : definition->references)
{
ObjectReferenceStatus& referenceStatus = reference.second;
@ -913,7 +922,7 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
ErrorAtToken(*referenceStatus.name,
"could not find generated header for referenced compile-time "
"function. Internal code error?\n");
foundHeaders = false;
foundRequiredComptimeFunctions = false;
continue;
}
@ -923,8 +932,24 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
StringOutMod_SurroundWithQuotes, referenceStatus.name);
addLangTokenOutput(autoIncludes.source, StringOutMod_NewlineAfter,
referenceStatus.name);
if (environment.isMsvcCompiler)
{
if (requiredDefinition->compileTimeImportLibraryName.empty())
{
ErrorAtToken(*referenceStatus.name,
"could not find import library name for referenced compile-time "
"function. Internal code error?\n");
foundRequiredComptimeFunctions = false;
continue;
}
buildObject.importLibraries.push_back(
requiredDefinition->compileTimeImportLibraryName);
}
}
if (!foundHeaders)
// Skip the object: Not all required definitions had headers (will error)
if (!foundRequiredComptimeFunctions)
continue;
outputSettings.sourceCakelispFilename = fileOutputName;
@ -945,7 +970,7 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
if (!writeGeneratorOutput(*definition->output, nameSettings, formatSettings,
outputSettings))
{
ErrorAtToken(*buildObject.definition->definitionInvocation,
ErrorAtToken(*definition->definitionInvocation,
"Failed to write to compile-time source file");
continue;
}
@ -969,6 +994,15 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
linkerDynamicLibraryExtension);
buildObject.dynamicLibraryPath = dynamicLibraryOut;
// Save our import library name for other functions to use
if (environment.isMsvcCompiler && definition->type == ObjectType_CompileTimeFunction)
{
char importLibraryName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(importLibraryName, "%s.%s", buildObject.artifactsName.c_str(),
compilerImportLibraryExtension);
definition->compileTimeImportLibraryName = importLibraryName;
}
char headerInclude[MAX_PATH_LENGTH] = {0};
if (environment.cakelispSrcDir.empty())
makeIncludeArgument(headerInclude, sizeof(headerInclude), "src/");
@ -980,6 +1014,14 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
makeObjectOutputArgument(buildObjectArgument, sizeof(buildObjectArgument),
buildObject.buildObjectName.c_str());
char debugSymbolsName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(debugSymbolsName, "%s/%s.%s", cakelispWorkingDir,
buildObject.artifactsName.c_str(), compilerDebugSymbolsExtension);
char debugSymbolsArgument[MAX_PATH_LENGTH] = {0};
makeDebugSymbolsOutputArgument(debugSymbolsArgument, sizeof(debugSymbolsArgument),
debugSymbolsName);
char compileTimeBuildExecutable[MAX_PATH_LENGTH] = {0};
if (!resolveExecutablePath(environment.compileTimeBuildCommand.fileToExecute.c_str(),
compileTimeBuildExecutable, sizeof(compileTimeBuildExecutable)))
@ -989,6 +1031,7 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
ProcessCommandInput compileTimeInputs[] = {
{ProcessCommandArgumentType_SourceInput, {sourceOutputName}},
{ProcessCommandArgumentType_ObjectOutput, {buildObjectArgument}},
{ProcessCommandArgumentType_DebugSymbolsOutput, {debugSymbolsArgument}},
{ProcessCommandArgumentType_CakelispHeadersInclude, {headerInclude}}};
const char** buildArguments = MakeProcessArgumentsFromCommand(
compileTimeBuildExecutable, environment.compileTimeBuildCommand.arguments,
@ -1027,6 +1070,11 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
}
}
// Annoying Windows workaround: delete PDB to fix fatal error C1052
// Technically we only need to do this for /DEBUG:fastlink
if (debugSymbolsArgument[0] && fileExists(debugSymbolsName))
remove(debugSymbolsName);
RunProcessArguments compileArguments = {};
compileArguments.fileToExecute = compileTimeBuildExecutable;
compileArguments.arguments = buildArguments;
@ -1083,6 +1131,18 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
if (logging.buildProcess)
Logf("Compiled %s successfully\n", buildObject.definition->name.c_str());
std::vector<std::string> importLibraryPaths;
// Need to be able to find imported dll import libraries
importLibraryPaths.push_back(cakelispWorkingDir);
std::vector<const char*> importLibraryPathsArgs;
std::vector<const char*> importLibrariesArgs;
BuildArgumentConverter convertedArguments[] = {
{&importLibraryPaths, {}, &importLibraryPathsArgs, makeImportLibraryPathArgument},
{&buildObject.importLibraries, {}, &importLibrariesArgs, nullptr}};
convertBuildArguments(convertedArguments, ArraySize(convertedArguments),
environment.compileTimeLinkCommand.fileToExecute.c_str());
char dynamicLibraryOutArgument[MAX_PATH_LENGTH] = {0};
makeDynamicLibraryOutputArgument(dynamicLibraryOutArgument,
sizeof(dynamicLibraryOutArgument),
@ -1096,7 +1156,9 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
ProcessCommandInput linkTimeInputs[] = {
{ProcessCommandArgumentType_DynamicLibraryOutput, {dynamicLibraryOutArgument}},
{ProcessCommandArgumentType_ObjectInput, {buildObject.buildObjectName.c_str()}}};
{ProcessCommandArgumentType_ObjectInput, {buildObject.buildObjectName.c_str()}},
{ProcessCommandArgumentType_ImportLibraryPaths, importLibraryPathsArgs},
{ProcessCommandArgumentType_ImportLibraries, importLibrariesArgs}};
const char** linkArgumentList = MakeProcessArgumentsFromCommand(
compileTimeLinkExecutable, environment.compileTimeLinkCommand.arguments, linkTimeInputs,
ArraySize(linkTimeInputs));
@ -1561,8 +1623,11 @@ bool EvaluateResolveReferences(EvaluatorEnvironment& environment)
}
}
buildWriteCacheFile(cakelispWorkingDir, environment.comptimeCachedCommandCrcs,
environment.comptimeNewCommandCrcs);
// Only write CRCs if we did some comptime compilation. Otherwise, the tokenizer will complain
// about loading a completely empty file
if (!environment.comptimeNewCommandCrcs.empty())
buildWriteCacheFile(cakelispWorkingDir, environment.comptimeCachedCommandCrcs,
environment.comptimeNewCommandCrcs);
return errors == 0 && numBuildResolveErrors == 0;
}


+ 25
- 13
src/Evaluator.hpp View File

@ -7,6 +7,7 @@
#include "Build.hpp"
#include "EvaluatorEnums.hpp"
#include "Exporting.hpp"
#include "FileTypes.hpp"
#include "RunProcess.hpp"
@ -84,6 +85,9 @@ struct EvaluatorContext
bool isRequired;
// Associate all unknown references with this definition
const Token* definitionName;
// When resolving references, have the context keep track of which reference is being resolved,
// to avoid creating additional references unnecessarily
const Token* resolvingReference;
Module* module;
// Insert delimiterTemplate between each expression/statement. Only recognized in
// EvaluateGenerateAll_Recursive()
@ -189,6 +193,8 @@ struct ObjectDefinition
bool isLoaded;
// Used by other compile-time functions to include this function's already output header
std::string compileTimeHeaderName;
// Only necessary for Windows builds; holds the import library for calling comptime functions
std::string compileTimeImportLibraryName;
// Arbitrary tags user may add for compile-time reference
std::vector<std::string> tags;
@ -317,6 +323,9 @@ struct EvaluatorEnvironment
// Generate code so that objects defined in Cakelisp can be loaded at runtime
bool useCLinkage;
// Whether to enable MSVC-specific hacks/conversions
bool isMsvcCompiler;
// Whether it is okay to skip an operation if the resultant file is already in the cache (and
// the source file hasn't been modified more recently)
bool useCachedFiles;
@ -369,9 +378,10 @@ struct EvaluatorEnvironment
// tokens. Essentially, call this as late as possible
void environmentDestroyInvalidateTokens(EvaluatorEnvironment& environment);
int EvaluateGenerate_Recursive(EvaluatorEnvironment& environment, const EvaluatorContext& context,
const std::vector<Token>& tokens, int startTokenIndex,
GeneratorOutput& output);
CAKELISP_API int EvaluateGenerate_Recursive(EvaluatorEnvironment& environment,
const EvaluatorContext& context,
const std::vector<Token>& tokens, int startTokenIndex,
GeneratorOutput& output);
// Delimiter template will be inserted between the outputs. Pass nullptr for no delimiter
int EvaluateGenerateAll_Recursive(EvaluatorEnvironment& environment,
@ -380,9 +390,9 @@ int EvaluateGenerateAll_Recursive(EvaluatorEnvironment& environment,
// For compile-time code modification.
// This destroys the old definition. Don't hold on to references to it for that reason
bool ReplaceAndEvaluateDefinition(EvaluatorEnvironment& environment,
const char* definitionToReplaceName,
const std::vector<Token>& newDefinitionTokens);
CAKELISP_API bool ReplaceAndEvaluateDefinition(EvaluatorEnvironment& environment,
const char* definitionToReplaceName,
const std::vector<Token>& newDefinitionTokens);
// Returns whether all references were resolved successfully
bool EvaluateResolveReferences(EvaluatorEnvironment& environment);
@ -400,17 +410,19 @@ bool registerEvaluateGenerator(EvaluatorEnvironment& environment, const char* ge
GeneratorFunc function);
GeneratorFunc findGenerator(EvaluatorEnvironment& environment, const char* functionName);
void* findCompileTimeFunction(EvaluatorEnvironment& environment, const char* functionName);
CAKELISP_API void* findCompileTimeFunction(EvaluatorEnvironment& environment,
const char* functionName);
bool findCompileTimeSymbol(EvaluatorEnvironment& environment, const char* symbolName);
ObjectDefinition* findObjectDefinition(EvaluatorEnvironment& environment, const char* name);
CAKELISP_API ObjectDefinition* findObjectDefinition(EvaluatorEnvironment& environment,
const char* name);
// These must take type as string in order to be address agnostic, making caching possible
// destroyFunc is necessary for any C++ type with a destructor. If nullptr, free() is used
bool CreateCompileTimeVariable(EvaluatorEnvironment& environment, const char* name,
const char* typeExpression, void* data,
const char* destroyCompileTimeFuncName);
bool GetCompileTimeVariable(EvaluatorEnvironment& environment, const char* name,
const char* typeExpression, void** dataOut);
CAKELISP_API bool CreateCompileTimeVariable(EvaluatorEnvironment& environment, const char* name,
const char* typeExpression, void* data,
const char* destroyCompileTimeFuncName);
CAKELISP_API bool GetCompileTimeVariable(EvaluatorEnvironment& environment, const char* name,
const char* typeExpression, void** dataOut);
// Whether the (reference) cached file is more recent than the filename, meaning whatever operation
// which made the cached file can be skipped. Basically fileIsMoreRecentlyModified(), but respects


+ 10
- 9
src/GeneratorHelpers.hpp View File

@ -1,7 +1,7 @@
#pragma once
#include <vector>
#include <string>
#include <vector>
#include "EvaluatorEnums.hpp"
#include "Exporting.hpp"
@ -30,15 +30,15 @@ bool ExpectInInvocation(const char* message, const std::vector<Token>& tokens, i
// Returns true if the symbol starts with :, &, or '
// TODO: Come up with better name
bool isSpecialSymbol(const Token& token);
CAKELISP_API bool isSpecialSymbol(const Token& token);
// startTokenIndex should be the opening parenthesis of the array you want to retrieve arguments
// from. For example, you should pass in the opening paren of a function invocation to get its name
// as argument 0 and first arg as argument 1 This function would be simpler and faster if there was
// an actual syntax tree, because we wouldn't be repeatedly traversing all the arguments
// Returns -1 if argument is not within range
int getArgument(const std::vector<Token>& tokens, int startTokenIndex, int desiredArgumentIndex,
int endTokenIndex);
CAKELISP_API int getArgument(const std::vector<Token>& tokens, int startTokenIndex,
int desiredArgumentIndex, int endTokenIndex);
CAKELISP_API int getExpectedArgument(const char* message, const std::vector<Token>& tokens,
int startTokenIndex, int desiredArgumentIndex,
int endTokenIndex);
@ -57,11 +57,11 @@ int getNextArgument(const std::vector<Token>& tokens, int currentTokenIndex,
// block, so it knows the scope comes from the generator invocation
int blockAbsorbScope(const std::vector<Token>& tokens, int startBlockIndex);
const Token* FindTokenExpressionEnd(const Token* startToken);
CAKELISP_API const Token* FindTokenExpressionEnd(const Token* startToken);
// This is useful for copying a definition, with macros expanded, for e.g. code modification
bool CreateDefinitionCopyMacroExpanded(const ObjectDefinition& definition,
std::vector<Token>& tokensOut);
CAKELISP_API bool CreateDefinitionCopyMacroExpanded(const ObjectDefinition& definition,
std::vector<Token>& tokensOut);
// Similar to Lisp's gensym, make a globally unique symbol for e.g. macro variables. Use prefix so
// it is still documented as to what it represents. Make sure your generated tokenToChange is
@ -73,8 +73,9 @@ void MakeUniqueSymbolName(EvaluatorEnvironment& environment, const char* prefix,
Token* tokenToChange);
// This should be stable as long as the context is managed properly. Code modification may make it
// unstable unless they reset the context on reevaluate, etc.
void MakeContextUniqueSymbolName(EvaluatorEnvironment& environment, const EvaluatorContext& context,
const char* prefix, Token* tokenToChange);
CAKELISP_API void MakeContextUniqueSymbolName(EvaluatorEnvironment& environment,
const EvaluatorContext& context, const char* prefix,
Token* tokenToChange);
CAKELISP_API void PushBackTokenExpression(std::vector<Token>& output, const Token* startToken);


+ 20
- 12
src/Generators.cpp View File

@ -65,6 +65,9 @@ bool SetProcessCommandArguments(EvaluatorEnvironment& environment, const std::ve
} symbolsToCommandTypes[] = {
{"'source-input", ProcessCommandArgumentType_SourceInput},
{"'object-output", ProcessCommandArgumentType_ObjectOutput},
{"'debug-symbols-output", ProcessCommandArgumentType_DebugSymbolsOutput},
{"'import-library-paths", ProcessCommandArgumentType_ImportLibraryPaths},
{"'import-libraries", ProcessCommandArgumentType_ImportLibraries},
{"'cakelisp-headers-include", ProcessCommandArgumentType_CakelispHeadersInclude},
{"'include-search-dirs", ProcessCommandArgumentType_IncludeSearchDirs},
{"'additional-options", ProcessCommandArgumentType_AdditionalOptions},
@ -958,9 +961,18 @@ bool DefunGenerator(EvaluatorEnvironment& environment, const EvaluatorContext& c
if (!parseFunctionSignature(tokens, argsIndex, arguments, returnTypeStart))
return false;
// Compile-time functions need to be exposed with C bindings so they can be found
if (isCompileTime && environment.isMsvcCompiler)
{
addStringOutput(functionOutput->source, "__declspec(dllexport)", StringOutMod_SpaceAfter,
&tokens[startTokenIndex]);
addStringOutput(functionOutput->header, "extern \"C\" __declspec(dllimport)",
StringOutMod_SpaceAfter, &tokens[startTokenIndex]);
// addStringOutput(functionOutput->header, "__declspec(dllimport)", StringOutMod_SpaceAfter,
// &tokens[startTokenIndex]);
}
// Compile-time functions need to be exposed with C bindings so they can be found (true?)
// Module-local functions are always marked static, which hides them from linking
if (!isModuleLocal && (isCompileTime || environment.useCLinkage))
else if (!isModuleLocal && (isCompileTime || environment.useCLinkage))
{
addStringOutput(functionOutput->header, "extern \"C\"", StringOutMod_SpaceAfter,
&tokens[startTokenIndex]);
@ -1740,11 +1752,9 @@ bool DefMacroGenerator(EvaluatorEnvironment& environment, const EvaluatorContext
// Macros will be found without headers thanks to dynamic linking
// bool isModuleLocal = tokens[startTokenIndex + 1].contents.compare("defmacro-local") == 0;
// TODO Only do this when using MSVC
#ifdef WINDOWS
addStringOutput(compTimeOutput->source, "__declspec(dllexport)", StringOutMod_SpaceAfter,
&tokens[startTokenIndex]);
#endif
if (environment.isMsvcCompiler)
addStringOutput(compTimeOutput->source, "__declspec(dllexport)", StringOutMod_SpaceAfter,
&tokens[startTokenIndex]);
// Macros must return success or failure
addStringOutput(compTimeOutput->source, "bool", StringOutMod_SpaceAfter,
@ -1839,11 +1849,9 @@ bool DefGeneratorGenerator(EvaluatorEnvironment& environment, const EvaluatorCon
// Generators will be found without headers thanks to dynamic linking
// bool isModuleLocal = tokens[startTokenIndex + 1].contents.compare("defgenerator-local") == 0;
// TODO Only do this when using MSVC
#ifdef WINDOWS
addStringOutput(compTimeOutput->source, "__declspec(dllexport)", StringOutMod_SpaceAfter,
&tokens[startTokenIndex]);
#endif
if (environment.isMsvcCompiler)
addStringOutput(compTimeOutput->source, "__declspec(dllexport)", StringOutMod_SpaceAfter,
&tokens[startTokenIndex]);
// Generators must return success or failure
addStringOutput(compTimeOutput->source, "bool", StringOutMod_SpaceAfter,


+ 36
- 49
src/ModuleManager.cpp View File

@ -75,6 +75,8 @@ void moduleManagerInitialize(ModuleManager& manager)
// Command defaults
{
#ifdef WINDOWS
manager.environment.isMsvcCompiler = true;
// MSVC by default
// Our lives could be easier by using Clang or MinGW, but it wouldn't be the ideal for
// hardcore Windows users, who we should support
@ -95,7 +97,8 @@ void moduleManagerInitialize(ModuleManager& manager)
{ProcessCommandArgumentType_String, "/Zi"},
{ProcessCommandArgumentType_String, "/c"},
{ProcessCommandArgumentType_SourceInput, EmptyString},
{ProcessCommandArgumentType_ObjectOutput, EmptyString},
{ProcessCommandArgumentType_ObjectOutput, EmptyString},
{ProcessCommandArgumentType_DebugSymbolsOutput, EmptyString},
{ProcessCommandArgumentType_CakelispHeadersInclude, EmptyString}};
manager.environment.compileTimeLinkCommand.fileToExecute = "link.exe";
@ -106,6 +109,8 @@ void moduleManagerInitialize(ModuleManager& manager)
// unresolved externals
{ProcessCommandArgumentType_String, "/LIBPATH:bin"},
{ProcessCommandArgumentType_String, "cakelisp.lib"},
{ProcessCommandArgumentType_ImportLibraryPaths, EmptyString},
{ProcessCommandArgumentType_ImportLibraries, EmptyString},
// Debug only
{ProcessCommandArgumentType_String, "/DEBUG:FASTLINK"},
{ProcessCommandArgumentType_DynamicLibraryOutput, EmptyString},
@ -118,7 +123,8 @@ void moduleManagerInitialize(ModuleManager& manager)
{ProcessCommandArgumentType_String, "/c"},
{ProcessCommandArgumentType_SourceInput, EmptyString},
{ProcessCommandArgumentType_ObjectOutput, EmptyString},
{ProcessCommandArgumentType_IncludeSearchDirs, EmptyString},
{ProcessCommandArgumentType_DebugSymbolsOutput, EmptyString},
{ProcessCommandArgumentType_IncludeSearchDirs, EmptyString},
{ProcessCommandArgumentType_AdditionalOptions, EmptyString}};
manager.environment.buildTimeLinkCommand.fileToExecute = "link.exe";
@ -277,7 +283,9 @@ bool moduleLoadTokenizeValidate(const char* filename, const std::vector<Token>**
if (tokens->empty())
{
Log("error: empty file. Please remove from system, or add (ignore)\n");
Logf("error: empty file or tokenization error with '%s'. Please remove from system, or add "
"(ignore)\n",
filename);
delete tokens;
return false;
}
@ -345,6 +353,7 @@ bool moduleManagerAddEvaluateFile(ModuleManager& manager, const char* filename,
makeSafeFilename(safePathBuffer, sizeof(safePathBuffer), resolvedPath);
const char* normalizedFilename = StrDuplicate(safePathBuffer);
Logf("'%s' normalized to '%s'\n", filename, normalizedFilename);
// Enabling this makes all file:line messages really long. For now, I'll keep it as relative to
// current working directory of this executable.
// const char* normalizedFilename = makeAbsolutePath_Allocated(".", filename);
@ -870,6 +879,13 @@ bool moduleManagerBuild(ModuleManager& manager, std::vector<BuildObject*>& build
objectOutput = &objectOutputOverride;
}
char debugSymbolsName[MAX_PATH_LENGTH] = {0};
PrintfBuffer(debugSymbolsName, "%s.%s", objectOutput->c_str(),
compilerDebugSymbolsExtension);
char debugSymbolsArgument[MAX_PATH_LENGTH] = {0};
makeDebugSymbolsOutputArgument(debugSymbolsArgument, sizeof(debugSymbolsArgument),
debugSymbolsName);
char buildTimeBuildExecutable[MAX_PATH_LENGTH] = {0};
if (!resolveExecutablePath(buildCommand.fileToExecute.c_str(), buildTimeBuildExecutable,
sizeof(buildTimeBuildExecutable)))
@ -881,7 +897,8 @@ bool moduleManagerBuild(ModuleManager& manager, std::vector<BuildObject*>& build
ProcessCommandInput buildTimeInputs[] = {
{ProcessCommandArgumentType_SourceInput, {object->sourceFilename.c_str()}},
{ProcessCommandArgumentType_ObjectOutput, {objectOutput->c_str()}},
{ProcessCommandArgumentType_IncludeSearchDirs, std::move(searchDirArgs)},
{ProcessCommandArgumentType_DebugSymbolsOutput, {debugSymbolsArgument}},
{ProcessCommandArgumentType_IncludeSearchDirs, std::move(searchDirArgs)},
{ProcessCommandArgumentType_AdditionalOptions, std::move(additionalOptions)}};
const char** buildArguments =
MakeProcessArgumentsFromCommand(buildTimeBuildExecutable, buildCommand.arguments,
@ -915,6 +932,11 @@ bool moduleManagerBuild(ModuleManager& manager, std::vector<BuildObject*>& build
}
}
// Annoying Windows workaround: delete PDB to fix fatal error C1052
// Technically we only need to do this for /DEBUG:fastlink
if (debugSymbolsArgument[0] && fileExists(debugSymbolsName))
remove(debugSymbolsName);
// Go through with the build
RunProcessArguments compileArguments = {};
compileArguments.fileToExecute = buildTimeBuildExecutable;
@ -1027,28 +1049,19 @@ bool moduleManagerLink(ModuleManager& manager, std::vector<BuildObject*>& buildO
// Copy it so hooks can modify it
ProcessCommand linkCommand = *buildOptions.linkCommand;
char executableOutput[MAX_PATH_LENGTH + 5] = {0};
makeExecutableOutputArgument(executableOutput, sizeof(executableOutput),
outputExecutableName.c_str(),
linkCommand.fileToExecute.c_str());
std::vector<std::string> executableOutputString;
executableOutputString.push_back(outputExecutableName);
// Various library and linker arguments need prefixes added. Do that here
// Various arguments need prefixes added. Do that here
std::vector<const char*> executableToArgs;
std::vector<const char*> librariesArgs;
std::vector<const char*> librarySearchDirsArgs;
std::vector<const char*> libraryRuntimeSearchDirsArgs;
std::vector<const char*> convertedLinkerArgs;
std::vector<const char*> compilerLinkArgs;
struct
{
std::vector<std::string>* stringsIn;
// Use C++ to manage our string memory, pointed to by argumentsOut
std::vector<std::string> argumentsOutMemory;
std::vector<const char*>* argumentsOut;
void (*argumentConversionFunc)(char* buffer, int bufferSize, const char* stringIn,
const char* linkExecutable);
} libraryArguments[] = {
{&buildOptions.linkLibraries, {}, &librariesArgs, makeLinkLibraryArgument},
BuildArgumentConverter convertedArguments[] = {
{&executableOutputString, {}, &executableToArgs, makeExecutableOutputArgument},
{&buildOptions.linkLibraries, {}, &librariesArgs, makeLinkLibraryArgument},
{&buildOptions.librarySearchDirs,
{},
&librarySearchDirsArgs,
@ -1060,37 +1073,11 @@ bool moduleManagerLink(ModuleManager& manager, std::vector<BuildObject*>& buildO
{&buildOptions.toLinkerOptions, {}, &convertedLinkerArgs, makeLinkerArgument},
// We can't know how to auto-convert these because they could be anything
{&buildOptions.compilerLinkOptions, {}, &compilerLinkArgs, nullptr}};
for (int inputIndex = 0; inputIndex < ArraySize(libraryArguments); ++inputIndex)
{
int numStrings = (int)libraryArguments[inputIndex].stringsIn->size();
libraryArguments[inputIndex].argumentsOutMemory.resize(numStrings);
libraryArguments[inputIndex].argumentsOut->resize(numStrings);
int currentString = 0;
for (const std::string& stringIn : *libraryArguments[inputIndex].stringsIn)
{
if (libraryArguments[inputIndex].argumentConversionFunc)
{
// Give some extra padding for prefixes
char argumentOut[MAX_PATH_LENGTH + 15] = {0};
libraryArguments[inputIndex].argumentConversionFunc(
argumentOut, sizeof(argumentOut), stringIn.c_str(),
linkCommand.fileToExecute.c_str());
libraryArguments[inputIndex].argumentsOutMemory[currentString] = argumentOut;
}
else
libraryArguments[inputIndex].argumentsOutMemory[currentString] = stringIn;
++currentString;
}
for (int stringIndex = 0; stringIndex < numStrings; ++stringIndex)
(*libraryArguments[inputIndex].argumentsOut)[stringIndex] =
libraryArguments[inputIndex].argumentsOutMemory[stringIndex].c_str();
}
convertBuildArguments(convertedArguments, ArraySize(convertedArguments),
linkCommand.fileToExecute.c_str());
ProcessCommandInput linkTimeInputs[] = {
{ProcessCommandArgumentType_ExecutableOutput, {executableOutput}},
{ProcessCommandArgumentType_ExecutableOutput, executableToArgs},
{ProcessCommandArgumentType_ObjectInput, objectsToLink},
{ProcessCommandArgumentType_AdditionalOptions, compilerLinkArgs},
{ProcessCommandArgumentType_LibrarySearchDirs, librarySearchDirsArgs},


+ 7
- 1
src/RunProcess.cpp View File

@ -318,7 +318,7 @@ int runProcess(const RunProcessArguments& arguments, int* statusOut)
ZeroMemory(processInfo, sizeof(PROCESS_INFORMATION));
// Start the child process.
if (!CreateProcess(fileToExecute, // No module name (use command line)
if (!CreateProcess(fileToExecute,
commandLineString, // Command line
nullptr, // No security attributes
nullptr, // Thread handle not inheritable
@ -494,6 +494,12 @@ static const char* ProcessCommandArgumentTypeToString(ProcessCommandArgumentType
return "SourceInput";
case ProcessCommandArgumentType_ObjectOutput:
return "ObjectOutput";
case ProcessCommandArgumentType_DebugSymbolsOutput:
return "DebugSymbolsOutput";
case ProcessCommandArgumentType_ImportLibraryPaths:
return "ImportLibraryPaths";
case ProcessCommandArgumentType_ImportLibraries:
return "ImportLibraries";
case ProcessCommandArgumentType_CakelispHeadersInclude:
return "CakelispHeadersInclude";
case ProcessCommandArgumentType_IncludeSearchDirs:


+ 6
- 5
src/RunProcess.hpp View File

@ -1,5 +1,6 @@
#pragma once
#include "Exporting.hpp"
#include "RunProcessEnums.hpp"
#include <string>
@ -13,11 +14,11 @@ struct RunProcessArguments
const char** arguments;
};
int runProcess(const RunProcessArguments& arguments, int* statusOut);
CAKELISP_API int runProcess(const RunProcessArguments& arguments, int* statusOut);
typedef void (*SubprocessOnOutputFunc)(const char* subprocessOutput);
void waitForAllProcessesClosed(SubprocessOnOutputFunc onOutput);
CAKELISP_API void waitForAllProcessesClosed(SubprocessOnOutputFunc onOutput);
//
// Helpers for programmatically constructing arguments
@ -45,8 +46,8 @@ void PrintProcessArguments(const char** processArguments);
// The array will need to be deleted, but the array members will not
// All strings need to exist and not be moved until after you call runProcess
const char** MakeProcessArgumentsFromCommand(const char* fileToExecute,
std::vector<ProcessCommandArgument>& arguments,
const ProcessCommandInput* inputs, int numInputs);
CAKELISP_API const char** MakeProcessArgumentsFromCommand(
const char* fileToExecute, std::vector<ProcessCommandArgument>& arguments,
const ProcessCommandInput* inputs, int numInputs);
extern const int maxProcessesRecommendedSpawned;

+ 3
- 0
src/RunProcessEnums.hpp View File

@ -9,6 +9,9 @@ enum ProcessCommandArgumentType
ProcessCommandArgumentType_SourceInput,
ProcessCommandArgumentType_ObjectOutput,
ProcessCommandArgumentType_ImportLibraryPaths,
ProcessCommandArgumentType_ImportLibraries,
ProcessCommandArgumentType_DebugSymbolsOutput,
ProcessCommandArgumentType_CakelispHeadersInclude,
ProcessCommandArgumentType_IncludeSearchDirs,
ProcessCommandArgumentType_AdditionalOptions,


+ 5
- 4
src/Tokenizer.hpp View File

@ -39,13 +39,14 @@ CAKELISP_API bool tokenizeLinePrintError(const char* inputLine, const char* sour
bool validateTokens(const std::vector<Token>& tokens);
void printTokens(const std::vector<Token>& tokens);
void prettyPrintTokens(const std::vector<Token>& tokens);
CAKELISP_API void prettyPrintTokens(const std::vector<Token>& tokens);
void prettyPrintTokensToFile(FILE* file, const std::vector<Token>& tokens);
bool writeCharToBufferErrorToken(char c, char** at, char* bufferStart, int bufferSize,
const Token& token);
bool writeStringToBufferErrorToken(const char* str, char** at, char* bufferStart, int bufferSize,
const Token& token);
bool appendTokenToString(const Token& token, char** at, char* bufferStart, int bufferSize);
CAKELISP_API bool writeStringToBufferErrorToken(const char* str, char** at, char* bufferStart,
int bufferSize, const Token& token);
CAKELISP_API bool appendTokenToString(const Token& token, char** at, char* bufferStart,
int bufferSize);
extern CAKELISP_API int g_totalLinesTokenized;

+ 11
- 9
test/BuildHelpers.cake View File

@ -8,18 +8,20 @@
(return 0))
(defun-comptime run-3rd-party-build (manager (& ModuleManager) module (* Module) &return bool)
(var cakelisp-executable (* (const char)) "bin/cakelisp")
(comptime-cond ('Windows (set cakelisp-executable "bin/cakelisp.exe")))
;; Sequential
(run-process-sequential-or ("ls" "-lh" :in-directory "src")
(Log "failed to run process")
(run-process-sequential-or (cakelisp-executable "--list-built-ins" :in-directory "src")
(Log "failed to run process\n")
(return false))
;; Parallel
(var ls-status int -1)
(run-process-start-or (addr ls-status) ("ls" "-lh")
(return false))
(var echo-status int -1)
(run-process-start-or (addr echo-status) ("echo" "Hello multiprocessing!")
(return false))
(var cakelisp-status int -1)
(run-process-start-or (addr cakelisp-status) (cakelisp-executable "--list-built-ins")
(return false))
(var cakelisp-2-status int -1)
(run-process-start-or (addr cakelisp-2-status) (cakelisp-executable "--list-built-ins" :in-directory "src")
(return false))
(waitForAllProcessesClosed null)
(return (and (= ls-status 0) (= echo-status 0))))
(return (and (= cakelisp-status 0) (= cakelisp-2-status 0))))
(add-compile-time-hook-module pre-build run-3rd-party-build)

+ 9
- 2
test/RunTests.cake View File

@ -24,19 +24,26 @@
(var tests ([] (const cakelisp-test))
(array
(array "Code modification" "test/CodeModification.cake")
(array "Build options" "test/BuildOptions.cake")
;; (array "Build options" "test/BuildOptions.cake")
(array "Execute" "test/Execute.cake")
(array "Defines" "test/Defines.cake")
(array "Multi-line strings" "test/MultiLineStrings.cake")
(array "Build helpers" "test/BuildHelpers.cake")))
(var platform-config (* (const char))
(comptime-cond
('Windows
"runtime/Config_Windows.cake")
(true
"runtime/Config_Linux.cake")))
(var i int 0)
(while (< i (array-size tests))
(var test-name (* (const char)) (field (at i tests) test-name))
(var test-file (* (const char)) (field (at i tests) test-file))
(Logf "\n===============\n%s\n\n" test-name)
(run-process-sequential-or
(cakelisp-executable "--execute" test-file)
(cakelisp-executable "--execute" platform-config test-file)
(Logf "error: test %s failed\n" test-name)
(return false))
(Logf "\n%s succeeded\n" test-name)


Loading…
Cancel
Save