Browse Source

Added run-process macros for build hooks

runtime/BuildTools.cake adds helper macros and comptime functions for
creating arbitrary processes. This makes it possible to easily
e.g. run CMake on a 3rd party dependency.

* Fix build/test script to exit on errors
* Update Debugging.org with correct solib-search-path
* resolveExecutablePath() now checks usual Linux path dirs. I am going
to replace this list with actually checking the PATH var
* Use prettyPrintTokens when macro output is erroneous (improves
  readability)
* Fix SpliceNoSpace not actually removing spaces
* Make hooks error print available hooks
* Add more default includes to comptime preamble
* Check for empty arguments instead of segfaulting in runProcess()
* Improve working directory error
* Hook up working directory to Windows (untested!)
* Make onOutput callback optional
build-system-unification
Macoy Madson 5 months ago
parent
commit
e7e6bd2839
12 changed files with 270 additions and 33 deletions
  1. +7
    -5
      BuildAndRunTests.sh
  2. +1
    -1
      doc/Debugging.org
  3. +163
    -0
      runtime/BuildTools.cake
  4. +44
    -2
      src/Build.cpp
  5. +2
    -2
      src/Evaluator.cpp
  6. +5
    -2
      src/GeneratorHelpers.cpp
  7. +2
    -1
      src/Generators.cpp
  8. +1
    -1
      src/Main.cpp
  9. +3
    -2
      src/OutputPreambles.cpp
  10. +26
    -16
      src/RunProcess.cpp
  11. +1
    -1
      src/RunProcess.hpp
  12. +15
    -0
      test/BuildHelpers.cake

+ 7
- 5
BuildAndRunTests.sh View File

@ -8,13 +8,13 @@
# clang -g -fPIC -x c++-header src/Evaluator.hpp -o src/Evaluator.hpp.pch
./bin/cakelisp --execute \
test/CodeModification.cake
test/CodeModification.cake || exit $?
./bin/cakelisp \
test/BuildOptions.cake
test/BuildOptions.cake || exit $?
./bin/cakelisp --execute \
test/Execute.cake
test/Execute.cake || exit $?
./bin/cakelisp \
runtime/Config_Linux.cake runtime/HotReloadingCodeModifier.cake runtime/TextAdventure.cake || exit $?
@ -27,8 +27,10 @@
runtime/Config_Linux.cake runtime/HotLoader.cake || exit $?
./bin/cakelisp --execute \
test/Defines.cake
test/Defines.cake || exit $?
# ./bin/cakelisp CrossCompile_Windows.cake
# ./bin/cakelisp CrossCompile_Windows.cake || exit $?
./bin/cakelisp --execute test/MultiLineStrings.cake || exit $?
./bin/cakelisp --execute --verbose-processes test/BuildHelpers.cake || exit $?

+ 1
- 1
doc/Debugging.org View File

@ -7,7 +7,7 @@ Run ~cakelisp --help~ to see what command-line arguments may be passed in to con
The following command may be run in order to tell GDB where the ~.so~ files you want to debug are located:
#+BEGIN_SRC sh
set solib-search-path ~/Development/code/repositories/cakelisp/
set solib-search-path ~/Development/code/repositories/cakelisp/cakelisp_cache
set cwd ~/Development/code/repositories/cakelisp/
#+END_SRC


+ 163
- 0
runtime/BuildTools.cake View File

@ -0,0 +1,163 @@
(skip-build)
(ignore
;; Straightline version
(defun build-sdl (&return bool)
(when (= target-platform 'Unix)
(unless (file-exists "Dependencies/SDL/build/lib/libSDL2.so")
(var sdl-build-dir (* (const char)) "Dependencies/SDL/build")
(unless (make-directory sdl-build-dir)
(return false))
(var sdl-install-dir (* (const char)) (makeAbsolutePath_Allocated sdl-build-dir))
(var sdl-install-prefix ([] MAX_PATH_LENGTH char) (array 0))
(PrintfBuffer sdl-install-prefix "--prefix=%s/install" sdl-install-dir)
(run-process "sh" "../configure" sdl-install-prefix
:in-directory sdl-build-dir (addr status))
(waitForAllProcessesClosed)
(unless (= 0 status)
(return false))
(run-process "make" :in-directory sdl-build-dir (addr status))
(waitForAllProcessesClosed)
(unless (= 0 status)
(return false))
(run-process "make" "install" :in-directory sdl-build-dir (addr status))
(waitForAllProcessesClosed)
(unless (= 0 status)
(return false))
(unless (file-exists "Dependencies/SDL/build/lib/libSDL2.so")
(Log "SDL still not built in expected location, despite build success. Configuration error?")
(return false)))))
;; Fancy version: DSL
;; Doesn't actually add that much over just adding a run-process-sequential macro
;; Another helpful macro: (create-and-format-string build-dir MAX_PATH_LENGTH "%s/build" sdl-dir)
(defun build-sdl-dsl (&return bool)
(unless (file-exists "Dependencies/SDL")
(Log "Failed to find SDL in Dependencies/SDL.\nIf you are using Git, run:\n
git submodule update --init --recursive\n
If you are not using Git, please download SDL via hg:\n
hg clone http://hg.libsdl.org/SDL\n
Or download a source release from https://www.libsdl.org/hg.php\n
Put it in Dependencies/SDL")
(return false))
(unless (file-exists "Dependencies/SDL/build/lib/libSDL2.so")
(make-directory build-dir)
(build-process "SDL (Unix, CMake, Ninja)"
(sequential
(run-process "sh" "../configure" sdl-install-prefix :in-directory build-dir
:on-fail "could not configure SDL. Is autotools installed?")
(run-process "make" :in-directory build-dir)
(run-process "make" "install" :in-directory build-dir))))
(unless (file-exists "Dependencies/SDL/build/lib/libSDL2.so")
(Log "did not find required files after successful SDL build. Does configuration need updating?")
(return false))
(return true))
(make-directory "Dependencies/ogre-next/build")
(run-process "cmake" ".." :in-directory "Dependencies/ogre-next/build" (addr status))
(waitForAllProcessesClosed)
(unless (= 0 status)
(return false))
(run-process "ninja" :in-directory "Dependencies/ogre-next/build" (addr status))
(waitForAllProcessesClosed)
(unless (= 0 status)
(return false))
) ;; End ignore
;; Returns exit code (0 = success)
(defun-comptime run-process-wait-for-completion (run-arguments (* RunProcessArguments)
&return int)
(var status int -1)
(unless (= 0 (runProcess (deref run-arguments) (addr status)))
(Log "error: failed to run process\n")
(return 1))
(waitForAllProcessesClosed nullptr)
(return status))
(defmacro gen-unique-symbol (token-var-name symbol prefix string reference-token any)
(tokenize-push
output
(var (token-splice token-var-name) Token (token-splice reference-token))
(MakeContextUniqueSymbolName environment context (token-splice prefix)
(addr (token-splice token-var-name))))
(return true))
;; Creates a variable arguments-out-name set up to run the given process
;; Use :in-directory to specify the working directory to run the process in
(defmacro run-process-make-arguments (arguments-out-name symbol executable-name any
&optional &rest arguments any)
(var specifier-tokens (<> std::vector Token))
(var command-arguments (<> std::vector Token))
(when arguments
(var current-token (* (const Token)) arguments)
(var end-paren-index int (FindCloseParenTokenIndex tokens startTokenIndex))
(var end-token (* (const Token)) (addr (at end-paren-index tokens)))
(while (< current-token end-token)
(cond
;; Special symbols to add optional specifications
((and (= TokenType_Symbol (path current-token > type))
(isSpecialSymbol (deref current-token)))
(cond
((= 0 (on-call (path current-token > contents) compare ":in-directory"))
(var next-token (* (const Token)) (+ 1 current-token))
(unless (< next-token end-token)
(ErrorAtToken (deref next-token) "expected expression for working directory")
(return false))
(gen-unique-symbol working-dir-str-var "working-dir-str" (deref next-token))
(tokenize-push
specifier-tokens
(var (token-splice-addr working-dir-str-var) (in std string) (token-splice next-token))
;; I thought I needed to make it absolute, but at least on Linux, chdir works with relative
;; TODO: Remove this if Windows is fine with it as well
;; (scope ;; Make the path absolute if necessary
;; (var working-dir-alloc (* (const char)) (makeAbsolutePath_Allocated null (token-splice next-token)))
;; (unless working-dir-alloc
;; (Logf "error: could not find expected directory %s" (token-splice next-token))
;; (return false))
;; ;; Copy it so we don't need to worry about freeing if something goes wrong
;; (set (token-splice-addr working-dir-str-var) working-dir-alloc)
;; (free (type-cast working-dir-alloc (* void))))
(set (field (token-splice arguments-out-name) working-directory)
(on-call (token-splice-addr working-dir-str-var) c_str)))
;; Absorb src for incr
(set current-token next-token))
(true
(ErrorAtToken (deref current-token) "unrecognized specifier. Valid specifiers: :in-directory")
(return false))))
;; Everything else is a argument to the command
(true
(on-call command-arguments push_back (deref current-token))))
(incr current-token)))
(gen-unique-symbol resolved-executable-var "resolved-executable" (deref executable-name))
(gen-unique-symbol command-array-var "command-arguments"
(? arguments (deref arguments) (deref executable-name)))
(tokenize-push
output
(var (token-splice arguments-out-name) RunProcessArguments (array 0))
(var (token-splice-addr resolved-executable-var) ([] MAX_PATH_LENGTH char) (array 0))
(unless (resolveExecutablePath (token-splice executable-name)
(token-splice-addr resolved-executable-var)
(sizeof (token-splice-addr resolved-executable-var)))
(Logf "error: failed to resolved executable %s. Is it installed?\\n"
(token-splice executable-name))
(return false))
(set (field (token-splice arguments-out-name) fileToExecute)
(token-splice-addr resolved-executable-var))
(token-splice-array specifier-tokens)
(var (token-splice-addr command-array-var) ([] (* (const char)))
(array (token-splice-addr resolved-executable-var)
(token-splice-array command-arguments) null))
(set (field (token-splice arguments-out-name) arguments)
(token-splice-addr command-array-var)))
(return true))

+ 44
- 2
src/Build.cpp View File

@ -1,5 +1,6 @@
#include "Build.hpp"
#include "FileUtilities.hpp"
#include "Logging.hpp"
#include "Utilities.hpp"
@ -126,8 +127,49 @@ bool resolveExecutablePath(const char* fileToExecute, char* resolvedPathOut,
}
#endif
// Default is to just keep it as-is. This may make sense to bolster in the future via using
// search paths, e.g. remove the need to specify /usr/bin
#ifdef UNIX
if (fileToExecute[0] == '/' || fileToExecute[0] == '.')
{
SafeSnprintf(resolvedPathOut, resolvedPathOutSize, "%s", fileToExecute);
if (!fileExists(resolvedPathOut))
{
Logf(
"error: failed to find '%s'. Checked exact path because it is directory-relative "
"or absolute\n",
fileToExecute);
return false;
}
}
else
{
// TODO: Use PATH environment variable instead
const char* pathsToCheck[] = {"/usr/local/sbin", "/usr/local/bin", "/usr/sbin",
"/usr/bin", "/sbin", "/bin"};
bool found = false;
for (int i = 0; i < ArraySize(pathsToCheck); ++i)
{
SafeSnprintf(resolvedPathOut, resolvedPathOutSize, "%s/%s", pathsToCheck[i],
fileToExecute);
if (fileExists(resolvedPathOut))
{
found = true;
break;
}
}
if (!found)
{
Logf("error: failed to find '%s'. Checked the following paths:\n", fileToExecute);
for (int i = 0; i < ArraySize(pathsToCheck); ++i)
Logf("\t%s\n", pathsToCheck[i]);
return false;
}
}
return true;
#endif
SafeSnprintf(resolvedPathOut, resolvedPathOutSize, "%s", fileToExecute);
return true;
}

+ 2
- 2
src/Evaluator.cpp View File

@ -311,7 +311,7 @@ bool HandleInvocation_Recursive(EvaluatorEnvironment& environment, const Evaluat
NoteAtToken(invocationStart,
"code was generated from macro. See erroneous macro "
"expansion below:");
printTokens(*macroOutputTokens);
prettyPrintTokens(*macroOutputTokens);
Log("\n");
// Deleting these tokens is only safe at this point because we know we have not
// evaluated them. As soon as they are evaluated, they must be kept around
@ -351,7 +351,7 @@ bool HandleInvocation_Recursive(EvaluatorEnvironment& environment, const Evaluat
{
NoteAtToken(invocationStart,
"code was generated from macro. See macro expansion below:");
printTokens(*macroOutputTokens);
prettyPrintTokens(*macroOutputTokens);
Log("\n");
return false;
}


+ 5
- 2
src/GeneratorHelpers.cpp View File

@ -934,8 +934,11 @@ bool CStatementOutput(EvaluatorEnvironment& environment, const EvaluatorContext&
bodyContext.scope = EvaluatorScope_ExpressionsOnly;
StringOutput spliceDelimiterTemplate = {};
spliceDelimiterTemplate.output = operation[i].keywordOrSymbol;
addModifierToStringOutput(spliceDelimiterTemplate, StringOutMod_SpaceBefore);
addModifierToStringOutput(spliceDelimiterTemplate, StringOutMod_SpaceAfter);
if (operation[i].type != SpliceNoSpace)
{
addModifierToStringOutput(spliceDelimiterTemplate, StringOutMod_SpaceBefore);
addModifierToStringOutput(spliceDelimiterTemplate, StringOutMod_SpaceAfter);
}
bodyContext.delimiterTemplate = spliceDelimiterTemplate;
int numErrors = EvaluateGenerateAll_Recursive(environment, bodyContext, tokens,
startSpliceListIndex, output);


+ 2
- 1
src/Generators.cpp View File

@ -425,7 +425,8 @@ bool AddCompileTimeHookGenerator(EvaluatorEnvironment& environment, const Evalua
}
ErrorAtToken(tokens[hookNameIndex],
"failed to set hook. Hook name not recognized or context mismatched");
"failed to set hook. Hook name not recognized or context mismatched. Available "
"hooks:\n\tpre-build (module only)\n\tpre-link\n\tpost-references-resolved\n");
return false;
}


+ 1
- 1
src/Main.cpp View File

@ -269,7 +269,7 @@ int main(int numArguments, char* arguments[])
char workingDirectory[MAX_PATH_LENGTH] = {0};
getDirectoryFromPath(arguments.fileToExecute, workingDirectory,
ArraySize(workingDirectory));
arguments.workingDir = workingDirectory;
arguments.workingDirectory = workingDirectory;
int status = 0;
if (runProcess(arguments, &status) != 0)


+ 3
- 2
src/OutputPreambles.cpp View File

@ -8,8 +8,9 @@ void makeCompileTimeHeaderFooter(GeneratorOutput& headerOut, GeneratorOutput& fo
GeneratorOutput* spliceAfterHeaders, const Token* blameToken)
{
const char* defaultIncludes[] = {
"Evaluator.hpp", "EvaluatorEnums.hpp", "Tokenizer.hpp", "GeneratorHelpers.hpp",
"Utilities.hpp", "ModuleManager.hpp", "Converters.hpp"};
"Evaluator.hpp", "EvaluatorEnums.hpp", "Tokenizer.hpp", "GeneratorHelpers.hpp",
"Utilities.hpp", "ModuleManager.hpp", "Converters.hpp", "RunProcess.hpp",
"Build.hpp", "FileUtilities.hpp"};
for (unsigned int i = 0; i < ArraySize(defaultIncludes); ++i)
{


+ 26
- 16
src/RunProcess.cpp View File

@ -64,6 +64,13 @@ void subprocessReceiveStdOut(const char* processOutputBuffer)
int runProcess(const RunProcessArguments& arguments, int* statusOut)
{
if (!arguments.arguments)
{
Log("error: runProcess() called with empty arguments. At a minimum, first argument must be "
"executable name\n");
return 1;
}
if (logging.processes)
{
Log("RunProcess command: ");
@ -122,16 +129,18 @@ int runProcess(const RunProcessArguments& arguments, int* statusOut)
nonConstArguments[numArgs] = nullptr;
}
if (arguments.workingDir)
if (arguments.workingDirectory)
{
if (chdir(arguments.workingDir) != 0)
if (chdir(arguments.workingDirectory) != 0)
{
perror("RunProcess chdir: ");
Logf("error: RunProcess failed to change directory to '%s'\n",
arguments.workingDirectory);
perror("RunProcess chdir");
goto childProcessFailed;
}
if (logging.processes)
Logf("Set working directory to %s\n", arguments.workingDir);
Logf("Set working directory to %s\n", arguments.workingDirectory);
}
systemExecute(arguments.fileToExecute, nonConstArguments);
@ -309,16 +318,16 @@ int runProcess(const RunProcessArguments& arguments, int* statusOut)
ZeroMemory(processInfo, sizeof(PROCESS_INFORMATION));
// Start the child process.
if (!CreateProcess(fileToExecute, // No module name (use command line)
commandLineString, // Command line
nullptr, // No security attributes
nullptr, // Thread handle not inheritable
true, // Set handle inheritance to true
0, // No creation flags
nullptr, // Use parent's environment block
nullptr, // Use parent's starting directory
&startupInfo, // Pointer to STARTUPINFO structure
processInfo)) // Pointer to PROCESS_INFORMATION structure
if (!CreateProcess(fileToExecute, // No module name (use command line)
commandLineString, // Command line
nullptr, // No security attributes
nullptr, // Thread handle not inheritable
true, // Set handle inheritance to true
0, // No creation flags
nullptr, // Use parent's environment block
arguments.workingDirectory, // If nullptr, use parent's starting directory
&startupInfo, // Pointer to STARTUPINFO structure
processInfo)) // Pointer to PROCESS_INFORMATION structure
{
CloseHandle(hChildStd_OUT_Rd);
CloseHandle(hChildStd_OUT_Wr);
@ -387,7 +396,7 @@ void readProcessPipe(Subprocess& process, SubprocessOnOutputFunc onOutput)
break;
}
if (bytesRead <= sizeof(buffer))
if (onOutput && bytesRead <= sizeof(buffer))
onOutput(buffer);
}
@ -415,7 +424,8 @@ void waitForAllProcessesClosed(SubprocessOnOutputFunc onOutput)
{
processOutputBuffer[numBytesRead] = '\0';
subprocessReceiveStdOut(processOutputBuffer);
onOutput(processOutputBuffer);
if (onOutput)
onOutput(processOutputBuffer);
numBytesRead = read(process.pipeReadFileDescriptor, processOutputBuffer,
sizeof(processOutputBuffer));
}


+ 1
- 1
src/RunProcess.hpp View File

@ -9,7 +9,7 @@ struct RunProcessArguments
{
const char* fileToExecute;
// nullptr = no change (use parent process's working dir)
const char* workingDir;
const char* workingDirectory;
const char** arguments;
};


+ 15
- 0
test/BuildHelpers.cake View File

@ -0,0 +1,15 @@
(add-cakelisp-search-directory "runtime")
(import &comptime-only "BuildTools.cake")
(c-import "<stdio.h>")
(defun main (&return int)
(printf "Hello, build tools!\n")
(return 0))
(defun-comptime run-3rd-party-build (manager (& ModuleManager) module (* Module) &return bool)
(run-process-make-arguments list-command "ls" "-lh" :in-directory "src")
(var result bool (= 0 (run-process-wait-for-completion (addr list-command))))
(return result))
(add-compile-time-hook-module pre-build run-3rd-party-build)

Loading…
Cancel
Save