Browse Source

Fix Windows build of DynamicLoader

* Added FileUtilities.cake for some helper functions
* Added PushBackAllTokenExpressions()
* Changed appendTokenToString a bit, but didn't actually need to
* Update tokenize-push documentation
build-system-unification
Macoy Madson 6 months ago
parent
commit
95933be0ed
11 changed files with 214 additions and 37 deletions
  1. +1
    -1
      .gitignore
  2. +1
    -1
      BuildAndRunTests.sh
  3. +3
    -1
      doc/Cakelisp.org
  4. +6
    -20
      runtime/DynamicLoader.cake
  5. +101
    -0
      runtime/FileUtilities.cake
  6. +5
    -1
      runtime/HotLoader.cake
  7. +26
    -0
      src/GeneratorHelpers.cpp
  8. +7
    -0
      src/GeneratorHelpers.hpp
  9. +40
    -5
      src/Generators.cpp
  10. +23
    -3
      src/Tokenizer.cpp
  11. +1
    -5
      test/BuildOptions.cake

+ 1
- 1
.gitignore View File

@ -34,7 +34,7 @@ src/dynamicLoadTest
src/runProcessTest
src/dependencyTest
runtime/HotReloadingTest
hot_loader
hot_loader*
test/ExecuteMe
test/MultiLineStrings


+ 1
- 1
BuildAndRunTests.sh View File

@ -8,7 +8,7 @@
# clang -g -fPIC -x c++-header src/Evaluator.hpp -o src/Evaluator.hpp.pch
./bin/cakelisp --execute \
test/CodeModification.cake
test/CodeModification.cake
./bin/cakelisp \
test/BuildOptions.cake


+ 3
- 1
doc/Cakelisp.org View File

@ -345,9 +345,11 @@ Macros must return ~true~ or ~false~ to denote whether the expansion was success
(var my-tokens (<> std::vector Token))
#+END_SRC
~tokenize-push~ treats all tokens as strings until it reaches one of the ~token*~ functions. These functions tell the tokenizer to unpack and insert the tokens in the variables rather than the symbol which is the variable name.
~tokenize-push~ treats all tokens as strings until it reaches one of the ~token*~ functions. These functions tell the tokenizer to unpack and insert the tokens in the variables rather than the symbol which is the variable name. Unless otherwise specified, these take any number of arguments:
- ~token-splice~: Given a token's address, insert a copy of that token. If the token is an open parenthesis, insert the whole expression (go until the closing parenthesis is found)
- ~token-splice-addr~: Like ~token-splice~, only it automatically takes the address of the given arguments
- ~token-splice-array~: Given an array of tokens, insert a copy of all tokens in the array
- ~token-splice-rest~: Given a token's address and token's source array (usually ~tokens~), output all expressions. It stops once a closing parenthesis is reached that wasn't counted, or the end of the source array is reached. /Accepts only one token argument/
The following is an example of ~tokenize-push~:
#+BEGIN_SRC lisp


+ 6
- 20
runtime/DynamicLoader.cake View File

@ -3,11 +3,12 @@
;; MIT, so this copy is justified (this comment is by Macoy Madson, so I can change the license)
(c-import "<stdio.h>" "<string>" "<unordered_map>")
(import "FileUtilities.cake")
(comptime-cond
('Unix
(c-import "<dlfcn.h>"))
('Windows
;; TODO: This is pretty annoying, because I'll need to add &with-decls etc.
(c-preprocessor-define WIN32_LEAN_AND_MEAN)
(c-import "<windows.h>"))
(true
@ -16,31 +17,16 @@
"This module requires platform-specific code. Please define your platform before importing" \
" this module, e.g.: (comptime-define-symbol 'Unix). Supported platforms: 'Unix, 'Windows")))
(c-preprocessor-define MAX_PATH_LENGTH 256)
(def-type-alias-global DynamicLibHandle (* void))
(c-preprocessor-define MAX_PATH_LENGTH 256)
(defstruct DynamicLibrary
handle DynamicLibHandle)
(def-type-alias DynamicLibraryMap (<> std::unordered_map std::string DynamicLibrary))
(var dynamicLibraries DynamicLibraryMap)
(defun-local make-backslash-filename (buffer (* char) bufferSize
int filename (* (const char)))
(var bufferWrite (* char) buffer)
(var currentChar (* (const char)) filename)
(while (deref currentChar)
(if (= (deref currentChar) '/')
(set (deref bufferWrite) '\\')
(set (deref bufferWrite) (deref currentChar)))
(incr bufferWrite)
(incr currentChar)
(when (>= (- bufferWrite buffer) bufferSize)
(fprintf stderr "error: could not make safe filename: buffer too small\n")
(break))))
;; allow-global-linking = Allow subsequently loaded libraries to resolve from this library You do
;; NOT want this if you intend to reload the library, because it may resolve to the old version
(defun dynamic-library-load (libraryPath (* (const char))
@ -69,12 +55,12 @@
;; TODO Clean this up! Only the cakelispBin is necessary I think (need to double check that)
;; TODO Clear added dirs after? (RemoveDllDirectory())
(var absoluteLibPath (* (const char))
(makeAbsolutePath_Allocated null libraryPath))
(make-absolute-path-allocated null libraryPath))
(var convertedPath ([] MAX_PATH_LENGTH char) (array 0))
;; TODO Remove, redundant with makeAbsolutePath_Allocated()
(make-backslash-filename convertedPath (sizeof convertedPath) absoluteLibPath)
(var dllDirectory ([] MAX_PATH_LENGTH char) (array 0))
(getDirectoryFromPath convertedPath dllDirectory (sizeof dllDirectory))
(get-directory-from-path convertedPath dllDirectory (sizeof dllDirectory))
(scope ;; DLL directory
(var wchars_num int (MultiByteToWideChar CP_UTF8 0 dllDirectory -1 null 0))
(var wstrDllDirectory (* wchar_t) (new-array wchars_num wchar_t))


+ 101
- 0
runtime/FileUtilities.cake View File

@ -0,0 +1,101 @@
(c-import "<stdio.h>" "<string>")
(c-preprocessor-define MAX_PATH_LENGTH 256)
(comptime-cond
('Unix
(c-import "<stdlib.h>" "<libgen.h>" "<string.h>"))
('Windows
(c-preprocessor-define WIN32_LEAN_AND_MEAN)
(c-import "<windows.h>"))
(true
;; If you're hitting this, you may need to port this over to whatever new platform you are on
(comptime-error
"This module requires platform-specific code. Please define your platform before importing" \
" this module, e.g.: (comptime-define-symbol 'Unix). Supported platforms: 'Unix, 'Windows")))
(defun make-absolute-path-allocated (fromDirectory (* (const char)) filePath (* (const char))
&return (* (const char)))
(comptime-cond
('Unix
;; Second condition allows for absolute paths
(if (and fromDirectory (!= (at 0 filePath) '/'))
(block
(var relativePath ([] MAX_PATH_LENGTH char) (array 0))
(safe-string-print relativePath (sizeof relativePath) "%s/%s" fromDirectory filePath)
(return (realpath relativePath null)))
(block
;; The path will be relative to the binary's working directory
(return (realpath filePath null)))))
('Windows
(var absolutePath (* char) (type-cast (calloc MAX_PATH_LENGTH (sizeof char)) (* char)))
(var isValid bool false)
(if fromDirectory
(block
(var relativePath ([] MAX_PATH_LENGTH char) (array 0))
(safe-string-print relativePath (sizeof relativePath) "%s/%s" fromDirectory filePath)
(set isValid (_fullpath absolutePath relativePath MAX_PATH_LENGTH)))
(set isValid (_fullpath absolutePath filePath MAX_PATH_LENGTH)))
(unless isValid
(free absolutePath)
(return null))
(return absolutePath))
(true
(comptime-error "Need to be able to normalize path on this platform")
(return null))))
(defun get-directory-from-path (path (* (const char)) bufferOut (* char) bufferSize int)
(comptime-cond
('Unix
(var pathCopy (* char) (string-duplicate path))
(var dirName (* (const char)) (dirname pathCopy))
(safe-string-print bufferOut bufferSize "%s" dirName)
(free pathCopy))
('Windows
(var drive ([] _MAX_DRIVE char))
(var dir ([] _MAX_DIR char))
;; char fname[_MAX_FNAME];
;; char ext[_MAX_EXT];
(_splitpath_s path drive (sizeof drive) dir (sizeof dir)
null 0 ;; fname
null 0) ;; extension
(_makepath_s bufferOut bufferSize drive dir
null ;; fname
null)) ;; extension
(true
(comptime-error "Need to be able to strip path on this platform"))))
(defun make-backslash-filename (buffer (* char) bufferSize
int filename (* (const char)))
(var bufferWrite (* char) buffer)
(var currentChar (* (const char)) filename)
(while (deref currentChar)
(if (= (deref currentChar) '/')
(set (deref bufferWrite) '\\')
(set (deref bufferWrite) (deref currentChar)))
(incr bufferWrite)
(incr currentChar)
(when (>= (- bufferWrite buffer) bufferSize)
(fprintf stderr "error: could not make safe filename: buffer too small\n")
(break))))
(defmacro safe-string-print (buffer any size any
format string &rest args any)
(tokenize-push output
(scope
(var num-printed int (snprintf (token-splice buffer size format)
(token-splice-rest args tokens)))
(set (at num-printed (token-splice buffer)) '\\0')))
(return true))
(defmacro string-duplicate (string-to-dup any)
(comptime-cond
('Unix
(tokenize-push output
(strdup (token-splice string-to-dup))))
('Windows
(tokenize-push output
(_strdup (token-splice string-to-dup)))))
(return true))

+ 5
- 1
runtime/HotLoader.cake View File

@ -24,4 +24,8 @@
(hot-reload-clean-up)
(return 0))
(set-cakelisp-option executable-output "hot_loader")
(comptime-cond
('Unix
(set-cakelisp-option executable-output "hot_loader"))
('Windows
(set-cakelisp-option executable-output "hot_loader.exe")))

+ 26
- 0
src/GeneratorHelpers.cpp View File

@ -368,6 +368,32 @@ void PushBackTokenExpression(std::vector<Token>& output, const Token* startToken
}
}
void PushBackAllTokenExpressions(std::vector<Token>& output, const Token* startToken,
const Token* finalToken)
{
if (!startToken)
{
Log("error: PushBackTokenExpression() received null token\n");
return;
}
int depth = 0;
for (const Token* currentToken = startToken; depth >= 0; ++currentToken)
{
if (currentToken->type == TokenType_OpenParen)
++depth;
else if (currentToken->type == TokenType_CloseParen)
--depth;
if (depth < 0)
break;
output.push_back(*currentToken);
if (currentToken == finalToken)
break;
}
}
//
// Outputting
//


+ 7
- 0
src/GeneratorHelpers.hpp View File

@ -78,6 +78,13 @@ void MakeContextUniqueSymbolName(EvaluatorEnvironment& environment, const Evalua
CAKELISP_API void PushBackTokenExpression(std::vector<Token>& output, const Token* startToken);
// e.g. output a whole block of expressions, stopping once we reach a closing paren that doesn't
// match any opening parens since startToken, or once finalToken is reached (which will be included
// if it isn't the extra closing paren). finalToken is only for the case where the expressions are
// not wrapped in a block; you should be fine to set it to the end of the tokens array
CAKELISP_API void PushBackAllTokenExpressions(std::vector<Token>& output, const Token* startToken,
const Token* finalToken);
void addModifierToStringOutput(StringOutput& operation, StringOutputModifierFlags flag);
void addStringOutput(std::vector<StringOutput>& output, const std::string& symbol,


+ 40
- 5
src/Generators.cpp View File

@ -2406,10 +2406,12 @@ bool TokenizePushGenerator(EvaluatorEnvironment& environment, const EvaluatorCon
if (currentToken.type == TokenType_OpenParen && nextToken.type == TokenType_Symbol &&
(nextToken.contents.compare("token-splice") == 0 ||
nextToken.contents.compare("token-splice-addr") == 0 ||
nextToken.contents.compare("token-splice-array") == 0))
nextToken.contents.compare("token-splice-array") == 0 ||
nextToken.contents.compare("token-splice-rest") == 0))
{
// TODO: Performance: remove extra string compare
// TODO: Performance: remove extra string compares
bool isArray = nextToken.contents.compare("token-splice-array") == 0;
bool isRest = nextToken.contents.compare("token-splice-rest") == 0;
bool tokenMakePointer = nextToken.contents.compare("token-splice-addr") == 0;
if (tokenToStringWrite != tokenToStringBuffer)
@ -2428,9 +2430,15 @@ bool TokenizePushGenerator(EvaluatorEnvironment& environment, const EvaluatorCon
for (int spliceArg = startSpliceArgs; spliceArg < endSpliceIndex;
spliceArg = getNextArgument(tokens, spliceArg, endSpliceIndex))
{
addStringOutput(output.source,
isArray ? "PushBackAll(" : "PushBackTokenExpression(",
StringOutMod_None, &tokens[spliceArg]);
if (isArray)
addStringOutput(output.source, "PushBackAll(", StringOutMod_None,
&tokens[spliceArg]);
else if (isRest)
addStringOutput(output.source, "PushBackAllTokenExpressions(",
StringOutMod_None, &tokens[spliceArg]);
else
addStringOutput(output.source, "PushBackTokenExpression(", StringOutMod_None,
&tokens[spliceArg]);
// Write output argument
addStringOutput(output.source, evaluateOutputTempVar.contents, StringOutMod_None,
@ -2448,8 +2456,35 @@ bool TokenizePushGenerator(EvaluatorEnvironment& environment, const EvaluatorCon
output) != 0)
return false;
// Second argument is tokens array. Need to get it for bounds check
bool shouldBreak = false;
if (isRest)
{
int tokenArrayArg =
getExpectedArgument("token-splice-rest array which holds given token",
tokens, i, 2, endSpliceIndex);
if (tokenArrayArg == -1)
return false;
addLangTokenOutput(output.source, StringOutMod_ListSeparator,
&tokens[spliceArg]);
addStringOutput(output.source, "&(", StringOutMod_None, &tokens[spliceArg]);
EvaluatorContext expressionContext = context;
expressionContext.scope = EvaluatorScope_ExpressionsOnly;
if (EvaluateGenerate_Recursive(environment, expressionContext, tokens,
tokenArrayArg, output) != 0)
return false;
addStringOutput(output.source, ".back())", StringOutMod_None,
&tokens[spliceArg]);
shouldBreak = true;
}
addLangTokenOutput(output.source, StringOutMod_CloseParen, &tokens[spliceArg]);
addLangTokenOutput(output.source, StringOutMod_EndStatement, &tokens[spliceArg]);
if (shouldBreak)
break;
}
// Finished splice list


+ 23
- 3
src/Tokenizer.cpp View File

@ -336,28 +336,48 @@ bool validateTokens(const std::vector<Token>& tokens)
bool appendTokenToString(const Token& token, char** at, char* bufferStart, int bufferSize)
{
char previousCharacter = 0;
if (*at != bufferStart)
previousCharacter = *(*at - 1);
switch (token.type)
{
case TokenType_OpenParen:
if (previousCharacter != '(' && previousCharacter != 0)
{
if (!writeCharToBufferErrorToken(' ', at, bufferStart, bufferSize, token))
return false;
}
return writeCharToBufferErrorToken('(', at, bufferStart, bufferSize, token);
case TokenType_CloseParen:
return writeCharToBufferErrorToken(')', at, bufferStart, bufferSize, token);
case TokenType_Symbol:
// Need space after paren for symbols
if (previousCharacter != '(' && previousCharacter != ' ' && previousCharacter != 0)
{
if (!writeCharToBufferErrorToken(' ', at, bufferStart, bufferSize, token))
return false;
}
if (!writeStringToBufferErrorToken(token.contents.c_str(), at, bufferStart, bufferSize,
token))
return false;
// Must add space after symbol to separate it from everything else
return writeCharToBufferErrorToken(' ', at, bufferStart, bufferSize, token);
case TokenType_String:
// Need space after paren for strings
if (previousCharacter != '(' && previousCharacter != ' ' && previousCharacter != 0)
{
if (!writeCharToBufferErrorToken(' ', at, bufferStart, bufferSize, token))
return false;
}
if (!writeStringToBufferErrorToken("\\\"", at, bufferStart, bufferSize, token))
return false;
// TODO Need to delimit quotes properly (try e.g. "test\"" and see it break)
if (!writeStringToBufferErrorToken(token.contents.c_str(), at, bufferStart, bufferSize,
token))
return false;
// Must add space after string to separate it from everything else
return writeStringToBufferErrorToken("\\\" ", at, bufferStart, bufferSize, token);
break;
default:
ErrorAtToken(token, "cannot append token of this type to string");
return false;


+ 1
- 5
test/BuildOptions.cake View File

@ -1,12 +1,8 @@
(c-import "Utilities.hpp")
(defstruct-local test-array-type
commands ([] int))
(var my-list-of-lists ([] test-array-type) (array (array 1 2 3) (array 4 5 6)))
(defun main (&return int)
(return 0))
(add-c-search-directory "src" "src" "notsrc")
(add-c-search-directory module "src" "notsrc")
(add-build-options "-Wall" "-Wextra" "-Wno-unused-parameter" "-O0")

Loading…
Cancel
Save