Browse Source

Hot-reloading: State variable implementation

* Register a module-local StateVariable for each variable declared at
module scope
* Automatically change StateVariable types to pointers
* Automatically change references to StateVariables to pointer
  dereferences
* Add NoEvalVariableGenerator for when references to state variables
need to access the state variable address
* Add token-splice-ref for ease of use (pairs well with quick-token-at)
* Add Converters.hpp to macro heading
* Fix PushBackTokenExpression not exiting at the final parenthesis of
the expression
* Update Debugging documentation to use cakelisp_cache folder
* Wrote hot-reload-make-state-variable-initializer and associated
experimentation for initializing StateVariables
* Experimented with using sizeof on arrays defined in different
  modules
HotReloadingState
Macoy Madson 5 months ago
parent
commit
13595f1808
10 changed files with 262 additions and 65 deletions
  1. +1
    -1
      doc/Debugging.org
  2. +6
    -0
      runtime/ArrayTest.cake
  3. +54
    -5
      runtime/HotReloading.cake
  4. +1
    -1
      runtime/Jamfile
  5. +62
    -18
      runtime/TextAdventure.cake
  6. +34
    -4
      src/Evaluator.cpp
  7. +8
    -4
      src/GeneratorHelpers.cpp
  8. +1
    -0
      src/GeneratorHelpers.hpp
  9. +93
    -31
      src/Generators.cpp
  10. +2
    -1
      src/OutputPreambles.cpp

+ 1
- 1
doc/Debugging.org View File

@ -7,7 +7,7 @@ Run ~cakelisp --help~ to see what command-line arguments may be passed in to con
The following command may be run in order to tell GDB where the ~.so~ files you want to debug are located:
#+BEGIN_SRC sh
set solib-search-path ~/Development/code/repositories/cakelisp/
set solib-search-path ~/Development/code/repositories/cakelisp/cakelisp_cache
#+END_SRC
(adjust that path to where you installed cakelisp, of course).


+ 6
- 0
runtime/ArrayTest.cake View File

@ -0,0 +1,6 @@
(import "TextAdventure.cake")
(c-import "<stdio.h>")
(defun test ()
(printf "%lu array\n" (sizeof rooms)))

+ 54
- 5
runtime/HotReloading.cake View File

@ -10,7 +10,7 @@
(def-type-alias FunctionReferenceMapIterator (in FunctionReferenceMap iterator))
(def-type-alias FunctionReferenceMapPair (<> std::pair (const std::string) FunctionReferenceArray))
(var registered-functions FunctionReferenceMap)
(var-noreload registered-functions FunctionReferenceMap)
;;
;; Data/state management
@ -18,14 +18,16 @@
(def-type-alias StateVariableMap (<> std::unordered_map std::string (* void)))
(def-type-alias StateVariableMapIterator (in StateVariableMap iterator))
(var registered-state-variables StateVariableMap)
(var-noreload registered-state-variables StateVariableMap)
;;
;; Library management
;;
(var hot-reload-lib-path (* (const char)) "libGeneratedCakelisp.so")
(var hot-reload-lib-init-symbol-name (* (const char)) "libGeneratedCakelisp_initialize")
(var current-lib DynamicLibHandle nullptr)
(var-noreload hot-reload-lib-path (* (const char)) "libGeneratedCakelisp.so")
(var-noreload hot-reload-lib-init-symbol-name (* (const char)) "libGeneratedCakelisp_initialize")
(var-noreload current-lib DynamicLibHandle nullptr)
(var-noreload verbose-variables bool true)
(defun register-function-pointer (function-pointer (* (* void))
function-name (* (const char)))
@ -45,10 +47,15 @@
(var find-it StateVariableMapIterator (on-call registered-state-variables find name))
(unless (!= find-it (on-call registered-state-variables end))
(set variable-address-out nullptr)
(when verbose-variables
(printf "Did not find variable %s\n" name))
(return false))
(when verbose-variables
(printf "Found variable %s at %p\n" name (path find-it > second)))
(set (deref variable-address-out) (path find-it > second))
(return true))
;; TODO: Free variables
(defun hot-reload-register-variable (name (* (const char)) variable-address (* void))
(set (at name registered-state-variables) variable-address))
@ -80,3 +87,45 @@
(for-in function-pointer (* (* void)) (path function-referent-it . second)
(set (deref function-pointer) loaded-symbol)))
(return true))
(defmacro hot-reload-make-state-variable-initializer ()
(destructure-arguments name-index type-index assignment-index)
(quick-token-at name name-index)
(quick-token-at type type-index)
(quick-token-at assignment assignment-index)
(var converted-name-buffer ([] 64 char) (array 0))
;; TODO: Need to pass this in somehow
(var name-style NameStyleSettings)
(lispNameStyleToCNameStyle (field name-style variableNameMode) (on-call (field name contents) c_str)
converted-name-buffer (sizeof converted-name-buffer) name)
(var init-function-name Token name)
(var init-function-name-buffer ([] 256 char) (array 0))
(PrintfBuffer init-function-name-buffer "%sInitialize" converted-name-buffer)
(set (field init-function-name contents) init-function-name-buffer)
(var string-var-name Token name)
(set (field string-var-name type) TokenType_String)
(tokenize-push
output
(defun-local (token-splice-ref init-function-name) ()
(var existing-value (* void) nullptr)
(if (hot-reload-find-variable (token-splice-ref string-var-name) (addr existing-value))
(set (no-eval-var (token-splice-ref name)) (type-cast existing-value (* (token-splice-ref type))))
(block
;; C can have an easier time with plain old malloc and cast
(set (no-eval-var (token-splice-ref name)) (new (token-splice-ref type)))
(set (token-splice-ref name) (token-splice-ref assignment))
(hot-reload-register-variable (token-splice-ref string-var-name)
(no-eval-var (token-splice-ref name)))))))
(return true))
;; (defun-local currentRoomInitialize ()
;; (var existing-value (* void )nullptr )
;; (if (hot-reload-find-variable "current-room" (addr existing-value ))
;; (set (no-eval-var current-room ) (type-cast existing-value (* (* (const room ))nullptr )))
;; (block (set (no-eval-var current-room )(new (* (const room ))nullptr ))
;; (set current-room 0 )
;; (hot-reload-register-variable "current-room" (no-eval-var current-room )))))

+ 1
- 1
runtime/Jamfile View File

@ -11,5 +11,5 @@ HotReloading.cake.cpp
../src/DynamicLoader.cpp
;
Main libGeneratedCakelisp$(SUFSHR) : TextAdventure.cake.cpp ;
Main libGeneratedCakelisp$(SUFSHR) : TextAdventure.cake.cpp ArrayTest.cake.cpp ;
LINKFLAGS on libGeneratedCakelisp$(SUFSHR) = -shared ;

+ 62
- 18
runtime/TextAdventure.cake View File

@ -3,6 +3,7 @@
;; This is to test hot-reloading
(import "HotReloading.cake")
(import "ArrayTest.cake")
(c-import "<stdio.h>"
"cctype" ;; For isdigit
@ -14,33 +15,65 @@
description (* (const char))
connected-rooms (<> std::vector room))
(var rooms (const ([] room))
(global-var rooms ([] 1 room)
(array
(array "front porch" "You're outside the front door of your home. The lights are off inside."
(array (array
"inside home" "Surprise! Your home is filled with cake. You look at your hands. You are cake."
(array))))))
;; TODO: Array
(var-noreload rooms-state (* room) nullptr)
;; TODO: Add no-eval-var
(defun-local rooms-state-init ()
(var existing-value (* void) nullptr)
(if (hot-reload-find-variable "rooms-state" (addr existing-value))
(set rooms-state (type-cast existing-value (* room)))
(block
(set rooms-state (new-array 1 room))
;; Have to handle array initialization ourselves, because the state variable is just a pointer now
(var current-elem int 0)
(while (< current-elem 1)
(set (at current-elem rooms-state)
(array "front porch" "You're outside the front door of your home. The lights are off inside."
(array
(array
"inside home"
"Surprise! Your home is filled with cake. You look at your hands. You are cake."
(array)))))
(incr current-elem))
(hot-reload-register-variable "rooms-state" rooms-state))))
;; This won't work because you must assign a value, but no function except a fancy template
;; function could return any value. Additionally, it may be unwise to rely on static initialization
;; (var static-init-test int (register-init-callback static-init-test-init))
;; The original declaration
;; (var static-init-test int 0)
;; The modified declaration
(var static-init-test (* int) 0)
(var num-times-hot-reloaded int 0)
;; The following two functions will be auto-generated
(defun-local static-init-test-init()
(var existing-value (* void) nullptr)
(if (hot-reload-find-variable "static-init-test" (addr existing-value))
(set static-init-test (type-cast existing-value (* int)))
(block
;; C can have an easier time with plain old malloc and cast
(set static-init-test (new int))
(set (deref static-init-test) 0)
(hot-reload-register-variable "static-init-test" static-init-test))))
;; (defun-local num-times-hot-reloaded-init ()
;; (var existing-value (* void) nullptr)
;; (if (hot-reload-find-variable "num-times-hot-reloaded" (addr existing-value))
;; (set (no-eval-var num-times-hot-reloaded) (type-cast existing-value (* int)))
;; (block
;; ;; C can have an easier time with plain old malloc and cast
;; (set (no-eval-var num-times-hot-reloaded) (new int))
;; (set num-times-hot-reloaded 0)
;; (hot-reload-register-variable "num-times-hot-reloaded" (no-eval-var num-times-hot-reloaded)))))
(hot-reload-make-state-variable-initializer num-times-hot-reloaded int 0)
;; This now makes order matter, because if we move it up we'll get a null pointer exception
(var current-room (* (const room)) nullptr)
(hot-reload-make-state-variable-initializer current-room (* (const room)) nullptr)
(defun libGeneratedCakelisp_initialize ()
(static-init-test-init))
(num-times-hot-reloaded-initialize)
(rooms-state-init)
(current-room-initialize))
(defun print-help ()
(printf "At any point, enter 'r' to reload the code, 'h' for help, or 'q' to quit\n")
@ -57,16 +90,27 @@
(when (on-call (path room-to-print > connected-rooms) empty)
(printf "There are no connected rooms. This must be the end of the game!\n")))
(defstruct TestStruct
a int
b int
c float)
;; Return true to hot-reload, or false to exit
(defun reloadable-entry-point (&return bool)
(printf "CAKE ADVENTURE\n\n")
(print-help)
(unless num-times-hot-reloaded
(printf "CAKE ADVENTURE\n\n")
(print-help))
;; (var my-test-struct ([] 5 TestStruct) (array))
;; (var my-ptr-to-test-struct (* TestStruct) my-test-struct)
;; (printf "Size experiment: %lu %lu %lu\n" (sizeof TestStruct)
;; (sizeof my-test-struct) (sizeof (deref my-ptr-to-test-struct)))
;; (test)
(var static-init-test-ref (& int) (deref static-init-test))
(printf "Num times reloaded: %d\n" static-init-test-ref)
(++ static-init-test-ref)
(printf "Num times reloaded: %d\n" num-times-hot-reloaded)
(++ num-times-hot-reloaded)
(var current-room (* (const room)) (addr (at 0 rooms)))
(print-room current-room)
(var input char 0)


+ 34
- 4
src/Evaluator.cpp View File

@ -39,6 +39,16 @@ MacroFunc findMacro(EvaluatorEnvironment& environment, const char* functionName)
return nullptr;
}
StateVariable* findModuleStateVariable(ModuleEnvironment* moduleEnvironment,
const char* stateVariableName)
{
StateVariableIterator findIt =
moduleEnvironment->stateVariables.find(std::string(stateVariableName));
if (findIt != moduleEnvironment->stateVariables.end())
return &findIt->second;
return nullptr;
}
bool isCompileTimeCodeLoaded(EvaluatorEnvironment& environment, const ObjectDefinition& definition)
{
if (definition.type == ObjectType_CompileTimeMacro)
@ -349,7 +359,7 @@ int EvaluateGenerate_Recursive(EvaluatorEnvironment& environment, const Evaluato
else
{
// The remaining token types evaluate to themselves. Output them directly.
if (ExpectEvaluatorScope("evaluated constant", token, context,
if (ExpectEvaluatorScope("evaluated constant or symbol", token, context,
EvaluatorScope_ExpressionsOnly))
{
switch (token.type)
@ -362,12 +372,32 @@ int EvaluateGenerate_Recursive(EvaluatorEnvironment& environment, const Evaluato
char secondChar = token.contents.size() > 1 ? token.contents[1] : 0;
if (firstChar == '\'' || isdigit(firstChar) ||
(firstChar == '-' && (secondChar == '.' || isdigit(secondChar))))
{
// Add as-is
addStringOutput(output.source, token.contents, StringOutMod_None, &token);
}
else
{
// Potential lisp name. Convert
addStringOutput(output.source, token.contents,
StringOutMod_ConvertVariableName, &token);
if (environment.enableHotReloading && context.moduleEnvironment &&
findModuleStateVariable(context.moduleEnvironment,
token.contents.c_str()))
{
// StateVariables are automatically converted to pointers, so all
// accessing of them needs to happen through a dereference
// Use (no-eval-var my-var) if you need to get around the auto
// dereference (if you are e.g. writing code which sets the pointer)
addLangTokenOutput(output.source, StringOutMod_OpenParen, &token);
addStringOutput(output.source, "*", StringOutMod_None, &token);
addStringOutput(output.source, token.contents,
StringOutMod_ConvertVariableName, &token);
addLangTokenOutput(output.source, StringOutMod_CloseParen, &token);
}
else
{
// Potential lisp name. Convert
addStringOutput(output.source, token.contents,
StringOutMod_ConvertVariableName, &token);
}
}
break;
}


+ 8
- 4
src/GeneratorHelpers.cpp View File

@ -238,17 +238,21 @@ void PushBackTokenExpression(std::vector<Token>& output, const Token* startToken
if (currentToken->type == TokenType_OpenParen)
++depth;
else if (currentToken->type == TokenType_CloseParen)
{
--depth;
if (depth < 0)
break;
}
output.push_back(*currentToken);
if (depth <= 0)
break;
}
}
}
void PushBackTokenExpressionRef(std::vector<Token>& output, const Token& startToken)
{
PushBackTokenExpression(output, &startToken);
}
//
// Outputting
//


+ 1
- 0
src/GeneratorHelpers.hpp View File

@ -57,6 +57,7 @@ int blockAbsorbScope(const std::vector<Token>& tokens, int startBlockIndex);
void MakeUniqueSymbolName(EvaluatorEnvironment& environment, const char* prefix,
Token* tokenToChange);
void PushBackTokenExpression(std::vector<Token>& output, const Token* startToken);
void PushBackTokenExpressionRef(std::vector<Token>& output, const Token& startToken);
void addModifierToStringOutput(StringOutput& operation, StringOutputModifierFlags flag);


+ 93
- 31
src/Generators.cpp View File

@ -373,8 +373,10 @@ bool VariableDeclarationGenerator(EvaluatorEnvironment& environment,
EvaluatorScope_Body))
return false;
bool hotReloadAllowed = funcNameToken.contents.compare("var-noreload") != 0;
// TODO: Eventually, static function variables could be automatically promoted to module scope
if (environment.enableHotReloading && isStaticFunctionVar &&
if (hotReloadAllowed && environment.enableHotReloading && isStaticFunctionVar &&
context.scope == EvaluatorScope_Body)
{
NoteAtToken(funcNameToken,
@ -394,11 +396,48 @@ bool VariableDeclarationGenerator(EvaluatorEnvironment& environment,
std::vector<StringOutput> typeOutput;
std::vector<StringOutput> typeAfterNameOutput;
// Arrays cannot be return types, they must be * instead
if (!tokenizedCTypeToString_Recursive(tokens, typeIndex,
/*allowArray=*/true, typeOutput, typeAfterNameOutput))
return false;
bool isArray = !typeAfterNameOutput.empty();
// Decide whether we should output the variable modified for hot-reloading or not
bool isStateVariable = context.scope == EvaluatorScope_Module;
bool isCustomInitialized = false;
// TODO Only reject arrays for now because they need their own handling
if (hotReloadAllowed && environment.enableHotReloading && isStateVariable && !isArray)
{
if (!context.moduleEnvironment)
{
printf(
"error: context had no module environment set. This is required for hot-reloading. "
"This is an internal error, not an error with the evaluated code\n");
return false;
}
StateVariableTable& stateVariables = context.moduleEnvironment->stateVariables;
StateVariableIterator findIt = stateVariables.find(tokens[varNameIndex].contents);
if (findIt != stateVariables.end())
{
const Token* firstDefinedNameToken = findIt->second.variableName;
ErrorAtTokenf(tokens[varNameIndex], "multiple definitions of state variable '%s'",
firstDefinedNameToken->contents.c_str());
NoteAtToken((*firstDefinedNameToken), "first defined here");
return false;
}
StateVariable newStateVariable = {};
newStateVariable.variableName = &tokens[varNameIndex];
newStateVariable.startType = &tokens[typeIndex];
stateVariables[tokens[varNameIndex].contents] = newStateVariable;
// Modify the type. All variables become pointers to the state variables stored on the heap
addStringOutput(typeOutput, "*", StringOutMod_None, &tokens[typeIndex]);
isCustomInitialized = true;
}
// At this point, we probably have a valid variable. Start outputting
addModifierToStringOutput(typeOutput.back(), StringOutMod_SpaceAfter);
@ -430,7 +469,15 @@ bool VariableDeclarationGenerator(EvaluatorEnvironment& environment,
int valueIndex = getNextArgument(tokens, typeIndex, endInvocationIndex);
// Initialized
if (valueIndex < endInvocationIndex)
if (isCustomInitialized)
{
// All state variables will have an initialization function executed once the module is
// loaded. We null out the handle here
addLangTokenOutput(output.source, StringOutMod_SpaceAfter, &tokens[startTokenIndex]);
addStringOutput(output.source, "=", StringOutMod_SpaceAfter, &tokens[startTokenIndex]);
addStringOutput(output.source, "nullptr", StringOutMod_None, &tokens[startTokenIndex]);
}
else if (valueIndex < endInvocationIndex)
{
addLangTokenOutput(output.source, StringOutMod_SpaceAfter, &tokens[valueIndex]);
addStringOutput(output.source, "=", StringOutMod_SpaceAfter, &tokens[valueIndex]);
@ -446,32 +493,16 @@ bool VariableDeclarationGenerator(EvaluatorEnvironment& environment,
if (isGlobal)
addLangTokenOutput(output.header, StringOutMod_EndStatement, &tokens[endInvocationIndex]);
bool isStateVariable = context.scope == EvaluatorScope_Module;
if (environment.enableHotReloading && isStateVariable)
// Generate initializer function
if (isCustomInitialized)
{
if (!context.moduleEnvironment)
{
printf(
"error: context had no module environment set. This is required for hot-reloading. "
"This is an internal error, not an error with the evaluated code\n");
return false;
}
StateVariableTable& stateVariables = context.moduleEnvironment->stateVariables;
StateVariableIterator findIt = stateVariables.find(tokens[varNameIndex].contents);
if (findIt != stateVariables.end())
{
const Token* firstDefinedNameToken = findIt->second.variableName;
ErrorAtTokenf(tokens[varNameIndex], "multiple definitions of state variable '%s'",
firstDefinedNameToken->contents.c_str());
NoteAtToken((*firstDefinedNameToken), "first defined here");
return false;
}
StateVariable newStateVariable = {};
newStateVariable.variableName = &tokens[varNameIndex];
newStateVariable.startType = &tokens[typeIndex];
stateVariables[tokens[varNameIndex].contents] = newStateVariable;
// const char* initializerTemplate =
// "static void %sInit()\n{\n\tvoid* existingValue = nullptr;\n\tif "
// "(hotReloadFindVariable(\"%s\", &existingValue))\n\t{\n\t\t%s = "
// "((%s)existingValue);\n\t}\n\telse\n\t{\n\t\t%s = new %s;\n\t\t(*%s)= "
// "%s;\n\t\thotReloadRegisterVariable(\"%s\", %s);\n\t}\n}";
// char initializerBuffer[1024] = {0};
// PrintfBuffer(initializerBuffer, );
}
return true;
@ -1014,6 +1045,24 @@ bool ObjectPathGenerator(EvaluatorEnvironment& environment, const EvaluatorConte
return true;
}
bool NoEvalVariableGenerator(EvaluatorEnvironment& environment, const EvaluatorContext& context,
const std::vector<Token>& tokens, int startTokenIndex,
GeneratorOutput& output)
{
int endInvocationIndex = FindCloseParenTokenIndex(tokens, startTokenIndex);
int symbolIndex = getExpectedArgument("expected variable name", tokens, startTokenIndex, 1,
endInvocationIndex);
if (symbolIndex == -1 ||
!ExpectTokenType("variable name", tokens[symbolIndex], TokenType_Symbol))
return false;
addStringOutput(output.source, tokens[symbolIndex].contents, StringOutMod_ConvertVariableName,
&tokens[symbolIndex]);
return true;
}
// I'm not too happy with this
static void tokenizeGenerateStringTokenize(const char* outputVarName, const Token& triggerToken,
const char* stringToTokenize, GeneratorOutput& output)
@ -1075,10 +1124,12 @@ bool TokenizePushGenerator(EvaluatorEnvironment& environment, const EvaluatorCon
const Token& nextToken = tokens[i + 1];
if (currentToken.type == TokenType_OpenParen && nextToken.type == TokenType_Symbol &&
(nextToken.contents.compare("token-splice") == 0 ||
nextToken.contents.compare("token-splice-ref") == 0 ||
nextToken.contents.compare("token-splice-array") == 0))
{
// TODO: Performance: remove extra string compare
bool isArray = nextToken.contents.compare("token-splice-array") == 0;
bool isRef = nextToken.contents.compare("token-splice-ref") == 0;
if (tokenToStringWrite != tokenToStringBuffer)
{
@ -1096,9 +1147,14 @@ bool TokenizePushGenerator(EvaluatorEnvironment& environment, const EvaluatorCon
for (int spliceArg = startSpliceArgs; spliceArg < endSpliceIndex;
spliceArg = getNextArgument(tokens, spliceArg, endSpliceIndex))
{
addStringOutput(output.source,
isArray ? "PushBackAll(" : "PushBackTokenExpression(",
StringOutMod_None, &tokens[spliceArg]);
const char* functionToCall = "PushBackTokenExpression(";
if (isRef)
functionToCall = "PushBackTokenExpressionRef(";
else if (isArray)
functionToCall = "PushBackAll(";
addStringOutput(output.source, functionToCall, StringOutMod_None,
&tokens[spliceArg]);
// Write output argument
addStringOutput(output.source, evaluateOutputTempVar.contents, StringOutMod_None,
@ -1653,6 +1709,12 @@ void importFundamentalGenerators(EvaluatorEnvironment& environment)
environment.generators["var"] = VariableDeclarationGenerator;
environment.generators["global-var"] = VariableDeclarationGenerator;
environment.generators["static-var"] = VariableDeclarationGenerator;
environment.generators["var-noreload"] = VariableDeclarationGenerator;
// Special case: Output the symbol without doing any additionall processing (e.g. StateVariable)
// This is only necessary for writing code internal to hot-reloading, or if you're going to do
// some weird stuff with state variables
environment.generators["no-eval-var"] = NoEvalVariableGenerator;
environment.generators["at"] = ArrayAccessGenerator;
environment.generators["nth"] = ArrayAccessGenerator;


+ 2
- 1
src/OutputPreambles.cpp View File

@ -2,7 +2,8 @@
// Must use extern "C" for dynamic symbols, because otherwise name mangling makes things hard
const char* macroSourceHeading =
"#include \"Evaluator.hpp\""
"#include \"Converters.hpp\""
"\n#include \"Evaluator.hpp\""
"\n#include \"EvaluatorEnums.hpp\""
"\n#include \"Tokenizer.hpp\""
"\n#include \"GeneratorHelpers.hpp\""


Loading…
Cancel
Save