Added WIP array state variable initializer

It still doesn't work, but I made good progress.

* Added prettyPrintTokens for slightly more readable token output
* Update readme and gitignore for ignore instructions

.gitignore

@@ -35,11 +35,8 @@ src/runProcessTest
 src/dependencyTest
 runtime/HotReloadingTest
 
-*.cake.cpp
-*.cake.hpp
-
-libcomptime*
-comptime_*
+*.cake.*
+cakelisp_cache/
 
 bin/
 lib/

ReadMe.org

@@ -72,6 +72,14 @@ Open ~.cake~ files in ~lisp-mode~:
 #+BEGIN_SRC lisp
 (add-to-list 'auto-mode-alist '("\\.cake?\\'" . lisp-mode))
 #+END_SRC
+** ~.gitignore~
+Add the following:
+#+BEGIN_SRC sh
+*.cake.*
+cakelisp_cache/
+#+END_SRC
+
+That will ignore your project's generated files as well as files generated for compile-time execution.
 ** Build systems
 A build system will work fine with Cakelisp, because Cakelisp outputs C/C++ source/header files. Note that Cakelisp is expected to be run before your regular build system runs, or in a stage where Cakelisp can create and add files to the build. This is because Cakelisp handles its own modules such that adding support to an existing build system would be challenging.
 ** Debugging

runtime/HotReloading.cake

@@ -88,25 +88,32 @@
                   (set (deref function-pointer) loaded-symbol)))
   (return true))
 
-(defmacro hot-reload-make-state-variable-initializer ()
-  (destructure-arguments name-index type-index assignment-index)
-  (quick-token-at name name-index)
-  (quick-token-at type type-index)
-  (quick-token-at assignment assignment-index)
-
-  (var converted-name-buffer ([] 64 char) (array 0))
-  ;; TODO: Need to pass this in somehow
-  (var name-style NameStyleSettings)
-  (lispNameStyleToCNameStyle (field name-style variableNameMode) (on-call (field name contents) c_str)
-                             converted-name-buffer (sizeof converted-name-buffer) name)
-
-  (var init-function-name Token name)
-  (var init-function-name-buffer ([] 256 char) (array 0))
-  (PrintfBuffer init-function-name-buffer "%sInitialize" converted-name-buffer)
-  (set (field init-function-name contents) init-function-name-buffer)
+(defmacro state-variable-initializer-preamble ()
+  (tokenize-push
+   output
+   (destructure-arguments name-index type-index assignment-index)
+   (quick-token-at name name-index)
+   (quick-token-at type type-index)
+   (quick-token-at assignment assignment-index)
+
+   (var init-function-name Token name)
+   (var string-var-name Token name)
+   (scope
+    (var converted-name-buffer ([] 64 char) (array 0))
+    ;; TODO: Need to pass this in somehow
+    (var name-style NameStyleSettings)
+    (lispNameStyleToCNameStyle (field name-style variableNameMode) (on-call (field name contents) c_str)
+                               converted-name-buffer (sizeof converted-name-buffer) name)
+
+    (var init-function-name-buffer ([] 256 char) (array 0))
+    (PrintfBuffer init-function-name-buffer "%sInitialize" converted-name-buffer)
+    (set (field init-function-name contents) init-function-name-buffer)
+
+    (set (field string-var-name type) TokenType_String)))
+  (return true))
 
-  (var string-var-name Token name)
-  (set (field string-var-name type) TokenType_String)
+(defmacro hot-reload-make-state-variable-initializer ()
+  (state-variable-initializer-preamble)
 
   (tokenize-push
    output
@@ -122,10 +129,58 @@
                                          (no-eval-var (token-splice-ref name)))))))
   (return true))
 
-;; (defun-local currentRoomInitialize ()
-;;   (var existing-value (* void )nullptr )
-;;   (if (hot-reload-find-variable "current-room" (addr existing-value ))
-;;       (set (no-eval-var current-room ) (type-cast existing-value (* (* (const room ))nullptr )))
-;;       (block (set (no-eval-var current-room )(new (* (const room ))nullptr ))
-;;         (set current-room 0 )
-;;         (hot-reload-register-variable "current-room" (no-eval-var current-room )))))
+;; Arrays must be handled specially because we need to know how big the array is to allocate it on
+;; the heap. We also need to declare a way to get the array size (we don't know with the heap array)
+(defmacro hot-reload-make-state-variable-array-initializer ()
+  (state-variable-initializer-preamble)
+
+  (unless (ExpectTokenType "hot-reload-make-state-variable-array-initializer"
+                          type TokenType_OpenParen)
+          (return false))
+
+  (unless (= 0 (on-call (field (at (+ 1 type-index) tokens) contents) compare "[]"))
+    (ErrorAtToken (at (+ 1 type-index) tokens)
+                  "expected array type. Use hot-reload-make-state-variable-initializer instead")
+    (return false))
+
+  (var end-type-index int (FindCloseParenTokenIndex tokens type-index))
+  (var last-argument-arg-index int (- (getNumArguments tokens type-index end-type-index) 1))
+  (var last-argument-token-index int (getExpectedArgument "expected type"
+                                      tokens type-index last-argument-arg-index end-type-index))
+  (when (= -1 last-argument-token-index)
+    (return false))
+  (var type-strip-array (& (const Token)) (at last-argument-token-index tokens))
+
+  (var array-length-name Token name)
+  (var array-length-name-buffer ([] 256 char) (array 0))
+  (PrintfBuffer array-length-name-buffer "%s-length" (on-call (field name contents) c_str))
+  (set (field array-length-name contents) array-length-name-buffer)
+
+  (var array-length-token Token assignment)
+  (set (field array-length-token type) TokenType_Symbol)
+  (var array-length int (- (getNumArguments tokens assignment-index
+                                            (FindCloseParenTokenIndex tokens assignment-index))
+                           1))
+  (set (field array-length-token contents) (call (in std to_string) array-length))
+
+  (tokenize-push
+   output
+   (var-noreload (token-splice-ref array-length-name) (const int) (token-splice-ref array-length-token))
+   (defun-local (token-splice-ref init-function-name) ()
+     (var existing-value (* void) nullptr)
+     (if (hot-reload-find-variable (token-splice-ref string-var-name) (addr existing-value))
+         (set (no-eval-var (token-splice-ref name)) (type-cast existing-value (* (token-splice-ref type-strip-array))))
+         (block
+             ;; C can have an easier time with plain old malloc and cast
+             (set (no-eval-var (token-splice-ref name)) (new-array (token-splice-ref array-length-name)
+                                                                   (token-splice-ref type-strip-array)))
+           (var static-intializer-full-array (const (token-splice-ref type)) (token-splice-ref assignment))
+           ;; Have to handle array initialization ourselves, because the state variable is just a pointer now
+           (var current-elem int 0)
+           (while (< current-elem (token-splice-ref array-length-name))
+             (set (at current-elem (no-eval-var (token-splice-ref name)))
+                  (at current-elem static-intializer-full-array))
+             (incr current-elem))
+           (hot-reload-register-variable (token-splice-ref string-var-name)
+                                         (no-eval-var (token-splice-ref name)))))))
+  (return true))

runtime/Macros.cake

@@ -71,6 +71,7 @@
   (quick-token-at array-token array-index)
   ;; This should evaluate its argument, but I'm just hacking it in right now anyways
   (unless (ExpectTokenType "array-size" array-token TokenType_Symbol)
+    (ErrorAtToken array-token "if you're seeing this, you should update array-size to evaluate its argument")
     (return false))
   (tokenize-push output (/ (sizeof (token-splice (addr array-token)))
                            (sizeof (at 0 (token-splice (addr array-token))))))

runtime/TextAdventure.cake

@@ -2,6 +2,10 @@
 ;; This is meant to be a dead simple "game" which you can modify while it is running.
 ;; This is to test hot-reloading
 
+;; GDB commands:
+;; set solib-search-path ~/Development/code/repositories/cakelisp/runtime
+;; set cwd ~/Development/code/repositories/cakelisp/runtime
+
 (import "HotReloading.cake")
 (import "ArrayTest.cake")
 
@@ -22,57 +26,23 @@
                      "inside home" "Surprise! Your home is filled with cake. You look at your hands. You are cake."
                      (array))))))
 
-;; TODO: Array
-(var-noreload rooms-state (* room) nullptr)
-
-;; TODO: Add no-eval-var
-(defun-local rooms-state-init ()
-  (var existing-value (* void) nullptr)
-  (if (hot-reload-find-variable "rooms-state" (addr existing-value))
-      (set rooms-state (type-cast existing-value (* room)))
-      (block
-          (set rooms-state (new-array 1 room))
-        ;; Have to handle array initialization ourselves, because the state variable is just a pointer now
-        (var current-elem int 0)
-        (while (< current-elem 1)
-          (set (at current-elem rooms-state)
-               (array "front porch" "You're outside the front door of your home. The lights are off inside."
-                      (array
-                       (array
-                        "inside home"
-                        "Surprise! Your home is filled with cake. You look at your hands. You are cake."
-                        (array)))))
-          (incr current-elem))
-        (hot-reload-register-variable "rooms-state" rooms-state))))
-
-;; This won't work because you must assign a value, but no function except a fancy template
-;; function could return any value. Additionally, it may be unwise to rely on static initialization
-;; (var static-init-test int (register-init-callback static-init-test-init))
-;; The original declaration
-;; (var static-init-test int 0)
-;; The modified declaration
-(var num-times-hot-reloaded int 0)
-
-;; The following two functions will be auto-generated
-;; (defun-local num-times-hot-reloaded-init ()
-;;   (var existing-value (* void) nullptr)
-;;   (if (hot-reload-find-variable "num-times-hot-reloaded" (addr existing-value))
-;;       (set (no-eval-var num-times-hot-reloaded) (type-cast existing-value (* int)))
-;;       (block
-;;           ;; C can have an easier time with plain old malloc and cast
-;;           (set (no-eval-var num-times-hot-reloaded) (new int))
-;;           (set num-times-hot-reloaded 0)
-;;           (hot-reload-register-variable "num-times-hot-reloaded" (no-eval-var num-times-hot-reloaded)))))
+(var rooms-state (* room) nullptr)
+(hot-reload-make-state-variable-array-initializer rooms-state ([] room) (array
+      (array "front porch" "You're outside the front door of your home. The lights are off inside."
+             (array (array
+                     "inside home" "Surprise! Your home is filled with cake. You look at your hands. You are cake."
+                     (array))))))
 
+(var num-times-hot-reloaded int 0)
 (hot-reload-make-state-variable-initializer num-times-hot-reloaded int 0)
 
 ;; This now makes order matter, because if we move it up we'll get a null pointer exception
-(var current-room (* (const room)) nullptr)
-(hot-reload-make-state-variable-initializer current-room (* (const room)) nullptr)
+(var current-room (* (const room)) (addr (at 0 rooms-state)))
+(hot-reload-make-state-variable-initializer current-room (* (const room)) (addr (at 0 rooms-state)))
 
 (defun libGeneratedCakelisp_initialize ()
   (num-times-hot-reloaded-initialize)
-  (rooms-state-init)
+  (rooms-state-initialize)
   (current-room-initialize))
 
 (defun print-help ()
@@ -101,16 +71,11 @@
     (printf "CAKE ADVENTURE\n\n")
     (print-help))
 
-  ;; (var my-test-struct ([] 5 TestStruct) (array))
-  ;; (var my-ptr-to-test-struct (* TestStruct) my-test-struct)
-  ;; (printf "Size experiment: %lu %lu %lu\n" (sizeof TestStruct)
-  ;;         (sizeof my-test-struct) (sizeof (deref my-ptr-to-test-struct)))
-
-  ;; (test)
-
   (printf "Num times reloaded: %d\n" num-times-hot-reloaded)
   (++ num-times-hot-reloaded)
 
+  (printf "Num intro rooms: %d\n" (sizeof rooms-state))
+
   (print-room current-room)
 
   (var input char 0)

src/Evaluator.cpp

@@ -235,7 +235,7 @@ bool HandleInvocation_Recursive(EvaluatorEnvironment& environment, const Evaluat
 			NoteAtToken(invocationStart,
 			            "code was generated from macro. See erroneous macro "
 			            "expansion below:");
-			printTokens(*macroOutputTokens);
+			prettyPrintTokens(*macroOutputTokens);
 			printf("\n");
 			// Deleting these tokens is only safe at this point because we know we have not
 			// evaluated them. As soon as they are evaluated, they must be kept around
@@ -258,7 +258,7 @@ bool HandleInvocation_Recursive(EvaluatorEnvironment& environment, const Evaluat
 		{
 			NoteAtToken(invocationStart,
 			            "code was generated from macro. See macro expansion below:");
-			printTokens(*macroOutputTokens);
+			prettyPrintTokens(*macroOutputTokens);
 			printf("\n");
 			return false;
 		}

src/Generators.cpp

@@ -373,7 +373,7 @@ bool VariableDeclarationGenerator(EvaluatorEnvironment& environment,
 	                          EvaluatorScope_Body))
 		return false;
 
-	bool hotReloadAllowed = funcNameToken.contents.compare("var-noreload") != 0;
+	bool hotReloadAllowed = !strstr(funcNameToken.contents.c_str(), "noreload");
 
 	// TODO: Eventually, static function variables could be automatically promoted to module scope
 	if (hotReloadAllowed && environment.enableHotReloading && isStaticFunctionVar &&
@@ -1710,6 +1710,7 @@ void importFundamentalGenerators(EvaluatorEnvironment& environment)
 	environment.generators["global-var"] = VariableDeclarationGenerator;
 	environment.generators["static-var"] = VariableDeclarationGenerator;
 	environment.generators["var-noreload"] = VariableDeclarationGenerator;
+	environment.generators["global-var-noreload"] = VariableDeclarationGenerator;
 
 	// Special case: Output the symbol without doing any additionall processing (e.g. StateVariable)
 	// This is only necessary for writing code internal to hot-reloading, or if you're going to do

src/Tokenizer.cpp

@@ -278,6 +278,24 @@ void printTokens(const std::vector<Token>& tokens)
 	printf("\n");
 }
 
+// TODO: Actually make it pretty
+void prettyPrintTokens(const std::vector<Token>& tokens)
+{
+	// Note that token parens could be invalid, so we shouldn't do things which rely on validity
+	const Token* previousToken = nullptr;
+	for (const Token& token : tokens)
+	{
+		if (previousToken && previousToken->type == TokenType_CloseParen &&
+		    token.type == TokenType_OpenParen)
+			printf("\n");
+
+		printFormattedToken(token);
+
+		previousToken = &token;
+	}
+	printf("\n");
+}
+
 bool writeCharToBuffer(char c, char** at, char* bufferStart, int bufferSize, const Token& token)
 {
 	**at = c;

src/Tokenizer.hpp

@@ -37,6 +37,7 @@ bool tokenizeLinePrintError(const char* inputLine, const char* source, unsigned
 bool validateParentheses(const std::vector<Token>& tokens);
 
 void printTokens(const std::vector<Token>& tokens);
+void prettyPrintTokens(const std::vector<Token>& tokens);
 
 bool writeCharToBuffer(char c, char** at, char* bufferStart, int bufferSize, const Token& token);
 bool writeStringToBuffer(const char* str, char** at, char* bufferStart, int bufferSize,