Got hot-reloading working on VocalGame

It's janky, but it's a good first step!

* Update Cakelisp for module vs. global C search directories. This was
necessary so HotReloading.hpp can be found by every module in the
hot-reloading library
* command-add-string-argument is now part of Cakelisp
* Created Loader, which handles main, hot-reloading, and links the
libraries the reloadable library needs. By having those libraries in
the loader, they will not be destroyed when the reloadable is
destroyed
* Moved state variables and changed array to pointer for VocalGame to
support hot-reloading

BuildHotReloadable.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+echo "\n\nVocal Game (reloadable)\n\n"
+./Dependencies/cakelisp/bin/cakelisp --verbose-processes \
+									 test/src/MakeHotReload.cake test/src/VocalGame.cake || exit $?

BuildLoader.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+echo "\n\nLoader\n\n"
+./Dependencies/cakelisp/bin/cakelisp --verbose-processes test/src/Loader.cake || exit $?
+cd test && ./Loader || exit $?

Build_Debug.sh

@@ -10,11 +10,11 @@
 echo "\n\nOgre\n\n"
 ./Dependencies/cakelisp/bin/cakelisp test/src/OgreApp.cake || exit $?
 echo "\n\nSDL Ogre\n\n"
-./Dependencies/cakelisp/bin/cakelisp test/src/SDLOgreApp.cake || exit $?
+./Dependencies/cakelisp/bin/cakelisp --verbose-file-system test/src/SDLOgreApp.cake || exit $?
 echo "\n\nAuto Test\n\n"
 ./Dependencies/cakelisp/bin/cakelisp src/AutoTest.cake src/SDL.cake src/Tracy.cake || exit $?
-echo "\n\nVocal Game\n\n"
-./Dependencies/cakelisp/bin/cakelisp test/src/VocalGame.cake || exit $?
+echo "\n\nVocal Game (no reload)\n\n"
+./Dependencies/cakelisp/bin/cakelisp test/src/NoHotReload.cake test/src/VocalGame.cake || exit $?
 echo "\n\nRun Vocal Game\n\n"
 cd test && ./VocalGame
 

Dependencies/cakelisp

@@ -1 +1 @@
-Subproject commit 373a6edc0d9d23cbe7672b936c32c70ee886af26
+Subproject commit e6f5f033061e8411a5e3e62f838d168ce704d994

src/Macros.cake

@@ -1,11 +1,3 @@
 (skip-build)
 
 (import &comptime-only "../Dependencies/cakelisp/runtime/Macros.cake")
-
-(defmacro command-add-string-argument ()
-  (destructure-arguments new-argument-index)
-  (quick-token-at new-argument new-argument-index)
-  (tokenize-push output (on-call (field linkCommand arguments) push_back
-                                 (array ProcessCommandArgumentType_String
-                                        (token-splice (addr new-argument)))))
-  (return true))

src/Ogre.cake

@@ -114,7 +114,8 @@
 (import &comptime-only "../Dependencies/cakelisp/runtime/Macros.cake")
 
 (add-build-options "-DOGRE_DEBUG_MODE=1")
-(add-c-search-directory "Dependencies/ogre-next/OgreMain/include"
+(add-c-search-directory module
+                        "Dependencies/ogre-next/OgreMain/include"
                         "Dependencies/ogre-next/Components/Hlms/Common/include"
                         "Dependencies/ogre-next/Components/Hlms/Pbs/include"
                         "Dependencies/ogre-next/Components/Hlms/Unlit/include"

src/OgreInitialize.cake

@@ -1,7 +1,8 @@
 (import &comptime-only "Macros.cake")
 
 (add-build-options "-DOGRE_DEBUG_MODE=1")
-(add-c-search-directory "Dependencies/ogre-next/OgreMain/include"
+(add-c-search-directory module
+                        "Dependencies/ogre-next/OgreMain/include"
                         "Dependencies/ogre-next/Components/Hlms/Common/include"
                         "Dependencies/ogre-next/Components/Hlms/Pbs/include"
                         "Dependencies/ogre-next/Components/Hlms/Unlit/include"

src/SDL.cake

@@ -89,7 +89,7 @@
 ;; For now, an easy way to build files which include SDL headers
 (defmacro module-use-sdl-build-options ()
   (tokenize-push output
-                 (add-c-search-directory "Dependencies/SDL/buildSDLBuild/include/SDL2"))
+                 (add-c-search-directory module "Dependencies/SDL/buildSDLBuild/include/SDL2"))
   (return true))
 
 (module-use-sdl-build-options)

src/Tracy.cake

@@ -29,7 +29,7 @@
 (defmacro module-use-tracy-build-options ()
   (tokenize-push output
                  (add-build-options "-DTRACY_ENABLE")
-                 (add-c-search-directory "Dependencies/tracy"))
+                 (add-c-search-directory module "Dependencies/tracy"))
   (return true))
 
 (module-use-tracy-build-options)

test/src/Loader.cake

@@ -0,0 +1,96 @@
+;; Export libraries included?
+;; (set-cakelisp-option use-c-linkage true)
+
+(set-cakelisp-option cakelisp-src-dir "Dependencies/cakelisp/src")
+(set-cakelisp-option executable-output "test/Loader")
+
+;; We only want the shared libraries these use
+;; (import "src/Ogre.cake"
+;;         "src/SDL.cake")
+
+
+(import &comptime-only "Dependencies/cakelisp/runtime/Macros.cake")
+;; TODO: Should this happen automatically, because import automatically adds current working dir?
+;; Should it add working dir?
+(add-c-search-directory module ".")
+
+;; So HotReloading.cake can find DynamicLoader.hpp
+(add-c-search-directory global "Dependencies/cakelisp/src")
+
+(import "Dependencies/cakelisp/runtime/HotReloading.cake")
+(c-import "stdio.h")
+
+(defun main (&return int)
+  (printf "Hello Hot-reloading!\n")
+
+  (def-function-signature reload-entry-point-signature (&return bool))
+  (var hot-reload-entry-point-func reload-entry-point-signature null)
+  (register-function-pointer (type-cast (addr hot-reload-entry-point-func) (* (* void)))
+                             ;; TODO Support name conversion at runtime (conversion requires tokens)
+                             "reloadableEntryPoint")
+
+  (unless (do-hot-reload)
+    (printf "error: failed to hot-reload\n")
+    (return 1))
+
+  (while (hot-reload-entry-point-func)
+    (unless (do-hot-reload)
+      (printf "error: failed to hot-reload\n")
+      (return 1)))
+  (return 0))
+
+
+;;
+;; Libraries
+;;
+(defun-comptime ogre-link-hook (manager (& ModuleManager)
+                                        linkCommand (& ProcessCommand)
+                                        linkTimeInputs (* ProcessCommandInput) numLinkTimeInputs int
+                                        &return bool)
+  ;; TODO: Expose this option somehow?
+  (var is-debug-build bool true)
+
+  (printf "OgreCore: Adding %s link arguments\n" (? is-debug-build "debug" "release"))
+
+  (if is-debug-build
+      (block
+          (command-add-string-argument "-LDependencies/ogre-next/build/Debug/lib")
+        (command-add-string-argument "-lOgreHlmsPbs_d")
+        (command-add-string-argument "-lOgreHlmsUnlit_d")
+        (command-add-string-argument "-lOgreMain_d")
+        (command-add-string-argument "-lOgreOverlay_d")
+        (command-add-string-argument "-Wl,-rpath,.:../Dependencies/ogre-next/build/Debug/lib"))
+      (block
+          (command-add-string-argument "-LDependencies/ogre-next/build/Release/lib")
+        (command-add-string-argument "-lOgreHlmsPbs")
+        (command-add-string-argument "-lOgreHlmsUnlit")
+        (command-add-string-argument "-lOgreMain")
+        (command-add-string-argument "-lOgreOverlay")
+        (command-add-string-argument "-Wl,-rpath,.:../Dependencies/ogre-next/build/Release/lib")))
+
+  (return true))
+(add-compile-time-hook pre-link ogre-link-hook)
+
+(defun-comptime sdl-link-hook (manager (& ModuleManager)
+                                       linkCommand (& ProcessCommand)
+                                       linkTimeInputs (* ProcessCommandInput) numLinkTimeInputs int
+                                       &return bool)
+  ;; TODO: Expose this option somehow?
+  (var is-debug-build bool true)
+
+  (printf "SDL: Adding %s link arguments\n" (? is-debug-build "debug" "release"))
+
+  (if is-debug-build
+      ;; TODO: Actually add debug build
+      (block
+          (command-add-string-argument "-LDependencies/SDL/buildSDLBuild/lib")
+        (command-add-string-argument "-lSDL2")
+        (command-add-string-argument "-Wl,-rpath,.:../Dependencies/SDL/buildSDLBuild/lib"))
+      (block
+          (command-add-string-argument "-LDependencies/sdl-next/build/Release/lib")
+        (command-add-string-argument "-lSDL2")
+        (command-add-string-argument "-Wl,-rpath,.:../Dependencies/SDL/buildSDLBuild/lib")))
+
+  (return true))
+
+(add-compile-time-hook pre-link sdl-link-hook)

test/src/MakeHotReload.cake

@@ -0,0 +1,7 @@
+(skip-build)
+
+(set-cakelisp-option executable-output "test/libGeneratedCakelisp.so")
+
+(import "Dependencies/cakelisp/runtime/HotReloadingCodeModifier.cake")
+
+(add-c-search-directory global "Dependencies/cakelisp/runtime")

test/src/NoHotReload.cake

@@ -0,0 +1,3 @@
+(defun main (&return int)
+  (reloadableEntryPoint)
+  (return 0))

test/src/OgreApp.cake

@@ -3,7 +3,7 @@
 (import "src/Ogre.cake")
 ;; TODO: Should this happen automatically, because import automatically adds current working dir?
 ;; Should it add working dir?
-(add-c-search-directory ".")
+(add-c-search-directory module ".")
 
 (c-import "<stdio.h>")
 

test/src/SDLOgreApp.cake

@@ -5,7 +5,7 @@
         "src/SDL.cake")
 ;; TODO: Should this happen automatically, because import automatically adds current working dir?
 ;; Should it add working dir?
-(add-c-search-directory ".")
+(add-c-search-directory module ".")
 
 (c-import "SDL.h" "SDL_syswm.h" "SDL_timer.h")
 ;; TODO: Somehow inherit this from SDL.cake?

test/src/VocalGame.cake

@@ -6,7 +6,7 @@
         &comptime-only "Dependencies/cakelisp/runtime/Macros.cake")
 ;; TODO: Should this happen automatically, because import automatically adds current working dir?
 ;; Should it add working dir?
-(add-c-search-directory ".")
+(add-c-search-directory module ".")
 
 (c-import "SDL.h" "SDL_syswm.h" "SDL_timer.h"
           "<math.h>" "<stdio.h>" "<string.h>")
@@ -15,7 +15,8 @@
 
 ;; These are read and written to from different threads (currently, without locking)
 (var audio-is-recording bool false)
-(var audio-input-buffer ([] 44100 Uint8) (array 0))
+(var audio-input-buffer (* Uint8) null)
+(var audio-input-buffer-size int 0)
 (var audio-input-write-head int 0)
 (var audio-input-read-head int 0)
 
@@ -38,12 +39,19 @@
   (fclose dest-file))
 
 (defun-local audio-input-buffer-initialize ()
+  (set audio-input-buffer-size 44100)
+  (set audio-input-buffer (type-cast (calloc audio-input-buffer-size (sizeof Uint8))
+                                     (* Uint8)))
   (var i int 0)
-  (while (< i (array-size audio-input-buffer))
+  (while (< i audio-input-buffer-size)
     ;; TODO: Use silence value from SDL audio spec
     (set (at i audio-input-buffer) 127)
     (incr i)))
 
+(defun-local audio-input-buffer-destroy ()
+  (free audio-input-buffer)
+  (set audio-input-buffer null))
+
 (defun-local audio-output-callback (userdata (* void) stream (* Uint8) stream-length int)
   ;; (printf "Audio len %d\n" stream-length)
   (static-var up bool false)
@@ -72,7 +80,7 @@
         (set mono-sample 127) ;; Silence
         (block ;; Else, play the recording
             (var recording-index int (+ (/ i num-channels) audio-input-read-head))
-          (set recording-index (mod recording-index (array-size audio-input-buffer)))
+          (set recording-index (mod recording-index audio-input-buffer-size))
           (set mono-sample (at recording-index audio-input-buffer))))
 
     ;; Channels are interleaved, e.g. LRLR, not LLRR
@@ -84,7 +92,7 @@
     (set i (+ i num-channels)))
   (set audio-input-read-head
        (mod (+ audio-input-read-head samples-per-channel)
-            (array-size audio-input-buffer))))
+            audio-input-buffer-size)))
 
 ;; Note: If input is sampled at a different rate, playback will be at a lower pitch. Use SDL's
 ;; audio conversion functions to handle that properly
@@ -98,7 +106,7 @@
   (var i int 0)
   (while (< i stream-length)
     (set (at audio-input-write-head audio-input-buffer) (at i stream))
-    (if (= audio-input-write-head (- (array-size audio-input-buffer) 1))
+    (if (= audio-input-write-head (- audio-input-buffer-size 1))
         (set audio-input-write-head 0)
         (incr audio-input-write-head))
     (incr i)))
@@ -269,32 +277,56 @@
   (when (>= output-device 2) (SDL_CloseAudioDevice output-device))
   (when (>= input-device 2) (SDL_CloseAudioDevice input-device)))
 
-(defun main (&return int)
-  (var window (* SDL_Window) null)
-  (unless (sdl-initialize (addr window))
-    (return 1))
+(var g-window (* SDL_Window) null)
+(var g-output-device SDL_AudioDeviceID 0)
+(var g-input-device SDL_AudioDeviceID 0)
+(var g-output-device-spec SDL_AudioSpec)
+(var g-input-device-spec SDL_AudioSpec)
 
-  (audio-input-buffer-initialize)
-  (var output-device SDL_AudioDeviceID 0)
-  (var input-device SDL_AudioDeviceID 0)
-  (var output-device-spec SDL_AudioSpec)
-  (var input-device-spec SDL_AudioSpec)
-  (when enable-audio
-    (unless (initialize-audio (addr output-device) (addr input-device)
-                              (addr output-device-spec) (addr input-device-spec))
-      (sdl-shutdown window)
-      (sdl-audio-close output-device input-device)
-      (return 1)))
+(var monkey-mesh mesh-handle)
+(var monkey-node scene-node)
+
+(var reload-sentinel int 2)
+
+;; TODO: Automatically make entry point
+(defun reloadableEntryPoint (&return bool)
+  (var result int (app-main))
+  (cond ((= 0 result)
+         (return false))
+        ((= 1 result)
+         (return false))
+        ((= reload-sentinel result)
+         (return true)))
+  (return false))
+
+(var initialized bool false)
+
+(defun app-main (&return int)
+  (unless initialized
+    ;; (defun main (&return int)
+    (unless (sdl-initialize (addr g-window))
+      (return 1))
 
-  ;; Ogre uses exceptions for error handling, so we can't gracefully close without getting all that
-  ;; stuff set up (which I don't really want to do; it belongs in Gamelib)
-  (unless (ogre-initialize-sdl)
-    (sdl-audio-close output-device
-                     input-device)
-    (return 1))
+    (audio-input-buffer-initialize)
 
-  (var monkey-mesh mesh-handle (ogre-load-mesh "Suzanne.mesh"))
-  (var monkey-node scene-node (ogre-node-from-item monkey-mesh))
+    ;; Ogre uses exceptions for error handling, so we can't gracefully close without getting all that
+    ;; stuff set up (which I don't really want to do; it belongs in Gamelib)
+    (unless (ogre-initialize-sdl)
+      (return 1))
+
+    (set monkey-mesh (ogre-load-mesh "Suzanne.mesh"))
+    (set monkey-node (ogre-node-from-item monkey-mesh))
+
+    (set initialized true))
+
+  ;; Audio needs to be re-initialized due to reload removing callbacks
+  ;; TODO: Some way to avoid the big wait time required when initializing audio would be good
+  (when enable-audio
+    (unless (initialize-audio (addr g-output-device) (addr g-input-device)
+                              (addr g-output-device-spec) (addr g-input-device-spec))
+      (sdl-shutdown g-window)
+      (sdl-audio-close g-output-device g-input-device)
+      (return 1)))
 
   (var exit-reason (* (const char)) null)
 
@@ -318,6 +350,15 @@
     (when (at SDL_SCANCODE_ESCAPE currentKeyStates)
       (set exit-reason "Escape pressed"))
 
+    (when (at SDL_SCANCODE_R currentKeyStates)
+      (printf "\nReloading\n\n")
+      ;; Our audio callbacks are going away!
+      ;; The SDL_QueueAudio() API would make this easier to deal with, but then I need to manage
+      ;; an audio thread on my own
+      (sdl-audio-close g-output-device
+                       g-input-device)
+      (return reload-sentinel))
+
     (var delta-position ([] 3 float) (array 0))
     (when (at SDL_SCANCODE_RIGHT currentKeyStates)
       (set (at 0 delta-position) (+ (at 0 delta-position) move-speed)))
@@ -339,14 +380,14 @@
        (var audio-conversion-settings SDL_AudioCVT)
        (SDL_BuildAudioCVT (addr audio-conversion-settings)
                           ;; Src
-                          (field input-device-spec format)
-                          (field input-device-spec channels)
-                          (field input-device-spec freq)
+                          (field g-input-device-spec format)
+                          (field g-input-device-spec channels)
+                          (field g-input-device-spec freq)
                           ;; Dest
-                          (field output-device-spec format)
-                          1 ;; (field output-device-spec channels) ;; We convert to stereo in playback callback
-                          (field output-device-spec freq))
-       (set (field audio-conversion-settings len) (array-size audio-input-buffer))
+                          (field g-output-device-spec format)
+                          1 ;; (field g-output-device-spec channels) ;; We convert to stereo in playback callback
+                          (field g-output-device-spec freq))
+       (set (field audio-conversion-settings len) audio-input-buffer-size)
        ;; TODO: Does the buffer really nead to be 8x the src? Am I doing something wrong with my len?
        ;; Could be conversion to float then 2x that for resampling = 4x * 2x = 8x
        (var converted-buffer-size-bytes int
@@ -360,7 +401,7 @@
             (type-cast (malloc converted-buffer-size-bytes) (* Uint8)))
        (scope ;; Copy src to in-place conversion buffer
         (var i int 0)
-        (while (< i (array-size audio-input-buffer))
+        (while (< i audio-input-buffer-size)
           (set (at i (field audio-conversion-settings buf)) (at i audio-input-buffer))
           (incr i)))
        (unless (= 0 (SDL_ConvertAudio (addr audio-conversion-settings)))
@@ -376,7 +417,7 @@
        (scope ;; Copy back to destination buffer. Note: this could drop samples, because it should
         ;; use the converted buffer size instead of the fixed array size!
         (var i int 0)
-        (while (< i (array-size audio-input-buffer))
+        (while (< i audio-input-buffer-size)
           (set (at i audio-input-buffer)
                (at i (type-cast (field audio-conversion-settings buf) (* Uint8))))
           ;; TODO: If my sample rate is higher when recorded, that means I need to record more
@@ -384,7 +425,7 @@
           ;; enough samples at the lower playback rate, and start hitting buffer workspace.
           ;; Zero them out for now. I should record with enough samples to make it exactly fit
           ;; after conversion
-          (when (>= i (- (array-size audio-input-buffer) 3900))
+          (when (>= i (- audio-input-buffer-size 3900))
             (set (at i audio-input-buffer) 127))
           (incr i)))
        (free (field audio-conversion-settings buf)))
@@ -393,7 +434,7 @@
       ;; Note: peak in both low and high, never greater than range / 2
       (var highest-peak Uint8 0)
       (var i int 0)
-      (while (< i (array-size audio-input-buffer))
+      (while (< i audio-input-buffer-size)
         (var current-value int (- (type-cast (at i audio-input-buffer) int) 127))
         (when (< highest-peak (abs current-value))
           (set highest-peak (abs current-value)))
@@ -411,7 +452,7 @@
           (set scaling-factor scaling-limit))
         (when scaling-factor
           (set i 0)
-          (while (< i (array-size audio-input-buffer))
+          (while (< i audio-input-buffer-size)
             (var current-value int (- (type-cast (at i audio-input-buffer) int) 127))
             (set (at i audio-input-buffer) (type-cast
                                             ;; Map back to 0-255
@@ -429,7 +470,7 @@
     ;; Visualize audio playback
     (var audio-read-head-to-world float (* 20.f
                                            (/ audio-input-read-head
-                                              (type-cast (array-size audio-input-buffer) float))))
+                                              (type-cast audio-input-buffer-size float))))
     (var audio-volume-to-world float (* 7.f
                                         (/ (- (at audio-input-read-head audio-input-buffer) 127)
                                            127.f)))
@@ -448,12 +489,14 @@
       (break)))
 
   (audio-dump-recorded-buffer "out.dat"
-                              audio-input-buffer (array-size audio-input-buffer))
+                              audio-input-buffer audio-input-buffer-size)
 
   (ogre-shutdown)
-  (sdl-audio-close output-device
-                   input-device)
-  (sdl-shutdown window)
+  (sdl-audio-close g-output-device
+                   g-input-device)
+  (sdl-shutdown g-window)
+
+  (audio-input-buffer-destroy)
 
   (when exit-reason
     (printf "Exit reason: %s\n" exit-reason))