You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
358 lines
18 KiB
358 lines
18 KiB
;; Hot-reloading library code modifier
|
|
;; - Changes all module and global variables to use the heap, thereby persisting state across reloads
|
|
;; (in case it isn't obvious, you cannot persist state on the stack of code you're going to reload)
|
|
;; - Enables C-linkage and shared library toggles so functions can be discovered by the loader
|
|
;;
|
|
;; To use:
|
|
;; - Build the "loader" with HotReloading.cake. See HotLoader for an example loader
|
|
;; - The loader should link libraries that you need to use and shouldn't reload
|
|
;; - Now, modify the library which will be reloaded:
|
|
;; - Define the following function: (defun reloadable-entry-point (&return bool))
|
|
;; (return true to reload, false to exit). This is the library's "main"
|
|
;; - Don't link any libraries which shouldn't be reloaded (they're in the loader)
|
|
;; - When building the library, put HotReloadingCodeModifier.cake as the *first* import
|
|
;; (this ensures your functions will output c-linkage, and that a library will be built)
|
|
;;
|
|
;; It seems a bit complicated, but it's a really awesome feature, which makes it worth the price
|
|
;; Note that this will increase build times, because all affected functions and variables need to be
|
|
;; re-evaluated. There will be no macro expansions, however, which saves some time
|
|
;;
|
|
;; The goal is to be plug-and-play, but there are some modifications you'll need to do to your program:
|
|
;; - Variables with state which needs to be persisted across reloads need to be in module or global
|
|
;; variables. This likely means moving function-local state variables into module scope
|
|
;; - Initialization needs to be gated by a module scope or global scope variable so the app doesn't
|
|
;; re-initialize after each reload. It could also be a separate function which the loader calls once
|
|
;; - Change your main function to match the reload entry point signature
|
|
;; - Add some control within the program to instruct it to reload (return true from entry point)
|
|
;; - TODO: Array sizes are different when hot-reloading
|
|
|
|
;; Make all non-module-local functions dynamically loadable (we mainly care about our entry point
|
|
;; and initializer function, but the user could have a customized loader which hooks more)
|
|
;; This needs to be set before any functions which need C linkage are evaluated by Cakelisp
|
|
(set-cakelisp-option use-c-linkage true)
|
|
|
|
;; Because we could be building things without using C linkage, we need to separate all this code
|
|
;; out to prevent oscillating rebuilds
|
|
(add-build-config-label "HotLoadingModifiedCode")
|
|
|
|
;; This is a hack: We want to generate HotReloading.cake.hpp, but we don't want to inherit all the
|
|
;; loader-specific options. HotReloading.cake will see this define and not add the options
|
|
;; TODO: Make a context variable for preventing environment changes during &decls-only?
|
|
(comptime-define-symbol 'No-Hot-Reload-Options)
|
|
|
|
(import "ComptimeHelpers.cake" "CHelpers.cake")
|
|
|
|
;; This is redefined by make-code-hot-reloadable
|
|
(defun hot-reload-initialize-state ())
|
|
|
|
;; High-level explanation:
|
|
;; - Find all global and module-local variables
|
|
;; - Convert them to pointers so they can be stored on the heap, surviving reloads
|
|
;; - Create initializers for these pointers, which are called right after loading/reloading.
|
|
;; - Change all references to those variables to automatic pointer dereferencing. This is the expensive part
|
|
;; - Create a function for the loader to call to initialize all the pointers
|
|
;;
|
|
;; TODO: Arrays
|
|
;; TODO: Destructors
|
|
;; TODO: Initializers which reference other modified variables
|
|
;; TODO: Automatically convert main if found to entry point? A NoReload.cake could also be made (easy)
|
|
;; TODO: Potential bug if vars miss modified-vars stage. Need to store list of modified vars instead
|
|
;; (would also need to keep track of modified references...yikes)
|
|
;; TODO: Need to take scope into account before changing a symbol (was it actually a module-private var)
|
|
(defun-comptime make-code-hot-reloadable (environment (ref EvaluatorEnvironment)
|
|
&return bool)
|
|
(var verbose bool false)
|
|
|
|
(get-or-create-comptime-var environment modified-vars bool false)
|
|
(when (deref modified-vars) ;; Modify variables only once
|
|
(return true))
|
|
(set (deref modified-vars) true)
|
|
|
|
(fprintf stderr "HotReloading: Modifying code for hot-reloading.\n")
|
|
(fprintf stderr "Subsequent modifications will not be hot-reloading safe\n")
|
|
|
|
(get-or-create-comptime-var environment modules-with-import
|
|
(template (in std unordered_map) (in std string) int))
|
|
|
|
(defstruct modify-definition
|
|
name (in std string)
|
|
expanded-definition (template (in std vector) Token)
|
|
module (addr Module))
|
|
(var variables-to-modify (template (in std vector) modify-definition))
|
|
|
|
;; Collect variables. It must be done separately from modification because modification will
|
|
;; invalidate definition iterators
|
|
(for-in definition-pair (ref ObjectDefinitionPair) (field environment definitions)
|
|
(unless (= (field definition-pair second type) ObjectType_Variable)
|
|
(continue))
|
|
|
|
;; Check if this variable is reloadable
|
|
(var variable-start-invocation (addr (const Token))
|
|
(field definition-pair second definitionInvocation))
|
|
(var variable-type (addr (const Token)) (+ variable-start-invocation 3))
|
|
(when (= TokenType_OpenParen (path variable-type > type))
|
|
(var top-level-type (addr (const Token)) (+ variable-type 1))
|
|
(cond
|
|
;; TODO: Add array support
|
|
((= 0 (call-on compare (path top-level-type > contents) "array"))
|
|
(NoteAtToken (deref variable-start-invocation) "hot-reloadable arrays not supported yet")
|
|
(continue))
|
|
;; Constants cannot be changed, so do not need to be reloaded
|
|
;; What happens when you reload the lib? Do the constants get updated?
|
|
((= 0 (call-on compare (path top-level-type > contents) "const"))
|
|
(continue))))
|
|
|
|
(when verbose (fprintf stderr ">>> Variable %s\n" (call-on c_str (field definition-pair first))))
|
|
(var definition (ref ObjectDefinition) (field definition-pair second))
|
|
(var var-to-modify modify-definition)
|
|
(unless (CreateDefinitionCopyMacroExpanded definition
|
|
(field var-to-modify expanded-definition))
|
|
(return false))
|
|
|
|
(set (field var-to-modify name) (field definition-pair first))
|
|
(set (field var-to-modify module) (field definition context module))
|
|
(call-on push_back variables-to-modify (call (in std move) var-to-modify)))
|
|
|
|
;; Collect references to variables we're going to need to auto-deref
|
|
;; TODO: variables can have initializers which reference modded variables, which the init functions
|
|
;; will need to take into account
|
|
(var references-to-modify (template (in std vector) modify-definition))
|
|
(for-in definition-pair (ref ObjectDefinitionPair) (field environment definitions)
|
|
(unless (= (field definition-pair second type) ObjectType_Function)
|
|
(continue))
|
|
;; This is pretty brutal: Expanding every single definition which might have a ref...
|
|
(var definition (ref ObjectDefinition) (field definition-pair second))
|
|
(var def-to-modify modify-definition)
|
|
(unless (CreateDefinitionCopyMacroExpanded definition
|
|
(field def-to-modify expanded-definition))
|
|
(return false))
|
|
|
|
(var reference-found bool false)
|
|
(for-in token (ref (const Token)) (field def-to-modify expanded-definition)
|
|
(unless (= (field token type) TokenType_Symbol)
|
|
(continue))
|
|
(for-in var-to-modify (ref (const modify-definition)) variables-to-modify
|
|
(when (= 0 (call-on compare (field token contents)
|
|
(field (at 2 (field var-to-modify expanded-definition)) contents)))
|
|
(set reference-found true)
|
|
(break))))
|
|
(unless reference-found
|
|
(continue))
|
|
|
|
(when verbose (fprintf stderr ">>> Reference(s) found in %s\n"
|
|
(call-on c_str (field definition-pair first))))
|
|
|
|
(set (field def-to-modify module) (field definition context module))
|
|
(set (field def-to-modify name) (field definition-pair first))
|
|
(call-on push_back references-to-modify (call (in std move) def-to-modify)))
|
|
|
|
(var initializer-names (template (in std vector) Token))
|
|
|
|
;; First = module filename. Second = initializer name token (for blaming)
|
|
(var modules-to-import (template (in std unordered_map) (in std string) Token))
|
|
|
|
;; Pointerify variables and create initializer functions
|
|
(for-in var-to-modify (ref modify-definition) variables-to-modify
|
|
(var expanded-var-tokens (ref (template (in std vector) Token))
|
|
(field var-to-modify expanded-definition))
|
|
(var module (addr Module) (field var-to-modify module))
|
|
|
|
;; Before
|
|
(when verbose (prettyPrintTokens expanded-var-tokens))
|
|
|
|
(var start-token-index int 0)
|
|
(var end-invocation-index int (- (call-on size expanded-var-tokens) 1))
|
|
(var var-name-index int
|
|
(getExpectedArgument "expected variable name"
|
|
expanded-var-tokens
|
|
start-token-index 1
|
|
end-invocation-index))
|
|
(when (= var-name-index -1)
|
|
(return false))
|
|
|
|
(var type-index int
|
|
(getExpectedArgument "expected variable type"
|
|
expanded-var-tokens
|
|
start-token-index 2
|
|
end-invocation-index))
|
|
(when (= type-index -1)
|
|
(return false))
|
|
|
|
(var var-invocation (ref Token) (at 1 expanded-var-tokens))
|
|
(var var-name (ref Token) (at var-name-index expanded-var-tokens))
|
|
(var type-start (ref Token) (at type-index expanded-var-tokens))
|
|
|
|
;; Pointerify, remove intializer
|
|
(var new-var-tokens (addr (template (in std vector) Token)) (new (template (in std vector) Token)))
|
|
(call-on push_back (field environment comptimeTokens) new-var-tokens)
|
|
(tokenize-push (deref new-var-tokens)
|
|
((token-splice-addr var-invocation)
|
|
(token-splice-addr var-name)
|
|
(addr (token-splice-addr type-start))
|
|
null))
|
|
|
|
;; After
|
|
(when verbose (prettyPrintTokens (deref new-var-tokens)))
|
|
|
|
;; Create intiailizer function
|
|
(var init-function-name Token var-name)
|
|
(var string-var-name Token var-name)
|
|
(set (field string-var-name type) TokenType_String)
|
|
(scope ;; Create initializer function name from variable name
|
|
(var converted-name-buffer (array 64 char) (array 0))
|
|
;; TODO: Need to pass this in somehow
|
|
(var name-style NameStyleSettings)
|
|
(lispNameStyleToCNameStyle (field name-style variableNameMode) (call-on c_str (field var-name contents))
|
|
converted-name-buffer (sizeof converted-name-buffer) var-name)
|
|
|
|
(var init-function-name-buffer (array 256 char) (array 0))
|
|
(PrintfBuffer init-function-name-buffer "hotReloadInitVar_%s" converted-name-buffer)
|
|
(set (field init-function-name contents) init-function-name-buffer))
|
|
|
|
;; Store it for making the global initializer, which will call all initializers
|
|
(call-on push_back initializer-names init-function-name)
|
|
(set (at (path module > filename) modules-to-import) init-function-name)
|
|
|
|
(var assignment-tokens (template (in std vector) Token))
|
|
(scope ;; Optional assignment
|
|
(var assignment-index int
|
|
(getArgument expanded-var-tokens start-token-index 3 endInvocationIndex))
|
|
(when (!= assignment-index -1)
|
|
(var assignment-token (addr Token) (addr (at assignment-index expanded-var-tokens)))
|
|
(tokenize-push assignment-tokens
|
|
(set (deref (token-splice-addr var-name)) (token-splice assignment-token)))))
|
|
|
|
(var initializer-procedure-tokens (addr (template (in std vector) Token)) (new (template (in std vector) Token)))
|
|
(call-on push_back (field environment comptimeTokens) initializer-procedure-tokens)
|
|
;; Note that we don't auto-deref this; this is the only place where that's the case
|
|
(tokenize-push
|
|
(deref initializer-procedure-tokens)
|
|
(defun (token-splice-addr init-function-name) ()
|
|
(var existing-value (addr void) nullptr)
|
|
(if (hot-reload-find-variable (token-splice-addr string-var-name) (addr existing-value))
|
|
(set (token-splice-addr var-name) (type-cast existing-value (addr (token-splice-addr type-start))))
|
|
(scope ;; Create the variable
|
|
;; C can have an easier time with plain old malloc and cast
|
|
(set (token-splice-addr var-name) (new (token-splice-addr type-start)))
|
|
(token-splice-array assignment-tokens)
|
|
;; (set (deref (token-splice-addr var-name)) (token-splice-addr assignment))
|
|
(hot-reload-register-variable (token-splice-addr string-var-name)
|
|
(token-splice-addr var-name))))))
|
|
(when verbose (prettyPrintTokens (deref initializer-procedure-tokens)))
|
|
|
|
;; Make the changes
|
|
|
|
;; Definition references invalid after this!
|
|
|
|
(unless (ReplaceAndEvaluateDefinition environment (call-on c_str (field var-name contents))
|
|
(deref new-var-tokens))
|
|
(return false))
|
|
|
|
(scope ;; Evaluate initializer
|
|
(unless module
|
|
(return false))
|
|
(var initializer-context EvaluatorContext (array))
|
|
(set (field initializer-context module) module)
|
|
(set (field initializer-context scope) EvaluatorScope_Module)
|
|
(set (field initializer-context definitionName)
|
|
(addr (path environment . moduleManager > globalPseudoInvocationName)))
|
|
(set (field initializer-context isRequired) true)
|
|
|
|
;; Make sure HotReloading header is included
|
|
(var module-filename (addr (const char)) (path module > filename))
|
|
(when (= (call-on find (deref modules-with-import) module-filename)
|
|
(call-on-ptr end modules-with-import))
|
|
(var import-hot-reloading-tokens (addr (template (in std vector) Token)) (new (template (in std vector) Token)))
|
|
(call-on push_back (field environment comptimeTokens) import-hot-reloading-tokens)
|
|
;; Make sure we don't build our own version of this. The loader needs to manage it
|
|
(tokenize-push (deref import-hot-reloading-tokens) (import &decls-only "HotReloading.cake"))
|
|
(unless (= 0 (EvaluateGenerate_Recursive
|
|
environment initializer-context
|
|
(deref import-hot-reloading-tokens) 0
|
|
(deref (path module > generatedOutput))))
|
|
(return false))
|
|
;; Meaningless number, only using hash table for fast lookup
|
|
(set (at module-filename (deref modules-with-import)) 1))
|
|
|
|
(unless (= 0 (EvaluateGenerate_Recursive
|
|
environment initializer-context
|
|
(deref initializer-procedure-tokens) 0
|
|
(deref (path module > generatedOutput))))
|
|
(return false))))
|
|
|
|
;; Auto-dereference any references to the variables we've converted to pointers
|
|
(for-in def-to-modify (ref modify-definition) references-to-modify
|
|
(var expanded-def-tokens (ref (template (in std vector) Token))
|
|
(field def-to-modify expanded-definition))
|
|
(var module (addr Module) (field def-to-modify module))
|
|
(var new-definition (addr (template (in std vector) Token)) (new (template (in std vector) Token)))
|
|
(call-on push_back (field environment comptimeTokens) new-definition)
|
|
(for-in token (ref (const Token)) expanded-def-tokens
|
|
(unless (= (field token type) TokenType_Symbol)
|
|
(call-on push_back (deref new-definition) token)
|
|
(continue))
|
|
|
|
;; Check for reference
|
|
(var reference-found bool false)
|
|
(for-in var-to-modify (ref (const modify-definition)) variables-to-modify
|
|
(when (= 0 (call-on compare (field token contents)
|
|
(field (at 2 (field var-to-modify expanded-definition)) contents)))
|
|
(set reference-found true)
|
|
(break)))
|
|
;; Just an uninteresting symbol
|
|
(unless reference-found
|
|
(call-on push_back (deref new-definition) token)
|
|
(continue))
|
|
|
|
;; Insert the deref
|
|
(var auto-deref-tokens (template (in std vector) Token))
|
|
(tokenize-push auto-deref-tokens (deref (token-splice-addr token)))
|
|
(PushBackAll (deref new-definition) auto-deref-tokens))
|
|
|
|
;; Replace it!
|
|
(unless (ReplaceAndEvaluateDefinition environment (call-on c_str (field def-to-modify name))
|
|
(deref new-definition))
|
|
(return false)))
|
|
|
|
;; Create global initializer function to initialize all pointers on load/reload
|
|
;; Use this module to house the initializer. Putting it in some other module could cause unnecessary
|
|
;; rebuilds if different subsets of files are built. If it is housed here, only this file will
|
|
;; need to be recompiled
|
|
(scope
|
|
(var new-initializer-def (addr (template (in std vector) Token)) (new (template (in std vector) Token)))
|
|
;; Environment will handle freeing tokens for us
|
|
(call-on push_back (field environment comptimeTokens) new-initializer-def)
|
|
|
|
(var invocations (template (in std vector) Token))
|
|
(for-in initializer-name (ref Token) initializer-names
|
|
(tokenize-push invocations
|
|
(declare-extern-function (token-splice-addr initializer-name) ())
|
|
(call (token-splice-addr initializer-name))))
|
|
|
|
(tokenize-push (deref new-initializer-def)
|
|
(defun hot-reload-initialize-state ()
|
|
(token-splice-array invocations)))
|
|
|
|
(unless (ReplaceAndEvaluateDefinition environment
|
|
"hot-reload-initialize-state" (deref new-initializer-def))
|
|
(return false)))
|
|
|
|
(return true))
|
|
|
|
;;
|
|
;; Building
|
|
;;
|
|
|
|
(add-compile-time-hook post-references-resolved make-code-hot-reloadable)
|
|
|
|
(comptime-cond
|
|
('Unix
|
|
(add-compiler-link-options "-shared")
|
|
;; TODO: Automatically make library if no main found?
|
|
(set-cakelisp-option executable-output "libGeneratedCakelisp.so"))
|
|
('Windows
|
|
(add-compiler-link-options "/DLL")
|
|
(set-cakelisp-option executable-output "libGeneratedCakelisp.dll")))
|
|
|
|
;; This modifies code extensively, so it makes sense to have its own label
|
|
(add-build-config-label "HotReloadable")
|
|
|