Browse Source

Added WIP introspection/serialization system

This code generates metadata tables for structs defined with a special
macro. These metadata tables can be used for a huge variety of things,
but my most pressing need now is for a basic text serialization
Macoy Madson 2 months ago
3 changed files with 237 additions and 2 deletions
  1. +3
  2. +233
  3. +1

+ 3
- 1
.gitignore View File

@ -69,4 +69,6 @@ test/data/Materials/Textures/*.dds

+ 233
- 0
src/Introspection.cake View File

@ -0,0 +1,233 @@
(import &comptime-only "CHelpers.cake" "ComptimeHelpers.cake")
(c-import "<stdio.h>"
"<stddef.h>") ;; For size_t, offsetof
;; (defmacro each-token-argument-in (token-array any
;; start-index any
;; iterator-name symbol
;; &rest body any)
;; (gen-unique-symbol end-token-var "end-token" (deref start-index))
;; (tokenize-push output
;; (scope
;; (var (token-splice-addr end-token-var) int
;; (FindCloseParenTokenIndex (token-splice token-array start-index)))
;; (c-for (var (token-splice iterator-name) int (token-splice start-index))
;; (< (token-splice iterator-name) (token-splice-addr end-token-var))
;; (set (token-splice iterator-name)
;; (getNextArgument (token-splice token-array iterator-name (addr end-token-var))))
;; (token-splice-rest body tokens))))
;; (return true))
(defmacro each-token-argument-in (token-array any
start-index any
end-token-index any
iterator-name symbol
&rest body any)
(tokenize-push output
(c-for (var (token-splice iterator-name) int (token-splice start-index))
(< (token-splice iterator-name) (token-splice end-token-index))
(set (token-splice iterator-name)
(getNextArgument (token-splice token-array iterator-name end-token-index)))
(token-splice-rest body tokens))))
(return true))
(defmacro def-serializable-struct (struct-name symbol &rest arguments any)
(var processed-arguments (* (<> std::vector Token)) (new (<> std::vector Token)))
(call-on push_back (field environment comptimeTokens) processed-arguments)
(var fields-metadata (* (<> std::vector Token)) (new (<> std::vector Token)))
(call-on push_back (field environment comptimeTokens) fields-metadata)
(var end-token-index int (FindCloseParenTokenIndex tokens startTokenIndex))
(var num-parens int 1) ;; Count the opening of the struct definition
;; TODO add enums
(var state-variable-name int 0)
(var state-variable-type int 1)
(var state-variable-annotations int 2)
(var state int state-variable-name)
;; TODO: Need better helper functions for getting tokens vs. elements (where element can be array or symbol)
(var start-members int (getExpectedArgument "first member" tokens startTokenIndex 2 end-token-index))
(when (= -1 start-members)
(return false))
(each-token-argument-in tokens start-members end-token-index i
(var name-token (* (const Token))
(addr (at i tokens)))
(set i (getNextArgument tokens i end-token-index))
(var type-token (* (const Token))
(addr (at i tokens)))
(var possible-annotation-index int (getNextArgument tokens i end-token-index))
(var annotations-token (* (const Token))
(addr (at possible-annotation-index tokens)))
(when (= TokenType_OpenParen (path annotations-token > type))
;; Process annotation
(NoteAtToken (deref annotations-token) "detected annotation")
;; Absorb the argument as part of this field
(set i possible-annotation-index))
(scope ;; Output field metadata
(var name-to-string-token Token (deref name-token))
(set (field name-to-string-token type) TokenType_String)
(var serialize-type Token (deref type-token))
(set (field serialize-type type) TokenType_Symbol)
(var determined-type-string (* (const char)) "serialization-type-unknown")
((std-str-equals (path type-token > contents) "int")
(set determined-type-string "serialization-type-int"))
((std-str-equals (path type-token > contents) "float")
(set determined-type-string "serialization-type-float"))
((std-str-equals (path type-token > contents) "bool")
(set determined-type-string "serialization-type-bool"))
((std-str-equals (path type-token > contents) "char")
(set determined-type-string "serialization-type-char"))
;; TODO What are you doing, bro?
;; Replace with e.g.:
;; (var char-type (<> (in std vector)))
;; (tokenize-push char-type (* (const char)))
;; (compare-token-expressions type-token (addr (at 0 char-type)))
((and (= TokenType_OpenParen (path type-token > type))
(std-str-equals (path (+ 1 type-token) > contents) "*")
(or (std-str-equals (path (+ 2 type-token) > contents) "char")
(and (std-str-equals (path (+ 3 type-token) > contents) "const")
(std-str-equals (path (+ 4 type-token) > contents) "char"))))
(set determined-type-string "serialization-type-string")))
(set (field serialize-type contents) determined-type-string)
(tokenize-push (deref fields-metadata)
(array (token-splice-addr name-to-string-token) (token-splice-addr serialize-type)
(offsetof (type (token-splice struct-name)) (token-splice name-token))
null null)))
;; Output regular struct fields
(PushBackTokenExpression (deref processed-arguments) name-token)
(PushBackTokenExpression (deref processed-arguments) type-token))
(var metadata-name Token (deref struct-name))
(var metadata-name-buffer ([] 128 char) (array 0))
(PrintfBuffer metadata-name-buffer "%s--metadata"
(call-on c_str (path struct-name > contents)))
(set (field metadata-name contents) metadata-name-buffer))
(var metadata-fields-name Token (deref struct-name))
(var metadata-fields-name-buffer ([] 128 char) (array 0))
(PrintfBuffer metadata-fields-name-buffer "%s--metadata-fields"
(call-on c_str (path struct-name > contents)))
(set (field metadata-fields-name contents) metadata-fields-name-buffer))
(var struct-name-to-str Token (deref struct-name))
(set (field struct-name-to-str type) TokenType_String)
(tokenize-push output
(defstruct (token-splice struct-name)
(token-splice-array (deref processed-arguments)))
(var (token-splice-addr metadata-fields-name) ([] metadata-field)
(array (token-splice-array (deref fields-metadata))))
(var (token-splice-addr metadata-name) metadata-struct
(array (token-splice-addr struct-name-to-str)
(token-splice-addr metadata-fields-name) ;; TODO
(array-size (token-splice-addr metadata-fields-name)))))
(return true))
;; TODO Enums (defenum serialization-type
(def-type-alias serialization-type int)
(var serialization-type-unknown int 0) ;; Assert if any field is encountered during reading with this (e.g. default all pointers)
(var serialization-type-no-serialize int 1) ;; E.g. runtime-only fields
(var serialization-type-override int 2) ;; Require custom callbacks. TODO: What about dynarray etc.?
(var serialization-type-int int 3)
(var serialization-type-float int 4)
(var serialization-type-bool int 5)
(var serialization-type-char int 6)
(var serialization-type-plain-old-data int 7)
(var serialization-type-string int 8)
(var serialization-type-field-by-field int 9) ;; Only for structs. Remove?
;; TODO Gross
(forward-declare (struct MetadataField)
(struct MetadataStruct))
(def-function-signature serialization-read-func (metadata (* metadata-field) data (* void) output (* FILE)))
(def-function-signature serialization-write-func (metadata (* metadata-field) data (* void) output (* FILE)))
(defstruct-local metadata-field
name (* (const char))
type serialization-type
offset size_t
override-read serialization-read-func
override-write serialization-write-func)
(defstruct-local metadata-struct
name (* (const char))
;; type serialization-type ;; Almost always field-by-field, but let them override the whole process
;; override-read serialization-read-func
;; override-write serialization-write-func
members (* metadata-field)
num-members size_t)
(def-serializable-struct my-struct
name (* (const char)) (string)
value int
decimal float
truthy bool
charry char)
(defmacro offset-pointer-to-type (start-address any offset-bytes any desired-type any)
(tokenize-push output
(type-cast (+ (type-cast (token-splice start-address) (* char))
(token-splice offset-bytes))
(token-splice desired-type)))
(return true))
;; TODO Make const correct
(defun-local write-serializable-struct (struct-metadata (* metadata-struct)
struct-to-write (* void)
out-file (* FILE))
(fprintf out-file "(%s" (path struct-metadata > name))
(each-in-range (path struct-metadata > num-members) i
(var field (* metadata-field) (addr (at i (path struct-metadata > members))))
(fprintf out-file " :%s" (path field > name))
((= (path field > type) serialization-type-int)
(var int-write (* int)
(offset-pointer-to-type struct-to-write (path field > offset) (* int)))
(fprintf out-file " %d" (deref int-write)))
((= (path field > type) serialization-type-float)
(var float-write (* float)
(offset-pointer-to-type struct-to-write (path field > offset) (* float)))
(fprintf out-file " %f" (deref float-write)))
((= (path field > type) serialization-type-bool)
(var bool-write (* bool)
(offset-pointer-to-type struct-to-write (path field > offset) (* bool)))
(fprintf out-file " %s" (? (deref bool-write) "true" "false")))
((= (path field > type) serialization-type-char)
(var char-write (* char)
(offset-pointer-to-type struct-to-write (path field > offset) (* char)))
;; Write chars as integers to avoid writing e.g. '\0' in text
(fprintf out-file " %d" (deref char-write)))
((= (path field > type) serialization-type-string)
(var str-write (* (* (const char)))
(offset-pointer-to-type struct-to-write (path field > offset) (* (* (const char)))))
;; TODO: This will require quoting any '"' in the string (etc.)
(fprintf out-file " \"%s\"" (deref str-write)))
(fprintf out-file " <unknown>"))))
(fprintf out-file ")\n"))
(defun test--serialization (&return int)
(var a my-struct (array "Test struct" 42
-0.33f false 'a'))
(write-serializable-struct (addr my-struct--metadata) (addr a) stderr)
(var out-file (* FILE) (fopen "TestSerialize.cake" "w"))
(unless out-file (return 1))
(write-serializable-struct (addr my-struct--metadata) (addr a) out-file)
(fclose out-file)
(return 0))

+ 1
- 1
test/src/GameLibTests.cake View File

@ -49,7 +49,7 @@
"Auto Test (data structures only)"
(array platform-config
"../src/AutoTest.cake" "../src/Dictionary.cake"
"../src/DynamicArray.cake" "../src/Introspection.cake")))