allocator: rework alloc_delete

Added a switch case at start of alloc_delete to only run it on
container (read: heap allocated) types.

If adding to free vector, iterate through free vector first to ensure
we've not added it already.  Reset the object before adding it, so
reuse is trivial.

.dir-locals.el

@@ -3,4 +3,6 @@
 
 ((nil . ((compile-command . "make MODE=debug test examples")
          (+license/license-choice . "GNU General Public License Version 2")))
- (c-mode . ((mode . clang-format))))
+ (c-mode . ((mode . clang-format)))
+ ("test" .
+  ((nil . ((compile-command . "make MODE=full test"))))))

Makefile

@@ -5,21 +5,22 @@ OUT=$(DIST)/alisp.out
 TEST=$(DIST)/test.out
 
 LDFLAGS=
-GFLAGS=-Wall -Wextra -Wpedantic -Werror -std=c23 -I./include/
-DFLAGS=-ggdb -fsanitize=address -fsanitize=undefined -DVERBOSE_LOGS=1
+GFLAGS=-Wall -Wextra -Wswitch-enum -Wpedantic -Werror -std=c23 -I./include/
+DFLAGS=-ggdb -fsanitize=address -fsanitize=undefined
 RFLAGS=-O3
 
 MODE=release
 ifeq ($(MODE), release)
 CFLAGS=$(GFLAGS) $(RFLAGS)
 else ifeq ($(MODE), debug)
-CFLAGS=$(GFLAGS) $(DFLAGS)
+CFLAGS=$(GFLAGS) $(DFLAGS) -DVERBOSE_LOGS=1
 else ifeq ($(MODE), full)
-CFLAGS=$(GFLAGS) $(DFLAGS) -DTEST_VERBOSE=1
+CFLAGS=$(GFLAGS) $(DFLAGS) -DVERBOSE_LOGS=2 -DTEST_VERBOSE=1
 endif
 
 # Units to compile
-UNITS=src/sv.c src/vec.c src/stream.c src/symtable.c src/tag.c src/lisp.c
+UNITS=src/sv.c src/vec.c src/string.c src/stream.c src/symtable.c src/lisp.c \
+		src/allocator.c src/sys.c src/reader.c
 OBJECTS:=$(patsubst src/%.c, $(DIST)/%.o, $(UNITS))
 
 TEST_UNITS=test/main.c

alisp.org

@@ -4,7 +4,16 @@
 #+filetags: :alisp:
 
 * Tasks
-** WIP Reader system
+** String views :strings:
+[[file:include/alisp/sv.h::/// String Views]]
+*** DONE sv_substr
+Takes an index and a size, returns a string view to that substring.
+*** DONE sv_chop_left and sv_chop_right
+Super obvious.
+*** TODO Design Strings for the Lisp :api:
+We have ~sv_t~ so our basic C API is done.  We just need pluggable
+functions to construct and deconstruct strings as lisps.
+** Reader system :reader:
 We need to design a reader system.  The big idea: given a "stream" of
 data, we can break out expressions from it.  An expression could be
 either an atomic value or a container.
@@ -45,80 +54,58 @@ i.e. no parsing.
 *** DONE Design what a "parser function" would look like
 The general function is something like ~stream -> T | Err~.  What
 other state do we need to encode?
-*** TODO Write a parser for integers
-*** TODO Write a parser for symbols
-*** TODO Write a parser for lists
-*** TODO Write a parser for vectors
+*** DONE Write a parser for integers
+*** DONE Write a parser for symbols
+*** DONE Write a parser for lists
+*** DONE Write a parser for vectors
+*** TODO Write a parser for strings
+Requires [[*Design Strings for the Lisp]] to be complete first.
 *** TODO Write the general parser
-** WIP Unit tests :tests:
-*** TODO Test streams
-**** DONE Test file init
-[[file:test/test_stream.c::void stream_test_file(void)]]
-***** DONE Test successful init from real files
-Ensure stream_size is 0 i.e. we don't read anything on creation.
-Also ensure stream_eoc is false.
-***** DONE Test failed init from fake files
-**** DONE Test peeking and next
-[[file:test/test_stream.c::void stream_test_peek_next(void)]]
-- Peeking with bad streams ('\0' return)
-- Peeking with good streams (no effect on position)
-- Next with bad streams ('\0' return, no effect on position)
-- Next with good streams (effects position)
-- Peeking after next (should just work)
-**** DONE Test seeking
-[[file:test/test_stream.c::void stream_test_seek(void)]]
-- Seeking forward/backward on a bad stream (should stop at 0)
-- Seeking forward/backward too far (should clamp)
-- Seeking forward/backward zero sum via relative index (stream_seek)
-**** DONE Test substring
-[[file:test/test_stream.c::void stream_test_substr(void)]]
-- Substr on bad stream (NULL sv)
-- Substr on bad position/size (NULL sv)
-- Substr relative/absolute (good SV)
-**** TODO Test till
-[[file:test/test_stream.c::void stream_test_till(void)]]
-- till on a bad stream (NULL SV)
-- till on an ended stream (NULL SV)
-- till on a stream with no items in search string (eoc)
-- till on a stream with all items in search string (no effect)
-- till on a stream with prefix being all search string (no effect)
-- till on a stream with suffix being all search string (stops at
-  suffix)
-**** TODO Test while
-[[file:test/test_stream.c::void stream_test_while(void)]]
-- while on a bad stream (NULL SV)
-- while on an ended stream (NULL SV)
-- while on a stream with no items in search string (no effect)
-- while on a stream with all items in search string (eoc)
-- while on a stream with prefix being all search string (effect)
-- while on a stream with suffix being all search string (no effect)
-**** TODO Test line_col
-[[file:test/test_stream.c::void stream_test_line_col(void)]]
-- line_col on bad stream (no effect on args)
-- line_col on eoc stream (should go right to the end)
-- line_col on random points in a stream
-*** DONE Test system registration of allocated units
-In particular, does clean up work as we expect?  Do we have situations
-where we may double free or not clean up something we should've?
-** Backlog
-*** TODO Design Big Integers
+** Design :design:
+*** TODO Design Big Integers :api:
 We currently have 62 bit integers implemented via immediate values
 embedded in a pointer.  We need to be able to support even _bigger_
 integers.  How do we do this?
-*** TODO Design garbage collection scheme :design:gc:
-Really, regardless of what I do, we need to have some kind of garbage
-collection header on whatever managed objects we allocate.
+*** TODO Capitalise symbols (TBD) :optimisation:
+Should we capitalise symbols?  This way, we limit the symbol table's
+possible options a bit (potentially we could design a better hashing
+algorithm?) and it would be kinda like an actual Lisp.
+*** TODO Consider reader macros :reader:
+Common Lisp has so-called "reader macros" which allows users to write
+Lisp code that affects further Lisp code reading.  It's quite
+powerful.
 
-Firstly, the distinction between managed and unmanaged objects:
+Scheme doesn't have it.  Should we implement this?
+** Allocator :allocator:
+*** Some definitions
 - Managed objects are allocations that are generated as part of
   evaluating user code i.e. strings, vectors, conses that are all made
   as part of evaluating code.
 - Unmanaged objects are allocations we do as part of the runtime.
   These are things that we expect to have near infinite lifetimes
   (such as the symbol table, vector of allocated objects, etc).
-
-We need to perform garbage collection against the managed objects, and
-leave the unmanaged objects to the runtime.
+*** DONE Design an allocator
+We'll need an allocator for all our managed objects.  Requirements:
+- Stable pointers (memory that has already been allocated should be
+  free to utilise via the same pointer for the lifetime of the
+  allocator)
+- Able to tag allocations as unused (i.e. "free") and able to reuse
+  these allocations
+  - This will link into the garbage collector, which should yield a
+    sequence of objects that were previously tagged as unfree and
+    should be "freed".
+- Able to allocate all the managed types we have
+**** DONE Design allocation data structures
+**** DONE Design allocation methods for different lisp types
+- Strings (when implemented)
+***** DONE Conses
+***** DONE Vectors
+**** DONE Design allocation freeing method
+*** TODO Design garbage collection scheme :gc:
+Really, regardless of what I do, we need to have some kind of garbage
+collection header on whatever managed objects we allocate.  We need to
+perform garbage collection against the managed objects, and leave the
+unmanaged objects to the runtime.
 **** TODO Mark stage
 We need to mark all objects that are currently accessible from the
 environment.  This means we need to have a root environment which we
@@ -189,19 +176,57 @@ Latter approach time complexity:
 
 Former approach is better time complexity wise, but latter is way
 better in terms of simplicity of code.  Must deliberate.
-*** TODO Design Strings
-We have ~sv_t~ so our basic C API is done.  We just need pluggable
-functions to construct and deconstruct strings as lisps.
-*** TODO Capitalise symbols (TBD) :optimisation:design:
-Should we capitalise symbols?  This way, we limit the symbol table's
-possible options a bit (potentially we could design a better hashing
-algorithm?) and it would be kinda like an actual Lisp.
-*** TODO sv_t
-[[file:include/alisp/sv.h::/// String Views]]
-**** TODO sv_substr
-Takes an index and a size, returns a string view to that substring.
-**** TODO sv_chop_left and sv_chop_right
-Super obvious.
+** Unit tests :tests:
+*** TODO Test streams :streams:
+**** DONE Test file init
+[[file:test/test_stream.c::void stream_test_file(void)]]
+***** DONE Test successful init from real files
+Ensure stream_size is 0 i.e. we don't read anything on creation.
+Also ensure stream_eoc is false.
+***** DONE Test failed init from fake files
+**** DONE Test peeking and next
+[[file:test/test_stream.c::void stream_test_peek_next(void)]]
+- Peeking with bad streams ('\0' return)
+- Peeking with good streams (no effect on position)
+- Next with bad streams ('\0' return, no effect on position)
+- Next with good streams (effects position)
+- Peeking after next (should just work)
+**** DONE Test seeking
+[[file:test/test_stream.c::void stream_test_seek(void)]]
+- Seeking forward/backward on a bad stream (should stop at 0)
+- Seeking forward/backward too far (should clamp)
+- Seeking forward/backward zero sum via relative index (stream_seek)
+**** DONE Test substring
+[[file:test/test_stream.c::void stream_test_substr(void)]]
+- Substr on bad stream (NULL sv)
+- Substr on bad position/size (NULL sv)
+- Substr relative/absolute (good SV)
+**** TODO Test till
+[[file:test/test_stream.c::void stream_test_till(void)]]
+- till on a bad stream (NULL SV)
+- till on an ended stream (NULL SV)
+- till on a stream with no items in search string (eoc)
+- till on a stream with all items in search string (no effect)
+- till on a stream with prefix being all search string (no effect)
+- till on a stream with suffix being all search string (stops at
+  suffix)
+**** TODO Test while
+[[file:test/test_stream.c::void stream_test_while(void)]]
+- while on a bad stream (NULL SV)
+- while on an ended stream (NULL SV)
+- while on a stream with no items in search string (no effect)
+- while on a stream with all items in search string (eoc)
+- while on a stream with prefix being all search string (effect)
+- while on a stream with suffix being all search string (no effect)
+**** TODO Test line_col
+[[file:test/test_stream.c::void stream_test_line_col(void)]]
+- line_col on bad stream (no effect on args)
+- line_col on eoc stream (should go right to the end)
+- line_col on random points in a stream
+*** TODO Test reader :reader:
+*** DONE Test system registration of allocated units
+In particular, does clean up work as we expect?  Do we have situations
+where we may double free or not clean up something we should've?
 ** Completed
 *** DONE Test value constructors and destructors :test:
 Test if ~make_int~ works with ~as_int,~ ~intern~ with ~as_sym~.

include/alisp/alisp.h

@@ -8,13 +8,13 @@
 #ifndef ALISP_H
 #define ALISP_H
 
+#include <alisp/allocator.h>
 #include <alisp/base.h>
 #include <alisp/lisp.h>
 #include <alisp/reader.h>
 #include <alisp/stream.h>
 #include <alisp/sv.h>
 #include <alisp/symtable.h>
-#include <alisp/tag.h>
 #include <alisp/vec.h>
 
 #endif

include/alisp/allocator.h

@@ -0,0 +1,61 @@
+/* allocator.h: Lisp Allocator
+ * Created: 2026-02-12
+ * Author: Aryadev Chavali
+ * License: See end of file
+ * Commentary:
+ */
+
+#ifndef ALLOCATOR_H
+#define ALLOCATOR_H
+
+#include <alisp/lisp.h>
+#include <alisp/vec.h>
+
+#define ALLOC_PAGE_DEFAULT_SIZE 512
+
+typedef struct
+{
+  u64 padding : 56;
+  tag_t tag   : 8;
+  u64 references;
+} alloc_metadata_t;
+
+static_assert(sizeof(alloc_metadata_t) == 16,
+              "16 byte metadata required for alignment purposes");
+
+typedef struct
+{
+  alloc_metadata_t metadata;
+  u8 data[];
+} alloc_node_t;
+
+typedef struct
+{
+  vec_t data;
+} page_t;
+
+typedef struct
+{
+  vec_t pages;
+  vec_t free_vec;
+} alloc_t;
+
+lisp_t *alloc_make(alloc_t *, tag_t type);
+void alloc_delete(alloc_t *, lisp_t *);
+u64 alloc_cost(alloc_t *);
+void alloc_free(alloc_t *);
+
+#endif
+
+/* Copyright (C) 2026 Aryadev Chavali
+
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License Version 2 for
+ * details.
+
+ * You may distribute and modify this code under the terms of the GNU General
+ * Public License Version 2, which you should have received a copy of along with
+ * this program.  If not, please go to <https://www.gnu.org/licenses/>.
+
+ */

include/alisp/lisp.h

@@ -8,7 +8,9 @@
 #ifndef LISP_H
 #define LISP_H
 
-#include <alisp/symtable.h>
+#include <stdio.h>
+
+#include <alisp/string.h>
 #include <alisp/vec.h>
 
 #define NIL 0
@@ -21,36 +23,53 @@ typedef struct
   lisp_t *car, *cdr;
 } cons_t;
 
-/// System context
-typedef struct
+/// Tagging system
+typedef enum Tag
 {
-  vec_t memory;
-  sym_table_t symtable;
-} sys_t;
+  TAG_SMI  = 0b00000001, // Atomic types
+  TAG_SYM  = 0b00000011,
+  TAG_NIL  = 0b00000000, // Container types (0 LSB)
+  TAG_CONS = 0b00000010,
+  TAG_VEC  = 0b00000100,
+  TAG_STR  = 0b00000110,
+  NUM_TAGS = 6,
+} tag_t;
 
-void sys_init(sys_t *);
-void sys_register(sys_t *, lisp_t *);
-void sys_free(sys_t *);
+// Some helper macros for tagging
+#define SHIFT_TAG (8)
+#define MASK_TAG  ((1 << SHIFT_TAG) - 1)
 
-/// Constructors and destructors
-lisp_t *make_int(i64);
-lisp_t *make_vec(sys_t *, u64);
-lisp_t *intern(sys_t *, sv_t);
-lisp_t *cons(sys_t *, lisp_t *, lisp_t *);
+#define TAG(PTR, TYPE)    ((lisp_t *)((((u64)(PTR)) << SHIFT_TAG) | TAG_##TYPE))
+#define UNTAG(PTR)        (((u64)PTR) >> SHIFT_TAG)
+#define GET_TAG(PTR)      ((tag_t)(((u64)(PTR)) & MASK_TAG))
+#define IS_TAG(PTR, TYPE) (GET_TAG(PTR) == TAG_##TYPE)
 
-i64 as_int(lisp_t *);
+#define INT_BITS ((sizeof(i64) * 8) - SHIFT_TAG)
+#define INT_MAX  ((((i64)1) << (INT_BITS - 1)) - 1)
+#define INT_MIN  (-(INT_MAX + 1))
+
+tag_t tag_get(const lisp_t *);
+u64 tag_sizeof(tag_t);
+u64 lisp_sizeof(lisp_t *);
+lisp_t *lisp_reset(lisp_t *);
+void lisp_print(FILE *, lisp_t *);
+
+lisp_t *tag_smi(const i64);
+lisp_t *tag_sym(const char *);
+lisp_t *tag_cons(const cons_t *);
+lisp_t *tag_vec(const vec_t *);
+lisp_t *tag_str(const str_t *);
+lisp_t *tag_generic(void *, tag_t);
+
+i64 as_smi(lisp_t *);
 char *as_sym(lisp_t *);
 cons_t *as_cons(lisp_t *);
 vec_t *as_vec(lisp_t *);
+str_t *as_str(lisp_t *);
 
 #define CAR(L) (as_cons(L)->car)
 #define CDR(L) (as_cons(L)->cdr)
 
-lisp_t *car(lisp_t *);
-lisp_t *cdr(lisp_t *);
-
-void lisp_free(lisp_t *);
-
 #endif
 
 /* Copyright (C) 2026 Aryadev Chavali

include/alisp/reader.h

@@ -8,13 +8,18 @@
 #ifndef READER_H
 #define READER_H
 
-#include <alisp/lisp.h>
 #include <alisp/stream.h>
+#include <alisp/sys.h>
 
 typedef enum
 {
   READ_ERR_OK = 0,
   READ_ERR_EOF,
+  READ_ERR_EXPECTED_CLOSED_BRACE,
+  READ_ERR_EXPECTED_CLOSED_SQUARE_BRACKET,
+  READ_ERR_EXPECTED_CLOSING_SPEECHMARKS,
+  READ_ERR_UNEXPECTED_CLOSED_BRACE,
+  READ_ERR_UNEXPECTED_CLOSED_SQUARE_BRACKET,
   READ_ERR_UNKNOWN_CHAR,
 } read_err_t;
 

include/alisp/stream.h

@@ -53,8 +53,11 @@ typedef struct
 
 stream_err_t stream_init_string(stream_t *, const char *, sv_t);
 stream_err_t stream_init_pipe(stream_t *, const char *, FILE *);
+// NOTE: stream_init_file will attempt to read all content from the FILE
+// descriptor.  Use with caution.
 stream_err_t stream_init_file(stream_t *, const char *, FILE *);
-void stream_stop(stream_t *);
+void stream_reset(stream_t *);
+void stream_free(stream_t *);
 
 // End of Content (i.e. we've consumed all cached content/file)
 bool stream_eoc(stream_t *);
@@ -70,6 +73,12 @@ u64 stream_seek(stream_t *, i64);
 u64 stream_seek_forward(stream_t *, u64);
 u64 stream_seek_backward(stream_t *, u64);
 
+// Return a string view to stream data relative to the current position of the
+// stream
+sv_t stream_sv(stream_t *);
+// Return a string view to data from the start of the stream
+sv_t stream_sv_abs(stream_t *);
+
 // Return a relative substring of a given size
 sv_t stream_substr(stream_t *, u64);
 // Return an absolute substring at given index and of given size.

include/alisp/string.h

@@ -0,0 +1,35 @@
+/* string.h: String library
+ * Created: 2026-03-05
+ * Author: Aryadev Chavali
+ * License: See end of file
+ * Commentary:
+ */
+
+#ifndef STRING_H
+#define STRING_H
+
+#include <alisp/sv.h>
+#include <alisp/vec.h>
+
+typedef struct
+{
+  vec_t data;
+} str_t;
+
+str_t string_make(sv_t sv);
+sv_t string_sv(str_t *);
+
+#endif
+
+/* Copyright (C) 2026 Aryadev Chavali
+
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License Version 2 for
+ * details.
+
+ * You may distribute and modify this code under the terms of the GNU General
+ * Public License Version 2, which you should have received a copy of along with
+ * this program.  If not, please go to <https://www.gnu.org/licenses/>.
+
+ */

include/alisp/sv.h

@@ -14,11 +14,12 @@
 typedef struct
 {
   u64 size;
-  char *data;
+  const char *data;
 } sv_t;
 
 // String view macro constructor
 #define SV(DATA, SIZE) ((sv_t){.data = (DATA), .size = (SIZE)})
+#define SV_AUTO(DATA)  ((sv_t){.data = (void *)(DATA), .size = sizeof(DATA) - 1})
 // Pretty printers
 #define SV_FMT(SV) (int)(SV).size, (SV).data
 #define PR_SV      "%.*s"
@@ -26,6 +27,13 @@ typedef struct
 
 // String view functions
 sv_t sv_copy(sv_t);
+sv_t sv_chop_left(sv_t, u64 size);
+sv_t sv_chop_right(sv_t, u64 size);
+sv_t sv_truncate(sv_t, u64 newsize);
+sv_t sv_substr(sv_t, u64 position, u64 size);
+
+sv_t sv_till(sv_t, const char *reject);
+sv_t sv_while(sv_t, const char *accept);
 
 #endif
 

include/alisp/symtable.h

@@ -21,9 +21,12 @@ typedef struct
 #define SYM_TABLE_INIT_SIZE (1 << 10)
 
 void sym_table_init(sym_table_t *);
-char *sym_table_find(sym_table_t *, sv_t);
+const char *sym_table_find(sym_table_t *, sv_t);
 void sym_table_free(sym_table_t *);
 
+// Debugging function: total memory used by symbol table.
+u64 sym_table_cost(sym_table_t *);
+
 #endif
 
 /* Copyright (C) 2026 Aryadev Chavali

include/alisp/sys.h

@@ -0,0 +1,57 @@
+/* sys.h: System context and constructors
+ * Created: 2026-02-12
+ * Author: Aryadev Chavali
+ * License: See end of file
+ * Commentary:
+ */
+
+#ifndef SYS_H
+#define SYS_H
+
+#include <alisp/allocator.h>
+#include <alisp/lisp.h>
+#include <alisp/symtable.h>
+
+/// System context
+typedef struct
+{
+  alloc_t memory;
+  sym_table_t symtable;
+} sys_t;
+
+void sys_init(sys_t *);
+lisp_t *sys_alloc(sys_t *, tag_t type);
+void sys_delete(sys_t *, lisp_t *);
+void sys_free(sys_t *);
+
+// Debugging function: provides total memory usage from system.
+u64 sys_cost(sys_t *);
+
+/// Constructors and general Lisp API
+lisp_t *make_int(i64);
+lisp_t *intern(sys_t *, sv_t);
+lisp_t *cons(sys_t *, lisp_t *, lisp_t *);
+lisp_t *make_list(sys_t *, lisp_t **, u64);
+lisp_t *make_vec(sys_t *, u64);
+lisp_t *make_str(sys_t *, u64);
+
+lisp_t *car(lisp_t *);
+lisp_t *cdr(lisp_t *);
+
+void lisp_free(sys_t *, lisp_t *);
+void lisp_free_rec(sys_t *, lisp_t *);
+
+#endif
+
+/* Copyright (C) 2026 Aryadev Chavali
+
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License Version 2 for
+ * details.
+
+ * You may distribute and modify this code under the terms of the GNU General
+ * Public License Version 2, which you should have received a copy of along with
+ * this program.  If not, please go to <https://www.gnu.org/licenses/>.
+
+ */

include/alisp/tag.h

@@ -1,68 +0,0 @@
-/* tag.h: Pointer tagging
- * Created: 2026-02-04
- * Author: Aryadev Chavali
- * License: See end of file
- * Commentary:
- */
-
-#ifndef TAG_H
-#define TAG_H
-
-#include <alisp/lisp.h>
-
-typedef enum Tag
-{
-  TAG_NIL  = 0b00000000, // Start of atomic types
-  TAG_INT  = 0b00000001, // Special tag so we can encode 63 bit integers
-  TAG_SYM  = 0b00000100,
-  TAG_CONS = 0b00000010, // Start of container types
-  TAG_VEC  = 0b00000110,
-  NUM_TAGS = 5,
-} tag_t;
-
-static_assert(NUM_TAGS == 5, "Expected NUM_TAGS == 5 for enum SHIFT");
-enum Shift
-{
-  SHIFT_INT  = 1,
-  SHIFT_SYM  = 8,
-  SHIFT_CONS = 8,
-  SHIFT_VEC  = 8,
-};
-
-static_assert(NUM_TAGS == 5, "Expected NUM_TAGS == 5 for enum MASK");
-enum Mask
-{
-  MASK_INT  = 0b00000001,
-  MASK_SYM  = 0b11111111,
-  MASK_CONS = 0b11111111,
-  MASK_VEC  = 0b11111111,
-};
-
-// Some helper macros for tagging
-#define TAG(PTR, TYPE)    ((lisp_t *)(((PTR) << SHIFT_##TYPE) | TAG_##TYPE))
-#define IS_TAG(PTR, TYPE) (((u64)(PTR) & MASK_##TYPE) == TAG_##TYPE)
-#define UNTAG(PTR, TYPE)  (((u64)PTR) >> SHIFT_##TYPE)
-
-#define INT_MAX ((1L << 62) - 1)
-#define INT_MIN (-(1L << 62))
-
-tag_t get_tag(lisp_t *);
-lisp_t *tag_int(i64);
-lisp_t *tag_sym(char *);
-lisp_t *tag_cons(cons_t *);
-lisp_t *tag_vec(vec_t *);
-
-#endif
-
-/* Copyright (C) 2026 Aryadev Chavali
-
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- * FOR A PARTICULAR PURPOSE.  See the GNU General Public License Version 2 for
- * details.
-
- * You may distribute and modify this code under the terms of the GNU General
- * Public License Version 2, which you should have received a copy of along with
- * this program.  If not, please go to <https://www.gnu.org/licenses/>.
-
- */

include/alisp/vec.h

@@ -31,12 +31,18 @@ static_assert(sizeof(vec_t) == 64, "vec_t has to be 64 bytes as part of SBO");
 
 #define VEC_GET(V, I, T) (((T *)vec_data(V))[I])
 #define VEC_SIZE(V, T)   ((V)->size / (sizeof(T)))
+#define FOR_VEC(INDEX, V, T) \
+  for (size_t INDEX = 0; INDEX < VEC_SIZE(V, T); ++INDEX)
 
 void vec_init(vec_t *, u64);
 void vec_free(vec_t *);
+void vec_reset(vec_t *);
 u8 *vec_data(vec_t *);
 
+// Append, possibly reallocating memory
 void vec_append(vec_t *, const void *const, u64);
+// Try to append without allocating memory
+bool vec_try_append(vec_t *, const void *const, u64);
 void vec_ensure_free(vec_t *, u64);
 void vec_clone(vec_t *, vec_t *);
 

src/allocator.c

@@ -0,0 +1,256 @@
+/* allocator.c: Allocator implementations
+ * Created: 2026-02-12
+ * Author: Aryadev Chavali
+ * License: See end of file
+ * Commentary:
+ */
+
+#include <stdlib.h>
+
+#include <alisp/allocator.h>
+#include <alisp/lisp.h>
+#include <alisp/vec.h>
+#include <string.h>
+
+page_t *make_page(u64 size)
+{
+  page_t *page = calloc(1, sizeof(*page));
+  vec_init(&page->data, MAX(size, ALLOC_PAGE_DEFAULT_SIZE));
+  return page;
+}
+
+alloc_node_t *make_node(page_t *page, tag_t type)
+{
+  alloc_node_t *node = NULL;
+  u64 size           = sizeof(*node);
+
+  static_assert(NUM_TAGS == 6);
+  switch (type)
+  {
+  case TAG_CONS:
+    size += sizeof(cons_t);
+    break;
+  case TAG_VEC:
+    size += sizeof(vec_t);
+    break;
+  case TAG_STR:
+    size += sizeof(str_t);
+    break;
+  case TAG_NIL:
+  case TAG_SMI:
+  case TAG_SYM:
+  default:
+    FAIL("Unreachable");
+    return node;
+  }
+
+  // We must ensure size is a multiple of 8 for alignment purposes
+  size = (size & 0b111) == 0 ? size : size + (8 - (size & 0b111));
+
+  if (!vec_try_append(&page->data, NULL, size))
+    return NULL;
+  node = (alloc_node_t *)(vec_data(&page->data) + page->data.size - size);
+  node->metadata = (alloc_metadata_t){.references = 0, .tag = type};
+  return node;
+}
+
+alloc_node_t *lisp_to_node(lisp_t *lisp)
+{
+  void *raw_ptr = NULL;
+
+  static_assert(NUM_TAGS == 6);
+  switch (tag_get(lisp))
+  {
+  case TAG_CONS:
+    raw_ptr = as_cons(lisp);
+    break;
+  case TAG_VEC:
+    raw_ptr = as_vec(lisp);
+    break;
+  case TAG_STR:
+    raw_ptr = as_str(lisp);
+    break;
+  case TAG_NIL: // These shouldn't be allocated
+  case TAG_SMI:
+  case TAG_SYM:
+  default:
+    FAIL("Unreachable");
+    return NIL;
+  }
+
+  alloc_node_t *node = raw_ptr;
+  return &node[-1];
+}
+
+lisp_t *alloc_make(alloc_t *alloc, tag_t type)
+{
+  static_assert(NUM_TAGS == 6);
+  switch (type)
+  {
+  case TAG_CONS:
+  case TAG_VEC:
+  case TAG_STR:
+    break;
+  case TAG_NIL: // These shouldn't be allocated
+  case TAG_SMI:
+  case TAG_SYM:
+  default:
+    FAIL("Unreachable");
+    return NIL;
+  }
+
+  // We want to try to fill this node with an allocation of this type.
+  alloc_node_t *node = NULL;
+
+  // Try to get something from the free vector
+  u64 free_vec_size = VEC_SIZE(&alloc->free_vec, alloc_node_t *);
+  for (u64 i = 0; i < free_vec_size; ++i)
+  {
+    alloc_node_t **nodeptr = &VEC_GET(&alloc->free_vec, i, alloc_node_t *);
+
+    // Skip any nodes that don't have the right type.
+    if (nodeptr[0]->metadata.tag != type)
+      continue;
+
+    assert("Expected free node to have no references" &&
+           nodeptr[0]->metadata.references == 0);
+
+    // Pop this node off the free vector by swapping it with the last item and
+    // decrementing the size of the vector.
+
+    alloc_node_t **lastptr =
+        &VEC_GET(&alloc->free_vec, free_vec_size - 1, alloc_node_t *);
+    alloc_node_t *val = *nodeptr;
+    *nodeptr          = *lastptr;
+    *lastptr          = val;
+
+    // Decrement the size of the free vector
+    alloc->free_vec.size -= sizeof(val);
+
+    // Then use that valid (and now unused) node as our return.
+    node = *lastptr;
+
+    goto end;
+  }
+
+  // We couldn't get anything from the free vector, so try to allocate a fresh
+  // one against one of the pages.
+  FOR_VEC(i, &alloc->pages, page_t *)
+  {
+    page_t *page = VEC_GET(&alloc->pages, i, page_t *);
+    node         = make_node(page, type);
+    if (node)
+      goto end;
+  }
+
+  // There aren't any pages we can allocate against, so we need to make a new
+  // page.
+  page_t *page = make_page(0);
+  vec_append(&alloc->pages, &page, sizeof(page));
+  node = make_node(page, type);
+
+end:
+  if (!node)
+    FAIL("Unexpected issue with allocating to a verifiably good page");
+
+  return tag_generic(node->data, type);
+}
+
+void alloc_delete(alloc_t *alloc, lisp_t *lisp)
+{
+  switch (tag_get(lisp))
+  {
+  case TAG_CONS:
+  case TAG_VEC:
+  case TAG_STR:
+    break;
+  case TAG_NIL: // These can't be deleted (not allocated)
+  case TAG_SMI:
+  case TAG_SYM:
+  default:
+    FAIL("Unreachable");
+    return;
+  }
+
+  alloc_node_t *node = lisp_to_node(lisp);
+  assert(node && node->metadata.references == 0);
+
+  // If already present in the free vector, stop.
+  FOR_VEC(i, &alloc->free_vec, alloc_node_t *)
+  {
+    alloc_node_t *other = VEC_GET(&alloc->pages, i, alloc_node_t *);
+    if (other == node)
+    {
+      return;
+    }
+  }
+
+  // Otherwise, add to the free vector.
+  lisp_reset(lisp);
+  vec_append(&alloc->free_vec, &node, sizeof(node));
+}
+
+u64 alloc_cost(alloc_t *alloc)
+{
+  u64 total_size = alloc->pages.size;
+  FOR_VEC(i, &alloc->pages, page_t *)
+  {
+    page_t *page = VEC_GET(&alloc->pages, i, page_t *);
+    total_size += page->data.size;
+  }
+  return total_size;
+}
+
+void alloc_free(alloc_t *alloc)
+{
+  FOR_VEC(i, &alloc->pages, page_t *)
+  {
+    page_t *page = VEC_GET(&alloc->pages, i, page_t *);
+    // Iterate through every alloc_node in this page (dynamic walk)
+    for (u64 j = 0; j < VEC_SIZE(&page->data, u8);)
+    {
+      alloc_node_t *node = (alloc_node_t *)(vec_data(&page->data) + j);
+      u64 next           = sizeof(*node) + tag_sizeof(node->metadata.tag);
+
+      static_assert(NUM_TAGS == 6);
+      switch (node->metadata.tag)
+      {
+      case TAG_CONS:
+        // Do nothing - will be cleaned by overall vec free anyway
+        break;
+      case TAG_VEC:
+        vec_free((vec_t *)node->data);
+        break;
+      case TAG_STR:
+        vec_free(&((str_t *)node->data)->data);
+        break;
+      case TAG_NIL:
+      case TAG_SMI:
+      case TAG_SYM:
+      default:
+        FAIL("Unreachable");
+      }
+      j += next;
+    }
+
+    // Each page was allocated on the heap.
+    vec_free(&page->data);
+    free(page);
+  }
+  vec_free(&alloc->pages);
+  vec_free(&alloc->free_vec);
+  memset(alloc, 0, sizeof(*alloc));
+}
+
+/* Copyright (C) 2026 Aryadev Chavali
+
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License Version 2 for
+ * details.
+
+ * You may distribute and modify this code under the terms of the GNU General
+ * Public License Version 2, which you should have received a copy of along with
+ * this program.  If not, please go to <https://www.gnu.org/licenses/>.
+
+ */

src/lisp.c

@@ -9,110 +9,267 @@
 #include <string.h>
 
 #include <alisp/lisp.h>
-#include <alisp/tag.h>
 
-void sys_init(sys_t *sys)
+lisp_t *tag_smi(i64 i)
 {
-  memset(sys, 0, sizeof(*sys));
+  return TAG(i, SMI);
 }
 
-void sys_register(sys_t *sys, lisp_t *ptr)
+lisp_t *tag_sym(const char *str)
 {
-  // Simply append it to the list of currently active conses
-  vec_append(&sys->memory, &ptr, sizeof(&ptr));
+  return TAG(str, SYM);
 }
 
-void sys_free(sys_t *sys)
+lisp_t *tag_vec(const vec_t *vec)
 {
-  static_assert(NUM_TAGS == 5);
+  return TAG(vec, VEC);
+}
 
-  sym_table_free(&sys->symtable);
-  if (sys->memory.size == 0)
-    return;
+lisp_t *tag_str(const str_t *str)
+{
+  return TAG(str, STR);
+}
 
-  // Iterate through each cell of memory currently allocated and free them
-  for (size_t i = 0; i < VEC_SIZE(&sys->memory, lisp_t **); ++i)
-  {
-    lisp_t *allocated = VEC_GET(&sys->memory, i, lisp_t *);
-    lisp_free(allocated);
-  }
-
-  // Free the container
-  vec_free(&sys->memory);
-
-  // Ensure no one treats this as active in any sense
-  memset(sys, 0, sizeof(*sys));
-}
-
-lisp_t *make_int(i64 i)
-{
-  return tag_int(i);
-}
-
-lisp_t *cons(sys_t *sys, lisp_t *car, lisp_t *cdr)
-{
-  cons_t *cons = calloc(1, sizeof(*cons));
-  cons->car    = car;
-  cons->cdr    = cdr;
-
-  lisp_t *lcons = tag_cons(cons);
-  sys_register(sys, lcons);
-  return lcons;
-}
-
-lisp_t *make_vec(sys_t *sys, u64 capacity)
-{
-  vec_t *vec = calloc(1, sizeof(*vec));
-  vec_init(vec, capacity);
-  lisp_t *ptr = tag_vec(vec);
-  sys_register(sys, ptr);
-  return ptr;
-}
-
-lisp_t *intern(sys_t *sys, sv_t sv)
-{
-  char *str = sym_table_find(&sys->symtable, sv);
-  return tag_sym(str);
-}
-
-lisp_t *car(lisp_t *lsp)
-{
-  if (!IS_TAG(lsp, CONS))
-    return NIL;
-  else
-    return CAR(lsp);
-}
-
-lisp_t *cdr(lisp_t *lsp)
-{
-  if (!IS_TAG(lsp, CONS))
-    return NIL;
-  else
-    return CDR(lsp);
-}
-
-void lisp_free(lisp_t *item)
-{
-  switch (get_tag(item))
+lisp_t *tag_cons(const cons_t *cons)
+{
+  return TAG(cons, CONS);
+}
+
+lisp_t *tag_generic(void *ptr, tag_t type)
+{
+  static_assert(NUM_TAGS == 6);
+  switch (type)
   {
+  case TAG_NIL:
+    return TAG(ptr, NIL);
+  case TAG_SMI:
+    return tag_smi((i64)ptr);
+  case TAG_SYM:
+    return tag_sym(ptr);
   case TAG_CONS:
-    // Delete the cons
-    free(as_cons(item));
+    return tag_cons(ptr);
+  case TAG_VEC:
+    return tag_vec(ptr);
+  case TAG_STR:
+    return tag_str(ptr);
+  default:
+    FAIL("Unreachable");
+    return NIL;
+  }
+}
+
+tag_t tag_get(const lisp_t *lisp)
+{
+  return GET_TAG(lisp);
+}
+
+i64 as_smi(lisp_t *obj)
+{
+  assert(IS_TAG(obj, SMI));
+  u64 raw_obj = UNTAG(obj);
+  u64 msb     = (NTH_BYTE(raw_obj, 6) & 0x80) >> 7;
+  msb         = ((1LU << 8) - msb) << 56;
+  return (i64)(raw_obj | msb);
+}
+
+char *as_sym(lisp_t *obj)
+{
+  assert(IS_TAG(obj, SYM));
+  return (char *)UNTAG(obj);
+}
+
+cons_t *as_cons(lisp_t *obj)
+{
+  assert(IS_TAG(obj, CONS));
+  return (cons_t *)UNTAG(obj);
+}
+
+str_t *as_str(lisp_t *obj)
+{
+  assert(IS_TAG(obj, STR));
+  return (str_t *)UNTAG(obj);
+}
+
+vec_t *as_vec(lisp_t *obj)
+{
+  assert(IS_TAG(obj, VEC));
+  return (vec_t *)UNTAG(obj);
+}
+
+void lisp_print(FILE *fp, lisp_t *lisp)
+{
+  if (!fp)
+    return;
+  static_assert(NUM_TAGS == 6);
+  switch (tag_get(lisp))
+  {
+  case TAG_NIL:
+    fprintf(fp, "NIL");
     break;
+  case TAG_SMI:
+#if VERBOSE_LOGS == 2
+    fprintf(fp, "INT[");
+#endif
+    fprintf(fp, "%ld", as_smi(lisp));
+#if VERBOSE_LOGS == 2
+    fprintf(fp, "]");
+#endif
+    break;
+  case TAG_SYM:
+#if VERBOSE_LOGS == 2
+    fprintf(fp, "SYM[");
+#endif
+    fprintf(fp, "%s", as_sym(lisp));
+#if VERBOSE_LOGS == 2
+    fprintf(fp, "]");
+#endif
+    break;
+  case TAG_CONS:
+  {
+#if VERBOSE_LOGS == 2
+    fprintf(fp, "LIST[");
+#else
+    fprintf(fp, "(");
+#endif
+    for (; lisp; lisp = CDR(lisp))
+    {
+      if (IS_TAG(lisp, CONS))
+      {
+        lisp_t *car = CAR(lisp);
+        lisp_t *cdr = CDR(lisp);
+
+        lisp_print(fp, car);
+        if (cdr && !IS_TAG(cdr, CONS))
+        {
+          fprintf(fp, " . ");
+        }
+        else if (cdr)
+        {
+          fprintf(fp, " ");
+        }
+      }
+      else
+      {
+        lisp_print(fp, lisp);
+        break;
+      }
+    }
+#if VERBOSE_LOGS == 2
+    fprintf(fp, "]");
+#else
+    fprintf(fp, ")");
+#endif
+    break;
+  }
   case TAG_VEC:
   {
-    vec_t *vec = as_vec(item);
-    vec_free(vec);
-    free(vec);
+#if VERBOSE_LOGS == 2
+    fprintf(fp, "VEC[");
+#else
+    fprintf(fp, "[");
+#endif
+
+    vec_t *vec = as_vec(lisp);
+    FOR_VEC(i, vec, lisp_t *)
+    {
+      lisp_t *item = VEC_GET(vec, i, lisp_t *);
+      lisp_print(fp, item);
+      if (i < VEC_SIZE(vec, lisp_t *) - 1)
+      {
+        fprintf(fp, " ");
+      }
+    }
+
+#if VERBOSE_LOGS == 2
+    fprintf(fp, "]");
+#else
+    fprintf(fp, "]");
+#endif
     break;
   }
-  case TAG_NIL:
-  case TAG_INT:
-  case TAG_SYM:
-  case NUM_TAGS:
-    // shouldn't be dealt with (either constant or dealt with elsewhere)
+  case TAG_STR:
+  {
+#if VERBOSE_LOGS == 2
+    fprintf(fp, "STR[");
+#else
+    fprintf(fp, "\"");
+#endif
+
+    sv_t sv = string_sv(as_str(lisp));
+    fprintf(fp, PR_SV, SV_FMT(sv));
+
+#if VERBOSE_LOGS == 2
+    fprintf(fp, "]");
+#else
+    fprintf(fp, "\"");
+#endif
     break;
   }
+  default:
+    FAIL("Unreachable");
+    break;
+  }
+}
+
+u64 tag_sizeof(tag_t tag)
+{
+  static_assert(NUM_TAGS == 6);
+  switch (tag)
+  {
+  case TAG_NIL:
+    return 0;
+  case TAG_SMI:
+  case TAG_SYM:
+    return sizeof(lisp_t *);
+  case TAG_CONS:
+    return sizeof(cons_t);
+  case TAG_VEC:
+    return sizeof(vec_t);
+  case TAG_STR:
+    return sizeof(str_t);
+  default:
+    FAIL("Unreachable");
+    return 0;
+  }
+}
+
+u64 lisp_sizeof(lisp_t *lisp)
+{
+  return tag_sizeof(tag_get(lisp));
+}
+
+lisp_t *lisp_reset(lisp_t *lisp)
+{
+  switch (tag_get(lisp))
+  {
+  case TAG_NIL:
+  case TAG_SMI:
+  case TAG_SYM:
+    // Nothing to "reset" here.
+    return lisp;
+  case TAG_CONS:
+  {
+    // Make `car` and `cons` NIL
+    CAR(lisp) = NIL;
+    CDR(lisp) = NIL;
+    return lisp;
+  }
+  case TAG_VEC:
+  {
+    vec_reset(as_vec(lisp));
+    return lisp;
+  }
+  case TAG_STR:
+  {
+    vec_reset(&as_str(lisp)->data);
+    return lisp;
+  }
+  default:
+  {
+    FAIL("Unreachable");
+    return lisp;
+  }
+  }
 }
 
 /* Copyright (C) 2025, 2026 Aryadev Chavali

src/main.c

@@ -11,27 +11,66 @@
 
 #include <alisp/alisp.h>
 
-void usage(FILE *fp)
-{
-  fprintf(fp, "Usage: alisp [OPTIONS...] FILE\n"
-              "Options:\n"
-              "\t--help        Print this usage and exit.\n"
-              "File:\n"
-              "\t<filename>  Read and interpret this file from filesystem.\n"
-              "\t--          Read and interpret from stdin using an EOF.\n");
-}
+void usage(FILE *fp);
+int init_stream_on_args(int argc, char *argv[], FILE **pipe, stream_t *stream);
 
 int main(int argc, char *argv[])
 {
   int ret         = 0;
   FILE *pipe      = NULL;
   stream_t stream = {0};
+  vec_t ast       = {0};
+  sys_t sys       = {0};
 
+  ret = init_stream_on_args(argc, argv, &pipe, &stream);
+  if (ret)
+    goto end;
+
+  LOG("[INFO]: Initialised stream for `%s`\n", stream.name);
+  {
+    read_err_t err = read_all(&sys, &stream, &ast);
+    if (err)
+    {
+      u64 line = 0, col = 0;
+      stream_line_col(&stream, &line, &col);
+      fprintf(stderr, "%s:%lu:%lu: ERROR: %s\n", stream.name, line, col,
+              read_err_to_cstr(err));
+      ret = 1;
+      goto end;
+    }
+  }
+
+  LOG("[INFO]: Utilised %lu bytes in parsing\n", sys_cost(&sys));
+  LOG("[INFO]: Parsed %lu %s\n", VEC_SIZE(&ast, lisp_t *),
+      VEC_SIZE(&ast, lisp_t *) == 1 ? "expr" : "exprs");
+
+  {
+    FOR_VEC(i, &ast, lisp_t *)
+    {
+#if VERBOSE_LOGS
+      lisp_t *expr = VEC_GET(&ast, i, lisp_t *);
+      printf("\t[%lu]: ", i);
+      lisp_print(stdout, expr);
+      printf("\n");
+#endif
+    }
+  }
+
+end:
+  sys_free(&sys);
+  vec_free(&ast);
+  stream_free(&stream);
+  if (pipe)
+    fclose(pipe);
+  return ret;
+}
+
+int init_stream_on_args(int argc, char *argv[], FILE **pipe, stream_t *stream)
+{
   if (argc == 1)
   {
     usage(stderr);
-    ret = 1;
-    goto end;
+    return 1;
   }
   else if (argc != 2)
   {
@@ -40,39 +79,42 @@ int main(int argc, char *argv[])
 
   if (strncmp(argv[1], "--", 2) == 0)
   {
-    stream_err_t err = stream_init_pipe(&stream, "stdin", stdin);
+    stream_err_t err = stream_init_pipe(stream, "stdin", stdin);
     if (err)
     {
       fprintf(stderr, "ERROR: %s from `%s`\n", stream_err_to_cstr(err),
               argv[1]);
-      ret = 1;
-      goto end;
+      return 1;
     }
   }
   else if (strncmp(argv[1], "--help", 6) == 0)
   {
     usage(stdout);
-    goto end;
+    return 0;
   }
   else
   {
-    pipe             = fopen(argv[1], "rb");
-    stream_err_t err = stream_init_file(&stream, argv[1], pipe);
+    *pipe            = fopen(argv[1], "rb");
+    stream_err_t err = stream_init_file(stream, argv[1], *pipe);
     if (err)
     {
       fprintf(stderr, "ERROR: %s from `%s`\n", stream_err_to_cstr(err),
               argv[1]);
-      ret = 1;
-      goto end;
+      return 1;
     }
   }
 
-  LOG("[INFO]: Initialised stream for `%s`\n", stream.name);
-end:
-  stream_stop(&stream);
-  if (pipe)
-    fclose(pipe);
-  return ret;
+  return 0;
+}
+
+void usage(FILE *fp)
+{
+  fprintf(fp, "Usage: alisp [OPTIONS...] FILE\n"
+              "Options:\n"
+              "\t--help        Print this usage and exit.\n"
+              "File:\n"
+              "\t<filename>  Read and interpret this file from filesystem.\n"
+              "\t--          Read and interpret from stdin using an EOF.\n");
 }
 
 /* Copyright (C) 2025, 2026 Aryadev Chavali

src/reader.c

@@ -8,7 +8,6 @@
 #include <ctype.h>
 #include <string.h>
 
-#include <alisp/base.h>
 #include <alisp/reader.h>
 
 const char *read_err_to_cstr(read_err_t err)
@@ -17,18 +16,271 @@ const char *read_err_to_cstr(read_err_t err)
   {
   case READ_ERR_OK:
     return "OK";
-    break;
   case READ_ERR_EOF:
     return "EOF";
-    break;
   case READ_ERR_UNKNOWN_CHAR:
     return "UNKNOWN_CHAR";
-    break;
+  case READ_ERR_EXPECTED_CLOSED_BRACE:
+    return "EXPECTED_CLOSED_BRACE";
+  case READ_ERR_EXPECTED_CLOSED_SQUARE_BRACKET:
+    return "EXPECTED_CLOSED_SQUARE_BRACKET";
+  case READ_ERR_EXPECTED_CLOSING_SPEECHMARKS:
+    return "EXPECTED_CLOSING_SPEECHMARKS";
+  case READ_ERR_UNEXPECTED_CLOSED_BRACE:
+    return "UNEXPECTED_CLOSED_BRACE";
+  case READ_ERR_UNEXPECTED_CLOSED_SQUARE_BRACKET:
+    return "UNEXPECTED_CLOSED_SQUARE_BRACKET";
   default:
     FAIL("Unreachable");
   }
 }
 
+// Accepted characters for symbols.
+static const char *SYMBOL_CHARS =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!$%&*+,-./"
+    ":<=>?@\\^_{|}~0123456789";
+
+// Little predicate using SYMBOL_CHARS
+bool is_sym(char c)
+{
+  return strchr(SYMBOL_CHARS, c) != NULL;
+}
+
+void skip_comments_and_whitespace(stream_t *stream)
+{
+  for (char c = stream_peek(stream); c != '\0' && (isspace(c) || c == ';');
+       c      = stream_peek(stream))
+  {
+    stream_while(stream, " \t\n\0");
+    if (stream_peek(stream) == ';')
+    {
+      // Skip till newline
+      stream_till(stream, "\n");
+    }
+  }
+}
+
+read_err_t read_sym(sys_t *sys, stream_t *stream, lisp_t **ret)
+{
+  sv_t sym_sv = stream_while(stream, SYMBOL_CHARS);
+  *ret        = intern(sys, sym_sv);
+  return READ_ERR_OK;
+}
+
+read_err_t read_int(sys_t *sys, stream_t *stream, lisp_t **ret)
+{
+  sv_t digits_sv = stream_while(stream, "0123456789");
+  if (is_sym(stream_peek(stream)))
+  {
+    // This is actually a symbol
+    stream_seek_backward(stream, digits_sv.size);
+    return read_sym(sys, stream, ret);
+  }
+
+  if (digits_sv.size >= 18)
+  {
+    TODO("alisp doesn't support big integers (bigger than 56 bits) yet");
+  }
+
+  i64 n = 0;
+  for (u64 i = 0; i < digits_sv.size; ++i)
+  {
+    char c   = digits_sv.data[i];
+    u8 digit = c - '0';
+
+    // NOTE: 10i + digit > INT_MAX
+    // => 10i > INT_MAX - digit
+    // => i > (INT_MAX - digit) / 10
+    if (n > (INT_MAX - digit) / 10)
+    {
+      TODO("alisp doesn't support big integers (bigger than 56 bits) yet");
+    }
+
+    n *= 10;
+    n += digit;
+  }
+
+  *ret = make_int(n);
+  return READ_ERR_OK;
+}
+
+read_err_t read_negative(sys_t *sys, stream_t *stream, lisp_t **ret)
+{
+  char c = stream_next(stream);
+  if (isdigit(c))
+  {
+    read_err_t err = read_int(sys, stream, ret);
+    if (err)
+      return err;
+    *ret = make_int(as_smi(*ret) * -1);
+    return READ_ERR_OK;
+  }
+  else if (is_sym(c) || isspace(c))
+  {
+    stream_seek_backward(stream, 1);
+    return read_sym(sys, stream, ret);
+  }
+  else
+  {
+    return READ_ERR_UNKNOWN_CHAR;
+  }
+}
+
+read_err_t read_list(sys_t *sys, stream_t *stream, lisp_t **ret)
+{
+  u64 old_pos = stream->position;
+  // skip past the open parentheses '('
+  (void)stream_next(stream);
+
+  lisp_t *top = NIL;
+  lisp_t *cur = NIL;
+  while (!stream_eoc(stream) && stream_peek(stream) != ')')
+  {
+    lisp_t *item   = NIL;
+    read_err_t err = read(sys, stream, &item);
+    if (err == READ_ERR_EOF)
+    {
+      goto no_close_brace;
+    }
+    else if (err)
+    {
+      return err;
+    }
+    else if (!top)
+    {
+      top = cons(sys, item, NIL);
+      cur = top;
+    }
+    else
+    {
+      as_cons(cur)->cdr = cons(sys, item, NIL);
+      cur               = cdr(cur);
+    }
+  }
+
+  if (stream_peek(stream) != ')')
+  {
+    goto no_close_brace;
+  }
+
+  stream_next(stream);
+  *ret = top;
+  return READ_ERR_OK;
+no_close_brace:
+  stream->position = old_pos;
+  return READ_ERR_EXPECTED_CLOSED_BRACE;
+}
+
+read_err_t read_vec(sys_t *sys, stream_t *stream, lisp_t **ret)
+{
+  u64 old_pos = stream->position;
+  (void)stream_next(stream);
+
+  lisp_t *container = make_vec(sys, 0);
+  while (!stream_eoc(stream) && stream_peek(stream) != ']')
+  {
+    lisp_t *item   = NIL;
+    read_err_t err = read(sys, stream, &item);
+    if (err == READ_ERR_EOF)
+    {
+      goto no_close_square_bracket;
+    }
+    else if (err)
+    {
+      return err;
+    }
+    else
+    {
+      vec_append(as_vec(container), &item, sizeof(item));
+    }
+  }
+
+  if (stream_peek(stream) != ']')
+    goto no_close_square_bracket;
+
+  stream_next(stream);
+  *ret = container;
+  return READ_ERR_OK;
+no_close_square_bracket:
+  stream->position = old_pos;
+  return READ_ERR_EXPECTED_CLOSED_SQUARE_BRACKET;
+}
+
+read_err_t read_str(sys_t *sys, stream_t *stream, lisp_t **ret)
+{
+  u64 old_pos = stream->position;
+
+  (void)stream_next(stream);
+  sv_t contents = stream_till(stream, "\"");
+  if (stream_eoc(stream) || stream_peek(stream) != '\"')
+  {
+    stream->position = old_pos;
+    return READ_ERR_EXPECTED_CLOSING_SPEECHMARKS;
+  }
+
+  stream_next(stream);
+  lisp_t *lisp = make_str(sys, contents.size);
+  vec_append(&as_str(lisp)->data, contents.data, contents.size);
+  *ret = lisp;
+  return READ_ERR_OK;
+}
+
+read_err_t read_quote(sys_t *sys, stream_t *stream, lisp_t **ret)
+{
+  lisp_t *to_quote = NIL;
+  stream_next(stream);
+  read_err_t err = read(sys, stream, &to_quote);
+  if (err)
+    return err;
+  lisp_t *items[] = {intern(sys, SV_AUTO("quote")), to_quote};
+  *ret            = make_list(sys, items, ARRSIZE(items));
+  return READ_ERR_OK;
+}
+
+read_err_t read_all(sys_t *sys, stream_t *stream, vec_t *out)
+{
+  while (!stream_eoc(stream))
+  {
+    lisp_t *item   = NIL;
+    read_err_t err = read(sys, stream, &item);
+    if (err)
+      return err;
+    else
+      vec_append(out, &item, sizeof(item));
+    skip_comments_and_whitespace(stream);
+  }
+
+  return READ_ERR_OK;
+}
+
+read_err_t read(sys_t *sys, stream_t *stream, lisp_t **ret)
+{
+  skip_comments_and_whitespace(stream);
+  if (stream_eoc(stream))
+    return READ_ERR_EOF;
+  char c = stream_peek(stream);
+  if (isdigit(c))
+    return read_int(sys, stream, ret);
+  else if (c == '-')
+    return read_negative(sys, stream, ret);
+  else if (is_sym(c))
+    return read_sym(sys, stream, ret);
+  else if (c == '\'')
+    return read_quote(sys, stream, ret);
+  else if (c == '(')
+    return read_list(sys, stream, ret);
+  else if (c == ')')
+    return READ_ERR_UNEXPECTED_CLOSED_BRACE;
+  else if (c == '[')
+    return read_vec(sys, stream, ret);
+  else if (c == ']')
+    return READ_ERR_UNEXPECTED_CLOSED_SQUARE_BRACKET;
+  else if (c == '\"')
+    return read_str(sys, stream, ret);
+
+  return READ_ERR_UNKNOWN_CHAR;
+}
+
 /* Copyright (C) 2026 Aryadev Chavali
 
  * This program is distributed in the hope that it will be useful, but WITHOUT

src/stream.c

@@ -11,6 +11,8 @@
 
 #include <alisp/base.h>
 #include <alisp/stream.h>
+#include <alisp/sv.h>
+#include <alisp/vec.h>
 
 const char *stream_err_to_cstr(stream_err_t err)
 {
@@ -18,19 +20,14 @@ const char *stream_err_to_cstr(stream_err_t err)
   {
   case STREAM_ERR_INVALID_PTR:
     return "INVALID PTR";
-    break;
   case STREAM_ERR_FILE_NONEXISTENT:
     return "FILE NONEXISTENT";
-    break;
   case STREAM_ERR_FILE_READ:
     return "FILE READ";
-    break;
   case STREAM_ERR_PIPE_NONEXISTENT:
     return "PIPE NONEXISTENT";
-    break;
   case STREAM_ERR_OK:
     return "OK";
-    break;
   default:
     FAIL("Unreachable");
   }
@@ -65,8 +62,6 @@ stream_err_t stream_init_pipe(stream_t *stream, const char *name, FILE *pipe)
   stream->name      = name;
   stream->pipe.file = pipe;
 
-  vec_init(&stream->pipe.cache, STREAM_DEFAULT_CHUNK);
-
   return STREAM_ERR_OK;
 }
 
@@ -82,26 +77,41 @@ stream_err_t stream_init_file(stream_t *stream, const char *name, FILE *pipe)
 
   stream->type      = STREAM_TYPE_FILE;
   stream->name      = name;
-  stream->pipe.file = pipe;
+  stream->pipe.file = NULL;
+
+  // NOTE: We're reading all the data from the file descriptor now.
+  fseek(pipe, 0, SEEK_END);
+  long size = ftell(pipe);
+  fseek(pipe, 0, SEEK_SET);
+  vec_ensure_free(&stream->pipe.cache, size);
+  int read = fread(vec_data(&stream->pipe.cache), 1, size, pipe);
+
+  // These must be equivalent for this function.
+  assert(read == size);
+  stream->pipe.cache.size += size;
 
   return STREAM_ERR_OK;
 }
 
-void stream_stop(stream_t *stream)
+void stream_reset(stream_t *stream)
+{
+  if (!stream)
+    return;
+  stream->position = 0;
+}
+
+void stream_free(stream_t *stream)
 {
   if (!stream)
     return;
   switch (stream->type)
   {
   case STREAM_TYPE_STRING:
-    free(stream->string.data);
+    free((char *)stream->string.data);
     break;
   case STREAM_TYPE_FILE:
-    // ensure we reset the FILE pointer to the start
-    fseek(stream->pipe.file, 0, SEEK_SET);
-    // fallthrough
   case STREAM_TYPE_PIPE:
-    // Must cleanup vector
+    // Must cleanup caching vector
     vec_free(&stream->pipe.cache);
     break;
   }
@@ -124,31 +134,15 @@ u64 stream_size(stream_t *stream)
   }
 }
 
-bool stream_eos(stream_t *stream)
-{
-  assert(stream);
-  switch (stream->type)
-  {
-  case STREAM_TYPE_STRING:
-    return stream->position >= stream->string.size;
-  case STREAM_TYPE_PIPE:
-  case STREAM_TYPE_FILE:
-    return feof(stream->pipe.file);
-  default:
-    FAIL("Unreachable");
-    return 0;
-  }
-}
-
 bool stream_eoc(stream_t *stream)
 {
   assert(stream);
   switch (stream->type)
   {
-  case STREAM_TYPE_STRING:
-    return stream->position >= stream->string.size;
-  case STREAM_TYPE_PIPE:
   case STREAM_TYPE_FILE:
+  case STREAM_TYPE_STRING:
+    return stream->position >= stream_size(stream);
+  case STREAM_TYPE_PIPE:
     return feof(stream->pipe.file) &&
            stream->position >= stream->pipe.cache.size;
   default:
@@ -160,23 +154,20 @@ bool stream_eoc(stream_t *stream)
 bool stream_chunk(stream_t *stream)
 {
   assert(stream);
-  u64 to_read = STREAM_DEFAULT_CHUNK;
   switch (stream->type)
   {
-  case STREAM_TYPE_STRING:
-    // vacuously true
-    return true;
-  case STREAM_TYPE_PIPE:
-    to_read = 1;
-    // fallthrough
   case STREAM_TYPE_FILE:
+  case STREAM_TYPE_STRING:
+    // nothing to chunk, hence false
+    return false;
+  case STREAM_TYPE_PIPE:
   {
     if (feof(stream->pipe.file))
       // We can't read anymore.  End of the line
       return false;
-    vec_ensure_free(&stream->pipe.cache, to_read);
+    vec_ensure_free(&stream->pipe.cache, STREAM_DEFAULT_CHUNK);
     int read = fread(vec_data(&stream->pipe.cache) + stream->pipe.cache.size, 1,
-                     to_read, stream->pipe.file);
+                     STREAM_DEFAULT_CHUNK, stream->pipe.file);
 
     // If we read something it's a good thing
     if (read > 0)
@@ -185,8 +176,10 @@ bool stream_chunk(stream_t *stream)
       return true;
     }
     else
+    {
       return false;
     }
+  }
   default:
     FAIL("Unreachable");
     return 0;
@@ -202,24 +195,20 @@ char stream_next(stream_t *stream)
 
 char stream_peek(stream_t *stream)
 {
-  // If we've reached end of stream, and end of content, there's really nothing
-  // to check here.
+  // End of the line?  We're done.
   if (stream_eoc(stream))
     return '\0';
 
   switch (stream->type)
   {
-  case STREAM_TYPE_STRING:
-    return stream->string.data[stream->position];
-  case STREAM_TYPE_PIPE:
   case STREAM_TYPE_FILE:
+  case STREAM_TYPE_STRING:
+    return stream_sv(stream).data[0];
+  case STREAM_TYPE_PIPE:
   {
     // Cached already?  We are done.
     if (stream->position < stream->pipe.cache.size)
-    {
-      const char *const str = (char *)vec_data(&stream->pipe.cache);
-      return str[stream->position];
-    }
+      return stream_sv(stream).data[0];
 
     // Try to read chunks in till we've reached it or we're at the end of the
     // file.
@@ -231,7 +220,7 @@ char stream_peek(stream_t *stream)
     // Same principle as the stream_eos(stream) check.
     if (stream->position >= stream->pipe.cache.size)
       return '\0';
-    return ((char *)vec_data(&stream->pipe.cache))[stream->position];
+    return stream_sv(stream).data[0];
   }
   default:
     FAIL("Unreachable");
@@ -246,36 +235,38 @@ u64 stream_seek(stream_t *stream, i64 offset)
   else if (offset > 0)
     return stream_seek_forward(stream, offset);
   else
-    // vacuously successful
-    return true;
+    return 0;
 }
 
 u64 stream_seek_forward(stream_t *stream, u64 offset)
 {
   if (stream_eoc(stream))
     return 0;
+  else if (stream->position + offset < stream_size(stream))
+  {
+    stream->position += offset;
+    return offset;
+  }
 
+  // NOTE: The only case not caught by the above branches is exact-to-end
+  // movement (i.e. offset puts us exactly at the end of the stream) or movement
+  // beyond what we've cached.
   switch (stream->type)
   {
+  case STREAM_TYPE_FILE:
   case STREAM_TYPE_STRING:
   {
-    if (stream->position + offset >= stream->string.size)
-      return 0;
-
+    // Clamp in the case of FILE and STRING movement since they're already
+    // fully cached.
+    if (stream->position + offset >= stream_size(stream))
+      offset = stream_size(stream) - stream->position;
     stream->position += offset;
     return offset;
   }
   case STREAM_TYPE_PIPE:
-  case STREAM_TYPE_FILE:
   {
-    // Similar principle as stream_peek really...
-
-    // Cached already?  We are done.
-    if (stream->position + offset < stream->pipe.cache.size)
-    {
-      stream->position += offset;
-      return offset;
-    }
+    // Pipes may have data remaining that hasn't been cached - we need to chunk
+    // before we can be sure to stop.
 
     // Try to read chunks in till we've reached it or we're at the end of the
     // file.
@@ -284,32 +275,59 @@ u64 stream_seek_forward(stream_t *stream, u64 offset)
          read_chunk = stream_chunk(stream))
       continue;
 
-    // Same principle as the stream_eoc(stream) check.
-    if (stream->position + offset > stream->pipe.cache.size)
-    {
-      offset = stream->pipe.cache.size - stream->position;
-    }
+    // NOTE: We've read everything from the pipe, but the offset is greater.  We
+    // must clamp here.
+    if (stream->position + offset > stream_size(stream))
+      offset = stream_size(stream) - stream->position;
+
     stream->position += offset;
     return offset;
   }
   default:
     FAIL("Unreachable");
-    return 0;
   }
+  return 0;
 }
 
 u64 stream_seek_backward(stream_t *stream, u64 offset)
 {
-  assert(stream);
+  if (!stream)
+    return 0;
   if (stream->position < offset)
-  {
     offset = stream->position;
-  }
 
   stream->position -= offset;
   return offset;
 }
 
+sv_t stream_sv(stream_t *stream)
+{
+  sv_t sv = stream_sv_abs(stream);
+  sv      = sv_chop_left(sv, stream->position);
+  return sv;
+}
+
+sv_t stream_sv_abs(stream_t *stream)
+{
+  if (!stream)
+    return SV(NULL, 0);
+  sv_t sv = {0};
+  switch (stream->type)
+  {
+  case STREAM_TYPE_STRING:
+    sv = stream->string;
+    break;
+  case STREAM_TYPE_FILE:
+  case STREAM_TYPE_PIPE:
+    sv = SV((char *)vec_data(&stream->pipe.cache), stream_size(stream));
+    break;
+  default:
+    FAIL("Unreachable");
+    return SV(NULL, 0);
+  }
+  return sv;
+}
+
 sv_t stream_substr(stream_t *stream, u64 size)
 {
   if (stream_eoc(stream))
@@ -324,104 +342,130 @@ sv_t stream_substr(stream_t *stream, u64 size)
   if (successful != size)
     return SV(NULL, 0);
 
-  char *ptr = NULL;
-  switch (stream->type)
-  {
-  case STREAM_TYPE_STRING:
-    ptr = stream->string.data;
-    break;
-  case STREAM_TYPE_PIPE:
-  case STREAM_TYPE_FILE:
-    ptr = (char *)vec_data(&stream->pipe.cache);
-    break;
-  default:
-    FAIL("Unreachable");
-    return SV(NULL, 0);
-  }
-
-  return SV(ptr + stream->position, size);
+  sv_t sv = stream_sv(stream);
+  sv      = sv_truncate(sv, size);
+  return sv;
 }
 
 sv_t stream_substr_abs(stream_t *stream, u64 index, u64 size)
 {
   switch (stream->type)
   {
-  case STREAM_TYPE_STRING:
-    if (index + size <= stream_size(stream))
-      return SV(stream->string.data + index, size);
-    return SV(NULL, 0);
   case STREAM_TYPE_PIPE:
-  case STREAM_TYPE_FILE:
   {
-    if (index + size <= stream_size(stream))
-      return SV((char *)vec_data(&stream->pipe.cache) + index, size);
-    // (index + size > stream_size(stream)) => try reading chunks
+    if (index + size > stream_size(stream))
+    {
+      // => try reading chunks till either we drop or we have enough space
       for (bool read_chunk = stream_chunk(stream);
            read_chunk && index + size >= stream->pipe.cache.size;
            read_chunk = stream_chunk(stream))
         continue;
-
-    if (index + size > stream_size(stream))
-      return SV(NULL, 0);
-    return SV((char *)vec_data(&stream->pipe.cache) + index, size);
     }
+    break;
+  }
+  case STREAM_TYPE_STRING:
+  case STREAM_TYPE_FILE:
+    break;
   default:
-    assert("Unreachable");
-    return SV(NULL, 0);
+    FAIL("Unreachable");
   }
+
+  sv_t sv = stream_sv_abs(stream);
+  sv      = sv_chop_left(sv, index);
+  sv      = sv_truncate(sv, size);
+  return sv;
 }
 
 sv_t stream_till(stream_t *stream, const char *str)
 {
   if (stream_eoc(stream))
     return SV(NULL, 0);
-  u64 current_position = stream->position;
-  for (char c = stream_peek(stream); c != '\0' && strchr(str, c) == NULL;
-       c      = stream_next(stream))
-    continue;
-  u64 size = stream->position - current_position;
-  if (size == 0)
-    return SV(NULL, 0);
-  return stream_substr_abs(stream, current_position, size - 1);
+
+  sv_t cur_sv = stream_sv(stream);
+  sv_t sv     = sv_till(cur_sv, str);
+  stream_seek_forward(stream, sv.size);
+  switch (stream->type)
+  {
+  case STREAM_TYPE_FILE:
+  case STREAM_TYPE_STRING:
+    return sv;
+  case STREAM_TYPE_PIPE:
+  {
+    if (cur_sv.size > sv.size)
+      return sv;
+
+    // Build a substring by hand while chunking data.
+    u64 index, size;
+    for (index = stream->position - sv.size, size = sv.size;
+         cur_sv.size == sv.size; size += sv.size)
+    {
+      cur_sv = stream_sv(stream);
+      sv     = sv_till(cur_sv, str);
+      stream_seek_forward(stream, sv.size);
+      if (sv.size == 0)
+        // Must stop if this has happened; nothing else to pick up.
+        break;
+    }
+    return stream_substr_abs(stream, index, size);
+  }
+  default:
+    FAIL("Unreachable");
+  }
 }
 
 sv_t stream_while(stream_t *stream, const char *str)
 {
   if (stream_eoc(stream))
     return SV(NULL, 0);
-  u64 current_position = stream->position;
-  for (char c = stream_peek(stream); c != '\0' && strchr(str, c);
-       c      = stream_next(stream))
-    continue;
-  u64 size = stream->position - current_position;
-  if (size == 0)
-    return SV(NULL, 0);
-  return stream_substr_abs(stream, current_position, size - 1);
+
+  sv_t cur_sv = stream_sv(stream);
+  sv_t sv     = sv_while(cur_sv, str);
+  stream_seek_forward(stream, sv.size);
+  switch (stream->type)
+  {
+  case STREAM_TYPE_FILE:
+  case STREAM_TYPE_STRING:
+    return sv;
+  case STREAM_TYPE_PIPE:
+  {
+    if (cur_sv.size > sv.size)
+      return sv;
+
+    // Build a substring by hand while chunking data.
+    u64 index, size;
+    for (index = stream->position - sv.size, size = sv.size;
+         cur_sv.size == sv.size; size += sv.size)
+    {
+      cur_sv = stream_sv(stream);
+      sv     = sv_while(cur_sv, str);
+      stream_seek_forward(stream, sv.size);
+      if (sv.size == 0)
+        // Must stop if this has happened; nothing else to pick up.
+        break;
+    }
+    return stream_substr_abs(stream, index, size);
+  }
+  default:
+    FAIL("Unreachable");
+  }
 }
 
 void stream_line_col(stream_t *stream, u64 *line, u64 *col)
 {
   if (!stream || !line || !col)
     return;
-  // Go through the cache, byte by byte.
-  char *cache = NULL;
-  u64 size    = 0;
-  if (stream->type == STREAM_TYPE_STRING)
-  {
-    cache = stream->string.data;
-    size  = stream->string.size;
-  }
-  else
-  {
-    cache = (char *)vec_data(&stream->pipe.cache);
-    size  = stream->pipe.cache.size;
-  }
+
+  // Generate a string view from the stream of exactly the content /upto/
+  // stream.postion.
+  sv_t sv = stream_sv_abs(stream);
+  sv      = sv_truncate(sv, stream->position + 1);
 
   *line = 1;
   *col  = 0;
-  for (u64 i = 0; i < size; ++i)
+  // TODO: Could this be faster?  Does it matter?
+  for (u64 i = 0; i < sv.size; ++i)
   {
-    char c = cache[i];
+    char c = sv.data[i];
     if (c == '\n')
     {
       *line += 1;

src/string.c

@@ -0,0 +1,44 @@
+/* string.c: String library implementation
+ * Created: 2026-03-05
+ * Author: Aryadev Chavali
+ * License: See end of file
+ * Commentary:
+ */
+
+#include <string.h>
+
+#include <alisp/string.h>
+
+str_t string_make(sv_t sv)
+{
+  str_t string = {0};
+  if (sv.size)
+  {
+    vec_init(&string.data, sv.size);
+    if (sv.data)
+    {
+      memcpy(vec_data(&string.data), sv.data, sv.size);
+    }
+  }
+  return string;
+}
+
+sv_t string_sv(str_t *str)
+{
+  if (!str)
+    return SV(NULL, 0);
+  return SV((char *)vec_data(&str->data), str->data.size);
+}
+
+/* Copyright (C) 2026 Aryadev Chavali
+
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License Version 2 for
+ * details.
+
+ * You may distribute and modify this code under the terms of the GNU General
+ * Public License Version 2, which you should have received a copy of along with
+ * this program.  If not, please go to <https://www.gnu.org/licenses/>.
+
+ */

src/sv.c

@@ -22,6 +22,59 @@ sv_t sv_copy(sv_t old)
   return SV(newstr, old.size);
 }
 
+sv_t sv_chop_left(sv_t sv, u64 size)
+{
+  if (sv.size <= size)
+    return SV(NULL, 0);
+  return SV(sv.data + size, sv.size - size);
+}
+
+sv_t sv_chop_right(sv_t sv, u64 size)
+{
+  if (sv.size <= size)
+    return SV(NULL, 0);
+  return SV(sv.data, sv.size - size);
+}
+
+sv_t sv_truncate(sv_t sv, u64 newsize)
+{
+  if (newsize > sv.size)
+    return SV(NULL, 0);
+  return SV(sv.data, newsize);
+}
+
+sv_t sv_substr(sv_t sv, u64 position, u64 size)
+{
+  sv_t result = sv_truncate(sv_chop_left(sv, position), size);
+  return result;
+}
+
+sv_t sv_till(sv_t sv, const char *reject)
+{
+  if (sv.size == 0 || !sv.data)
+    return SV(NULL, 0);
+
+  u64 offset;
+  for (offset = 0; offset < sv.size && strchr(reject, sv.data[offset]) == NULL;
+       ++offset)
+    continue;
+
+  return sv_truncate(sv, offset);
+}
+
+sv_t sv_while(sv_t sv, const char *accept)
+{
+  if (sv.size == 0 || !sv.data)
+    return SV(NULL, 0);
+
+  u64 offset;
+  for (offset = 0; offset < sv.size && strchr(accept, sv.data[offset]) != NULL;
+       ++offset)
+    continue;
+
+  return sv_truncate(sv, offset);
+}
+
 /* Copyright (C) 2025, 2026 Aryadev Chavali
 
  * This program is distributed in the hope that it will be useful, but WITHOUT

src/symtable.c

@@ -29,7 +29,7 @@ void sym_table_init(sym_table_t *table)
   vec_init(&table->entries, table->capacity * sizeof(sv_t));
 }
 
-char *sym_table_find(sym_table_t *table, sv_t sv)
+const char *sym_table_find(sym_table_t *table, sv_t sv)
 {
   // Initialise the table if it's not done already
   if (table->entries.capacity == 0)
@@ -62,13 +62,29 @@ void sym_table_free(sym_table_t *table)
   {
     current = ENTRY_GET(table, i);
     if (current.data)
-      free(current.data);
+    {
+      // NOTE: We clone all data here, so it's okay to free by hand.
+      free((void *)current.data);
+    }
   }
   // Free the underlying container
   vec_free(&table->entries);
   memset(table, 0, sizeof(*table));
 }
 
+u64 sym_table_cost(sym_table_t *table)
+{
+  if (!table || !table->count)
+    return 0;
+  else
+  {
+    u64 total_size = 0;
+    for (u64 i = 0; i < table->capacity; ++i)
+      total_size += ENTRY_GET(table, i).size;
+    return total_size;
+  }
+}
+
 /* Copyright (C) 2025, 2026 Aryadev Chavali
 
  * This program is distributed in the hope that it will be useful, but WITHOUT

src/sys.c

@@ -0,0 +1,180 @@
+/* sys.c: System implementation
+ * Created: 2026-02-12
+ * Author: Aryadev Chavali
+ * License: See end of file
+ * Commentary:
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#include <alisp/sys.h>
+
+void sys_init(sys_t *sys)
+{
+  memset(sys, 0, sizeof(*sys));
+}
+
+lisp_t *sys_alloc(sys_t *sys, tag_t type)
+{
+  static_assert(NUM_TAGS == 6);
+  switch (type)
+  {
+  case TAG_CONS:
+  case TAG_VEC:
+  case TAG_STR:
+    return alloc_make(&sys->memory, type);
+  // Shouldn't be allocated
+  case TAG_NIL:
+  case TAG_SMI:
+  case TAG_SYM:
+  default:
+    FAIL("Unreachable");
+  }
+  return NIL;
+}
+
+void sys_delete(sys_t *sys, lisp_t *lisp)
+{
+  alloc_delete(&sys->memory, lisp);
+}
+
+u64 sys_cost(sys_t *sys)
+{
+  return alloc_cost(&sys->memory) + sym_table_cost(&sys->symtable);
+}
+
+void sys_free(sys_t *sys)
+{
+  sym_table_free(&sys->symtable);
+  alloc_free(&sys->memory);
+  memset(sys, 0, sizeof(*sys));
+}
+
+lisp_t *make_int(i64 i)
+{
+  return tag_smi(i);
+}
+
+lisp_t *cons(sys_t *sys, lisp_t *car, lisp_t *cdr)
+{
+  lisp_t *cons = sys_alloc(sys, TAG_CONS);
+  CAR(cons)    = car;
+  CDR(cons)    = cdr;
+  return cons;
+}
+
+lisp_t *make_list(sys_t *sys, lisp_t **lisps, u64 size)
+{
+  lisp_t *root = NIL;
+  for (u64 i = size; i > 0; --i)
+  {
+    lisp_t *node = lisps[i - 1];
+    root         = cons(sys, node, root);
+  }
+  return root;
+}
+
+lisp_t *make_vec(sys_t *sys, u64 capacity)
+{
+  lisp_t *vec = sys_alloc(sys, TAG_VEC);
+  vec_init(as_vec(vec), capacity);
+  return vec;
+}
+
+lisp_t *make_str(sys_t *sys, u64 capacity)
+{
+  lisp_t *str = sys_alloc(sys, TAG_STR);
+  vec_init(&as_str(str)->data, capacity);
+  return str;
+}
+
+lisp_t *intern(sys_t *sys, sv_t sv)
+{
+  const char *str = sym_table_find(&sys->symtable, sv);
+  return tag_sym(str);
+}
+
+lisp_t *car(lisp_t *lsp)
+{
+  if (!IS_TAG(lsp, CONS))
+    return NIL;
+  else
+    return CAR(lsp);
+}
+
+lisp_t *cdr(lisp_t *lsp)
+{
+  if (!IS_TAG(lsp, CONS))
+    return NIL;
+  else
+    return CDR(lsp);
+}
+
+void lisp_free(sys_t *sys, lisp_t *lisp)
+{
+  static_assert(NUM_TAGS == 6);
+  switch (tag_get(lisp))
+  {
+  case TAG_STR:
+  case TAG_VEC:
+  case TAG_CONS:
+    // Delete the underlying data
+    alloc_delete(&sys->memory, lisp);
+    break;
+  case TAG_NIL:
+  case TAG_SMI:
+  case TAG_SYM:
+    // shouldn't be dealt with (either constant or dealt with elsewhere)
+    break;
+  }
+}
+
+void lisp_free_rec(sys_t *sys, lisp_t *item)
+{
+  static_assert(NUM_TAGS == 6);
+  switch (tag_get(item))
+  {
+  case TAG_CONS:
+  {
+    lisp_free_rec(sys, car(item));
+    lisp_free_rec(sys, cdr(item));
+    lisp_free(sys, item);
+    break;
+  }
+  case TAG_VEC:
+  {
+    vec_t *vec = as_vec(item);
+    FOR_VEC(i, vec, lisp_t *)
+    {
+      lisp_t *allocated = VEC_GET(vec, i, lisp_t *);
+      lisp_free_rec(sys, allocated);
+    }
+    lisp_free(sys, item);
+    break;
+  }
+  case TAG_STR:
+  {
+    lisp_free(sys, item);
+    break;
+  }
+  case TAG_NIL:
+  case TAG_SMI:
+  case TAG_SYM:
+    // shouldn't be dealt with (either constant or dealt with elsewhere)
+    break;
+  }
+}
+
+/* Copyright (C) 2026 Aryadev Chavali
+
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the GNU General Public License Version 2 for
+ * details.
+
+ * You may distribute and modify this code under the terms of the GNU General
+ * Public License Version 2, which you should have received a copy of along with
+ * this program.  If not, please go to <https://www.gnu.org/licenses/>.
+
+ */

src/tag.c

@@ -1,82 +0,0 @@
-/* tag.c: Pointer tagging
- * Created: 2025-08-19
- * Author: Aryadev Chavali
- * License: See end of file
- * Commentary:
- */
-
-#include <assert.h>
-#include <stdlib.h>
-
-#include <alisp/tag.h>
-
-lisp_t *tag_int(i64 i)
-{
-  return TAG((u64)i, INT);
-}
-
-lisp_t *tag_sym(char *str)
-{
-  return TAG((u64)str, SYM);
-}
-
-lisp_t *tag_vec(vec_t *vec)
-{
-  return TAG((u64)vec, VEC);
-}
-
-lisp_t *tag_cons(cons_t *cons)
-{
-  return TAG((u64)cons, CONS);
-}
-
-tag_t get_tag(lisp_t *lisp)
-{
-  static_assert(NUM_TAGS == 5);
-  if (!lisp)
-    return TAG_NIL;
-  else if (IS_TAG(lisp, INT))
-    return TAG_INT;
-
-  return (u64)lisp & 0xFF;
-}
-
-i64 as_int(lisp_t *obj)
-{
-  assert(IS_TAG(obj, INT));
-  u64 p_obj = (u64)obj;
-  return UNTAG(p_obj, INT) |               // Delete the tag
-         (NTH_BYTE(p_obj, 7) & 0x80) << 56 // duplicate the MSB (preserve sign)
-      ;
-}
-
-char *as_sym(lisp_t *obj)
-{
-  assert(IS_TAG(obj, SYM));
-  return (char *)UNTAG(obj, SYM);
-}
-
-cons_t *as_cons(lisp_t *obj)
-{
-  assert(IS_TAG(obj, CONS));
-  return (cons_t *)UNTAG(obj, CONS);
-}
-
-vec_t *as_vec(lisp_t *obj)
-{
-  assert(IS_TAG(obj, VEC));
-  return (vec_t *)UNTAG(obj, VEC);
-}
-
-/* Copyright (C) 2025, 2026 Aryadev Chavali
-
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- * FOR A PARTICULAR PURPOSE.  See the GNU General Public License Version 2 for
- * details.
-
- * You may distribute and modify this code under the terms of the GNU General
- * Public License Version 2, which you should have received a copy of along with
- * this program.  If not, please go to <https://www.gnu.org/licenses/>.
-
- */

src/vec.c

@@ -38,6 +38,14 @@ void vec_free(vec_t *vec)
   memset(vec, 0, sizeof(*vec));
 }
 
+void vec_reset(vec_t *vec)
+{
+  if (!vec)
+    return;
+  memset(vec_data(vec), 0, vec->capacity);
+  vec->size = 0;
+}
+
 u8 *vec_data(vec_t *vec)
 {
   return vec->not_inlined ? vec->ptr : vec->inlined;
@@ -81,10 +89,24 @@ void vec_append(vec_t *vec, const void *const ptr, u64 size)
   if (!vec)
     return;
   vec_ensure_free(vec, size);
+  if (ptr)
     memcpy(vec_data(vec) + vec->size, ptr, size);
   vec->size += size;
 }
 
+bool vec_try_append(vec_t *vec, const void *const ptr, u64 size)
+{
+  if (!vec || vec->capacity - vec->size < size)
+    return false;
+  if (ptr)
+  {
+    void *newptr = vec_data(vec) + vec->size;
+    memcpy(newptr, ptr, size);
+  }
+  vec->size += size;
+  return true;
+}
+
 void vec_clone(vec_t *dest, vec_t *src)
 {
   if (!src || !dest)

test/test_lisp_api.c

@@ -23,7 +23,7 @@ void smi_test(void)
   {
     i64 in       = ints[i];
     lisp_t *lisp = make_int(in);
-    i64 out      = as_int(lisp);
+    i64 out      = as_smi(lisp);
 
     TEST(in == out, "%ld == %ld", in, out);
   }
@@ -47,7 +47,7 @@ void smi_oob_test(void)
   {
     i64 in       = ints[i];
     lisp_t *lisp = make_int(in);
-    i64 out      = as_int(lisp);
+    i64 out      = as_smi(lisp);
 
     TEST(in != out, "%ld != %ld", in, out);
   }
@@ -84,10 +84,10 @@ void sym_unique_test(void)
   sys_init(&system);
 
   sv_t symbols[] = {
-      SV("hello", 5),
-      SV("goodbye", 7),
-      SV("display", 7),
-      SV("@xs'a_sh;d::a-h]", 16),
+      SV_AUTO("hello"),
+      SV_AUTO("goodbye"),
+      SV_AUTO("display"),
+      SV_AUTO("@xs'a_sh;d::a-h]"),
   };
 
   lisp_t *ptrs[ARRSIZE(symbols)];
@@ -151,44 +151,42 @@ void sys_test(void)
   TEST_START();
   sys_t sys = {0};
   sys_init(&sys);
-  u64 old_memory_size = sys.memory.size;
+  u64 old_memory_size = sys_cost(&sys);
 
   // Creating integers doesn't affect memory size
   (void)make_int(2000);
-  TEST(sys.memory.size == old_memory_size,
+  TEST(sys_cost(&sys) == old_memory_size,
        "Making integers doesn't affect system memory size");
 
-  // Creating symbols won't affect memory size, but does affect the symbol table
-  (void)intern(&sys, SV("hello world!", 12));
-  TEST(sys.memory.size == old_memory_size,
+  // Creating symbols does affect memory size and memory table
+  (void)intern(&sys, SV_AUTO("hello world!"));
+  TEST(sys_cost(&sys) > old_memory_size,
        "Interning doesn't affect system memory size");
   TEST(sys.symtable.count > 0, "Interning affects symbol table");
+  old_memory_size = sys_cost(&sys);
 
   // Creating conses do affect memory size
   (void)cons(&sys, make_int(1), make_int(2));
-  TEST(sys.memory.size > 0, "Creating conses affects memory size");
-  old_memory_size = sys.memory.size;
+  TEST(sys_cost(&sys) > 0, "Creating conses affects memory size");
+  old_memory_size = sys_cost(&sys);
 
-  (void)cons(&sys, intern(&sys, SV("test", 4)), NIL);
-  TEST(sys.memory.size > old_memory_size,
+  (void)cons(&sys, intern(&sys, SV_AUTO("test")), NIL);
+  TEST(sys_cost(&sys) > old_memory_size,
        "Creating conses back to back affects memory size");
-  old_memory_size = sys.memory.size;
+  old_memory_size = sys_cost(&sys);
 
   // Creating vectors does affect memory size
   (void)make_vec(&sys, 8);
-  TEST(sys.memory.size > old_memory_size,
+  TEST(sys_cost(&sys) > old_memory_size,
        "Creating vectors (size 8) affects memory size");
-  old_memory_size = sys.memory.size;
+  old_memory_size = sys_cost(&sys);
 
   (void)make_vec(&sys, 1000);
-  TEST(sys.memory.size > old_memory_size,
+  TEST(sys_cost(&sys) > old_memory_size,
        "Creating vectors (size 1000) affects memory size");
-  old_memory_size = sys.memory.size;
+  old_memory_size = sys_cost(&sys);
 
   sys_free(&sys);
-  TEST(sys.memory.size == 0, "sys_free cleans up memory (shallow check)");
-  TEST(sys.symtable.count == 0, "sys_free cleans up symtable (shallow check)");
-
   TEST_END();
 }
 

test/test_stream.c

@@ -54,7 +54,7 @@ void stream_test_prologue(void)
 
 void stream_test_epilogue(void)
 {
-  TEST_INFO("Freeing resources and deleting file `%s`\n", valid_filename);
+  TEST_INFO("Deleting file `%s`\n", valid_filename);
   assert(valid_fp);
   fclose(valid_fp);
   remove(valid_filename);
@@ -64,9 +64,9 @@ void stream_test_string(void)
 {
   TEST_START();
   sv_t test_strings[] = {
-      SV("hello, world!", 13),
-      SV("another string", 14),
-      SV((char *)text, ARRSIZE(text) / 2),
+      SV_AUTO("hello, world!"),
+      SV_AUTO("another string"),
+      sv_truncate(SV_AUTO(text), ARRSIZE(text) / 2),
   };
 
   for (u64 i = 0; i < ARRSIZE(test_strings); ++i)
@@ -82,12 +82,13 @@ void stream_test_string(void)
          test_strings[i].size);
     TEST(!stream_eoc(&stream), "Not end of content already");
 
-    stream_stop(&stream);
+    stream_free(&stream);
     TEST(strncmp(copy.data, test_strings[i].data, copy.size) == 0,
          "Freeing a stream does not free the underlying memory it was derived "
          "from");
 
-    free(copy.data);
+    // NOTE: Okay to free since we own copy.
+    free((void *)copy.data);
   }
 
   stream_t stream  = {0};
@@ -96,7 +97,7 @@ void stream_test_string(void)
        stream_err_to_cstr(err));
   TEST(stream_size(&stream) == 0, "NULL stream size is 0");
   TEST(stream_eoc(&stream), "NULL stream is always at end of content");
-  stream_stop(&stream);
+  stream_free(&stream);
 
   TEST_END();
 }
@@ -113,9 +114,8 @@ void stream_test_file(void)
       TEST(err == STREAM_ERR_OK, "Expected initialisating to be okay: %s",
            stream_err_to_cstr(err));
     }
-    TEST(stream_size(&stream) == 0, "Stream doesn't read on init: size = %lu",
-         stream_size(&stream));
     TEST(!stream_eoc(&stream), "Stream should not be at the EoC from init.");
+    stream_free(&stream);
   }
 
   // try to initialise the stream again but against a nonexistent file - we're
@@ -164,7 +164,7 @@ void stream_test_peek_next(void)
          "(%c)",
          c3, c2);
 
-    stream_stop(&stream);
+    stream_free(&stream);
   }
 
   // Invalid streams
@@ -181,7 +181,7 @@ void stream_test_peek_next(void)
          "Next on an invalid stream should not affect position (%lu -> %lu)",
          old_position, stream.position);
 
-    stream_stop(&stream);
+    stream_free(&stream);
   }
   TEST_END();
 }
@@ -210,7 +210,7 @@ void stream_test_seek(void)
          "stream (%lu -> %lu)",
          old_position, stream.position);
 
-    stream_stop(&stream);
+    stream_free(&stream);
   }
 
   // Valid streams
@@ -268,7 +268,7 @@ void stream_test_seek(void)
         "above.",
         stream.position);
 
-    stream_stop(&stream);
+    stream_free(&stream);
   }
 
   TEST_END();
@@ -302,7 +302,7 @@ void stream_test_substr(void)
            position, size);
     }
 
-    stream_stop(&stream);
+    stream_free(&stream);
   }
 
   // Taking substrings of valid streams
@@ -320,7 +320,7 @@ void stream_test_substr(void)
 
       TEST(result.size == size, "Substring has right size (%lu)", result.size);
 
-      sv_t expected = SV((char *)words_text + position, size);
+      sv_t expected = sv_substr(SV_AUTO(words_text), position, size);
       TEST(strncmp(result.data, expected.data, result.size) == 0,
            "Expect the substring to be the same as the data we put in");
     }
@@ -333,7 +333,7 @@ void stream_test_substr(void)
 
       TEST(result.size == size, "Substring has right size (%lu)", result.size);
 
-      sv_t expected = SV((char *)words_text, size);
+      sv_t expected = sv_truncate(SV_AUTO(words_text), size);
       TEST(strncmp(result.data, expected.data, result.size) == 0,
            "Expect the substring to be the same as the data we put in");
     }
@@ -350,7 +350,7 @@ void stream_test_substr(void)
 
       TEST(result.size == size, "Substring has right size (%lu)", result.size);
 
-      sv_t expected = SV((char *)words_text + position, size);
+      sv_t expected = sv_substr(SV_AUTO(words_text), position, size);
       TEST(strncmp(result.data, expected.data, result.size) == 0,
            "Expect the substring to be the same as the data we put in");
 
@@ -377,7 +377,7 @@ void stream_test_substr(void)
       }
     }
 
-    stream_stop(&stream);
+    stream_free(&stream);
   }
   TEST_END();
 }

test/test_sv.c

@@ -24,8 +24,8 @@ void sv_copy_test(void)
     TEST(strncmp(word.data, copy.data, copy.size) == 0, "`%s` == `%s`",
          word.data, copy.data);
 
-    // Obviously we can't just have this lying around.
-    free(copy.data);
+    // NOTE: Okay to free since we own copy.
+    free((void *)copy.data);
   }
 }
 

test/test_vec.c

@@ -50,13 +50,13 @@ void vec_test_gen_substr(void)
       {0, 16},
       {0, 32},
       {32, 64},
-      {0, ARRSIZE(text)},
+      {0, ARRSIZE(text) - 1},
   };
 
   for (u64 i = 0; i < ARRSIZE(tests); ++i)
   {
     struct Test test  = tests[i];
-    const sv_t substr = SV((char *)text + test.start, test.size);
+    const sv_t substr = sv_substr(SV_AUTO(text), test.start, test.size);
     const u64 size    = test.size / 2;
 
     lisp_t *lvec = make_vec(&system, size);