parser -> lexer

That's the real purpose of this module; it's not really generating an
AST since ARL's syntax isn't tree like whatsoever.

The next stage will be something closer to an AST, in the sense we'll
be introducing:
- Syntactical analysis
- Type Checking

include/arl/lexer/lexer.h

@@ -1,38 +1,38 @@
-/* parser.h: Parser which takes character buffers and yields an AST
+/* lexer.h: Lexer which takes character buffers and yields a sequence of tokens.
  * Created: 2026-01-22
  * Author: Aryadev Chavali
  * License: See end of file
  * Commentary:
  */
 
-#ifndef PARSER_H
-#define PARSER_H
+#ifndef LEXER_H
+#define LEXER_H
 
-#include <arl/parser/ast.h>
+#include <arl/lexer/token.h>
 
-/// Parser streams, utilised when generating an AST.
+/// Token streams, utilised when lexing.
 typedef struct
 {
   u64 byte;
   sv_t contents;
-} parse_stream_t;
+} lex_stream_t;
 
-/// Types of errors that may occur during parsing
+/// Types of errors that may occur during lexing
 typedef enum
 {
-  PARSE_ERR_OK = 0,
-  PARSE_ERR_EXPECTED_SPEECH_MARKS,
-  PARSE_ERR_UNKNOWN_CHAR,
-} parse_err_t;
-const char *parse_err_to_string(parse_err_t err);
+  LEX_ERR_OK = 0,
+  LEX_ERR_EXPECTED_SPEECH_MARKS,
+  LEX_ERR_UNKNOWN_CHAR,
+} lex_err_t;
+const char *lex_err_to_string(lex_err_t err);
 
-// Generates an AST from STREAM, storing it in OUT.  Returns any errors it may
-// generate.
-parse_err_t parse(ast_t *out, parse_stream_t *stream);
+// Generates a token stream from a lex_stream_t, storing it in OUT.  Returns any
+// errors it may generate.
+lex_err_t lex_stream(token_stream_t *out, lex_stream_t *stream);
 
 // Computes the line and column that STREAM is currently pointing at in its
 // buffer, storing it in LINE and COL.
-void parse_stream_get_line_col(parse_stream_t *stream, u64 *line, u64 *col);
+void lex_stream_get_line_col(lex_stream_t *stream, u64 *line, u64 *col);
 
 #endif
 

include/arl/lexer/token.h

@@ -0,0 +1,73 @@
+/* token.h: General definition of tokens, and a sequence of them.
+ * Created: 2026-01-22
+ * Author: Aryadev Chavali
+ * License: See end of file
+ * Commentary:
+ */
+
+#ifndef TOKEN_H
+#define TOKEN_H
+
+#include <arl/lib/base.h>
+#include <arl/lib/sv.h>
+#include <arl/lib/vec.h>
+
+/// Types of tokens
+typedef enum
+{
+  TOKEN_TYPE_KNOWN = 0,
+  TOKEN_TYPE_SYMBOL,
+  TOKEN_TYPE_STRING,
+
+  NUM_TOKEN_TYPES,
+} token_type_t;
+
+/// Known symbols which later stages would benefit from.
+typedef enum
+{
+  TOKEN_KNOWN_PUTSTR,
+  NUM_TOKEN_KNOWNS,
+} token_known_t;
+
+const char *token_known_to_cstr(token_known_t);
+
+/// Tokens are a tagged union
+typedef struct
+{
+  u64 byte_location;
+  token_type_t type;
+  union
+  {
+    token_known_t as_known;
+    sv_t as_symbol;
+    sv_t as_string;
+  };
+} token_t;
+
+token_t token_known(u64 byte, token_known_t known);
+token_t token_symbol(u64 byte, sv_t symbol);
+token_t token_string(u64 byte, sv_t string);
+void token_print(FILE *fp, token_t *token);
+
+/// Sequence of tokens
+typedef struct
+{
+  vec_t vec;
+} token_stream_t;
+
+void token_stream_free(token_stream_t *token);
+void token_stream_print(FILE *fp, token_stream_t *token);
+
+#endif
+
+/* Copyright (C) 2026 Aryadev Chavali
+
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE.  See the MIT License for details.
+
+ * You may distribute and modify this code under the terms of the MIT License,
+ * which you should have received a copy of along with this program.  If not,
+ * please go to <https://opensource.org/license/MIT>.
+
+ */

include/arl/parser/ast.h

@@ -1,74 +0,0 @@
-/* ast.h: General definition of the AST and nodes within it.
- * Created: 2026-01-22
- * Author: Aryadev Chavali
- * License: See end of file
- * Commentary:
- */
-
-#ifndef AST_H
-#define AST_H
-
-#include <arl/lib/base.h>
-#include <arl/lib/sv.h>
-#include <arl/lib/vec.h>
-
-/// Types the AST can encode
-typedef enum
-{
-  AST_NODE_TYPE_KNOWN = 0,
-  AST_NODE_TYPE_SYMBOL,
-  AST_NODE_TYPE_STRING,
-
-  NUM_AST_NODE_TYPES,
-} ast_node_type_t;
-
-/// Known symbols - may reference callables or values.
-typedef enum
-{
-  AST_KNOWN_PUTSTR,
-
-  NUM_AST_KNOWNS,
-} ast_known_t;
-
-const char *ast_known_to_cstr(ast_known_t);
-
-/// Node of the AST as a tagged union
-typedef struct
-{
-  u64 byte_location;
-  ast_node_type_t type;
-  union
-  {
-    ast_known_t as_known;
-    sv_t as_symbol;
-    sv_t as_string;
-  };
-} ast_node_t;
-
-ast_node_t ast_node_known(u64 byte, ast_known_t known);
-ast_node_t ast_node_symbol(u64 byte, sv_t symbol);
-ast_node_t ast_node_string(u64 byte, sv_t string);
-void ast_node_print(FILE *fp, ast_node_t *node);
-
-/// The AST as a flat collection of nodes
-typedef struct
-{
-  vec_t nodes;
-} ast_t;
-
-void ast_free(ast_t *ast);
-void ast_print(FILE *fp, ast_t *ast);
-
-#endif
-
-/* Copyright (C) 2026 Aryadev Chavali
-
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- * FOR A PARTICULAR PURPOSE.  See the MIT License for details.
-
- * You may distribute and modify this code under the terms of the MIT License,
- * which you should have received a copy of along with this program.  If not,
- * please go to <https://opensource.org/license/MIT>.
-
- */