From 036ac0317665fbd89af60b2a7f591d9c174409d6 Mon Sep 17 00:00:00 2001
From: Aryadev Chavali <aryadev@aryadevchavali.com>
Date: Fri, 5 Jul 2024 18:18:11 +0100
Subject: [PATCH] Lexer tokens now include source name as part of the token

---
 src/lexer.cpp | 21 +++++++++++++--------
 src/lexer.hpp | 17 +++++++++++------
 src/main.cpp  |  2 +-
 3 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/src/lexer.cpp b/src/lexer.cpp
index 6e8665d..19de4fa 100644
--- a/src/lexer.cpp
+++ b/src/lexer.cpp
@@ -482,7 +482,8 @@ namespace Lexer
     return token;
   }
 
-  Err tokenise_buffer(string_view source, std::vector<Token *> &tokens)
+  Err tokenise_buffer(string_view source_name, string_view source,
+                      std::vector<Token *> &tokens)
   {
     size_t column = 0, line = 1;
     while (source.size() > 0)
@@ -571,8 +572,9 @@ namespace Lexer
 
       if (is_token)
       {
-        t.line     = line;
-        Token *acc = new Token{t};
+        t.source_name = source_name;
+        t.line        = line;
+        Token *acc    = new Token{t};
         tokens.push_back(acc);
       }
     }
@@ -580,17 +582,20 @@ namespace Lexer
   }
 
   Token::Token()
-  {}
+  {
+  }
 
   Token::Token(Token::Type type, string_view content, size_t col, size_t line,
                OperandType optype)
-      : type{type}, column{col}, line{line}, content{content},
-        operand_type{optype}
-  {}
+      : type{type}, operand_type{optype}, column{col}, line{line},
+        content{content}
+  {
+  }
 
   Err::Err(Err::Type type, size_t col, size_t line)
       : col{col}, line{line}, type{type}
-  {}
+  {
+  }
 
   std::string to_string(const Token::Type &type)
   {
diff --git a/src/lexer.hpp b/src/lexer.hpp
index f467a78..092139e 100644
--- a/src/lexer.hpp
+++ b/src/lexer.hpp
@@ -27,15 +27,20 @@ namespace Lexer
   {
     enum class Type
     {
+      // Preprocessor and other parse time constants
       PP_CONST,     // %const(<symbol>)...
       PP_USE,       // %use <string>
       PP_END,       // %end
       PP_REFERENCE, // $<symbol>
       GLOBAL,
       STAR,
+      // Literals
       LITERAL_NUMBER,
       LITERAL_CHAR,
       LITERAL_STRING,
+      SYMBOL,
+
+      // Instruction
       NOOP,
       HALT,
       PUSH,
@@ -70,11 +75,7 @@ namespace Lexer
       CALL,
       CALL_STACK,
       RET,
-      SYMBOL,
     } type;
-    size_t column, line;
-    std::string content;
-
     enum class OperandType
     {
       NIL,
@@ -86,9 +87,12 @@ namespace Lexer
       LONG
     } operand_type;
 
+    size_t column, line;
+    std::string source_name, content;
+
     Token();
     Token(Token::Type, std::string_view content = "", size_t col = 0,
-          size_t line = 0, OperandType optype = OperandType::NIL);
+          size_t line = 0, OperandType type = OperandType::NIL);
   };
 
   struct Err
@@ -110,7 +114,8 @@ namespace Lexer
     Err(Type type = Type::OK, size_t col = 0, size_t line = 0);
   };
 
-  Err tokenise_buffer(std::string_view, std::vector<Token *> &);
+  Err tokenise_buffer(std::string_view source_name, std::string_view content,
+                      std::vector<Token *> &vec);
 
   std::string to_string(const Token::Type &);
   std::string to_string(const Token::OperandType &);
diff --git a/src/main.cpp b/src/main.cpp
index fc91a4b..f699236 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -85,7 +85,7 @@ int main(int argc, const char *argv[])
   }
   original = string_view{source_str};
   src      = string_view{source_str};
-  lerr     = tokenise_buffer(src, tokens);
+  lerr     = tokenise_buffer(source_name, src, tokens);
 
   if (lerr.type != Lexer ::Err::Type::OK)
   {