Added string literals in tokeniser

Doesn't do much, invalid for most operations.
This commit is contained in:
2023-11-11 10:16:06 +00:00
parent bd6fb54e31
commit c9f684cc7d
3 changed files with 24 additions and 0 deletions

View File

@@ -34,6 +34,8 @@ const char *token_type_as_cstr(token_type_t type)
return "GLOBAL"; return "GLOBAL";
case TOKEN_STAR: case TOKEN_STAR:
return "STAR"; return "STAR";
case TOKEN_LITERAL_STRING:
return "LITERAL_STRING";
case TOKEN_LITERAL_NUMBER: case TOKEN_LITERAL_NUMBER:
return "LITERAL_NUMBER"; return "LITERAL_NUMBER";
case TOKEN_LITERAL_CHAR: case TOKEN_LITERAL_CHAR:
@@ -461,6 +463,24 @@ token_t tokenise_char_literal(buffer_t *buffer, size_t *column)
return token; return token;
} }
token_t tokenise_string_literal(buffer_t *buffer, size_t *column)
{
++buffer->used;
size_t string_size;
for (string_size = 0; string_size + buffer->used < buffer->available &&
buffer->data[buffer->used + string_size] != '\"';
++string_size)
continue;
token_t t = {.type = TOKEN_LITERAL_STRING,
.column = *column,
.str = malloc(string_size + 1),
.str_size = string_size};
memcpy(t.str, buffer->data + (buffer->used - string_size), string_size);
t.str[string_size] = '\0';
*column += string_size + 1;
return t;
}
lerr_t tokenise_buffer(buffer_t *buffer, token_stream_t *tokens_ptr) lerr_t tokenise_buffer(buffer_t *buffer, token_stream_t *tokens_ptr)
{ {
size_t column = 0, line = 1; size_t column = 0, line = 1;
@@ -507,6 +527,8 @@ lerr_t tokenise_buffer(buffer_t *buffer, token_stream_t *tokens_ptr)
t.str[0] = '\0'; t.str[0] = '\0';
++buffer->used; ++buffer->used;
} }
else if (c == '\"')
t = tokenise_string_literal(buffer, &column);
else if (isdigit(c) || (space_left(buffer) > 1 && c == '-' && else if (isdigit(c) || (space_left(buffer) > 1 && c == '-' &&
isdigit(buffer->data[buffer->used + 1]))) isdigit(buffer->data[buffer->used + 1])))
t = tokenise_number_literal(buffer, &column); t = tokenise_number_literal(buffer, &column);

View File

@@ -24,6 +24,7 @@ typedef enum TokenType
TOKEN_STAR, TOKEN_STAR,
TOKEN_LITERAL_NUMBER, TOKEN_LITERAL_NUMBER,
TOKEN_LITERAL_CHAR, TOKEN_LITERAL_CHAR,
TOKEN_LITERAL_STRING,
TOKEN_NOOP, TOKEN_NOOP,
TOKEN_HALT, TOKEN_HALT,
TOKEN_PUSH, TOKEN_PUSH,

View File

@@ -469,6 +469,7 @@ perr_t parse_next(token_stream_t *stream, presult_t *ret)
perr_t perr = PERR_OK; perr_t perr = PERR_OK;
switch (token.type) switch (token.type)
{ {
case TOKEN_LITERAL_STRING:
case TOKEN_PP_CONST: case TOKEN_PP_CONST:
case TOKEN_PP_REFERENCE: case TOKEN_PP_REFERENCE:
case TOKEN_PP_END: case TOKEN_PP_END: