Rename files

token.h

@@ -0,0 +1,85 @@
+#ifndef INCLUDED_PARSE_H
+#define INCLUDED_PARSE_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+typedef struct {
+    size_t line;
+    size_t column;
+    size_t line_offset; // first char of line
+    size_t offset;
+} SrcPos;
+
+typedef enum {
+    // keywords
+    TOKEN_LET,
+    TOKEN_EVAL,
+    TOKEN_CONF,
+
+    // punctuation
+    TOKEN_EQUALS,
+    TOKEN_BACKSLASH,
+    TOKEN_COLON,
+    TOKEN_OPEN_PAREN,
+    TOKEN_CLOSE_PAREN,
+    TOKEN_DEFINE, // -> in lambda definitions
+
+    // variable or macro
+    TOKEN_IDENT,
+
+    // =abc>
+    TOKEN_REDUCE,
+
+    // eof
+    TOKEN_EOF,
+    N_TOKEN_TYPES,
+} TokenType;
+
+const char *token_type_to_string(TokenType type);
+
+typedef struct {
+    SrcPos pos;
+    TokenType type;
+    size_t length;
+} Token;
+
+typedef struct {
+    SrcPos pos;
+    const char *src;
+    size_t src_len;
+} TokenStream;
+
+typedef struct {
+    bool set;
+    SrcPos pos;
+    char message[64];
+} ParseError;
+
+inline static void token_stream_init(TokenStream *stream, const char *src,
+                                     size_t src_len) {
+    stream->pos.line = 1;
+    stream->pos.column = 0;
+    stream->pos.offset = 0;
+    stream->pos.line_offset = 0;
+    stream->src = src;
+    stream->src_len = src_len;
+}
+
+inline static bool token_stream_is_eof(TokenStream *stream) {
+    return stream->pos.offset == stream->src_len;
+}
+
+// return true on success, false on error
+bool token_stream_next(TokenStream *restrict stream, Token *restrict out,
+                       ParseError *restrict error);
+static inline void parse_error_clear(ParseError *error) {
+    error->set = false;
+}
+static inline bool parse_error_is_set(const ParseError *error) {
+    return error->set;
+}
+
+#endif