First commit, porting from PaCO

tokenizer.h

@@ -0,0 +1,65 @@
+#pragma once
+
+typedef unsigned char byte;
+typedef unsigned short word;
+
+enum t_tokentype {
+	TOKEN_ERROR,
+	TOKEN_NUMBER,
+	TOKEN_STRING,
+
+	TOKEN_IDENTIFIER,
+
+	TOKEN_STRUCT,
+	TOKEN_AS,
+	TOKEN_END,
+	TOKEN_FUNCTION,
+	TOKEN_VAR,
+	TOKEN_ARRAY,
+	TOKEN_OF,
+	TOKEN_CONST,
+	TOKEN_IF,
+	TOKEN_THEN,
+	TOKEN_ELSE,
+	TOKEN_WHILE,
+	TOKEN_DO,
+	TOKEN_REPEAT,
+	TOKEN_UNTIL,
+	TOKEN_FOR,
+	TOKEN_TO,
+	TOKEN_STEP,
+	TOKEN_RETURN,
+	TOKEN_REM,
+
+	TOKEN_AND,
+	TOKEN_OR,
+	TOKEN_NOT,
+	TOKEN_MOD,
+	TOKEN_PLUS,
+	TOKEN_MINUS,
+	TOKEN_ASTERISC,
+	TOKEN_SLASH,
+	TOKEN_DOT,
+	TOKEN_LPAR,
+	TOKEN_RPAR,
+	TOKEN_LBRACKET,
+	TOKEN_RBRACKET,
+	TOKEN_COMMA,
+	TOKEN_LT,
+	TOKEN_GT,
+	TOKEN_LEQ,
+	TOKEN_GEQ,
+	TOKEN_EQ,
+	TOKEN_NEQ,
+	TOKEN_ENDLINE,
+	TOKEN_ENDFILE
+};
+
+void tkn_init(const char* buffer);
+void tkn_next();
+t_tokentype tkn_get_token();
+char* tkn_get_string();
+unsigned char* tkn_get_array();
+float tkn_get_value();
+int tkn_get_line();
+int tkn_get_row();