X-Git-Url: https://git.m6w6.name/?p=m6w6%2Fext-psi;a=blobdiff_plain;f=src%2Fparser.re;h=ebd1e1628de46248bf97d3cafe3589a189786d4f;hp=fb460803bf37af692f85bdbaa5180bfdfcc573be;hb=35060621f2fd5079502543d17942127c1a602f72;hpb=0eb2c3dc52fd9fde9413a544c3ea7ebfe1ec0e89

diff --git a/src/parser.re b/src/parser.re
index fb46080..ebd1e16 100644
--- a/src/parser.re
+++ b/src/parser.re
@@ -26,188 +26,220 @@
 #include "php_psi_stdinc.h"
 #include <sys/mman.h>
 #include <assert.h>
+#include <errno.h>
+#include <stdarg.h>
+
+#include <Zend/zend_smart_str.h>
 
 #include "parser.h"
 
-void *psi_parser_proc_init(void);
-void psi_parser_proc_free(void **parser_proc);
-void psi_parser_proc_parse(void *parser_proc, token_t r, struct psi_token *token, struct psi_parser *parser);
-void psi_parser_proc_trace(FILE *out, char *prefix);
+/*!max:re2c*/
+#ifndef YYMAXFILL
+# define YYMAXFILL 256
+#endif
 
 struct psi_parser *psi_parser_init(struct psi_parser *P, psi_error_cb error, unsigned flags)
 {
 	if (!P) {
-		P = malloc(sizeof(*P));
+		P = pemalloc(sizeof(*P), 1);
 	}
 	memset(P, 0, sizeof(*P));
 
 	psi_data_ctor_with_dtors(PSI_DATA(P), error, flags);
 
-	P->col = 1;
-	P->line = 1;
-	P->proc = psi_parser_proc_init();
-
-	if (flags & PSI_DEBUG) {
-		psi_parser_proc_trace(stderr, "PSI> ");
-	}
+	P->preproc = psi_cpp_init(P);
 
 	return P;
 }
 
-bool psi_parser_open_file(struct psi_parser *P, const char *filename)
+struct psi_parser_input *psi_parser_open_file(struct psi_parser *P, const char *filename, bool report_errors)
 {
-	FILE *fp = fopen(filename, "r");
-
-	if (!fp) {
-		P->error(PSI_DATA(P), NULL, PSI_WARNING,
-				"Could not open '%s' for reading: %s",
-				filename, strerror(errno));
-		return false;
+	struct stat sb;
+	FILE *fp;
+	struct psi_parser_input *fb;
+
+	if (stat(filename, &sb)) {
+		if (report_errors) {
+			P->error(PSI_DATA(P), NULL, PSI_WARNING,
+					"Could not stat '%s': %s",
+					filename, strerror(errno));
+		}
+		return NULL;
 	}
 
-	P->input.type = PSI_PARSE_FILE;
-	P->input.data.file.handle = fp;
-
-#if HAVE_MMAP
-	struct stat sb;
-	int fd = fileno(fp);
+	if (!(fb = pemalloc(sizeof(*fb) + sb.st_size + YYMAXFILL, 1))) {
+		if (report_errors) {
+			P->error(PSI_DATA(P), NULL, PSI_WARNING,
+					"Could not allocate %zu bytes for reading '%s': %s",
+					sb.st_size + YYMAXFILL, filename, strerror(errno));
+		}
+		return NULL;
+	}
 
-	if (fstat(fd, &sb)) {
-		P->error(PSI_DATA(P), NULL, PSI_WARNING,
-				"Could not stat '%s': %s",
-				filename, strerror(errno));
-		return false;
+	if (!(fp = fopen(filename, "r"))) {
+		free(fb);
+		if (report_errors) {
+			P->error(PSI_DATA(P), NULL, PSI_WARNING,
+					"Could not open '%s' for reading: %s",
+					filename, strerror(errno));
+		}
+		return NULL;
 	}
 
-	P->input.data.file.buffer = mmap(NULL, sb.st_size, PROT_READ, MAP_SHARED, fd, 0);
-	if (MAP_FAILED == P->input.data.file.buffer) {
-		P->error(PSI_DATA(P), NULL, PSI_WARNING,
-				"Could not map '%s' for reading: %s",
-				filename, strerror(errno));
-		return false;
+	if (sb.st_size != fread(fb->buffer, 1, sb.st_size, fp)) {
+		free(fb);
+		fclose(fp);
+		if (report_errors) {
+			P->error(PSI_DATA(P), NULL, PSI_WARNING,
+					"Could not read %zu bytes from '%s': %s",
+					sb.st_size + YYMAXFILL, filename, strerror(errno));
+		}
+		return NULL;
 	}
-	P->input.data.file.length = sb.st_size;
-#else
-	P->input.data.file.buffer = malloc(BSIZE);
-#endif
 
-	P->file.fn = strdup(filename);
+	fb->length = sb.st_size;
+	fb->file = zend_string_init_interned(filename, strlen(filename), 1);
 
-	return true;
+	return fb;
 }
 
-bool psi_parser_open_string(struct psi_parser *P, const char *string, size_t length)
+struct psi_parser_input *psi_parser_open_string(struct psi_parser *P, const char *string, size_t length)
 {
-	P->input.type = PSI_PARSE_STRING;
-	P->input.data.string.length = length;
-	if (!(P->input.data.string.buffer = strndup(string, length))) {
-		return false;
+	struct psi_parser_input *sb;
+
+	if (!(sb = pemalloc(sizeof(*sb) + length + YYMAXFILL, 1))) {
+		P->error(PSI_DATA(P), NULL, PSI_WARNING,
+				"Could not allocate %zu bytes: %s",
+				length + YYMAXFILL, strerror(errno));
+		return NULL;
 	}
 
-	P->file.fn = strdup("<input>");
+	memcpy(sb->buffer, string, length);
+	memset(sb->buffer + length, 0, YYMAXFILL);
 
-	return true;
+	sb->length = length;
+	sb->file = zend_string_init_interned("<stdin>", strlen("<stdin>"), 1);
+
+	return sb;
 }
 
-static ssize_t psi_parser_fill(struct psi_parser *P, size_t n)
+struct psi_plist *psi_parser_preprocess(struct psi_parser *P, struct psi_plist **tokens)
 {
-	PSI_DEBUG_PRINT(P, "PSI< Fill: n=%zu (input.type=%d)\n", n, P->input.type);
-
-	/* init if n==0 */
-	if (!n) {
-		switch (P->input.type) {
-		case PSI_PARSE_FILE:
-			P->cur = P->tok = P->mrk = P->input.data.file.buffer;
-#if HAVE_MMAP
-			P->eof = P->input.data.file.buffer + P->input.data.file.length;
-			P->lim = P->eof;
-#else
-			P->eof = NULL;
-			P->lim = P->input.data.file.buffer;
-#endif
-			break;
-
-		case PSI_PARSE_STRING:
-			P->cur = P->tok = P->mrk = P->input.data.string.buffer;
-			P->eof = P->input.data.string.buffer + P->input.data.string.length;
-			P->lim = P->eof;
-			break;
-		}
+	if (psi_cpp_process(P->preproc, tokens)) {
+		return *tokens;
+	}
+	return NULL;
+}
 
-		PSI_DEBUG_PRINT(P, "PSI< Fill: cur=%p lim=%p eof=%p\n", P->cur, P->lim, P->eof);
+bool psi_parser_process(struct psi_parser *P, struct psi_plist *tokens, size_t *processed)
+{
+	if (psi_plist_count(tokens)) {
+		return 0 == psi_parser_proc_parse(P, tokens, processed);
 	}
+	return true;
+}
 
-	switch (P->input.type) {
-	case PSI_PARSE_STRING:
-		break;
-
-	case PSI_PARSE_FILE:
-#if !HAVE_MMAP
-		if (!P->eof) {
-			size_t consumed = P->tok - P->buf;
-			size_t reserved = P->lim - P->tok;
-			size_t available = BSIZE - reserved;
-			size_t didread;
-
-			if (consumed) {
-				memmove(P->buf, P->tok, reserved);
-				P->tok -= consumed;
-				P->cur -= consumed;
-				P->lim -= consumed;
-				P->mrk -= consumed;
+void psi_parser_postprocess(struct psi_parser *P)
+{
+	unsigned flags;
+	zend_string *name;
+	struct psi_validate_scope scope = {0};
+
+	psi_validate_scope_ctor(&scope);
+	scope.defs = &P->preproc->defs;
+
+	flags = P->flags;
+	P->flags |= PSI_SILENT;
+
+	/* register const macros */
+	ZEND_HASH_FOREACH_STR_KEY_PTR(&P->preproc->defs, name, scope.macro)
+	{
+		if (scope.macro->sig) {
+		} else if (scope.macro->exp) {
+			if (psi_num_exp_validate(PSI_DATA(P), scope.macro->exp, &scope)) {
+				struct psi_impl_type *type;
+				struct psi_impl_def_val *def;
+				struct psi_const *cnst;
+				struct psi_num_exp *num;
+				smart_str ns_name = {0};
+				zend_string *name_str, *type_str;
+
+				smart_str_appendl_ex(&ns_name, ZEND_STRL("psi\\"), 1);
+				smart_str_append_ex(&ns_name, name, 1);
+				name_str = smart_str_extract(&ns_name);
+				type_str = zend_string_init_interned(ZEND_STRL("<eval number>"), 1);
+
+				num = psi_num_exp_copy(scope.macro->exp);
+				def = psi_impl_def_val_init(PSI_T_NUMBER, num);
+				type = psi_impl_type_init(PSI_T_NUMBER, type_str);
+				cnst = psi_const_init(type, name_str, def);
+				P->consts = psi_plist_add(P->consts, &cnst);
+				zend_string_release(name_str);
+				zend_string_release(type_str);
 			}
-
-			didread = fread(P->lim, 1, available, P->fp);
-			P->lim += didread;
-			if (didread < available) {
-				P->eof = P->lim;
+		} else {
+			if (psi_plist_count(scope.macro->tokens) == 1) {
+				struct psi_token *t;
+
+				if (psi_plist_get(scope.macro->tokens, 0, &t)) {
+					if (t->type == PSI_T_QUOTED_STRING) {
+						struct psi_impl_type *type;
+						struct psi_impl_def_val *def;
+						struct psi_const *cnst;
+						smart_str ns_name = {0};
+						zend_string *name_str, *type_str;
+
+						smart_str_appendl_ex(&ns_name, ZEND_STRL("psi\\"), 1);
+						smart_str_append_ex(&ns_name, name, 1);
+						name_str = smart_str_extract(&ns_name);
+						type_str = zend_string_init_interned(ZEND_STRL("string"), 1);
+
+						type = psi_impl_type_init(PSI_T_STRING, type_str);
+						def = psi_impl_def_val_init(PSI_T_QUOTED_STRING, t->text);
+						cnst = psi_const_init(type, name_str, def);
+						P->consts = psi_plist_add(P->consts, &cnst);
+						zend_string_release(name_str);
+						zend_string_release(type_str);
+					}
+				}
 			}
-			PSI_DEBUG_PRINT(P, "PSI< Fill: consumed=%zu reserved=%zu available=%zu didread=%zu\n",
-					consumed, reserved, available, didread);
 		}
-#endif
-		break;
 	}
+	ZEND_HASH_FOREACH_END();
 
-	PSI_DEBUG_PRINT(P, "PSI< Fill: avail=%td\n", P->lim - P->cur);
+	P->flags = flags;
 
-	return P->lim - P->cur;
+	psi_validate_scope_dtor(&scope);
 }
 
-void psi_parser_parse(struct psi_parser *P, struct psi_token *T)
+bool psi_parser_parse(struct psi_parser *P, struct psi_parser_input *I)
 {
-	if (T) {
-		psi_parser_proc_parse(P->proc, T->type, T, P);
-	} else {
-		psi_parser_proc_parse(P->proc, 0, NULL, P);
+	struct psi_plist *scanned, *preproc;
+	size_t processed = 0;
+
+	if (!(scanned = psi_parser_scan(P, I))) {
+		return false;
 	}
-}
 
-void psi_parser_dtor(struct psi_parser *P)
-{
-	psi_parser_proc_free(&P->proc);
-
-	switch (P->input.type) {
-	case PSI_PARSE_FILE:
-		if (P->input.data.file.buffer) {
-#if HAVE_MMAP
-			munmap(P->input.data.file.buffer, P->input.data.file.length);
-#else
-			free(P->input.data.file.buffer);
-#endif
-		}
-		if (P->input.data.file.handle) {
-			fclose(P->input.data.file.handle);
-		}
-		break;
+	if (!(preproc = psi_parser_preprocess(P, &scanned))) {
+		psi_plist_free(scanned);
+		return false;
+	}
 
-	case PSI_PARSE_STRING:
-		if (P->input.data.string.buffer) {
-			free(P->input.data.string.buffer);
-		}
-		break;
+	if (!psi_parser_process(P, preproc, &processed)) {
+		psi_plist_free(preproc);
+		return false;
 	}
 
+	psi_parser_postprocess(P);
+
+	psi_plist_free(preproc);
+	return true;
+}
+
+void psi_parser_dtor(struct psi_parser *P)
+{
+	psi_cpp_free(&P->preproc);
 	psi_data_dtor(PSI_DATA(P));
 
 	memset(P, 0, sizeof(*P));
@@ -222,162 +254,300 @@ void psi_parser_free(struct psi_parser **P)
 	}
 }
 
-/*!max:re2c*/
-#if BSIZE < YYMAXFILL
-# error BSIZE must be greater than YYMAXFILL
-#endif
+#define NEWLINE() \
+	eol = cur; \
+	++I->lines
+
+#define NEWTOKEN(t) \
+	if (t == PSI_T_COMMENT || t == PSI_T_WHITESPACE) { \
+		token = psi_token_init(t, "", 0, tok - eol + 1, I->lines, I->file); \
+	} else { \
+		token = psi_token_init(t, tok, cur - tok, tok - eol + 1, I->lines, I->file); \
+	} \
+	tokens = psi_plist_add(tokens, &token); \
+	if (P->flags & PSI_DEBUG) { \
+		fprintf(stderr, "PSI< "); \
+		psi_token_dump(2, token); \
+	}
 
-#define RETURN(t) do { \
-	P->num = t; \
-	PSI_DEBUG_PRINT(P, "PSI< TOKEN: %d %.*s (EOF=%d %s:%u:%u)\n", \
-				P->num, (int) (P->cur-P->tok), P->tok, P->num == PSI_T_EOF, \
-				P->file.fn, P->line, P->col); \
-	return t; \
-} while(1)
 
-#define ADDCOLS \
-	P->col += P->cur - P->tok
 
-#define NEWLINE(label) \
-	P->col = 1; \
-	++P->line; \
-	goto label
 
-token_t psi_parser_scan(struct psi_parser *P)
+struct psi_plist *psi_parser_scan(struct psi_parser *P, struct psi_parser_input *I)
 {
-	if (!P->cur) {
-		psi_parser_fill(P, 0);
-	}
-	for (;;) {
-		ADDCOLS;
-	nextline:
-		P->tok = P->cur;
+	struct psi_plist *tokens;
+	struct psi_token *token;
+	const char *tok, *cur, *lim, *mrk, *eol, *ctxmrk;
+	unsigned parens;
+	bool escaped;
+	token_t char_width;
+
+	PSI_DEBUG_PRINT(P, "PSI: scanning %s\n", I->file->val);
+
+	tok = mrk = eol = cur = I->buffer;
+	lim = I->buffer + I->length;
+	I->lines = 1;
+	tokens = psi_plist_init((psi_plist_dtor) psi_token_free);
+
+	start: ;
+		char_width = 1;
+		ctxmrk = NULL;
+		tok = cur;
+
+		(void) ctxmrk;
+
 		/*!re2c
+
 		re2c:indent:top = 2;
 		re2c:define:YYCTYPE = "unsigned char";
-		re2c:define:YYCURSOR = P->cur;
-		re2c:define:YYLIMIT = P->lim;
-		re2c:define:YYMARKER = P->mrk;
-		re2c:define:YYFILL = "{ if (!psi_parser_fill(P,@@)) RETURN(PSI_T_EOF); }";
+		re2c:define:YYCURSOR = cur;
+		re2c:define:YYLIMIT = lim;
+		re2c:define:YYMARKER = mrk;
+		re2c:define:YYCTXMARKER = ctxmrk;
+		re2c:define:YYFILL = "if (cur >= lim) goto done;";
 		re2c:yyfill:parameter = 0;
 
-		B = [^a-zA-Z0-9_];
-		W = [a-zA-Z0-9_];
-		NAME = [a-zA-Z_]W*;
+		W = [a-zA-Z0-9_\x80-\xff];
+		SP = [ \t\f];
+		EOL = [\r\n];
+		NAME = [a-zA-Z_\x80-\xff] W*;
 		NSNAME = (NAME)? ("\\" NAME)+;
 		DOLLAR_NAME = '$' W+;
-		QUOTED_STRING = "\"" ([^\"])+ "\"";
-		NUMBER = [+-]? [0-9]* "."? [0-9]+ ([eE] [+-]? [0-9]+)?;
-
-		"/*" { goto comment; }
-		("#"|"//") .* "\n" { NEWLINE(nextline); }
-		"(" {RETURN(PSI_T_LPAREN);}
-		")" {RETURN(PSI_T_RPAREN);}
-		";" {RETURN(PSI_T_EOS);}
-		"," {RETURN(PSI_T_COMMA);}
-		":" {RETURN(PSI_T_COLON);}
-		"{" {RETURN(PSI_T_LBRACE);}
-		"}" {RETURN(PSI_T_RBRACE);}
-		"[" {RETURN(PSI_T_LBRACKET);}
-		"]" {RETURN(PSI_T_RBRACKET);}
-		"!=" {RETURN(PSI_T_CMP_NE);}
-		"==" {RETURN(PSI_T_CMP_EQ);}
-		"&&" {RETURN(PSI_T_AND);}
-		"||" {RETURN(PSI_T_OR);}
-		"=" {RETURN(PSI_T_EQUALS);}
-		"*" {RETURN(PSI_T_ASTERISK);}
-		"~" {RETURN(PSI_T_TILDE);}
-		"!" {RETURN(PSI_T_NOT);}
-		"%" {RETURN(PSI_T_MODULO);}
-		"&" {RETURN(PSI_T_AMPERSAND);}
-		"+" {RETURN(PSI_T_PLUS);}
-		"-" {RETURN(PSI_T_MINUS);}
-		"/" {RETURN(PSI_T_SLASH);}
-		"|" {RETURN(PSI_T_PIPE);}
-		"^" {RETURN(PSI_T_CARET);}
-		"<<" {RETURN(PSI_T_LSHIFT);}
-		">>" {RETURN(PSI_T_RSHIFT);}
-		"<=" {RETURN(PSI_T_CMP_LE);}
-		">=" {RETURN(PSI_T_CMP_GE);}
-		"<" {RETURN(PSI_T_LCHEVR);}
-		">" {RETURN(PSI_T_RCHEVR);}
-		"..." {RETURN(PSI_T_ELLIPSIS);}
-		[\r\n] { NEWLINE(nextline); }
-		[\t ]+ { continue; }
-		'TRUE' {RETURN(PSI_T_TRUE);}
-		'FALSE' {RETURN(PSI_T_FALSE);}
-		'NULL' {RETURN(PSI_T_NULL);}
-		'MIXED' {RETURN(PSI_T_MIXED);}
-		'CALLABLE' {RETURN(PSI_T_CALLABLE);}
-		'VOID' {RETURN(PSI_T_VOID);}
-		'BOOL' {RETURN(PSI_T_BOOL);}
-		'CHAR' {RETURN(PSI_T_CHAR);}
-		'SHORT' {RETURN(PSI_T_SHORT);}
-		'INT' {RETURN(PSI_T_INT);}
-		'LONG' {RETURN(PSI_T_LONG);}
-		'FLOAT' {RETURN(PSI_T_FLOAT);}
-		'DOUBLE' {RETURN(PSI_T_DOUBLE);}
-		'INT8_T' {RETURN(PSI_T_INT8);}
-		'UINT8_T' {RETURN(PSI_T_UINT8);}
-		'INT16_T' {RETURN(PSI_T_INT16);}
-		'UINT16_T' {RETURN(PSI_T_UINT16);}
-		'INT32_T' {RETURN(PSI_T_INT32);}
-		'UINT32_T' {RETURN(PSI_T_UINT32);}
-		'INT64_T' {RETURN(PSI_T_INT64);}
-		'UINT64_T' {RETURN(PSI_T_UINT64);}
-		'UNSIGNED' {RETURN(PSI_T_UNSIGNED);}
-		'SIGNED' {RETURN(PSI_T_SIGNED);}
-		'STRING' {RETURN(PSI_T_STRING);}
-		'ARRAY' {RETURN(PSI_T_ARRAY);}
-		'OBJECT' {RETURN(PSI_T_OBJECT);}
-		'CALLBACK' {RETURN(PSI_T_CALLBACK);}
-		'STATIC' {RETURN(PSI_T_STATIC);}
-		'FUNCTION' {RETURN(PSI_T_FUNCTION);}
-		'TYPEDEF' {RETURN(PSI_T_TYPEDEF);}
-		'STRUCT' {RETURN(PSI_T_STRUCT);}
-		'UNION' {RETURN(PSI_T_UNION);}
-		'ENUM' {RETURN(PSI_T_ENUM);}
-		'CONST' {RETURN(PSI_T_CONST);}
-		'LIB' {RETURN(PSI_T_LIB);}
-		'LET' {RETURN(PSI_T_LET);}
-		'SET' {RETURN(PSI_T_SET);}
-		'PRE_ASSERT' {RETURN(PSI_T_PRE_ASSERT);}
-		'POST_ASSERT' {RETURN(PSI_T_POST_ASSERT);}
-		'RETURN' {RETURN(PSI_T_RETURN);}
-		'FREE' {RETURN(PSI_T_FREE);}
-		'TEMP' {RETURN(PSI_T_TEMP);}
-		'STRLEN' {RETURN(PSI_T_STRLEN);}
-		'STRVAL' {RETURN(PSI_T_STRVAL);}
-		'PATHVAL' {RETURN(PSI_T_PATHVAL);}
-		'INTVAL' {RETURN(PSI_T_INTVAL);}
-		'FLOATVAL' {RETURN(PSI_T_FLOATVAL);}
-		'BOOLVAL' {RETURN(PSI_T_BOOLVAL);}
-		'ARRVAL' {RETURN(PSI_T_ARRVAL);}
-		'OBJVAL' {RETURN(PSI_T_OBJVAL);}
-		'ZVAL' {RETURN(PSI_T_ZVAL);}
-		'COUNT' {RETURN(PSI_T_COUNT);}
-		'CALLOC' {RETURN(PSI_T_CALLOC);}
-		'TO_OBJECT' {RETURN(PSI_T_TO_OBJECT);}
-		'TO_ARRAY' {RETURN(PSI_T_TO_ARRAY);}
-		'TO_STRING' {RETURN(PSI_T_TO_STRING);}
-		'TO_INT' {RETURN(PSI_T_TO_INT);}
-		'TO_FLOAT' {RETURN(PSI_T_TO_FLOAT);}
-		'TO_BOOL' {RETURN(PSI_T_TO_BOOL);}
-		NUMBER {RETURN(PSI_T_NUMBER);}
-		NAME {RETURN(PSI_T_NAME);}
-		NSNAME {RETURN(PSI_T_NSNAME);}
-		DOLLAR_NAME {RETURN(PSI_T_DOLLAR_NAME);}
-		QUOTED_STRING {RETURN(PSI_T_QUOTED_STRING);}
-		[^] {break;}
+		CPP_HEADER = "<" [-._/a-zA-Z0-9]+ ">";
+		CPP_ATTRIBUTE = "__attribute__" SP* "((";
+
+		DEC_CONST = [1-9] [0-9]*;
+		OCT_CONST = "0" [0-7]*;
+		HEX_CONST = '0x' [0-9a-fA-F]+;
+		INT_CONST = (DEC_CONST | OCT_CONST | HEX_CONST);
+
+		FLT_HEX_CONST = HEX_CONST ("." [0-9a-fA-F]*)? 'p' [+-]? [0-9]+;
+		FLT_DEC_NUM = "0" | DEC_CONST;
+		FLT_DEC_CONST = (FLT_DEC_NUM ("." [0-9]*)? 'e' [+-]? [0-9]+) | (FLT_DEC_NUM "." [0-9]*) | ("." [0-9]+);
+		FLT_CONST = (FLT_DEC_CONST | FLT_HEX_CONST);
+
+		[+-]? INT_CONST						{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_INT; goto start; }
+		[+-]? INT_CONST / 'u'				{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_INT | PSI_NUMBER_U; cur += 1; goto start; }
+		[+-]? INT_CONST / 'l'				{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_INT | PSI_NUMBER_L; cur += 1; goto start; }
+		[+-]? INT_CONST / ('lu' | 'ul')		{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_INT | PSI_NUMBER_UL; cur += 2; goto start; }
+		[+-]? INT_CONST / ('llu' | 'ull')	{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_INT | PSI_NUMBER_ULL; cur += 3; goto start; }
+
+		[+-]? FLT_CONST					{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT; goto start; }
+		[+-]? FLT_CONST / 'f'			{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT | PSI_NUMBER_F; cur += 1; goto start; }
+		[+-]? FLT_CONST / 'l'			{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT | PSI_NUMBER_L; cur += 1; goto start; }
+		[+-]? FLT_CONST	/ 'df'			{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT | PSI_NUMBER_DF; cur += 2; goto start; }
+		[+-]? FLT_CONST	/ 'dd'			{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT | PSI_NUMBER_DD; cur += 2; goto start; }
+		[+-]? FLT_CONST	/ 'dl'			{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT | PSI_NUMBER_DL; cur += 2; goto start; }
+
+		"'"				{ escaped = false; tok += 1; goto character; }
+		"\""			{ escaped = false; tok += 1; goto string; }
+		"u8" / "\""		{ char_width = 1; }
+		"u" / ['"]		{ char_width = 2; }
+		"U" / ['"]		{ char_width = 4; }
+		"L" / ['"]		{ char_width = sizeof(wchar_t)/8; }
+
+		"/*"			{ goto comment; }
+		"//"			{ goto comment_sl; }
+
+		"##"			{ NEWTOKEN(PSI_T_CPP_PASTE); goto start; }
+		"#"				{ NEWTOKEN(PSI_T_HASH); goto start; }
+		"("				{ NEWTOKEN(PSI_T_LPAREN); goto start; }
+		")"				{ NEWTOKEN(PSI_T_RPAREN); goto start; }
+		";"				{ NEWTOKEN(PSI_T_EOS); goto start; }
+		","				{ NEWTOKEN(PSI_T_COMMA); goto start; }
+		":"				{ NEWTOKEN(PSI_T_COLON); goto start; }
+		"{"				{ NEWTOKEN(PSI_T_LBRACE); goto start; }
+		"}"				{ NEWTOKEN(PSI_T_RBRACE); goto start; }
+		"["				{ NEWTOKEN(PSI_T_LBRACKET); goto start; }
+		"]"				{ NEWTOKEN(PSI_T_RBRACKET); goto start; }
+		"!=" 			{ NEWTOKEN(PSI_T_CMP_NE); goto start; }
+		"=="			{ NEWTOKEN(PSI_T_CMP_EQ); goto start; }
+		"&&"			{ NEWTOKEN(PSI_T_AND); goto start; }
+		"||"			{ NEWTOKEN(PSI_T_OR); goto start; }
+		"="				{ NEWTOKEN(PSI_T_EQUALS); goto start; }
+		"*"				{ NEWTOKEN(PSI_T_ASTERISK); goto start; }
+		"~"				{ NEWTOKEN(PSI_T_TILDE); goto start; }
+		"!"				{ NEWTOKEN(PSI_T_NOT); goto start; }
+		"%"				{ NEWTOKEN(PSI_T_MODULO); goto start; }
+		"&"				{ NEWTOKEN(PSI_T_AMPERSAND); goto start; }
+		"+"				{ NEWTOKEN(PSI_T_PLUS); goto start; }
+		"-"				{ NEWTOKEN(PSI_T_MINUS); goto start; }
+		"/"				{ NEWTOKEN(PSI_T_SLASH); goto start; }
+		"\\"			{ NEWTOKEN(PSI_T_BSLASH); goto start; }
+		"|"				{ NEWTOKEN(PSI_T_PIPE); goto start; }
+		"^"				{ NEWTOKEN(PSI_T_CARET); goto start; }
+		"<<"			{ NEWTOKEN(PSI_T_LSHIFT); goto start; }
+		">>"			{ NEWTOKEN(PSI_T_RSHIFT); goto start; }
+		"<="			{ NEWTOKEN(PSI_T_CMP_LE); goto start; }
+		">="			{ NEWTOKEN(PSI_T_CMP_GE); goto start; }
+		"<"				{ NEWTOKEN(PSI_T_LCHEVR); goto start; }
+		">"				{ NEWTOKEN(PSI_T_RCHEVR); goto start; }
+		"."				{ NEWTOKEN(PSI_T_PERIOD); goto start; }
+		"..."			{ NEWTOKEN(PSI_T_ELLIPSIS); goto start; }
+		"?"				{ NEWTOKEN(PSI_T_IIF); goto start; }
+		"pragma"		{ NEWTOKEN(PSI_T_PRAGMA); goto start; }
+		"pragma" W+ "once"	{ NEWTOKEN(PSI_T_PRAGMA_ONCE); goto start; }
+		"__inline"		{ NEWTOKEN(PSI_T_CPP_INLINE); goto start; }
+		"__restrict"	{ NEWTOKEN(PSI_T_CPP_RESTRICT); goto start; }
+		"__extension__"	{ NEWTOKEN(PSI_T_CPP_EXTENSION); goto start; }
+		"__asm__"		{ NEWTOKEN(PSI_T_CPP_ASM); goto start; }
+		"volatile"		{ NEWTOKEN(PSI_T_VOLATILE); goto start; }
+		"sizeof"		{ NEWTOKEN(PSI_T_SIZEOF); goto start; }
+		"line"			{ NEWTOKEN(PSI_T_LINE); goto start; }
+		"typedef"		{ NEWTOKEN(PSI_T_TYPEDEF); goto start; }
+		"struct"		{ NEWTOKEN(PSI_T_STRUCT); goto start; }
+		"union"			{ NEWTOKEN(PSI_T_UNION); goto start; }
+		"enum"			{ NEWTOKEN(PSI_T_ENUM); goto start; }
+		"const"			{ NEWTOKEN(PSI_T_CONST); goto start; }
+		"void"			{ NEWTOKEN(PSI_T_VOID); goto start; }
+		"bool"			{ NEWTOKEN(PSI_T_BOOL); goto start; }
+		"char"			{ NEWTOKEN(PSI_T_CHAR); goto start; }
+		"short"			{ NEWTOKEN(PSI_T_SHORT); goto start; }
+		"int"			{ NEWTOKEN(PSI_T_INT); goto start; }
+		"long"			{ NEWTOKEN(PSI_T_LONG); goto start; }
+		"float"			{ NEWTOKEN(PSI_T_FLOAT); goto start; }
+		"double"		{ NEWTOKEN(PSI_T_DOUBLE); goto start; }
+		"unsigned"		{ NEWTOKEN(PSI_T_UNSIGNED); goto start; }
+		"signed"		{ NEWTOKEN(PSI_T_SIGNED); goto start; }
+		'IF'			{ NEWTOKEN(PSI_T_IF); goto start; }
+		'IFDEF'			{ NEWTOKEN(PSI_T_IFDEF); goto start; }
+		'IFNDEF'		{ NEWTOKEN(PSI_T_IFNDEF); goto start; }
+		'ELSE'			{ NEWTOKEN(PSI_T_ELSE); goto start; }
+		'ELIF'			{ NEWTOKEN(PSI_T_ELIF); goto start; }
+		'ENDIF'			{ NEWTOKEN(PSI_T_ENDIF); goto start; }
+		'DEFINE'		{ NEWTOKEN(PSI_T_DEFINE); goto start; }
+		'DEFINED'		{ NEWTOKEN(PSI_T_DEFINED); goto start; }
+		'UNDEF'			{ NEWTOKEN(PSI_T_UNDEF); goto start; }
+		'WARNING'		{ NEWTOKEN(PSI_T_WARNING); goto start; }
+		'ERROR'			{ NEWTOKEN(PSI_T_ERROR); goto start; }
+		'INCLUDE'		{ NEWTOKEN(PSI_T_INCLUDE); goto start; }
+		'INCLUDE_NEXT'	{ NEWTOKEN(PSI_T_INCLUDE_NEXT); goto start; }
+		'TRUE'			{ NEWTOKEN(PSI_T_TRUE); goto start; }
+		'FALSE'			{ NEWTOKEN(PSI_T_FALSE); goto start; }
+		'NULL'			{ NEWTOKEN(PSI_T_NULL); goto start; }
+		'MIXED'			{ NEWTOKEN(PSI_T_MIXED); goto start; }
+		'CALLABLE'		{ NEWTOKEN(PSI_T_CALLABLE); goto start; }
+		'STRING'		{ NEWTOKEN(PSI_T_STRING); goto start; }
+		'ARRAY'			{ NEWTOKEN(PSI_T_ARRAY); goto start; }
+		'OBJECT'		{ NEWTOKEN(PSI_T_OBJECT); goto start; }
+		'CALLBACK'		{ NEWTOKEN(PSI_T_CALLBACK); goto start; }
+		'STATIC'		{ NEWTOKEN(PSI_T_STATIC); goto start; }
+		'FUNCTION'		{ NEWTOKEN(PSI_T_FUNCTION); goto start; }
+		'LIB'			{ NEWTOKEN(PSI_T_LIB); goto start; }
+		'LET'			{ NEWTOKEN(PSI_T_LET); goto start; }
+		'SET'			{ NEWTOKEN(PSI_T_SET); goto start; }
+		'PRE_ASSERT'	{ NEWTOKEN(PSI_T_PRE_ASSERT); goto start; }
+		'POST_ASSERT'	{ NEWTOKEN(PSI_T_POST_ASSERT); goto start; }
+		'RETURN'		{ NEWTOKEN(PSI_T_RETURN); goto start; }
+		'AS'			{ NEWTOKEN(PSI_T_AS); goto start; }
+		'FREE'			{ NEWTOKEN(PSI_T_FREE); goto start; }
+		'TEMP'			{ NEWTOKEN(PSI_T_TEMP); goto start; }
+		'STRLEN'		{ NEWTOKEN(PSI_T_STRLEN); goto start; }
+		'STRVAL'		{ NEWTOKEN(PSI_T_STRVAL); goto start; }
+		'PATHVAL'		{ NEWTOKEN(PSI_T_PATHVAL); goto start; }
+		'INTVAL'		{ NEWTOKEN(PSI_T_INTVAL); goto start; }
+		'FLOATVAL'		{ NEWTOKEN(PSI_T_FLOATVAL); goto start; }
+		'BOOLVAL'		{ NEWTOKEN(PSI_T_BOOLVAL); goto start; }
+		'ARRVAL'		{ NEWTOKEN(PSI_T_ARRVAL); goto start; }
+		'OBJVAL'		{ NEWTOKEN(PSI_T_OBJVAL); goto start; }
+		'ZVAL'			{ NEWTOKEN(PSI_T_ZVAL); goto start; }
+		'COUNT'			{ NEWTOKEN(PSI_T_COUNT); goto start; }
+		'CALLOC'		{ NEWTOKEN(PSI_T_CALLOC); goto start; }
+		'TO_OBJECT'		{ NEWTOKEN(PSI_T_TO_OBJECT); goto start; }
+		'TO_ARRAY'		{ NEWTOKEN(PSI_T_TO_ARRAY); goto start; }
+		'TO_STRING'		{ NEWTOKEN(PSI_T_TO_STRING); goto start; }
+		'TO_INT'		{ NEWTOKEN(PSI_T_TO_INT); goto start; }
+		'TO_FLOAT'		{ NEWTOKEN(PSI_T_TO_FLOAT); goto start; }
+		'TO_BOOL'		{ NEWTOKEN(PSI_T_TO_BOOL); goto start; }
+		NAME			{ NEWTOKEN(PSI_T_NAME); goto start; }
+		NSNAME			{ NEWTOKEN(PSI_T_NSNAME); goto start; }
+		DOLLAR_NAME		{ NEWTOKEN(PSI_T_DOLLAR_NAME); goto start; }
+		CPP_HEADER		{ tok += 1; cur -= 1; NEWTOKEN(PSI_T_CPP_HEADER); cur += 1; goto start; }
+		CPP_ATTRIBUTE	{ parens = 2; goto cpp_attribute; }
+		EOL				{ NEWTOKEN(PSI_T_EOL); NEWLINE(); goto start; }
+		SP+				{ NEWTOKEN(PSI_T_WHITESPACE); goto start; }
+		[^]				{ NEWTOKEN(-2); goto error; }
+		*				{ NEWTOKEN(-1); goto error; }
+
 		*/
 
-	comment:
-		P->tok = P->cur;
+	character: ;
 		/*!re2c
-		"\n" { NEWLINE(comment); }
-		"*" "/" { continue; }
-		[^] { goto comment; }
+
+		EOL		{ NEWLINE(); goto character; }
+		"\\"	{ escaped = !escaped;  goto character; }
+		"'"		{
+			if (escaped) {
+				escaped = false;
+				goto character;
+			}
+			cur -= 1;
+			NEWTOKEN(PSI_T_QUOTED_CHAR);
+			cur += 1;
+			token->flags = char_width;
+			goto start;
+		}
+		*		{ escaped = false; goto character; }
+
 		*/
-	}
-	return -1;
+
+	string: ;
+		/*!re2c
+
+		EOL		{ NEWLINE(); goto string; }
+		"\\" 	{ escaped = !escaped; goto string; }
+		"\""	{
+			if (escaped) {
+				escaped = false;
+				goto string;
+			}
+			cur -= 1;
+			NEWTOKEN(PSI_T_QUOTED_STRING);
+			cur += 1;
+			token->flags = char_width;
+			goto start;
+		}
+		*		{ escaped = false; goto string; }
+
+		*/
+
+	comment: ;
+		/*!re2c
+
+		EOL		{ NEWLINE(); goto comment; }
+		"*" "/"	{ NEWTOKEN(PSI_T_COMMENT); goto start; }
+		 *		{ goto comment; }
+
+		*/
+
+	comment_sl: ;
+		/*!re2c
+
+		EOL	{ NEWTOKEN(PSI_T_COMMENT); NEWLINE(); goto start; }
+		*	{ goto comment_sl; }
+
+		*/
+
+	cpp_attribute: ;
+
+		/*!re2c
+
+		"("	{ ++parens; goto cpp_attribute; }
+		")" { if (parens == 1) { NEWTOKEN(PSI_T_CPP_ATTRIBUTE); goto start; } else { --parens; goto cpp_attribute; } }
+		EOL	{ NEWLINE(); goto cpp_attribute; }
+		 *	{ goto cpp_attribute; }
+
+		*/
+error: ;
+
+	P->error(PSI_DATA(P), token, PSI_WARNING, "PSI syntax error: unexpected input (%d) '%.*s' at col %tu",
+			token->type, token->text->len, token->text->val, tok - eol + 1);
+	psi_plist_free(tokens);
+	return NULL;
+
+done:
+
+	PSI_DEBUG_PRINT(P, "PSI: EOF cur=%p lim=%p\n", cur, lim);
+
+	return tokens;
 }