X-Git-Url: https://git.m6w6.name/?p=m6w6%2Fext-psi;a=blobdiff_plain;f=src%2Fparser.re;h=fb4e35bb3933e3fa4847ad04eb0f98d89bf2e12e;hp=59c8bdaf141d12136b62afe2b794d884166fb2e9;hb=3d77b5b65efb78c1da9485dfde1fcd417066b803;hpb=ba906e039ffe9e57842ce5135aa43efa00b8a4c6

diff --git a/src/parser.re b/src/parser.re
index 59c8bda..fb4e35b 100644
--- a/src/parser.re
+++ b/src/parser.re
@@ -181,10 +181,10 @@ static void psi_parser_register_constants(struct psi_parser *P)
 }
 #endif
 
-struct psi_plist *psi_parser_preprocess(struct psi_parser *P, struct psi_plist *tokens)
+struct psi_plist *psi_parser_preprocess(struct psi_parser *P, struct psi_plist **tokens)
 {
-	if (psi_cpp_process(P->preproc, &tokens)) {
-		return tokens;
+	if (psi_cpp_process(P->preproc, tokens)) {
+		return *tokens;
 	}
 	return NULL;
 }
@@ -206,7 +206,7 @@ bool psi_parser_parse(struct psi_parser *P, struct psi_parser_input *I)
 		return false;
 	}
 
-	if (!(preproc = psi_parser_preprocess(P, scanned))) {
+	if (!(preproc = psi_parser_preprocess(P, &scanned))) {
 		psi_plist_free(scanned);
 		return false;
 	}
@@ -260,13 +260,18 @@ struct psi_plist *psi_parser_scan(struct psi_parser *P, struct psi_parser_input
 	struct psi_token *token;
 	const char *tok, *cur, *lim, *mrk, *eol, *ctxmrk;
 	unsigned parens;
+	bool escaped;
+	token_t char_width;
+
+	PSI_DEBUG_PRINT(P, "PSI: scanning %s\n", I->file);
 
 	tok = mrk = eol = cur = I->buffer;
 	lim = I->buffer + I->length;
 	I->lines = 1;
-	tokens = psi_plist_init((void (*)(void *)) psi_token_free);
+	tokens = psi_plist_init((psi_plist_dtor) psi_token_free);
 
 	start: ;
+		char_width = 1;
 		ctxmrk = NULL;
 		tok = cur;
 
@@ -282,36 +287,23 @@ struct psi_plist *psi_parser_scan(struct psi_parser *P, struct psi_parser_input
 		re2c:yyfill:parameter = 0;
 
 		W = [a-zA-Z0-9_\x80-\xff];
-		SP = [ \t];
+		SP = [ \t\f];
 		EOL = [\r\n];
-		NAME = [a-zA-Z_\x80-\xff]W*;
+		NAME = [a-zA-Z_\x80-\xff] W*;
 		NSNAME = (NAME)? ("\\" NAME)+;
 		DOLLAR_NAME = '$' W+;
-		QUOTED_STRING = "L"? "\"" ([^"])+ "\"";
-		QUOTED_CHAR = "L"? "'" ([^']+ "\\'"?)+ "'";
 		CPP_HEADER = "<" [-._/a-zA-Z0-9]+ ">";
-		CPP_ATTRIBUTE = "__attribute__" W* "((";
-		CPP_PRAGMA_ONCE = "pragma" W+ "once";
+		CPP_ATTRIBUTE = "__attribute__" SP* "((";
 
 		DEC_CONST = [1-9] [0-9]*;
 		OCT_CONST = "0" [0-7]*;
 		HEX_CONST = '0x' [0-9a-fA-F]+;
 		INT_CONST = (DEC_CONST | OCT_CONST | HEX_CONST);
-		INT_SUFFIX = 'u'('l' 'l'? )? | 'l'('l'? 'u')?;
-		INT_NUMBER = (DEC_CONST | OCT_CONST | HEX_CONST) INT_SUFFIX?;
 
-		FLT_HEX_FRAC = [0-9a-fA-F]*;
-		FLT_HEX_SIG = HEX_CONST ("." FLT_HEX_FRAC)?;
-		FLT_HEX_EXPO = 'p' [+-]? [0-9]+;
-		FLT_HEX_CONST = FLT_HEX_SIG FLT_HEX_EXPO;
+		FLT_HEX_CONST = HEX_CONST ("." [0-9a-fA-F]*)? 'p' [+-]? [0-9]+;
 		FLT_DEC_NUM = "0" | DEC_CONST;
-		FLT_DEC_FRAC = [0-9]*;
-		FLT_DEC_SIG = FLT_DEC_NUM ("." FLT_DEC_FRAC)?;
-		FLT_DEC_EXPO = 'e' [+-]? [0-9]+;
-		FLT_DEC_CONST = (FLT_DEC_SIG FLT_DEC_EXPO) | (FLT_DEC_NUM "." FLT_DEC_FRAC) | ("." [0-9]+);
+		FLT_DEC_CONST = (FLT_DEC_NUM ("." [0-9]*)? 'e' [+-]? [0-9]+) | (FLT_DEC_NUM "." [0-9]*) | ("." [0-9]+);
 		FLT_CONST = (FLT_DEC_CONST | FLT_HEX_CONST);
-		FLT_SUFFIX = 'f' | 'l' | ('d' ('f' | 'd' | 'l'));
-		FLT_NUMBER = (FLT_DEC_CONST | FLT_HEX_CONST) FLT_SUFFIX?;
 
 		[+-]? INT_CONST						{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_INT; goto start; }
 		[+-]? INT_CONST / 'u'				{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_INT | PSI_NUMBER_U; cur += 1; goto start; }
@@ -326,8 +318,17 @@ struct psi_plist *psi_parser_scan(struct psi_parser *P, struct psi_parser_input
 		[+-]? FLT_CONST	/ 'dd'			{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT | PSI_NUMBER_DD; cur += 2; goto start; }
 		[+-]? FLT_CONST	/ 'dl'			{ NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT | PSI_NUMBER_DL; cur += 2; goto start; }
 
+		"'"				{ escaped = false; tok += 1; goto character; }
+		"\""			{ escaped = false; tok += 1; goto string; }
+		"u8" / "\""		{ char_width = 1; }
+		"u" / ['"]		{ char_width = 2; }
+		"U" / ['"]		{ char_width = 4; }
+		"L" / ['"]		{ char_width = SIZEOF_WCHAR_T/8; }
+
 		"/*"			{ goto comment; }
 		"//"			{ goto comment_sl; }
+
+		"##"			{ NEWTOKEN(PSI_T_CPP_PASTE); goto start; }
 		"#"				{ NEWTOKEN(PSI_T_HASH); goto start; }
 		"("				{ NEWTOKEN(PSI_T_LPAREN); goto start; }
 		")"				{ NEWTOKEN(PSI_T_RPAREN); goto start; }
@@ -364,7 +365,34 @@ struct psi_plist *psi_parser_scan(struct psi_parser *P, struct psi_parser_input
 		"..."			{ NEWTOKEN(PSI_T_ELLIPSIS); goto start; }
 		"?"				{ NEWTOKEN(PSI_T_IIF); goto start; }
 		"pragma"		{ NEWTOKEN(PSI_T_PRAGMA); goto start; }
-		"once"			{ NEWTOKEN(PSI_T_ONCE); goto start; }
+		"pragma" W+ "once"	{ NEWTOKEN(PSI_T_PRAGMA_ONCE); goto start; }
+		"__restrict"	{ NEWTOKEN(PSI_T_CPP_RESTRICT); goto start; }
+		"__extension__"	{ NEWTOKEN(PSI_T_CPP_EXTENSION); goto start; }
+		"__asm__"		{ NEWTOKEN(PSI_T_CPP_ASM); goto start; }
+		"line"			{ NEWTOKEN(PSI_T_LINE); goto start; }
+		"typedef"		{ NEWTOKEN(PSI_T_TYPEDEF); goto start; }
+		"struct"		{ NEWTOKEN(PSI_T_STRUCT); goto start; }
+		"union"			{ NEWTOKEN(PSI_T_UNION); goto start; }
+		"enum"			{ NEWTOKEN(PSI_T_ENUM); goto start; }
+		"const"			{ NEWTOKEN(PSI_T_CONST); goto start; }
+		"void"			{ NEWTOKEN(PSI_T_VOID); goto start; }
+		"bool"			{ NEWTOKEN(PSI_T_BOOL); goto start; }
+		"char"			{ NEWTOKEN(PSI_T_CHAR); goto start; }
+		"short"			{ NEWTOKEN(PSI_T_SHORT); goto start; }
+		"int"			{ NEWTOKEN(PSI_T_INT); goto start; }
+		"long"			{ NEWTOKEN(PSI_T_LONG); goto start; }
+		"float"			{ NEWTOKEN(PSI_T_FLOAT); goto start; }
+		"double"		{ NEWTOKEN(PSI_T_DOUBLE); goto start; }
+		"int8_t"		{ NEWTOKEN(PSI_T_INT8); goto start; }
+		"uint8_t"		{ NEWTOKEN(PSI_T_UINT8); goto start; }
+		"int16_t"		{ NEWTOKEN(PSI_T_INT16); goto start; }
+		"uint16_t"		{ NEWTOKEN(PSI_T_UINT16); goto start; }
+		"int32_t"		{ NEWTOKEN(PSI_T_INT32); goto start; }
+		"uint32_t"		{ NEWTOKEN(PSI_T_UINT32); goto start; }
+		"int64_t"		{ NEWTOKEN(PSI_T_INT64); goto start; }
+		"uint64_t"		{ NEWTOKEN(PSI_T_UINT64); goto start; }
+		"unsigned"		{ NEWTOKEN(PSI_T_UNSIGNED); goto start; }
+		"signed"		{ NEWTOKEN(PSI_T_SIGNED); goto start; }
 		'IF'			{ NEWTOKEN(PSI_T_IF); goto start; }
 		'IFDEF'			{ NEWTOKEN(PSI_T_IFDEF); goto start; }
 		'IFNDEF'		{ NEWTOKEN(PSI_T_IFNDEF); goto start; }
@@ -383,35 +411,12 @@ struct psi_plist *psi_parser_scan(struct psi_parser *P, struct psi_parser_input
 		'NULL'			{ NEWTOKEN(PSI_T_NULL); goto start; }
 		'MIXED'			{ NEWTOKEN(PSI_T_MIXED); goto start; }
 		'CALLABLE'		{ NEWTOKEN(PSI_T_CALLABLE); goto start; }
-		'VOID'			{ NEWTOKEN(PSI_T_VOID); goto start; }
-		'BOOL'			{ NEWTOKEN(PSI_T_BOOL); goto start; }
-		'CHAR'			{ NEWTOKEN(PSI_T_CHAR); goto start; }
-		'SHORT'			{ NEWTOKEN(PSI_T_SHORT); goto start; }
-		'INT'			{ NEWTOKEN(PSI_T_INT); goto start; }
-		'LONG'			{ NEWTOKEN(PSI_T_LONG); goto start; }
-		'FLOAT'			{ NEWTOKEN(PSI_T_FLOAT); goto start; }
-		'DOUBLE'		{ NEWTOKEN(PSI_T_DOUBLE); goto start; }
-		'INT8_T'		{ NEWTOKEN(PSI_T_INT8); goto start; }
-		'UINT8_T'		{ NEWTOKEN(PSI_T_UINT8); goto start; }
-		'INT16_T'		{ NEWTOKEN(PSI_T_INT16); goto start; }
-		'UINT16_T'		{ NEWTOKEN(PSI_T_UINT16); goto start; }
-		'INT32_T'		{ NEWTOKEN(PSI_T_INT32); goto start; }
-		'UINT32_T'		{ NEWTOKEN(PSI_T_UINT32); goto start; }
-		'INT64_T'		{ NEWTOKEN(PSI_T_INT64); goto start; }
-		'UINT64_T'		{ NEWTOKEN(PSI_T_UINT64); goto start; }
-		'UNSIGNED'		{ NEWTOKEN(PSI_T_UNSIGNED); goto start; }
-		'SIGNED'		{ NEWTOKEN(PSI_T_SIGNED); goto start; }
 		'STRING'		{ NEWTOKEN(PSI_T_STRING); goto start; }
 		'ARRAY'			{ NEWTOKEN(PSI_T_ARRAY); goto start; }
 		'OBJECT'		{ NEWTOKEN(PSI_T_OBJECT); goto start; }
 		'CALLBACK'		{ NEWTOKEN(PSI_T_CALLBACK); goto start; }
 		'STATIC'		{ NEWTOKEN(PSI_T_STATIC); goto start; }
 		'FUNCTION'		{ NEWTOKEN(PSI_T_FUNCTION); goto start; }
-		'TYPEDEF'		{ NEWTOKEN(PSI_T_TYPEDEF); goto start; }
-		'STRUCT'		{ NEWTOKEN(PSI_T_STRUCT); goto start; }
-		'UNION'			{ NEWTOKEN(PSI_T_UNION); goto start; }
-		'ENUM'			{ NEWTOKEN(PSI_T_ENUM); goto start; }
-		'CONST'			{ NEWTOKEN(PSI_T_CONST); goto start; }
 		'LIB'			{ NEWTOKEN(PSI_T_LIB); goto start; }
 		'LET'			{ NEWTOKEN(PSI_T_LET); goto start; }
 		'SET'			{ NEWTOKEN(PSI_T_SET); goto start; }
@@ -440,9 +445,7 @@ struct psi_plist *psi_parser_scan(struct psi_parser *P, struct psi_parser_input
 		NAME			{ NEWTOKEN(PSI_T_NAME); goto start; }
 		NSNAME			{ NEWTOKEN(PSI_T_NSNAME); goto start; }
 		DOLLAR_NAME		{ NEWTOKEN(PSI_T_DOLLAR_NAME); goto start; }
-		QUOTED_STRING	{ NEWTOKEN(PSI_T_QUOTED_STRING); goto start; }
-		QUOTED_CHAR		{ NEWTOKEN(PSI_T_QUOTED_CHAR); goto start; }
-		CPP_HEADER		{ NEWTOKEN(PSI_T_CPP_HEADER); goto start; }
+		CPP_HEADER		{ tok += 1; cur -= 1; NEWTOKEN(PSI_T_CPP_HEADER); cur += 1; goto start; }
 		CPP_ATTRIBUTE	{ parens = 2; goto cpp_attribute; }
 		EOL				{ NEWTOKEN(PSI_T_EOL); NEWLINE(); goto start; }
 		SP+				{ NEWTOKEN(PSI_T_WHITESPACE); goto start; }
@@ -451,6 +454,46 @@ struct psi_plist *psi_parser_scan(struct psi_parser *P, struct psi_parser_input
 
 		*/
 
+	character: ;
+		/*!re2c
+
+		EOL		{ NEWLINE(); goto character; }
+		"\\"	{ escaped = !escaped;  goto character; }
+		"'"		{
+			if (escaped) {
+				escaped = false;
+				goto character;
+			}
+			cur -= 1;
+			NEWTOKEN(PSI_T_QUOTED_CHAR);
+			cur += 1;
+			token->flags = char_width;
+			goto start;
+		}
+		*		{ escaped = false; goto character; }
+
+		*/
+
+	string: ;
+		/*!re2c
+
+		EOL		{ NEWLINE(); goto string; }
+		"\\" 	{ escaped = !escaped; goto string; }
+		"\""	{
+			if (escaped) {
+				escaped = false;
+				goto string;
+			}
+			cur -= 1;
+			NEWTOKEN(PSI_T_QUOTED_STRING);
+			cur += 1;
+			token->flags = char_width;
+			goto start;
+		}
+		*		{ escaped = false; goto string; }
+
+		*/
+
 	comment: ;
 		/*!re2c