git.m6w6.name Git - m6w6/ext-psi/blob - src/parser.re

   1 /*******************************************************************************
   2  Copyright (c) 2016, Michael Wallner <mike@php.net>.
   3  All rights reserved.
   4
   5  Redistribution and use in source and binary forms, with or without
   6  modification, are permitted provided that the following conditions are met:
   7
   8      * Redistributions of source code must retain the above copyright notice,
   9        this list of conditions and the following disclaimer.
  10      * Redistributions in binary form must reproduce the above copyright
  11        notice, this list of conditions and the following disclaimer in the
  12        documentation and/or other materials provided with the distribution.
  13
  14  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  15  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  17  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  18  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  20  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  21  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  22  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24 *******************************************************************************/
  25
  26 #include "php_psi_stdinc.h"
  27 #include <sys/mman.h>
  28 #include <assert.h>
  29 #include <stdarg.h>
  30
  31 #include "parser.h"
  32
  33 /*!max:re2c*/
  34 #ifndef YYMAXFILL
  35 # define YYMAXFILL 256
  36 #endif
  37
  38 struct psi_parser *psi_parser_init(struct psi_parser *P, psi_error_cb error, unsigned flags)
  39 {
  40         if (!P) {
  41                 P = malloc(sizeof(*P));
  42         }
  43         memset(P, 0, sizeof(*P));
  44
  45         psi_data_ctor_with_dtors(PSI_DATA(P), error, flags);
  46
  47         P->preproc = psi_cpp_init(P);
  48
  49         psi_cpp_load_defaults(P->preproc);
  50
  51         return P;
  52 }
  53
  54 struct psi_parser_input *psi_parser_open_file(struct psi_parser *P, const char *filename, bool report_errors)
  55 {
  56         struct stat sb;
  57         FILE *fp;
  58         struct psi_parser_input *fb;
  59
  60         if (stat(filename, &sb)) {
  61                 if (report_errors) {
  62                         P->error(PSI_DATA(P), NULL, PSI_WARNING,
  63                                         "Could not stat '%s': %s",
  64                                         filename, strerror(errno));
  65                 }
  66                 return NULL;
  67         }
  68
  69         if (!(fb = malloc(sizeof(*fb) + strlen(filename) + 1 + sb.st_size + YYMAXFILL))) {
  70                 if (report_errors) {
  71                         P->error(PSI_DATA(P), NULL, PSI_WARNING,
  72                                         "Could not allocate %zu bytes for reading '%s': %s",
  73                                         sb.st_size + YYMAXFILL, filename, strerror(errno));
  74                 }
  75                 return NULL;
  76         }
  77
  78         if (!(fp = fopen(filename, "r"))) {
  79                 free(fb);
  80                 if (report_errors) {
  81                         P->error(PSI_DATA(P), NULL, PSI_WARNING,
  82                                         "Could not open '%s' for reading: %s",
  83                                         filename, strerror(errno));
  84                 }
  85                 return NULL;
  86         }
  87
  88         if (sb.st_size != fread(fb->buffer, 1, sb.st_size, fp)) {
  89                 free(fb);
  90                 fclose(fp);
  91                 if (report_errors) {
  92                         P->error(PSI_DATA(P), NULL, PSI_WARNING,
  93                                         "Could not read %zu bytes from '%s': %s",
  94                                         sb.st_size + YYMAXFILL, filename, strerror(errno));
  95                 }
  96                 return NULL;
  97         }
  98
  99         memset(fb->buffer + sb.st_size, 0, YYMAXFILL);
 100         fb->length = sb.st_size;
 101         fb->file = &fb->buffer[sb.st_size + YYMAXFILL];
 102         memcpy(fb->file, filename, strlen(filename) + 1);
 103
 104         return fb;
 105 }
 106
 107 struct psi_parser_input *psi_parser_open_string(struct psi_parser *P, const char *string, size_t length)
 108 {
 109         struct psi_parser_input *sb;
 110
 111         if (!(sb = malloc(sizeof(*sb) + sizeof("<stdin>") + length + YYMAXFILL))) {
 112                 P->error(PSI_DATA(P), NULL, PSI_WARNING,
 113                                 "Could not allocate %zu bytes: %s",
 114                                 length + YYMAXFILL, strerror(errno));
 115                 return NULL;
 116         }
 117
 118         memcpy(sb->buffer, string, length);
 119         memset(sb->buffer + length, 0, YYMAXFILL);
 120
 121         sb->length = length;
 122         sb->file = &sb->buffer[length + YYMAXFILL];
 123         memcpy(sb->file, "<stdin>", sizeof("<stdin>"));
 124
 125         return sb;
 126 }
 127
 128 #if 0
 129 static void psi_parser_register_constants(struct psi_parser *P)
 130 {
 131         zend_string *key;
 132         zval *val;
 133
 134         ZEND_HASH_FOREACH_STR_KEY_VAL(&P->cpp.defs, key, val)
 135         {
 136                 struct psi_impl_def_val *iv;
 137                 struct psi_const_type *ct;
 138                 struct psi_const *c;
 139                 const char *ctn;
 140                 token_t ctt;
 141                 impl_val tmp;
 142                 zend_string *str;
 143
 144                 ZVAL_DEREF(val);
 145                 switch (Z_TYPE_P(val)) {
 146                 case IS_TRUE:
 147                 case IS_FALSE:
 148                         ctt = PSI_T_BOOL;
 149                         ctn = "bool";
 150                         tmp.zend.bval = Z_TYPE_P(val) == IS_TRUE;
 151                         break;
 152                 case IS_LONG:
 153                         ctt = PSI_T_INT;
 154                         ctn = "int";
 155                         tmp.zend.lval = Z_LVAL_P(val);
 156                         break;
 157                 case IS_DOUBLE:
 158                         ctt = PSI_T_FLOAT;
 159                         ctn = "float";
 160                         tmp.dval = Z_DVAL_P(val);
 161                         break;
 162                 default:
 163                         ctt = PSI_T_STRING;
 164                         ctn = "string";
 165                         str = zval_get_string(val);
 166                         tmp.zend.str = zend_string_dup(str, 1);
 167                         zend_string_release(str);
 168                         break;
 169                 }
 170
 171                 iv = psi_impl_def_val_init(ctt, NULL);
 172                 iv->ival = tmp;
 173                 ct = psi_const_type_init(ctt, ctn);
 174                 c = psi_const_init(ct, key->val, iv);
 175                 if (!P->consts) {
 176                         P->consts = psi_plist_init((psi_plist_dtor) psi_const_free);
 177                 }
 178                 P->consts = psi_plist_add(P->consts, &c);
 179         }
 180         ZEND_HASH_FOREACH_END();
 181 }
 182 #endif
 183
 184 struct psi_plist *psi_parser_preprocess(struct psi_parser *P, struct psi_plist **tokens)
 185 {
 186         if (psi_cpp_process(P->preproc, tokens)) {
 187                 return *tokens;
 188         }
 189         return NULL;
 190 }
 191
 192 bool psi_parser_process(struct psi_parser *P, struct psi_plist *tokens, size_t *processed)
 193 {
 194         if (psi_plist_count(tokens)) {
 195                 return 0 == psi_parser_proc_parse(P, tokens, processed);
 196         }
 197         return true;
 198 }
 199
 200 void psi_parser_postprocess(struct psi_parser *P)
 201 {
 202         unsigned flags;
 203         zend_string *name;
 204         struct psi_validate_scope scope = {0};
 205
 206         psi_validate_scope_ctor(&scope);
 207         scope.defs = &P->preproc->defs;
 208
 209         flags = P->flags;
 210         P->flags |= PSI_SILENT;
 211
 212         /* register const macros */
 213         ZEND_HASH_FOREACH_STR_KEY_PTR(&P->preproc->defs, name, scope.macro)
 214         {
 215                 if (scope.macro->sig) {
 216                 } else if (scope.macro->exp) {
 217                         if (psi_num_exp_validate(PSI_DATA(P), scope.macro->exp, &scope)) {
 218                                 struct psi_impl_type *type;
 219                                 struct psi_impl_def_val *def;
 220                                 struct psi_const *cnst;
 221                                 struct psi_num_exp *num;
 222                                 char *name_str = malloc(name->len + sizeof("psi\\"));
 223
 224                                 strcat(strcpy(name_str, "psi\\"), name->val);
 225                                 num = psi_num_exp_copy(scope.macro->exp);
 226                                 def = psi_impl_def_val_init(PSI_T_NUMBER, num);
 227                                 type = psi_impl_type_init(PSI_T_NUMBER, "<eval number>");
 228                                 cnst = psi_const_init(type, name_str, def);
 229                                 P->consts = psi_plist_add(P->consts, &cnst);
 230                                 free(name_str);
 231                         }
 232                 } else {
 233                         if (psi_plist_count(scope.macro->tokens) == 1) {
 234                                 struct psi_token *t;
 235
 236                                 if (psi_plist_get(scope.macro->tokens, 0, &t)) {
 237                                         if (t->type == PSI_T_QUOTED_STRING) {
 238                                                 struct psi_impl_type *type;
 239                                                 struct psi_impl_def_val *def;
 240                                                 struct psi_const *cnst;
 241                                                 char *name_str = malloc(name->len + sizeof("psi\\"));
 242
 243                                                 strcat(strcpy(name_str, "psi\\"), name->val);
 244                                                 type = psi_impl_type_init(PSI_T_STRING, "string");
 245                                                 def = psi_impl_def_val_init(PSI_T_QUOTED_STRING, t->text);
 246                                                 cnst = psi_const_init(type, name_str, def);
 247                                                 P->consts = psi_plist_add(P->consts, &cnst);
 248                                                 free(name_str);
 249                                         }
 250                                 }
 251                         }
 252                 }
 253         }
 254         ZEND_HASH_FOREACH_END();
 255
 256         P->flags = flags;
 257
 258         psi_validate_scope_dtor(&scope);
 259 }
 260
 261 bool psi_parser_parse(struct psi_parser *P, struct psi_parser_input *I)
 262 {
 263         struct psi_plist *scanned, *preproc;
 264         size_t processed = 0;
 265
 266         if (!(scanned = psi_parser_scan(P, I))) {
 267                 return false;
 268         }
 269
 270         if (!(preproc = psi_parser_preprocess(P, &scanned))) {
 271                 psi_plist_free(scanned);
 272                 return false;
 273         }
 274
 275         if (!psi_parser_process(P, preproc, &processed)) {
 276                 psi_plist_free(preproc);
 277                 return false;
 278         }
 279
 280         psi_parser_postprocess(P);
 281
 282         psi_plist_free(preproc);
 283         return true;
 284 }
 285
 286 void psi_parser_dtor(struct psi_parser *P)
 287 {
 288         psi_cpp_free(&P->preproc);
 289         psi_data_dtor(PSI_DATA(P));
 290
 291         memset(P, 0, sizeof(*P));
 292 }
 293
 294 void psi_parser_free(struct psi_parser **P)
 295 {
 296         if (*P) {
 297                 psi_parser_dtor(*P);
 298                 free(*P);
 299                 *P = NULL;
 300         }
 301 }
 302
 303 #define NEWLINE() \
 304         eol = cur; \
 305         ++I->lines
 306
 307 #define NEWTOKEN(t) \
 308         token = psi_token_init(t, tok, cur - tok, tok - eol + 1, I->lines, I->file); \
 309         tokens = psi_plist_add(tokens, &token); \
 310         if (P->flags & PSI_DEBUG) { \
 311                 fprintf(stderr, "PSI< "); \
 312                 psi_token_dump(2, token); \
 313         }
 314
 315 union int_suffix {
 316         char s[4];
 317         uint32_t i;
 318 };
 319
 320 struct psi_plist *psi_parser_scan(struct psi_parser *P, struct psi_parser_input *I)
 321 {
 322         struct psi_plist *tokens;
 323         struct psi_token *token;
 324         const char *tok, *cur, *lim, *mrk, *eol, *ctxmrk;
 325         unsigned parens;
 326         bool escaped;
 327         token_t char_width;
 328
 329         PSI_DEBUG_PRINT(P, "PSI: scanning %s\n", I->file);
 330
 331         tok = mrk = eol = cur = I->buffer;
 332         lim = I->buffer + I->length;
 333         I->lines = 1;
 334         tokens = psi_plist_init((psi_plist_dtor) psi_token_free);
 335
 336         start: ;
 337                 char_width = 1;
 338                 ctxmrk = NULL;
 339                 tok = cur;
 340
 341                 (void) ctxmrk;
 342
 343                 /*!re2c
 344
 345                 re2c:indent:top = 2;
 346                 re2c:define:YYCTYPE = "unsigned char";
 347                 re2c:define:YYCURSOR = cur;
 348                 re2c:define:YYLIMIT = lim;
 349                 re2c:define:YYMARKER = mrk;
 350                 re2c:define:YYCTXMARKER = ctxmrk;
 351                 re2c:define:YYFILL = "if (cur >= lim) goto done;";
 352                 re2c:yyfill:parameter = 0;
 353
 354                 W = [a-zA-Z0-9_\x80-\xff];
 355                 SP = [ \t\f];
 356                 EOL = [\r\n];
 357                 NAME = [a-zA-Z_\x80-\xff] W*;
 358                 NSNAME = (NAME)? ("\\" NAME)+;
 359                 DOLLAR_NAME = '$' W+;
 360                 CPP_HEADER = "<" [-._/a-zA-Z0-9]+ ">";
 361                 CPP_ATTRIBUTE = "__attribute__" SP* "((";
 362
 363                 DEC_CONST = [1-9] [0-9]*;
 364                 OCT_CONST = "0" [0-7]*;
 365                 HEX_CONST = '0x' [0-9a-fA-F]+;
 366                 INT_CONST = (DEC_CONST | OCT_CONST | HEX_CONST);
 367
 368                 FLT_HEX_CONST = HEX_CONST ("." [0-9a-fA-F]*)? 'p' [+-]? [0-9]+;
 369                 FLT_DEC_NUM = "0" | DEC_CONST;
 370                 FLT_DEC_CONST = (FLT_DEC_NUM ("." [0-9]*)? 'e' [+-]? [0-9]+) | (FLT_DEC_NUM "." [0-9]*) | ("." [0-9]+);
 371                 FLT_CONST = (FLT_DEC_CONST | FLT_HEX_CONST);
 372
 373                 [+-]? INT_CONST                                         { NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_INT; goto start; }
 374                 [+-]? INT_CONST / 'u'                           { NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_INT | PSI_NUMBER_U; cur += 1; goto start; }
 375                 [+-]? INT_CONST / 'l'                           { NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_INT | PSI_NUMBER_L; cur += 1; goto start; }
 376                 [+-]? INT_CONST / ('lu' | 'ul')         { NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_INT | PSI_NUMBER_UL; cur += 2; goto start; }
 377                 [+-]? INT_CONST / ('llu' | 'ull')       { NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_INT | PSI_NUMBER_ULL; cur += 3; goto start; }
 378
 379                 [+-]? FLT_CONST                                 { NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT; goto start; }
 380                 [+-]? FLT_CONST / 'f'                   { NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT | PSI_NUMBER_F; cur += 1; goto start; }
 381                 [+-]? FLT_CONST / 'l'                   { NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT | PSI_NUMBER_L; cur += 1; goto start; }
 382                 [+-]? FLT_CONST / 'df'                  { NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT | PSI_NUMBER_DF; cur += 2; goto start; }
 383                 [+-]? FLT_CONST / 'dd'                  { NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT | PSI_NUMBER_DD; cur += 2; goto start; }
 384                 [+-]? FLT_CONST / 'dl'                  { NEWTOKEN(PSI_T_NUMBER); token->flags = PSI_NUMBER_FLT | PSI_NUMBER_DL; cur += 2; goto start; }
 385
 386                 "'"                             { escaped = false; tok += 1; goto character; }
 387                 "\""                    { escaped = false; tok += 1; goto string; }
 388                 "u8" / "\""             { char_width = 1; }
 389                 "u" / ['"]              { char_width = 2; }
 390                 "U" / ['"]              { char_width = 4; }
 391                 "L" / ['"]              { char_width = sizeof(wchar_t)/8; }
 392
 393                 "/*"                    { goto comment; }
 394                 "//"                    { goto comment_sl; }
 395
 396                 "##"                    { NEWTOKEN(PSI_T_CPP_PASTE); goto start; }
 397                 "#"                             { NEWTOKEN(PSI_T_HASH); goto start; }
 398                 "("                             { NEWTOKEN(PSI_T_LPAREN); goto start; }
 399                 ")"                             { NEWTOKEN(PSI_T_RPAREN); goto start; }
 400                 ";"                             { NEWTOKEN(PSI_T_EOS); goto start; }
 401                 ","                             { NEWTOKEN(PSI_T_COMMA); goto start; }
 402                 ":"                             { NEWTOKEN(PSI_T_COLON); goto start; }
 403                 "{"                             { NEWTOKEN(PSI_T_LBRACE); goto start; }
 404                 "}"                             { NEWTOKEN(PSI_T_RBRACE); goto start; }
 405                 "["                             { NEWTOKEN(PSI_T_LBRACKET); goto start; }
 406                 "]"                             { NEWTOKEN(PSI_T_RBRACKET); goto start; }
 407                 "!="                    { NEWTOKEN(PSI_T_CMP_NE); goto start; }
 408                 "=="                    { NEWTOKEN(PSI_T_CMP_EQ); goto start; }
 409                 "&&"                    { NEWTOKEN(PSI_T_AND); goto start; }
 410                 "||"                    { NEWTOKEN(PSI_T_OR); goto start; }
 411                 "="                             { NEWTOKEN(PSI_T_EQUALS); goto start; }
 412                 "*"                             { NEWTOKEN(PSI_T_ASTERISK); goto start; }
 413                 "~"                             { NEWTOKEN(PSI_T_TILDE); goto start; }
 414                 "!"                             { NEWTOKEN(PSI_T_NOT); goto start; }
 415                 "%"                             { NEWTOKEN(PSI_T_MODULO); goto start; }
 416                 "&"                             { NEWTOKEN(PSI_T_AMPERSAND); goto start; }
 417                 "+"                             { NEWTOKEN(PSI_T_PLUS); goto start; }
 418                 "-"                             { NEWTOKEN(PSI_T_MINUS); goto start; }
 419                 "/"                             { NEWTOKEN(PSI_T_SLASH); goto start; }
 420                 "\\"                    { NEWTOKEN(PSI_T_BSLASH); goto start; }
 421                 "|"                             { NEWTOKEN(PSI_T_PIPE); goto start; }
 422                 "^"                             { NEWTOKEN(PSI_T_CARET); goto start; }
 423                 "<<"                    { NEWTOKEN(PSI_T_LSHIFT); goto start; }
 424                 ">>"                    { NEWTOKEN(PSI_T_RSHIFT); goto start; }
 425                 "<="                    { NEWTOKEN(PSI_T_CMP_LE); goto start; }
 426                 ">="                    { NEWTOKEN(PSI_T_CMP_GE); goto start; }
 427                 "<"                             { NEWTOKEN(PSI_T_LCHEVR); goto start; }
 428                 ">"                             { NEWTOKEN(PSI_T_RCHEVR); goto start; }
 429                 "."                             { NEWTOKEN(PSI_T_PERIOD); goto start; }
 430                 "..."                   { NEWTOKEN(PSI_T_ELLIPSIS); goto start; }
 431                 "?"                             { NEWTOKEN(PSI_T_IIF); goto start; }
 432                 "pragma"                { NEWTOKEN(PSI_T_PRAGMA); goto start; }
 433                 "pragma" W+ "once"      { NEWTOKEN(PSI_T_PRAGMA_ONCE); goto start; }
 434                 "__inline"              { NEWTOKEN(PSI_T_CPP_INLINE); goto start; }
 435                 "__restrict"    { NEWTOKEN(PSI_T_CPP_RESTRICT); goto start; }
 436                 "__extension__" { NEWTOKEN(PSI_T_CPP_EXTENSION); goto start; }
 437                 "__asm__"               { NEWTOKEN(PSI_T_CPP_ASM); goto start; }
 438                 "volatile"              { NEWTOKEN(PSI_T_VOLATILE); goto start; }
 439                 "sizeof"                { NEWTOKEN(PSI_T_SIZEOF); goto start; }
 440                 "line"                  { NEWTOKEN(PSI_T_LINE); goto start; }
 441                 "typedef"               { NEWTOKEN(PSI_T_TYPEDEF); goto start; }
 442                 "struct"                { NEWTOKEN(PSI_T_STRUCT); goto start; }
 443                 "union"                 { NEWTOKEN(PSI_T_UNION); goto start; }
 444                 "enum"                  { NEWTOKEN(PSI_T_ENUM); goto start; }
 445                 "const"                 { NEWTOKEN(PSI_T_CONST); goto start; }
 446                 "void"                  { NEWTOKEN(PSI_T_VOID); goto start; }
 447                 "bool"                  { NEWTOKEN(PSI_T_BOOL); goto start; }
 448                 "char"                  { NEWTOKEN(PSI_T_CHAR); goto start; }
 449                 "short"                 { NEWTOKEN(PSI_T_SHORT); goto start; }
 450                 "int"                   { NEWTOKEN(PSI_T_INT); goto start; }
 451                 "long"                  { NEWTOKEN(PSI_T_LONG); goto start; }
 452                 "float"                 { NEWTOKEN(PSI_T_FLOAT); goto start; }
 453                 "double"                { NEWTOKEN(PSI_T_DOUBLE); goto start; }
 454                 "unsigned"              { NEWTOKEN(PSI_T_UNSIGNED); goto start; }
 455                 "signed"                { NEWTOKEN(PSI_T_SIGNED); goto start; }
 456                 'IF'                    { NEWTOKEN(PSI_T_IF); goto start; }
 457                 'IFDEF'                 { NEWTOKEN(PSI_T_IFDEF); goto start; }
 458                 'IFNDEF'                { NEWTOKEN(PSI_T_IFNDEF); goto start; }
 459                 'ELSE'                  { NEWTOKEN(PSI_T_ELSE); goto start; }
 460                 'ELIF'                  { NEWTOKEN(PSI_T_ELIF); goto start; }
 461                 'ENDIF'                 { NEWTOKEN(PSI_T_ENDIF); goto start; }
 462                 'DEFINE'                { NEWTOKEN(PSI_T_DEFINE); goto start; }
 463                 'DEFINED'               { NEWTOKEN(PSI_T_DEFINED); goto start; }
 464                 'UNDEF'                 { NEWTOKEN(PSI_T_UNDEF); goto start; }
 465                 'WARNING'               { NEWTOKEN(PSI_T_WARNING); goto start; }
 466                 'ERROR'                 { NEWTOKEN(PSI_T_ERROR); goto start; }
 467                 'INCLUDE'               { NEWTOKEN(PSI_T_INCLUDE); goto start; }
 468                 'INCLUDE_NEXT'  { NEWTOKEN(PSI_T_INCLUDE_NEXT); goto start; }
 469                 'TRUE'                  { NEWTOKEN(PSI_T_TRUE); goto start; }
 470                 'FALSE'                 { NEWTOKEN(PSI_T_FALSE); goto start; }
 471                 'NULL'                  { NEWTOKEN(PSI_T_NULL); goto start; }
 472                 'MIXED'                 { NEWTOKEN(PSI_T_MIXED); goto start; }
 473                 'CALLABLE'              { NEWTOKEN(PSI_T_CALLABLE); goto start; }
 474                 'STRING'                { NEWTOKEN(PSI_T_STRING); goto start; }
 475                 'ARRAY'                 { NEWTOKEN(PSI_T_ARRAY); goto start; }
 476                 'OBJECT'                { NEWTOKEN(PSI_T_OBJECT); goto start; }
 477                 'CALLBACK'              { NEWTOKEN(PSI_T_CALLBACK); goto start; }
 478                 'STATIC'                { NEWTOKEN(PSI_T_STATIC); goto start; }
 479                 'FUNCTION'              { NEWTOKEN(PSI_T_FUNCTION); goto start; }
 480                 'LIB'                   { NEWTOKEN(PSI_T_LIB); goto start; }
 481                 'LET'                   { NEWTOKEN(PSI_T_LET); goto start; }
 482                 'SET'                   { NEWTOKEN(PSI_T_SET); goto start; }
 483                 'PRE_ASSERT'    { NEWTOKEN(PSI_T_PRE_ASSERT); goto start; }
 484                 'POST_ASSERT'   { NEWTOKEN(PSI_T_POST_ASSERT); goto start; }
 485                 'RETURN'                { NEWTOKEN(PSI_T_RETURN); goto start; }
 486                 'AS'                    { NEWTOKEN(PSI_T_AS); goto start; }
 487                 'FREE'                  { NEWTOKEN(PSI_T_FREE); goto start; }
 488                 'TEMP'                  { NEWTOKEN(PSI_T_TEMP); goto start; }
 489                 'STRLEN'                { NEWTOKEN(PSI_T_STRLEN); goto start; }
 490                 'STRVAL'                { NEWTOKEN(PSI_T_STRVAL); goto start; }
 491                 'PATHVAL'               { NEWTOKEN(PSI_T_PATHVAL); goto start; }
 492                 'INTVAL'                { NEWTOKEN(PSI_T_INTVAL); goto start; }
 493                 'FLOATVAL'              { NEWTOKEN(PSI_T_FLOATVAL); goto start; }
 494                 'BOOLVAL'               { NEWTOKEN(PSI_T_BOOLVAL); goto start; }
 495                 'ARRVAL'                { NEWTOKEN(PSI_T_ARRVAL); goto start; }
 496                 'OBJVAL'                { NEWTOKEN(PSI_T_OBJVAL); goto start; }
 497                 'ZVAL'                  { NEWTOKEN(PSI_T_ZVAL); goto start; }
 498                 'COUNT'                 { NEWTOKEN(PSI_T_COUNT); goto start; }
 499                 'CALLOC'                { NEWTOKEN(PSI_T_CALLOC); goto start; }
 500                 'TO_OBJECT'             { NEWTOKEN(PSI_T_TO_OBJECT); goto start; }
 501                 'TO_ARRAY'              { NEWTOKEN(PSI_T_TO_ARRAY); goto start; }
 502                 'TO_STRING'             { NEWTOKEN(PSI_T_TO_STRING); goto start; }
 503                 'TO_INT'                { NEWTOKEN(PSI_T_TO_INT); goto start; }
 504                 'TO_FLOAT'              { NEWTOKEN(PSI_T_TO_FLOAT); goto start; }
 505                 'TO_BOOL'               { NEWTOKEN(PSI_T_TO_BOOL); goto start; }
 506                 NAME                    { NEWTOKEN(PSI_T_NAME); goto start; }
 507                 NSNAME                  { NEWTOKEN(PSI_T_NSNAME); goto start; }
 508                 DOLLAR_NAME             { NEWTOKEN(PSI_T_DOLLAR_NAME); goto start; }
 509                 CPP_HEADER              { tok += 1; cur -= 1; NEWTOKEN(PSI_T_CPP_HEADER); cur += 1; goto start; }
 510                 CPP_ATTRIBUTE   { parens = 2; goto cpp_attribute; }
 511                 EOL                             { NEWTOKEN(PSI_T_EOL); NEWLINE(); goto start; }
 512                 SP+                             { NEWTOKEN(PSI_T_WHITESPACE); goto start; }
 513                 [^]                             { NEWTOKEN(-2); goto error; }
 514                 *                               { NEWTOKEN(-1); goto error; }
 515
 516                 */
 517
 518         character: ;
 519                 /*!re2c
 520
 521                 EOL             { NEWLINE(); goto character; }
 522                 "\\"    { escaped = !escaped;  goto character; }
 523                 "'"             {
 524                         if (escaped) {
 525                                 escaped = false;
 526                                 goto character;
 527                         }
 528                         cur -= 1;
 529                         NEWTOKEN(PSI_T_QUOTED_CHAR);
 530                         cur += 1;
 531                         token->flags = char_width;
 532                         goto start;
 533                 }
 534                 *               { escaped = false; goto character; }
 535
 536                 */
 537
 538         string: ;
 539                 /*!re2c
 540
 541                 EOL             { NEWLINE(); goto string; }
 542                 "\\"    { escaped = !escaped; goto string; }
 543                 "\""    {
 544                         if (escaped) {
 545                                 escaped = false;
 546                                 goto string;
 547                         }
 548                         cur -= 1;
 549                         NEWTOKEN(PSI_T_QUOTED_STRING);
 550                         cur += 1;
 551                         token->flags = char_width;
 552                         goto start;
 553                 }
 554                 *               { escaped = false; goto string; }
 555
 556                 */
 557
 558         comment: ;
 559                 /*!re2c
 560
 561                 EOL             { NEWLINE(); goto comment; }
 562                 "*" "/" { NEWTOKEN(PSI_T_COMMENT); goto start; }
 563                  *              { goto comment; }
 564
 565                 */
 566
 567         comment_sl: ;
 568                 /*!re2c
 569
 570                 EOL     { NEWTOKEN(PSI_T_COMMENT); NEWLINE(); goto start; }
 571                 *       { goto comment_sl; }
 572
 573                 */
 574
 575         cpp_attribute: ;
 576
 577                 /*!re2c
 578
 579                 "("     { ++parens; goto cpp_attribute; }
 580                 ")" { if (parens == 1) { NEWTOKEN(PSI_T_CPP_ATTRIBUTE); goto start; } else { --parens; goto cpp_attribute; } }
 581                 EOL     { NEWLINE(); goto cpp_attribute; }
 582                  *      { goto cpp_attribute; }
 583
 584                 */
 585 error: ;
 586
 587         P->error(PSI_DATA(P), token, PSI_WARNING, "PSI syntax error: unexpected input (%d) '%.*s' at col %tu",
 588                         token->type, token->size, token->text, tok - eol + 1);
 589         psi_plist_free(tokens);
 590         return NULL;
 591
 592 done:
 593
 594         PSI_DEBUG_PRINT(P, "PSI: EOF cur=%p lim=%p\n", cur, lim);
 595
 596         return tokens;
 597 }