X-Git-Url: https://git.m6w6.name/?p=m6w6%2Fext-psi;a=blobdiff_plain;f=src%2Fparser.h;h=30388ad7d3e7bea7a2ea4255b4fcf9b2dffd0552;hp=b27fdafccbe96ec22286b64f83f3d39b117267a1;hb=8b652242a4ef920f2ab82bfb822596de25bc5a63;hpb=39831a9cf4a4aa9f126bc9a949f03ae232e3794b

diff --git a/src/parser.h b/src/parser.h
index b27fdaf..30388ad 100644
--- a/src/parser.h
+++ b/src/parser.h
@@ -7,6 +7,7 @@
 #include <string.h>
 
 #include <Zend/zend_types.h>
+#include <Zend/zend_API.h> /* fcall */
 
 #include "parser_proc.h"
 
@@ -17,17 +18,23 @@
 
 typedef int token_t;
 
-/* in php_psi.h */
 size_t psi_t_alignment(token_t);
 size_t psi_t_size(token_t);
 
 typedef struct PSI_Token {
 	token_t type;
-	unsigned size, *line;
+	unsigned size, line, col;
 	char *text, *file;
 	char buf[1];
 } PSI_Token;
 
+static inline PSI_Token *PSI_TokenCopy(PSI_Token *src);
+
+typedef struct zend_fcall {
+	zend_fcall_info fci;
+	zend_fcall_info_cache fcc;
+} zend_fcall;
+
 typedef union impl_val {
 	char cval;
 	int8_t i8;
@@ -43,10 +50,14 @@ typedef union impl_val {
 	uint64_t u64;
 	float fval;
 	double dval;
+#ifdef HAVE_LONG_DOUBLE
+	long double ldval;
+#endif
 	union {
 		zend_bool bval;
 		zend_long lval;
 		zend_string *str;
+		zend_fcall *cb;
 	} zend;
 	void *ptr;
 	uint8_t _dbg[sizeof(void *)];
@@ -58,6 +69,9 @@ typedef struct decl_type {
 	token_t type;
 	struct decl_type *real;
 	struct decl_struct *strct;
+	struct decl_union *unn;
+	struct decl_enum *enm;
+	struct decl *func;
 } decl_type;
 
 static inline decl_type *init_decl_type(token_t type, const char *name) {
@@ -67,40 +81,6 @@ static inline decl_type *init_decl_type(token_t type, const char *name) {
 	return t;
 }
 
-static inline decl_type *init_decl_type_ex(token_t type, int argc, ...) {
-	va_list argv;
-	char *ptr, *arg;
-	unsigned i;
-	size_t len, pos = 0, all = 0;
-	decl_type *t = calloc(1, sizeof(*t));
-
-	va_start(argv, argc);
-	for (i = 0; i < argc; ++i) {
-		arg = va_arg(argv, char *);
-		len = va_arg(argv, size_t);
-
-		if (len) {
-			if (all) {
-				pos = all;
-				ptr = realloc(ptr, 1 + (all += len));
-			} else {
-				ptr = malloc(1 + (all = len));
-			}
-			memcpy(ptr + pos, arg, len);
-		}
-	}
-	va_end(argv);
-
-	if (!all) {
-		ptr = calloc(1, 1);
-	} else {
-		ptr[all] = 0;
-	}
-	t->type = type;
-	t->name = ptr;
-	return t;
-}
-
 static inline decl_type *real_decl_type(decl_type *type) {
 	while (type->real) {
 		type = type->real;
@@ -108,60 +88,18 @@ static inline decl_type *real_decl_type(decl_type *type) {
 	return type;
 }
 
+static inline void free_decl(struct decl *decl);
 static inline void free_decl_type(decl_type *type) {
 	if (type->token) {
 		free(type->token);
 	}
+	if (type->type == PSI_T_FUNCTION) {
+		free_decl(type->func);
+	}
 	free(type->name);
 	free(type);
 }
 
-typedef struct decl_typedef {
-	PSI_Token *token;
-	char *alias;
-	decl_type *type;
-} decl_typedef;
-
-static inline decl_typedef *init_decl_typedef(const char *name, decl_type *type) {
-	decl_typedef *t = calloc(1, sizeof(*t));
-	t->alias = strdup(name);
-	t->type = type;
-	return t;
-}
-
-static inline void free_decl_typedef(decl_typedef *t) {
-	if (t->token) {
-		free(t->token);
-	}
-	free(t->alias);
-	free_decl_type(t->type);
-	free(t);
-}
-
-typedef struct decl_typedefs {
-	size_t count;
-	decl_typedef **list;
-} decl_typedefs;
-
-static inline decl_typedefs *add_decl_typedef(decl_typedefs *defs, decl_typedef *def) {
-	if (!defs) {
-		defs = calloc(1, sizeof(*defs));
-	}
-	defs->list = realloc(defs->list, ++defs->count * sizeof(*defs->list));
-	defs->list[defs->count-1] = def;
-	return defs;
-}
-
-static void free_decl_typedefs(decl_typedefs *defs) {
-	size_t i;
-
-	for (i = 0; i < defs->count; ++i) {
-		free_decl_typedef(defs->list[i]);
-	}
-	free(defs->list);
-	free(defs);
-}
-
 typedef struct decl_var {
 	PSI_Token *token;
 	char *name;
@@ -178,6 +116,17 @@ static inline decl_var *init_decl_var(const char *name, unsigned pl, unsigned as
 	return v;
 }
 
+static inline decl_var *copy_decl_var(decl_var *src) {
+	decl_var *dest = calloc(1, sizeof(*dest));
+
+	memcpy(dest, src, sizeof(*dest));
+	dest->name = strdup(dest->name);
+	if (dest->token) {
+		dest->token = PSI_TokenCopy(dest->token);
+	}
+	return dest;
+}
+
 static inline void free_decl_var(decl_var *var) {
 	if (var->token) {
 		free(var->token);
@@ -193,7 +142,7 @@ typedef struct decl_struct_layout {
 
 static inline decl_struct_layout *init_decl_struct_layout(size_t pos, size_t len) {
 	decl_struct_layout *l = calloc(1, sizeof(*l));
-
+	ZEND_ASSERT(pos+len);
 	l->pos = pos;
 	l->len = len;
 	return l;
@@ -225,6 +174,9 @@ static inline decl_arg *init_decl_arg(decl_type *type, decl_var *var) {
 }
 
 static inline void free_decl_arg(decl_arg *arg) {
+	if (arg->token && arg->token != arg->var->token) {
+		free(arg->token);
+	}
 	free_decl_type(arg->type);
 	free_decl_var(arg->var);
 	if (arg->layout) {
@@ -233,6 +185,30 @@ static inline void free_decl_arg(decl_arg *arg) {
 	free(arg);
 }
 
+typedef struct decl_typedefs {
+	size_t count;
+	decl_arg **list;
+} decl_typedefs;
+
+static inline decl_typedefs *add_decl_typedef(decl_typedefs *defs, decl_arg *def) {
+	if (!defs) {
+		defs = calloc(1, sizeof(*defs));
+	}
+	defs->list = realloc(defs->list, ++defs->count * sizeof(*defs->list));
+	defs->list[defs->count-1] = def;
+	return defs;
+}
+
+static void free_decl_typedefs(decl_typedefs *defs) {
+	size_t i;
+
+	for (i = 0; i < defs->count; ++i) {
+		free_decl_arg(defs->list[i]);
+	}
+	free(defs->list);
+	free(defs);
+}
+
 typedef struct decl_vars {
 	decl_var **vars;
 	size_t count;
@@ -320,7 +296,11 @@ typedef struct decl_callinfo {
 	void *info;
 	size_t argc;
 	void **args;
-	void *rval;
+	void **rval;
+	struct {
+		void *data;
+		void (*dtor)(void *data);
+	} closure;
 } decl_callinfo;
 
 typedef struct decl {
@@ -372,11 +352,17 @@ static inline void free_decls(decls *decls) {
 	free(decls);
 }
 
+
 typedef struct decl_struct {
 	PSI_Token *token;
 	char *name;
 	decl_args *args;
 	size_t size;
+	size_t align;
+	struct {
+		void *type;
+		void (*dtor)(void *type);
+	} engine;
 } decl_struct;
 
 static inline decl_struct *init_decl_struct(const char *name, decl_args *args) {
@@ -393,6 +379,9 @@ static inline void free_decl_struct(decl_struct *s) {
 	if (s->args) {
 		free_decl_args(s->args);
 	}
+	if (s->engine.type && s->engine.dtor) {
+		s->engine.dtor(s->engine.type);
+	}
 	free(s->name);
 	free(s);
 }
@@ -421,6 +410,56 @@ static inline void free_decl_structs(decl_structs *ss) {
 	free(ss);
 }
 
+typedef struct decl_union {
+	PSI_Token *token;
+	char *name;
+	decl_args *args;
+	size_t size;
+	size_t align;
+} decl_union;
+
+static inline decl_union *init_decl_union(const char *name, decl_args *args) {
+	decl_union *u = calloc(1, sizeof(*u));
+	u->name = strdup(name);
+	u->args = args;
+	return u;
+}
+
+static inline void free_decl_union(decl_union *u) {
+	if (u->token) {
+		free(u->token);
+	}
+	if (u->args) {
+		free_decl_args(u->args);
+	}
+	free(u->name);
+	free(u);
+}
+
+typedef struct decl_unions {
+	decl_union **list;
+	size_t count;
+} decl_unions;
+
+static inline decl_unions *add_decl_union(decl_unions *uu, decl_union *u) {
+	if (!uu) {
+		uu = calloc(1, sizeof(*uu));
+	}
+	uu->list = realloc(uu->list, ++uu->count * sizeof(*uu->list));
+	uu->list[uu->count-1] = u;
+	return uu;
+}
+
+static inline void free_decl_unions(decl_unions *uu) {
+	size_t i;
+
+	for (i = 0; i < uu->count; ++i) {
+		free_decl_union(uu->list[i]);
+	}
+	free(uu->list);
+	free(uu);
+}
+
 typedef struct impl_type {
 	char *name;
 	token_t type;
@@ -452,6 +491,17 @@ static inline impl_var *init_impl_var(const char *name, int is_reference) {
 	return var;
 }
 
+static inline impl_var *copy_impl_var(impl_var *var) {
+	impl_var *cpy = malloc(sizeof(*cpy));
+
+	memcpy(cpy, var, sizeof(*cpy));
+	cpy->name = strdup(cpy->name);
+	if (cpy->token) {
+		cpy->token = PSI_TokenCopy(cpy->token);
+	}
+	return cpy;
+}
+
 static inline void free_impl_var(impl_var *var) {
 	if (var->token) {
 		free(var->token);
@@ -641,6 +691,7 @@ typedef struct num_exp {
 		char *numb;
 		constant *cnst;
 		decl_var *dvar;
+		struct decl_enum_item *enm;
 	} u;
 	token_t operator;
 	int (*calculator)(int t1, impl_val *v1, int t2, impl_val *v2, impl_val *res);
@@ -662,6 +713,35 @@ static inline num_exp *init_num_exp(token_t t, void *num) {
 	return exp;
 }
 
+static inline num_exp *copy_num_exp(num_exp *exp) {
+	decl_var *dvar;
+	num_exp *num = calloc(1, sizeof(*num));
+
+	memcpy(num, exp, sizeof(*num));
+
+	if (num->token) {
+		num->token = PSI_TokenCopy(num->token);
+	}
+	if (num->operand) {
+		num->operand = copy_num_exp(num->operand);
+	}
+	switch (num->t) {
+	case PSI_T_NUMBER:
+	case PSI_T_NSNAME:
+		num->u.numb = strdup(num->u.numb);
+		break;
+	case PSI_T_NAME:
+		dvar = init_decl_var(num->u.dvar->name, num->u.dvar->pointer_level, num->u.dvar->array_size);
+		dvar->arg = num->u.dvar->arg;
+		if (num->u.dvar->token) {
+			dvar->token = PSI_TokenCopy(num->u.dvar->token);
+		}
+		num->u.dvar = dvar;
+		break;
+	}
+	return num;
+}
+
 static inline void free_num_exp(num_exp *exp) {
 	if (exp->token) {
 		free(exp->token);
@@ -675,6 +755,8 @@ static inline void free_num_exp(num_exp *exp) {
 	case PSI_T_NAME:
 		free_decl_var(exp->u.dvar);
 		break;
+	case PSI_T_ENUM:
+		break;
 	EMPTY_SWITCH_DEFAULT_CASE();
 	}
 	if (exp->operand) {
@@ -683,6 +765,122 @@ static inline void free_num_exp(num_exp *exp) {
 	free(exp);
 }
 
+typedef struct decl_enum_item {
+	PSI_Token *token;
+	char *name;
+	num_exp *num;
+	num_exp inc;
+	struct decl_enum_item *prev;
+} decl_enum_item;
+
+static inline decl_enum_item *init_decl_enum_item(const char *name, num_exp *num) {
+	decl_enum_item *i = calloc(1, sizeof(*i));
+
+	i->name = strdup(name);
+	i->num = num;
+	return i;
+}
+
+static inline void free_decl_enum_item(decl_enum_item *i) {
+	if (i->token) {
+		free(i->token);
+	}
+	if (i->num && i->num != &i->inc) {
+		free_num_exp(i->num);
+	}
+	free(i->name);
+	free(i);
+}
+
+typedef struct decl_enum_items {
+	decl_enum_item **list;
+	size_t count;
+} decl_enum_items;
+
+static inline decl_enum_items *init_decl_enum_items(decl_enum_item *i) {
+	decl_enum_items *l = calloc(1, sizeof(*l));
+
+	if (i) {
+		l->count = 1;
+		l->list = calloc(1, sizeof(*l->list));
+		l->list[0] = i;
+	}
+	return l;
+}
+
+static inline decl_enum_items *add_decl_enum_item(decl_enum_items *l, decl_enum_item *i) {
+	l->list = realloc(l->list, sizeof(*l->list) * (l->count + 1));
+	l->list[l->count] = i;
+	if (l->count) {
+		i->prev = l->list[l->count - 1];
+	}
+	++l->count;
+	return l;
+}
+
+static inline void free_decl_enum_items(decl_enum_items *l) {
+	if (l->list) {
+		size_t j;
+
+		for (j = 0; j < l->count; ++j) {
+			free_decl_enum_item(l->list[j]);
+		}
+		free(l->list);
+	}
+	free(l);
+}
+
+typedef struct decl_enum {
+	PSI_Token *token;
+	char *name;
+	decl_enum_items *items;
+} decl_enum;
+
+static inline decl_enum *init_decl_enum(const char *name, decl_enum_items *l) {
+	decl_enum *e = calloc(1, sizeof(*e));
+
+	e->name = strdup(name);
+	e->items = l;
+	return e;
+}
+
+static inline void free_decl_enum(decl_enum *e) {
+	if (e->token) {
+		free(e->token);
+	}
+	if (e->items) {
+		free_decl_enum_items(e->items);
+	}
+	free(e->name);
+	free(e);
+}
+
+typedef struct decl_enums {
+	decl_enum **list;
+	size_t count;
+} decl_enums;
+
+static inline decl_enums* add_decl_enum(decl_enums *es, decl_enum *e) {
+	if (!es) {
+		es = calloc(1, sizeof(*es));
+	}
+	es->list = realloc(es->list, ++es->count * sizeof(*es->list));
+	es->list[es->count-1] = e;
+	return es;
+}
+
+static inline void free_decl_enums(decl_enums *es) {
+	if (es->list) {
+		size_t j;
+
+		for (j = 0; j < es->count; ++j) {
+			free_decl_enum(es->list[j]);
+		}
+	}
+	free(es->list);
+	free(es);
+}
+
 typedef struct let_calloc {
 	num_exp *nmemb;
 	num_exp *size;
@@ -701,11 +899,33 @@ static inline void free_let_calloc(let_calloc *alloc) {
 	free(alloc);
 }
 
+typedef struct let_callback {
+	struct let_func *func;
+	struct set_values *args;
+} let_callback;
+
+static inline void free_let_func(struct let_func *func);
+static inline void free_set_values(struct set_values *vals);
+static inline let_callback *init_let_callback(struct let_func *func, struct set_values *args) {
+	let_callback *cb = calloc(1, sizeof(*cb));
+
+	cb->func = func;
+	cb->args = args;
+	return cb;
+}
+
+static inline void free_let_callback(let_callback *cb) {
+	free_let_func(cb->func);
+	free_set_values(cb->args);
+	free(cb);
+}
+
 typedef struct let_func {
 	token_t type;
 	char *name;
 	impl_var *var;
 	impl_arg *arg;
+	impl_val *(*handler)(impl_val *tmp, decl_type *type, void *val, void **to_free);
 } let_func;
 
 static inline let_func *init_let_func(token_t type, const char *name, impl_var *var) {
@@ -728,12 +948,14 @@ typedef struct let_val {
 		PSI_LET_NULL,
 		PSI_LET_NUMEXP,
 		PSI_LET_CALLOC,
+		PSI_LET_CALLBACK,
 		PSI_LET_FUNC,
 		PSI_LET_TMP,
 	} kind;
 	union {
 		num_exp *num;
 		let_calloc *alloc;
+		let_callback *callback;
 		let_func *func;
 		decl_var *var;
 	} data;
@@ -756,6 +978,9 @@ static inline let_val *init_let_val(enum let_val_kind kind, void *data) {
 	case PSI_LET_CALLOC:
 		let->data.alloc = data;
 		break;
+	case PSI_LET_CALLBACK:
+		let->data.callback = data;
+		break;
 	case PSI_LET_FUNC:
 		let->data.func = data;
 		break;
@@ -777,6 +1002,9 @@ static inline void free_let_val(let_val *let) {
 	case PSI_LET_CALLOC:
 		free_let_calloc(let->data.alloc);
 		break;
+	case PSI_LET_CALLBACK:
+		free_let_callback(let->data.callback);
+		break;
 	case PSI_LET_FUNC:
 		free_let_func(let->data.func);
 		break;
@@ -845,10 +1073,15 @@ typedef struct set_value {
 		struct set_value *set;
 		impl_val *val;
 	} outer;
-	struct set_value **inner;
-	size_t count;
+	struct set_values *inner;
 } set_value;
 
+typedef struct set_values {
+	set_value **vals;
+	size_t count;
+} set_values;
+
+
 static inline set_value *init_set_value(set_func *func, decl_vars *vars) {
 	set_value *val = calloc(1, sizeof(*val));
 	val->func = func;
@@ -856,8 +1089,8 @@ static inline set_value *init_set_value(set_func *func, decl_vars *vars) {
 	return val;
 }
 static inline set_value *add_inner_set_value(set_value *val, set_value *inner) {
-	val->inner = realloc(val->inner, ++val->count * sizeof(*val->inner));
-	val->inner[val->count-1] = inner;
+	val->inner->vals = realloc(val->inner->vals, ++val->inner->count * sizeof(*val->inner->vals));
+	val->inner->vals[val->inner->count-1] = inner;
 	inner->outer.set = val;
 	return val;
 }
@@ -871,10 +1104,10 @@ static inline void free_set_value(set_value *val) {
 	}
 	if (val->inner && (!val->outer.set || val->outer.set->inner != val->inner)) {
 		size_t i;
-		for (i = 0; i < val->count; ++i) {
-			free_set_value(val->inner[i]);
+		for (i = 0; i < val->inner->count; ++i) {
+			free_set_value(val->inner->vals[i]);
 		}
-		free(val->inner);
+		free(val->inner->vals);
 	}
 	if (val->num) {
 		free_num_exp(val->num);
@@ -882,6 +1115,29 @@ static inline void free_set_value(set_value *val) {
 	free(val);
 }
 
+static inline set_values *init_set_values(set_value *val) {
+	set_values *vals = calloc(1, sizeof(*vals));
+	if (val) {
+		vals->count = 1;
+		vals->vals = calloc(1, sizeof(val));
+		vals->vals[0] = val;
+	}
+	return vals;
+}
+
+static inline set_values *add_set_value(set_values *vals, set_value *val) {
+	vals->vals = realloc(vals->vals, ++vals->count * sizeof(val));
+	vals->vals[vals->count-1] = val;
+	return vals;
+}
+
+static inline void free_set_values(set_values *vals) {
+	if (vals->vals) {
+		free(vals->vals);
+	}
+	free(vals);
+}
+
 typedef struct set_stmt {
 	impl_var *var;
 	set_value *val;
@@ -1206,7 +1462,7 @@ static inline impl_val *enref_impl_val(void *ptr, decl_var *var) {
 			var->name, var->pointer_level, var->arg->var->pointer_level,
 			var->array_size, var->arg->var->array_size);
 #endif
-	if (!var->pointer_level && real_decl_type(var->arg->type)->type != PSI_T_STRUCT) {
+	if (!var->pointer_level ){//&& real_decl_type(var->arg->type)->type != PSI_T_STRUCT) {
 		return ptr;
 	}
 
@@ -1233,20 +1489,24 @@ static inline impl_val *struct_member_ref(decl_arg *set_arg, impl_val *struct_pt
 
 #define PSI_ERROR 16
 #define PSI_WARNING 32
-typedef void (*psi_error_cb)(PSI_Token *token, int type, const char *msg, ...);
+typedef void (*psi_error_cb)(void *context, PSI_Token *token, int type, const char *msg, ...);
 
 #define PSI_DATA(D) ((PSI_Data *) (D))
 #define PSI_DATA_MEMBERS \
 	constants *consts; \
 	decl_typedefs *defs; \
 	decl_structs *structs; \
+	decl_unions *unions; \
+	decl_enums *enums; \
 	decls *decls; \
 	impls *impls; \
 	union { \
 		decl_file file; \
 		decl_libs libs; \
 	} psi; \
-	psi_error_cb error
+	psi_error_cb error; \
+	unsigned errors; \
+	unsigned flags
 typedef struct PSI_Data {
 	PSI_DATA_MEMBERS;
 } PSI_Data;
@@ -1270,6 +1530,12 @@ static inline void PSI_DataDtor(PSI_Data *data) {
 	if (data->structs) {
 		free_decl_structs(data->structs);
 	}
+	if (data->unions) {
+		free_decl_unions(data->unions);
+	}
+	if (data->enums) {
+		free_decl_enums(data->enums);
+	}
 	if (data->decls) {
 		free_decls(data->decls);
 	}
@@ -1284,12 +1550,12 @@ typedef struct PSI_Parser {
 	FILE *fp;
 	token_t num;
 	void *proc;
-	unsigned flags, errors, line, col;
+	unsigned line, col;
 	char *cur, *tok, *lim, *eof, *ctx, *mrk, buf[BSIZE];
 } PSI_Parser;
 
 static inline size_t PSI_TokenAllocSize(size_t token_len, size_t fname_len) {
-	return sizeof(PSI_Token) + token_len + fname_len + sizeof(unsigned) + 2;
+	return sizeof(PSI_Token) + token_len + fname_len + 2;
 }
 
 static inline PSI_Token *PSI_TokenAlloc(PSI_Parser *P) {
@@ -1310,11 +1576,11 @@ static inline PSI_Token *PSI_TokenAlloc(PSI_Parser *P) {
 	T->size = token_len;
 	T->text = &T->buf[0];
 	T->file = &T->buf[token_len + 1];
-	T->line = (void *) &T->buf[fname_len + token_len + 2];
+	T->line = P->line;
+	T->col = P->col;
 
 	memcpy(T->text, P->tok, token_len);
 	memcpy(T->file, P->psi.file.fn, fname_len);
-	memcpy(T->line, &P->line, sizeof(unsigned));
 
 	return T;
 }
@@ -1341,12 +1607,14 @@ static inline PSI_Token *PSI_TokenCat(unsigned argc, ...) {
 		PSI_Token *arg = va_arg(argv, PSI_Token *);
 
 		if (T) {
-			size_t fname_len = strlen(T->file);
+			size_t token_len = T->size, fname_len = strlen(T->file);
 
-			T = realloc(T, PSI_TokenAllocSize(T->size + arg->size, fname_len));
-			memmove(&T->buf[T->size + 1], T->file, fname_len + 1);
-			memcpy(T->file - 1, arg->text, arg->size + 1);
+			T = realloc(T, PSI_TokenAllocSize(T->size += arg->size + 1, fname_len));
+			T->text = &T->buf[0];
 			T->file = &T->buf[T->size + 1];
+			T->buf[token_len] = ' ';
+			memmove(&T->buf[T->size + 1], &T->buf[token_len + 1], fname_len + 1);
+			memcpy(&T->buf[token_len + 1], arg->text, arg->size + 1);
 		} else {
 			T = PSI_TokenCopy(arg);
 			T->type = PSI_T_NAME;
@@ -1357,11 +1625,64 @@ static inline PSI_Token *PSI_TokenCat(unsigned argc, ...) {
 	return T;
 }
 
-static inline const char *PSI_TokenLocation(PSI_Token *t) {
-	return t ? t->file : "<builtin>:0:0";
+static inline PSI_Token *PSI_TokenAppend(PSI_Token *T, unsigned argc, ...) {
+	va_list argv;
+	unsigned i;
+
+	va_start(argv, argc);
+	for (i = 0; i < argc; ++i) {
+		char *str = va_arg(argv, char *);
+		size_t str_len = strlen(str), token_len = T->size, fname_len = strlen(T->file);
+
+		T = realloc(T, PSI_TokenAllocSize(T->size += str_len + 1, fname_len));
+		T->text = &T->buf[0];
+		T->file = &T->buf[T->size + 1];
+		T->buf[token_len] = ' ';
+		memmove(&T->buf[T->size + 1], &T->buf[token_len + 1], fname_len + 1);
+		memcpy(&T->buf[token_len + 1], str, str_len + 1);
+	}
+	va_end(argv);
+
+	return T;
+}
+
+char *php_strtr(char *str, size_t len, char *str_from, char *str_to, size_t trlen);
+static inline PSI_Token *PSI_TokenTranslit(PSI_Token *T, char *from, char *to) {
+	php_strtr(T->text, T->size, from, to, MIN(strlen(from), strlen(to)));
+	return T;
+}
+
+static inline uint64_t psi_hash(char *digest_buf, ...)
+{
+    uint64_t hash = 5381;
+    uint8_t c;
+    const uint8_t *ptr;
+    va_list argv;
+
+    va_start(argv, digest_buf);
+    while ((ptr = va_arg(argv, const uint8_t *))) {
+		while ((c = *ptr++)) {
+			hash = ((hash << 5) + hash) + c;
+		}
+    }
+    va_end(argv);
+
+    if (digest_buf) {
+    	sprintf(digest_buf, "%" PRIx64, hash);
+    }
+
+    return hash;
+}
+
+static inline uint64_t PSI_TokenHash(PSI_Token *t, char *digest_buf) {
+	char loc_buf[48];
+
+	sprintf(loc_buf, "%u%u", t->line, t->col);
+	return psi_hash(digest_buf, t->file, loc_buf, NULL);
 }
 
 #define PSI_PARSER_DEBUG 0x1
+#define PSI_PARSER_SILENT 0x2
 
 PSI_Parser *PSI_ParserInit(PSI_Parser *P, const char *filename, psi_error_cb error, unsigned flags);
 void PSI_ParserSyntaxError(PSI_Parser *P, const char *fn, size_t ln, const char *msg, ...);