From: Michael Wallner <mike@php.net>
Date: Wed, 5 Dec 2018 10:46:02 +0000 (+0100)
Subject: flush
X-Git-Url: https://git.m6w6.name/?p=m6w6%2Fext-psi;a=commitdiff_plain;h=cd0567b4052ee24259bc6b8a9858af9d26f3de48

flush
---

diff --git a/README.md b/README.md
index 6c91e51..8d7a235 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,8 @@ The acronym PSI may be read as:
 ## Installing
 
 > **WARNING:**  
-> This is heavy WIP. Installation only works from a source checkout with php-src@master yet.
+> This is heavy WIP. Installation only works from a source checkout 
+> with php-src@master on x86_64-linux yet.
 
 ### PECL
 
diff --git a/src/cpp.c b/src/cpp.c
index eaf43f8..26e324f 100644
--- a/src/cpp.c
+++ b/src/cpp.c
@@ -111,6 +111,7 @@ struct psi_cpp *psi_cpp_init(struct psi_parser *P)
 	zend_hash_init(&cpp->once, 0, NULL, NULL, 1);
 	zend_hash_init(&cpp->defs, 0, NULL, free_cpp_def, 1);
 	zend_hash_copy(&cpp->defs, &psi_cpp_defaults, NULL);
+	zend_hash_init(&cpp->expanding, 0, NULL, NULL, 1);
 
 	return cpp;
 }
@@ -129,6 +130,7 @@ void psi_cpp_free(struct psi_cpp **cpp_ptr)
 		*cpp_ptr = NULL;
 		zend_hash_destroy(&cpp->defs);
 		zend_hash_destroy(&cpp->once);
+		zend_hash_destroy(&cpp->expanding);
 		free(cpp);
 	}
 }
@@ -230,8 +232,7 @@ static bool psi_cpp_stage1(struct psi_cpp *cpp)
 
 static bool psi_cpp_stage2(struct psi_cpp *cpp)
 {
-	struct psi_plist *parser_tokens = psi_plist_init((psi_plist_dtor) psi_token_free);
-	bool is_eol = true, do_cpp = false, do_expansion = true, skip_paren = false, skip_all = false;
+	bool is_eol = true, do_expansion = true, skip_paren = false, skip_all = false;
 
 	PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP %s\n", "stage2");
 
@@ -241,7 +242,7 @@ static bool psi_cpp_stage2(struct psi_cpp *cpp)
 
 		if (current->type == PSI_T_HASH) {
 			if (is_eol) {
-				do_cpp = true;
+				cpp->do_cpp = true;
 				is_eol = false;
 			}
 		} else if (current->type == PSI_T_EOL) {
@@ -251,14 +252,14 @@ static bool psi_cpp_stage2(struct psi_cpp *cpp)
 			is_eol = true;
 			skip_all = false;
 			do_expansion = true;
-			if (!do_cpp) {
+			if (!cpp->do_cpp) {
 				psi_cpp_tokiter_del_cur(cpp, true);
 				continue;
 			}
 		} else {
 			is_eol = false;
 
-			if (do_cpp) {
+			if (cpp->do_cpp) {
 				switch (current->type) {
 				case PSI_T_DEFINE:
 #if PSI_CPP_DEBUG
@@ -303,20 +304,22 @@ static bool psi_cpp_stage2(struct psi_cpp *cpp)
 		}
 
 		if (cpp->skip) {
-			if (!do_cpp) {
+			if (!cpp->do_cpp) {
 #if PSI_CPP_DEBUG
+				psi_debug_lock(PSI_DATA(cpp->parser));
 				PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP skip ");
 				PSI_DEBUG_DUMP(cpp->parser, psi_token_dump, current);
+				psi_debug_unlock(PSI_DATA(cpp->parser));
 #endif
 				psi_cpp_tokiter_del_cur(cpp, true);
 				continue;
 			}
 		}
 
-		if (do_expansion && current->type == PSI_T_NAME && psi_cpp_tokiter_defined(cpp)) {
+		if (do_expansion && psi_cpp_defined(cpp, current)) {
 			bool expanded = false;
 
-			while (psi_cpp_tokiter_expand(cpp)) {
+			if (psi_cpp_tokiter_expand(cpp)) {
 				expanded = true;
 			}
 			if (expanded) {
@@ -324,55 +327,58 @@ static bool psi_cpp_stage2(struct psi_cpp *cpp)
 			}
 		}
 
-		if (do_cpp) {
-			parser_tokens = psi_plist_add(parser_tokens, &current);
+		psi_cpp_tokiter_add_cur(cpp);
 
-			if (is_eol) {
-				size_t processed = 0;
-				bool parsed = psi_parser_process(cpp->parser, parser_tokens, &processed);
+		if (cpp->do_cpp && is_eol) {
+			size_t processed = 0;
+			bool parsed;
 
-				/* EOL */
-				psi_plist_pop(parser_tokens, NULL);
-				psi_plist_clean(parser_tokens);
-				do_cpp = false;
+			cpp->do_cpp = false;
 
-				if (!parsed) {
-					psi_plist_free(parser_tokens);
-					return false;
-				}
-			} else {
-				/* leave EOLs in the input stream, else we might end up
-				 * with a hash not preceded with a new line after include */
-				psi_cpp_tokiter_del_cur(cpp, false);
+			parsed = psi_parser_process(cpp->parser, cpp->tokens.exec, &processed);
+
+			/* leave EOLs in the input stream, else we might end up
+			 * with a hash not preceded with a new line after include */
+			psi_plist_pop(cpp->tokens.exec, NULL);
+			psi_plist_clean(cpp->tokens.exec);
+
+			if (!parsed) {
+				psi_plist_free(cpp->tokens.exec);
+				return false;
 			}
 
 #if PSI_CPP_DEBUG > 1
 			PSI_DEBUG_DUMP(cpp->parser, psi_cpp_tokiter_dump, cpp);
 #endif
-
-			continue;
 		}
 
-		psi_cpp_tokiter_add_cur(cpp);
 		psi_cpp_tokiter_next(cpp);
 	}
 
-	psi_plist_free(parser_tokens);
+	psi_plist_free(cpp->tokens.exec);
 
 	return true;
 }
 
-bool psi_cpp_process(struct psi_cpp *cpp, struct psi_plist **tokens)
+bool psi_cpp_process(struct psi_cpp *cpp, struct psi_plist **tokens,
+		struct psi_token *expanding)
 {
 	bool parsed = false;
-	struct psi_cpp temp = *cpp;	cpp->level = temp.level;
+	struct psi_cpp temp = *cpp;
 
 	cpp->tokens.iter = *tokens;
 	cpp->tokens.next = NULL;
+	cpp->tokens.exec = NULL;
 
+	if (expanding) {
+		zend_hash_add_empty_element(&cpp->expanding, expanding->text);
+	}
 	if (psi_cpp_stage1(cpp) && psi_cpp_stage2(cpp)) {
 		parsed = true;
 	}
+	if (expanding) {
+		zend_hash_del(&cpp->expanding, expanding->text);
+	}
 
 	if (cpp->tokens.next) {
 		free(cpp->tokens.iter);
@@ -385,36 +391,54 @@ bool psi_cpp_process(struct psi_cpp *cpp, struct psi_plist **tokens)
 	if (temp.tokens.iter) {
 		cpp->tokens.iter = temp.tokens.iter;
 		cpp->tokens.next = temp.tokens.next;
+		cpp->tokens.exec = temp.tokens.exec;
 	}
 	cpp->index = temp.index;
 	cpp->skip = temp.skip;
 	cpp->level = temp.level;
 	cpp->seen = temp.seen;
+	cpp->do_cpp = temp.do_cpp;
 
 	return parsed;
 }
 
 bool psi_cpp_defined(struct psi_cpp *cpp, struct psi_token *tok)
 {
-	bool defined;
+	bool defined = false;
 
 	if (tok->type == PSI_T_NAME) {
-		defined = zend_hash_exists(&cpp->defs, tok->text)
-				|| psi_builtin_exists(tok->text);
-	} else {
-		defined = false;
-	}
+		if (psi_builtin_exists(tok->text)) {
+			defined = true;
+		} else if (!zend_hash_exists(&cpp->expanding, tok->text)) {
+			struct psi_macro_decl *macro = zend_hash_find_ptr(&cpp->defs, tok->text);
 
+			if (macro) {
+				defined = true;
+			}
+		}
 #if PSI_CPP_DEBUG
-	PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP defined -> %s ", defined ? "true" : "false");
-	if (defined) {
-		struct psi_cpp_macro_decl *macro = zend_hash_find_ptr(&cpp->defs, tok->text);
-		if (macro) {
-			PSI_DEBUG_PRINT(cpp->parser, " @ %s:%u ", macro->token->file->val, macro->token->line);
+		psi_debug_lock(PSI_DATA(cpp->parser));
+		PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP defined -> %s ", defined ? "true" : "false");
+		if (defined) {
+			struct psi_cpp_macro_decl *macro = zend_hash_find_ptr(&cpp->defs, tok->text);
+			if (macro) {
+				PSI_DEBUG_PRINT(cpp->parser, " @ %s:%u ", macro->token->file->val, macro->token->line);
+			}
+		} else {
+			zend_string *key;
+
+			PSI_DEBUG_PRINT(cpp->parser, " expanding=");
+			ZEND_HASH_FOREACH_STR_KEY(&cpp->expanding, key)
+			{
+				PSI_DEBUG_PRINT(cpp->parser, "%s,", key->val);
+			}
+			ZEND_HASH_FOREACH_END();
+			PSI_DEBUG_PRINT(cpp->parser, "\t");
 		}
-	}
-	PSI_DEBUG_DUMP(cpp->parser, psi_token_dump, tok);
+		PSI_DEBUG_DUMP(cpp->parser, psi_token_dump, tok);
+		psi_debug_unlock(PSI_DATA(cpp->parser));
 #endif
+	}
 
 	return defined;
 }
@@ -430,6 +454,7 @@ void psi_cpp_define(struct psi_cpp *cpp, struct psi_cpp_macro_decl *decl)
 				"'%s' previously defined", old->token->text->val);
 	}
 #if PSI_CPP_DEBUG
+	psi_debug_lock(PSI_DATA(cpp->parser));
 	if (decl->exp) {
 		PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP MACRO num_exp -> ");
 	} else {
@@ -437,6 +462,7 @@ void psi_cpp_define(struct psi_cpp *cpp, struct psi_cpp_macro_decl *decl)
 	}
 	PSI_DEBUG_DUMP(cpp->parser, psi_cpp_macro_decl_dump, decl);
 	PSI_DEBUG_PRINT(cpp->parser, "\n");
+	psi_debug_unlock(PSI_DATA(cpp->parser));
 #endif
 	zend_hash_update_ptr(&cpp->defs, decl->token->text, decl);
 }
@@ -496,7 +522,7 @@ bool psi_cpp_include(struct psi_cpp *cpp, const struct psi_token *file, unsigned
 		return false;
 	}
 
-	parsed = psi_cpp_process(cpp, &tokens);
+	parsed = psi_cpp_process(cpp, &tokens, NULL);
 	if (!parsed) {
 		psi_plist_free(tokens);
 		return false;
diff --git a/src/cpp.h b/src/cpp.h
index 671d2da..be7a54c 100644
--- a/src/cpp.h
+++ b/src/cpp.h
@@ -29,7 +29,7 @@
 #include "data.h"
 
 #ifndef PSI_CPP_DEBUG
-# define PSI_CPP_DEBUG 0
+# define PSI_CPP_DEBUG 1
 #endif
 
 struct psi_cpp {
@@ -39,7 +39,9 @@ struct psi_cpp {
 	struct {
 		struct psi_plist *iter;
 		struct psi_plist *next;
+		struct psi_plist *exec;
 	} tokens;
+	HashTable expanding;
 	const char *search;
 	size_t index;
 	unsigned level;
@@ -47,10 +49,12 @@ struct psi_cpp {
 	unsigned seen;
 	unsigned expanded;
 	unsigned counter;
+	bool do_cpp;
 };
 
 struct psi_cpp *psi_cpp_init(struct psi_parser *parser);
-bool psi_cpp_process(struct psi_cpp *cpp, struct psi_plist **tokens);
+bool psi_cpp_process(struct psi_cpp *cpp, struct psi_plist **tokens,
+		struct psi_token *expanding);
 void psi_cpp_free(struct psi_cpp **cpp_ptr);
 
 bool psi_cpp_if(struct psi_cpp *cpp, struct psi_cpp_exp *exp);
diff --git a/src/cpp_tokiter.c b/src/cpp_tokiter.c
index a28fcf9..2a8fc39 100644
--- a/src/cpp_tokiter.c
+++ b/src/cpp_tokiter.c
@@ -73,12 +73,14 @@ void psi_cpp_tokiter_reset(struct psi_cpp *cpp)
 	cpp->skip = 0;
 	cpp->seen = 0;
 	cpp->level = 0;
+	cpp->do_cpp = false;
 
 	if (cpp->tokens.next) {
 		free(cpp->tokens.iter);
 		cpp->tokens.iter = cpp->tokens.next;
 	}
 	cpp->tokens.next = psi_plist_init((psi_plist_dtor) psi_token_free);
+	cpp->tokens.exec = psi_plist_init((psi_plist_dtor) psi_token_free);
 }
 
 bool psi_cpp_tokiter_seek(struct psi_cpp *cpp, size_t index)
@@ -110,19 +112,21 @@ bool psi_cpp_tokiter_add_cur(struct psi_cpp *cpp)
 	struct psi_token *cur = NULL;
 
 	if (psi_plist_get(cpp->tokens.iter, cpp->index, &cur)) {
-		struct psi_plist *tokens = psi_plist_add(cpp->tokens.next, &cur);
-
-		if (tokens) {
-			cpp->tokens.next = tokens;
+		if (cpp->do_cpp) {
+			cpp->tokens.exec = psi_plist_add(cpp->tokens.exec, &cur);
+		} else {
+			cpp->tokens.next = psi_plist_add(cpp->tokens.next, &cur);
+		}
 
 #if PSI_CPP_DEBUG
-			PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP add_cur -> index=%zu, iter.count=%zu, next.count=%zu	",
-					cpp->index, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next));
-			PSI_DEBUG_DUMP(cpp->parser, psi_token_dump, cur);
+		PSI_DEBUG_PRINT(cpp->parser,
+				"PSI: CPP add_cur -> index=%zu, iter.count=%zu, next.count=%zu, exec.count=%zu	",
+				cpp->index, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next),
+				psi_plist_count(cpp->tokens.exec));
+		PSI_DEBUG_DUMP(cpp->parser, psi_token_dump, cur);
 #endif
 
-			return true;
-		}
+		return true;
 	}
 
 	return false;
@@ -130,16 +134,17 @@ bool psi_cpp_tokiter_add_cur(struct psi_cpp *cpp)
 
 bool psi_cpp_tokiter_add(struct psi_cpp *cpp, struct psi_token *tok)
 {
-	struct psi_plist *tokens = psi_plist_add(cpp->tokens.next, &tok);
-
-	if (!tokens) {
-		return false;
+	if (cpp->do_cpp) {
+		cpp->tokens.exec = psi_plist_add(cpp->tokens.exec, &tok);
+	} else {
+		cpp->tokens.next = psi_plist_add(cpp->tokens.next, &tok);
 	}
-	cpp->tokens.next = tokens;
 
 #if PSI_CPP_DEBUG
-	PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP add -> index=%zu, iter.count=%zu, next.count=%zu	",
-			cpp->index, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next));
+	PSI_DEBUG_PRINT(cpp->parser,
+			"PSI: CPP add -> index=%zu, iter.count=%zu, next.count=%zu, exec.count=%zu	",
+			cpp->index, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next),
+			psi_plist_count(cpp->tokens.exec));
 	PSI_DEBUG_DUMP(cpp->parser, psi_token_dump, tok);
 #endif
 
@@ -155,15 +160,17 @@ bool psi_cpp_tokiter_add_range(struct psi_cpp *cpp, size_t num_eles, void **eles
 		return true;
 	}
 
-	tokens = psi_plist_add_r(cpp->tokens.next, num_eles, eles);
-	if (!tokens) {
-		return false;
+	if (cpp->do_cpp) {
+		cpp->tokens.exec = psi_plist_add_r(cpp->tokens.exec, num_eles, eles);
+	} else {
+		cpp->tokens.next = psi_plist_add_r(cpp->tokens.next, num_eles, eles);
 	}
-	cpp->tokens.next = tokens;
 
 #if PSI_CPP_DEBUG
-	PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP add_range -> index=%zu, num_eles=%zu, iter.count=%zu, next.count=%zu\n",
-			cpp->index, num_eles, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next));
+	PSI_DEBUG_PRINT(cpp->parser,
+			"PSI: CPP add_range -> index=%zu, num_eles=%zu, iter.count=%zu, next.count=%zu, exec.count=%zu\n",
+			cpp->index, num_eles, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next),
+			psi_plist_count(cpp->tokens.exec));
 #endif
 
 	return true;
@@ -193,27 +200,39 @@ bool psi_cpp_tokiter_del_prev(struct psi_cpp *cpp, bool free_token)
 	struct psi_token *cur = NULL;
 
 #if PSI_CPP_DEBUG
-	PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP del_prev -> index=%zu, iter.count=%zu, next.count=%zu\n",
-			cpp->index, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next));
+	PSI_DEBUG_PRINT(cpp->parser,
+			"PSI: CPP del_prev -> index=%zu, iter.count=%zu, next.count=%zu, exec.count=%zu\n",
+			cpp->index, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next),
+			psi_plist_count(cpp->tokens.exec));
 #endif
 
-	if (psi_plist_pop(cpp->tokens.next, NULL) && psi_plist_get(cpp->tokens.iter, cpp->index - 1, &cur)) {
-		psi_plist_unset(cpp->tokens.iter, cpp->index - 1);
-		if (free_token && cur) {
-			psi_token_free(&cur);
+	if (cpp->do_cpp) {
+		if (!psi_plist_pop(cpp->tokens.exec, NULL)) {
+			return false;
 		}
-		return true;
+	} else if (!psi_plist_pop(cpp->tokens.next, NULL)) {
+		return false;
 	}
 
-	return false;
+	if (!psi_plist_get(cpp->tokens.iter, cpp->index - 1, &cur)) {
+		return false;
+	}
+	psi_plist_unset(cpp->tokens.iter, cpp->index - 1);
+	if (free_token && cur) {
+		psi_token_free(&cur);
+	}
+	return true;
 }
+
 bool psi_cpp_tokiter_del_cur(struct psi_cpp *cpp, bool free_token)
 {
 	struct psi_token *cur = NULL;
 
 #if PSI_CPP_DEBUG
-	PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP del_cur -> index=%zu, iter.count=%zu, next.count=%zu	",
-			cpp->index, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next));
+	PSI_DEBUG_PRINT(cpp->parser,
+			"PSI: CPP del_cur -> index=%zu, iter.count=%zu, next.count=%zu, exec.count=%zu	",
+			cpp->index, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next),
+			psi_plist_count(cpp->tokens.exec));
 #endif
 
 	if (psi_plist_get(cpp->tokens.iter, cpp->index, &cur)) {
@@ -237,8 +256,10 @@ bool psi_cpp_tokiter_del_range(struct psi_cpp *cpp, size_t offset, size_t num_el
 	size_t i;
 
 #if PSI_CPP_DEBUG
-	PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP del_range -> index=%zu, offset=%zu, num_eles=%zu, iter.count=%zu, next.count=%zu\n",
-			cpp->index, offset, num_eles, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next));
+	PSI_DEBUG_PRINT(cpp->parser,
+			"PSI: CPP del_range -> index=%zu, offset=%zu, num_eles=%zu, iter.count=%zu, next.count=%zu, exec.count=%zu\n",
+			cpp->index, offset, num_eles, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next),
+			psi_plist_count(cpp->tokens.exec));
 #endif
 
 	for (i = offset; i < offset + num_eles; ++i) {
@@ -267,15 +288,13 @@ bool psi_cpp_tokiter_ins_range(struct psi_cpp *cpp, size_t num_eles, void **eles
 		return true;
 	}
 
-	tokens = psi_plist_ins_r(cpp->tokens.iter, cpp->index, num_eles, eles);
-	if (!tokens) {
-		return false;
-	}
-	cpp->tokens.iter = tokens;
+	cpp->tokens.iter = psi_plist_ins_r(cpp->tokens.iter, cpp->index, num_eles, eles);
 
 #if PSI_CPP_DEBUG
-	PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP ins_range -> index=%zu, num_eles=%zu, iter.count=%zu, next.count=%zu\n",
-			cpp->index, num_eles, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next));
+	PSI_DEBUG_PRINT(cpp->parser,
+			"PSI: CPP ins_range -> index=%zu, num_eles=%zu, iter.count=%zu, next.count=%zu, exec.count=%zu\n",
+			cpp->index, num_eles, psi_plist_count(cpp->tokens.iter), psi_plist_count(cpp->tokens.next),
+			psi_plist_count(cpp->tokens.exec));
 #endif
 
 	return true;
@@ -377,8 +396,11 @@ static inline size_t psi_cpp_tokiter_expand_tokens(struct psi_cpp *cpp,
 		stringify = false;
 	}
 
+	bool processed = psi_cpp_process(cpp, &exp, target);
+	assert(processed);
+
 	n = psi_plist_count(exp);
-	psi_cpp_tokiter_ins_range(cpp, n, psi_plist_eles(exp));
+	psi_cpp_tokiter_add_range(cpp, n, psi_plist_eles(exp));
 	free(exp);
 
 	return n;
@@ -497,7 +519,8 @@ static inline void psi_cpp_tokiter_expand_call_tokens(struct psi_cpp *cpp,
 			for (s = 0; psi_plist_get(macro->sig, s, &arg_name); ++s) {
 				if (zend_string_equals(arg_name->text, tok->text)) {
 					if (prescan) {
-						bool processed = psi_cpp_process(cpp, &arg_tokens_list[s]);
+						bool processed = psi_cpp_process(cpp,
+								&arg_tokens_list[s], target);
 
 						assert(processed);
 					}
@@ -555,7 +578,8 @@ static inline void psi_cpp_tokiter_expand_builtin_tokens(struct psi_cpp *cpp,
 
 	/* prescan */
 	for (s = 0; s < argc; ++s) {
-		bool processed = psi_cpp_process(cpp, &arg_tokens_list[s]);
+		bool processed = psi_cpp_process(cpp, &arg_tokens_list[s],
+				target);
 		assert(processed);
 	}
 
@@ -563,11 +587,11 @@ static inline void psi_cpp_tokiter_expand_builtin_tokens(struct psi_cpp *cpp,
 	if (!builtin->func(cpp, target, arg_tokens_list, &res)) {
 		struct psi_token *zero = psi_token_init(PSI_T_NUMBER, "0", 1,
 				target->col, target->line, target->file);
-		psi_cpp_tokiter_ins_range(cpp, 1, (void *) &zero);
+		psi_cpp_tokiter_add(cpp, zero);
 	} else if (!res) {
 		struct psi_token *one = psi_token_init(PSI_T_NUMBER, "1", 1,
 				target->col, target->line, target->file);
-		psi_cpp_tokiter_ins_range(cpp, 1, (void *) &one);
+		psi_cpp_tokiter_add(cpp, one);
 	} else {
 		psi_cpp_tokiter_expand_tokens(cpp, target, res);
 		psi_plist_free(res);
@@ -608,11 +632,20 @@ static inline bool psi_cpp_tokiter_expand_def(struct psi_cpp *cpp,
 {
 	/* delete current token from stream */
 	psi_cpp_tokiter_del_cur(cpp, false);
-	/* replace with tokens from macro */
-	psi_cpp_tokiter_expand_tokens(cpp, target, macro->tokens);
+
+	if (macro->tokens) {
+		struct psi_plist *tokens = psi_plist_copy(macro->tokens,
+					(void (*)(void *)) psi_token_copy_ctor);
+		bool processed = psi_cpp_process(cpp, &tokens, target);
+
+		if (processed) {
+			/* replace with tokens from macro */
+			psi_cpp_tokiter_expand_tokens(cpp, target, tokens);
+		}
+		++cpp->expanded;
+	}
 
 	psi_token_free(&target);
-	++cpp->expanded;
 	return true;
 }
 
@@ -637,9 +670,10 @@ static inline bool psi_cpp_tokiter_expand_decl(struct psi_cpp *cpp,
 	}
 
 	/* don't expand itself */
-	if (macro->token == target) {
-		return false;
-	}
+//	if (zend_string_equals(macro->token->text, target->text)) {
+//		PSI_DEBUG_PRINT(cpp->parser, "PSI: CPP expand ~ skipping self token\n");
+//		return false;
+//	}
 
 	if (macro->sig) {
 		return psi_cpp_tokiter_expand_call(cpp, target, macro);
diff --git a/src/error.c b/src/error.c
index c130e71..50ca092 100644
--- a/src/error.c
+++ b/src/error.c
@@ -75,11 +75,13 @@ void psi_error_wrapper(struct psi_data *context, struct psi_token *t, int type,
 
 	va_start(argv, msg);
 	psi_verror(type, fn, ln, msg, argv);
-#ifdef __clang__
 	va_end(argv);
+
 	va_start(argv, msg);
-#endif
+	psi_debug_lock(context);
 	PSI_DEBUG_PRINTV(context, msg, argv);
+	PSI_DEBUG_PRINT(context, "\n");
+	psi_debug_unlock(context);
 	va_end(argv);
 
 	if (context) {
diff --git a/src/parser.c b/src/parser.c
index 9781ed1..2a89068 100644
--- a/src/parser.c
+++ b/src/parser.c
@@ -131,7 +131,7 @@ struct psi_parser_input *psi_parser_open_string(struct psi_parser *P,
 struct psi_plist *psi_parser_preprocess(struct psi_parser *P,
 		struct psi_plist **tokens)
 {
-	if (psi_cpp_process(P->preproc, tokens)) {
+	if (psi_cpp_process(P->preproc, tokens, NULL)) {
 		return *tokens;
 	}
 	return NULL;
@@ -238,7 +238,7 @@ void psi_parser_postprocess(struct psi_parser *P)
 		if (!cnst) {
 			continue;
 		}
-//fprintf(stderr, "PARSE: %s", ZSTR_VAL(cnst));
+
 		I = psi_parser_open_string(P, ZSTR_VAL(cnst), ZSTR_LEN(cnst));
 		zend_string_release(cnst);
 
diff --git a/src/parser_proc.c b/src/parser_proc.c
index 6b8b817..206056a 100644
--- a/src/parser_proc.c
+++ b/src/parser_proc.c
@@ -9475,11 +9475,22 @@ static void psi_parser_proc_error(struct psi_parser *P, struct psi_plist *tokens
 		
 		P->error(PSI_DATA(P), T, PSI_WARNING, "PSI %s at col %u", msg, T->col);
 		while (i <= last || T->type != PSI_T_EOS) {
+			const char *pos;
+			
 			if (!psi_plist_get(tokens, i++, &T)) {
 				break;
 			}
+			
+			if (i < last + 1) {
+				pos = "preceding";
+			} else if (i > last + 1) {
+				pos = "following";
+			} else {
+				pos = "offending";
+			}
+			
 			P->error(PSI_DATA(P), T, PSI_WARNING, "PSI %s token '%s' at col %u", 
-					i<last+1?"preceding":i>last+1?"following":"offending", T->text->val, T->col);
+					pos, T ? T->text->val : "<deleted>", T ? T->col : 0);
 		}
 	} else {
 		P->error(PSI_DATA(P), NULL, PSI_WARNING, "PSI %s", msg);
diff --git a/src/parser_proc_grammar.y b/src/parser_proc_grammar.y
index b23683c..7a3cf55 100644
--- a/src/parser_proc_grammar.y
+++ b/src/parser_proc_grammar.y
@@ -2329,11 +2329,22 @@ static void psi_parser_proc_error(struct psi_parser *P, struct psi_plist *tokens
 		
 		P->error(PSI_DATA(P), T, PSI_WARNING, "PSI %s at col %u", msg, T->col);
 		while (i <= last || T->type != PSI_T_EOS) {
+			const char *pos;
+			
 			if (!psi_plist_get(tokens, i++, &T)) {
 				break;
 			}
+			
+			if (i < last + 1) {
+				pos = "preceding";
+			} else if (i > last + 1) {
+				pos = "following";
+			} else {
+				pos = "offending";
+			}
+			
 			P->error(PSI_DATA(P), T, PSI_WARNING, "PSI %s token '%s' at col %u", 
-					i<last+1?"preceding":i>last+1?"following":"offending", T->text->val, T->col);
+					pos, T ? T->text->val : "<deleted>", T ? T->col : 0);
 		}
 	} else {
 		P->error(PSI_DATA(P), NULL, PSI_WARNING, "PSI %s", msg);
diff --git a/src/parser_scan.c b/src/parser_scan.c
index ebf47ea..3f8e51c 100644
--- a/src/parser_scan.c
+++ b/src/parser_scan.c
@@ -1,4 +1,4 @@
-/* Generated by re2c 1.1.1 on Tue Dec  4 14:09:03 2018 */
+/* Generated by re2c 1.1.1 on Tue Dec  4 17:43:28 2018 */
 #line 1 "src/parser_scan.re"
 /*******************************************************************************
  Copyright (c) 2016, Michael Wallner <mike@php.net>.