From 6ebcb60416cad353f5aeacebe919cb6990ef5a7d Mon Sep 17 00:00:00 2001
From: Michael Wallner <mike@php.net>
Date: Tue, 7 Mar 2017 12:43:37 +0100
Subject: [PATCH] cpp: token stringification and pasting

---
 src/cpp.h                  |  2 +-
 src/cpp_tokiter.c          | 60 ++++++++++++++++++++++++++++----------
 src/parser_proc.c          |  6 ++--
 src/parser_proc.y          | 16 +++++-----
 src/parser_proc_def.h      | 16 +++++-----
 src/plist.c                | 15 ++++++++--
 src/plist.h                |  5 ++--
 src/token.c                | 44 +++++++++++++++++++++-------
 src/token.h                |  5 ++--
 src/types/cpp_macro_decl.c | 12 ++++++++
 10 files changed, 129 insertions(+), 52 deletions(-)

diff --git a/src/cpp.h b/src/cpp.h
index 1a8fc2c..81f27b3 100644
--- a/src/cpp.h
+++ b/src/cpp.h
@@ -29,7 +29,7 @@
 #include "data.h"
 
 #ifndef PSI_CPP_DEBUG
-# define PSI_CPP_DEBUG 0
+# define PSI_CPP_DEBUG 1
 #endif
 
 struct psi_cpp_data {
diff --git a/src/cpp_tokiter.c b/src/cpp_tokiter.c
index e1d45f3..5753af7 100644
--- a/src/cpp_tokiter.c
+++ b/src/cpp_tokiter.c
@@ -138,7 +138,7 @@ bool psi_cpp_tokiter_del_range(struct psi_cpp_data *cpp, size_t offset, size_t n
 	fprintf(stderr, "PSI: CPP del_range -> index=%zu, offset=%zu, num_eles=%zu\n", cpp->index, offset, num_eles);
 #endif
 
-	deleted = psi_plist_del_range(cpp->tokens, offset, num_eles, (void *) ptr);
+	deleted = psi_plist_del_r(cpp->tokens, offset, num_eles, (void *) ptr);
 
 	if (deleted) {
 		if (cpp->index >= psi_plist_count(cpp->tokens)) {
@@ -175,7 +175,7 @@ bool psi_cpp_tokiter_ins_cur(struct psi_cpp_data *cpp, struct psi_token *tok)
 bool psi_cpp_tokiter_ins_range(struct psi_cpp_data *cpp, size_t offset,
 		size_t num_eles, void **eles)
 {
-	struct psi_plist *tokens = psi_plist_ins_range(cpp->tokens, offset,
+	struct psi_plist *tokens = psi_plist_ins_r(cpp->tokens, offset,
 			num_eles, eles);
 
 #if PSI_CPP_DEBUG
@@ -203,18 +203,46 @@ bool psi_cpp_tokiter_defined(struct psi_cpp_data *cpp)
 void psi_cpp_tokiter_expand_tokens(struct psi_cpp_data *cpp, struct psi_plist *tokens)
 {
 	if (tokens && psi_plist_count(tokens)) {
-		size_t i;
-		struct psi_token **exp_tokens = calloc(psi_plist_count(tokens), sizeof(*exp_tokens));
+		size_t i = 0, n = 0;
+		bool stringify = false, paste = false;
+		struct psi_token *tok, **exp_tokens = calloc(psi_plist_count(tokens), sizeof(*exp_tokens));
 
-		for (i = 0; psi_plist_get(tokens, i, &exp_tokens[i]); ++i) {
-			exp_tokens[i] = psi_token_copy(exp_tokens[i]);
+		while (psi_plist_get(tokens, i++, &tok)) {
+			struct psi_token *new_tok;
+
+			if (tok->type == PSI_T_HASH) {
+				if (stringify) {
+					stringify = false;
+					paste = true;
+				} else {
+					stringify = true;
+				}
+				continue;
+			}
+
+			if (paste && n > 0 && exp_tokens[n - 1] &&
+					(new_tok = psi_token_cat(NULL, 2, exp_tokens[n - 1], tok))) {
+				free(exp_tokens[n - 1]);
+				exp_tokens[n - 1] = new_tok;
+			} else {
+				struct psi_token *cpy = psi_token_copy(tok);
+
+				if (stringify) {
+					cpy = psi_token_append(NULL,
+							psi_token_prepend(NULL, cpy, 1, "\""), 1, "\"");
+					cpy->type = PSI_T_QUOTED_STRING;
+				}
+				exp_tokens[n++] = cpy;
+			}
 
 #if PSI_CPP_DEBUG
 			fprintf(stderr, "PSI: CPP expand > ");
-			psi_token_dump(2, exp_tokens[i]);
+			psi_token_dump(2, tok);
 #endif
+			paste = false;
+			stringify = false;
 		}
-		psi_cpp_tokiter_ins_range(cpp, psi_cpp_tokiter_index(cpp), i, (void *) exp_tokens);
+		psi_cpp_tokiter_ins_range(cpp, psi_cpp_tokiter_index(cpp), n, (void *) exp_tokens);
 		free(exp_tokens);
 	}
 }
@@ -316,6 +344,7 @@ static void psi_cpp_tokiter_expand_call_tokens(struct psi_cpp_data *cpp,
 {
 	size_t i;
 	struct psi_token *tok;
+	struct psi_plist *tokens = psi_plist_init(NULL);
 
 	for (i = 0; psi_plist_get(macro->tokens, i, &tok); ++i) {
 		struct psi_plist *arg_tokens = NULL;
@@ -335,17 +364,16 @@ static void psi_cpp_tokiter_expand_call_tokens(struct psi_cpp_data *cpp,
 		}
 
 		if (arg_tokens) {
-			size_t tok_count = psi_plist_count(arg_tokens);
-
-			if (tok_count) {
-				psi_cpp_tokiter_expand_tokens(cpp, arg_tokens);
-				psi_cpp_tokiter_seek(cpp, psi_cpp_tokiter_index(cpp) + tok_count);
-			}
+			tokens = psi_plist_add_r(tokens, psi_plist_count(arg_tokens), psi_plist_eles(arg_tokens));
 		} else {
-			psi_cpp_tokiter_ins_cur(cpp, psi_token_copy(tok));
-			psi_cpp_tokiter_next(cpp);
+			tokens = psi_plist_add(tokens, &tok);
 		}
 	}
+
+	psi_cpp_tokiter_expand_tokens(cpp, tokens);
+	psi_cpp_tokiter_seek(cpp, psi_cpp_tokiter_index(cpp) + psi_plist_count(tokens));
+	psi_plist_free(tokens);
+
 }
 
 static bool psi_cpp_tokiter_expand_call(struct psi_cpp_data *cpp,
diff --git a/src/parser_proc.c b/src/parser_proc.c
index a3b6119..81d3ca2 100644
--- a/src/parser_proc.c
+++ b/src/parser_proc.c
@@ -2438,7 +2438,7 @@ static void yy_reduce(
  } else {
   char digest[17];
   psi_token_hash(yymsp[-1].minor.yy0, digest);
-  yylhsminor.yy0 = psi_token_translit(psi_token_append(yymsp[-1].minor.yy0, 1, digest), " ", "@");
+  yylhsminor.yy0 = psi_token_append("@", yymsp[-1].minor.yy0, 1, digest);
  }
 }
 #line 2445 "src/parser_proc.c"
@@ -2956,7 +2956,7 @@ static void yy_reduce(
 #line 712 "src/parser_proc.y"
 {
  if (yymsp[0].minor.yy0) {
-  yylhsminor.yy0 = psi_token_cat(2, yymsp[-1].minor.yy0, yymsp[0].minor.yy0);
+  yylhsminor.yy0 = psi_token_cat(" ", 2, yymsp[-1].minor.yy0, yymsp[0].minor.yy0);
   free(yymsp[-1].minor.yy0);
   free(yymsp[0].minor.yy0);
  } else {
@@ -2970,7 +2970,7 @@ static void yy_reduce(
       case 105: /* decl_type ::= SIGNED decl_scalar_type */ yytestcase(yyruleno==105);
 #line 760 "src/parser_proc.y"
 {
- struct psi_token *T = psi_token_cat(2, yymsp[-1].minor.yy0, yymsp[0].minor.yy0);
+ struct psi_token *T = psi_token_cat(" ", 2, yymsp[-1].minor.yy0, yymsp[0].minor.yy0);
  yylhsminor.yy172 = psi_decl_type_init(T->type, T->text);
  yylhsminor.yy172->token = T;
  free(yymsp[-1].minor.yy0);
diff --git a/src/parser_proc.y b/src/parser_proc.y
index 4dab6c9..bac0d38 100644
--- a/src/parser_proc.y
+++ b/src/parser_proc.y
@@ -416,7 +416,7 @@ enum_name(n) ::= ENUM(E) optional_name(N). {
  } else {
   char digest[17];
   psi_token_hash(E, digest);
-  n = psi_token_translit(psi_token_append(E, 1, digest), " ", "@");
+  n = psi_token_append("@", E, 1, digest);
  }
 }
 struct_name(n) ::= STRUCT(S) optional_name(N). {
@@ -426,7 +426,7 @@ struct_name(n) ::= STRUCT(S) optional_name(N). {
  } else {
   char digest[17];
   psi_token_hash(S, digest);
-  n = psi_token_translit(psi_token_append(S, 1, digest), " ", "@");
+  n = psi_token_append("@", S, 1, digest);
  }
 }
 union_name(n) ::= UNION(U) optional_name(N). {
@@ -436,7 +436,7 @@ union_name(n) ::= UNION(U) optional_name(N). {
  } else {
   char digest[17];
   psi_token_hash(U, digest);
-  n = psi_token_translit(psi_token_append(U, 1, digest), " ", "@");
+  n = psi_token_append("@", U, 1, digest);
  }
 }
 decl_enum(e) ::= enum_name(N) LBRACE decl_enum_items(list) RBRACE. {
@@ -711,7 +711,7 @@ decl_scalar_type(type_) ::= CHAR(C). {
 }
 decl_scalar_type(type_) ::= SHORT(S) decl_scalar_type_short(s). {
  if (s) {
-  type_ = psi_token_cat(2, S, s);
+  type_ = psi_token_cat(" ", 2, S, s);
   free(S);
   free(s);
  } else {
@@ -729,7 +729,7 @@ decl_scalar_type(type_) ::= INT(I). {
 }
 decl_scalar_type(type_) ::= LONG(L) decl_scalar_type_long(l). {
  if (l) {
-  type_ = psi_token_cat(2, L, l);
+  type_ = psi_token_cat(" ", 2, L, l);
   free(L);
   free(l);
  } else {
@@ -744,7 +744,7 @@ decl_scalar_type_long(l) ::= DOUBLE(D). {
 }
 decl_scalar_type_long(l) ::= LONG(L) decl_scalar_type_long_long(ll). {
  if (ll) {
-  l = psi_token_cat(2, L, ll);
+  l = psi_token_cat(" ", 2, L, ll);
   free(L);
   free(ll);
  } else {
@@ -758,14 +758,14 @@ decl_scalar_type_long_long(ll) ::= INT(I). {
  ll = I;
 }
 decl_type(type_) ::= UNSIGNED(U) decl_scalar_type(N). {
- struct psi_token *T = psi_token_cat(2, U, N);
+ struct psi_token *T = psi_token_cat(" ", 2, U, N);
  type_ = psi_decl_type_init(T->type, T->text);
  type_->token = T;
  free(U);
  free(N);
 }
 decl_type(type_) ::= SIGNED(S) decl_scalar_type(N). {
- struct psi_token *T = psi_token_cat(2, S, N);
+ struct psi_token *T = psi_token_cat(" ", 2, S, N);
  type_ = psi_decl_type_init(T->type, T->text);
  type_->token = T;
  free(S);
diff --git a/src/parser_proc_def.h b/src/parser_proc_def.h
index bee1bc4..ed1455a 100644
--- a/src/parser_proc_def.h
+++ b/src/parser_proc_def.h
@@ -648,7 +648,7 @@ PARSE_NAMED(enum_name, n,
 		char digest[17];
 
 		psi_token_hash(E, digest);
-		n = psi_token_translit(psi_token_append(E, 1, digest), " ", "@");
+		n = psi_token_append("@", E, 1, digest);
 	}
 }
 
@@ -665,7 +665,7 @@ PARSE_NAMED(struct_name, n,
 		char digest[17];
 
 		psi_token_hash(S, digest);
-		n = psi_token_translit(psi_token_append(S, 1, digest), " ", "@");
+		n = psi_token_append("@", S, 1, digest);
 	}
 }
 
@@ -682,7 +682,7 @@ PARSE_NAMED(union_name, n,
 		char digest[17];
 
 		psi_token_hash(U, digest);
-		n = psi_token_translit(psi_token_append(U, 1, digest), " ", "@");
+		n = psi_token_append("@", U, 1, digest);
 	}
 }
 
@@ -1296,7 +1296,7 @@ PARSE_NAMED(decl_scalar_type, type_,
 		NAMED(SHORT, S)
 		NAMED(decl_scalar_type_short, s)) {
 	if (s) {
-		type_ = psi_token_cat(2, S, s);
+		type_ = psi_token_cat(" ", 2, S, s);
 		free(S);
 		free(s);
 	} else {
@@ -1334,7 +1334,7 @@ PARSE_NAMED(decl_scalar_type, type_,
 		NAMED(LONG, L)
 		NAMED(decl_scalar_type_long, l)) {
 	if (l) {
-		type_ = psi_token_cat(2, L, l);
+		type_ = psi_token_cat(" ", 2, L, l);
 		free(L);
 		free(l);
 	} else {
@@ -1364,7 +1364,7 @@ PARSE_NAMED(decl_scalar_type_long, l,
 		NAMED(LONG, L)
 		NAMED(decl_scalar_type_long_long, ll)) {
 	if (ll) {
-		l = psi_token_cat(2, L, ll);
+		l = psi_token_cat(" ", 2, L, ll);
 		free(L);
 		free(ll);
 	} else {
@@ -1393,7 +1393,7 @@ PARSE_NAMED(decl_scalar_type_long_long, ll,
 PARSE_TYPED(decl_type, type_,
 		NAMED(UNSIGNED, U)
 		NAMED(decl_scalar_type, N)) {
-	struct psi_token *T = psi_token_cat(2, U, N);
+	struct psi_token *T = psi_token_cat(" ", 2, U, N);
 	type_ = psi_decl_type_init(T->type, T->text);
 	type_->token = T;
 	free(U);
@@ -1406,7 +1406,7 @@ PARSE_TYPED(decl_type, type_,
 PARSE_TYPED(decl_type, type_,
 		NAMED(SIGNED, S)
 		NAMED(decl_scalar_type, N)) {
-	struct psi_token *T = psi_token_cat(2, S, N);
+	struct psi_token *T = psi_token_cat(" ", 2, S, N);
 	type_ = psi_decl_type_init(T->type, T->text);
 	type_->token = T;
 	free(S);
diff --git a/src/plist.c b/src/plist.c
index 7b298be..b5acd76 100644
--- a/src/plist.c
+++ b/src/plist.c
@@ -111,6 +111,17 @@ struct psi_plist *psi_plist_add(struct psi_plist *list, void *ptr) {
 	return list;
 }
 
+struct psi_plist *psi_plist_add_r(struct psi_plist *list, size_t num_eles, void **eles) {
+	if (list && list->count) {
+		list = realloc(list, sizeof(*list) + list->size + (num_eles + list->count) * list->size);
+	}
+	if (list) {
+		memcpy(PLIST_ELE(list, list->count), eles, num_eles * list->size);
+		list->count += num_eles;
+	}
+	return list;
+}
+
 bool psi_plist_get(struct psi_plist *list, size_t index, void *ptr) {
 	if (list && list->count > index) {
 		PLIST_CPY(list, ptr, PLIST_ELE(list, index));
@@ -132,7 +143,7 @@ bool psi_plist_del(struct psi_plist *list, size_t index, void *ptr) {
 	return false;
 }
 
-bool psi_plist_del_range(struct psi_plist *list, size_t offset_start, size_t num_eles, void **eles) {
+bool psi_plist_del_r(struct psi_plist *list, size_t offset_start, size_t num_eles, void **eles) {
 	if (list) {
 		size_t offset_end = offset_start + num_eles - 1;
 
@@ -169,7 +180,7 @@ struct psi_plist *psi_plist_ins(struct psi_plist *list, size_t index, void *ptr)
 	return list;
 }
 
-struct psi_plist *psi_plist_ins_range(struct psi_plist *list, size_t offset_start, size_t num_eles, void **eles) {
+struct psi_plist *psi_plist_ins_r(struct psi_plist *list, size_t offset_start, size_t num_eles, void **eles) {
 	size_t new_count = MAX(offset_start, list->count) + num_eles;
 
 	if (list && new_count) {
diff --git a/src/plist.h b/src/plist.h
index bca0d85..4bb76b5 100644
--- a/src/plist.h
+++ b/src/plist.h
@@ -39,11 +39,12 @@ size_t psi_plist_count(struct psi_plist *list);
 void **psi_plist_eles(struct psi_plist *list);
 
 struct psi_plist *psi_plist_add(struct psi_plist *list, void *ptr);
+struct psi_plist *psi_plist_add_r(struct psi_plist *list, size_t num_eles, void **eles);
 bool psi_plist_get(struct psi_plist *list, size_t index, void *ptr);
 bool psi_plist_del(struct psi_plist *list, size_t index, void *ptr);
-bool psi_plist_del_range(struct psi_plist *list, size_t offset_start, size_t offset_end, void **eles);
+bool psi_plist_del_r(struct psi_plist *list, size_t offset_start, size_t offset_end, void **eles);
 struct psi_plist *psi_plist_ins(struct psi_plist *list, size_t index, void *ptr);
-struct psi_plist *psi_plist_ins_range(struct psi_plist *list, size_t offset_start, size_t num_eles, void **eles);
+struct psi_plist *psi_plist_ins_r(struct psi_plist *list, size_t offset_start, size_t num_eles, void **eles);
 
 bool psi_plist_shift(struct psi_plist *list, void *ptr);
 bool psi_plist_pop(struct psi_plist *list, void *ptr);
diff --git a/src/token.c b/src/token.c
index ec8c4e9..8cfc0d5 100644
--- a/src/token.c
+++ b/src/token.c
@@ -86,9 +86,10 @@ void psi_token_copy_ctor(struct psi_token **tok) {
 	*tok = psi_token_copy(*tok);
 }
 
-struct psi_token *psi_token_cat(unsigned argc, ...) {
+struct psi_token *psi_token_cat(const char *sep, unsigned argc, ...) {
 	va_list argv;
 	unsigned i;
+	size_t sep_len = sep ? strlen(sep) : 0;
 	struct psi_token *T = NULL;
 
 	va_start(argv, argc);
@@ -97,7 +98,7 @@ struct psi_token *psi_token_cat(unsigned argc, ...) {
 
 		if (T) {
 			size_t token_len = T->size, fname_len = strlen(T->file);
-			struct psi_token *tmp = realloc(T, psi_token_alloc_size(T->size += arg->size + 1, fname_len));
+			struct psi_token *tmp = realloc(T, psi_token_alloc_size(T->size += arg->size + sep_len, fname_len));
 
 			if (tmp) {
 				T = tmp;
@@ -109,9 +110,9 @@ struct psi_token *psi_token_cat(unsigned argc, ...) {
 
 			T->text = &T->buf[0];
 			T->file = &T->buf[T->size + 1];
-			T->buf[token_len] = ' ';
-			memmove(&T->buf[T->size + 1], &T->buf[token_len + 1], fname_len + 1);
-			memcpy(&T->buf[token_len + 1], arg->text, arg->size + 1);
+			memmove(&T->buf[T->size + 1], &T->buf[token_len + sep_len], fname_len + 1);
+			memcpy(&T->buf[token_len], sep, sep_len);
+			memcpy(&T->buf[token_len + sep_len], arg->text, arg->size + 1);
 		} else {
 			T = psi_token_copy(arg);
 			T->type = PSI_T_NAME;
@@ -122,21 +123,44 @@ struct psi_token *psi_token_cat(unsigned argc, ...) {
 	return T;
 }
 
-struct psi_token *psi_token_append(struct psi_token *T, unsigned argc, ...) {
+struct psi_token *psi_token_prepend(const char *sep, struct psi_token *T, unsigned argc, ...) {
 	va_list argv;
 	unsigned i;
+	size_t sep_len = sep ? strlen(sep) : 0;
 
 	va_start(argv, argc);
 	for (i = 0; i < argc; ++i) {
 		char *str = va_arg(argv, char *);
 		size_t str_len = strlen(str), token_len = T->size, fname_len = strlen(T->file);
 
-		T = realloc(T, psi_token_alloc_size(T->size += str_len + 1, fname_len));
+		T = realloc(T, psi_token_alloc_size(T->size += str_len + sep_len, fname_len));
 		T->text = &T->buf[0];
 		T->file = &T->buf[T->size + 1];
-		T->buf[token_len] = ' ';
-		memmove(&T->buf[T->size + 1], &T->buf[token_len + 1], fname_len + 1);
-		memcpy(&T->buf[token_len + 1], str, str_len + 1);
+		memmove(&T->buf[str_len + sep_len], &T->buf[0], T->size + 1 + fname_len + 1);
+		memcpy(&T->buf[0], str, str_len);
+		memcpy(&T->buf[str_len], sep, sep_len);
+		T->buf[T->size] = '\0';
+	}
+	va_end(argv);
+
+	return T;
+}
+struct psi_token *psi_token_append(const char *sep, struct psi_token *T, unsigned argc, ...) {
+	va_list argv;
+	unsigned i;
+	size_t sep_len = sep ? strlen(sep) : 0;
+
+	va_start(argv, argc);
+	for (i = 0; i < argc; ++i) {
+		char *str = va_arg(argv, char *);
+		size_t str_len = strlen(str), token_len = T->size, fname_len = strlen(T->file);
+
+		T = realloc(T, psi_token_alloc_size(T->size += str_len + sep_len, fname_len));
+		T->text = &T->buf[0];
+		T->file = &T->buf[T->size + 1];
+		memmove(&T->buf[T->size + 1], &T->buf[token_len + sep_len], fname_len + 1);
+		memcpy(&T->buf[token_len], sep, sep_len);
+		memcpy(&T->buf[token_len + sep_len], str, str_len + 1);
 	}
 	va_end(argv);
 
diff --git a/src/token.h b/src/token.h
index 93625da..322843b 100644
--- a/src/token.h
+++ b/src/token.h
@@ -140,8 +140,9 @@ struct psi_token *psi_token_alloc(struct psi_parser *P);
 size_t psi_token_alloc_size(size_t token_len, size_t fname_len);
 struct psi_token *psi_token_copy(struct psi_token *src);
 void psi_token_copy_ctor(struct psi_token **src);
-struct psi_token *psi_token_cat(unsigned argc, ...);
-struct psi_token *psi_token_append(struct psi_token *T, unsigned argc, ...);
+struct psi_token *psi_token_cat(const char *sep, unsigned argc, ...);
+struct psi_token *psi_token_prepend(const char *sep, struct psi_token *T, unsigned argc, ...);
+struct psi_token *psi_token_append(const char *sep, struct psi_token *T, unsigned argc, ...);
 struct psi_token *psi_token_translit(struct psi_token *T, char *from, char *to);
 uint64_t psi_token_hash(struct psi_token *t, char *digest_buf);
 void psi_token_dump(int fd, struct psi_token *t);
diff --git a/src/types/cpp_macro_decl.c b/src/types/cpp_macro_decl.c
index bfa230c..d5e195c 100644
--- a/src/types/cpp_macro_decl.c
+++ b/src/types/cpp_macro_decl.c
@@ -74,4 +74,16 @@ void psi_cpp_macro_decl_dump(int fd, struct psi_cpp_macro_decl *macro)
 		}
 		dprintf(fd, ")");
 	}
+
+	if (macro->exp) {
+		dprintf(fd, " ");
+		psi_num_exp_dump(fd, macro->exp);
+	} else if (macro->tokens) {
+		size_t i = 0;
+		struct psi_token *tok;
+
+		while (psi_plist_get(macro->tokens, i++, &tok)) {
+			dprintf(fd, " %s", tok->text);
+		}
+	}
 }
-- 
2.30.2