hush: heredoc support, based on patch by Mike Frysinger (vapier AT gentoo.org)

 some TODOs are to be attacked later

function                                             old     new   delta
parse_stream                                        1461    1866    +405
parse_redirect                                         -     265    +265
setup_heredoc                                          -     116    +116
setup_redirects                                      151     196     +45
builtin_exit                                          49      47      -2
redirect_opt_num                                      61      55      -6
setup_redirect                                       212       -    -212
------------------------------------------------------------------------------
(add/remove: 2/1 grow/shrink: 2/2 up/down: 831/-220)          Total: 611 bytes

4 files changed, 319 insertions, 115 deletions

diff --git a/shell/hush.c b/shell/hush.c
index 14ca52ec0..792886944 100644
--- a/shell/hush.c
+++ b/shell/hush.c
@@ -21,8 +21,8 @@
  *      rewrites.
  *
  * Other credits:
- *      o_addchr() derived from similar w_addchar function in glibc-2.2.
- *      setup_redirect(), redirect_opt_num(), and big chunks of main()
+ *      o_addchr derived from similar w_addchar function in glibc-2.2.
+ *      setup_redirect, redirect_opt_num, and big chunks of main
  *      and many builtins derived from contributions by Erik Andersen.
  *      Miscellaneous bugfixes from Matt Kraai.
  *
@@ -249,11 +249,13 @@ static const char hush_version_str[] ALIGN1 = "HUSH_VERSION="BB_VER;
 #define SPECIAL_VAR_SYMBOL 3
 
 typedef enum redir_type {
+	REDIRECT_INVALID   = 0,
 	REDIRECT_INPUT     = 1,
 	REDIRECT_OVERWRITE = 2,
 	REDIRECT_APPEND    = 3,
-	REDIRECT_HEREIS    = 4,
-	REDIRECT_IO        = 5
+	REDIRECT_HEREDOC   = 4,
+	REDIRECT_IO        = 5,
+	REDIRECT_HEREDOC2  = 6, /* REDIRECT_HEREDOC after heredoc load */
 } redir_type;
 
 /* The descrip member of this structure is only used to make
@@ -263,12 +265,14 @@ static const struct {
 	signed char default_fd;
 	char descrip[3];
 } redir_table[] = {
-	{ 0,                         0, "()" },
+	{ 0,                         0, "??" },
 	{ O_RDONLY,                  0, "<"  },
 	{ O_CREAT|O_TRUNC|O_WRONLY,  1, ">"  },
 	{ O_CREAT|O_APPEND|O_WRONLY, 1, ">>" },
-	{ O_RDONLY,                 -1, "<<" },
-	{ O_RDWR,                    1, "<>" }
+	{ O_RDONLY,                  0, "<<" },
+	{ O_CREAT|O_RDWR,            1, "<>" },
+/* Should not be needed. Bogus default_fd helps in debugging */
+/*	{ O_RDONLY,                 77, "<<" }, */
 };
 
 typedef enum reserved_style {
@@ -2030,20 +2034,56 @@ static char **expand_assignments(char **argv, int count)
 }
 
 
+//TODO: fix big string case!
+static void setup_heredoc(int fd, const char *heredoc)
+{
+	struct fd_pair pair;
+	pid_t pid;
+
+	xpiped_pair(pair);
+	pid = vfork();
+	if (pid < 0)
+		bb_perror_msg_and_die("vfork");
+	if (pid == 0) { /* child */
+		die_sleep = 0;
+		close(pair.rd);
+		xwrite_str(pair.wr, heredoc);
+		_exit(0);
+	}
+	/* parent */
+	die_sleep = -1;
+	close(pair.wr);
+	xmove_fd(pair.rd, fd);
+}
+
 /* squirrel != NULL means we squirrel away copies of stdin, stdout,
  * and stderr if they are redirected. */
 static int setup_redirects(struct command *prog, int squirrel[])
 {
+//TODO: no callers ever check return value - ?!
 	int openfd, mode;
 	struct redir_struct *redir;
 
 	for (redir = prog->redirects; redir; redir = redir->next) {
-		if (redir->dup == -1 && redir->rd_filename == NULL) {
-			/* something went wrong in the parse.  Pretend it didn't happen */
+		if (redir->rd_type == REDIRECT_HEREDOC2) {
+			if (squirrel && redir->fd < 3) {
+				squirrel[redir->fd] = dup(redir->fd);
+			}
+			/* for REDIRECT_HEREDOC2, rd_filename holds _contents_
+			 * of the heredoc */
+			debug_printf_parse("set heredoc '%s'\n",
+					redir->rd_filename);
+			setup_heredoc(redir->fd, redir->rd_filename);
 			continue;
 		}
+
 		if (redir->dup == -1) {
 			char *p;
+			if (redir->rd_filename == NULL) {
+				/* Something went wrong in the parse.
+				 * Pretend it didn't happen */
+				continue;
+			}
 			mode = redir_table[redir->rd_type].mode;
 //TODO: check redir for names like '\\'
 			p = expand_string_to_string(redir->rd_filename);
@@ -2051,7 +2091,8 @@ static int setup_redirects(struct command *prog, int squirrel[])
 			free(p);
 			if (openfd < 0) {
 			/* this could get lost if stderr has been redirected, but
-			   bash and ash both lose it as well (though zsh doesn't!) */
+			 * bash and ash both lose it as well (though zsh doesn't!) */
+//what the above comment tries to say?
 				return 1;
 			}
 		} else {
@@ -3407,83 +3448,6 @@ static int run_and_free_list(struct pipe *pi)
 }
 
 
-/* Peek ahead in the in_str to find out if we have a "&n" construct,
- * as in "2>&1", that represents duplicating a file descriptor.
- * Return either -2 (syntax error), -1 (no &), or the number found.
- */
-static int redirect_dup_num(struct in_str *input)
-{
-	int ch, d = 0, ok = 0;
-	ch = i_peek(input);
-	if (ch != '&') return -1;
-
-	i_getch(input);  /* get the & */
-	ch = i_peek(input);
-	if (ch == '-') {
-		i_getch(input);
-		return -3;  /* "-" represents "close me" */
-	}
-	while (isdigit(ch)) {
-		d = d*10 + (ch-'0');
-		ok = 1;
-		i_getch(input);
-		ch = i_peek(input);
-	}
-	if (ok) return d;
-
-	bb_error_msg("ambiguous redirect");
-	return -2;
-}
-
-/* The src parameter allows us to peek forward to a possible &n syntax
- * for file descriptor duplication, e.g., "2>&1".
- * Return code is 0 normally, 1 if a syntax error is detected in src.
- * Resource errors (in xmalloc) cause the process to exit */
-static int setup_redirect(struct parse_context *ctx,
-		int fd,
-		redir_type style,
-		struct in_str *input)
-{
-	struct command *command = ctx->command;
-	struct redir_struct *redir;
-	struct redir_struct **redirp;
-	int dup_num;
-
-	/* Check for a '2>&1' type redirect */
-	dup_num = redirect_dup_num(input);
-	if (dup_num == -2)
-		return 1;  /* syntax error */
-
-	/* Create a new redir_struct and drop it onto the end of the linked list */
-	redirp = &command->redirects;
-	while ((redir = *redirp) != NULL) {
-		redirp = &(redir->next);
-	}
-	*redirp = redir = xzalloc(sizeof(*redir));
-	/* redir->next = NULL; */
-	/* redir->rd_filename = NULL; */
-	redir->rd_type = style;
-	redir->fd = (fd == -1) ? redir_table[style].default_fd : fd;
-
-	debug_printf("Redirect type %d%s\n", redir->fd, redir_table[style].descrip);
-
-	redir->dup = dup_num;
-	if (dup_num != -1) {
-		/* Erik had a check here that the file descriptor in question
-		 * is legit; I postpone that to "run time"
-		 * A "-" representation of "close me" shows up as a -3 here */
-		debug_printf("Duplicating redirect '%d>&%d'\n", redir->fd, redir->dup);
-	} else {
-		/* We do _not_ try to open the file that src points to,
-		 * since we need to return and let src be expanded first.
-		 * Set ctx->pending_redirect, so we know what to do at the
-		 * end of the next parsed word. */
-		ctx->pending_redirect = redir;
-	}
-	return 0;
-}
-
-
 static struct pipe *new_pipe(void)
 {
 	struct pipe *pi;
@@ -3602,7 +3566,6 @@ static void initialize_context(struct parse_context *ctx)
 	done_command(ctx);
 }
 
-
 /* If a reserved word is found and processed, parse context is modified
  * and 1 is returned.
  */
@@ -3766,14 +3729,26 @@ static int done_word(o_string *word, struct parse_context *ctx)
 	if (ctx->pending_redirect) {
 		/* We do not glob in e.g. >*.tmp case. bash seems to glob here
 		 * only if run as "bash", not "sh" */
+		/* http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html
+		 * "2.7 Redirection
+		 * ...the word that follows the redirection operator
+		 * shall be subjected to tilde expansion, parameter expansion,
+		 * command substitution, arithmetic expansion, and quote
+		 * removal. Pathname expansion shall not be performed
+		 * on the word by a non-interactive shell; an interactive
+		 * shell may perform it, but shall do so only when
+		 * the expansion would result in one word."
+		 */
 		ctx->pending_redirect->rd_filename = xstrdup(word->data);
 		word->o_assignment = NOT_ASSIGNMENT;
-		debug_printf("word stored in rd_filename: '%s'\n", word->data);
+		debug_printf_parse("word stored in rd_filename: '%s'\n", word->data);
 	} else {
 		/* "{ echo foo; } echo bar" - bad */
 		/* NB: bash allows e.g.:
 		 * if true; then { echo foo; } fi
 		 * while if false; then false; fi do break; done
+		 * and disallows:
+		 * while if false; then false; fi; do; break; done
 		 * TODO? */
 		if (command->group) {
 			syntax(word->data);
@@ -3855,6 +3830,117 @@ static int done_word(o_string *word, struct parse_context *ctx)
 	return 0;
 }
 
+
+/* Peek ahead in the input to find out if we have a "&n" construct,
+ * as in "2>&1", that represents duplicating a file descriptor.
+ * Return: -3 if >&- "close fd" construct is seen,
+ * -2 (syntax error), -1 if no & was seen, or the number found.
+ */
+#if BB_MMU
+#define redirect_dup_num(as_string, input) \
+	redirect_dup_num(input)
+#endif
+static int redirect_dup_num(o_string *as_string, struct in_str *input)
+{
+	int ch, d, ok;
+
+	ch = i_peek(input);
+	if (ch != '&')
+		return -1;
+
+	ch = i_getch(input);  /* get the & */
+#if !BB_MMU
+	o_addchr(as_string, ch);
+#endif
+	ch = i_peek(input);
+	if (ch == '-') {
+		ch = i_getch(input);
+#if !BB_MMU
+		o_addchr(as_string, ch);
+#endif
+		return -3;  /* "-" represents "close me" */
+	}
+	d = 0;
+	ok = 0;
+	while (ch != EOF && isdigit(ch)) {
+		d = d*10 + (ch-'0');
+		ok = 1;
+		ch = i_getch(input);
+#if !BB_MMU
+		o_addchr(as_string, ch);
+#endif
+		ch = i_peek(input);
+	}
+	if (ok) return d;
+
+//TODO: this is the place to catch ">&file" bashism (redirect both fd 1 and 2)
+
+	bb_error_msg("ambiguous redirect");
+	return -2;
+}
+
+/* Return code is 0 normally, 1 if a syntax error is detected
+ */
+static int parse_redirect(struct parse_context *ctx,
+		int fd,
+		redir_type style,
+		struct in_str *input)
+{
+	struct command *command = ctx->command;
+	struct redir_struct *redir;
+	struct redir_struct **redirp;
+	int dup_num;
+
+	dup_num = -1;
+	if (style != REDIRECT_HEREDOC) {
+		/* Check for a '2>&1' type redirect */
+		dup_num = redirect_dup_num(&ctx->as_string, input);
+		if (dup_num == -2)
+			return 1;  /* syntax error */
+	}
+//TODO: else { check for <<-word }
+
+	if (style == REDIRECT_OVERWRITE && dup_num == -1) {
+		int ch = i_peek(input);
+		if (ch == '|') {
+			/* >|FILE redirect ("clobbering" >).
+			 * Since we do not support "set -o noclobber" yet,
+			 * >| and > are the same for now. Just eat |.
+			 */
+			ch = i_getch(input);
+#if !BB_MMU
+			o_addchr(&ctx->as_string, ch);
+#endif
+		}
+	}
+
+	/* Create a new redir_struct and append it to the linked list */
+	redirp = &command->redirects;
+	while ((redir = *redirp) != NULL) {
+		redirp = &(redir->next);
+	}
+	*redirp = redir = xzalloc(sizeof(*redir));
+	/* redir->next = NULL; */
+	/* redir->rd_filename = NULL; */
+	redir->rd_type = style;
+	redir->fd = (fd == -1) ? redir_table[style].default_fd : fd;
+
+	debug_printf_parse("redirect type %d %s\n", redir->fd, redir_table[style].descrip);
+
+	redir->dup = dup_num;
+	if (dup_num != -1) {
+		/* Erik had a check here that the file descriptor in question
+		 * is legit; I postpone that to "run time"
+		 * A "-" representation of "close me" shows up as a -3 here */
+		debug_printf_parse("duplicating redirect '%d>&%d'\n", redir->fd, redir->dup);
+	} else {
+		/* Set ctx->pending_redirect, so we know what to do at the
+		 * end of the next parsed word. */
+		ctx->pending_redirect = redir;
+	}
+	return 0;
+}
+
 /* If a redirect is immediately preceded by a number, that number is
  * supposed to tell which file descriptor to redirect.  This routine
  * looks for such preceding numbers.  In an ideal world this routine
@@ -3863,25 +3949,97 @@ static int done_word(o_string *word, struct parse_context *ctx)
  *     echo 49>foo    # redirects fd 49 to file "foo", nothing passed to echo
  *     echo -2>foo    # redirects fd  1 to file "foo",    "-2" passed to echo
  *     echo 49x>foo   # redirects fd  1 to file "foo",   "49x" passed to echo
- * A -1 output from this program means no valid number was found, so the
- * caller should use the appropriate default for this redirection.
+ *
+ * http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html
+ * "2.7 Redirection
+ * ... If n is quoted, the number shall not be recognized as part of
+ * the redirection expression. For example:
+ * echo \2>a
+ * writes the character 2 into file a"
+ * I am not sure we do it right (and not sure we care)
+ *
+ * A -1 return means no valid number was found,
+ * the caller should use the appropriate default for this redirection.
  */
 static int redirect_opt_num(o_string *o)
 {
 	int num;
 
-	if (o->length == 0)
+	if (o->data == NULL)
+		return -1;
+	num = bb_strtou(o->data, NULL, 10);
+	if (errno || num < 0)
 		return -1;
-	for (num = 0; num < o->length; num++) {
-		if (!isdigit(o->data[num])) {
-			return -1;
-		}
-	}
-	num = atoi(o->data);
 	o_reset(o);
 	return num;
 }
 
+//TODO: add NOMMU as_string fill
+static char *fetch_till_str(struct in_str *input, const char *word)
+{
+	o_string heredoc = NULL_O_STRING;
+	int past_EOL = 0;
+	int ch;
+
+	while (1) {
+		ch = i_getch(input);
+		if (ch == EOF) {
+			o_free_unsafe(&heredoc);
+			return NULL;
+		}
+		if (ch == '\n') {
+			if (strcmp(heredoc.data + past_EOL, word) == 0) {
+				heredoc.data[past_EOL] = '\0';
+				debug_printf_parse("parsed heredoc '%s'\n", heredoc.data);
+				return heredoc.data;
+			}
+			past_EOL = heredoc.length + 1;
+		}
+		o_addchr(&heredoc, ch);
+	}
+}
+
+static int fetch_heredocs(int heredoc_cnt, struct parse_context *ctx, struct in_str *input)
+{
+	struct pipe *pi = ctx->list_head;
+
+	while (pi) {
+		int i;
+		struct command *cmd = pi->cmds;
+
+		debug_printf_parse("fetch_heredocs: num_cmds:%d cmd argv0:'%s'\n",
+				pi->num_cmds,
+				cmd->argv ? cmd->argv[0] : "NONE");
+		for (i = 0; i < pi->num_cmds; i++) {
+			struct redir_struct *redirect = cmd->redirects;
+
+			debug_printf_parse("fetch_heredocs: %d cmd argv0:'%s'\n",
+					i, cmd->argv ? cmd->argv[0] : "NONE");
+			while (redirect) {
+				if (redirect->rd_type == REDIRECT_HEREDOC) {
+					char *p;
+
+					if (heredoc_cnt <= 0)
+						return 1; /* error */
+					redirect->rd_type = REDIRECT_HEREDOC2;
+					p = fetch_till_str(input, redirect->rd_filename);
+					if (!p)
+						return 1; /* unexpected EOF */
+					free(redirect->rd_filename);
+					redirect->rd_filename = p;
+					heredoc_cnt--;
+				}
+				redirect = redirect->next;
+			}
+			cmd++;
+		}
+		pi = pi->next;
+	}
+	/* Should be 0. If it isn't, it's a parse error */
+	return heredoc_cnt;
+}
+
+
 #if BB_MMU
 #define parse_stream(pstring, input, end_trigger) \
 	parse_stream(input, end_trigger)
@@ -4391,8 +4549,7 @@ static int parse_stream_dquoted(o_string *as_string,
  again:
 	ch = i_getch(input);
 #if !BB_MMU
-	if (as_string && ch != EOF)
-		o_addchr(as_string, ch);
+	if (as_string && ch != EOF) o_addchr(as_string, ch);
 #endif
 	if (ch == dquote_end) { /* may be only '"' or EOF */
 		dest->nonnull = 1;
@@ -4479,6 +4636,7 @@ static struct pipe *parse_stream(char **pstring,
 	struct parse_context ctx;
 	o_string dest = NULL_O_STRING;
 	int is_in_dquote;
+	int heredoc_cnt;
 
 	/* Double-quote state is handled in the state variable is_in_dquote.
 	 * A single-quote triggers a bypass of the main loop until its mate is
@@ -4498,6 +4656,7 @@ static struct pipe *parse_stream(char **pstring,
 	/* dest.o_assignment = MAYBE_ASSIGNMENT; - already is */
 	initialize_context(&ctx);
 	is_in_dquote = 0;
+	heredoc_cnt = 0;
 	while (1) {
 		const char *is_ifs;
 		const char *is_special;
@@ -4518,6 +4677,11 @@ static struct pipe *parse_stream(char **pstring,
 						ch, ch, dest.o_escape);
 		if (ch == EOF) {
 			struct pipe *pi;
+
+			if (heredoc_cnt) {
+				syntax("unterminated here document");
+				goto parse_error;
+			}
 			if (done_word(&dest, &ctx)) {
 				goto parse_error;
 			}
@@ -4578,14 +4742,36 @@ static struct pipe *parse_stream(char **pstring,
 #endif
 				/* Treat newline as a command separator. */
 				done_pipe(&ctx, PIPE_SEQ);
+				debug_printf_parse("heredoc_cnt:%d\n", heredoc_cnt);
+				if (heredoc_cnt) {
+					if (fetch_heredocs(heredoc_cnt, &ctx, input))
+						goto parse_error;
+					heredoc_cnt = 0;
+				}
 				dest.o_assignment = MAYBE_ASSIGNMENT;
 				ch = ';';
 				/* note: if (is_ifs) continue;
 				 * will still trigger for us */
 			}
 		}
-		if (end_trigger && end_trigger == ch) {
+		if (end_trigger && end_trigger == ch
+		 && (heredoc_cnt == 0 || end_trigger != ';')
+		) {
 //TODO: disallow "{ cmd }" without semicolon
+			if (heredoc_cnt) {
+				/* This is technically valid:
+				 * { cat <<HERE; }; echo Ok
+				 * heredoc
+				 * heredoc
+				 * heredoc
+				 * HERE
+				 * but we don't support this.
+				 * We require heredoc to be in enclosing {}/(),
+				 * if any.
+				 */
+				syntax("unterminated here document");
+				goto parse_error;
+			}
 			if (done_word(&dest, &ctx)) {
 				goto parse_error;
 			}
@@ -4715,7 +4901,8 @@ static struct pipe *parse_stream(char **pstring,
 				goto parse_error;
 			}
 #endif
-			setup_redirect(&ctx, redir_fd, redir_style, input);
+			if (parse_redirect(&ctx, redir_fd, redir_style, input))
+				goto parse_error;
 			break;
 		case '<':
 			redir_fd = redirect_opt_num(&dest);
@@ -4724,7 +4911,9 @@ static struct pipe *parse_stream(char **pstring,
 			}
 			redir_style = REDIRECT_INPUT;
 			if (next == '<') {
-				redir_style = REDIRECT_HEREIS;
+				redir_style = REDIRECT_HEREDOC;
+				heredoc_cnt++;
+				debug_printf_parse("++heredoc_cnt=%d\n", heredoc_cnt);
 				ch = i_getch(input);
 #if !BB_MMU
 				o_addchr(&ctx.as_string, ch);
@@ -4742,7 +4931,8 @@ static struct pipe *parse_stream(char **pstring,
 				goto parse_error;
 			}
 #endif
-			setup_redirect(&ctx, redir_fd, redir_style, input);
+			if (parse_redirect(&ctx, redir_fd, redir_style, input))
+				goto parse_error;
 			break;
 		case ';':
 #if ENABLE_HUSH_CASE
diff --git a/shell/hush_test/hush-misc/exec.tests b/shell/hush_test/hush-misc/exec.tests
index 01a7f591c..6de50fa7b 100755
--- a/shell/hush_test/hush-misc/exec.tests
+++ b/shell/hush_test/hush-misc/exec.tests
@@ -1,30 +1,30 @@
 # make sure we have a way of checking these things
-[ ! -e /dev/fd ] && exit 1
+cd /proc/$$/fd || cd /dev/fd || exit 1
 
-[ -e /dev/fd/44 ] && exit 1
+[ -e 44 ] && exit 1
 exec 44>/dev/null
-[ -e /dev/fd/44 ] || exit 1
+[ -e 44 ] || exit 1
 echo pass fd out open
 
-[ -e /dev/fd/55 ] && exit 1
+[ -e 55 ] && exit 1
 exec 55>&44
-[ -e /dev/fd/55 ] || exit 1
+[ -e 55 ] || exit 1
 echo pass fd out dup
 
 exec 44>&-
-[ -e /dev/fd/44 ] && exit 1
+[ -e 44 ] && exit 1
 echo pass fd out close
 
-[ -e /dev/fd/66 ] && exit 1
+[ -e 66 ] && exit 1
 exec 66</dev/null
-[ -e /dev/fd/66 ] || exit 1
+[ -e 66 ] || exit 1
 echo pass fd in open
 
-[ -e /dev/fd/77 ] && exit 1
+[ -e 77 ] && exit 1
 exec 77<&66
-[ -e /dev/fd/77 ] || exit 1
+[ -e 77 ] || exit 1
 echo pass fd in dup
 
 exec 66<&-
-[ -e /dev/fd/66 ] && exit 1
+[ -e 66 ] && exit 1
 echo pass fd in close
diff --git a/shell/hush_test/hush-misc/heredoc1.right b/shell/hush_test/hush-misc/heredoc1.right
new file mode 100644
index 000000000..7fc68f3e1
--- /dev/null
+++ b/shell/hush_test/hush-misc/heredoc1.right
@@ -0,0 +1,5 @@
+qwe
+asd
+123
+456
+Ok
diff --git a/shell/hush_test/hush-misc/heredoc1.tests b/shell/hush_test/hush-misc/heredoc1.tests
new file mode 100755
index 000000000..2eeb4726b
--- /dev/null
+++ b/shell/hush_test/hush-misc/heredoc1.tests
@@ -0,0 +1,9 @@
+cat <<000; cat <<www; cat <<eee
+000
+qwe
+asd
+www
+123
+456
+eee
+echo Ok