--- /dev/null	2013-02-24 04:11:48.000000000 +0000
+++ new/usr/src/cmd/iconv/scanner.c	2013-02-24 04:11:48.491431527 +0000
@@ -0,0 +1,728 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2010 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2013 David Hoeppner.  All rights reserved.
+ */
+
+/*
+ * Functions to charmap .
+ */
+
+#include <assert.h>
+#include <ctype.h>
+#include <limits.h>
+#include <widec.h>
+
+#include "iconv.h"
+#include "parser.tab.h"
+
+/*
+ * Helper macros.
+ */
+#define	hex(x)	\
+	(isdigit(x) ? (x - '0') : ((islower(x) ? (x - 'a') : (x - 'A')) + 10))
+
+#define	isodigit(x)	((x >= '0') && (x <= '7'))
+
+/*
+ * Charmap specific.
+ */
+int		com_char = '#';
+int		esc_char = '\\';
+int		mb_cur_max = 1;
+int		mb_cur_min = 1;
+
+int		lineno = 1;
+static FILE	*input = stdin;
+static const char *filename = "<stdin>";
+static int	escaped = 0;
+static int	instring = 0;
+static int	nextline;
+
+/*
+ * Tokens.
+ */
+static char	*token = NULL;
+static int	tokidx;
+static int	toksz = 0;
+static int	hadtok = 0;
+
+/*
+ * Wide strings.
+ */
+static wchar_t	*widestr = NULL;
+static int	wideidx = 0;
+static int	widesz = 0;
+
+/*
+ * Keywords related.
+ */
+int		last_kw = 0;
+static int	category = T_END;
+
+static struct token {
+	int	id;
+	const char *name;
+} keywords[] = {
+	{ T_COM_CHAR,		"comment_char" },
+	{ T_ESC_CHAR,		"escape_char" },
+	{ T_END,		"END" },
+	{ T_CHARMAP,		"CHARMAP" },
+	{ T_WIDTH,		"WIDTH" },
+	{ T_WIDTH_DEFAULT,	"WIDTH_DEFAULT" },
+	{ -1, NULL },
+};
+
+/*
+ * Charmap reserved keywords.
+ */
+static struct token symwords[] = {
+	{ T_COM_CHAR,		"comment_char" },
+	{ T_ESC_CHAR,		"escape_char" },
+	{ T_CODE_SET,		"code_set_name" },
+	{ T_MB_CUR_MAX,		"mb_cur_max" },
+	{ T_MB_CUR_MIN,		"mb_cur_min" },
+	{ -1, NULL },
+};
+
+static int categories[] = {
+	T_CHARMAP,
+	T_WIDTH,
+	0,
+};
+
+char *
+to_mb_string(const wchar_t *wcs)
+{
+	return (NULL);
+}
+
+void
+set_wide_encoding(const char *encoding)
+{
+}
+
+/*
+ * Reset the scanner variables and open the supplied charmap file.
+ */
+void
+reset_scanner(const char *fname)
+{
+	input = fopen(fname, "r");
+	if (input == NULL) {
+		perror("fopen");
+		exit(4);
+	}
+
+	filename = fname;
+	com_char = '#';
+	esc_char = '\\';
+	instring = 0;
+	escaped = 0;
+	lineno = 1;
+	nextline = 1;
+	tokidx = 0;
+	wideidx = 0;
+}
+
+static int
+scanc(void)
+{
+	int	c;
+
+	c = getc(input);
+	lineno = nextline;
+	if (c == '\n') {
+		nextline++;
+	}
+
+	return (c);
+}
+
+static void
+unscanc(int c)
+{
+	if (c == '\n') {
+		nextline--;
+	}
+
+	if (ungetc(c, input) < 0) {
+		yyerror(_("ungetc failed"));
+	}
+}
+
+static int
+scan_hex_byte(void)
+{
+	int	c1, c2;
+	int	v;
+
+	c1 = scanc();
+	if (!isxdigit(c1)) {
+		yyerror(_("malformed hex digit"));
+		return (0);
+	}
+	c2 = scanc();
+	if (!isxdigit(c2)) {
+		yyerror(_("malformed hex digit"));
+		return (0);
+	}
+	v = ((hex(c1) << 4) | hex(c2));
+	return (v);
+}
+
+static int
+scan_dec_byte(void)
+{
+	int	c1, c2, c3;
+	int	b;
+
+	c1 = scanc();
+	if (!isdigit(c1)) {
+		yyerror(_("malformed decimal digit"));
+		return (0);
+	}
+	b = c1 - '0';
+	c2 = scanc();
+	if (!isdigit(c2)) {
+		yyerror(_("malformed decimal digit"));
+		return (0);
+	}
+	b *= 10;
+	b += (c2 - '0');
+	c3 = scanc();
+	if (!isdigit(c3)) {
+		unscanc(c3);
+	} else {
+		b *= 10;
+		b += (c3 - '0');
+	}
+	return (b);
+}
+
+static int
+scan_oct_byte(void)
+{
+	int	c1, c2, c3;
+	int	b;
+
+	b = 0;
+
+	c1 = scanc();
+	if (!isodigit(c1)) {
+		yyerror(_("malformed octal digit"));
+		return (0);
+	}
+	b = c1 - '0';
+	c2 = scanc();
+	if (!isodigit(c2)) {
+		yyerror(_("malformed octal digit"));
+		return (0);
+	}
+	b *= 8;
+	b += (c2 - '0');
+	c3 = scanc();
+	if (!isodigit(c3)) {
+		unscanc(c3);
+	} else {
+		b *= 8;
+		b += (c3 - '0');
+	}
+	return (b);
+}
+
+void
+add_tok(int c)
+{
+	if ((tokidx + 1) >= toksz) {
+		toksz += 64;
+
+		if ((token = realloc(token, toksz)) == NULL) {
+			yyerror(_("out of memory"));
+			tokidx = 0;
+			toksz = 0;
+			return;
+		}
+	}
+
+	token[tokidx++] = (char)c;
+	token[tokidx] = 0;
+}
+
+void
+add_wcs(wchar_t c)
+{
+	if ((wideidx + 1) >= widesz) {
+		widesz += 64;
+		widestr = realloc(widestr, (widesz * sizeof (wchar_t)));
+		if (widestr == NULL) {
+			yyerror(_("out of memory"));
+			wideidx = 0;
+			widesz = 0;
+			return;
+		}
+	}
+
+	widestr[wideidx++] = c;
+	widestr[wideidx] = 0;
+}
+
+wchar_t *
+get_wcs(void)
+{
+	wchar_t	*ws = widestr;
+
+	wideidx = 0;
+	widestr = NULL;
+	widesz = 0;
+
+	if (ws == NULL) {
+		if ((ws = wsdup(L"")) == NULL) {
+			yyerror(_("out of memory"));
+		}
+	}
+
+	return (ws);
+}
+
+static int
+get_byte(void)
+{
+	int	c;
+
+	if ((c = scanc()) != esc_char) {
+		unscanc(c);
+		return (EOF);
+	}
+
+	c = scanc();
+
+	switch (c) {
+	case 'd':
+	case 'D':
+		return (scan_dec_byte());
+	case 'x':
+	case 'X':
+		return (scan_hex_byte());
+	case '0' ... '7':
+		/* Put the character back so we can get it */
+		unscanc(c);
+		return (scan_oct_byte());
+	default:
+		unscanc(c);
+		unscanc(esc_char);
+		return (EOF);
+	}
+}
+
+int
+get_escaped(int c)
+{
+	switch (c) {
+	case 'n':
+		return ('\n');
+	case 'r':
+		return ('\r');
+	case 't':
+		return ('\t');
+	case 'f':
+		return ('\f');
+	case 'v':
+		return ('\v');
+	case 'b':
+		return ('\b');
+	case 'a':
+		return ('\a');
+	default:
+		return (c);
+	}
+}
+
+int
+get_wide(void)
+{
+	char	mbs[MB_LEN_MAX + 1] = "";
+	int	mbi = 0;
+	int	c;
+	wchar_t	wc;
+
+	if (mb_cur_max >= sizeof (mbs)) {
+		yyerror(_("max multibyte character size too big"));
+		mbi = 0;
+		return (T_NULL);
+	}
+
+	for (;;) {
+		if ((mbi == mb_cur_max) || ((c = get_byte()) == EOF)) {
+			/*
+			 * End of the byte sequence reached, but no
+			 * valid wide decoding.  Fatal error.
+			 */
+			mbi = 0;
+			yyerror(_("not a valid character encoding"));
+			return (T_NULL);
+		}
+
+		mbs[mbi++] = c;
+		mbs[mbi] = 0;
+
+		if (mbi == mb_cur_max) {
+			break;
+		}
+	}
+
+	mbi = 0;
+	/* XXX */
+	yylval.wc = (uint8_t)*mbs;
+
+	return (T_CHAR);
+}
+
+int
+get_symbol(void)
+{
+	int	c;
+
+	while ((c = scanc()) != EOF) {
+		if (escaped == 1) {
+			escaped = 0;
+			if (c == '\n') {
+				continue;
+			}
+
+			add_tok(get_escaped(c));
+			continue;
+		}
+
+		if (c == esc_char) {
+			escaped = 1;
+			continue;
+		}
+
+		if (c == '\n') {	/* Well that's strange! */
+			yyerror(_("unterminated symbolic name"));
+			continue;
+		}
+
+		if (c == '>') {		/* End of symbol */
+			/*
+			 * This restarts the token from the beginning
+			 * the next time we scan a character.  (This
+			 * token is complete.)
+			 */
+			if (token == NULL) {
+				yyerror(_("missing symbolic name"));
+				return (T_NULL);
+			}
+
+			tokidx = 0;
+
+			/*
+			 * A few symbols are handled as keywords outside
+			 * of the normal categories.
+			 */
+			if (category == T_END) {
+				int	i;
+
+				for (i = 0; symwords[i].name != 0; i++) {
+					if (strcmp(token, symwords[i].name) ==
+					    0) {
+						last_kw = symwords[i].id;
+						return (last_kw);
+					}
+				}
+			}
+
+			/* XXX */
+
+			/* Its an undefined symbol */
+			yylval.token = strdup(token);
+			token = NULL;
+			toksz = 0;
+			tokidx = 0;
+printf("returning SYMBOL %s\n", yylval.token);
+			return (T_SYMBOL);
+		}
+
+		add_tok(c);
+	}
+
+	yyerror(_("unterminated symbolic name"));
+
+	return (EOF);
+}
+
+static int
+consume_token(void)
+{
+	int	len = tokidx;
+	int	i;
+
+	tokidx = 0;
+	if (token == NULL) {
+		return (T_NULL);
+	}
+
+	/*
+	 * This one is special, because we don't want it to alter the
+	 * last_kw field.
+	 */
+	if (strcmp(token, "...") == 0) {
+		return (T_ELLIPSIS);
+	}
+
+	/* Search for reserved words first */
+	for (i = 0; keywords[i].name; i++) {
+		int	j;
+
+		if (strcmp(keywords[i].name, token)) {
+			continue;
+		}
+
+		last_kw = keywords[i].id;
+
+		/* Clear the top level category if we're done with it */
+		if (last_kw == T_END) {
+			category = T_END;
+		}
+
+		/* Set the top level category if we're changing */
+		for (j = 0; categories[j]; j++) {
+			if (categories[j] != last_kw) {
+				continue;
+			}
+			category = last_kw;
+		}
+
+		return (keywords[i].id);
+	}
+
+	/* Maybe its a numeric constant? */
+	if (isdigit(*token) || (*token == '-' && isdigit(token[1]))) {
+		char	*eptr;
+
+		yylval.num = strtol(token, &eptr, 10);
+		if (*eptr != 0) {
+			yyerror(_("malformed number"));
+		}
+
+		return (T_NUMBER);
+	}
+
+	/*
+	 * A single lone character is treated as a character literal.
+	 * To avoid duplication of effort, we stick in the charmap.
+	 */
+	if (len == 1) {
+		yylval.wc = token[0];
+		return (T_CHAR);
+	}
+
+	/* Anything else is treated as a symbolic name */
+	yylval.token = strdup(token);
+	token = NULL;
+	toksz = 0;
+	tokidx = 0;
+
+	return (T_NAME);
+}
+
+void
+scan_to_eol(void)
+{
+	int	c;
+
+	while ((c = scanc()) != '\n') {
+		if (c == EOF) {
+			/* end of file without newline! */
+			errf(_("missing newline"));
+			return;
+		}
+	}
+
+	assert(c == '\n');
+}
+
+int
+yylex(void)
+{
+	int	c;
+
+	while ((c = scanc()) != EOF) {
+printf("--- yylex --%c--\n", c);
+
+		/* Special handling for quoted strings */
+		if (instring == 1) {
+			if (escaped == 1) {
+				escaped = 0;
+
+				/* If newline, just eat and forget it */
+				if (c == '\n') {
+					continue;
+				}
+
+				if (strchr("xd01234567", c)) {
+					unscanc(c);
+					unscanc(esc_char);
+					return (get_wide());
+				}
+
+				yylval.wc = get_escaped(c);
+				return (T_CHAR);
+			}
+
+			if (c == esc_char) {
+				escaped = 1;
+				continue;
+			}
+
+			switch (c) {
+			case '<':
+				return (get_symbol());
+			case '>':
+				/* Opps! Should generate syntax error */
+				return (T_GT);
+			case '"':
+				instring = 0;
+				return (T_QUOTE);
+			default:
+				yylval.wc = c;
+				return (T_CHAR);
+			}
+		}
+
+		/* Escaped characters first */
+		if (escaped == 1) {
+			escaped = 0;
+			if (c == '\n') {
+				/* Eat the newline */
+				continue;
+			}
+			hadtok = 1;
+			if (tokidx != 0) {
+				/* An escape mid-token is nonsense */
+				return (T_NULL);
+			}
+
+			/* Numeric escapes are treated as wide characters */
+			if (strchr("xXd01234567", c)) {
+				unscanc(c);
+				unscanc(esc_char);
+				return (get_wide());
+			}
+
+			add_tok(get_escaped(c));
+			continue;
+		}
+
+		/* If it is the escape character itself note it */
+		if (c == esc_char) {
+			escaped = 1;
+			continue;
+		}
+
+		/* Remove from the comment character to end of line */
+		if (c == com_char) {
+			while (c != '\n') {
+				if ((c = scanc()) == EOF) {
+					/* End of file without newline */
+					return (EOF);
+				}
+			}
+
+			assert(c == '\n');
+
+			if (hadtok == 0) {
+				/*
+				 * If there were no tokens on this line,
+				 * then just pretend it didn't exist at all.
+				 */
+				continue;
+			}
+
+			hadtok = 0;
+			return (T_NL);
+		}
+
+		if (strchr(" \t\n;()<>,\"", c) && (tokidx != 0)) {
+			/*
+			 * These are all token delimiters.  If there
+			 * is a token already in progress, we need to
+			 * process it.
+			 */
+			unscanc(c);
+			return (consume_token());
+		}
+
+		switch (c) {
+		case '\n':
+			if (hadtok == 0) {
+				/*
+				 * If the line was completely devoid of tokens,
+				 * then just ignore it.
+				 */
+				continue;
+			}
+
+			/* We're starting a new line, reset the token state */
+			hadtok = 0;
+			return (T_NL);
+		case '>':
+			hadtok = 1;
+			return (T_GT);
+		case '<':
+			/* Symbol start! */
+			hadtok = 1;
+			return (get_symbol());
+		case ' ':
+		case '\t':
+			/* Whitespace, just ignore */
+			continue;
+		case '"':
+			hadtok = 1;
+			instring = 1;
+			return (T_QUOTE);
+		default:
+//printf("--- adding %c to token\n", c);
+			hadtok = 1;
+			add_tok(c);
+			continue;
+		}
+	}
+
+	return (EOF);
+}
+
+void
+yyerror(const char *msg)
+{
+	(void) fprintf(stderr, _("%s: %d: error: %s\n"),
+	    filename, lineno, msg);
+	exit(4);
+}
+
+void
+errf(const char *fmt, ...)
+{
+	char	*msg;
+	va_list	va;
+
+	va_start(va, fmt);
+	(void) vasprintf(&msg, fmt, va);
+	va_end(va);
+
+	(void) fprintf(stderr, _("%s: %d: error: %s\n"),
+	    filename, lineno, msg);
+	free(msg);
+	exit(4);
+}
