1 /* $Id: preconv.c,v 1.5 2011/07/24 18:15:14 kristaps Exp $ */
2 /*
3 * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
17 #ifdef HAVE_CONFIG_H
18 #include "config.h"
19 #endif
20
21 #ifdef HAVE_MMAP
22 #include <sys/stat.h>
23 #include <sys/mman.h>
24 #endif
25
26 #include <assert.h>
27 #include <fcntl.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <string.h>
31 #include <unistd.h>
32
33 /*
34 * The read_whole_file() and resize_buf() functions are copied from
35 * read.c, including all dependency code (MAP_FILE, etc.).
36 */
37
38 #ifndef MAP_FILE
39 #define MAP_FILE 0
40 #endif
41
42 enum enc {
43 ENC_UTF_8, /* UTF-8 */
44 ENC_US_ASCII, /* US-ASCII */
45 ENC_LATIN_1, /* Latin-1 */
46 ENC__MAX
47 };
48
49 struct buf {
50 char *buf; /* binary input buffer */
51 size_t sz; /* size of binary buffer */
52 size_t offs; /* starting buffer offset */
53 };
54
55 struct encode {
56 const char *name;
57 int (*conv)(const struct buf *);
58 };
59
60 static int cue_enc(const struct buf *, size_t *, enum enc *);
61 static int conv_latin_1(const struct buf *);
62 static int conv_us_ascii(const struct buf *);
63 static int conv_utf_8(const struct buf *);
64 static int read_whole_file(const char *, int,
65 struct buf *, int *);
66 static void resize_buf(struct buf *, size_t);
67 static void usage(void);
68
69 static const struct encode encs[ENC__MAX] = {
70 { "utf-8", conv_utf_8 }, /* ENC_UTF_8 */
71 { "us-ascii", conv_us_ascii }, /* ENC_US_ASCII */
72 { "latin-1", conv_latin_1 }, /* ENC_LATIN_1 */
73 };
74
75 static const char *progname;
76
77 static void
78 usage(void)
79 {
80
81 fprintf(stderr, "usage: %s "
82 "[-D enc] "
83 "[-e ENC] "
84 "[file]\n", progname);
85 }
86
87 static int
88 conv_latin_1(const struct buf *b)
89 {
90 size_t i;
91 unsigned char cu;
92 const char *cp;
93
94 cp = b->buf + (int)b->offs;
95
96 /*
97 * Latin-1 falls into the first 256 code-points of Unicode, so
98 * there's no need for any sort of translation. Just make the
99 * 8-bit characters use the Unicode escape.
100 * Note that binary values 128 < v < 160 are passed through
101 * unmodified to mandoc.
102 */
103
104 for (i = b->offs; i < b->sz; i++) {
105 cu = (unsigned char)*cp++;
106 cu < 160U ? putchar(cu) : printf("\\[u%.4X]", cu);
107 }
108
109 return(1);
110 }
111
112 static int
113 conv_us_ascii(const struct buf *b)
114 {
115
116 /*
117 * US-ASCII has no conversion since it falls into the first 128
118 * bytes of Unicode.
119 */
120
121 fwrite(b->buf, 1, b->sz, stdout);
122 return(1);
123 }
124
125 static int
126 conv_utf_8(const struct buf *b)
127 {
128 int state, be;
129 unsigned int accum;
130 size_t i;
131 unsigned char cu;
132 const char *cp;
133 const long one = 1L;
134
135 cp = b->buf + (int)b->offs;
136 state = 0;
137 accum = 0U;
138 be = 0;
139
140 /* Quick test for big-endian value. */
141
142 if ( ! (*((const char *)(&one))))
143 be = 1;
144
145 for (i = b->offs; i < b->sz; i++) {
146 cu = (unsigned char)*cp++;
147 if (state) {
148 if ( ! (cu & 128) || (cu & 64)) {
149 /* Bad sequence header. */
150 return(0);
151 }
152
153 /* Accept only legitimate bit patterns. */
154
155 if (cu > 191 || cu < 128) {
156 /* Bad in-sequence bits. */
157 return(0);
158 }
159
160 accum |= (cu & 63) << --state * 6;
161
162 /*
163 * Accum is held in little-endian order as
164 * stipulated by the UTF-8 sequence coding. We
165 * need to convert to a native big-endian if our
166 * architecture requires it.
167 */
168
169 if (0 == state && be)
170 accum = (accum >> 24) |
171 ((accum << 8) & 0x00FF0000) |
172 ((accum >> 8) & 0x0000FF00) |
173 (accum << 24);
174
175 if (0 == state) {
176 accum < 128U ? putchar(accum) :
177 printf("\\[u%.4X]", accum);
178 accum = 0U;
179 }
180 } else if (cu & (1 << 7)) {
181 /*
182 * Entering a UTF-8 state: if we encounter a
183 * UTF-8 bitmask, calculate the expected UTF-8
184 * state from it.
185 */
186 for (state = 0; state < 7; state++)
187 if ( ! (cu & (1 << (7 - state))))
188 break;
189
190 /* Accept only legitimate bit patterns. */
191
192 switch (state) {
193 case (4):
194 if (cu <= 244 && cu >= 240) {
195 accum = (cu & 7) << 18;
196 break;
197 }
198 /* Bad 4-sequence start bits. */
199 return(0);
200 case (3):
201 if (cu <= 239 && cu >= 224) {
202 accum = (cu & 15) << 12;
203 break;
204 }
205 /* Bad 3-sequence start bits. */
206 return(0);
207 case (2):
208 if (cu <= 223 && cu >= 194) {
209 accum = (cu & 31) << 6;
210 break;
211 }
212 /* Bad 2-sequence start bits. */
213 return(0);
214 default:
215 /* Bad sequence bit mask. */
216 return(0);
217 }
218 state--;
219 } else
220 putchar(cu);
221 }
222
223 if (0 != state) {
224 /* Bad trailing bits. */
225 return(0);
226 }
227
228 return(1);
229 }
230
231 static void
232 resize_buf(struct buf *buf, size_t initial)
233 {
234
235 buf->sz = buf->sz > initial / 2 ?
236 2 * buf->sz : initial;
237
238 buf->buf = realloc(buf->buf, buf->sz);
239 if (NULL == buf->buf) {
240 perror(NULL);
241 exit(EXIT_FAILURE);
242 }
243 }
244
245 static int
246 read_whole_file(const char *f, int fd,
247 struct buf *fb, int *with_mmap)
248 {
249 size_t off;
250 ssize_t ssz;
251
252 #ifdef HAVE_MMAP
253 struct stat st;
254 if (-1 == fstat(fd, &st)) {
255 perror(f);
256 return(0);
257 }
258
259 /*
260 * If we're a regular file, try just reading in the whole entry
261 * via mmap(). This is faster than reading it into blocks, and
262 * since each file is only a few bytes to begin with, I'm not
263 * concerned that this is going to tank any machines.
264 */
265
266 if (S_ISREG(st.st_mode) && st.st_size >= (1U << 31)) {
267 fprintf(stderr, "%s: input too large\n", f);
268 return(0);
269 }
270
271 if (S_ISREG(st.st_mode)) {
272 *with_mmap = 1;
273 fb->sz = (size_t)st.st_size;
274 fb->buf = mmap(NULL, fb->sz, PROT_READ,
275 MAP_FILE|MAP_SHARED, fd, 0);
276 if (fb->buf != MAP_FAILED)
277 return(1);
278 }
279 #endif
280
281 /*
282 * If this isn't a regular file (like, say, stdin), then we must
283 * go the old way and just read things in bit by bit.
284 */
285
286 *with_mmap = 0;
287 off = 0;
288 fb->sz = 0;
289 fb->buf = NULL;
290 for (;;) {
291 if (off == fb->sz && fb->sz == (1U << 31)) {
292 fprintf(stderr, "%s: input too large\n", f);
293 break;
294 }
295
296 if (off == fb->sz)
297 resize_buf(fb, 65536);
298
299 ssz = read(fd, fb->buf + (int)off, fb->sz - off);
300 if (ssz == 0) {
301 fb->sz = off;
302 return(1);
303 }
304 if (ssz == -1) {
305 perror(f);
306 break;
307 }
308 off += (size_t)ssz;
309 }
310
311 free(fb->buf);
312 fb->buf = NULL;
313 return(0);
314 }
315
316 static int
317 cue_enc(const struct buf *b, size_t *offs, enum enc *enc)
318 {
319 const char *ln, *eoln, *eoph;
320 size_t sz, phsz, nsz;
321 int i;
322
323 ln = b->buf + (int)*offs;
324 sz = b->sz - *offs;
325
326 /* Look for the end-of-line. */
327
328 if (NULL == (eoln = memchr(ln, '\n', sz)))
329 return(-1);
330
331 /* Set next-line marker. */
332
333 *offs = (size_t)((eoln + 1) - b->buf);
334
335 /* Check if we have the correct header/trailer. */
336
337 if ((sz = (size_t)(eoln - ln)) < 10 ||
338 memcmp(ln, ".\\\" -*-", 7) ||
339 memcmp(eoln - 3, "-*-", 3))
340 return(0);
341
342 /* Move after the header and adjust for the trailer. */
343
344 ln += 7;
345 sz -= 10;
346
347 while (sz > 0) {
348 while (sz > 0 && ' ' == *ln) {
349 ln++;
350 sz--;
351 }
352 if (0 == sz)
353 break;
354
355 /* Find the end-of-phrase marker (or eoln). */
356
357 if (NULL == (eoph = memchr(ln, ';', sz)))
358 eoph = eoln - 3;
359 else
360 eoph++;
361
362 /* Only account for the "coding" phrase. */
363
364 if ((phsz = (size_t)(eoph - ln)) < 7 ||
365 strncasecmp(ln, "coding:", 7)) {
366 sz -= phsz;
367 ln += phsz;
368 continue;
369 }
370
371 sz -= 7;
372 ln += 7;
373
374 while (sz > 0 && ' ' == *ln) {
375 ln++;
376 sz--;
377 }
378 if (0 == sz)
379 break;
380
381 /* Check us against known encodings. */
382
383 for (i = 0; i < (int)ENC__MAX; i++) {
384 nsz = strlen(encs[i].name);
385 if (phsz < nsz)
386 continue;
387 if (strncasecmp(ln, encs[i].name, nsz))
388 continue;
389
390 *enc = (enum enc)i;
391 return(1);
392 }
393
394 /* Unknown encoding. */
395
396 *enc = ENC__MAX;
397 return(1);
398 }
399
400 return(0);
401 }
402
403 int
404 main(int argc, char *argv[])
405 {
406 int i, ch, map, fd, rc;
407 struct buf b;
408 const char *fn;
409 enum enc enc, def;
410 unsigned char bom[3] = { 0xEF, 0xBB, 0xBF };
411 size_t offs;
412 extern int optind;
413 extern char *optarg;
414
415 progname = strrchr(argv[0], '/');
416 if (progname == NULL)
417 progname = argv[0];
418 else
419 ++progname;
420
421 fn = "<stdin>";
422 fd = STDIN_FILENO;
423 rc = EXIT_FAILURE;
424 enc = def = ENC__MAX;
425 map = 0;
426
427 memset(&b, 0, sizeof(struct buf));
428
429 while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
430 switch (ch) {
431 case ('D'):
432 /* FALLTHROUGH */
433 case ('e'):
434 for (i = 0; i < (int)ENC__MAX; i++) {
435 if (strcasecmp(optarg, encs[i].name))
436 continue;
437 break;
438 }
439 if (i < (int)ENC__MAX) {
440 if ('D' == ch)
441 def = (enum enc)i;
442 else
443 enc = (enum enc)i;
444 break;
445 }
446
447 fprintf(stderr, "%s: Bad encoding\n", optarg);
448 return(EXIT_FAILURE);
449 case ('r'):
450 /* FALLTHROUGH */
451 case ('d'):
452 /* FALLTHROUGH */
453 case ('v'):
454 /* Compatibility with GNU preconv. */
455 break;
456 case ('h'):
457 /* Compatibility with GNU preconv. */
458 /* FALLTHROUGH */
459 default:
460 usage();
461 return(EXIT_FAILURE);
462 }
463
464 argc -= optind;
465 argv += optind;
466
467 /*
468 * Open and read the first argument on the command-line.
469 * If we don't have one, we default to stdin.
470 */
471
472 if (argc > 0) {
473 fn = *argv;
474 fd = open(fn, O_RDONLY, 0);
475 if (-1 == fd) {
476 perror(fn);
477 return(EXIT_FAILURE);
478 }
479 }
480
481 if ( ! read_whole_file(fn, fd, &b, &map))
482 goto out;
483
484 /* Try to read the UTF-8 BOM. */
485
486 if (ENC__MAX == enc)
487 if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) {
488 b.offs = 3;
489 enc = ENC_UTF_8;
490 }
491
492 /* Try reading from the "-*-" cue. */
493
494 if (ENC__MAX == enc) {
495 offs = b.offs;
496 ch = cue_enc(&b, &offs, &enc);
497 if (0 == ch)
498 ch = cue_enc(&b, &offs, &enc);
499 }
500
501 /*
502 * No encoding has been detected.
503 * Thus, we either fall into our default encoder, if specified,
504 * or use Latin-1 if all else fails.
505 */
506
507 if (ENC__MAX == enc)
508 enc = ENC__MAX == def ? ENC_LATIN_1 : def;
509
510 if ( ! (*encs[(int)enc].conv)(&b)) {
511 fprintf(stderr, "%s: Bad encoding\n", fn);
512 goto out;
513 }
514
515 rc = EXIT_SUCCESS;
516 out:
517 #ifdef HAVE_MMAP
518 if (map)
519 munmap(b.buf, b.sz);
520 else
521 #endif
522 free(b.buf);
523
524 if (fd > STDIN_FILENO)
525 close(fd);
526
527 return(rc);
528 }