1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
14 * Copyright 2012 Garrett D'Amore <garrett@damore.org> All rights reserved.
15 */
16
17 /*
18 * The functions in this file convert from the standard multibyte forms
19 * to the wide character forms used internally by libc. Unfortunately,
20 * this approach means that we need a method for each and every encoding.
21 */
22
23 #include <stdlib.h>
24 #include <wchar.h>
25 #include <string.h>
26 #include <note.h>
27 #include <sys/types.h>
28 #include "localedef.h"
29
30 static int towide_none(wchar_t *, const char *, unsigned);
31 static int towide_utf8(wchar_t *, const char *, unsigned);
32 static int towide_big5(wchar_t *, const char *, unsigned);
33 static int towide_gbk(wchar_t *, const char *, unsigned);
34 static int towide_gb2312(wchar_t *, const char *, unsigned);
35 static int towide_gb18030(wchar_t *, const char *, unsigned);
36 static int towide_mskanji(wchar_t *, const char *, unsigned);
37 static int towide_euccn(wchar_t *, const char *, unsigned);
38 static int towide_eucjp(wchar_t *, const char *, unsigned);
39 static int towide_euckr(wchar_t *, const char *, unsigned);
40 static int towide_euctw(wchar_t *, const char *, unsigned);
41
42 static int tomb_none(char *, wchar_t);
43 static int tomb_utf8(char *, wchar_t);
44 static int tomb_mbs(char *, wchar_t);
45
46 static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none;
47 static int (*_tomb)(char *, wchar_t) = tomb_none;
48 static const char *_encoding = "NONE";
49 static int _nbits = 7;
50
51 /*
52 * Table of supported encodings. We only bother to list the multibyte
53 * encodings here, because single byte locales are handed by "NONE".
54 */
55 static struct {
56 const char *name;
57 /* the name that the underlying libc implemenation uses */
58 const char *cname;
59 /* the maximum number of bits required for priorities */
60 int nbits;
61 int (*towide)(wchar_t *, const char *, unsigned);
62 int (*tomb)(char *, wchar_t);
63 } mb_encodings[] = {
64 /*
65 * UTF8 values max out at 0x1fffff (although in theory there could
66 * be later extensions, but it won't happen.) This means we only need
67 * 21 bits to be able to encode the entire range of priorities.
68 */
69 { "UTF-8", "UTF-8", 21, towide_utf8, tomb_utf8 },
70 { "UTF8", "UTF-8", 21, towide_utf8, tomb_utf8 },
71 { "utf8", "UTF-8", 21, towide_utf8, tomb_utf8 },
72 { "utf-8", "UTF-8", 21, towide_utf8, tomb_utf8 },
73
74 { "EUC-CN", "EUC-CN", 16, towide_euccn, tomb_mbs },
75 { "eucCN", "EUC-CN", 16, towide_euccn, tomb_mbs },
76 /*
77 * Becuase the 3-byte form of EUC-JP use the same leading byte,
78 * only 17 bits required to provide unique priorities. (The low
79 * bit of that first byte is set.) By setting this value low,
80 * we can get by with only 3 bytes in the strxfrm expansion.
81 */
82 { "EUC-JP", "EUC-JP", 17, towide_eucjp, tomb_mbs },
83 { "eucJP", "EUC-JP", 17, towide_eucjp, tomb_mbs },
84
85 { "EUC-KR", "EUC-KR", 16, towide_euckr, tomb_mbs },
86 { "eucKR", "EUC-KR", 16, towide_euckr, tomb_mbs },
87 /*
88 * EUC-TW uses 2 bytes most of the time, but 4 bytes if the
89 * high order byte is 0x8E. However, with 4 byte encodings,
90 * the third byte will be A0-B0. So we only need to consider
91 * the lower order 24 bits for collation.
92 */
93 { "EUC-TW", "EUC-TW", 24, towide_euctw, tomb_mbs },
94 { "eucTW", "EUC-TW", 24, towide_euctw, tomb_mbs },
95
96 { "MS_Kanji", "MSKanji", 16, towide_mskanji, tomb_mbs },
97 { "MSKanji", "MSKanji", 16, towide_mskanji, tomb_mbs },
98 { "PCK", "MSKanji", 16, towide_mskanji, tomb_mbs },
99 { "SJIS", "MSKanji", 16, towide_mskanji, tomb_mbs },
100 { "Shift_JIS", "MSKanji", 16, towide_mskanji, tomb_mbs },
101
102 { "BIG5", "BIG5", 16, towide_big5, tomb_mbs },
103 { "big5", "BIG5", 16, towide_big5, tomb_mbs },
104 { "Big5", "BIG5", 16, towide_big5, tomb_mbs },
105
106 { "GBK", "GBK", 16, towide_gbk, tomb_mbs },
107
108 /*
109 * GB18030 can get away with just 31 bits. This is because the
110 * high order bit is always set for 4 byte values, and the
111 * at least one of the other bits in that 4 byte value will
112 * be non-zero.
113 */
114 { "GB18030", "GB18030", 31, towide_gb18030, tomb_mbs },
115
116 /*
117 * This should probably be an aliase for euc-cn, or vice versa.
118 */
119 { "GB2312", "GB2312", 16, towide_gb2312, tomb_mbs },
120
121 { NULL, NULL },
122 };
123
124 static char *
125 show_mb(const char *mb)
126 {
127 static char buf[64];
128
129 /* ASCII stuff we just print */
130 if (isascii(*mb) && isgraph(*mb)) {
131 buf[0] = *mb;
132 buf[1] = 0;
133 return (buf);
134 }
135 buf[0] = 0;
136 while (*mb != 0) {
137 char scr[8];
138 (void) snprintf(scr, sizeof (scr), "\\x%02x", *mb);
139 (void) strlcat(buf, scr, sizeof (buf));
140 mb++;
141 }
142 return (buf);
143 }
144
145 static char *widemsg;
146
147 void
148 werr(const char *fmt, ...)
149 {
150 char *msg;
151
152 va_list va;
153 va_start(va, fmt);
154 (void) vasprintf(&msg, fmt, va);
155 va_end(va);
156
157 free(widemsg);
158 widemsg = msg;
159 }
160
161 /*
162 * This is used for 8-bit encodings.
163 */
164 /* ARGSUSED */
165 int
166 towide_none(wchar_t *c, const char *mb, unsigned n)
167 {
168 _NOTE(ARGUNUSED(n));
169
170 if (mb_cur_max != 1) {
171 werr("invalid or unsupported multibyte locale");
172 return (-1);
173 }
174 *c = (uint8_t)*mb;
175 return (1);
176 }
177
178 int
179 tomb_none(char *mb, wchar_t wc)
180 {
181 if (mb_cur_max != 1) {
182 werr("invalid or unsupported multibyte locale");
183 return (-1);
184 }
185 *(uint8_t *)mb = (wc & 0xff);
186 mb[1] = 0;
187 return (1);
188 }
189
190 /*
191 * UTF-8 stores wide characters in UTF-32 form.
192 */
193 int
194 towide_utf8(wchar_t *wc, const char *mb, unsigned n)
195 {
196 wchar_t c;
197 int nb;
198 int lv; /* lowest legal value */
199 int i;
200 const uint8_t *s = (const uint8_t *)mb;
201
202 c = *s;
203
204 if ((c & 0x80) == 0) {
205 /* 7-bit ASCII */
206 *wc = c;
207 return (1);
208 } else if ((c & 0xe0) == 0xc0) {
209 /* u80-u7ff - two bytes encoded */
210 nb = 2;
211 lv = 0x80;
212 c &= ~0xe0;
213 } else if ((c & 0xf0) == 0xe0) {
214 /* u800-uffff - three bytes encoded */
215 nb = 3;
216 lv = 0x800;
217 c &= ~0xf0;
218 } else if ((c & 0xf8) == 0xf0) {
219 /* u1000-u1fffff - four bytes encoded */
220 nb = 4;
221 lv = 0x1000;
222 c &= ~0xf8;
223 } else {
224 /* 5 and 6 byte encodings are not legal unicode */
225 werr("utf8 encoding too large (%s)", show_mb(mb));
226 return (-1);
227 }
228 if (nb > n) {
229 werr("incomplete utf8 sequence (%s)", show_mb(mb));
230 return (-1);
231 }
232
233 for (i = 1; i < nb; i++) {
234 if (((s[i]) & 0xc0) != 0x80) {
235 werr("illegal utf8 byte (%x)", s[i]);
236 return (-1);
237 }
238 c <<= 6;
239 c |= (s[i] & 0x3f);
240 }
241
242 if (c < lv) {
243 werr("illegal redundant utf8 encoding (%s)", show_mb(mb));
244 return (-1);
245 }
246 *wc = c;
247 return (nb);
248 }
249
250 int
251 tomb_utf8(char *mb, wchar_t wc)
252 {
253 uint8_t *s = (uint8_t *)mb;
254 uint8_t msk;
255 int cnt;
256 int i;
257
258 if (wc <= 0x7f) {
259 s[0] = wc & 0x7f;
260 s[1] = 0;
261 return (1);
262 }
263 if (wc <= 0x7ff) {
264 cnt = 2;
265 msk = 0xc0;
266 } else if (wc <= 0xffff) {
267 cnt = 3;
268 msk = 0xe0;
269 } else if (wc <= 0x1fffff) {
270 cnt = 4;
271 msk = 0xf0;
272 } else {
273 werr("illegal uf8 char (%x)", wc);
274 return (-1);
275 }
276 for (i = cnt - 1; i; i--) {
277 s[i] = (wc & 0x3f) | 0x80;
278 wc >>= 6;
279 }
280 s[0] = (msk) | wc;
281 s[cnt] = 0;
282 return (cnt);
283 }
284
285 /*
286 * Several encodings share a simplistic dual byte encoding. In these
287 * forms, they all indicate that a two byte sequence is to be used if
288 * the first byte has its high bit set. They all store this simple
289 * encoding as a 16-bit value, although a great many of the possible
290 * code points are not used in most character sets. This gives a possible
291 * set of just over 32,000 valid code points.
292 *
293 * 0x00 - 0x7f - 1 byte encoding
294 * 0x80 - 0x7fff - illegal
295 * 0x8000 - 0xffff - 2 byte encoding
296 */
297 static int
298 towide_dbcs(wchar_t *wc, const char *mb, unsigned n)
299 {
300 wchar_t c;
301
302 c = *(uint8_t *)mb;
303
304 if ((c & 0x80) == 0) {
305 /* 7-bit */
306 *wc = c;
307 return (1);
308 }
309 if (n < 2) {
310 werr("incomplete character sequence (%s)", show_mb(mb));
311 return (-1);
312 }
313
314 /* Store both bytes as a single 16-bit wide. */
315 c <<= 8;
316 c |= (uint8_t)(mb[1]);
317 *wc = c;
318 return (2);
319 }
320
321 /*
322 * Most multibyte locales just convert the wide character to the multibyte
323 * form by stripping leading null bytes, and writing the 32-bit quantity
324 * in big-endian order.
325 */
326 int
327 tomb_mbs(char *mb, wchar_t wc)
328 {
329 uint8_t *s = (uint8_t *)mb;
330 int n = 0, c;
331
332 if ((wc & 0xff000000U) != 0) {
333 n = 4;
334 } else if ((wc & 0x00ff0000U) != 0) {
335 n = 3;
336 } else if ((wc & 0x0000ff00U) != 0) {
337 n = 2;
338 } else {
339 n = 1;
340 }
341 c = n;
342 while (n) {
343 n--;
344 s[n] = wc & 0xff;
345 wc >>= 8;
346 }
347 /* ensure null termination */
348 s[c] = 0;
349 return (c);
350 }
351
352
353 /*
354 * big5 is a simple dual byte character set.
355 */
356 int
357 towide_big5(wchar_t *wc, const char *mb, unsigned n)
358 {
359 return (towide_dbcs(wc, mb, n));
360 }
361
362 /*
363 * GBK encodes wides in the same way that big5 does, the high order
364 * bit of the first byte indicates a double byte character.
365 */
366 int
367 towide_gbk(wchar_t *wc, const char *mb, unsigned n)
368 {
369 return (towide_dbcs(wc, mb, n));
370 }
371
372 /*
373 * GB2312 is another DBCS. Its cleaner than others in that the second
374 * byte does not encode ASCII, but it supports characters.
375 */
376 int
377 towide_gb2312(wchar_t *wc, const char *mb, unsigned n)
378 {
379 return (towide_dbcs(wc, mb, n));
380 }
381
382 /*
383 * GB18030. This encodes as 8, 16, or 32-bits.
384 * 7-bit values are in 1 byte, 4 byte sequences are used when
385 * the second byte encodes 0x30-39 and all other sequences are 2 bytes.
386 */
387 int
388 towide_gb18030(wchar_t *wc, const char *mb, unsigned n)
389 {
390 wchar_t c;
391
392 c = *(uint8_t *)mb;
393
394 if ((c & 0x80) == 0) {
395 /* 7-bit */
396 *wc = c;
397 return (1);
398 }
399 if (n < 2) {
400 werr("incomplete character sequence (%s)", show_mb(mb));
401 return (-1);
402 }
403
404 /* pull in the second byte */
405 c <<= 8;
406 c |= (uint8_t)(mb[1]);
407
408 if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) {
409 if (n < 4) {
410 werr("incomplete 4-byte character sequence (%s)",
411 show_mb(mb));
412 return (-1);
413 }
414 c <<= 8;
415 c |= (uint8_t)(mb[2]);
416 c <<= 8;
417 c |= (uint8_t)(mb[3]);
418 *wc = c;
419 return (4);
420 }
421
422 *wc = c;
423 return (2);
424 }
425
426 /*
427 * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it
428 * also has a range of single byte characters above 0x80. (0xa1-0xdf).
429 */
430 int
431 towide_mskanji(wchar_t *wc, const char *mb, unsigned n)
432 {
433 wchar_t c;
434
435 c = *(uint8_t *)mb;
436
437 if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) {
438 /* 7-bit */
439 *wc = c;
440 return (1);
441 }
442
443 if (n < 2) {
444 werr("incomplete character sequence (%s)", show_mb(mb));
445 return (-1);
446 }
447
448 /* Store both bytes as a single 16-bit wide. */
449 c <<= 8;
450 c |= (uint8_t)(mb[1]);
451 *wc = c;
452 return (2);
453 }
454
455 /*
456 * EUC forms. EUC encodings are "variable". FreeBSD carries some additional
457 * variable data to encode these, but we're going to treat each as independent
458 * instead. Its the only way we can sensibly move forward.
459 *
460 * Note that the way in which the different EUC forms vary is how wide
461 * CS2 and CS3 are and what the first byte of them is.
462 */
463 static int
464 towide_euc_impl(wchar_t *wc, const char *mb, unsigned n,
465 uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width)
466 {
467 int i;
468 int width;
469 wchar_t c;
470
471 c = *(uint8_t *)mb;
472
473 /*
474 * All variations of EUC encode 7-bit ASCII as one byte, and use
475 * additional bytes for more than that.
476 */
477 if ((c & 0x80) == 0) {
478 /* 7-bit */
479 *wc = c;
480 return (1);
481 }
482
483 /*
484 * All EUC variants reserve 0xa1-0xff to identify CS1, which
485 * is always two bytes wide. Note that unused CS will be zero,
486 * and that cannot be true because we know that the high order
487 * bit must be set.
488 */
489 if (c >= 0xa1) {
490 width = 2;
491 } else if (c == cs2) {
492 width = cs2width;
493 } else if (c == cs3) {
494 width = cs3width;
495 }
496
497 if (n < width) {
498 werr("incomplete character sequence (%s)", show_mb(mb));
499 return (-1);
500 }
501
502 for (i = 1; i < width; i++) {
503 /* pull in the next byte */
504 c <<= 8;
505 c |= (uint8_t)(mb[i]);
506 }
507
508 *wc = c;
509 return (width);
510 }
511
512 /*
513 * EUC-CN encodes as follows:
514 *
515 * Code set 0 (ASCII): 0x21-0x7E
516 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE
517 * Code set 2: unused
518 * Code set 3: unused
519 */
520 int
521 towide_euccn(wchar_t *wc, const char *mb, unsigned n)
522 {
523 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
524 }
525
526 /*
527 * EUC-JP encodes as follows:
528 *
529 * Code set 0 (ASCII or JIS X 0201-1976 Roman): 0x21-0x7E
530 * Code set 1 (JIS X 0208): 0xA1A1-0xFEFE
531 * Code set 2 (half-width katakana): 0x8EA1-0x8EDF
532 * Code set 3 (JIS X 0212-1990): 0x8FA1A1-0x8FFEFE
533 */
534 int
535 towide_eucjp(wchar_t *wc, const char *mb, unsigned n)
536 {
537 return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3));
538 }
539
540 /*
541 * EUC-KR encodes as follows:
542 *
543 * Code set 0 (ASCII or KS C 5636-1993): 0x21-0x7E
544 * Code set 1 (KS C 5601-1992): 0xA1A1-0xFEFE
545 * Code set 2: unused
546 * Code set 3: unused
547 */
548 int
549 towide_euckr(wchar_t *wc, const char *mb, unsigned n)
550 {
551 return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0));
552 }
553
554 /*
555 * EUC-TW encodes as follows:
556 *
557 * Code set 0 (ASCII): 0x21-0x7E
558 * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE
559 * Code set 2 (CNS 11643-1992 Planes 1-16): 0x8EA1A1A1-0x8EB0FEFE
560 * Code set 3: unused
561 */
562 int
563 towide_euctw(wchar_t *wc, const char *mb, unsigned n)
564 {
565 return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
566 }
567
568 /*
569 * Public entry points.
570 */
571
572 int
573 to_wide(wchar_t *wc, const char *mb)
574 {
575 /* this won't fail hard */
576 return (_towide(wc, mb, strlen(mb)));
577 }
578
579 int
580 to_mb(char *mb, wchar_t wc)
581 {
582 int rv;
583
584 if ((rv = _tomb(mb, wc)) < 0) {
585 errf(widemsg);
586 free(widemsg);
587 widemsg = NULL;
588 }
589 return (rv);
590 }
591
592 char *
593 to_mb_string(const wchar_t *wcs)
594 {
595 char *mbs;
596 char *ptr;
597 int len;
598
599 mbs = malloc((wcslen(wcs) * mb_cur_max) + 1);
600 if (mbs == NULL) {
601 errf("out of memory");
602 return (NULL);
603 }
604 ptr = mbs;
605 while (*wcs) {
606 if ((len = to_mb(ptr, *wcs)) < 0) {
607 INTERR;
608 free(mbs);
609 return (NULL);
610 }
611 wcs++;
612 ptr += len;
613 }
614 *ptr = 0;
615 return (mbs);
616 }
617
618 void
619 set_wide_encoding(const char *encoding)
620 {
621 int i;
622
623 _towide = towide_none;
624 _tomb = tomb_none;
625 _encoding = "NONE";
626 _nbits = 8;
627
628 for (i = 0; mb_encodings[i].name; i++) {
629 if (strcasecmp(encoding, mb_encodings[i].name) == 0) {
630 _towide = mb_encodings[i].towide;
631 _tomb = mb_encodings[i].tomb;
632 _encoding = mb_encodings[i].cname;
633 _nbits = mb_encodings[i].nbits;
634 break;
635 }
636 }
637 }
638
639 const char *
640 get_wide_encoding(void)
641 {
642 return (_encoding);
643 }
644
645 int
646 max_wide(void)
647 {
648 return ((int)((1U << _nbits) - 1));
649 }