Print this page
2964 need POSIX 2008 locale object support
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Gordon Ross <gordon.ross@nexenta.com>
Approved by: TBD

*** 1,7 **** /* ! * Copright 2010 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 1995 Alex Tatmanjants <alex@elvisti.kiev.ua> * at Electronni Visti IA, Kiev, Ukraine. * All rights reserved. * * Redistribution and use in source and binary forms, with or without --- 1,8 ---- /* ! * Copyright 2014 Garrett D'Amore <garrett@damore.org> ! * Copyright 2010 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 1995 Alex Tatmanjants <alex@elvisti.kiev.ua> * at Electronni Visti IA, Kiev, Ukraine. * All rights reserved. * * Redistribution and use in source and binary forms, with or without
*** 42,126 **** #include <sys/stat.h> #include <sys/mman.h> #include "collate.h" #include "setlocale.h" - #include "ldpart.h" /* * See the comments in usr/src/cmd/localedef/collate.c for further * information. It would also be very helpful to have a copy of the * POSIX standard for collation (in the locale format manual page) * handy (www.opengroup.org). */ ! static collate_subst_t *subst_table[COLL_WEIGHTS_MAX]; ! static collate_char_t *char_pri_table; ! static collate_large_t *large_pri_table; ! static collate_chain_t *chain_pri_table; ! static char *cache = NULL; ! static size_t cachesz; ! static char collate_encoding[ENCODING_LEN + 1]; ! /* Exposed externally to other parts of libc. */ ! collate_info_t *_collate_info; ! int _collate_load_error = 1; ! int ! _collate_load_tables(const char *encoding) { int i, chains, z; char buf[PATH_MAX]; char *TMP; char *map; collate_info_t *info; struct stat sbuf; int fd; - /* 'encoding' must be already checked. */ - if (strcmp(encoding, "C") == 0 || strcmp(encoding, "POSIX") == 0) { - _collate_load_error = 1; - return (_LDP_CACHE); - } - /* - * If the locale name is the same as our cache, use the cache. - */ - if (cache && (strncmp(encoding, collate_encoding, ENCODING_LEN) == 0)) { - _collate_load_error = 0; - return (_LDP_CACHE); - } - - /* * Slurp the locale file into the cache. */ (void) snprintf(buf, sizeof (buf), "%s/%s/LC_COLLATE/LCL_DATA", ! _PathLocale, encoding); ! if ((fd = open(buf, O_RDONLY)) < 0) ! return (_LDP_ERROR); if (fstat(fd, &sbuf) < 0) { (void) close(fd); ! return (_LDP_ERROR); } if (sbuf.st_size < (COLLATE_STR_LEN + sizeof (info))) { (void) close(fd); errno = EINVAL; ! return (_LDP_ERROR); } map = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0); (void) close(fd); if ((TMP = map) == NULL) { ! return (_LDP_ERROR); } if (strncmp(TMP, COLLATE_VERSION, COLLATE_STR_LEN) != 0) { (void) munmap(map, sbuf.st_size); errno = EINVAL; ! return (_LDP_ERROR); } TMP += COLLATE_STR_LEN; info = (void *)TMP; TMP += sizeof (*info); --- 43,118 ---- #include <sys/stat.h> #include <sys/mman.h> #include "collate.h" #include "setlocale.h" /* * See the comments in usr/src/cmd/localedef/collate.c for further * information. It would also be very helpful to have a copy of the * POSIX standard for collation (in the locale format manual page) * handy (www.opengroup.org). */ ! /* ! * POSIX uses empty tables and falls down to strcmp. ! */ ! struct lc_collate lc_collate_posix = { ! .lc_is_posix = 1, ! }; ! struct locdata __posix_collate_locdata = { ! .l_lname = "C", ! .l_data = { &lc_collate_posix } ! }; ! struct locdata * ! __lc_collate_load(const char *locname) { int i, chains, z; char buf[PATH_MAX]; char *TMP; char *map; collate_info_t *info; struct stat sbuf; int fd; + struct locdata *ldata; + struct lc_collate *lcc; /* * Slurp the locale file into the cache. */ (void) snprintf(buf, sizeof (buf), "%s/%s/LC_COLLATE/LCL_DATA", ! _PathLocale, locname); ! if ((fd = open(buf, O_RDONLY)) < 0) { ! errno = EINVAL; ! return (NULL); ! } if (fstat(fd, &sbuf) < 0) { (void) close(fd); ! errno = EINVAL; ! return (NULL); } if (sbuf.st_size < (COLLATE_STR_LEN + sizeof (info))) { (void) close(fd); errno = EINVAL; ! return (NULL); } map = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0); (void) close(fd); if ((TMP = map) == NULL) { ! errno = EINVAL; ! return (NULL); } if (strncmp(TMP, COLLATE_VERSION, COLLATE_STR_LEN) != 0) { (void) munmap(map, sbuf.st_size); errno = EINVAL; ! return (NULL); } TMP += COLLATE_STR_LEN; info = (void *)TMP; TMP += sizeof (*info);
*** 128,203 **** if ((info->directive_count < 1) || (info->directive_count >= COLL_WEIGHTS_MAX) || ((chains = info->chain_count) < 0)) { (void) munmap(map, sbuf.st_size); errno = EINVAL; ! return (_LDP_ERROR); } i = (sizeof (collate_char_t) * (UCHAR_MAX + 1)) + (sizeof (collate_chain_t) * chains) + (sizeof (collate_large_t) * info->large_count); ! for (z = 0; z < (info->directive_count); z++) { i += sizeof (collate_subst_t) * info->subst_count[z]; } if (i != (sbuf.st_size - (TMP - map))) { (void) munmap(map, sbuf.st_size); errno = EINVAL; ! return (_LDP_ERROR); } ! char_pri_table = (void *)TMP; TMP += sizeof (collate_char_t) * (UCHAR_MAX + 1); ! for (z = 0; z < info->directive_count; z++) { ! if (info->subst_count[z] > 0) { ! subst_table[z] = (void *)TMP; ! TMP += info->subst_count[z] * sizeof (collate_subst_t); } else { ! subst_table[z] = NULL; } } if (chains > 0) { ! chain_pri_table = (void *)TMP; TMP += chains * sizeof (collate_chain_t); } else ! chain_pri_table = NULL; ! if (info->large_count > 0) ! large_pri_table = (void *)TMP; else ! large_pri_table = NULL; ! (void) strlcpy(collate_encoding, encoding, ENCODING_LEN); ! _collate_info = info; ! ! if (cache) ! (void) munmap(cache, cachesz); ! ! cache = map; ! cachesz = sbuf.st_size; ! _collate_load_error = 0; ! ! return (_LDP_LOADED); } ! static int32_t * ! substsearch(const wchar_t key, int pass) { ! collate_subst_t *p; ! int n = _collate_info->subst_count[pass]; if (n == 0) return (NULL); ! if (pass >= _collate_info->directive_count) return (NULL); if (!(key & COLLATE_SUBST_PRIORITY)) return (NULL); ! p = subst_table[pass] + (key & ~COLLATE_SUBST_PRIORITY); assert(p->key == key); return (p->pri); } /* --- 120,207 ---- if ((info->directive_count < 1) || (info->directive_count >= COLL_WEIGHTS_MAX) || ((chains = info->chain_count) < 0)) { (void) munmap(map, sbuf.st_size); errno = EINVAL; ! return (NULL); } i = (sizeof (collate_char_t) * (UCHAR_MAX + 1)) + (sizeof (collate_chain_t) * chains) + (sizeof (collate_large_t) * info->large_count); ! for (z = 0; z < info->directive_count; z++) { i += sizeof (collate_subst_t) * info->subst_count[z]; } if (i != (sbuf.st_size - (TMP - map))) { (void) munmap(map, sbuf.st_size); errno = EINVAL; ! return (NULL); } ! ! if ((ldata = __locdata_alloc(locname, sizeof (*lcc))) == NULL) { ! (void) munmap(map, sbuf.st_size); ! return (NULL); ! } ! lcc = ldata->l_data[0]; ! ldata->l_map = map; ! ldata->l_map_len = sbuf.st_size; ! ! lcc->lc_info = info; ! lcc->lc_directive_count = info->directive_count; ! lcc->lc_large_count = info->large_count; ! ! for (z = 0; z < COLL_WEIGHTS_MAX; z++) { ! lcc->lc_directive[z] = info->directive[z]; ! lcc->lc_subst_count[z] = info->subst_count[z]; ! lcc->lc_pri_count[z] = info->pri_count[z]; ! lcc->lc_undef_pri[z] = info->undef_pri[z]; ! } ! ! lcc->lc_char_table = (void *)TMP; TMP += sizeof (collate_char_t) * (UCHAR_MAX + 1); ! for (z = 0; z < lcc->lc_directive_count; z++) { ! int count; ! if ((count = lcc->lc_subst_count[z]) > 0) { ! lcc->lc_subst_table[z] = (void *)TMP; ! TMP += count * sizeof (collate_subst_t); } else { ! lcc->lc_subst_table[z] = NULL; } } if (chains > 0) { ! lcc->lc_chain_table = (void *)TMP; TMP += chains * sizeof (collate_chain_t); } else ! lcc->lc_chain_table = NULL; ! lcc->lc_chain_count = chains; ! if (lcc->lc_large_count > 0) ! lcc->lc_large_table = (void *)TMP; else ! lcc->lc_large_table = NULL; ! return (ldata); } ! static const int32_t * ! substsearch(const struct lc_collate *lcc, const wchar_t key, int pass) { ! const collate_subst_t *p; ! int n = lcc->lc_subst_count[pass]; if (n == 0) return (NULL); ! if (pass >= lcc->lc_directive_count) return (NULL); if (!(key & COLLATE_SUBST_PRIORITY)) return (NULL); ! p = lcc->lc_subst_table[pass] + (key & ~COLLATE_SUBST_PRIORITY); assert(p->key == key); return (p->pri); } /*
*** 204,227 **** * Note: for performance reasons, we have expanded bsearch here. This avoids * function call overhead with each comparison. */ static collate_chain_t * ! chainsearch(const wchar_t *key, int *len) { int low; int high; int next, compar, l; collate_chain_t *p; collate_chain_t *tab; ! if (_collate_info->chain_count == 0) return (NULL); low = 0; ! high = _collate_info->chain_count - 1; ! tab = chain_pri_table; while (low <= high) { next = (low + high) / 2; p = tab + next; compar = *key - *p->str; --- 208,231 ---- * Note: for performance reasons, we have expanded bsearch here. This avoids * function call overhead with each comparison. */ static collate_chain_t * ! chainsearch(const struct lc_collate *lcc, const wchar_t *key, int *len) { int low; int high; int next, compar, l; collate_chain_t *p; collate_chain_t *tab; ! if (lcc->lc_info->chain_count == 0) return (NULL); low = 0; ! high = lcc->lc_info->chain_count - 1; ! tab = lcc->lc_chain_table; while (low <= high) { next = (low + high) / 2; p = tab + next; compar = *key - *p->str;
*** 240,258 **** } return (NULL); } static collate_large_t * ! largesearch(const wchar_t key) { int low = 0; ! int high = _collate_info->large_count - 1; int next, compar; collate_large_t *p; ! collate_large_t *tab = large_pri_table; ! if (_collate_info->large_count == 0) return (NULL); while (low <= high) { next = (low + high) / 2; p = tab + next; --- 244,262 ---- } return (NULL); } static collate_large_t * ! largesearch(const struct lc_collate *lcc, const wchar_t key) { int low = 0; ! int high = lcc->lc_info->large_count - 1; int next, compar; collate_large_t *p; ! collate_large_t *tab = lcc->lc_large_table; ! if (lcc->lc_info->large_count == 0) return (NULL); while (low <= high) { next = (low + high) / 2; p = tab + next;
*** 266,288 **** } return (NULL); } void ! _collate_lookup(const wchar_t *t, int *len, int *pri, int which, int **state) { collate_chain_t *p2; collate_large_t *match; - collate_info_t *info = _collate_info; int p, l; ! int *sptr; /* * If this is the "last" pass for the UNDEFINED, then * we just return the priority itself. */ ! if (which >= info->directive_count) { *pri = *t; *len = 1; *state = NULL; return; } --- 270,292 ---- } return (NULL); } void ! _collate_lookup(const struct lc_collate *lcc, const wchar_t *t, ! int *len, int *pri, int which, const int **state) { collate_chain_t *p2; collate_large_t *match; int p, l; ! const int *sptr; /* * If this is the "last" pass for the UNDEFINED, then * we just return the priority itself. */ ! if (which >= lcc->lc_directive_count) { *pri = *t; *len = 1; *state = NULL; return; }
*** 304,314 **** /* * Check for composites such as dipthongs that collate as a * single element (aka chains or collating-elements). */ ! if (((p2 = chainsearch(t, &l)) != NULL) && ((p = p2->pri[which]) >= 0)) { *len = l; *pri = p; --- 308,318 ---- /* * Check for composites such as dipthongs that collate as a * single element (aka chains or collating-elements). */ ! if (((p2 = chainsearch(lcc, t, &l)) != NULL) && ((p = p2->pri[which]) >= 0)) { *len = l; *pri = p;
*** 316,329 **** /* * Character is a small (8-bit) character. * We just look these up directly for speed. */ ! *pri = char_pri_table[*t].pri[which]; ! } else if ((info->large_count > 0) && ! ((match = largesearch(*t)) != NULL)) { /* * Character was found in the extended table. */ *pri = match->pri.pri[which]; --- 320,333 ---- /* * Character is a small (8-bit) character. * We just look these up directly for speed. */ ! *pri = lcc->lc_char_table[*t].pri[which]; ! } else if ((lcc->lc_info->large_count > 0) && ! ((match = largesearch(lcc, *t)) != NULL)) { /* * Character was found in the extended table. */ *pri = match->pri.pri[which];
*** 330,344 **** } else { /* * Character lacks a specific definition. */ ! if (info->directive[which] & DIRECTIVE_UNDEFINED) { /* Mask off sign bit to prevent ordering confusion. */ *pri = (*t & COLLATE_MAX_PRIORITY); } else { ! *pri = info->undef_pri[which]; } /* No substitutions for undefined characters! */ return; } --- 334,348 ---- } else { /* * Character lacks a specific definition. */ ! if (lcc->lc_directive[which] & DIRECTIVE_UNDEFINED) { /* Mask off sign bit to prevent ordering confusion. */ *pri = (*t & COLLATE_MAX_PRIORITY); } else { ! *pri = lcc->lc_undef_pri[which]; } /* No substitutions for undefined characters! */ return; }
*** 352,362 **** * to identify a single key value. In order for this to work, * its important that the priority assigned to a given element * to be substituted be unique for that level. The localedef * code ensures this for us. */ ! if ((sptr = substsearch(*pri, which)) != NULL) { if ((*pri = *sptr) != 0) { sptr++; *state = *sptr ? sptr : NULL; } } --- 356,366 ---- * to identify a single key value. In order for this to work, * its important that the priority assigned to a given element * to be substituted be unique for that level. The localedef * code ensures this for us. */ ! if ((sptr = substsearch(lcc, *pri, which)) != NULL) { if ((*pri = *sptr) != 0) { sptr++; *state = *sptr ? sptr : NULL; } }
*** 366,390 **** /* * This is the meaty part of wcsxfrm & strxfrm. Note that it does * NOT NULL terminate. That is left to the caller. */ size_t ! _collate_wxfrm(const wchar_t *src, wchar_t *xf, size_t room) { int pri; int len; const wchar_t *t; wchar_t *tr = NULL; int direc; int pass; ! int32_t *state; size_t want = 0; size_t need = 0; assert(src); ! for (pass = 0; pass <= _collate_info->directive_count; pass++) { state = NULL; if (pass != 0) { /* insert level separator from the previous pass */ --- 370,396 ---- /* * This is the meaty part of wcsxfrm & strxfrm. Note that it does * NOT NULL terminate. That is left to the caller. */ size_t ! _collate_wxfrm(const struct lc_collate *lcc, const wchar_t *src, wchar_t *xf, ! size_t room) { int pri; int len; const wchar_t *t; wchar_t *tr = NULL; int direc; int pass; ! const int32_t *state; size_t want = 0; size_t need = 0; + int ndir = lcc->lc_directive_count; assert(src); ! for (pass = 0; pass <= ndir; pass++) { state = NULL; if (pass != 0) { /* insert level separator from the previous pass */
*** 394,407 **** } want++; } /* special pass for undefined */ ! if (pass == _collate_info->directive_count) { direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED; } else { ! direc = _collate_info->directive[pass]; } t = src; if (direc & DIRECTIVE_BACKWARD) { --- 400,413 ---- } want++; } /* special pass for undefined */ ! if (pass == ndir) { direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED; } else { ! direc = lcc->lc_directive[pass]; } t = src; if (direc & DIRECTIVE_BACKWARD) {
*** 422,432 **** t = (const wchar_t *)tr; } if (direc & DIRECTIVE_POSITION) { while (*t || state) { ! _collate_lookup(t, &len, &pri, pass, &state); t += len; if (pri <= 0) { if (pri < 0) { errno = EINVAL; goto fail; --- 428,439 ---- t = (const wchar_t *)tr; } if (direc & DIRECTIVE_POSITION) { while (*t || state) { ! _collate_lookup(lcc, t, &len, &pri, pass, ! &state); t += len; if (pri <= 0) { if (pri < 0) { errno = EINVAL; goto fail;
*** 440,450 **** want++; need = want; } } else { while (*t || state) { ! _collate_lookup(t, &len, &pri, pass, &state); t += len; if (pri <= 0) { if (pri < 0) { errno = EINVAL; goto fail; --- 447,458 ---- want++; need = want; } } else { while (*t || state) { ! _collate_lookup(lcc, t, &len, &pri, pass, ! &state); t += len; if (pri <= 0) { if (pri < 0) { errno = EINVAL; goto fail;
*** 495,508 **** #define XFRM_SHIFT 6 #define XFRM_MASK ((1 << XFRM_SHIFT) - 1) #define XFRM_SEP ('.') /* chosen to be less than XFRM_OFFSET */ static int ! xfrm(unsigned char *p, int pri, int pass) { /* we use unsigned to ensure zero fill on right shift */ ! uint32_t val = (uint32_t)_collate_info->pri_count[pass]; int nc = 0; while (val) { *p = (pri & XFRM_MASK) + XFRM_OFFSET; pri >>= XFRM_SHIFT; --- 503,516 ---- #define XFRM_SHIFT 6 #define XFRM_MASK ((1 << XFRM_SHIFT) - 1) #define XFRM_SEP ('.') /* chosen to be less than XFRM_OFFSET */ static int ! xfrm(locale_t loc, unsigned char *p, int pri, int pass) { /* we use unsigned to ensure zero fill on right shift */ ! uint32_t val = (uint32_t)loc->collate->lc_pri_count[pass]; int nc = 0; while (val) { *p = (pri & XFRM_MASK) + XFRM_OFFSET; pri >>= XFRM_SHIFT;
*** 512,538 **** } return (nc); } size_t ! _collate_sxfrm(const wchar_t *src, char *xf, size_t room) { int pri; int len; const wchar_t *t; wchar_t *tr = NULL; int direc; int pass; ! int32_t *state; size_t want = 0; size_t need = 0; int b; uint8_t buf[XFRM_BYTES]; assert(src); ! for (pass = 0; pass <= _collate_info->directive_count; pass++) { state = NULL; if (pass != 0) { /* insert level separator from the previous pass */ --- 520,548 ---- } return (nc); } size_t ! _collate_sxfrm(const wchar_t *src, char *xf, size_t room, locale_t loc) { int pri; int len; const wchar_t *t; wchar_t *tr = NULL; int direc; int pass; ! const int32_t *state; size_t want = 0; size_t need = 0; int b; uint8_t buf[XFRM_BYTES]; + const struct lc_collate *lcc = loc->collate; + int ndir = lcc->lc_directive_count; assert(src); ! for (pass = 0; pass <= ndir; pass++) { state = NULL; if (pass != 0) { /* insert level separator from the previous pass */
*** 542,555 **** } want++; } /* special pass for undefined */ ! if (pass == _collate_info->directive_count) { direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED; } else { ! direc = _collate_info->directive[pass]; } t = src; if (direc & DIRECTIVE_BACKWARD) { --- 552,565 ---- } want++; } /* special pass for undefined */ ! if (pass == ndir) { direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED; } else { ! direc = lcc->lc_directive[pass]; } t = src; if (direc & DIRECTIVE_BACKWARD) {
*** 571,591 **** } if (direc & DIRECTIVE_POSITION) { while (*t || state) { ! _collate_lookup(t, &len, &pri, pass, &state); t += len; if (pri <= 0) { if (pri < 0) { errno = EINVAL; goto fail; } pri = COLLATE_MAX_PRIORITY; } ! b = xfrm(buf, pri, pass); want += b; if (room) { while (b) { b--; if (room) { --- 581,602 ---- } if (direc & DIRECTIVE_POSITION) { while (*t || state) { ! _collate_lookup(lcc, t, &len, &pri, pass, ! &state); t += len; if (pri <= 0) { if (pri < 0) { errno = EINVAL; goto fail; } pri = COLLATE_MAX_PRIORITY; } ! b = xfrm(loc, buf, pri, pass); want += b; if (room) { while (b) { b--; if (room) {
*** 596,616 **** } need = want; } } else { while (*t || state) { ! _collate_lookup(t, &len, &pri, pass, &state); t += len; if (pri <= 0) { if (pri < 0) { errno = EINVAL; goto fail; } continue; } ! b = xfrm(buf, pri, pass); want += b; if (room) { while (b) { b--; --- 607,628 ---- } need = want; } } else { while (*t || state) { ! _collate_lookup(lcc, t, &len, &pri, pass, ! &state); t += len; if (pri <= 0) { if (pri < 0) { errno = EINVAL; goto fail; } continue; } ! b = xfrm(loc, buf, pri, pass); want += b; if (room) { while (b) { b--;