Print this page
2964 need POSIX 2008 locale object support
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Gordon Ross <gordon.ross@nexenta.com>
Approved by: TBD
*** 1,7 ****
/*
! * Copright 2010 Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 1995 Alex Tatmanjants <alex@elvisti.kiev.ua>
* at Electronni Visti IA, Kiev, Ukraine.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
--- 1,8 ----
/*
! * Copyright 2014 Garrett D'Amore <garrett@damore.org>
! * Copyright 2010 Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 1995 Alex Tatmanjants <alex@elvisti.kiev.ua>
* at Electronni Visti IA, Kiev, Ukraine.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
*** 42,126 ****
#include <sys/stat.h>
#include <sys/mman.h>
#include "collate.h"
#include "setlocale.h"
- #include "ldpart.h"
/*
* See the comments in usr/src/cmd/localedef/collate.c for further
* information. It would also be very helpful to have a copy of the
* POSIX standard for collation (in the locale format manual page)
* handy (www.opengroup.org).
*/
! static collate_subst_t *subst_table[COLL_WEIGHTS_MAX];
! static collate_char_t *char_pri_table;
! static collate_large_t *large_pri_table;
! static collate_chain_t *chain_pri_table;
! static char *cache = NULL;
! static size_t cachesz;
! static char collate_encoding[ENCODING_LEN + 1];
! /* Exposed externally to other parts of libc. */
! collate_info_t *_collate_info;
! int _collate_load_error = 1;
! int
! _collate_load_tables(const char *encoding)
{
int i, chains, z;
char buf[PATH_MAX];
char *TMP;
char *map;
collate_info_t *info;
struct stat sbuf;
int fd;
- /* 'encoding' must be already checked. */
- if (strcmp(encoding, "C") == 0 || strcmp(encoding, "POSIX") == 0) {
- _collate_load_error = 1;
- return (_LDP_CACHE);
- }
-
/*
- * If the locale name is the same as our cache, use the cache.
- */
- if (cache && (strncmp(encoding, collate_encoding, ENCODING_LEN) == 0)) {
- _collate_load_error = 0;
- return (_LDP_CACHE);
- }
-
- /*
* Slurp the locale file into the cache.
*/
(void) snprintf(buf, sizeof (buf), "%s/%s/LC_COLLATE/LCL_DATA",
! _PathLocale, encoding);
! if ((fd = open(buf, O_RDONLY)) < 0)
! return (_LDP_ERROR);
if (fstat(fd, &sbuf) < 0) {
(void) close(fd);
! return (_LDP_ERROR);
}
if (sbuf.st_size < (COLLATE_STR_LEN + sizeof (info))) {
(void) close(fd);
errno = EINVAL;
! return (_LDP_ERROR);
}
map = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
(void) close(fd);
if ((TMP = map) == NULL) {
! return (_LDP_ERROR);
}
if (strncmp(TMP, COLLATE_VERSION, COLLATE_STR_LEN) != 0) {
(void) munmap(map, sbuf.st_size);
errno = EINVAL;
! return (_LDP_ERROR);
}
TMP += COLLATE_STR_LEN;
info = (void *)TMP;
TMP += sizeof (*info);
--- 43,119 ----
#include <sys/stat.h>
#include <sys/mman.h>
#include "collate.h"
#include "setlocale.h"
/*
* See the comments in usr/src/cmd/localedef/collate.c for further
* information. It would also be very helpful to have a copy of the
* POSIX standard for collation (in the locale format manual page)
* handy (www.opengroup.org).
*/
! /*
! * POSIX uses empty tables and falls down to strcmp.
! */
! struct lc_collate lc_collate_posix = {
! .lc_is_posix = 1,
! };
! struct locdata __posix_collate_locdata = {
! .l_lname = "C",
! .l_refcnt = (uint32_t)-1,
! .l_data = { &lc_collate_posix }
! };
! struct locdata *
! __lc_collate_load(const char *locname)
{
int i, chains, z;
char buf[PATH_MAX];
char *TMP;
char *map;
collate_info_t *info;
struct stat sbuf;
int fd;
+ struct locdata *ldata;
+ struct lc_collate *lcc;
/*
* Slurp the locale file into the cache.
*/
(void) snprintf(buf, sizeof (buf), "%s/%s/LC_COLLATE/LCL_DATA",
! _PathLocale, locname);
! if ((fd = open(buf, O_RDONLY)) < 0) {
! errno = EINVAL;
! return (NULL);
! }
if (fstat(fd, &sbuf) < 0) {
(void) close(fd);
! errno = EINVAL;
! return (NULL);
}
if (sbuf.st_size < (COLLATE_STR_LEN + sizeof (info))) {
(void) close(fd);
errno = EINVAL;
! return (NULL);
}
map = mmap(NULL, sbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
(void) close(fd);
if ((TMP = map) == NULL) {
! errno = EINVAL;
! return (NULL);
}
if (strncmp(TMP, COLLATE_VERSION, COLLATE_STR_LEN) != 0) {
(void) munmap(map, sbuf.st_size);
errno = EINVAL;
! return (NULL);
}
TMP += COLLATE_STR_LEN;
info = (void *)TMP;
TMP += sizeof (*info);
*** 128,203 ****
if ((info->directive_count < 1) ||
(info->directive_count >= COLL_WEIGHTS_MAX) ||
((chains = info->chain_count) < 0)) {
(void) munmap(map, sbuf.st_size);
errno = EINVAL;
! return (_LDP_ERROR);
}
i = (sizeof (collate_char_t) * (UCHAR_MAX + 1)) +
(sizeof (collate_chain_t) * chains) +
(sizeof (collate_large_t) * info->large_count);
! for (z = 0; z < (info->directive_count); z++) {
i += sizeof (collate_subst_t) * info->subst_count[z];
}
if (i != (sbuf.st_size - (TMP - map))) {
(void) munmap(map, sbuf.st_size);
errno = EINVAL;
! return (_LDP_ERROR);
}
! char_pri_table = (void *)TMP;
TMP += sizeof (collate_char_t) * (UCHAR_MAX + 1);
! for (z = 0; z < info->directive_count; z++) {
! if (info->subst_count[z] > 0) {
! subst_table[z] = (void *)TMP;
! TMP += info->subst_count[z] * sizeof (collate_subst_t);
} else {
! subst_table[z] = NULL;
}
}
if (chains > 0) {
! chain_pri_table = (void *)TMP;
TMP += chains * sizeof (collate_chain_t);
} else
! chain_pri_table = NULL;
! if (info->large_count > 0)
! large_pri_table = (void *)TMP;
else
! large_pri_table = NULL;
! (void) strlcpy(collate_encoding, encoding, ENCODING_LEN);
! _collate_info = info;
!
! if (cache)
! (void) munmap(cache, cachesz);
!
! cache = map;
! cachesz = sbuf.st_size;
! _collate_load_error = 0;
!
! return (_LDP_LOADED);
}
! static int32_t *
! substsearch(const wchar_t key, int pass)
{
! collate_subst_t *p;
! int n = _collate_info->subst_count[pass];
if (n == 0)
return (NULL);
! if (pass >= _collate_info->directive_count)
return (NULL);
if (!(key & COLLATE_SUBST_PRIORITY))
return (NULL);
! p = subst_table[pass] + (key & ~COLLATE_SUBST_PRIORITY);
assert(p->key == key);
return (p->pri);
}
/*
--- 121,208 ----
if ((info->directive_count < 1) ||
(info->directive_count >= COLL_WEIGHTS_MAX) ||
((chains = info->chain_count) < 0)) {
(void) munmap(map, sbuf.st_size);
errno = EINVAL;
! return (NULL);
}
i = (sizeof (collate_char_t) * (UCHAR_MAX + 1)) +
(sizeof (collate_chain_t) * chains) +
(sizeof (collate_large_t) * info->large_count);
! for (z = 0; z < info->directive_count; z++) {
i += sizeof (collate_subst_t) * info->subst_count[z];
}
if (i != (sbuf.st_size - (TMP - map))) {
(void) munmap(map, sbuf.st_size);
errno = EINVAL;
! return (NULL);
}
!
! if ((ldata = __locdata_alloc(locname, sizeof (*lcc))) == NULL) {
! (void) munmap(map, sbuf.st_size);
! return (NULL);
! }
! lcc = ldata->l_data[0];
! ldata->l_map = map;
! ldata->l_map_len = sbuf.st_size;
!
! lcc->lc_info = info;
! lcc->lc_directive_count = info->directive_count;
! lcc->lc_large_count = info->large_count;
!
! for (z = 0; z < COLL_WEIGHTS_MAX; z++) {
! lcc->lc_directive[z] = info->directive[z];
! lcc->lc_subst_count[z] = info->subst_count[z];
! lcc->lc_pri_count[z] = info->pri_count[z];
! lcc->lc_undef_pri[z] = info->undef_pri[z];
! }
!
! lcc->lc_char_table = (void *)TMP;
TMP += sizeof (collate_char_t) * (UCHAR_MAX + 1);
! for (z = 0; z < lcc->lc_directive_count; z++) {
! int count;
! if ((count = lcc->lc_subst_count[z]) > 0) {
! lcc->lc_subst_table[z] = (void *)TMP;
! TMP += count * sizeof (collate_subst_t);
} else {
! lcc->lc_subst_table[z] = NULL;
}
}
if (chains > 0) {
! lcc->lc_chain_table = (void *)TMP;
TMP += chains * sizeof (collate_chain_t);
} else
! lcc->lc_chain_table = NULL;
! lcc->lc_chain_count = chains;
! if (lcc->lc_large_count > 0)
! lcc->lc_large_table = (void *)TMP;
else
! lcc->lc_large_table = NULL;
! return (ldata);
}
! static const int32_t *
! substsearch(const struct lc_collate *lcc, const wchar_t key, int pass)
{
! const collate_subst_t *p;
! int n = lcc->lc_subst_count[pass];
if (n == 0)
return (NULL);
! if (pass >= lcc->lc_directive_count)
return (NULL);
if (!(key & COLLATE_SUBST_PRIORITY))
return (NULL);
! p = lcc->lc_subst_table[pass] + (key & ~COLLATE_SUBST_PRIORITY);
assert(p->key == key);
return (p->pri);
}
/*
*** 204,227 ****
* Note: for performance reasons, we have expanded bsearch here. This avoids
* function call overhead with each comparison.
*/
static collate_chain_t *
! chainsearch(const wchar_t *key, int *len)
{
int low;
int high;
int next, compar, l;
collate_chain_t *p;
collate_chain_t *tab;
! if (_collate_info->chain_count == 0)
return (NULL);
low = 0;
! high = _collate_info->chain_count - 1;
! tab = chain_pri_table;
while (low <= high) {
next = (low + high) / 2;
p = tab + next;
compar = *key - *p->str;
--- 209,232 ----
* Note: for performance reasons, we have expanded bsearch here. This avoids
* function call overhead with each comparison.
*/
static collate_chain_t *
! chainsearch(const struct lc_collate *lcc, const wchar_t *key, int *len)
{
int low;
int high;
int next, compar, l;
collate_chain_t *p;
collate_chain_t *tab;
! if (lcc->lc_info->chain_count == 0)
return (NULL);
low = 0;
! high = lcc->lc_info->chain_count - 1;
! tab = lcc->lc_chain_table;
while (low <= high) {
next = (low + high) / 2;
p = tab + next;
compar = *key - *p->str;
*** 240,258 ****
}
return (NULL);
}
static collate_large_t *
! largesearch(const wchar_t key)
{
int low = 0;
! int high = _collate_info->large_count - 1;
int next, compar;
collate_large_t *p;
! collate_large_t *tab = large_pri_table;
! if (_collate_info->large_count == 0)
return (NULL);
while (low <= high) {
next = (low + high) / 2;
p = tab + next;
--- 245,263 ----
}
return (NULL);
}
static collate_large_t *
! largesearch(const struct lc_collate *lcc, const wchar_t key)
{
int low = 0;
! int high = lcc->lc_info->large_count - 1;
int next, compar;
collate_large_t *p;
! collate_large_t *tab = lcc->lc_large_table;
! if (lcc->lc_info->large_count == 0)
return (NULL);
while (low <= high) {
next = (low + high) / 2;
p = tab + next;
*** 266,288 ****
}
return (NULL);
}
void
! _collate_lookup(const wchar_t *t, int *len, int *pri, int which, int **state)
{
collate_chain_t *p2;
collate_large_t *match;
- collate_info_t *info = _collate_info;
int p, l;
! int *sptr;
/*
* If this is the "last" pass for the UNDEFINED, then
* we just return the priority itself.
*/
! if (which >= info->directive_count) {
*pri = *t;
*len = 1;
*state = NULL;
return;
}
--- 271,293 ----
}
return (NULL);
}
void
! _collate_lookup(const struct lc_collate *lcc, const wchar_t *t,
! int *len, int *pri, int which, const int **state)
{
collate_chain_t *p2;
collate_large_t *match;
int p, l;
! const int *sptr;
/*
* If this is the "last" pass for the UNDEFINED, then
* we just return the priority itself.
*/
! if (which >= lcc->lc_directive_count) {
*pri = *t;
*len = 1;
*state = NULL;
return;
}
*** 304,314 ****
/*
* Check for composites such as dipthongs that collate as a
* single element (aka chains or collating-elements).
*/
! if (((p2 = chainsearch(t, &l)) != NULL) &&
((p = p2->pri[which]) >= 0)) {
*len = l;
*pri = p;
--- 309,319 ----
/*
* Check for composites such as dipthongs that collate as a
* single element (aka chains or collating-elements).
*/
! if (((p2 = chainsearch(lcc, t, &l)) != NULL) &&
((p = p2->pri[which]) >= 0)) {
*len = l;
*pri = p;
*** 316,329 ****
/*
* Character is a small (8-bit) character.
* We just look these up directly for speed.
*/
! *pri = char_pri_table[*t].pri[which];
! } else if ((info->large_count > 0) &&
! ((match = largesearch(*t)) != NULL)) {
/*
* Character was found in the extended table.
*/
*pri = match->pri.pri[which];
--- 321,334 ----
/*
* Character is a small (8-bit) character.
* We just look these up directly for speed.
*/
! *pri = lcc->lc_char_table[*t].pri[which];
! } else if ((lcc->lc_info->large_count > 0) &&
! ((match = largesearch(lcc, *t)) != NULL)) {
/*
* Character was found in the extended table.
*/
*pri = match->pri.pri[which];
*** 330,344 ****
} else {
/*
* Character lacks a specific definition.
*/
! if (info->directive[which] & DIRECTIVE_UNDEFINED) {
/* Mask off sign bit to prevent ordering confusion. */
*pri = (*t & COLLATE_MAX_PRIORITY);
} else {
! *pri = info->undef_pri[which];
}
/* No substitutions for undefined characters! */
return;
}
--- 335,349 ----
} else {
/*
* Character lacks a specific definition.
*/
! if (lcc->lc_directive[which] & DIRECTIVE_UNDEFINED) {
/* Mask off sign bit to prevent ordering confusion. */
*pri = (*t & COLLATE_MAX_PRIORITY);
} else {
! *pri = lcc->lc_undef_pri[which];
}
/* No substitutions for undefined characters! */
return;
}
*** 352,362 ****
* to identify a single key value. In order for this to work,
* its important that the priority assigned to a given element
* to be substituted be unique for that level. The localedef
* code ensures this for us.
*/
! if ((sptr = substsearch(*pri, which)) != NULL) {
if ((*pri = *sptr) != 0) {
sptr++;
*state = *sptr ? sptr : NULL;
}
}
--- 357,367 ----
* to identify a single key value. In order for this to work,
* its important that the priority assigned to a given element
* to be substituted be unique for that level. The localedef
* code ensures this for us.
*/
! if ((sptr = substsearch(lcc, *pri, which)) != NULL) {
if ((*pri = *sptr) != 0) {
sptr++;
*state = *sptr ? sptr : NULL;
}
}
*** 366,390 ****
/*
* This is the meaty part of wcsxfrm & strxfrm. Note that it does
* NOT NULL terminate. That is left to the caller.
*/
size_t
! _collate_wxfrm(const wchar_t *src, wchar_t *xf, size_t room)
{
int pri;
int len;
const wchar_t *t;
wchar_t *tr = NULL;
int direc;
int pass;
! int32_t *state;
size_t want = 0;
size_t need = 0;
assert(src);
! for (pass = 0; pass <= _collate_info->directive_count; pass++) {
state = NULL;
if (pass != 0) {
/* insert level separator from the previous pass */
--- 371,397 ----
/*
* This is the meaty part of wcsxfrm & strxfrm. Note that it does
* NOT NULL terminate. That is left to the caller.
*/
size_t
! _collate_wxfrm(const struct lc_collate *lcc, const wchar_t *src, wchar_t *xf,
! size_t room)
{
int pri;
int len;
const wchar_t *t;
wchar_t *tr = NULL;
int direc;
int pass;
! const int32_t *state;
size_t want = 0;
size_t need = 0;
+ int ndir = lcc->lc_directive_count;
assert(src);
! for (pass = 0; pass <= ndir; pass++) {
state = NULL;
if (pass != 0) {
/* insert level separator from the previous pass */
*** 394,407 ****
}
want++;
}
/* special pass for undefined */
! if (pass == _collate_info->directive_count) {
direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED;
} else {
! direc = _collate_info->directive[pass];
}
t = src;
if (direc & DIRECTIVE_BACKWARD) {
--- 401,414 ----
}
want++;
}
/* special pass for undefined */
! if (pass == ndir) {
direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED;
} else {
! direc = lcc->lc_directive[pass];
}
t = src;
if (direc & DIRECTIVE_BACKWARD) {
*** 422,432 ****
t = (const wchar_t *)tr;
}
if (direc & DIRECTIVE_POSITION) {
while (*t || state) {
! _collate_lookup(t, &len, &pri, pass, &state);
t += len;
if (pri <= 0) {
if (pri < 0) {
errno = EINVAL;
goto fail;
--- 429,440 ----
t = (const wchar_t *)tr;
}
if (direc & DIRECTIVE_POSITION) {
while (*t || state) {
! _collate_lookup(lcc, t, &len, &pri, pass,
! &state);
t += len;
if (pri <= 0) {
if (pri < 0) {
errno = EINVAL;
goto fail;
*** 440,450 ****
want++;
need = want;
}
} else {
while (*t || state) {
! _collate_lookup(t, &len, &pri, pass, &state);
t += len;
if (pri <= 0) {
if (pri < 0) {
errno = EINVAL;
goto fail;
--- 448,459 ----
want++;
need = want;
}
} else {
while (*t || state) {
! _collate_lookup(lcc, t, &len, &pri, pass,
! &state);
t += len;
if (pri <= 0) {
if (pri < 0) {
errno = EINVAL;
goto fail;
*** 495,508 ****
#define XFRM_SHIFT 6
#define XFRM_MASK ((1 << XFRM_SHIFT) - 1)
#define XFRM_SEP ('.') /* chosen to be less than XFRM_OFFSET */
static int
! xfrm(unsigned char *p, int pri, int pass)
{
/* we use unsigned to ensure zero fill on right shift */
! uint32_t val = (uint32_t)_collate_info->pri_count[pass];
int nc = 0;
while (val) {
*p = (pri & XFRM_MASK) + XFRM_OFFSET;
pri >>= XFRM_SHIFT;
--- 504,517 ----
#define XFRM_SHIFT 6
#define XFRM_MASK ((1 << XFRM_SHIFT) - 1)
#define XFRM_SEP ('.') /* chosen to be less than XFRM_OFFSET */
static int
! xfrm(locale_t loc, unsigned char *p, int pri, int pass)
{
/* we use unsigned to ensure zero fill on right shift */
! uint32_t val = (uint32_t)loc->collate->lc_pri_count[pass];
int nc = 0;
while (val) {
*p = (pri & XFRM_MASK) + XFRM_OFFSET;
pri >>= XFRM_SHIFT;
*** 512,538 ****
}
return (nc);
}
size_t
! _collate_sxfrm(const wchar_t *src, char *xf, size_t room)
{
int pri;
int len;
const wchar_t *t;
wchar_t *tr = NULL;
int direc;
int pass;
! int32_t *state;
size_t want = 0;
size_t need = 0;
int b;
uint8_t buf[XFRM_BYTES];
assert(src);
! for (pass = 0; pass <= _collate_info->directive_count; pass++) {
state = NULL;
if (pass != 0) {
/* insert level separator from the previous pass */
--- 521,549 ----
}
return (nc);
}
size_t
! _collate_sxfrm(const wchar_t *src, char *xf, size_t room, locale_t loc)
{
int pri;
int len;
const wchar_t *t;
wchar_t *tr = NULL;
int direc;
int pass;
! const int32_t *state;
size_t want = 0;
size_t need = 0;
int b;
uint8_t buf[XFRM_BYTES];
+ const struct lc_collate *lcc = loc->collate;
+ int ndir = lcc->lc_directive_count;
assert(src);
! for (pass = 0; pass <= ndir; pass++) {
state = NULL;
if (pass != 0) {
/* insert level separator from the previous pass */
*** 542,555 ****
}
want++;
}
/* special pass for undefined */
! if (pass == _collate_info->directive_count) {
direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED;
} else {
! direc = _collate_info->directive[pass];
}
t = src;
if (direc & DIRECTIVE_BACKWARD) {
--- 553,566 ----
}
want++;
}
/* special pass for undefined */
! if (pass == ndir) {
direc = DIRECTIVE_FORWARD | DIRECTIVE_UNDEFINED;
} else {
! direc = lcc->lc_directive[pass];
}
t = src;
if (direc & DIRECTIVE_BACKWARD) {
*** 571,591 ****
}
if (direc & DIRECTIVE_POSITION) {
while (*t || state) {
! _collate_lookup(t, &len, &pri, pass, &state);
t += len;
if (pri <= 0) {
if (pri < 0) {
errno = EINVAL;
goto fail;
}
pri = COLLATE_MAX_PRIORITY;
}
! b = xfrm(buf, pri, pass);
want += b;
if (room) {
while (b) {
b--;
if (room) {
--- 582,603 ----
}
if (direc & DIRECTIVE_POSITION) {
while (*t || state) {
! _collate_lookup(lcc, t, &len, &pri, pass,
! &state);
t += len;
if (pri <= 0) {
if (pri < 0) {
errno = EINVAL;
goto fail;
}
pri = COLLATE_MAX_PRIORITY;
}
! b = xfrm(loc, buf, pri, pass);
want += b;
if (room) {
while (b) {
b--;
if (room) {
*** 596,616 ****
}
need = want;
}
} else {
while (*t || state) {
! _collate_lookup(t, &len, &pri, pass, &state);
t += len;
if (pri <= 0) {
if (pri < 0) {
errno = EINVAL;
goto fail;
}
continue;
}
! b = xfrm(buf, pri, pass);
want += b;
if (room) {
while (b) {
b--;
--- 608,629 ----
}
need = want;
}
} else {
while (*t || state) {
! _collate_lookup(lcc, t, &len, &pri, pass,
! &state);
t += len;
if (pri <= 0) {
if (pri < 0) {
errno = EINVAL;
goto fail;
}
continue;
}
! b = xfrm(loc, buf, pri, pass);
want += b;
if (room) {
while (b) {
b--;