Print this page
9083 replace regex implementation with tre
   1 /*
   2  * Copyright 2010 Nexenta Systems, Inc.  All rights reserved.
   3  * Copyright (c) 1992, 1993, 1994 Henry Spencer.
   4  * Copyright (c) 1992, 1993, 1994
   5  *      The Regents of the University of California.  All rights reserved.
   6  *
   7  * This code is derived from software contributed to Berkeley by
   8  * Henry Spencer.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:

  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.

  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. Neither the name of the University nor the names of its contributors
  19  *    may be used to endorse or promote products derived from this software
  20  *    without specific prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  23  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  26  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  27  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  28  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  29  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  30  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  31  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  */
  34 
  35 /*
  36  * the outer shell of regexec()
  37  *
  38  * This file includes engine.c three times, after muchos fiddling with the
  39  * macros that code uses.  This lets the same code operate on two different
  40  * representations for state sets and characters.
  41  */
  42 #include "lint.h"
  43 #include "file64.h"
  44 #include <sys/types.h>
  45 #include <stdio.h>
  46 #include <stdlib.h>
  47 #include <string.h>
  48 #include <limits.h>
  49 #include <ctype.h>
  50 #include <regex.h>
  51 #include <wchar.h>
  52 #include <wctype.h>
  53 #include <note.h>
  54 #include <assert.h>
  55 
  56 #include "utils.h"
  57 #include "regex2.h"

  58 
  59 /* we want _NOTE, but not NOTE (which collides with our own use) */
  60 #undef  NOTE
  61 
  62 static size_t
  63 xmbrtowc(wint_t *wi, const char *s, size_t n, mbstate_t *mbs, wint_t dummy)



  64 {
  65         size_t nr;
  66         wchar_t wc;
  67 
  68         nr = mbrtowc(&wc, s, n, mbs);
  69         if (wi != NULL)
  70                 *wi = wc;
  71         if (nr == 0)
  72                 return (1);
  73         else if (nr == (size_t)-1 || nr == (size_t)-2) {
  74                 (void) memset(mbs, 0, sizeof (*mbs));
  75                 if (wi != NULL)
  76                         *wi = dummy;
  77                 return (1);
  78         } else
  79                 return (nr);

































































  80 }
  81 
  82 static size_t
  83 xmbrtowc_dummy(wint_t *wi, const char *s, size_t n, mbstate_t *mbs,
  84     wint_t dummy)


  85 {
  86         _NOTE(ARGUNUSED(n));
  87         _NOTE(ARGUNUSED(mbs));
  88         _NOTE(ARGUNUSED(dummy));
  89 
  90         if (wi != NULL)
  91                 *wi = (unsigned char)*s;
  92         return (1);
  93 }
  94 
  95 /* macros for manipulating states, small version */
  96 #define states  long
  97 #define states1 states          /* for later use in regexec() decision */
  98 #define CLEAR(v)        ((v) = 0)
  99 #define SET0(v, n)      ((v) &= ~((unsigned long)1 << (n)))
 100 #define SET1(v, n)      ((v) |= (unsigned long)1 << (n))
 101 #define ISSET(v, n)     (((v) & ((unsigned long)1 << (n))) != 0)
 102 #define ASSIGN(d, s)    ((d) = (s))
 103 #define EQ(a, b)        ((a) == (b))
 104 #define STATEVARS       long dummy      /* dummy version */
 105 #define STATESETUP(m, n)        /* nothing */
 106 #define STATETEARDOWN(m)        /* nothing */
 107 #define SETUP(v)        ((v) = 0)
 108 #define onestate        long
 109 #define INIT(o, n)      ((o) = (unsigned long)1 << (n))
 110 #define INC(o)  ((o) <<= 1)
 111 #define ISSTATEIN(v, o) (((v) & (o)) != 0)
 112 /* some abbreviations; note that some of these know variable names! */
 113 /* do "if I'm here, I can also be there" etc without branches */
 114 #define FWD(dst, src, n)        ((dst) |= ((unsigned long)(src)&(here)) << (n))
 115 #define BACK(dst, src, n)       ((dst) |= ((unsigned long)(src)&(here)) >> (n))
 116 #define ISSETBACK(v, n) (((v) & ((unsigned long)here >> (n))) != 0)
 117 /* no multibyte support */
 118 #define XMBRTOWC        xmbrtowc_dummy
 119 #define ZAPSTATE(mbs)   ((void)(mbs))
 120 /* function names */
 121 #define SNAMES                  /* engine.c looks after details */
 122 
 123 #include "engine.c"





















 124 
 125 /* now undo things */
 126 #undef  states
 127 #undef  CLEAR
 128 #undef  SET0
 129 #undef  SET1
 130 #undef  ISSET
 131 #undef  ASSIGN
 132 #undef  EQ
 133 #undef  STATEVARS
 134 #undef  STATESETUP
 135 #undef  STATETEARDOWN
 136 #undef  SETUP
 137 #undef  onestate
 138 #undef  INIT
 139 #undef  INC
 140 #undef  ISSTATEIN
 141 #undef  FWD
 142 #undef  BACK
 143 #undef  ISSETBACK
 144 #undef  SNAMES
 145 #undef  XMBRTOWC
 146 #undef  ZAPSTATE
 147 
 148 /* macros for manipulating states, large version */
 149 #define states  char *
 150 #define CLEAR(v)        (void) memset(v, 0, m->g->nstates)
 151 #define SET0(v, n)      ((v)[n] = 0)
 152 #define SET1(v, n)      ((v)[n] = 1)
 153 #define ISSET(v, n)     ((v)[n])
 154 #define ASSIGN(d, s)    (void) memcpy(d, s, m->g->nstates)
 155 #define EQ(a, b)        (memcmp(a, b, m->g->nstates) == 0)
 156 #define STATEVARS       long vn; char *space
 157 #define STATESETUP(m, nv) { (m)->space = malloc((nv)*(m)->g->nstates); \
 158         if ((m)->space == NULL) \
 159                 return (REG_ESPACE); \
 160         (m)->vn = 0; }
 161 #define STATETEARDOWN(m)        { free((m)->space); }
 162 #define SETUP(v)        ((v) = &m->space[m->vn++ * m->g->nstates])
 163 #define onestate        long
 164 #define INIT(o, n)      ((o) = (n))
 165 #define INC(o)  ((o)++)
 166 #define ISSTATEIN(v, o) ((v)[o])
 167 /* some abbreviations; note that some of these know variable names! */
 168 /* do "if I'm here, I can also be there" etc without branches */
 169 #define FWD(dst, src, n)        ((dst)[here+(n)] |= (src)[here])
 170 #define BACK(dst, src, n)       ((dst)[here-(n)] |= (src)[here])
 171 #define ISSETBACK(v, n) ((v)[here - (n)])
 172 /* no multibyte support */
 173 #define XMBRTOWC        xmbrtowc_dummy
 174 #define ZAPSTATE(mbs)   ((void)(mbs))
 175 /* function names */
 176 #define LNAMES                  /* flag */
 177 
 178 #include "engine.c"






 179 
 180 /* multibyte character & large states version */
 181 #undef  LNAMES
 182 #undef  XMBRTOWC
 183 #undef  ZAPSTATE
 184 #define XMBRTOWC        xmbrtowc
 185 #define ZAPSTATE(mbs)   (void) memset((mbs), 0, sizeof (*(mbs)))
 186 #define MNAMES
 187 
 188 #include "engine.c"

 189 
 190 /*
 191  * regexec - interface for matching
 192  *
 193  * We put this here so we can exploit knowledge of the state representation
 194  * when choosing which matcher to call.  Also, by this point the matchers
 195  * have been prototyped.
 196  */
 197 int                             /* 0 success, REG_NOMATCH failure */
 198 regexec(const regex_t *_RESTRICT_KYWD preg, const char *_RESTRICT_KYWD string,
 199     size_t nmatch, regmatch_t pmatch[_RESTRICT_KYWD], int eflags)
 200 {
 201         struct re_guts *g = preg->re_g;
 202 #ifdef REDEBUG
 203 #define GOODFLAGS(f)    (f)



































 204 #else
 205 #ifdef  REG_STARTEND
 206 #define GOODFLAGS(f)    ((f)&(REG_NOTBOL|REG_NOTEOL|REG_STARTEND))
 207 #else
 208 #define GOODFLAGS(f)    ((f)&(REG_NOTBOL|REG_NOTEOL))
 209 #endif
 210 #endif










 211 
 212         if (preg->re_magic != MAGIC1 || g->magic != MAGIC2)
 213                 return (REG_BADPAT);
 214         assert(!(g->iflags&BAD));
 215         if (g->iflags&BAD)               /* backstop for no-debug case */
 216                 return (REG_BADPAT);
 217         eflags = GOODFLAGS(eflags);

















 218 
 219         if (MB_CUR_MAX > 1)
 220                 return (mmatcher(g, string, nmatch, pmatch, eflags));
 221         else if (g->nstates <= CHAR_BIT*sizeof (states1) && !(eflags&REG_LARGE))
 222                 return (smatcher(g, string, nmatch, pmatch, eflags));
 223         else
 224                 return (lmatcher(g, string, nmatch, pmatch, eflags));











 225 }
   1 /*
   2  * Copyright (c) 2001-2009 Ville Laurikari <vl@iki.fi>
   3  * All rights reserved.


   4  *



   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  *
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.



  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
  17  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19  * A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT
  20  * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28 
  29 /*
  30   tre_regexec.c - TRE POSIX compatible matching functions (and more).
  31 */
  32 
  33 #include <assert.h>
  34 #include <limits.h>





  35 #include <stdlib.h>
  36 #include <string.h>



  37 #include <wchar.h>
  38 #include <wctype.h>


  39 
  40 #include "tre-internal.h"
  41 #include "tre-match-utils.h"
  42 #include "tre.h"
  43 
  44 /* For each tre_last_matched_t in the lm array, find the last matched branch by
  45    comparing the touch value of the cmp_tag's.  For all other branches, reset
  46    the corresponding tags.  If reset_all is non-zero, reset all tags in all
  47    branches.  Recurse into the nested last matched structures, clearing tags as
  48    apprpriate. */
  49 static void
  50 tre_reset_last_matched_branches(tre_tag_t *tags, const tre_last_matched_t *lm,
  51                                 int n, int start_tag, int reset_all)
  52 {
  53   int max, i, reset;
  54   tre_last_matched_branch_t *b;
  55 
  56   DPRINT(("tre_reset_last_matched_branches: n=%d start_tag=%d reset_all=%d\n",
  57           n, start_tag, reset_all));
  58   for (; n-- > 0; lm++)
  59     {
  60       if (lm->n_branches == 1)
  61         {
  62           b = lm->branches;
  63           if (start_tag > 0)
  64             {
  65               DPRINT(("  b->cmp_tag=%d %d <? %d\n", b->cmp_tag,
  66                       tre_tag_touch_get(tags, b->cmp_tag),
  67                       tre_tag_touch_get(tags, start_tag)));
  68               reset = (reset_all || tre_tag_touch_get(tags, b->cmp_tag) <
  69                        tre_tag_touch_get(tags, start_tag));
  70             }
  71           else
  72             reset = 0;
  73 
  74           if (reset)
  75             {
  76               int *t;
  77 
  78               for (i = b->n_tags, t = b->tags; i > 0; i--, t++)
  79                 {
  80                   DPRINT(("  Resetting t%d\n", *t));
  81                   tre_tag_reset(tags, *t);
  82                 }
  83             }
  84           if (b->n_last_matched > 0)
  85             tre_reset_last_matched_branches(tags, b->last_matched,
  86                                                 b->n_last_matched, 
  87                                                 lm->start_tag, reset);
  88         }
  89       else
  90         {
  91           if (!reset_all)
  92             {
  93 #ifdef TRE_DEBUG
  94               int last;
  95 #endif /* TRE_DEBUG */
  96               max = 0;
  97               for (i = lm->n_branches, b = lm->branches; i > 0; i--, b++)
  98                 {
  99                   int t = b->cmp_tag;
 100                   int touch = tre_tag_touch_get(tags, t);
 101                   if (touch > max)
 102                     {
 103                       max = touch;
 104 #ifdef TRE_DEBUG
 105                       last = t;
 106 #endif /* TRE_DEBUG */
 107                     }
 108                 }
 109               DPRINT(("  Last touched end tag t%d=%d\n", last, max));
 110             }
 111 
 112           for (i = lm->n_branches, b = lm->branches; i > 0; i--, b++)
 113             {
 114               reset = (reset_all || tre_tag_touch_get(tags, b->cmp_tag) < max);
 115               if (reset)
 116                 {
 117                   int j;
 118                   int *t;
 119 
 120                   for (j = b->n_tags, t = b->tags; j > 0; j--, t++)
 121                     {
 122                       DPRINT(("  Resetting t%d\n", *t));
 123                       tre_tag_reset(tags, *t);
 124                     }
 125                 }
 126               if (b->n_last_matched > 0)
 127                 tre_reset_last_matched_branches(tags, b->last_matched,
 128                                                 b->n_last_matched,
 129                                                 lm->start_tag, reset);
 130             }
 131         }
 132     }
 133 }
 134 
 135 /* Fills the POSIX.2 regmatch_t array according to the TNFA tag and match
 136    endpoint values. */
 137 reg_errcode_t
 138 tre_fill_pmatch(size_t nmatch, regmatch_t pmatch[], int cflags,
 139                 const tre_tnfa_t *tnfa, const tre_tag_t *intags, int match_eo)
 140 {
 141   unsigned int i;


 142 
 143   if (cflags & REG_NOSUB) return REG_OK;



 144 
 145   i = 0;
 146   if (match_eo >= 0 && intags)
 147     {
 148       const tre_tag_t *tags = intags;
 149       tre_submatch_data_t *submatch_data;






















 150 
 151       if (tnfa->last_matched_branch &&
 152           tnfa->last_matched_branch->n_last_matched > 0)
 153         {
 154           tre_tag_t *t;
 155           t = malloc(sizeof(*t) * tnfa->num_tags);
 156           if (!t) return REG_ESPACE;
 157           memcpy(t, intags, tnfa->num_tags * sizeof(tre_tag_t));
 158           tre_reset_last_matched_branches(t,
 159                                     tnfa->last_matched_branch->last_matched,
 160                                     tnfa->last_matched_branch->n_last_matched,
 161                                     0, 0);
 162           tags = t;
 163         }
 164       /* Construct submatch offsets from the tags. */
 165       DPRINT(("end tag = t%d = %d\n", tnfa->end_tag, match_eo));
 166       submatch_data = tnfa->submatch_data;
 167       while (i < tnfa->num_submatches && i < nmatch)
 168         {
 169           if (submatch_data[i].so_tag == tnfa->end_tag)
 170             pmatch[i].rm_so = match_eo;
 171           else
 172             pmatch[i].rm_so = tre_tag_get(tags, submatch_data[i].so_tag);
 173 
 174           if (submatch_data[i].eo_tag == tnfa->end_tag)
 175             pmatch[i].rm_eo = match_eo;
 176           else
 177             pmatch[i].rm_eo = tre_tag_get(tags, submatch_data[i].eo_tag);


















 178 
 179           /* If either of the endpoints were not used, this submatch
 180              was not part of the match. */
 181           if (pmatch[i].rm_so == -1 || pmatch[i].rm_eo == -1)
 182             pmatch[i].rm_so = pmatch[i].rm_eo = -1;

























 183 
 184           DPRINT(("pmatch[%d] = {t%d = %qd, t%d = %qd}\n", i,
 185                   submatch_data[i].so_tag, pmatch[i].rm_so,
 186                   submatch_data[i].eo_tag, pmatch[i].rm_eo));
 187           i++;
 188         }
 189         if (tags != intags) free(__DECONST(tre_tag_t *,tags));
 190     }
 191 
 192   while (i < nmatch)
 193     {
 194       pmatch[i].rm_so = -1;
 195       pmatch[i].rm_eo = -1;
 196       i++;
 197     }

 198 
 199   return REG_OK;
 200 }
 201 
 202 /*
 203   Wrapper functions for POSIX compatible regexp matching.
 204 */
 205 
 206 int
 207 tre_have_backrefs(const regex_t *preg)




 208 {
 209   tre_tnfa_t *tnfa = (void *)preg->TRE_REGEX_T_FIELD;
 210   return tnfa->have_backrefs;
 211 }
 212 
 213 static int
 214 tre_match(const tre_tnfa_t *tnfa, const void *string, size_t len,
 215           tre_str_type_t type, size_t nmatch, regmatch_t pmatch[],
 216           int eflags)
 217 {
 218   reg_errcode_t status;
 219   tre_tag_t *tags = NULL;
 220   int eo;
 221   size_t offset = 0, count = 0;
 222   if (tnfa->num_tags > 0 && nmatch > 0)
 223     {
 224       tags = malloc(sizeof(*tags) * tnfa->num_tags);
 225       if (tags == NULL)
 226         return REG_ESPACE;
 227     }
 228 
 229   if (
 230       (eflags & REG_STARTEND) && pmatch)
 231     {
 232       if (pmatch->rm_so < 0)
 233         return REG_INVARG;
 234       if (len == (size_t)-1)
 235         {
 236           if (pmatch->rm_eo < 0 || pmatch->rm_so > pmatch->rm_eo)
 237             return REG_INVARG;
 238           len = pmatch->rm_eo - pmatch->rm_so;
 239         }
 240       count = offset = pmatch->rm_so;
 241       if (type == STR_WIDE) offset *= sizeof(wchar_t);
 242     }
 243 
 244   /* Dispatch to the appropriate matcher. */
 245 #ifdef REG_BACKTRACKING_MATCHER
 246   if (tnfa->have_backrefs || eflags & REG_BACKTRACKING_MATCHER)
 247 #else
 248   if (tnfa->have_backrefs)



 249 #endif
 250     {
 251       /* The regex has back references, use the backtracking matcher. */
 252       status = tre_tnfa_run_backtrack(tnfa, string + offset, (int)len, type,
 253                                       tags, eflags, &eo);
 254     }
 255   else
 256     {
 257       /* Exact matching, no back references, use the parallel matcher. */
 258       status = tre_tnfa_run_parallel(tnfa, string + offset, (int)len, type,
 259                                      tags, eflags, &eo);
 260     }
 261 
 262   if (status == REG_OK)
 263     {
 264       /* A match was found, so fill the submatch registers. */
 265       status = tre_fill_pmatch(nmatch, pmatch, tnfa->cflags, tnfa, tags, eo);
 266       /* If doing REG_STARTEND, adjust the pmatch array (we can't build
 267          this into tre_fill_pmatch, because tre_tnfa_run_backtrack calls
 268          tre_fill_pmatch itself). */
 269       if (status == REG_OK && !(tnfa->cflags & REG_NOSUB) &&
 270           (eflags & REG_STARTEND) && pmatch && nmatch > 0)
 271         {
 272           size_t i;
 273           regmatch_t *p;
 274           for (i = nmatch, p = pmatch; i > 0; p++, i--)
 275             {
 276               if (p->rm_so >= 0) p->rm_so += count;
 277               if (p->rm_eo >= 0) p->rm_eo += count;
 278             }
 279         }
 280     }
 281   if (tags)
 282     free(tags);
 283   return status;
 284 }
 285 
 286 int
 287 tre_regnexec(const regex_t *preg, const char *str, size_t len,
 288          size_t nmatch, regmatch_t pmatch[], int eflags)
 289 {
 290   tre_tnfa_t *tnfa = (void *)preg->TRE_REGEX_T_FIELD;
 291   tre_str_type_t type = (TRE_MB_CUR_MAX_L(tnfa->loc) == 1) ? STR_BYTE : STR_MBS;
 292 
 293   if (preg->re_magic != RE_MAGIC) return REG_BADPAT;
 294 
 295   return tre_match(tnfa, str, len, type, nmatch, pmatch, eflags);
 296 }
 297 
 298 int
 299 regexec(const regex_t *_RESTRICT_KYWD preg, const char *_RESTRICT_KYWD str,
 300     size_t nmatch, regmatch_t pmatch[_RESTRICT_KYWD], int eflags)
 301 {
 302   return tre_regnexec(preg, str, (size_t)-1, nmatch, pmatch, eflags);
 303 }