/* regcomp.c */ /* * "A fair jaw-cracker dwarf-language must be." --Samwise Gamgee */ /* This file contains functions for compiling a regular expression. See * also regexec.c which funnily enough, contains functions for executing * a regular expression. * * This file is also copied at build time to ext/re/re_comp.c, where * it's built with -DPERL_EXT_RE_BUILD -DPERL_EXT_RE_DEBUG -DPERL_EXT. * This causes the main functions to be compiled under new names and with * debugging support added, which makes "use re 'debug'" work. */ /* NOTE: this is derived from Henry Spencer's regexp code, and should not * confused with the original package (see point 3 below). Thanks, Henry! */ /* Additional note: this code is very heavily munged from Henry's version * in places. In some spots I've traded clarity for efficiency, so don't * blame Henry for some of the lack of readability. */ /* The names of the functions have been changed from regcomp and * regexec to pregcomp and pregexec in order to avoid conflicts * with the POSIX routines of the same names. */ #ifdef PERL_EXT_RE_BUILD /* need to replace pregcomp et al, so enable that */ # ifndef PERL_IN_XSUB_RE # define PERL_IN_XSUB_RE # endif /* need access to debugger hooks */ # if defined(PERL_EXT_RE_DEBUG) && !defined(DEBUGGING) # define DEBUGGING # endif #endif #ifdef PERL_IN_XSUB_RE /* We *really* need to overwrite these symbols: */ # define Perl_pregcomp my_regcomp # define Perl_regdump my_regdump # define Perl_regprop my_regprop # define Perl_pregfree my_regfree # define Perl_re_intuit_string my_re_intuit_string /* *These* symbols are masked to allow static link. */ # define Perl_regnext my_regnext # define Perl_save_re_context my_save_re_context # define Perl_reginitcolors my_reginitcolors # define PERL_NO_GET_CONTEXT #endif /* * pregcomp and pregexec -- regsub and regerror are not used in perl * * Copyright (c) 1986 by University of Toronto. * Written by Henry Spencer. Not derived from licensed software. * * Permission is granted to anyone to use this software for any * purpose on any computer system, and to redistribute it freely, * subject to the following restrictions: * * 1. The author is not responsible for the consequences of use of * this software, no matter how awful, even if they arise * from defects in it. * * 2. The origin of this software must not be misrepresented, either * by explicit claim or by omission. * * 3. Altered versions must be plainly marked as such, and must not * be misrepresented as being the original software. * * **** Alterations to Henry's code are... **** **** Copyright (C) 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, **** 2000, 2001, 2002, 2003, 2004, 2005, by Larry Wall and others **** **** You may distribute under the terms of either the GNU General Public **** License or the Artistic License, as specified in the README file. * * Beware that some of this code is subtly aware of the way operator * precedence is structured in regular expressions. Serious changes in * regular-expression syntax might require a total rethink. */ #include "EXTERN.h" #define PERL_IN_REGCOMP_C #include "perl.h" #ifndef PERL_IN_XSUB_RE # include "INTERN.h" #endif #define REG_COMP_C #include "regcomp.h" #ifdef op #undef op #endif /* op */ #ifdef MSDOS # if defined(BUGGY_MSC6) /* MSC 6.00A breaks on op/regexp.t test 85 unless we turn this off */ # pragma optimize("a",off) /* But MSC 6.00A is happy with 'w', for aliases only across function calls*/ # pragma optimize("w",on ) # endif /* BUGGY_MSC6 */ #endif /* MSDOS */ #ifndef STATIC #define STATIC static #endif typedef struct RExC_state_t { U32 flags; /* are we folding, multilining? */ char *precomp; /* uncompiled string. */ regexp *rx; char *start; /* Start of input for compile */ char *end; /* End of input for compile */ char *parse; /* Input-scan pointer. */ I32 whilem_seen; /* number of WHILEM in this expr */ regnode *emit_start; /* Start of emitted-code area */ regnode *emit; /* Code-emit pointer; ®dummy = don't = compiling */ I32 naughty; /* How bad is this pattern? */ I32 sawback; /* Did we see \1, ...? */ U32 seen; I32 size; /* Code size. */ I32 npar; /* () count. */ I32 extralen; I32 seen_zerolen; I32 seen_evals; I32 utf8; #if ADD_TO_REGEXEC char *starttry; /* -Dr: where regtry was called. */ #define RExC_starttry (pRExC_state->starttry) #endif } RExC_state_t; #define RExC_flags (pRExC_state->flags) #define RExC_precomp (pRExC_state->precomp) #define RExC_rx (pRExC_state->rx) #define RExC_start (pRExC_state->start) #define RExC_end (pRExC_state->end) #define RExC_parse (pRExC_state->parse) #define RExC_whilem_seen (pRExC_state->whilem_seen) #define RExC_offsets (pRExC_state->rx->offsets) /* I am not like the others */ #define RExC_emit (pRExC_state->emit) #define RExC_emit_start (pRExC_state->emit_start) #define RExC_naughty (pRExC_state->naughty) #define RExC_sawback (pRExC_state->sawback) #define RExC_seen (pRExC_state->seen) #define RExC_size (pRExC_state->size) #define RExC_npar (pRExC_state->npar) #define RExC_extralen (pRExC_state->extralen) #define RExC_seen_zerolen (pRExC_state->seen_zerolen) #define RExC_seen_evals (pRExC_state->seen_evals) #define RExC_utf8 (pRExC_state->utf8) #define ISMULT1(c) ((c) == '*' || (c) == '+' || (c) == '?') #define ISMULT2(s) ((*s) == '*' || (*s) == '+' || (*s) == '?' || \ ((*s) == '{' && regcurly(s))) #ifdef SPSTART #undef SPSTART /* dratted cpp namespace... */ #endif /* * Flags to be passed up and down. */ #define WORST 0 /* Worst case. */ #define HASWIDTH 0x1 /* Known to match non-null strings. */ #define SIMPLE 0x2 /* Simple enough to be STAR/PLUS operand. */ #define SPSTART 0x4 /* Starts with * or +. */ #define TRYAGAIN 0x8 /* Weeded out a declaration. */ /* Length of a variant. */ typedef struct scan_data_t { I32 len_min; I32 len_delta; I32 pos_min; I32 pos_delta; SV *last_found; I32 last_end; /* min value, <0 unless valid. */ I32 last_start_min; I32 last_start_max; SV **longest; /* Either &l_fixed, or &l_float. */ SV *longest_fixed; I32 offset_fixed; SV *longest_float; I32 offset_float_min; I32 offset_float_max; I32 flags; I32 whilem_c; I32 *last_closep; struct regnode_charclass_class *start_class; } scan_data_t; /* * Forward declarations for pregcomp()'s friends. */ static const scan_data_t zero_scan_data = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; #define SF_BEFORE_EOL (SF_BEFORE_SEOL|SF_BEFORE_MEOL) #define SF_BEFORE_SEOL 0x1 #define SF_BEFORE_MEOL 0x2 #define SF_FIX_BEFORE_EOL (SF_FIX_BEFORE_SEOL|SF_FIX_BEFORE_MEOL) #define SF_FL_BEFORE_EOL (SF_FL_BEFORE_SEOL|SF_FL_BEFORE_MEOL) #ifdef NO_UNARY_PLUS # define SF_FIX_SHIFT_EOL (0+2) # define SF_FL_SHIFT_EOL (0+4) #else # define SF_FIX_SHIFT_EOL (+2) # define SF_FL_SHIFT_EOL (+4) #endif #define SF_FIX_BEFORE_SEOL (SF_BEFORE_SEOL << SF_FIX_SHIFT_EOL) #define SF_FIX_BEFORE_MEOL (SF_BEFORE_MEOL << SF_FIX_SHIFT_EOL) #define SF_FL_BEFORE_SEOL (SF_BEFORE_SEOL << SF_FL_SHIFT_EOL) #define SF_FL_BEFORE_MEOL (SF_BEFORE_MEOL << SF_FL_SHIFT_EOL) /* 0x20 */ #define SF_IS_INF 0x40 #define SF_HAS_PAR 0x80 #define SF_IN_PAR 0x100 #define SF_HAS_EVAL 0x200 #define SCF_DO_SUBSTR 0x400 #define SCF_DO_STCLASS_AND 0x0800 #define SCF_DO_STCLASS_OR 0x1000 #define SCF_DO_STCLASS (SCF_DO_STCLASS_AND|SCF_DO_STCLASS_OR) #define SCF_WHILEM_VISITED_POS 0x2000 #define UTF (RExC_utf8 != 0) #define LOC ((RExC_flags & PMf_LOCALE) != 0) #define FOLD ((RExC_flags & PMf_FOLD) != 0) #define OOB_UNICODE 12345678 #define OOB_NAMEDCLASS -1 #define CHR_SVLEN(sv) (UTF ? sv_len_utf8(sv) : SvCUR(sv)) #define CHR_DIST(a,b) (UTF ? utf8_distance(a,b) : a - b) /* length of regex to show in messages that don't mark a position within */ #define RegexLengthToShowInErrorMessages 127 /* * If MARKER[12] are adjusted, be sure to adjust the constants at the top * of t/op/regmesg.t, the tests in t/op/re_tests, and those in * op/pragma/warn/regcomp. */ #define MARKER1 "<-- HERE" /* marker as it appears in the description */ #define MARKER2 " <-- HERE " /* marker as it appears within the regex */ #define REPORT_LOCATION " in regex; marked by " MARKER1 " in m/%.*s" MARKER2 "%s/" /* * Calls SAVEDESTRUCTOR_X if needed, then calls Perl_croak with the given * arg. Show regex, up to a maximum length. If it's too long, chop and add * "...". */ #define FAIL(msg) STMT_START { \ const char *ellipses = ""; \ IV len = RExC_end - RExC_precomp; \ \ if (!SIZE_ONLY) \ SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \ if (len > RegexLengthToShowInErrorMessages) { \ /* chop 10 shorter than the max, to ensure meaning of "..." */ \ len = RegexLengthToShowInErrorMessages - 10; \ ellipses = "..."; \ } \ Perl_croak(aTHX_ "%s in regex m/%.*s%s/", \ msg, (int)len, RExC_precomp, ellipses); \ } STMT_END /* * Calls SAVEDESTRUCTOR_X if needed, then calls Perl_croak with the given * args. Show regex, up to a maximum length. If it's too long, chop and add * "...". */ #define FAIL2(pat,msg) STMT_START { \ const char *ellipses = ""; \ IV len = RExC_end - RExC_precomp; \ \ if (!SIZE_ONLY) \ SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \ if (len > RegexLengthToShowInErrorMessages) { \ /* chop 10 shorter than the max, to ensure meaning of "..." */ \ len = RegexLengthToShowInErrorMessages - 10; \ ellipses = "..."; \ } \ S_re_croak2(aTHX_ pat, " in regex m/%.*s%s/", \ msg, (int)len, RExC_precomp, ellipses); \ } STMT_END /* * Simple_vFAIL -- like FAIL, but marks the current location in the scan */ #define Simple_vFAIL(m) STMT_START { \ const IV offset = RExC_parse - RExC_precomp; \ Perl_croak(aTHX_ "%s" REPORT_LOCATION, \ m, (int)offset, RExC_precomp, RExC_precomp + offset); \ } STMT_END /* * Calls SAVEDESTRUCTOR_X if needed, then Simple_vFAIL() */ #define vFAIL(m) STMT_START { \ if (!SIZE_ONLY) \ SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \ Simple_vFAIL(m); \ } STMT_END /* * Like Simple_vFAIL(), but accepts two arguments. */ #define Simple_vFAIL2(m,a1) STMT_START { \ const IV offset = RExC_parse - RExC_precomp; \ S_re_croak2(aTHX_ m, REPORT_LOCATION, a1, \ (int)offset, RExC_precomp, RExC_precomp + offset); \ } STMT_END /* * Calls SAVEDESTRUCTOR_X if needed, then Simple_vFAIL2(). */ #define vFAIL2(m,a1) STMT_START { \ if (!SIZE_ONLY) \ SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \ Simple_vFAIL2(m, a1); \ } STMT_END /* * Like Simple_vFAIL(), but accepts three arguments. */ #define Simple_vFAIL3(m, a1, a2) STMT_START { \ const IV offset = RExC_parse - RExC_precomp; \ S_re_croak2(aTHX_ m, REPORT_LOCATION, a1, a2, \ (int)offset, RExC_precomp, RExC_precomp + offset); \ } STMT_END /* * Calls SAVEDESTRUCTOR_X if needed, then Simple_vFAIL3(). */ #define vFAIL3(m,a1,a2) STMT_START { \ if (!SIZE_ONLY) \ SAVEDESTRUCTOR_X(clear_re,(void*)RExC_rx); \ Simple_vFAIL3(m, a1, a2); \ } STMT_END /* * Like Simple_vFAIL(), but accepts four arguments. */ #define Simple_vFAIL4(m, a1, a2, a3) STMT_START { \ const IV offset = RExC_parse - RExC_precomp; \ S_re_croak2(aTHX_ m, REPORT_LOCATION, a1, a2, a3, \ (int)offset, RExC_precomp, RExC_precomp + offset); \ } STMT_END #define vWARN(loc,m) STMT_START { \ const IV offset = loc - RExC_precomp; \ Perl_warner(aTHX_ packWARN(WARN_REGEXP), "%s" REPORT_LOCATION, \ m, (int)offset, RExC_precomp, RExC_precomp + offset); \ } STMT_END #define vWARNdep(loc,m) STMT_START { \ const IV offset = loc - RExC_precomp; \ Perl_warner(aTHX_ packWARN2(WARN_DEPRECATED, WARN_REGEXP), \ "%s" REPORT_LOCATION, \ m, (int)offset, RExC_precomp, RExC_precomp + offset); \ } STMT_END #define vWARN2(loc, m, a1) STMT_START { \ const IV offset = loc - RExC_precomp; \ Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \ a1, (int)offset, RExC_precomp, RExC_precomp + offset); \ } STMT_END #define vWARN3(loc, m, a1, a2) STMT_START { \ const IV offset = loc - RExC_precomp; \ Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \ a1, a2, (int)offset, RExC_precomp, RExC_precomp + offset); \ } STMT_END #define vWARN4(loc, m, a1, a2, a3) STMT_START { \ const IV offset = loc - RExC_precomp; \ Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \ a1, a2, a3, (int)offset, RExC_precomp, RExC_precomp + offset); \ } STMT_END #define vWARN5(loc, m, a1, a2, a3, a4) STMT_START { \ const IV offset = loc - RExC_precomp; \ Perl_warner(aTHX_ packWARN(WARN_REGEXP), m REPORT_LOCATION, \ a1, a2, a3, a4, (int)offset, RExC_precomp, RExC_precomp + offset); \ } STMT_END /* Allow for side effects in s */ #define REGC(c,s) STMT_START { \ if (!SIZE_ONLY) *(s) = (c); else (void)(s); \ } STMT_END /* Macros for recording node offsets. 20001227 mjd@plover.com * Nodes are numbered 1, 2, 3, 4. Node #n's position is recorded in * element 2*n-1 of the array. Element #2n holds the byte length node #n. * Element 0 holds the number n. */ #define MJD_OFFSET_DEBUG(x) /* #define MJD_OFFSET_DEBUG(x) DEBUG_r(Perl_warn_nocontext x) */ #define Set_Node_Offset_To_R(node,byte) STMT_START { \ if (! SIZE_ONLY) { \ MJD_OFFSET_DEBUG(("** (%d) offset of node %d is %d.\n", \ __LINE__, (node), (byte))); \ if((node) < 0) { \ Perl_croak(aTHX_ "value of node is %d in Offset macro", node); \ } else { \ RExC_offsets[2*(node)-1] = (byte); \ } \ } \ } STMT_END #define Set_Node_Offset(node,byte) \ Set_Node_Offset_To_R((node)-RExC_emit_start, (byte)-RExC_start) #define Set_Cur_Node_Offset Set_Node_Offset(RExC_emit, RExC_parse) #define Set_Node_Length_To_R(node,len) STMT_START { \ if (! SIZE_ONLY) { \ MJD_OFFSET_DEBUG(("** (%d) size of node %d is %d.\n", \ __LINE__, (node), (len))); \ if((node) < 0) { \ Perl_croak(aTHX_ "value of node is %d in Length macro", node); \ } else { \ RExC_offsets[2*(node)] = (len); \ } \ } \ } STMT_END #define Set_Node_Length(node,len) \ Set_Node_Length_To_R((node)-RExC_emit_start, len) #define Set_Cur_Node_Length(len) Set_Node_Length(RExC_emit, len) #define Set_Node_Cur_Length(node) \ Set_Node_Length(node, RExC_parse - parse_start) /* Get offsets and lengths */ #define Node_Offset(n) (RExC_offsets[2*((n)-RExC_emit_start)-1]) #define Node_Length(n) (RExC_offsets[2*((n)-RExC_emit_start)]) static void clear_re(pTHX_ void *r); /* Mark that we cannot extend a found fixed substring at this point. Updata the longest found anchored substring and the longest found floating substrings if needed. */ STATIC void S_scan_commit(pTHX_ RExC_state_t *pRExC_state, scan_data_t *data) 4127328 { 4127328 const STRLEN l = CHR_SVLEN(data->last_found); 4127328 const STRLEN old_l = CHR_SVLEN(*data->longest); 4127328 if ((l >= old_l) && ((l > old_l) || (data->flags & SF_BEFORE_EOL))) { 236273 SvSetMagicSV(*data->longest, data->last_found); 236273 if (*data->longest == data->longest_fixed) { 117024 data->offset_fixed = l ? data->last_start_min : data->pos_min; 117024 if (data->flags & SF_BEFORE_EOL) 26077 data->flags |= ((data->flags & SF_BEFORE_EOL) << SF_FIX_SHIFT_EOL); else 90947 data->flags &= ~SF_FIX_BEFORE_EOL; } else { 119249 data->offset_float_min = l ? data->last_start_min : data->pos_min; 119249 data->offset_float_max = (l ? data->last_start_max : data->pos_min + data->pos_delta); 119249 if ((U32)data->offset_float_max > (U32)I32_MAX) 165 data->offset_float_max = I32_MAX; 119249 if (data->flags & SF_BEFORE_EOL) 36670 data->flags |= ((data->flags & SF_BEFORE_EOL) << SF_FL_SHIFT_EOL); else 82579 data->flags &= ~SF_FL_BEFORE_EOL; } } 4127328 SvCUR_set(data->last_found, 0); { 4127328 SV * const sv = data->last_found; 4127328 MAGIC * const mg = 4127328 SvUTF8(sv) && SvMAGICAL(sv) ? mg_find(sv, PERL_MAGIC_utf8) : NULL; 4127328 if (mg && mg->mg_len > 0) 3116 mg->mg_len = 0; } 4127328 data->last_end = -1; 4127328 data->flags &= ~SF_BEFORE_EOL; } /* Can match anything (initialization) */ STATIC void S_cl_anything(pTHX_ RExC_state_t *pRExC_state, struct regnode_charclass_class *cl) 355067 { 355067 ANYOF_CLASS_ZERO(cl); 355067 ANYOF_BITMAP_SETALL(cl); 355067 cl->flags = ANYOF_EOS|ANYOF_UNICODE_ALL; 355067 if (LOC) 587 cl->flags |= ANYOF_LOCALE; } /* Can match anything (initialization) */ STATIC int S_cl_is_anything(pTHX_ const struct regnode_charclass_class *cl) 30671 { 30671 int value; 552078 for (value = 0; value <= ANYOF_MAX; value += 2) 521407 if (ANYOF_CLASS_TEST(cl, value) && ANYOF_CLASS_TEST(cl, value + 1)) ###### return 1; 30671 if (!(cl->flags & ANYOF_UNICODE_ALL)) 8672 return 0; 21999 if (!ANYOF_BITMAP_TESTALLSET(cl)) 20539 return 0; 1460 return 1; } /* Can match anything (initialization) */ STATIC void S_cl_init(pTHX_ RExC_state_t *pRExC_state, struct regnode_charclass_class *cl) 329336 { 329336 Zero(cl, 1, struct regnode_charclass_class); 329336 cl->type = ANYOF; 329336 cl_anything(pRExC_state, cl); } STATIC void S_cl_init_zero(pTHX_ RExC_state_t *pRExC_state, struct regnode_charclass_class *cl) 15477 { 15477 Zero(cl, 1, struct regnode_charclass_class); 15477 cl->type = ANYOF; 15477 cl_anything(pRExC_state, cl); 15477 if (LOC) 19 cl->flags |= ANYOF_LOCALE; } /* 'And' a given class with another one. Can create false positives */ /* We assume that cl is not inverted */ STATIC void S_cl_and(pTHX_ struct regnode_charclass_class *cl, const struct regnode_charclass_class *and_with) 90580 { 90580 if (!(and_with->flags & ANYOF_CLASS) && !(cl->flags & ANYOF_CLASS) && (and_with->flags & ANYOF_LOCALE) == (cl->flags & ANYOF_LOCALE) && !(and_with->flags & ANYOF_FOLD) && !(cl->flags & ANYOF_FOLD)) { 90269 int i; 90269 if (and_with->flags & ANYOF_INVERT) 3465 for (i = 0; i < ANYOF_BITMAP_SIZE; i++) 3360 cl->bitmap[i] &= ~and_with->bitmap[i]; else 2975412 for (i = 0; i < ANYOF_BITMAP_SIZE; i++) 2885248 cl->bitmap[i] &= and_with->bitmap[i]; } /* XXXX: logic is complicated otherwise, leave it along for a moment. */ 90580 if (!(and_with->flags & ANYOF_EOS)) 31316 cl->flags &= ~ANYOF_EOS; 90580 if (cl->flags & ANYOF_UNICODE_ALL && and_with->flags & ANYOF_UNICODE && !(and_with->flags & ANYOF_INVERT)) { 8 cl->flags &= ~ANYOF_UNICODE_ALL; 8 cl->flags |= ANYOF_UNICODE; 8 ARG_SET(cl, ARG(and_with)); } 90580 if (!(and_with->flags & ANYOF_UNICODE_ALL) && !(and_with->flags & ANYOF_INVERT)) 19465 cl->flags &= ~ANYOF_UNICODE_ALL; 90580 if (!(and_with->flags & (ANYOF_UNICODE|ANYOF_UNICODE_ALL)) && !(and_with->flags & ANYOF_INVERT)) 19457 cl->flags &= ~ANYOF_UNICODE; } /* 'OR' a given class with another one. Can create false positives */ /* We assume that cl is not inverted */ STATIC void S_cl_or(pTHX_ RExC_state_t *pRExC_state, struct regnode_charclass_class *cl, const struct regnode_charclass_class *or_with) 108420 { 108420 if (or_with->flags & ANYOF_INVERT) { /* We do not use * (B1 | CL1) | (!B2 & !CL2) = (B1 | !B2 & !CL2) | (CL1 | (!B2 & !CL2)) * <= (B1 | !B2) | (CL1 | !CL2) * which is wasteful if CL2 is small, but we ignore CL2: * (B1 | CL1) | (!B2 & !CL2) <= (B1 | CL1) | !B2 = (B1 | !B2) | CL1 * XXXX Can we handle case-fold? Unclear: * (OK1(i) | OK1(i')) | !(OK1(i) | OK1(i')) = * (OK1(i) | OK1(i')) | (!OK1(i) & !OK1(i')) */ 400 if ( (or_with->flags & ANYOF_LOCALE) == (cl->flags & ANYOF_LOCALE) && !(or_with->flags & ANYOF_FOLD) && !(cl->flags & ANYOF_FOLD) ) { 400 int i; 13200 for (i = 0; i < ANYOF_BITMAP_SIZE; i++) 12800 cl->bitmap[i] |= ~or_with->bitmap[i]; } /* XXXX: logic is complicated otherwise */ else { ###### cl_anything(pRExC_state, cl); } } else { /* (B1 | CL1) | (B2 | CL2) = (B1 | B2) | (CL1 | CL2)) */ 108020 if ( (or_with->flags & ANYOF_LOCALE) == (cl->flags & ANYOF_LOCALE) && (!(or_with->flags & ANYOF_FOLD) || (cl->flags & ANYOF_FOLD)) ) { 97826 int i; /* OR char bitmap and class bitmap separately */ 3228258 for (i = 0; i < ANYOF_BITMAP_SIZE; i++) 3130432 cl->bitmap[i] |= or_with->bitmap[i]; 97826 if (or_with->flags & ANYOF_CLASS) { ###### for (i = 0; i < ANYOF_CLASSBITMAP_SIZE; i++) ###### cl->classflags[i] |= or_with->classflags[i]; ###### cl->flags |= ANYOF_CLASS; } } else { /* XXXX: logic is complicated, leave it along for a moment. */ 10194 cl_anything(pRExC_state, cl); } } 108420 if (or_with->flags & ANYOF_EOS) 6291 cl->flags |= ANYOF_EOS; 108420 if (cl->flags & ANYOF_UNICODE && or_with->flags & ANYOF_UNICODE && ARG(cl) != ARG(or_with)) { ###### cl->flags |= ANYOF_UNICODE_ALL; ###### cl->flags &= ~ANYOF_UNICODE; } 108420 if (or_with->flags & ANYOF_UNICODE_ALL) { 46210 cl->flags |= ANYOF_UNICODE_ALL; 46210 cl->flags &= ~ANYOF_UNICODE; } } /* make_trie(startbranch,first,last,tail,flags) startbranch: the first branch in the whole branch sequence first : start branch of sequence of branch-exact nodes. May be the same as startbranch last : Thing following the last branch. May be the same as tail. tail : item following the branch sequence flags : currently the OP() type we will be building one of /EXACT(|F|Fl)/ Inplace optimizes a sequence of 2 or more Branch-Exact nodes into a TRIE node. A trie is an N'ary tree where the branches are determined by digital decomposition of the key. IE, at the root node you look up the 1st character and follow that branch repeat until you find the end of the branches. Nodes can be marked as "accepting" meaning they represent a complete word. Eg: /he|she|his|hers/ would convert into the following structure. Numbers represent states, letters following numbers represent valid transitions on the letter from that state, if the number is in square brackets it represents an accepting state, otherwise it will be in parenthesis. +-h->+-e->[3]-+-r->(8)-+-s->[9] | | | (2) | | (1) +-i->(6)-+-s->[7] | +-s->(3)-+-h->(4)-+-e->[5] Accept Word Mapping: 3=>1 (he),5=>2 (she), 7=>3 (his), 9=>4 (hers) This shows that when matching against the string 'hers' we will begin at state 1 read 'h' and move to state 2, read 'e' and move to state 3 which is accepting, then read 'r' and go to state 8 followed by 's' which takes us to state 9 which is also accepting. Thus we know that we can match both 'he' and 'hers' with a single traverse. We store a mapping from accepting to state to which word was matched, and then when we have multiple possibilities we try to complete the rest of the regex in the order in which they occured in the alternation. The only prior NFA like behaviour that would be changed by the TRIE support is the silent ignoring of duplicate alternations which are of the form: / (DUPE|DUPE) X? (?{ ... }) Y /x Thus EVAL blocks follwing a trie may be called a different number of times with and without the optimisation. With the optimisations dupes will be silently ignored. This inconsistant behaviour of EVAL type nodes is well established as the following demonstrates: 'words'=~/(word|word|word)(?{ print $1 })[xyz]/ which prints out 'word' three times, but 'words'=~/(word|word|word)(?{ print $1 })S/ which doesnt print it out at all. This is due to other optimisations kicking in. Example of what happens on a structural level: The regexp /(ac|ad|ab)+/ will produce the folowing debug output: 1: CURLYM[1] {1,32767}(18) 5: BRANCH(8) 6: EXACT (16) 8: BRANCH(11) 9: EXACT (16) 11: BRANCH(14) 12: EXACT (16) 16: SUCCEED(0) 17: NOTHING(18) 18: END(0) This would be optimizable with startbranch=5, first=5, last=16, tail=16 and should turn into: 1: CURLYM[1] {1,32767}(18) 5: TRIE(16) [Words:3 Chars Stored:6 Unique Chars:4 States:5 NCP:1] 16: SUCCEED(0) 17: NOTHING(18) 18: END(0) Cases where tail != last would be like /(?foo|bar)baz/: 1: BRANCH(4) 2: EXACT (8) 4: BRANCH(7) 5: EXACT (8) 7: TAIL(8) 8: EXACT (10) 10: END(0) which would be optimizable with startbranch=1, first=1, last=7, tail=8 and would end up looking like: 1: TRIE(8) [Words:2 Chars Stored:6 Unique Chars:5 States:7 NCP:1] 7: TAIL(8) 8: EXACT (10) 10: END(0) */ #define TRIE_DEBUG_CHAR \ DEBUG_TRIE_COMPILE_r({ \ SV *tmp; \ if ( UTF ) { \ tmp = newSVpvn( "", 0 ); \ pv_uni_display( tmp, uc, len, 60, UNI_DISPLAY_REGEX ); \ } else { \ tmp = Perl_newSVpvf_nocontext( "%c", (int)uvc ); \ } \ av_push( trie->revcharmap, tmp ); \ }) #define TRIE_READ_CHAR STMT_START { \ if ( UTF ) { \ if ( folder ) { \ if ( foldlen > 0 ) { \ uvc = utf8n_to_uvuni( scan, UTF8_MAXLEN, &len, uniflags ); \ foldlen -= len; \ scan += len; \ len = 0; \ } else { \ uvc = utf8n_to_uvuni( (const U8*)uc, UTF8_MAXLEN, &len, uniflags);\ uvc = to_uni_fold( uvc, foldbuf, &foldlen ); \ foldlen -= UNISKIP( uvc ); \ scan = foldbuf + UNISKIP( uvc ); \ } \ } else { \ uvc = utf8n_to_uvuni( (const U8*)uc, UTF8_MAXLEN, &len, uniflags);\ } \ } else { \ uvc = (U32)*uc; \ len = 1; \ } \ } STMT_END #define TRIE_LIST_ITEM(state,idx) (trie->states[state].trans.list)[ idx ] #define TRIE_LIST_CUR(state) ( TRIE_LIST_ITEM( state, 0 ).forid ) #define TRIE_LIST_LEN(state) ( TRIE_LIST_ITEM( state, 0 ).newstate ) #define TRIE_LIST_USED(idx) ( trie->states[state].trans.list ? (TRIE_LIST_CUR( idx ) - 1) : 0 ) #define TRIE_LIST_PUSH(state,fid,ns) STMT_START { \ if ( TRIE_LIST_CUR( state ) >=TRIE_LIST_LEN( state ) ) { \ TRIE_LIST_LEN( state ) *= 2; \ Renew( trie->states[ state ].trans.list, \ TRIE_LIST_LEN( state ), reg_trie_trans_le ); \ } \ TRIE_LIST_ITEM( state, TRIE_LIST_CUR( state ) ).forid = fid; \ TRIE_LIST_ITEM( state, TRIE_LIST_CUR( state ) ).newstate = ns; \ TRIE_LIST_CUR( state )++; \ } STMT_END #define TRIE_LIST_NEW(state) STMT_START { \ Newz( 1023, trie->states[ state ].trans.list, \ 4, reg_trie_trans_le ); \ TRIE_LIST_CUR( state ) = 1; \ TRIE_LIST_LEN( state ) = 4; \ } STMT_END STATIC I32 S_make_trie(pTHX_ RExC_state_t *pRExC_state, regnode *startbranch, regnode *first, regnode *last, regnode *tail, U32 flags) 15392 { dVAR; /* first pass, loop through and scan words */ 15392 reg_trie_data *trie; 15392 regnode *cur; 15392 const U32 uniflags = ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY; 15392 STRLEN len = 0; 15392 UV uvc = 0; 15392 U16 curword = 0; 15392 U32 next_alloc = 0; /* we just use folder as a flag in utf8 */ 15392 const U8 * const folder = ( flags == EXACTF ? PL_fold : ( flags == EXACTFL ? PL_fold_locale : NULL ) 15392 ); 15392 const U32 data_slot = add_data( pRExC_state, 1, "t" ); 15392 SV *re_trie_maxbuff; 15392 GET_RE_DEBUG_FLAGS_DECL; 15392 Newz( 848200, trie, 1, reg_trie_data ); 15392 trie->refcount = 1; 15392 RExC_rx->data->data[ data_slot ] = (void*)trie; 15392 Newz( 848201, trie->charmap, 256, U16 ); DEBUG_r({ trie->words = newAV(); trie->revcharmap = newAV(); 15392 }); 15392 re_trie_maxbuff = get_sv(RE_TRIE_MAXBUF_NAME, 1); 15392 if (!SvIOK(re_trie_maxbuff)) { ###### sv_setiv(re_trie_maxbuff, RE_TRIE_MAXBUF_INIT); } /* -- First loop and Setup -- We first traverse the branches and scan each word to determine if it contains widechars, and how many unique chars there are, this is important as we have to build a table with at least as many columns as we have unique chars. We use an array of integers to represent the character codes 0..255 (trie->charmap) and we use a an HV* to store unicode characters. We use the native representation of the character value as the key and IV's for the coded index. *TODO* If we keep track of how many times each character is used we can remap the columns so that the table compression later on is more efficient in terms of memory by ensuring most common value is in the middle and the least common are on the outside. IMO this would be better than a most to least common mapping as theres a decent chance the most common letter will share a node with the least common, meaning the node will not be compressable. With a middle is most common approach the worst case is when we have the least common nodes twice. */ 99503 for ( cur = first ; cur < last ; cur = regnext( cur ) ) { 84111 regnode *noper = NEXTOPER( cur ); 84111 const U8 *uc = (U8*)STRING( noper ); 84111 const U8 * const e = uc + STR_LEN( noper ); 84111 STRLEN foldlen = 0; 84111 U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ]; 84111 const U8 *scan = (U8*)NULL; 856085 for ( ; uc < e ; uc += len ) { 385987 trie->charcount++; 385987 TRIE_READ_CHAR; 385987 if ( uvc < 256 ) { 366238 if ( !trie->charmap[ uvc ] ) { 163021 trie->charmap[ uvc ]=( ++trie->uniquecharcount ); 163021 if ( folder ) 31759 trie->charmap[ folder[ uvc ] ] = trie->charmap[ uvc ]; 163021 TRIE_DEBUG_CHAR; } } else { 19749 SV** svpp; 19749 if ( !trie->widecharmap ) 5 trie->widecharmap = newHV(); 19749 svpp = hv_fetch( trie->widecharmap, (char*)&uvc, sizeof( UV ), 1 ); 19749 if ( !svpp ) ###### Perl_croak( aTHX_ "error creating/fetching widecharmap entry for 0x%"UVXf, uvc ); 19749 if ( !SvTRUE( *svpp ) ) { 19749 sv_setiv( *svpp, ++trie->uniquecharcount ); 19749 TRIE_DEBUG_CHAR; } } } 84111 trie->wordcount++; } /* end first pass */ DEBUG_TRIE_COMPILE_r( PerlIO_printf( Perl_debug_log, "TRIE(%s): W:%d C:%d Uq:%d \n", ( trie->widecharmap ? "UTF8" : "NATIVE" ), trie->wordcount, (int)trie->charcount, trie->uniquecharcount ) 15392 ); /* We now know what we are dealing with in terms of unique chars and string sizes so we can calculate how much memory a naive representation using a flat table will take. If it's over a reasonable limit (as specified by ${^RE_TRIE_MAXBUF}) we use a more memory conservative but potentially much slower representation using an array of lists. At the end we convert both representations into the same compressed form that will be used in regexec.c for matching with. The latter is a form that cannot be used to construct with but has memory properties similar to the list form and access properties similar to the table form making it both suitable for fast searches and small enough that its feasable to store for the duration of a program. See the comment in the code where the compressed table is produced inplace from the flat tabe representation for an explanation of how the compression works. */ 15392 if ( (IV)( ( trie->charcount + 1 ) * trie->uniquecharcount + 1) > SvIV(re_trie_maxbuff) ) { /* Second Pass -- Array Of Lists Representation Each state will be represented by a list of charid:state records (reg_trie_trans_le) the first such element holds the CUR and LEN points of the allocated array. (See defines above). We build the initial structure using the lists, and then convert it into the compressed table form which allows faster lookups (but cant be modified once converted). */ 1 STRLEN transcount = 1; 1 Newz( 848204, trie->states, trie->charcount + 2, reg_trie_state ); 1 TRIE_LIST_NEW(1); 1 next_alloc = 2; 19752 for ( cur = first ; cur < last ; cur = regnext( cur ) ) { 19751 regnode *noper = NEXTOPER( cur ); 19751 U8 *uc = (U8*)STRING( noper ); 19751 const U8 * const e = uc + STR_LEN( noper ); 19751 U32 state = 1; /* required init */ 19751 U16 charid = 0; /* sanity init */ 19751 U8 *scan = (U8*)NULL; /* sanity init */ 19751 STRLEN foldlen = 0; /* required init */ 39520 U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ]; 59289 for ( ; uc < e ; uc += len ) { 19769 TRIE_READ_CHAR; 19769 if ( uvc < 256 ) { 24 charid = trie->charmap[ uvc ]; } else { 19745 SV** svpp=(SV**)NULL; 19745 svpp = hv_fetch( trie->widecharmap, (char*)&uvc, sizeof( UV ), 0); 19745 if ( !svpp ) { ###### charid = 0; } else { 19745 charid=(U16)SvIV( *svpp ); } } 19769 if ( charid ) { 19769 U16 check; 19769 U32 newstate = 0; 19769 charid--; 19769 if ( !trie->states[ state ].trans.list ) { 18 TRIE_LIST_NEW( state ); } 195060894 for ( check = 1; check <= TRIE_LIST_USED( state ); check++ ) { 195041125 if ( TRIE_LIST_ITEM( state, check ).forid == charid ) { ###### newstate = TRIE_LIST_ITEM( state, check ).newstate; ###### break; } } 19769 if ( ! newstate ) { 19769 newstate = next_alloc++; 19769 TRIE_LIST_PUSH( state, charid, newstate ); 19769 transcount++; } 19769 state = newstate; } else { ###### Perl_croak( aTHX_ "panic! In trie construction, no char mapping for %"IVdf, uvc ); } /* charid is now 0 if we dont know the char read, or nonzero if we do */ } 19751 if ( !trie->states[ state ].wordnum ) { /* we havent inserted this word into the structure yet. */ 19751 trie->states[ state ].wordnum = ++curword; DEBUG_r({ /* store the word for dumping */ SV* tmp = newSVpvn( STRING( noper ), STR_LEN( noper ) ); if ( UTF ) SvUTF8_on( tmp ); av_push( trie->words, tmp ); 19751 }); } else { /* Its a dupe. So ignore it. */ } } /* end second pass */ 1 trie->laststate = next_alloc; 1 Renew( trie->states, next_alloc, reg_trie_state ); DEBUG_TRIE_COMPILE_MORE_r({ U32 state; /* print out the table precompression. */ PerlIO_printf( Perl_debug_log, "\nState :Word | Transition Data\n" ); PerlIO_printf( Perl_debug_log, "------:-----+-----------------" ); for( state=1 ; state < next_alloc ; state ++ ) { U16 charid; PerlIO_printf( Perl_debug_log, "\n %04"UVXf" :", (UV)state ); if ( ! trie->states[ state ].wordnum ) { PerlIO_printf( Perl_debug_log, "%5s| ",""); } else { PerlIO_printf( Perl_debug_log, "W%04x| ", trie->states[ state ].wordnum ); } for( charid = 1 ; charid <= TRIE_LIST_USED( state ) ; charid++ ) { SV **tmp = av_fetch( trie->revcharmap, TRIE_LIST_ITEM(state,charid).forid, 0); PerlIO_printf( Perl_debug_log, "%s:%3X=%04"UVXf" | ", SvPV_nolen_const( *tmp ), TRIE_LIST_ITEM(state,charid).forid, (UV)TRIE_LIST_ITEM(state,charid).newstate ); } } PerlIO_printf( Perl_debug_log, "\n\n" ); 1 }); 1 Newz( 848203, trie->trans, transcount ,reg_trie_trans ); { 1 U32 state; 1 U32 tp = 0; 1 U32 zp = 0; 19771 for( state=1 ; state < next_alloc ; state ++ ) { 19770 U32 base=0; /* DEBUG_TRIE_COMPILE_MORE_r( PerlIO_printf( Perl_debug_log, "tp: %d zp: %d ",tp,zp) ); */ 19770 if (trie->states[state].trans.list) { 19 U16 minid=TRIE_LIST_ITEM( state, 1).forid; 19 U16 maxid=minid; 19 U16 idx; 19769 for( idx = 2 ; idx <= TRIE_LIST_USED( state ) ; idx++ ) { 19750 if ( TRIE_LIST_ITEM( state, idx).forid < minid ) { ###### minid=TRIE_LIST_ITEM( state, idx).forid; 19750 } else if ( TRIE_LIST_ITEM( state, idx).forid > maxid ) { 19749 maxid=TRIE_LIST_ITEM( state, idx).forid; } } 19 if ( transcount < tp + maxid - minid + 1) { ###### transcount *= 2; ###### Renew( trie->trans, transcount, reg_trie_trans ); ###### Zero( trie->trans + (transcount / 2), transcount / 2 , reg_trie_trans ); } 19 base = trie->uniquecharcount + tp - minid; 19 if ( maxid == minid ) { 18 U32 set = 0; 39534 for ( ; zp < tp ; zp++ ) { 19765 if ( ! trie->trans[ zp ].next ) { 7 base = trie->uniquecharcount + zp - minid; 7 trie->trans[ zp ].next = TRIE_LIST_ITEM( state, 1).newstate; 7 trie->trans[ zp ].check = state; 7 set = 1; 7 break; } } 18 if ( !set ) { 11 trie->trans[ tp ].next = TRIE_LIST_ITEM( state, 1).newstate; 11 trie->trans[ tp ].check = state; 11 tp++; 11 zp = tp; } } else { 19752 for ( idx=1; idx <= TRIE_LIST_USED( state ) ; idx++ ) { 19751 U32 tid = base - trie->uniquecharcount + TRIE_LIST_ITEM( state, idx ).forid; 19751 trie->trans[ tid ].next = TRIE_LIST_ITEM( state, idx ).newstate; 19751 trie->trans[ tid ].check = state; } 1 tp += ( maxid - minid + 1 ); } 19 Safefree(trie->states[ state ].trans.list); } /* DEBUG_TRIE_COMPILE_MORE_r( PerlIO_printf( Perl_debug_log, " base: %d\n",base); ); */ 19770 trie->states[ state ].trans.base=base; } 1 trie->lasttrans = tp + 1; } } else { /* Second Pass -- Flat Table Representation. we dont use the 0 slot of either trans[] or states[] so we add 1 to each. We know that we will need Charcount+1 trans at most to store the data (one row per char at worst case) So we preallocate both structures assuming worst case. We then construct the trie using only the .next slots of the entry structs. We use the .check field of the first entry of the node temporarily to make compression both faster and easier by keeping track of how many non zero fields are in the node. Since trans are numbered from 1 any 0 pointer in the table is a FAIL transition. There are two terms at use here: state as a TRIE_NODEIDX() which is a number representing the first entry of the node, and state as a TRIE_NODENUM() which is the trans number. state 1 is TRIE_NODEIDX(1) and TRIE_NODENUM(1), state 2 is TRIE_NODEIDX(2) and TRIE_NODENUM(3) if there are 2 entrys per node. eg: A B A B 1. 2 4 1. 3 7 2. 0 3 3. 0 5 3. 0 0 5. 0 0 4. 0 0 7. 0 0 The table is internally in the right hand, idx form. However as we also have to deal with the states array which is indexed by nodenum we have to use TRIE_NODENUM() to convert. */ Newz( 848203, trie->trans, ( trie->charcount + 1 ) * trie->uniquecharcount + 1, 15391 reg_trie_trans ); 15391 Newz( 848204, trie->states, trie->charcount + 2, reg_trie_state ); 15391 next_alloc = trie->uniquecharcount + 1; 79751 for ( cur = first ; cur < last ; cur = regnext( cur ) ) { 64360 regnode *noper = NEXTOPER( cur ); 64360 const U8 *uc = (U8*)STRING( noper ); 64360 const U8 * const e = uc + STR_LEN( noper ); 64360 U32 state = 1; /* required init */ 64360 U16 charid = 0; /* sanity init */ 64360 U32 accept_state = 0; /* sanity init */ 64360 U8 *scan = (U8*)NULL; /* sanity init */ 64360 STRLEN foldlen = 0; /* required init */ 430578 U8 foldbuf[ UTF8_MAXBYTES_CASE + 1 ]; 796796 for ( ; uc < e ; uc += len ) { 366218 TRIE_READ_CHAR; 366218 if ( uvc < 256 ) { 366214 charid = trie->charmap[ uvc ]; } else { 4 SV** svpp=(SV**)NULL; 4 svpp = hv_fetch( trie->widecharmap, (char*)&uvc, sizeof( UV ), 0); 4 if ( !svpp ) { ###### charid = 0; } else { 4 charid=(U16)SvIV( *svpp ); } } 366218 if ( charid ) { 366218 charid--; 366218 if ( !trie->trans[ state + charid ].next ) { 338225 trie->trans[ state + charid ].next = next_alloc; 338225 trie->trans[ state ].check++; 338225 next_alloc += trie->uniquecharcount; } 366218 state = trie->trans[ state + charid ].next; } else { ###### Perl_croak( aTHX_ "panic! In trie construction, no char mapping for %"IVdf, uvc ); } /* charid is now 0 if we dont know the char read, or nonzero if we do */ } 64360 accept_state = TRIE_NODENUM( state ); 64360 if ( !trie->states[ accept_state ].wordnum ) { /* we havent inserted this word into the structure yet. */ 64305 trie->states[ accept_state ].wordnum = ++curword; DEBUG_r({ /* store the word for dumping */ SV* tmp = newSVpvn( STRING( noper ), STR_LEN( noper ) ); if ( UTF ) SvUTF8_on( tmp ); av_push( trie->words, tmp ); 64305 }); } else { /* Its a dupe. So ignore it. */ } } /* end second pass */ DEBUG_TRIE_COMPILE_MORE_r({ /* print out the table precompression so that we can do a visual check that they are identical. */ U32 state; U16 charid; PerlIO_printf( Perl_debug_log, "\nChar : " ); for( charid = 0 ; charid < trie->uniquecharcount ; charid++ ) { SV **tmp = av_fetch( trie->revcharmap, charid, 0); if ( tmp ) { PerlIO_printf( Perl_debug_log, "%4.4s ", SvPV_nolen_const( *tmp ) ); } } PerlIO_printf( Perl_debug_log, "\nState+-" ); for( charid=0 ; charid < trie->uniquecharcount ; charid++ ) { PerlIO_printf( Perl_debug_log, "%4s-", "----" ); } PerlIO_printf( Perl_debug_log, "\n" ); for( state=1 ; state < next_alloc ; state += trie->uniquecharcount ) { PerlIO_printf( Perl_debug_log, "%04"UVXf" : ", (UV)TRIE_NODENUM( state ) ); for( charid = 0 ; charid < trie->uniquecharcount ; charid++ ) { PerlIO_printf( Perl_debug_log, "%04"UVXf" ", (UV)SAFE_TRIE_NODENUM( trie->trans[ state + charid ].next ) ); } if ( ! trie->states[ TRIE_NODENUM( state ) ].wordnum ) { PerlIO_printf( Perl_debug_log, " (%04"UVXf")\n", (UV)trie->trans[ state ].check ); } else { PerlIO_printf( Perl_debug_log, " (%04"UVXf") W%04X\n", (UV)trie->trans[ state ].check, trie->states[ TRIE_NODENUM( state ) ].wordnum ); } } PerlIO_printf( Perl_debug_log, "\n\n" ); 15391 }); { /* * Inplace compress the table.* For sparse data sets the table constructed by the trie algorithm will be mostly 0/FAIL transitions or to put it another way mostly empty. (Note that leaf nodes will not contain any transitions.) This algorithm compresses the tables by eliminating most such transitions, at the cost of a modest bit of extra work during lookup: - Each states[] entry contains a .base field which indicates the index in the state[] array wheres its transition data is stored. - If .base is 0 there are no valid transitions from that node. - If .base is nonzero then charid is added to it to find an entry in the trans array. -If trans[states[state].base+charid].check!=state then the transition is taken to be a 0/Fail transition. Thus if there are fail transitions at the front of the node then the .base offset will point somewhere inside the previous nodes data (or maybe even into a node even earlier), but the .check field determines if the transition is valid. The following process inplace converts the table to the compressed table: We first do not compress the root node 1,and mark its all its .check pointers as 1 and set its .base pointer as 1 as well. This allows to do a DFA construction from the compressed table later, and ensures that any .base pointers we calculate later are greater than 0. - We set 'pos' to indicate the first entry of the second node. - We then iterate over the columns of the node, finding the first and last used entry at l and m. We then copy l..m into pos..(pos+m-l), and set the .check pointers accordingly, and advance pos appropriately and repreat for the next node. Note that when we copy the next pointers we have to convert them from the original NODEIDX form to NODENUM form as the former is not valid post compression. - If a node has no transitions used we mark its base as 0 and do not advance the pos pointer. - If a node only has one transition we use a second pointer into the structure to fill in allocated fail transitions from other states. This pointer is independent of the main pointer and scans forward looking for null transitions that are allocated to a state. When it finds one it writes the single transition into the "hole". If the pointer doesnt find one the single transition is appeneded as normal. - Once compressed we can Renew/realloc the structures to release the excess space. See "Table-Compression Methods" in sec 3.9 of the Red Dragon, specifically Fig 3.47 and the associated pseudocode. demq */ 15391 const U32 laststate = TRIE_NODENUM( next_alloc ); 15391 U32 state, charid; 15391 U32 pos = 0, zp=0; 15391 trie->laststate = laststate; 369007 for ( state = 1 ; state < laststate ; state++ ) { 353616 U8 flag = 0; 353616 const U32 stateidx = TRIE_NODEIDX( state ); 353616 const U32 o_used = trie->trans[ stateidx ].check; 353616 U32 used = trie->trans[ stateidx ].check; 353616 trie->trans[ stateidx ].check = 0; 2182545 for ( charid = 0 ; used && charid < trie->uniquecharcount ; charid++ ) { 2096004 if ( flag || trie->trans[ stateidx + charid ].next ) { 442259 if ( trie->trans[ stateidx + charid ].next ) { 338225 if (o_used == 1) { 403473 for ( ; zp < pos ; zp++ ) { 172037 if ( ! trie->trans[ zp ].next ) { 103838 break; } } 267075 trie->states[ state ].trans.base = zp + trie->uniquecharcount - charid ; 267075 trie->trans[ zp ].next = SAFE_TRIE_NODENUM( trie->trans[ stateidx + charid ].next ); 267075 trie->trans[ zp ].check = state; 267075 if ( ++zp > pos ) pos = zp; 163237 break; } 71150 used--; } 175184 if ( !flag ) { 24640 flag = 1; 24640 trie->states[ state ].trans.base = pos + trie->uniquecharcount - charid ; } 175184 trie->trans[ pos ].next = SAFE_TRIE_NODENUM( trie->trans[ stateidx + charid ].next ); 175184 trie->trans[ pos ].check = state; 175184 pos++; } } } 15391 trie->lasttrans = pos + 1; 15391 Renew( trie->states, laststate + 1, reg_trie_state); DEBUG_TRIE_COMPILE_MORE_r( PerlIO_printf( Perl_debug_log, " Alloc: %d Orig: %"IVdf" elements, Final:%"IVdf". Savings of %%%5.2f\n", (int)( ( trie->charcount + 1 ) * trie->uniquecharcount + 1 ), (IV)next_alloc, (IV)pos, ( ( next_alloc - pos ) * 100 ) / (double)next_alloc ); 15391 ); } /* end table compress */ } /* resize the trans array to remove unused space */ 15392 Renew( trie->trans, trie->lasttrans, reg_trie_trans); DEBUG_TRIE_COMPILE_r({ U32 state; /* Now we print it out again, in a slightly different form as there is additional info we want to be able to see when its compressed. They are close enough for visual comparison though. */ PerlIO_printf( Perl_debug_log, "\nChar : %-6s%-6s%-4s ","Match","Base","Ofs" ); for( state = 0 ; state < trie->uniquecharcount ; state++ ) { SV **tmp = av_fetch( trie->revcharmap, state, 0); if ( tmp ) { PerlIO_printf( Perl_debug_log, "%4.4s ", SvPV_nolen_const( *tmp ) ); } } PerlIO_printf( Perl_debug_log, "\n-----:-----------------------"); for( state = 0 ; state < trie->uniquecharcount ; state++ ) PerlIO_printf( Perl_debug_log, "-----"); PerlIO_printf( Perl_debug_log, "\n"); for( state = 1 ; state < trie->laststate ; state++ ) { const U32 base = trie->states[ state ].trans.base; PerlIO_printf( Perl_debug_log, "#%04"UVXf" ", (UV)state); if ( trie->states[ state ].wordnum ) { PerlIO_printf( Perl_debug_log, " W%04X", trie->states[ state ].wordnum ); } else { PerlIO_printf( Perl_debug_log, "%6s", "" ); } PerlIO_printf( Perl_debug_log, " @%04"UVXf" ", (UV)base ); if ( base ) { U32 ofs = 0; while( ( base + ofs < trie->uniquecharcount ) || ( base + ofs - trie->uniquecharcount < trie->lasttrans && trie->trans[ base + ofs - trie->uniquecharcount ].check != state)) ofs++; PerlIO_printf( Perl_debug_log, "+%02"UVXf"[ ", (UV)ofs); for ( ofs = 0 ; ofs < trie->uniquecharcount ; ofs++ ) { if ( ( base + ofs >= trie->uniquecharcount ) && ( base + ofs - trie->uniquecharcount < trie->lasttrans ) && trie->trans[ base + ofs - trie->uniquecharcount ].check == state ) { PerlIO_printf( Perl_debug_log, "%04"UVXf" ", (UV)trie->trans[ base + ofs - trie->uniquecharcount ].next ); } else { PerlIO_printf( Perl_debug_log, "%4s "," 0" ); } } PerlIO_printf( Perl_debug_log, "]"); } PerlIO_printf( Perl_debug_log, "\n" ); } 15392 }); { /* now finally we "stitch in" the new TRIE node This means we convert either the first branch or the first Exact, depending on whether the thing following (in 'last') is a branch or not and whther first is the startbranch (ie is it a sub part of the alternation or is it the whole thing.) Assuming its a sub part we conver the EXACT otherwise we convert the whole branch sequence, including the first. */ 15392 regnode *convert; 15392 if ( first == startbranch && OP( last ) != BRANCH ) { 14480 convert = first; } else { 912 convert = NEXTOPER( first ); 912 NEXT_OFF( first ) = (U16)(last - first); } 15392 OP( convert ) = TRIE + (U8)( flags - EXACT ); 15392 NEXT_OFF( convert ) = (U16)(tail - convert); 15392 ARG_SET( convert, data_slot ); /* tells us if we need to handle accept buffers specially */ 15392 convert->flags = ( RExC_seen_evals ? 1 : 0 ); /* needed for dumping*/ DEBUG_r({ regnode *optimize = convert + NODE_STEP_REGNODE + regarglen[ TRIE ]; /* We now need to mark all of the space originally used by the branches as optimized away. This keeps the dumpuntil from throwing a wobbly as it doesnt use regnext() to traverse the opcodes. */ while( optimize < last ) { OP( optimize ) = OPTIMIZED; optimize++; } 15392 }); } /* end node insert */ 15392 return 1; } /* * There are strange code-generation bugs caused on sparc64 by gcc-2.95.2. * These need to be revisited when a newer toolchain becomes available. */ #if defined(__sparc64__) && defined(__GNUC__) # if __GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) # undef SPARC64_GCC_WORKAROUND # define SPARC64_GCC_WORKAROUND 1 # endif #endif /* REx optimizer. Converts nodes into quickier variants "in place". Finds fixed substrings. */ /* Stops at toplevel WHILEM as well as at "last". At end *scanp is set to the position after last scanned or to NULL. */ STATIC I32 S_study_chunk(pTHX_ RExC_state_t *pRExC_state, regnode **scanp, I32 *deltap, regnode *last, scan_data_t *data, U32 flags, U32 depth) /* scanp: Start here (read-write). */ /* deltap: Write maxlen-minlen here. */ /* last: Stop before this one. */ 2296461 { 2296461 I32 min = 0, pars = 0, code; 2296461 regnode *scan = *scanp, *next; 2296461 I32 delta = 0; 2296461 int is_inf = (flags & SCF_DO_SUBSTR) && (data->flags & SF_IS_INF); 2296461 int is_inf_internal = 0; /* The studied chunk is infinite */ 2296461 I32 is_par = OP(scan) == OPEN ? ARG(scan) : 0; 2296461 scan_data_t data_fake; 2296461 struct regnode_charclass_class and_with; /* Valid if flags & SCF_DO_STCLASS_OR */ 2296461 SV *re_trie_maxbuff = NULL; 2296461 GET_RE_DEBUG_FLAGS_DECL; 5625604 while (scan && OP(scan) != END && scan < last) { /* Peephole optimizer: */ DEBUG_OPTIMISE_r({ SV *mysv=sv_newmortal(); regprop( mysv, scan); PerlIO_printf(Perl_debug_log, "%*speep: %s (0x%08"UVXf")\n", (int)depth*2, "", SvPV_nolen_const(mysv), PTR2UV(scan)); 3408651 }); 3408651 if (PL_regkind[(U8)OP(scan)] == EXACT) { /* Merge several consecutive EXACTish nodes into one. */ 2162619 regnode *n = regnext(scan); 2162619 U32 stringok = 1; #ifdef DEBUGGING 2162619 regnode *stop = scan; #endif 2162619 next = scan + NODE_SZ_STR(scan); /* Skip NOTHING, merge EXACT*. */ 2185523 while (n && ( PL_regkind[(U8)OP(n)] == NOTHING || (stringok && (OP(n) == OP(scan)))) && NEXT_OFF(n) && NEXT_OFF(scan) + NEXT_OFF(n) < I16_MAX) { 25416 if (OP(n) == TAIL || n > next) 14998 stringok = 0; 25416 if (PL_regkind[(U8)OP(n)] == NOTHING) { 20224 NEXT_OFF(scan) += NEXT_OFF(n); 20224 next = n + NODE_STEP_REGNODE; #ifdef DEBUGGING 20224 if (stringok) 5186 stop = n; #endif 20224 n = regnext(n); } 5192 else if (stringok) { 5192 const int oldl = STR_LEN(scan); 5192 regnode *nnext = regnext(n); 5192 if (oldl + STR_LEN(n) > U8_MAX) 2512 break; 2680 NEXT_OFF(scan) += NEXT_OFF(n); 2680 STR_LEN(scan) += STR_LEN(n); 2680 next = n + NODE_SZ_STR(n); /* Now we can overwrite *n : */ 2680 Move(STRING(n), STRING(scan) + oldl, STR_LEN(n), char); #ifdef DEBUGGING 2680 stop = next - 1; #endif 2680 n = nnext; } } 2162619 if (UTF && ( OP(scan) == EXACTF ) && ( STR_LEN(scan) >= 6 ) ) { /* Two problematic code points in Unicode casefolding of EXACT nodes: U+0390 - GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS U+03B0 - GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS which casefold to Unicode UTF-8 U+03B9 U+0308 U+0301 0xCE 0xB9 0xCC 0x88 0xCC 0x81 U+03C5 U+0308 U+0301 0xCF 0x85 0xCC 0x88 0xCC 0x81 This means that in case-insensitive matching (or "loose matching", as Unicode calls it), an EXACTF of length six (the UTF-8 encoded byte length of the above casefolded versions) can match a target string of length two (the byte length of UTF-8 encoded U+0390 or U+03B0). This would rather mess up the minimum length computation. What we'll do is to look for the tail four bytes, and then peek at the preceding two bytes to see whether we need to decrease the minimum length by four (six minus two). Thanks to the design of UTF-8, there cannot be false matches: A sequence of valid UTF-8 bytes cannot be a subsequence of another valid sequence of UTF-8 bytes. */ 385 char *s0 = STRING(scan), *s, *t; 385 char *s1 = s0 + STR_LEN(scan) - 1, *s2 = s1 - 4; 385 const char * const t0 = "\xcc\x88\xcc\x81"; 385 const char * const t1 = t0 + 3; 403 for (s = s0 + 2; s < s2 && (t = ninstr(s, s1, t0, t1)); s = t + 4) { 18 if (((U8)t[-1] == 0xB9 && (U8)t[-2] == 0xCE) || ((U8)t[-1] == 0x85 && (U8)t[-2] == 0xCF)) 18 min -= 4; } } #ifdef DEBUGGING /* Allow dumping */ 2162619 n = scan + NODE_SZ_STR(scan); 2170556 while (n <= stop) { 7937 if (PL_regkind[(U8)OP(n)] != NOTHING || OP(n) == NOTHING) { 2810 OP(n) = OPTIMIZED; 2810 NEXT_OFF(n) = 0; } 7937 n++; } #endif } /* Follow the next-chain of the current node and optimize away all the NOTHINGs from it. */ 3408651 if (OP(scan) != CURLYX) { 3329151 const int max = (reg_off_by_arg[OP(scan)] ? I32_MAX /* I32 may be smaller than U16 on CRAYs! */ 3329151 : (I32_MAX < U16_MAX ? I32_MAX : U16_MAX)); 3329151 int off = (reg_off_by_arg[OP(scan)] ? ARG(scan) : NEXT_OFF(scan)); 3329151 int noff; 3329151 regnode *n = scan; /* Skip NOTHING and LONGJMP. */ 3362404 while ((n = regnext(n)) && ((PL_regkind[(U8)OP(n)] == NOTHING && (noff = NEXT_OFF(n))) || ((OP(n) == LONGJMP) && (noff = ARG(n)))) && off + noff < max) 33253 off += noff; 3329151 if (reg_off_by_arg[OP(scan)]) 30401 ARG(scan) = off; else 3298750 NEXT_OFF(scan) = off; } /* The principal pseudo-switch. Cannot be a switch, since we look into several different things. */ 3408651 if (OP(scan) == BRANCH || OP(scan) == BRANCHJ || OP(scan) == IFTHEN || OP(scan) == SUSPEND) { 26824 next = regnext(scan); 26824 code = OP(scan); /* demq: the op(next)==code check is to see if we have "branch-branch" AFAICT */ 26824 if (OP(next) == code || code == IFTHEN || code == SUSPEND) { 26824 I32 max1 = 0, min1 = I32_MAX, num = 0; 26824 struct regnode_charclass_class accum; 26824 regnode *startbranch=scan; 26824 if (flags & SCF_DO_SUBSTR) /* XXXX Add !SUSPEND? */ 18518 scan_commit(pRExC_state, data); /* Cannot merge strings after this. */ 26824 if (flags & SCF_DO_STCLASS) 15477 cl_init_zero(pRExC_state, &accum); 177009 while (OP(scan) == code) { 151051 I32 deltanext, minnext, f = 0, fake; 151051 struct regnode_charclass_class this_class; 151051 num++; 151051 data_fake.flags = 0; 151051 if (data) { 150941 data_fake.whilem_c = data->whilem_c; 150941 data_fake.last_closep = data->last_closep; } else 110 data_fake.last_closep = &fake; 151051 next = regnext(scan); 151051 scan = NEXTOPER(scan); 151051 if (code != BRANCH) 42430 scan = NEXTOPER(scan); 151051 if (flags & SCF_DO_STCLASS) { 85193 cl_init(pRExC_state, &this_class); 85193 data_fake.start_class = &this_class; 85193 f = SCF_DO_STCLASS_AND; } 151051 if (flags & SCF_WHILEM_VISITED_POS) 109745 f |= SCF_WHILEM_VISITED_POS; /* we suppose the run is continuous, last=next...*/ 151051 minnext = study_chunk(pRExC_state, &scan, &deltanext, next, &data_fake, f,depth+1); 151051 if (min1 > minnext) 39907 min1 = minnext; 151051 if (max1 < minnext + deltanext) 41042 max1 = minnext + deltanext; 151051 if (deltanext == I32_MAX) 6999 is_inf = is_inf_internal = 1; 151051 scan = next; 151051 if (data_fake.flags & (SF_HAS_PAR|SF_IN_PAR)) 4061 pars++; 151051 if (data && (data_fake.flags & SF_HAS_EVAL)) 756 data->flags |= SF_HAS_EVAL; 151051 if (data) 150941 data->whilem_c = data_fake.whilem_c; 151051 if (flags & SCF_DO_STCLASS) 85193 cl_or(pRExC_state, &accum, &this_class); 151051 if (code == SUSPEND) 26824 break; } 26824 if (code == IFTHEN && num < 2) /* Empty ELSE branch */ 112 min1 = 0; 26824 if (flags & SCF_DO_SUBSTR) { 18518 data->pos_min += min1; 18518 data->pos_delta += max1 - min1; 18518 if (max1 != min1 || is_inf) 16719 data->longest = &(data->longest_float); } 26824 min += min1; 26824 delta += max1 - min1; 26824 if (flags & SCF_DO_STCLASS_OR) { 4866 cl_or(pRExC_state, data->start_class, &accum); 4866 if (min1) { 4652 cl_and(data->start_class, &and_with); 4652 flags &= ~SCF_DO_STCLASS; } } 21958 else if (flags & SCF_DO_STCLASS_AND) { 10611 if (min1) { 10098 cl_and(data->start_class, &accum); 10098 flags &= ~SCF_DO_STCLASS; } else { /* Switch to OR mode: cache the old value of * data->start_class */ StructCopy(data->start_class, &and_with, 513 struct regnode_charclass_class); 513 flags &= ~SCF_DO_STCLASS_AND; StructCopy(&accum, data->start_class, 513 struct regnode_charclass_class); 513 flags |= SCF_DO_STCLASS_OR; 513 data->start_class->flags |= ANYOF_EOS; } } /* demq. Assuming this was/is a branch we are dealing with: 'scan' now points at the item that follows the branch sequence, whatever it is. We now start at the beginning of the sequence and look for subsequences of BRANCH->EXACT=>X BRANCH->EXACT=>X which would be constructed from a pattern like /A|LIST|OF|WORDS/ If we can find such a subseqence we need to turn the first element into a trie and then add the subsequent branch exact strings to the trie. We have two cases 1. patterns where the whole set of branch can be converted to a trie, 2. patterns where only a subset of the alternations can be converted to a trie. In case 1 we can replace the whole set with a single regop for the trie. In case 2 we need to keep the start and end branchs so 'BRANCH EXACT; BRANCH EXACT; BRANCH X' becomes BRANCH TRIE; BRANCH X; Hypthetically when we know the regex isnt anchored we can turn a case 1 into a DFA and let it rip... Every time it finds a match it would just call its tail, no WHILEM/CURLY needed. */ 26824 if (DO_TRIE) { 26824 if (!re_trie_maxbuff) { 26416 re_trie_maxbuff = get_sv(RE_TRIE_MAXBUF_NAME, 1); 26416 if (!SvIOK(re_trie_maxbuff)) 1826 sv_setiv(re_trie_maxbuff, RE_TRIE_MAXBUF_INIT); } 26824 if ( SvIV(re_trie_maxbuff)>=0 && OP( startbranch )==BRANCH ) { 25644 regnode *cur; 25644 regnode *first = (regnode *)NULL; 25644 regnode *last = (regnode *)NULL; 25644 regnode *tail = scan; 25644 U8 optype = 0; 25644 U32 count=0; #ifdef DEBUGGING 25644 SV *mysv = sv_newmortal(); /* for dumping */ #endif /* var tail is used because there may be a TAIL regop in the way. Ie, the exacts will point to the thing following the TAIL, but the last branch will point at the TAIL. So we advance tail. If we have nested (?:) we may have to move through several tails. */ 32035 while ( OP( tail ) == TAIL ) { /* this is the TAIL generated by (?:) */ 6391 tail = regnext( tail ); } DEBUG_OPTIMISE_r({ regprop( mysv, tail ); PerlIO_printf( Perl_debug_log, "%*s%s%s%s\n", (int)depth * 2 + 2, "", "Tail node is:", SvPV_nolen_const( mysv ), (RExC_seen_evals) ? "[EVAL]" : "" ); 25644 }); /* step through the branches, cur represents each branch, noper is the first thing to be matched as part of that branch and noper_next is the regnext() of that node. if noper is an EXACT and noper_next is the same as scan (our current position in the regex) then the EXACT branch is a possible optimization target. Once we have two or more consequetive such branches we can create a trie of the EXACT's contents and stich it in place. If the sequence represents all of the branches we eliminate the whole thing and replace it with a single TRIE. If it is a subsequence then we need to stitch it in. This means the first branch has to remain, and needs to be repointed at the item on the branch chain following the last branch optimized. This could be either a BRANCH, in which case the subsequence is internal, or it could be the item following the branch sequence in which case the subsequence is at the end. */ /* dont use tail as the end marker for this traverse */ 134265 for ( cur = startbranch ; cur != scan ; cur = regnext( cur ) ) { 108621 regnode *noper = NEXTOPER( cur ); 108621 regnode *noper_next = regnext( noper ); DEBUG_OPTIMISE_r({ regprop( mysv, cur); PerlIO_printf( Perl_debug_log, "%*s%s", (int)depth * 2 + 2," ", SvPV_nolen_const( mysv ) ); regprop( mysv, noper); PerlIO_printf( Perl_debug_log, " -> %s", SvPV_nolen_const(mysv)); if ( noper_next ) { regprop( mysv, noper_next ); PerlIO_printf( Perl_debug_log,"\t=> %s\t", SvPV_nolen_const(mysv)); } PerlIO_printf( Perl_debug_log, "0x%p,0x%p,0x%p)\n", first, last, cur ); 108621 }); 108621 if ( ( first ? OP( noper ) == optype : PL_regkind[ (U8)OP( noper ) ] == EXACT ) && noper_next == tail && count %s\n", SvPV_nolen_const( mysv ) ); } 68719 ); 68719 last = cur; DEBUG_OPTIMISE_r({ regprop( mysv, cur); PerlIO_printf( Perl_debug_log, "%*s%s", (int)depth * 2 + 2, "N:", SvPV_nolen_const( mysv ) ); regprop( mysv, noper ); PerlIO_printf( Perl_debug_log, " -> %s\n", SvPV_nolen_const( mysv ) ); 68719 }); } } else { 17999 if ( last ) { DEBUG_OPTIMISE_r( PerlIO_printf( Perl_debug_log, "%*s%s\n", (int)depth * 2 + 2, "E:", "**END**" ); 537 ); 537 make_trie( pRExC_state, startbranch, first, cur, tail, optype ); } 17999 if ( PL_regkind[ (U8)OP( noper ) ] == EXACT && noper_next == tail ) { ###### count = 1; ###### first = cur; ###### optype = OP( noper ); } else { 17999 count = 0; 17999 first = NULL; 17999 optype = 0; } 17999 last = NULL; } } DEBUG_OPTIMISE_r({ regprop( mysv, cur); PerlIO_printf( Perl_debug_log, "%*s%s\t(0x%p,0x%p,0x%p)\n", (int)depth * 2 + 2, " ", SvPV_nolen_const( mysv ), first, last, cur); 25644 }); 25644 if ( last ) { DEBUG_OPTIMISE_r( PerlIO_printf( Perl_debug_log, "%*s%s\n", (int)depth * 2 + 2, "E:", "==END==" ); 14855 ); 14855 make_trie( pRExC_state, startbranch, first, scan, tail, optype ); } } } } ###### else if ( code == BRANCHJ ) { /* single branch is optimized. */ ###### scan = NEXTOPER(NEXTOPER(scan)); } else /* single branch is optimized. */ ###### scan = NEXTOPER(scan); ###### continue; } 3381827 else if (OP(scan) == EXACT) { 419993 I32 l = STR_LEN(scan); 419993 UV uc = *((U8*)STRING(scan)); 419993 if (UTF) { 22938 const U8 * const s = (U8*)STRING(scan); 22938 l = utf8_length(s, s + l); 22938 uc = utf8_to_uvchr(s, NULL); } 419993 min += l; 419993 if (flags & SCF_DO_SUBSTR) { /* Update longest substr. */ /* The code below prefers earlier match for fixed offset, later match for variable offset. */ 209710 if (data->last_end == -1) { /* Update the start info. */ 101070 data->last_start_min = data->pos_min; 101070 data->last_start_max = is_inf ? I32_MAX : data->pos_min + data->pos_delta; } 209710 sv_catpvn(data->last_found, STRING(scan), STR_LEN(scan)); { 209710 SV * sv = data->last_found; 209710 MAGIC * const mg = SvUTF8(sv) && SvMAGICAL(sv) ? 209710 mg_find(sv, PERL_MAGIC_utf8) : NULL; 209710 if (mg && mg->mg_len >= 0) 1015 mg->mg_len += utf8_length((U8*)STRING(scan), (U8*)STRING(scan)+STR_LEN(scan)); } 209710 if (UTF) 3117 SvUTF8_on(data->last_found); 209710 data->last_end = data->pos_min + l; 209710 data->pos_min += l; /* As in the first entry. */ 209710 data->flags &= ~SF_BEFORE_EOL; } 419993 if (flags & SCF_DO_STCLASS_AND) { /* Check whether it is compatible with what we know already! */ 180522 int compat = 1; 180522 if (uc >= 0x100 || (!(data->start_class->flags & (ANYOF_CLASS | ANYOF_LOCALE)) && !ANYOF_BITMAP_TEST(data->start_class, uc) && (!(data->start_class->flags & ANYOF_FOLD) || !ANYOF_BITMAP_TEST(data->start_class, PL_fold[uc]))) ) 19802 compat = 0; 180522 ANYOF_CLASS_ZERO(data->start_class); 180522 ANYOF_BITMAP_ZERO(data->start_class); 180522 if (compat) 160720 ANYOF_BITMAP_SET(data->start_class, uc); 180522 data->start_class->flags &= ~ANYOF_EOS; 180522 if (uc < 0x100) 160720 data->start_class->flags &= ~ANYOF_UNICODE_ALL; } 239471 else if (flags & SCF_DO_STCLASS_OR) { /* false positive possible if the class is case-folded */ 17954 if (uc < 0x100) 17954 ANYOF_BITMAP_SET(data->start_class, uc); else ###### data->start_class->flags |= ANYOF_UNICODE_ALL; 17954 data->start_class->flags &= ~ANYOF_EOS; 17954 cl_and(data->start_class, &and_with); } 419993 flags &= ~SCF_DO_STCLASS; } 2961834 else if (PL_regkind[(U8)OP(scan)] == EXACT) { /* But OP != EXACT! */ 1742626 I32 l = STR_LEN(scan); 1742626 UV uc = *((U8*)STRING(scan)); /* Search for fixed substrings supports EXACT only. */ 1742626 if (flags & SCF_DO_SUBSTR) 1721678 scan_commit(pRExC_state, data); 1742626 if (UTF) { 8886 U8 *s = (U8 *)STRING(scan); 8886 l = utf8_length(s, s + l); 8886 uc = utf8_to_uvchr(s, NULL); } 1742626 min += l; 1742626 if (data && (flags & SCF_DO_SUBSTR)) 1721678 data->pos_min += l; 1742626 if (flags & SCF_DO_STCLASS_AND) { /* Check whether it is compatible with what we know already! */ 10399 int compat = 1; 10399 if (uc >= 0x100 || (!(data->start_class->flags & (ANYOF_CLASS | ANYOF_LOCALE)) && !ANYOF_BITMAP_TEST(data->start_class, uc) && !ANYOF_BITMAP_TEST(data->start_class, PL_fold[uc]))) 4 compat = 0; 10399 ANYOF_CLASS_ZERO(data->start_class); 10399 ANYOF_BITMAP_ZERO(data->start_class); 10399 if (compat) { 10395 ANYOF_BITMAP_SET(data->start_class, uc); 10395 data->start_class->flags &= ~ANYOF_EOS; 10395 data->start_class->flags |= ANYOF_FOLD; 10395 if (OP(scan) == EXACTFL) 9 data->start_class->flags |= ANYOF_LOCALE; } } 1732227 else if (flags & SCF_DO_STCLASS_OR) { 423 if (data->start_class->flags & ANYOF_FOLD) { /* false positive possible if the class is case-folded. Assume that the locale settings are the same... */ 188 if (uc < 0x100) 188 ANYOF_BITMAP_SET(data->start_class, uc); 188 data->start_class->flags &= ~ANYOF_EOS; } 423 cl_and(data->start_class, &and_with); } 1742626 flags &= ~SCF_DO_STCLASS; } 1219208 else if (strchr((const char*)PL_varies,OP(scan))) { 338524 I32 mincount, maxcount, minnext, deltanext, fl = 0; 338524 I32 f = flags, pos_before = 0; 338524 regnode *oscan = scan; 338524 struct regnode_charclass_class this_class; 338524 struct regnode_charclass_class *oclass = NULL; 338524 I32 next_is_eval = 0; 338524 switch (PL_regkind[(U8)OP(scan)]) { case WHILEM: /* End of (?:...)* . */ 79500 scan = NEXTOPER(scan); 79500 goto finish; case PLUS: 54918 if (flags & (SCF_DO_SUBSTR | SCF_DO_STCLASS)) { 47249 next = NEXTOPER(scan); 47249 if (OP(next) == EXACT || (flags & SCF_DO_STCLASS)) { 14092 mincount = 1; 14092 maxcount = REG_INFTY; 14092 next = regnext(scan); 14092 scan = NEXTOPER(scan); 14092 goto do_curly; } } 40826 if (flags & SCF_DO_SUBSTR) 33157 data->pos_min++; 40826 min++; /* Fall through. */ case STAR: 147004 if (flags & SCF_DO_STCLASS) { 39235 mincount = 0; 39235 maxcount = REG_INFTY; 39235 next = regnext(scan); 39235 scan = NEXTOPER(scan); 39235 goto do_curly; } 107769 is_inf = is_inf_internal = 1; 107769 scan = regnext(scan); 107769 if (flags & SCF_DO_SUBSTR) { 85561 scan_commit(pRExC_state, data); /* Cannot extend fixed substrings */ 85561 data->longest = &(data->longest_float); } 85561 goto optimize_curly_tail; case CURLY: 96170 mincount = ARG1(scan); 96170 maxcount = ARG2(scan); 96170 next = regnext(scan); 96170 if (OP(scan) == CURLYX) { 79500 I32 lp = (data ? *(data->last_closep) : 0); 79500 scan->flags = ((lp <= U8_MAX) ? (U8)lp : U8_MAX); } 96170 scan = NEXTOPER(scan) + EXTRA_STEP_2ARGS; 96170 next_is_eval = (OP(scan) == EVAL); do_curly: 149497 if (flags & SCF_DO_SUBSTR) { 134469 if (mincount == 0) scan_commit(pRExC_state,data); /* Cannot extend fixed substrings */ 134469 pos_before = data->pos_min; } 149497 if (data) { 149444 fl = data->flags; 149444 data->flags &= ~(SF_HAS_PAR|SF_IN_PAR|SF_HAS_EVAL); 149444 if (is_inf) 24847 data->flags |= SF_IS_INF; } 149497 if (flags & SCF_DO_STCLASS) { 76633 cl_init(pRExC_state, &this_class); 76633 oclass = data->start_class; 76633 data->start_class = &this_class; 76633 f |= SCF_DO_STCLASS_AND; 76633 f &= ~SCF_DO_STCLASS_OR; } /* These are the cases when once a subexpression fails at a particular position, it cannot succeed even after backtracking at the enclosing scope. XXXX what if minimal match and we are at the initial run of {n,m}? */ 149497 if ((mincount != maxcount - 1) && (maxcount != REG_INFTY)) 2135 f &= ~SCF_WHILEM_VISITED_POS; /* This will finish on WHILEM, setting scan, or on NULL: */ 149497 minnext = study_chunk(pRExC_state, &scan, &deltanext, last, data, (mincount == 0 ? (f & ~SCF_DO_SUBSTR) : f),depth+1); 149497 if (flags & SCF_DO_STCLASS) 76633 data->start_class = oclass; 149497 if (mincount == 0 || minnext == 0) { 127180 if (flags & SCF_DO_STCLASS_OR) { 9875 cl_or(pRExC_state, data->start_class, &this_class); } 117305 else if (flags & SCF_DO_STCLASS_AND) { /* Switch to OR mode: cache the old value of * data->start_class */ StructCopy(data->start_class, &and_with, 48116 struct regnode_charclass_class); 48116 flags &= ~SCF_DO_STCLASS_AND; StructCopy(&this_class, data->start_class, 48116 struct regnode_charclass_class); 48116 flags |= SCF_DO_STCLASS_OR; 48116 data->start_class->flags |= ANYOF_EOS; } } else { /* Non-zero len */ 22317 if (flags & SCF_DO_STCLASS_OR) { 2629 cl_or(pRExC_state, data->start_class, &this_class); 2629 cl_and(data->start_class, &and_with); } 19688 else if (flags & SCF_DO_STCLASS_AND) 16013 cl_and(data->start_class, &this_class); 22317 flags &= ~SCF_DO_STCLASS; } 149497 if (!scan) /* It was not CURLYX, but CURLY. */ 69997 scan = next; 149497 if (ckWARN(WARN_REGEXP) /* ? quantifier ok, except for (?{ ... }) */ && (next_is_eval || !(mincount == 0 && maxcount == 1)) && (minnext == 0) && (deltanext == 0) && data && !(data->flags & (SF_HAS_PAR|SF_IN_PAR)) && maxcount <= REG_INFTY/3) /* Complement check for big count */ { vWARN(RExC_parse, ###### "Quantifier unexpected on zero-length expression"); } 149497 min += minnext * mincount; 149497 is_inf_internal |= ((maxcount == REG_INFTY && (minnext + deltanext) > 0) || deltanext == I32_MAX); 149497 is_inf |= is_inf_internal; 149497 delta += (minnext + deltanext) * maxcount - minnext * mincount; /* Try powerful optimization CURLYX => CURLYN. */ 149497 if ( OP(oscan) == CURLYX && data && data->flags & SF_IN_PAR && !(data->flags & SF_HAS_EVAL) && !deltanext && minnext == 1 ) { /* Try to optimize to CURLYN. */ 1970 regnode *nxt = NEXTOPER(oscan) + EXTRA_STEP_2ARGS; 1970 regnode *nxt1 = nxt; #ifdef DEBUGGING 1970 regnode *nxt2; #endif /* Skip open. */ 1970 nxt = regnext(nxt); 1970 if (!strchr((const char*)PL_simple,OP(nxt)) && !(PL_regkind[(U8)OP(nxt)] == EXACT && STR_LEN(nxt) == 1)) 1823 goto nogo; #ifdef DEBUGGING 1823 nxt2 = nxt; #endif 1823 nxt = regnext(nxt); 1823 if (OP(nxt) != CLOSE) ###### goto nogo; /* Now we know that nxt2 is the only contents: */ 1823 oscan->flags = (U8)ARG(nxt); 1823 OP(oscan) = CURLYN; 1823 OP(nxt1) = NOTHING; /* was OPEN. */ #ifdef DEBUGGING 1823 OP(nxt1 + 1) = OPTIMIZED; /* was count. */ 1823 NEXT_OFF(nxt1+ 1) = 0; /* just for consistancy. */ 1823 NEXT_OFF(nxt2) = 0; /* just for consistancy with CURLY. */ 1823 OP(nxt) = OPTIMIZED; /* was CLOSE. */ 1823 OP(nxt + 1) = OPTIMIZED; /* was count. */ 1823 NEXT_OFF(nxt+ 1) = 0; /* just for consistancy. */ #endif } nogo: /* Try optimization CURLYX => CURLYM. */ 149497 if ( OP(oscan) == CURLYX && data && !(data->flags & SF_HAS_PAR) && !(data->flags & SF_HAS_EVAL) && !deltanext /* atom is fixed width */ && minnext != 0 /* CURLYM can't handle zero width */ ) { /* XXXX How to optimize if data == 0? */ /* Optimize to a simpler form. */ 52131 regnode *nxt = NEXTOPER(oscan) + EXTRA_STEP_2ARGS; /* OPEN */ 52131 regnode *nxt2; 52131 OP(oscan) = CURLYM; 64313 while ( (nxt2 = regnext(nxt)) /* skip over embedded stuff*/ && (OP(nxt2) != WHILEM)) 12182 nxt = nxt2; 52131 OP(nxt2) = SUCCEED; /* Whas WHILEM */ /* Need to optimize away parenths. */ 52131 if (data->flags & SF_IN_PAR) { /* Set the parenth number. */ 5280 regnode *nxt1 = NEXTOPER(oscan) + EXTRA_STEP_2ARGS; /* OPEN*/ 5280 if (OP(nxt) != CLOSE) ###### FAIL("Panic opt close"); 5280 oscan->flags = (U8)ARG(nxt); 5280 OP(nxt1) = OPTIMIZED; /* was OPEN. */ 5280 OP(nxt) = OPTIMIZED; /* was CLOSE. */ #ifdef DEBUGGING 5280 OP(nxt1 + 1) = OPTIMIZED; /* was count. */ 5280 OP(nxt + 1) = OPTIMIZED; /* was count. */ 5280 NEXT_OFF(nxt1 + 1) = 0; /* just for consistancy. */ 5280 NEXT_OFF(nxt + 1) = 0; /* just for consistancy. */ #endif #if 0 while ( nxt1 && (OP(nxt1) != WHILEM)) { regnode *nnxt = regnext(nxt1); if (nnxt == nxt) { if (reg_off_by_arg[OP(nxt1)]) ARG_SET(nxt1, nxt2 - nxt1); else if (nxt2 - nxt1 < U16_MAX) NEXT_OFF(nxt1) = nxt2 - nxt1; else OP(nxt) = NOTHING; /* Cannot beautify */ } nxt1 = nnxt; } #endif /* Optimize again: */ 5280 study_chunk(pRExC_state, &nxt1, &deltanext, nxt, NULL, 0,depth+1); } else 46851 oscan->flags = 0; } 97366 else if ((OP(oscan) == CURLYX) && (flags & SCF_WHILEM_VISITED_POS) /* See the comment on a similar expression above. However, this time it not a subexpression we care about, but the expression itself. */ && (maxcount == REG_INFTY) && data && ++data->whilem_c < 16) { /* This stays as CURLYX, we can put the count/of pair. */ /* Find WHILEM (as in regexec.c) */ 2414 regnode *nxt = oscan + NEXT_OFF(oscan); 2414 if (OP(PREVOPER(nxt)) == NOTHING) /* LONGJMP */ ###### nxt += ARG(nxt); 2414 PREVOPER(nxt)->flags = (U8)(data->whilem_c | (RExC_whilem_seen << 4)); /* On WHILEM */ } 149497 if (data && fl & (SF_HAS_PAR|SF_IN_PAR)) 7005 pars++; 149497 if (flags & SCF_DO_SUBSTR) { 134469 SV *last_str = Nullsv; 134469 int counted = mincount != 0; 134469 if (data->last_end > 0 && mincount != 0) { /* Ends with a string. */ #if defined(SPARC64_GCC_WORKAROUND) I32 b = 0; STRLEN l = 0; const char *s = NULL; I32 old = 0; if (pos_before >= data->last_start_min) b = pos_before; else b = data->last_start_min; l = 0; s = SvPV_const(data->last_found, l); old = b - data->last_start_min; #else 7976 I32 b = pos_before >= data->last_start_min 7976 ? pos_before : data->last_start_min; 7976 STRLEN l; 7976 const char *s = SvPV_const(data->last_found, l); 7976 I32 old = b - data->last_start_min; #endif 7976 if (UTF) 5 old = utf8_hop((U8*)s, old) - (U8*)s; 7976 l -= old; /* Get the added string: */ 7976 last_str = newSVpvn(s + old, l); 7976 if (UTF) 5 SvUTF8_on(last_str); 7976 if (deltanext == 0 && pos_before == b) { /* What was added is a constant string */ 7768 if (mincount > 1) { 800 SvGROW(last_str, (mincount * l) + 1); 800 repeatcpy(SvPVX(last_str) + l, SvPVX_const(last_str), l, mincount - 1); 800 SvCUR_set(last_str, SvCUR(last_str) * mincount); /* Add additional parts. */ SvCUR_set(data->last_found, 800 SvCUR(data->last_found) - l); 800 sv_catsv(data->last_found, last_str); { 800 SV * sv = data->last_found; 800 MAGIC *mg = SvUTF8(sv) && SvMAGICAL(sv) ? 800 mg_find(sv, PERL_MAGIC_utf8) : NULL; 800 if (mg && mg->mg_len >= 0) ###### mg->mg_len += CHR_SVLEN(last_str); } 800 data->last_end += l * (mincount - 1); } } else { /* start offset must point into the last copy */ 208 data->last_start_min += minnext * (mincount - 1); 208 data->last_start_max += is_inf ? I32_MAX : (maxcount - 1) * (minnext + data->pos_delta); } } /* It is counted once already... */ 134469 data->pos_min += minnext * (mincount - counted); 134469 data->pos_delta += - counted * deltanext + (minnext + deltanext) * maxcount - minnext * mincount; 134469 if (mincount != maxcount) { /* Cannot extend fixed substrings found inside the group. */ 133279 scan_commit(pRExC_state,data); 133279 if (mincount && last_str) { 7165 sv_setsv(data->last_found, last_str); 7165 data->last_end = data->pos_min; 7165 data->last_start_min = data->pos_min - CHR_SVLEN(last_str); 7165 data->last_start_max = is_inf ? I32_MAX : data->pos_min + data->pos_delta - CHR_SVLEN(last_str); } 133279 data->longest = &(data->longest_float); } 134469 SvREFCNT_dec(last_str); } 149497 if (data && (fl & SF_HAS_EVAL)) 76 data->flags |= SF_HAS_EVAL; optimize_curly_tail: 257266 if (OP(oscan) != CURLYX) { 285848 while (PL_regkind[(U8)OP(next = regnext(oscan))] == NOTHING && NEXT_OFF(next)) 54128 NEXT_OFF(oscan) += NEXT_OFF(next); } 1758 continue; default: /* REF and CLUMP only? */ 1758 if (flags & SCF_DO_SUBSTR) { 1436 scan_commit(pRExC_state,data); /* Cannot expect anything... */ 1436 data->longest = &(data->longest_float); } 1758 is_inf = is_inf_internal = 1; 1758 if (flags & SCF_DO_STCLASS_OR) 60 cl_anything(pRExC_state, data->start_class); 1758 flags &= ~SCF_DO_STCLASS; 1758 break; } } 880684 else if (strchr((const char*)PL_simple,OP(scan))) { 128277 int value = 0; 128277 if (flags & SCF_DO_SUBSTR) { 71144 scan_commit(pRExC_state,data); 71144 data->pos_min++; } 128277 min++; 128277 if (flags & SCF_DO_STCLASS) { 67462 data->start_class->flags &= ~ANYOF_EOS; /* No match on empty */ /* Some of the logic below assumes that switching locale on will only add false positives. */ 67462 switch (PL_regkind[(U8)OP(scan)]) { case SANY: default: do_default: /* Perl_croak(aTHX_ "panic: unexpected simple REx opcode %d", OP(scan)); */ 12717 if (flags & SCF_DO_STCLASS_OR) /* Allow everything */ ###### cl_anything(pRExC_state, data->start_class); ###### break; case REG_ANY: 18713 if (OP(scan) == SANY) 12717 goto do_default; 5996 if (flags & SCF_DO_STCLASS_OR) { /* Everything but \n */ ###### value = (ANYOF_BITMAP_TEST(data->start_class,'\n') || (data->start_class->flags & ANYOF_CLASS)); ###### cl_anything(pRExC_state, data->start_class); } 5996 if (flags & SCF_DO_STCLASS_AND || !value) 5996 ANYOF_BITMAP_CLEAR(data->start_class,'\n'); 5996 break; case ANYOF: 21438 if (flags & SCF_DO_STCLASS_AND) 15581 cl_and(data->start_class, (struct regnode_charclass_class*)scan); else 5857 cl_or(pRExC_state, data->start_class, (struct regnode_charclass_class*)scan); 5857 break; case ALNUM: 2760 if (flags & SCF_DO_STCLASS_AND) { 2697 if (!(data->start_class->flags & ANYOF_LOCALE)) { 2681 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NALNUM); 689017 for (value = 0; value < 256; value++) 686336 if (!isALNUM(value)) 517433 ANYOF_BITMAP_CLEAR(data->start_class, value); } } else { 63 if (data->start_class->flags & ANYOF_LOCALE) ###### ANYOF_CLASS_SET(data->start_class,ANYOF_ALNUM); else { 16191 for (value = 0; value < 256; value++) 16128 if (isALNUM(value)) 3969 ANYOF_BITMAP_SET(data->start_class, value); } } ###### break; case ALNUML: ###### if (flags & SCF_DO_STCLASS_AND) { ###### if (data->start_class->flags & ANYOF_LOCALE) ###### ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NALNUM); } else { ###### ANYOF_CLASS_SET(data->start_class,ANYOF_ALNUM); ###### data->start_class->flags |= ANYOF_LOCALE; } ###### break; case NALNUM: 1289 if (flags & SCF_DO_STCLASS_AND) { 1269 if (!(data->start_class->flags & ANYOF_LOCALE)) { 1269 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_ALNUM); 326133 for (value = 0; value < 256; value++) 324864 if (isALNUM(value)) 79947 ANYOF_BITMAP_CLEAR(data->start_class, value); } } else { 20 if (data->start_class->flags & ANYOF_LOCALE) ###### ANYOF_CLASS_SET(data->start_class,ANYOF_NALNUM); else { 5140 for (value = 0; value < 256; value++) 5120 if (!isALNUM(value)) 3860 ANYOF_BITMAP_SET(data->start_class, value); } } ###### break; case NALNUML: ###### if (flags & SCF_DO_STCLASS_AND) { ###### if (data->start_class->flags & ANYOF_LOCALE) ###### ANYOF_CLASS_CLEAR(data->start_class,ANYOF_ALNUM); } else { ###### data->start_class->flags |= ANYOF_LOCALE; ###### ANYOF_CLASS_SET(data->start_class,ANYOF_NALNUM); } ###### break; case SPACE: 20185 if (flags & SCF_DO_STCLASS_AND) { 20184 if (!(data->start_class->flags & ANYOF_LOCALE)) { 20157 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NSPACE); 5180349 for (value = 0; value < 256; value++) 5160192 if (!isSPACE(value)) 5059407 ANYOF_BITMAP_CLEAR(data->start_class, value); } } else { 1 if (data->start_class->flags & ANYOF_LOCALE) ###### ANYOF_CLASS_SET(data->start_class,ANYOF_SPACE); else { 257 for (value = 0; value < 256; value++) 256 if (isSPACE(value)) 5 ANYOF_BITMAP_SET(data->start_class, value); } } ###### break; case SPACEL: ###### if (flags & SCF_DO_STCLASS_AND) { ###### if (data->start_class->flags & ANYOF_LOCALE) ###### ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NSPACE); } else { ###### data->start_class->flags |= ANYOF_LOCALE; ###### ANYOF_CLASS_SET(data->start_class,ANYOF_SPACE); } ###### break; case NSPACE: 510 if (flags & SCF_DO_STCLASS_AND) { 327 if (!(data->start_class->flags & ANYOF_LOCALE)) { 324 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_SPACE); 83268 for (value = 0; value < 256; value++) 82944 if (isSPACE(value)) 1620 ANYOF_BITMAP_CLEAR(data->start_class, value); } } else { 183 if (data->start_class->flags & ANYOF_LOCALE) ###### ANYOF_CLASS_SET(data->start_class,ANYOF_NSPACE); else { 47031 for (value = 0; value < 256; value++) 46848 if (!isSPACE(value)) 45933 ANYOF_BITMAP_SET(data->start_class, value); } } ###### break; case NSPACEL: ###### if (flags & SCF_DO_STCLASS_AND) { ###### if (data->start_class->flags & ANYOF_LOCALE) { ###### ANYOF_CLASS_CLEAR(data->start_class,ANYOF_SPACE); ###### for (value = 0; value < 256; value++) ###### if (!isSPACE(value)) ###### ANYOF_BITMAP_CLEAR(data->start_class, value); } } else { ###### data->start_class->flags |= ANYOF_LOCALE; ###### ANYOF_CLASS_SET(data->start_class,ANYOF_NSPACE); } ###### break; case DIGIT: 2567 if (flags & SCF_DO_STCLASS_AND) { 2559 ANYOF_CLASS_CLEAR(data->start_class,ANYOF_NDIGIT); 657663 for (value = 0; value < 256; value++) 655104 if (!isDIGIT(value)) 629514 ANYOF_BITMAP_CLEAR(data->start_class, value); } else { 8 if (data->start_class->flags & ANYOF_LOCALE) ###### ANYOF_CLASS_SET(data->start_class,ANYOF_DIGIT); else { 2056 for (value = 0; value < 256; value++) 2048 if (isDIGIT(value)) 80 ANYOF_BITMAP_SET(data->start_class, value); } } ###### break; case NDIGIT: ###### if (flags & SCF_DO_STCLASS_AND) { ###### ANYOF_CLASS_CLEAR(data->start_class,ANYOF_DIGIT); ###### for (value = 0; value < 256; value++) ###### if (isDIGIT(value)) ###### ANYOF_BITMAP_CLEAR(data->start_class, value); } else { ###### if (data->start_class->flags & ANYOF_LOCALE) ###### ANYOF_CLASS_SET(data->start_class,ANYOF_NDIGIT); else { ###### for (value = 0; value < 256; value++) ###### if (!isDIGIT(value)) ###### ANYOF_BITMAP_SET(data->start_class, value); } } 67462 break; } 67462 if (flags & SCF_DO_STCLASS_OR) 6132 cl_and(data->start_class, &and_with); 67462 flags &= ~SCF_DO_STCLASS; } } 752407 else if (PL_regkind[(U8)OP(scan)] == EOL && flags & SCF_DO_SUBSTR) { 66783 data->flags |= (OP(scan) == MEOL ? SF_BEFORE_MEOL : SF_BEFORE_SEOL); } 685624 else if ( PL_regkind[(U8)OP(scan)] == BRANCHJ /* Lookbehind, or need to calculate parens/evals/stclass: */ && (scan->flags || data || (flags & SCF_DO_STCLASS)) && (OP(scan) == IFMATCH || OP(scan) == UNLESSM)) { /* Lookahead/lookbehind */ 6893 I32 deltanext, minnext, fake = 0; 6893 regnode *nscan; 6893 struct regnode_charclass_class intrnl; 6893 int f = 0; 6893 data_fake.flags = 0; 6893 if (data) { 6893 data_fake.whilem_c = data->whilem_c; 6893 data_fake.last_closep = data->last_closep; } else ###### data_fake.last_closep = &fake; 6893 if ( flags & SCF_DO_STCLASS && !scan->flags && OP(scan) == IFMATCH ) { /* Lookahead */ 319 cl_init(pRExC_state, &intrnl); 319 data_fake.start_class = &intrnl; 319 f |= SCF_DO_STCLASS_AND; } 6893 if (flags & SCF_WHILEM_VISITED_POS) 6893 f |= SCF_WHILEM_VISITED_POS; 6893 next = regnext(scan); 6893 nscan = NEXTOPER(NEXTOPER(scan)); 6893 minnext = study_chunk(pRExC_state, &nscan, &deltanext, last, &data_fake, f,depth+1); 6893 if (scan->flags) { 816 if (deltanext) { 6 vFAIL("Variable length lookbehind not implemented"); } 810 else if (minnext > U8_MAX) { 2 vFAIL2("Lookbehind longer than %"UVuf" not implemented", (UV)U8_MAX); } 808 scan->flags = (U8)minnext; } 6885 if (data && data_fake.flags & (SF_HAS_PAR|SF_IN_PAR)) 219 pars++; 6885 if (data && (data_fake.flags & SF_HAS_EVAL)) ###### data->flags |= SF_HAS_EVAL; 6885 if (data) 6885 data->whilem_c = data_fake.whilem_c; 6885 if (f & SCF_DO_STCLASS_AND) { 319 const int was = (data->start_class->flags & ANYOF_EOS); 319 cl_and(data->start_class, &intrnl); 319 if (was) 319 data->start_class->flags |= ANYOF_EOS; } } 678731 else if (OP(scan) == OPEN) { 165577 pars++; } 513154 else if (OP(scan) == CLOSE) { 165577 if ((I32)ARG(scan) == is_par) { 39129 next = regnext(scan); 39129 if ( next && (OP(next) != WHILEM) && next < last) 20794 is_par = 0; /* Disable optimization */ } 165577 if (data) 165577 *(data->last_closep) = ARG(scan); } 347577 else if (OP(scan) == EVAL) { 1069 if (data) 1069 data->flags |= SF_HAS_EVAL; } 346508 else if (OP(scan) == LOGICAL && scan->flags == 2) { /* Embedded follows */ 825 if (flags & SCF_DO_SUBSTR) { 217 scan_commit(pRExC_state,data); 217 data->longest = &(data->longest_float); } 825 is_inf = is_inf_internal = 1; 825 if (flags & SCF_DO_STCLASS_OR) /* Allow everything */ ###### cl_anything(pRExC_state, data->start_class); 825 flags &= ~SCF_DO_STCLASS; } /* Else: zero-length, ignore. */ 3045053 scan = regnext(scan); } finish: 2296453 *scanp = scan; 2296453 *deltap = is_inf_internal ? I32_MAX : delta; 2296453 if (flags & SCF_DO_SUBSTR && is_inf) 100441 data->pos_delta = I32_MAX - data->pos_min; 2296453 if (is_par > U8_MAX) ###### is_par = 0; 2296453 if (is_par && pars==1 && data) { 18049 data->flags |= SF_IN_PAR; 18049 data->flags &= ~SF_HAS_PAR; } 2278404 else if (pars && data) { 124980 data->flags |= SF_HAS_PAR; 124980 data->flags &= ~SF_IN_PAR; } 2296453 if (flags & SCF_DO_STCLASS_OR) 16779 cl_and(data->start_class, &and_with); 2296453 return min; } STATIC I32 S_add_data(pTHX_ RExC_state_t *pRExC_state, I32 n, const char *s) 124348 { 124348 if (RExC_rx->data) { Renewc(RExC_rx->data, sizeof(*RExC_rx->data) + sizeof(void*) * (RExC_rx->data->count + n - 1), 31952 char, struct reg_data); 31952 Renew(RExC_rx->data->what, RExC_rx->data->count + n, U8); 31952 RExC_rx->data->count += n; } else { Newc(1207, RExC_rx->data, sizeof(*RExC_rx->data) + sizeof(void*) * (n - 1), 92396 char, struct reg_data); 92396 New(1208, RExC_rx->data->what, n, U8); 92396 RExC_rx->data->count = n; } 124348 Copy(s, RExC_rx->data->what + RExC_rx->data->count - n, n, U8); 124348 return RExC_rx->data->count - n; } void Perl_reginitcolors(pTHX) ###### { ###### const char * const s = PerlEnv_getenv("PERL_RE_COLORS"); ###### if (s) { ###### char *t = savepv(s); ###### int i = 0; ###### PL_colors[0] = t; ###### while (++i < 6) { ###### t = strchr(t, '\t'); ###### if (t) { ###### *t = '\0'; ###### PL_colors[i] = ++t; } else ###### PL_colors[i] = t = (char *)""; } } else { ###### int i = 0; ###### while (i < 6) ###### PL_colors[i++] = (char *)""; } ###### PL_colorset = 1; } /* - pregcomp - compile a regular expression into internal code * * We can't allocate space until we know how big the compiled form will be, * but we can't compile it (and thus know how big it is) until we've got a * place to put the code. So we cheat: we compile it twice, once with code * generation turned off and size counting turned on, and once "for real". * This also means that we don't allocate space until we are sure that the * thing really will compile successfully, and we never have to move the * code and thus invalidate pointers into it. (Note that it has to be in * one piece because free() must be able to free it all.) [NB: not true in perl] * * Beware that the optimization-preparation code in here knows about some * of the structure of the compiled regexp. [I'll say.] */ regexp * Perl_pregcomp(pTHX_ char *exp, char *xend, PMOP *pm) 1985013 { 1985013 register regexp *r; 1985013 regnode *scan; 1985013 regnode *first; 1985013 I32 flags; 1985013 I32 minlen = 0; 1985013 I32 sawplus = 0; 1985013 I32 sawopen = 0; 1985013 scan_data_t data; 1985013 RExC_state_t RExC_state; 1985013 RExC_state_t *pRExC_state = &RExC_state; 1985013 GET_RE_DEBUG_FLAGS_DECL; 1985013 if (exp == NULL) ###### FAIL("NULL regexp argument"); 1985013 RExC_utf8 = pm->op_pmdynflags & PMdf_CMP_UTF8; 1985013 RExC_precomp = exp; 1985013 DEBUG_r(if (!PL_colorset) reginitcolors()); DEBUG_COMPILE_r({ PerlIO_printf(Perl_debug_log, "%sCompiling REx%s \"%s%*s%s\"\n", PL_colors[4],PL_colors[5],PL_colors[0], (int)(xend - exp), RExC_precomp, PL_colors[1]); 1985013 }); 1985013 RExC_flags = pm->op_pmflags; 1985013 RExC_sawback = 0; 1985013 RExC_seen = 0; 1985013 RExC_seen_zerolen = *exp == '^' ? -1 : 0; 1985013 RExC_seen_evals = 0; 1985013 RExC_extralen = 0; /* First pass: determine size, legality. */ 1985013 RExC_parse = exp; 1985013 RExC_start = exp; 1985013 RExC_end = xend; 1985013 RExC_naughty = 0; 1985013 RExC_npar = 1; 1985013 RExC_size = 0L; 1985013 RExC_emit = &PL_regdummy; 1985013 RExC_whilem_seen = 0; #if 0 /* REGC() is (currently) a NOP at the first pass. * Clever compilers notice this and complain. --jhi */ REGC((U8)REG_MAGIC, (char*)RExC_emit); #endif 1985013 if (reg(pRExC_state, 0, &flags) == NULL) { ###### RExC_precomp = Nullch; ###### return(NULL); } 1983768 DEBUG_COMPILE_r(PerlIO_printf(Perl_debug_log, "size %"IVdf" ", (IV)RExC_size)); /* Small enough for pointer-storage convention? If extralen==0, this means that we will not need long jumps. */ 1983768 if (RExC_size >= 0x10000L && RExC_extralen) 2 RExC_size += RExC_extralen; else 1983766 RExC_extralen = 0; 1983768 if (RExC_whilem_seen > 15) ###### RExC_whilem_seen = 15; /* Allocate space and initialize. */ Newc(1001, r, sizeof(regexp) + (unsigned)RExC_size * sizeof(regnode), 1983768 char, regexp); 1983768 if (r == NULL) ###### FAIL("Regexp out of space"); #ifdef DEBUGGING /* avoid reading uninitialized memory in DEBUGGING code in study_chunk() */ 1983768 Zero(r, sizeof(regexp) + (unsigned)RExC_size * sizeof(regnode), char); #endif 1983768 r->refcnt = 1; 1983768 r->prelen = xend - exp; 1983768 r->precomp = savepvn(RExC_precomp, r->prelen); 1983768 r->subbeg = NULL; #ifdef PERL_OLD_COPY_ON_WRITE r->saved_copy = Nullsv; #endif 1983768 r->reganch = pm->op_pmflags & PMf_COMPILETIME; 1983768 r->nparens = RExC_npar - 1; /* set early to validate backrefs */ 1983768 r->substrs = 0; /* Useful during FAIL. */ 1983768 r->startp = 0; /* Useful during FAIL. */ 1983768 r->endp = 0; /* Useful during FAIL. */ 1983768 Newz(1304, r->offsets, 2*RExC_size+1, U32); /* MJD 20001228 */ 1983768 if (r->offsets) { 1983768 r->offsets[0] = RExC_size; } DEBUG_OFFSETS_r(PerlIO_printf(Perl_debug_log, "%s %"UVuf" bytes for offset annotations.\n", r->offsets ? "Got" : "Couldn't get", 1983768 (UV)((2*RExC_size+1) * sizeof(U32)))); 1983768 RExC_rx = r; /* Second pass: emit code. */ 1983768 RExC_flags = pm->op_pmflags; /* don't let top level (?i) bleed */ 1983768 RExC_parse = exp; 1983768 RExC_end = xend; 1983768 RExC_naughty = 0; 1983768 RExC_npar = 1; 1983768 RExC_emit_start = r->program; 1983768 RExC_emit = r->program; /* Store the count of eval-groups for security checks: */ 1983768 RExC_emit->next_off = (U16)((RExC_seen_evals > U16_MAX) ? U16_MAX : RExC_seen_evals); 1983768 REGC((U8)REG_MAGIC, (char*) RExC_emit++); 1983768 r->data = 0; 1983768 if (reg(pRExC_state, 0, &flags) == NULL) ###### return(NULL); /* Dig out information for optimizations. */ 1983740 r->reganch = pm->op_pmflags & PMf_COMPILETIME; /* Again? */ 1983740 pm->op_pmflags = RExC_flags; 1983740 if (UTF) 8051 r->reganch |= ROPT_UTF8; /* Unicode in it? */ 1983740 r->regstclass = NULL; 1983740 if (RExC_naughty >= 10) /* Probably an expensive pattern. */ 40283 r->reganch |= ROPT_NAUGHTY; 1983740 scan = r->program + 1; /* First BRANCH. */ /* XXXX To minimize changes to RE engine we always allocate 3-units-long substrs field. */ 1983740 Newz(1004, r->substrs, 1, struct reg_substr_data); 1983740 StructCopy(&zero_scan_data, &data, scan_data_t); /* XXXX Should not we check for something else? Usually it is OPEN1... */ 1983740 if (OP(scan) != BRANCH) { /* Only one top-level choice. */ 1981534 I32 fake; 1981534 STRLEN longest_float_length, longest_fixed_length; 1981534 struct regnode_charclass_class ch_class; 1981534 int stclass_flag; 1981534 I32 last_close = 0; 1981534 first = scan; /* Skip introductions and multiplicators >= 1. */ 2021421 while ((OP(first) == OPEN && (sawopen = 1)) || /* An OR of *one* alternative - should not happen now. */ (OP(first) == BRANCH && OP(regnext(first)) != BRANCH) || (OP(first) == PLUS) || (OP(first) == MINMOD) || /* An {n,m} with n>0 */ (PL_regkind[(U8)OP(first)] == CURLY && ARG1(first) > 0) ) { 39887 if (OP(first) == PLUS) 14189 sawplus = 1; else 25698 first += regarglen[(U8)OP(first)]; 39887 first = NEXTOPER(first); } /* Starting-point info. */ again: 2097431 if (PL_regkind[(U8)OP(first)] == EXACT) { 1809611 if (OP(first) == EXACT) ; /* Empty, get anchored substr later. */ 1714269 else if ((OP(first) == EXACTF || OP(first) == EXACTFL)) 1714269 r->regstclass = first; } 287820 else if (strchr((const char*)PL_simple,OP(first))) 54146 r->regstclass = first; 233674 else if (PL_regkind[(U8)OP(first)] == BOUND || PL_regkind[(U8)OP(first)] == NBOUND) 48134 r->regstclass = first; 185540 else if (PL_regkind[(U8)OP(first)] == BOL) { 102672 r->reganch |= (OP(first) == MBOL ? ROPT_ANCH_MBOL : (OP(first) == SBOL ? ROPT_ANCH_SBOL : ROPT_ANCH_BOL)); 102672 first = NEXTOPER(first); 102672 goto again; } 82868 else if (OP(first) == GPOS) { 9961 r->reganch |= ROPT_ANCH_GPOS; 9961 first = NEXTOPER(first); 9961 goto again; } 72907 else if (!sawopen && (OP(first) == STAR && PL_regkind[(U8)OP(NEXTOPER(first))] == REG_ANY) && !(r->reganch & ROPT_ANCH) ) { /* turn .* into ^.* with an implied $*=1 */ 3264 const int type = (OP(NEXTOPER(first)) == REG_ANY) ? ROPT_ANCH_MBOL 3264 : ROPT_ANCH_SBOL; 3264 r->reganch |= type | ROPT_IMPLICIT; 3264 first = NEXTOPER(first); 3264 goto again; } 1981534 if (sawplus && (!sawopen || !RExC_sawback) && !(RExC_seen & REG_SEEN_EVAL)) /* May examine pos and $& */ /* x+ must match at the 1st pos of run of x's */ 14189 r->reganch |= ROPT_SKIP; /* Scan is after the zeroth branch, first is atomic matcher. */ DEBUG_COMPILE_r(PerlIO_printf(Perl_debug_log, "first at %"IVdf"\n", 1981534 (IV)(first - scan + 1))); /* * If there's something expensive in the r.e., find the * longest literal string that must appear and make it the * regmust. Resolve ties in favor of later strings, since * the regstart check works with the beginning of the r.e. * and avoiding duplication strengthens checking. Not a * strong reason, but sufficient in the absence of others. * [Now we resolve ties in favor of the earlier string if * it happens that c_offset_min has been invalidated, since the * earlier string may buy us something the later one won't.] */ 1981534 minlen = 0; 1981534 data.longest_fixed = newSVpvn("",0); 1981534 data.longest_float = newSVpvn("",0); 1981534 data.last_found = newSVpvn("",0); 1981534 data.longest = &(data.longest_fixed); 1981534 first = scan; 1981534 if (!r->regstclass) { 164985 cl_init(pRExC_state, &ch_class); 164985 data.start_class = &ch_class; 164985 stclass_flag = SCF_DO_STCLASS_AND; } else /* XXXX Check for BOUND? */ 1816549 stclass_flag = 0; 1981534 data.last_closep = &last_close; 1981534 minlen = study_chunk(pRExC_state, &first, &fake, scan + RExC_size, /* Up to end */ &data, SCF_DO_SUBSTR | SCF_WHILEM_VISITED_POS | stclass_flag,0); 1981526 if ( RExC_npar == 1 && data.longest == &(data.longest_fixed) && data.last_start_min == 0 && data.last_end > 0 && !RExC_seen_zerolen && (!(RExC_seen & REG_SEEN_GPOS) || (r->reganch & ROPT_ANCH_GPOS))) 40539 r->reganch |= ROPT_CHECK_ALL; 1981526 scan_commit(pRExC_state, &data); 1981526 SvREFCNT_dec(data.last_found); 1981526 longest_float_length = CHR_SVLEN(data.longest_float); 1981526 if (longest_float_length || (data.flags & SF_FL_BEFORE_EOL && (!(data.flags & SF_FL_BEFORE_MEOL) || (RExC_flags & PMf_MULTILINE)))) { 117553 int t; 117553 if (SvCUR(data.longest_fixed) /* ok to leave SvCUR */ && data.offset_fixed == data.offset_float_min && SvCUR(data.longest_fixed) == SvCUR(data.longest_float)) 5360 goto remove_float; /* As in (a)+. */ 112193 if (SvUTF8(data.longest_float)) { 43 r->float_utf8 = data.longest_float; 43 r->float_substr = Nullsv; } else { 112150 r->float_substr = data.longest_float; 112150 r->float_utf8 = Nullsv; } 112193 r->float_min_offset = data.offset_float_min; 112193 r->float_max_offset = data.offset_float_max; 112193 t = (data.flags & SF_FL_BEFORE_EOL /* Can't have SEOL and MULTI */ && (!(data.flags & SF_FL_BEFORE_MEOL) || (RExC_flags & PMf_MULTILINE))); 112193 fbm_compile(data.longest_float, t ? FBMcf_TAIL : 0); } else { remove_float: 1869333 r->float_substr = r->float_utf8 = Nullsv; 1869333 SvREFCNT_dec(data.longest_float); 1869333 longest_float_length = 0; } 1981526 longest_fixed_length = CHR_SVLEN(data.longest_fixed); 1981526 if (longest_fixed_length || (data.flags & SF_FIX_BEFORE_EOL /* Cannot have SEOL and MULTI */ && (!(data.flags & SF_FIX_BEFORE_MEOL) || (RExC_flags & PMf_MULTILINE)))) { 116816 int t; 116816 if (SvUTF8(data.longest_fixed)) { 2092 r->anchored_utf8 = data.longest_fixed; 2092 r->anchored_substr = Nullsv; } else { 114724 r->anchored_substr = data.longest_fixed; 114724 r->anchored_utf8 = Nullsv; } 116816 r->anch