lib/search/regex.c

   1 /*
   2    Search text engine.
   3    Regex search
   4
   5    Copyright (C) 2009-2016
   6    Free Software Foundation, Inc.
   7
   8    Written by:
   9    Slava Zanko <slavazanko@gmail.com>, 2009, 2010, 2011, 2013
  10    Vitaliy Filippov <vitalif@yourcmc.ru>, 2011
  11    Andrew Borodin <aborodin@vmail.ru>, 2013-2015
  12
  13    This file is part of the Midnight Commander.
  14
  15    The Midnight Commander is free software: you can redistribute it
  16    and/or modify it under the terms of the GNU General Public License as
  17    published by the Free Software Foundation, either version 3 of the License,
  18    or (at your option) any later version.
  19
  20    The Midnight Commander is distributed in the hope that it will be useful,
  21    but WITHOUT ANY WARRANTY; without even the implied warranty of
  22    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  23    GNU General Public License for more details.
  24
  25    You should have received a copy of the GNU General Public License
  26    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  27  */
  28
  29 #include <config.h>
  30
  31 #include <stdlib.h>
  32
  33 #include "lib/global.h"
  34 #include "lib/strutil.h"
  35 #include "lib/search.h"
  36 #include "lib/strescape.h"
  37 #include "lib/util.h"           /* MC_PTR_FREE */
  38
  39 #include "internal.h"
  40
  41 /*** global variables ****************************************************************************/
  42
  43 /*** file scope macro definitions ****************************************************************/
  44
  45 #define REPLACE_PREPARE_T_NOTHING_SPECIAL -1
  46 #define REPLACE_PREPARE_T_REPLACE_FLAG    -2
  47 #define REPLACE_PREPARE_T_ESCAPE_SEQ      -3
  48
  49 /*** file scope type declarations ****************************************************************/
  50
  51 typedef enum
  52 {
  53     REPLACE_T_NO_TRANSFORM = 0,
  54     REPLACE_T_UPP_TRANSFORM_CHAR = 1,
  55     REPLACE_T_LOW_TRANSFORM_CHAR = 2,
  56     REPLACE_T_UPP_TRANSFORM = 4,
  57     REPLACE_T_LOW_TRANSFORM = 8
  58 } replace_transform_type_t;
  59
  60
  61 /*** file scope variables ************************************************************************/
  62
  63 /*** file scope functions ************************************************************************/
  64
  65 static gboolean
  66 mc_search__regex_str_append_if_special (GString * copy_to, const GString * regex_str,
  67                                         gsize * offset)
  68 {
  69     const char *special_chars[] = {
  70         "\\s", "\\S",
  71         "\\d", "\\D",
  72         "\\b", "\\B",
  73         "\\w", "\\W",
  74         "\\t", "\\n",
  75         "\\r", "\\f",
  76         "\\a", "\\e",
  77         "\\x", "\\X",
  78         "\\c", "\\C",
  79         "\\l", "\\L",
  80         "\\u", "\\U",
  81         "\\E", "\\Q",
  82         NULL
  83     };
  84
  85     char *tmp_regex_str;
  86     const char **spec_chr;
  87
  88     tmp_regex_str = &(regex_str->str[*offset]);
  89
  90     for (spec_chr = special_chars; *spec_chr != NULL; spec_chr++)
  91     {
  92         gsize spec_chr_len;
  93
  94         spec_chr_len = strlen (*spec_chr);
  95
  96         if (strncmp (tmp_regex_str, *spec_chr, spec_chr_len) == 0
  97             && !strutils_is_char_escaped (regex_str->str, tmp_regex_str))
  98         {
  99             if (strncmp ("\\x", *spec_chr, spec_chr_len) == 0)
 100             {
 101                 if (tmp_regex_str[spec_chr_len] != '{')
 102                     spec_chr_len += 2;
 103                 else
 104                 {
 105                     while ((spec_chr_len < regex_str->len - *offset)
 106                            && tmp_regex_str[spec_chr_len] != '}')
 107                         spec_chr_len++;
 108                     if (tmp_regex_str[spec_chr_len] == '}')
 109                         spec_chr_len++;
 110                 }
 111             }
 112             g_string_append_len (copy_to, tmp_regex_str, spec_chr_len);
 113             *offset += spec_chr_len;
 114             return TRUE;
 115         }
 116     }
 117
 118     return FALSE;
 119 }
 120
 121 /* --------------------------------------------------------------------------------------------- */
 122
 123 static void
 124 mc_search__cond_struct_new_regex_hex_add (const char *charset, GString * str_to,
 125                                           const char *one_char, gsize str_len)
 126 {
 127     GString *upp, *low;
 128     gsize loop;
 129
 130     upp = mc_search__toupper_case_str (charset, one_char, str_len);
 131     low = mc_search__tolower_case_str (charset, one_char, str_len);
 132
 133     for (loop = 0; loop < upp->len; loop++)
 134     {
 135         gchar tmp_str[10 + 1];  /* longest content is "[\\x%02X\\x%02X]" */
 136         gint tmp_len;
 137
 138         if (loop >= low->len || upp->str[loop] == low->str[loop])
 139             tmp_len =
 140                 g_snprintf (tmp_str, sizeof (tmp_str), "\\x%02X", (unsigned char) upp->str[loop]);
 141         else
 142             tmp_len =
 143                 g_snprintf (tmp_str, sizeof (tmp_str), "[\\x%02X\\x%02X]",
 144                             (unsigned char) upp->str[loop], (unsigned char) low->str[loop]);
 145
 146         g_string_append_len (str_to, tmp_str, tmp_len);
 147     }
 148
 149     g_string_free (upp, TRUE);
 150     g_string_free (low, TRUE);
 151 }
 152
 153 /* --------------------------------------------------------------------------------------------- */
 154
 155 static void
 156 mc_search__cond_struct_new_regex_accum_append (const char *charset, GString * str_to,
 157                                                GString * str_from)
 158 {
 159     GString *recoded_part;
 160     gsize loop = 0;
 161
 162     recoded_part = g_string_sized_new (32);
 163
 164     while (loop < str_from->len)
 165     {
 166         gchar *one_char;
 167         gsize one_char_len;
 168         gboolean just_letters;
 169
 170         one_char =
 171             mc_search__get_one_symbol (charset, &(str_from->str[loop]),
 172                                        MIN (str_from->len - loop, 6), &just_letters);
 173         one_char_len = strlen (one_char);
 174
 175         if (one_char_len == 0)
 176             loop++;
 177         else
 178         {
 179             loop += one_char_len;
 180
 181             if (just_letters)
 182                 mc_search__cond_struct_new_regex_hex_add (charset, recoded_part, one_char,
 183                                                           one_char_len);
 184             else
 185                 g_string_append_len (recoded_part, one_char, one_char_len);
 186         }
 187
 188         g_free (one_char);
 189     }
 190
 191     g_string_append_len (str_to, recoded_part->str, recoded_part->len);
 192     g_string_free (recoded_part, TRUE);
 193     g_string_set_size (str_from, 0);
 194 }
 195
 196 /* --------------------------------------------------------------------------------------------- */
 197
 198 static GString *
 199 mc_search__cond_struct_new_regex_ci_str (const char *charset, const GString * astr)
 200 {
 201     GString *accumulator, *spec_char, *ret_str;
 202     gsize loop;
 203
 204     ret_str = g_string_sized_new (64);
 205     accumulator = g_string_sized_new (64);
 206     spec_char = g_string_sized_new (64);
 207     loop = 0;
 208
 209     while (loop <= astr->len)
 210     {
 211         if (mc_search__regex_str_append_if_special (spec_char, astr, &loop))
 212         {
 213             mc_search__cond_struct_new_regex_accum_append (charset, ret_str, accumulator);
 214             g_string_append_len (ret_str, spec_char->str, spec_char->len);
 215             g_string_set_size (spec_char, 0);
 216             continue;
 217         }
 218
 219         if (astr->str[loop] == '[' && !strutils_is_char_escaped (astr->str, &(astr->str[loop])))
 220         {
 221             mc_search__cond_struct_new_regex_accum_append (charset, ret_str, accumulator);
 222
 223             while (loop < astr->len && !(astr->str[loop] == ']'
 224                                          && !strutils_is_char_escaped (astr->str,
 225                                                                        &(astr->str[loop]))))
 226             {
 227                 g_string_append_c (ret_str, astr->str[loop]);
 228                 loop++;
 229             }
 230
 231             g_string_append_c (ret_str, astr->str[loop]);
 232             loop++;
 233             continue;
 234         }
 235         /*
 236            TODO: handle [ and ]
 237          */
 238         g_string_append_c (accumulator, astr->str[loop]);
 239         loop++;
 240     }
 241     mc_search__cond_struct_new_regex_accum_append (charset, ret_str, accumulator);
 242
 243     g_string_free (accumulator, TRUE);
 244     g_string_free (spec_char, TRUE);
 245
 246     return ret_str;
 247 }
 248
 249 /* --------------------------------------------------------------------------------------------- */
 250
 251 #ifdef SEARCH_TYPE_GLIB
 252 /* A thin wrapper above g_regex_match_full that makes sure the string passed
 253  * to it is valid UTF-8 (unless G_REGEX_RAW compile flag was set), as it is a
 254  * requirement by glib and it might crash otherwise. See: mc ticket 3449.
 255  * Be careful: there might be embedded NULs in the strings. */
 256 static gboolean
 257 mc_search__g_regex_match_full_safe (const GRegex * regex,
 258                                     const gchar * string,
 259                                     gssize string_len,
 260                                     gint start_position,
 261                                     GRegexMatchFlags match_options,
 262                                     GMatchInfo ** match_info, GError ** error)
 263 {
 264     char *string_safe, *p, *end;
 265     gboolean ret;
 266
 267     if ((g_regex_get_compile_flags (regex) & G_REGEX_RAW)
 268         || g_utf8_validate (string, string_len, NULL))
 269     {
 270         return g_regex_match_full (regex, string, string_len, start_position, match_options,
 271                                    match_info, error);
 272     }
 273
 274     if (string_len < 0)
 275     {
 276         string_len = strlen (string);
 277     }
 278
 279     /* Correctly handle embedded NULs while copying */
 280     p = string_safe = g_malloc (string_len);
 281     memcpy (string_safe, string, string_len);
 282     end = p + string_len;
 283
 284     while (p < end)
 285     {
 286         gunichar c = g_utf8_get_char_validated (p, -1);
 287         if (c != (gunichar) (-1) && c != (gunichar) (-2))
 288         {
 289             p = g_utf8_next_char (p);
 290         }
 291         else
 292         {
 293             /* U+FFFD would be the proper choice, but then we'd have to
 294                maintain mapping between old and new offsets.
 295                So rather do a byte by byte replacement. */
 296             *p++ = '\0';
 297         }
 298     }
 299
 300     ret =
 301         g_regex_match_full (regex, string_safe, string_len, start_position, match_options,
 302                             match_info, error);
 303     g_free (string_safe);
 304     return ret;
 305 }
 306 #endif /* SEARCH_TYPE_GLIB */
 307
 308 /* --------------------------------------------------------------------------------------------- */
 309
 310 static mc_search__found_cond_t
 311 mc_search__regex_found_cond_one (mc_search_t * lc_mc_search, mc_search_regex_t * regex,
 312                                  GString * search_str)
 313 {
 314 #ifdef SEARCH_TYPE_GLIB
 315     GError *mcerror = NULL;
 316
 317     if (!mc_search__g_regex_match_full_safe
 318         (regex, search_str->str, search_str->len, 0, G_REGEX_MATCH_NEWLINE_ANY,
 319          &lc_mc_search->regex_match_info, &mcerror))
 320     {
 321         g_match_info_free (lc_mc_search->regex_match_info);
 322         lc_mc_search->regex_match_info = NULL;
 323         if (mcerror != NULL)
 324         {
 325             lc_mc_search->error = MC_SEARCH_E_REGEX;
 326             g_free (lc_mc_search->error_str);
 327             lc_mc_search->error_str =
 328                 str_conv_gerror_message (mcerror, _("Regular expression error"));
 329             g_error_free (mcerror);
 330             return COND__FOUND_ERROR;
 331         }
 332         return COND__NOT_FOUND;
 333     }
 334     lc_mc_search->num_results = g_match_info_get_match_count (lc_mc_search->regex_match_info);
 335 #else /* SEARCH_TYPE_GLIB */
 336     lc_mc_search->num_results = pcre_exec (regex, lc_mc_search->regex_match_info,
 337                                            search_str->str, search_str->len, 0, 0,
 338                                            lc_mc_search->iovector, MC_SEARCH__NUM_REPLACE_ARGS);
 339     if (lc_mc_search->num_results < 0)
 340     {
 341         return COND__NOT_FOUND;
 342     }
 343 #endif /* SEARCH_TYPE_GLIB */
 344     return COND__FOUND_OK;
 345
 346 }
 347
 348 /* --------------------------------------------------------------------------------------------- */
 349
 350 static mc_search__found_cond_t
 351 mc_search__regex_found_cond (mc_search_t * lc_mc_search, GString * search_str)
 352 {
 353     gsize loop1;
 354
 355     for (loop1 = 0; loop1 < lc_mc_search->conditions->len; loop1++)
 356     {
 357         mc_search_cond_t *mc_search_cond;
 358         mc_search__found_cond_t ret;
 359
 360         mc_search_cond = (mc_search_cond_t *) g_ptr_array_index (lc_mc_search->conditions, loop1);
 361
 362         if (!mc_search_cond->regex_handle)
 363             continue;
 364
 365         ret =
 366             mc_search__regex_found_cond_one (lc_mc_search, mc_search_cond->regex_handle,
 367                                              search_str);
 368         if (ret != COND__NOT_FOUND)
 369             return ret;
 370     }
 371     return COND__NOT_ALL_FOUND;
 372 }
 373
 374 /* --------------------------------------------------------------------------------------------- */
 375
 376 static int
 377 mc_search_regex__get_max_num_of_replace_tokens (const gchar * str, gsize len)
 378 {
 379     int max_token = 0;
 380     gsize loop;
 381     for (loop = 0; loop < len - 1; loop++)
 382     {
 383         if (str[loop] == '\\' && g_ascii_isdigit (str[loop + 1]))
 384         {
 385             if (strutils_is_char_escaped (str, &str[loop]))
 386                 continue;
 387             if (max_token < str[loop + 1] - '0')
 388                 max_token = str[loop + 1] - '0';
 389             continue;
 390         }
 391         if (str[loop] == '$' && str[loop + 1] == '{')
 392         {
 393             gsize tmp_len;
 394
 395             if (strutils_is_char_escaped (str, &str[loop]))
 396                 continue;
 397
 398             for (tmp_len = 0;
 399                  loop + tmp_len + 2 < len && (str[loop + 2 + tmp_len] & (char) 0xf0) == 0x30;
 400                  tmp_len++);
 401
 402             if (str[loop + 2 + tmp_len] == '}')
 403             {
 404                 int tmp_token;
 405                 char *tmp_str;
 406
 407                 tmp_str = g_strndup (&str[loop + 2], tmp_len);
 408                 tmp_token = atoi (tmp_str);
 409                 if (max_token < tmp_token)
 410                     max_token = tmp_token;
 411                 g_free (tmp_str);
 412             }
 413         }
 414     }
 415     return max_token;
 416 }
 417
 418 /* --------------------------------------------------------------------------------------------- */
 419
 420 static char *
 421 mc_search_regex__get_token_by_num (const mc_search_t * lc_mc_search, gsize lc_index)
 422 {
 423     int fnd_start = 0, fnd_end = 0;
 424
 425 #ifdef SEARCH_TYPE_GLIB
 426     g_match_info_fetch_pos (lc_mc_search->regex_match_info, lc_index, &fnd_start, &fnd_end);
 427 #else /* SEARCH_TYPE_GLIB */
 428     fnd_start = lc_mc_search->iovector[lc_index * 2 + 0];
 429     fnd_end = lc_mc_search->iovector[lc_index * 2 + 1];
 430 #endif /* SEARCH_TYPE_GLIB */
 431
 432     if (fnd_end == fnd_start)
 433         return g_strdup ("");
 434
 435     return g_strndup (lc_mc_search->regex_buffer->str + fnd_start, fnd_end - fnd_start);
 436
 437 }
 438
 439 /* --------------------------------------------------------------------------------------------- */
 440
 441 static gboolean
 442 mc_search_regex__replace_handle_esc_seq (const GString * replace_str, const gsize current_pos,
 443                                          gsize * skip_len, int *ret)
 444 {
 445     char *curr_str = &(replace_str->str[current_pos]);
 446     char c = curr_str[1];
 447
 448     if (replace_str->len > current_pos + 2)
 449     {
 450         if (c == '{')
 451         {
 452             for (*skip_len = 2; /* \{ */
 453                  current_pos + *skip_len < replace_str->len && curr_str[*skip_len] >= '0'
 454                  && curr_str[*skip_len] <= '7'; (*skip_len)++)
 455                 ;
 456
 457             if (current_pos + *skip_len < replace_str->len && curr_str[*skip_len] == '}')
 458             {
 459                 (*skip_len)++;
 460                 *ret = REPLACE_PREPARE_T_ESCAPE_SEQ;
 461                 return FALSE;
 462             }
 463             else
 464             {
 465                 *ret = REPLACE_PREPARE_T_NOTHING_SPECIAL;
 466                 return TRUE;
 467             }
 468         }
 469
 470         if (c == 'x')
 471         {
 472             *skip_len = 2;      /* \x */
 473             c = curr_str[2];
 474             if (c == '{')
 475             {
 476                 for (*skip_len = 3;     /* \x{ */
 477                      current_pos + *skip_len < replace_str->len
 478                      && g_ascii_isxdigit ((guchar) curr_str[*skip_len]); (*skip_len)++)
 479                     ;
 480
 481                 if (current_pos + *skip_len < replace_str->len && curr_str[*skip_len] == '}')
 482                 {
 483                     (*skip_len)++;
 484                     *ret = REPLACE_PREPARE_T_ESCAPE_SEQ;
 485                     return FALSE;
 486                 }
 487                 else
 488                 {
 489                     *ret = REPLACE_PREPARE_T_NOTHING_SPECIAL;
 490                     return TRUE;
 491                 }
 492             }
 493             else if (!g_ascii_isxdigit ((guchar) c))
 494             {
 495                 *skip_len = 2;  /* \x without number behind */
 496                 *ret = REPLACE_PREPARE_T_NOTHING_SPECIAL;
 497                 return FALSE;
 498             }
 499             else
 500             {
 501                 c = curr_str[3];
 502                 if (!g_ascii_isxdigit ((guchar) c))
 503                     *skip_len = 3;      /* \xH */
 504                 else
 505                     *skip_len = 4;      /* \xHH */
 506                 *ret = REPLACE_PREPARE_T_ESCAPE_SEQ;
 507                 return FALSE;
 508             }
 509         }
 510     }
 511
 512     if (strchr ("ntvbrfa", c) != NULL)
 513     {
 514         *skip_len = 2;
 515         *ret = REPLACE_PREPARE_T_ESCAPE_SEQ;
 516         return FALSE;
 517     }
 518     return TRUE;
 519 }
 520
 521 /* --------------------------------------------------------------------------------------------- */
 522
 523 static int
 524 mc_search_regex__process_replace_str (const GString * replace_str, const gsize current_pos,
 525                                       gsize * skip_len, replace_transform_type_t * replace_flags)
 526 {
 527     int ret = -1;
 528     const char *curr_str = &(replace_str->str[current_pos]);
 529
 530     if (current_pos > replace_str->len)
 531         return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 532
 533     *skip_len = 0;
 534
 535     if (replace_str->len > current_pos + 2 && curr_str[0] == '$' && curr_str[1] == '{'
 536         && (curr_str[2] & (char) 0xf0) == 0x30)
 537     {
 538         char *tmp_str;
 539
 540         if (strutils_is_char_escaped (replace_str->str, curr_str))
 541         {
 542             *skip_len = 1;
 543             return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 544         }
 545
 546         for (*skip_len = 0;
 547              current_pos + *skip_len + 2 < replace_str->len
 548              && (curr_str[2 + *skip_len] & (char) 0xf0) == 0x30; (*skip_len)++)
 549             ;
 550
 551         if (curr_str[2 + *skip_len] != '}')
 552             return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 553
 554         tmp_str = g_strndup (curr_str + 2, *skip_len);
 555         if (tmp_str == NULL)
 556             return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 557
 558         ret = atoi (tmp_str);
 559         g_free (tmp_str);
 560
 561         *skip_len += 3;         /* ${} */
 562         return ret;             /* capture buffer index >= 0 */
 563     }
 564
 565     if (curr_str[0] == '\\' && replace_str->len > current_pos + 1)
 566     {
 567         if (strutils_is_char_escaped (replace_str->str, curr_str))
 568         {
 569             *skip_len = 1;
 570             return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 571         }
 572
 573         if (g_ascii_isdigit (curr_str[1]))
 574         {
 575             ret = g_ascii_digit_value (curr_str[1]);    /* capture buffer index >= 0 */
 576             *skip_len = 2;      /* \\ and one digit */
 577             return ret;
 578         }
 579
 580         if (!mc_search_regex__replace_handle_esc_seq (replace_str, current_pos, skip_len, &ret))
 581             return ret;
 582
 583         ret = REPLACE_PREPARE_T_REPLACE_FLAG;
 584         *skip_len += 2;
 585
 586         switch (curr_str[1])
 587         {
 588         case 'U':
 589             *replace_flags |= REPLACE_T_UPP_TRANSFORM;
 590             *replace_flags &= ~REPLACE_T_LOW_TRANSFORM;
 591             break;
 592         case 'u':
 593             *replace_flags |= REPLACE_T_UPP_TRANSFORM_CHAR;
 594             break;
 595         case 'L':
 596             *replace_flags |= REPLACE_T_LOW_TRANSFORM;
 597             *replace_flags &= ~REPLACE_T_UPP_TRANSFORM;
 598             break;
 599         case 'l':
 600             *replace_flags |= REPLACE_T_LOW_TRANSFORM_CHAR;
 601             break;
 602         case 'E':
 603             *replace_flags = REPLACE_T_NO_TRANSFORM;
 604             break;
 605         default:
 606             ret = REPLACE_PREPARE_T_NOTHING_SPECIAL;
 607             break;
 608         }
 609     }
 610     return ret;
 611 }
 612
 613 /* --------------------------------------------------------------------------------------------- */
 614
 615 static void
 616 mc_search_regex__process_append_str (GString * dest_str, const char *from, gsize len,
 617                                      replace_transform_type_t * replace_flags)
 618 {
 619     gsize loop;
 620     gsize char_len;
 621
 622     if (len == (gsize) (-1))
 623         len = strlen (from);
 624
 625     if (*replace_flags == REPLACE_T_NO_TRANSFORM)
 626     {
 627         g_string_append_len (dest_str, from, len);
 628         return;
 629     }
 630
 631     for (loop = 0; loop < len; loop += char_len)
 632     {
 633         GString *tmp_string = NULL;
 634         char *tmp_str;
 635
 636         tmp_str = mc_search__get_one_symbol (NULL, from + loop, len - loop, NULL);
 637         char_len = strlen (tmp_str);
 638
 639         if ((*replace_flags & REPLACE_T_UPP_TRANSFORM_CHAR) != 0)
 640         {
 641             *replace_flags &= ~REPLACE_T_UPP_TRANSFORM_CHAR;
 642             tmp_string = mc_search__toupper_case_str (NULL, tmp_str, char_len);
 643             g_string_append_len (dest_str, tmp_string->str, tmp_string->len);
 644             g_string_free (tmp_string, TRUE);
 645         }
 646         else if ((*replace_flags & REPLACE_T_LOW_TRANSFORM_CHAR) != 0)
 647         {
 648             *replace_flags &= ~REPLACE_T_LOW_TRANSFORM_CHAR;
 649             tmp_string = mc_search__tolower_case_str (NULL, tmp_str, char_len);
 650             g_string_append_len (dest_str, tmp_string->str, tmp_string->len);
 651             g_string_free (tmp_string, TRUE);
 652         }
 653         else if ((*replace_flags & REPLACE_T_UPP_TRANSFORM) != 0)
 654         {
 655             tmp_string = mc_search__toupper_case_str (NULL, tmp_str, char_len);
 656             g_string_append_len (dest_str, tmp_string->str, tmp_string->len);
 657             g_string_free (tmp_string, TRUE);
 658         }
 659         else if ((*replace_flags & REPLACE_T_LOW_TRANSFORM) != 0)
 660         {
 661             tmp_string = mc_search__tolower_case_str (NULL, tmp_str, char_len);
 662             g_string_append_len (dest_str, tmp_string->str, tmp_string->len);
 663             g_string_free (tmp_string, TRUE);
 664         }
 665
 666         g_free (tmp_str);
 667     }
 668 }
 669
 670 /* --------------------------------------------------------------------------------------------- */
 671
 672 static void
 673 mc_search_regex__process_escape_sequence (GString * dest_str, const char *from, gsize len,
 674                                           replace_transform_type_t * replace_flags,
 675                                           gboolean is_utf8)
 676 {
 677     gsize i = 0;
 678     unsigned int c = 0;
 679     char b;
 680
 681     if (len == (gsize) (-1))
 682         len = strlen (from);
 683     if (len == 0)
 684         return;
 685
 686     if (from[i] == '{')
 687         i++;
 688     if (i >= len)
 689         return;
 690
 691     if (from[i] == 'x')
 692     {
 693         i++;
 694         if (i < len && from[i] == '{')
 695             i++;
 696         for (; i < len; i++)
 697         {
 698             if (from[i] >= '0' && from[i] <= '9')
 699                 c = c * 16 + from[i] - '0';
 700             else if (from[i] >= 'a' && from[i] <= 'f')
 701                 c = c * 16 + 10 + from[i] - 'a';
 702             else if (from[i] >= 'A' && from[i] <= 'F')
 703                 c = c * 16 + 10 + from[i] - 'A';
 704             else
 705                 break;
 706         }
 707     }
 708     else if (from[i] >= '0' && from[i] <= '7')
 709         for (; i < len && from[i] >= '0' && from[i] <= '7'; i++)
 710             c = c * 8 + from[i] - '0';
 711     else
 712     {
 713         switch (from[i])
 714         {
 715         case 'n':
 716             c = '\n';
 717             break;
 718         case 't':
 719             c = '\t';
 720             break;
 721         case 'v':
 722             c = '\v';
 723             break;
 724         case 'b':
 725             c = '\b';
 726             break;
 727         case 'r':
 728             c = '\r';
 729             break;
 730         case 'f':
 731             c = '\f';
 732             break;
 733         case 'a':
 734             c = '\a';
 735             break;
 736         default:
 737             mc_search_regex__process_append_str (dest_str, from, len, replace_flags);
 738             return;
 739         }
 740     }
 741
 742     if (c < 0x80 || !is_utf8)
 743         g_string_append_c (dest_str, (char) c);
 744     else if (c < 0x800)
 745     {
 746         b = 0xC0 | (c >> 6);
 747         g_string_append_c (dest_str, b);
 748         b = 0x80 | (c & 0x3F);
 749         g_string_append_c (dest_str, b);
 750     }
 751     else if (c < 0x10000)
 752     {
 753         b = 0xE0 | (c >> 12);
 754         g_string_append_c (dest_str, b);
 755         b = 0x80 | ((c >> 6) & 0x3F);
 756         g_string_append_c (dest_str, b);
 757         b = 0x80 | (c & 0x3F);
 758         g_string_append_c (dest_str, b);
 759     }
 760     else if (c < 0x10FFFF)
 761     {
 762         b = 0xF0 | (c >> 16);
 763         g_string_append_c (dest_str, b);
 764         b = 0x80 | ((c >> 12) & 0x3F);
 765         g_string_append_c (dest_str, b);
 766         b = 0x80 | ((c >> 6) & 0x3F);
 767         g_string_append_c (dest_str, b);
 768         b = 0x80 | (c & 0x3F);
 769         g_string_append_c (dest_str, b);
 770     }
 771 }
 772
 773 /* --------------------------------------------------------------------------------------------- */
 774 /*** public functions ****************************************************************************/
 775 /* --------------------------------------------------------------------------------------------- */
 776
 777 void
 778 mc_search__cond_struct_new_init_regex (const char *charset, mc_search_t * lc_mc_search,
 779                                        mc_search_cond_t * mc_search_cond)
 780 {
 781     if (lc_mc_search->whole_words && !lc_mc_search->is_entire_line)
 782     {
 783         /* NOTE: \b as word boundary doesn't allow search
 784          * whole words with non-ASCII symbols.
 785          * Update: Is it still true nowadays? Probably not. #2396, #3524 */
 786         g_string_prepend (mc_search_cond->str, "(?<![\\p{L}\\p{N}_])");
 787         g_string_append (mc_search_cond->str, "(?![\\p{L}\\p{N}_])");
 788     }
 789
 790     {
 791 #ifdef SEARCH_TYPE_GLIB
 792         GError *mcerror = NULL;
 793         GRegexCompileFlags g_regex_options = G_REGEX_OPTIMIZE | G_REGEX_DOTALL;
 794
 795         if (str_isutf8 (charset) && mc_global.utf8_display)
 796         {
 797             if (!lc_mc_search->is_case_sensitive)
 798                 g_regex_options |= G_REGEX_CASELESS;
 799         }
 800         else
 801         {
 802             g_regex_options |= G_REGEX_RAW;
 803
 804             if (!lc_mc_search->is_case_sensitive)
 805             {
 806                 GString *tmp;
 807
 808                 tmp = mc_search_cond->str;
 809                 mc_search_cond->str = mc_search__cond_struct_new_regex_ci_str (charset, tmp);
 810                 g_string_free (tmp, TRUE);
 811
 812             }
 813         }
 814
 815         mc_search_cond->regex_handle =
 816             g_regex_new (mc_search_cond->str->str, g_regex_options, 0, &mcerror);
 817
 818         if (mcerror != NULL)
 819         {
 820             lc_mc_search->error = MC_SEARCH_E_REGEX_COMPILE;
 821             g_free (lc_mc_search->error_str);
 822             lc_mc_search->error_str =
 823                 str_conv_gerror_message (mcerror, _("Regular expression error"));
 824             g_error_free (mcerror);
 825             return;
 826         }
 827 #else /* SEARCH_TYPE_GLIB */
 828         const char *error;
 829         int erroffset;
 830         int pcre_options = PCRE_EXTRA | PCRE_MULTILINE;
 831
 832         if (str_isutf8 (charset) && mc_global.utf8_display)
 833         {
 834             pcre_options |= PCRE_UTF8;
 835             if (!lc_mc_search->is_case_sensitive)
 836                 pcre_options |= PCRE_CASELESS;
 837         }
 838         else
 839         {
 840             if (!lc_mc_search->is_case_sensitive)
 841             {
 842                 GString *tmp;
 843
 844                 tmp = mc_search_cond->str;
 845                 mc_search_cond->str = mc_search__cond_struct_new_regex_ci_str (charset, tmp);
 846                 g_string_free (tmp, TRUE);
 847             }
 848         }
 849
 850         mc_search_cond->regex_handle =
 851             pcre_compile (mc_search_cond->str->str, pcre_options, &error, &erroffset, NULL);
 852         if (mc_search_cond->regex_handle == NULL)
 853         {
 854             mc_search_set_error (lc_mc_search, MC_SEARCH_E_REGEX_COMPILE, "%s", error);
 855             return;
 856         }
 857         lc_mc_search->regex_match_info = pcre_study (mc_search_cond->regex_handle, 0, &error);
 858         if (lc_mc_search->regex_match_info == NULL && error != NULL)
 859         {
 860             mc_search_set_error (lc_mc_search, MC_SEARCH_E_REGEX_COMPILE, "%s", error);
 861             MC_PTR_FREE (mc_search_cond->regex_handle);
 862             return;
 863         }
 864 #endif /* SEARCH_TYPE_GLIB */
 865     }
 866
 867     lc_mc_search->is_utf8 = str_isutf8 (charset);
 868 }
 869
 870 /* --------------------------------------------------------------------------------------------- */
 871
 872 gboolean
 873 mc_search__run_regex (mc_search_t * lc_mc_search, const void *user_data,
 874                       gsize start_search, gsize end_search, gsize * found_len)
 875 {
 876     mc_search_cbret_t ret = MC_SEARCH_CB_NOTFOUND;
 877     gsize current_pos, virtual_pos;
 878     gint start_pos;
 879     gint end_pos;
 880
 881     if (lc_mc_search->regex_buffer != NULL)
 882         g_string_set_size (lc_mc_search->regex_buffer, 0);
 883     else
 884         lc_mc_search->regex_buffer = g_string_sized_new (64);
 885
 886     virtual_pos = current_pos = start_search;
 887     while (virtual_pos <= end_search)
 888     {
 889         g_string_set_size (lc_mc_search->regex_buffer, 0);
 890         lc_mc_search->start_buffer = current_pos;
 891
 892         if (lc_mc_search->search_fn != NULL)
 893         {
 894             while (TRUE)
 895             {
 896                 int current_chr = '\n'; /* stop search symbol */
 897
 898                 ret = lc_mc_search->search_fn (user_data, current_pos, &current_chr);
 899
 900                 if (ret == MC_SEARCH_CB_ABORT)
 901                     break;
 902
 903                 if (ret == MC_SEARCH_CB_INVALID)
 904                     continue;
 905
 906                 current_pos++;
 907
 908                 if (ret == MC_SEARCH_CB_SKIP)
 909                     continue;
 910
 911                 virtual_pos++;
 912
 913                 g_string_append_c (lc_mc_search->regex_buffer, (char) current_chr);
 914
 915                 if ((char) current_chr == '\n' || virtual_pos > end_search)
 916                     break;
 917             }
 918         }
 919         else
 920         {
 921             /* optimization for standard case (for search from file manager)
 922              *  where there is no MC_SEARCH_CB_INVALID or MC_SEARCH_CB_SKIP
 923              *  return codes, so we can copy line at regex buffer all at once
 924              */
 925             while (TRUE)
 926             {
 927                 const char current_chr = ((const char *) user_data)[current_pos];
 928
 929                 if (current_chr == '\0')
 930                     break;
 931
 932                 current_pos++;
 933
 934                 if (current_chr == '\n' || current_pos > end_search)
 935                     break;
 936             }
 937
 938             /* use virtual_pos as index of start of current chunk */
 939             g_string_append_len (lc_mc_search->regex_buffer, (const char *) user_data + virtual_pos,
 940                                  current_pos - virtual_pos);
 941             virtual_pos = current_pos;
 942         }
 943
 944         switch (mc_search__regex_found_cond (lc_mc_search, lc_mc_search->regex_buffer))
 945         {
 946         case COND__FOUND_OK:
 947 #ifdef SEARCH_TYPE_GLIB
 948             g_match_info_fetch_pos (lc_mc_search->regex_match_info, 0, &start_pos, &end_pos);
 949 #else /* SEARCH_TYPE_GLIB */
 950             start_pos = lc_mc_search->iovector[0];
 951             end_pos = lc_mc_search->iovector[1];
 952 #endif /* SEARCH_TYPE_GLIB */
 953             if (found_len != NULL)
 954                 *found_len = end_pos - start_pos;
 955             lc_mc_search->normal_offset = lc_mc_search->start_buffer + start_pos;
 956             return TRUE;
 957         case COND__NOT_ALL_FOUND:
 958             break;
 959         default:
 960             g_string_free (lc_mc_search->regex_buffer, TRUE);
 961             lc_mc_search->regex_buffer = NULL;
 962             return FALSE;
 963         }
 964
 965         if ((lc_mc_search->update_fn != NULL) &&
 966             ((lc_mc_search->update_fn) (user_data, current_pos) == MC_SEARCH_CB_ABORT))
 967             ret = MC_SEARCH_CB_ABORT;
 968
 969         if (ret == MC_SEARCH_CB_ABORT || ret == MC_SEARCH_CB_NOTFOUND)
 970             break;
 971     }
 972
 973     g_string_free (lc_mc_search->regex_buffer, TRUE);
 974     lc_mc_search->regex_buffer = NULL;
 975
 976     MC_PTR_FREE (lc_mc_search->error_str);
 977     lc_mc_search->error = ret == MC_SEARCH_CB_ABORT ? MC_SEARCH_E_ABORT : MC_SEARCH_E_NOTFOUND;
 978
 979     return FALSE;
 980 }
 981
 982 /* --------------------------------------------------------------------------------------------- */
 983
 984 GString *
 985 mc_search_regex_prepare_replace_str (mc_search_t * lc_mc_search, GString * replace_str)
 986 {
 987     GString *ret;
 988
 989     int num_replace_tokens;
 990     gsize loop;
 991     gsize prev = 0;
 992     replace_transform_type_t replace_flags = REPLACE_T_NO_TRANSFORM;
 993
 994     num_replace_tokens =
 995         mc_search_regex__get_max_num_of_replace_tokens (replace_str->str, replace_str->len);
 996
 997     if (lc_mc_search->num_results < 0)
 998         return g_string_new_len (replace_str->str, replace_str->len);
 999
1000     if (num_replace_tokens > lc_mc_search->num_results - 1
1001         || num_replace_tokens > MC_SEARCH__NUM_REPLACE_ARGS)
1002     {
1003         mc_search_set_error (lc_mc_search, MC_SEARCH_E_REGEX_REPLACE, "%s",
1004                              _(STR_E_RPL_NOT_EQ_TO_FOUND));
1005         return NULL;
1006     }
1007
1008     ret = g_string_sized_new (64);
1009
1010     for (loop = 0; loop < replace_str->len - 1; loop++)
1011     {
1012         int lc_index;
1013         gchar *tmp_str;
1014         gsize len = 0;
1015
1016         lc_index = mc_search_regex__process_replace_str (replace_str, loop, &len, &replace_flags);
1017
1018         if (lc_index == REPLACE_PREPARE_T_NOTHING_SPECIAL)
1019         {
1020             if (len != 0)
1021             {
1022                 mc_search_regex__process_append_str (ret, replace_str->str + prev, loop - prev,
1023                                                      &replace_flags);
1024                 mc_search_regex__process_append_str (ret, replace_str->str + loop + 1, len - 1,
1025                                                      &replace_flags);
1026                 prev = loop + len;
1027                 loop = prev - 1;        /* prepare to loop++ */
1028             }
1029
1030             continue;
1031         }
1032
1033         if (lc_index == REPLACE_PREPARE_T_REPLACE_FLAG)
1034         {
1035             if (loop != 0)
1036                 mc_search_regex__process_append_str (ret, replace_str->str + prev, loop - prev,
1037                                                      &replace_flags);
1038             prev = loop + len;
1039             loop = prev - 1;    /* prepare to loop++ */
1040             continue;
1041         }
1042
1043         /* escape sequence */
1044         if (lc_index == REPLACE_PREPARE_T_ESCAPE_SEQ)
1045         {
1046             mc_search_regex__process_append_str (ret, replace_str->str + prev, loop - prev,
1047                                                  &replace_flags);
1048             /* call process_escape_sequence without starting '\\' */
1049             mc_search_regex__process_escape_sequence (ret, replace_str->str + loop + 1, len - 1,
1050                                                       &replace_flags, lc_mc_search->is_utf8);
1051             prev = loop + len;
1052             loop = prev - 1;    /* prepare to loop++ */
1053             continue;
1054         }
1055
1056         /* invalid capture buffer number */
1057         if (lc_index > lc_mc_search->num_results)
1058         {
1059             g_string_free (ret, TRUE);
1060             mc_search_set_error (lc_mc_search, MC_SEARCH_E_REGEX_REPLACE,
1061                                  _(STR_E_RPL_INVALID_TOKEN), lc_index);
1062             return NULL;
1063         }
1064
1065         tmp_str = mc_search_regex__get_token_by_num (lc_mc_search, lc_index);
1066
1067         if (loop != 0)
1068             mc_search_regex__process_append_str (ret, replace_str->str + prev, loop - prev,
1069                                                  &replace_flags);
1070
1071         mc_search_regex__process_append_str (ret, tmp_str, -1, &replace_flags);
1072         g_free (tmp_str);
1073
1074         prev = loop + len;
1075         loop = prev - 1;        /* prepare to loop++ */
1076     }
1077
1078     mc_search_regex__process_append_str (ret, replace_str->str + prev, replace_str->len - prev,
1079                                          &replace_flags);
1080
1081     return ret;
1082 }