lib/search/regex.c

   1 /*
   2    Search text engine.
   3    Regex search
   4
   5    Copyright (C) 2009-2022
   6    Free Software Foundation, Inc.
   7
   8    Written by:
   9    Slava Zanko <slavazanko@gmail.com>, 2009, 2010, 2011, 2013
  10    Vitaliy Filippov <vitalif@yourcmc.ru>, 2011
  11    Andrew Borodin <aborodin@vmail.ru>, 2013-2015
  12
  13    This file is part of the Midnight Commander.
  14
  15    The Midnight Commander is free software: you can redistribute it
  16    and/or modify it under the terms of the GNU General Public License as
  17    published by the Free Software Foundation, either version 3 of the License,
  18    or (at your option) any later version.
  19
  20    The Midnight Commander is distributed in the hope that it will be useful,
  21    but WITHOUT ANY WARRANTY; without even the implied warranty of
  22    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  23    GNU General Public License for more details.
  24
  25    You should have received a copy of the GNU General Public License
  26    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  27  */
  28
  29 #include <config.h>
  30
  31 #include <stdlib.h>
  32
  33 #include "lib/global.h"
  34 #include "lib/strutil.h"
  35 #include "lib/search.h"
  36 #include "lib/strescape.h"
  37 #include "lib/util.h"           /* MC_PTR_FREE */
  38
  39 #include "internal.h"
  40
  41 /*** global variables ****************************************************************************/
  42
  43 /*** file scope macro definitions ****************************************************************/
  44
  45 #define REPLACE_PREPARE_T_NOTHING_SPECIAL -1
  46 #define REPLACE_PREPARE_T_REPLACE_FLAG    -2
  47 #define REPLACE_PREPARE_T_ESCAPE_SEQ      -3
  48
  49 /*** file scope type declarations ****************************************************************/
  50
  51 typedef enum
  52 {
  53     REPLACE_T_NO_TRANSFORM = 0,
  54     REPLACE_T_UPP_TRANSFORM_CHAR = 1,
  55     REPLACE_T_LOW_TRANSFORM_CHAR = 2,
  56     REPLACE_T_UPP_TRANSFORM = 4,
  57     REPLACE_T_LOW_TRANSFORM = 8
  58 } replace_transform_type_t;
  59
  60
  61 /*** file scope variables ************************************************************************/
  62
  63 /*** file scope functions ************************************************************************/
  64
  65 static gboolean
  66 mc_search__regex_str_append_if_special (GString * copy_to, const GString * regex_str,
  67                                         gsize * offset)
  68 {
  69     const char *special_chars[] = {
  70         "\\s", "\\S",
  71         "\\d", "\\D",
  72         "\\b", "\\B",
  73         "\\w", "\\W",
  74         "\\t", "\\n",
  75         "\\r", "\\f",
  76         "\\a", "\\e",
  77         "\\x", "\\X",
  78         "\\c", "\\C",
  79         "\\l", "\\L",
  80         "\\u", "\\U",
  81         "\\E", "\\Q",
  82         NULL
  83     };
  84
  85     char *tmp_regex_str;
  86     const char **spec_chr;
  87
  88     tmp_regex_str = &(regex_str->str[*offset]);
  89
  90     for (spec_chr = special_chars; *spec_chr != NULL; spec_chr++)
  91     {
  92         gsize spec_chr_len;
  93
  94         spec_chr_len = strlen (*spec_chr);
  95
  96         if (strncmp (tmp_regex_str, *spec_chr, spec_chr_len) == 0
  97             && !strutils_is_char_escaped (regex_str->str, tmp_regex_str))
  98         {
  99             if (strncmp ("\\x", *spec_chr, spec_chr_len) == 0)
 100             {
 101                 if (tmp_regex_str[spec_chr_len] != '{')
 102                     spec_chr_len += 2;
 103                 else
 104                 {
 105                     while ((spec_chr_len < regex_str->len - *offset)
 106                            && tmp_regex_str[spec_chr_len] != '}')
 107                         spec_chr_len++;
 108                     if (tmp_regex_str[spec_chr_len] == '}')
 109                         spec_chr_len++;
 110                 }
 111             }
 112             g_string_append_len (copy_to, tmp_regex_str, spec_chr_len);
 113             *offset += spec_chr_len;
 114             return TRUE;
 115         }
 116     }
 117
 118     return FALSE;
 119 }
 120
 121 /* --------------------------------------------------------------------------------------------- */
 122
 123 static void
 124 mc_search__cond_struct_new_regex_hex_add (const char *charset, GString * str_to,
 125                                           const GString * one_char)
 126 {
 127     GString *upp, *low;
 128     gsize loop;
 129
 130     upp = mc_search__toupper_case_str (charset, one_char);
 131     low = mc_search__tolower_case_str (charset, one_char);
 132
 133     for (loop = 0; loop < upp->len; loop++)
 134     {
 135         gchar tmp_str[10 + 1];  /* longest content is "[\\x%02X\\x%02X]" */
 136         gint tmp_len;
 137
 138         if (loop >= low->len || upp->str[loop] == low->str[loop])
 139             tmp_len =
 140                 g_snprintf (tmp_str, sizeof (tmp_str), "\\x%02X", (unsigned char) upp->str[loop]);
 141         else
 142             tmp_len =
 143                 g_snprintf (tmp_str, sizeof (tmp_str), "[\\x%02X\\x%02X]",
 144                             (unsigned char) upp->str[loop], (unsigned char) low->str[loop]);
 145
 146         g_string_append_len (str_to, tmp_str, tmp_len);
 147     }
 148
 149     g_string_free (upp, TRUE);
 150     g_string_free (low, TRUE);
 151 }
 152
 153 /* --------------------------------------------------------------------------------------------- */
 154
 155 static void
 156 mc_search__cond_struct_new_regex_accum_append (const char *charset, GString * str_to,
 157                                                GString * str_from)
 158 {
 159     GString *recoded_part;
 160     gsize loop = 0;
 161
 162     recoded_part = g_string_sized_new (32);
 163
 164     while (loop < str_from->len)
 165     {
 166         GString *one_char;
 167         gboolean just_letters;
 168
 169         one_char =
 170             mc_search__get_one_symbol (charset, str_from->str + loop,
 171                                        MIN (str_from->len - loop, 6), &just_letters);
 172
 173         if (one_char->len == 0)
 174             loop++;
 175         else
 176         {
 177             loop += one_char->len;
 178
 179             if (just_letters)
 180                 mc_search__cond_struct_new_regex_hex_add (charset, recoded_part, one_char);
 181             else
 182                 g_string_append_len (recoded_part, one_char->str, one_char->len);
 183         }
 184
 185         g_string_free (one_char, TRUE);
 186     }
 187
 188     g_string_append_len (str_to, recoded_part->str, recoded_part->len);
 189     g_string_free (recoded_part, TRUE);
 190     g_string_set_size (str_from, 0);
 191 }
 192
 193 /* --------------------------------------------------------------------------------------------- */
 194
 195 /**
 196  * Creates a case-insensitive version of a regex pattern.
 197  *
 198  * For example (assuming ASCII charset): given "\\bHello!\\xAB", returns
 199  * "\\b[Hh][Ee][Ll][Ll][Oo]!\\xAB" (this example is for easier reading; in
 200  * reality hex codes are used instead of letters).
 201  *
 202  * This function knows not to ruin special regex symbols.
 203  *
 204  * This function is used when working with non-UTF-8 charsets: GLib's
 205  * regex engine doesn't understand such charsets and therefore can't do
 206  * this job itself.
 207  */
 208 static GString *
 209 mc_search__cond_struct_new_regex_ci_str (const char *charset, const GString * astr)
 210 {
 211     GString *accumulator, *spec_char, *ret_str;
 212     gsize loop;
 213
 214     ret_str = g_string_sized_new (64);
 215     accumulator = g_string_sized_new (64);
 216     spec_char = g_string_sized_new (64);
 217     loop = 0;
 218
 219     while (loop < astr->len)
 220     {
 221         if (mc_search__regex_str_append_if_special (spec_char, astr, &loop))
 222         {
 223             mc_search__cond_struct_new_regex_accum_append (charset, ret_str, accumulator);
 224             g_string_append_len (ret_str, spec_char->str, spec_char->len);
 225             g_string_set_size (spec_char, 0);
 226             continue;
 227         }
 228
 229         if (astr->str[loop] == '[' && !strutils_is_char_escaped (astr->str, &(astr->str[loop])))
 230         {
 231             mc_search__cond_struct_new_regex_accum_append (charset, ret_str, accumulator);
 232
 233             while (loop < astr->len && !(astr->str[loop] == ']'
 234                                          && !strutils_is_char_escaped (astr->str,
 235                                                                        &(astr->str[loop]))))
 236             {
 237                 g_string_append_c (ret_str, astr->str[loop]);
 238                 loop++;
 239             }
 240
 241             g_string_append_c (ret_str, astr->str[loop]);
 242             loop++;
 243             continue;
 244         }
 245         /*
 246            TODO: handle [ and ]
 247          */
 248         g_string_append_c (accumulator, astr->str[loop]);
 249         loop++;
 250     }
 251     mc_search__cond_struct_new_regex_accum_append (charset, ret_str, accumulator);
 252
 253     g_string_free (accumulator, TRUE);
 254     g_string_free (spec_char, TRUE);
 255
 256     return ret_str;
 257 }
 258
 259 /* --------------------------------------------------------------------------------------------- */
 260
 261 #ifdef SEARCH_TYPE_GLIB
 262 /* A thin wrapper above g_regex_match_full that makes sure the string passed
 263  * to it is valid UTF-8 (unless G_REGEX_RAW compile flag was set), as it is a
 264  * requirement by glib and it might crash otherwise. See: mc ticket 3449.
 265  * Be careful: there might be embedded NULs in the strings. */
 266 static gboolean
 267 mc_search__g_regex_match_full_safe (const GRegex * regex,
 268                                     const gchar * string,
 269                                     gssize string_len,
 270                                     gint start_position,
 271                                     GRegexMatchFlags match_options,
 272                                     GMatchInfo ** match_info, GError ** error)
 273 {
 274     char *string_safe, *p, *end;
 275     gboolean ret;
 276
 277     if (string_len < 0)
 278         string_len = strlen (string);
 279
 280     if ((g_regex_get_compile_flags (regex) & G_REGEX_RAW)
 281         || g_utf8_validate (string, string_len, NULL))
 282     {
 283         return g_regex_match_full (regex, string, string_len, start_position, match_options,
 284                                    match_info, error);
 285     }
 286
 287     /* Correctly handle embedded NULs while copying */
 288     p = string_safe = g_malloc (string_len + 1);
 289     memcpy (string_safe, string, string_len);
 290     string_safe[string_len] = '\0';
 291     end = p + string_len;
 292
 293     while (p < end)
 294     {
 295         gunichar c = g_utf8_get_char_validated (p, -1);
 296         if (c != (gunichar) (-1) && c != (gunichar) (-2))
 297         {
 298             p = g_utf8_next_char (p);
 299         }
 300         else
 301         {
 302             /* U+FFFD would be the proper choice, but then we'd have to
 303                maintain mapping between old and new offsets.
 304                So rather do a byte by byte replacement. */
 305             *p++ = '\0';
 306         }
 307     }
 308
 309     ret =
 310         g_regex_match_full (regex, string_safe, string_len, start_position, match_options,
 311                             match_info, error);
 312     g_free (string_safe);
 313     return ret;
 314 }
 315 #endif /* SEARCH_TYPE_GLIB */
 316
 317 /* --------------------------------------------------------------------------------------------- */
 318
 319 static mc_search__found_cond_t
 320 mc_search__regex_found_cond_one (mc_search_t * lc_mc_search, mc_search_regex_t * regex,
 321                                  GString * search_str)
 322 {
 323 #ifdef SEARCH_TYPE_GLIB
 324     GError *mcerror = NULL;
 325
 326     if (!mc_search__g_regex_match_full_safe
 327         (regex, search_str->str, search_str->len, 0, G_REGEX_MATCH_NEWLINE_ANY,
 328          &lc_mc_search->regex_match_info, &mcerror))
 329     {
 330         g_match_info_free (lc_mc_search->regex_match_info);
 331         lc_mc_search->regex_match_info = NULL;
 332         if (mcerror != NULL)
 333         {
 334             lc_mc_search->error = MC_SEARCH_E_REGEX;
 335             g_free (lc_mc_search->error_str);
 336             lc_mc_search->error_str =
 337                 str_conv_gerror_message (mcerror, _("Regular expression error"));
 338             g_error_free (mcerror);
 339             return COND__FOUND_ERROR;
 340         }
 341         return COND__NOT_FOUND;
 342     }
 343     lc_mc_search->num_results = g_match_info_get_match_count (lc_mc_search->regex_match_info);
 344 #else /* SEARCH_TYPE_GLIB */
 345     lc_mc_search->num_results = pcre_exec (regex, lc_mc_search->regex_match_info,
 346                                            search_str->str, search_str->len, 0, 0,
 347                                            lc_mc_search->iovector, MC_SEARCH__NUM_REPLACE_ARGS);
 348     if (lc_mc_search->num_results < 0)
 349     {
 350         return COND__NOT_FOUND;
 351     }
 352 #endif /* SEARCH_TYPE_GLIB */
 353     return COND__FOUND_OK;
 354
 355 }
 356
 357 /* --------------------------------------------------------------------------------------------- */
 358
 359 static mc_search__found_cond_t
 360 mc_search__regex_found_cond (mc_search_t * lc_mc_search, GString * search_str)
 361 {
 362     gsize loop1;
 363
 364     for (loop1 = 0; loop1 < lc_mc_search->prepared.conditions->len; loop1++)
 365     {
 366         mc_search_cond_t *mc_search_cond;
 367         mc_search__found_cond_t ret;
 368
 369         mc_search_cond =
 370             (mc_search_cond_t *) g_ptr_array_index (lc_mc_search->prepared.conditions, loop1);
 371
 372         if (!mc_search_cond->regex_handle)
 373             continue;
 374
 375         ret =
 376             mc_search__regex_found_cond_one (lc_mc_search, mc_search_cond->regex_handle,
 377                                              search_str);
 378         if (ret != COND__NOT_FOUND)
 379             return ret;
 380     }
 381     return COND__NOT_ALL_FOUND;
 382 }
 383
 384 /* --------------------------------------------------------------------------------------------- */
 385
 386 static int
 387 mc_search_regex__get_max_num_of_replace_tokens (const gchar * str, gsize len)
 388 {
 389     int max_token = 0;
 390     gsize loop;
 391     for (loop = 0; loop < len - 1; loop++)
 392     {
 393         if (str[loop] == '\\' && g_ascii_isdigit (str[loop + 1]))
 394         {
 395             if (strutils_is_char_escaped (str, &str[loop]))
 396                 continue;
 397             if (max_token < str[loop + 1] - '0')
 398                 max_token = str[loop + 1] - '0';
 399             continue;
 400         }
 401         if (str[loop] == '$' && str[loop + 1] == '{')
 402         {
 403             gsize tmp_len;
 404
 405             if (strutils_is_char_escaped (str, &str[loop]))
 406                 continue;
 407
 408             for (tmp_len = 0;
 409                  loop + tmp_len + 2 < len && (str[loop + 2 + tmp_len] & (char) 0xf0) == 0x30;
 410                  tmp_len++);
 411
 412             if (str[loop + 2 + tmp_len] == '}')
 413             {
 414                 int tmp_token;
 415                 char *tmp_str;
 416
 417                 tmp_str = g_strndup (&str[loop + 2], tmp_len);
 418                 tmp_token = atoi (tmp_str);
 419                 if (max_token < tmp_token)
 420                     max_token = tmp_token;
 421                 g_free (tmp_str);
 422             }
 423         }
 424     }
 425     return max_token;
 426 }
 427
 428 /* --------------------------------------------------------------------------------------------- */
 429
 430 static char *
 431 mc_search_regex__get_token_by_num (const mc_search_t * lc_mc_search, gsize lc_index)
 432 {
 433     int fnd_start = 0, fnd_end = 0;
 434
 435 #ifdef SEARCH_TYPE_GLIB
 436     g_match_info_fetch_pos (lc_mc_search->regex_match_info, lc_index, &fnd_start, &fnd_end);
 437 #else /* SEARCH_TYPE_GLIB */
 438     fnd_start = lc_mc_search->iovector[lc_index * 2 + 0];
 439     fnd_end = lc_mc_search->iovector[lc_index * 2 + 1];
 440 #endif /* SEARCH_TYPE_GLIB */
 441
 442     if (fnd_end == fnd_start)
 443         return g_strdup ("");
 444
 445     return g_strndup (lc_mc_search->regex_buffer->str + fnd_start, fnd_end - fnd_start);
 446
 447 }
 448
 449 /* --------------------------------------------------------------------------------------------- */
 450
 451 static gboolean
 452 mc_search_regex__replace_handle_esc_seq (const GString * replace_str, const gsize current_pos,
 453                                          gsize * skip_len, int *ret)
 454 {
 455     char *curr_str = &(replace_str->str[current_pos]);
 456     char c = curr_str[1];
 457
 458     if (replace_str->len > current_pos + 2)
 459     {
 460         if (c == '{')
 461         {
 462             for (*skip_len = 2; /* \{ */
 463                  current_pos + *skip_len < replace_str->len && curr_str[*skip_len] >= '0'
 464                  && curr_str[*skip_len] <= '7'; (*skip_len)++)
 465                 ;
 466
 467             if (current_pos + *skip_len < replace_str->len && curr_str[*skip_len] == '}')
 468             {
 469                 (*skip_len)++;
 470                 *ret = REPLACE_PREPARE_T_ESCAPE_SEQ;
 471                 return FALSE;
 472             }
 473             else
 474             {
 475                 *ret = REPLACE_PREPARE_T_NOTHING_SPECIAL;
 476                 return TRUE;
 477             }
 478         }
 479
 480         if (c == 'x')
 481         {
 482             *skip_len = 2;      /* \x */
 483             c = curr_str[2];
 484             if (c == '{')
 485             {
 486                 for (*skip_len = 3;     /* \x{ */
 487                      current_pos + *skip_len < replace_str->len
 488                      && g_ascii_isxdigit ((guchar) curr_str[*skip_len]); (*skip_len)++)
 489                     ;
 490
 491                 if (current_pos + *skip_len < replace_str->len && curr_str[*skip_len] == '}')
 492                 {
 493                     (*skip_len)++;
 494                     *ret = REPLACE_PREPARE_T_ESCAPE_SEQ;
 495                     return FALSE;
 496                 }
 497                 else
 498                 {
 499                     *ret = REPLACE_PREPARE_T_NOTHING_SPECIAL;
 500                     return TRUE;
 501                 }
 502             }
 503             else if (!g_ascii_isxdigit ((guchar) c))
 504             {
 505                 *skip_len = 2;  /* \x without number behind */
 506                 *ret = REPLACE_PREPARE_T_NOTHING_SPECIAL;
 507                 return FALSE;
 508             }
 509             else
 510             {
 511                 c = curr_str[3];
 512                 if (!g_ascii_isxdigit ((guchar) c))
 513                     *skip_len = 3;      /* \xH */
 514                 else
 515                     *skip_len = 4;      /* \xHH */
 516                 *ret = REPLACE_PREPARE_T_ESCAPE_SEQ;
 517                 return FALSE;
 518             }
 519         }
 520     }
 521
 522     if (strchr ("ntvbrfa", c) != NULL)
 523     {
 524         *skip_len = 2;
 525         *ret = REPLACE_PREPARE_T_ESCAPE_SEQ;
 526         return FALSE;
 527     }
 528     return TRUE;
 529 }
 530
 531 /* --------------------------------------------------------------------------------------------- */
 532
 533 static int
 534 mc_search_regex__process_replace_str (const GString * replace_str, const gsize current_pos,
 535                                       gsize * skip_len, replace_transform_type_t * replace_flags)
 536 {
 537     int ret = -1;
 538     const char *curr_str = &(replace_str->str[current_pos]);
 539
 540     if (current_pos > replace_str->len)
 541         return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 542
 543     *skip_len = 0;
 544
 545     if (replace_str->len > current_pos + 2 && curr_str[0] == '$' && curr_str[1] == '{'
 546         && (curr_str[2] & (char) 0xf0) == 0x30)
 547     {
 548         char *tmp_str;
 549
 550         if (strutils_is_char_escaped (replace_str->str, curr_str))
 551         {
 552             *skip_len = 1;
 553             return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 554         }
 555
 556         for (*skip_len = 0;
 557              current_pos + *skip_len + 2 < replace_str->len
 558              && (curr_str[2 + *skip_len] & (char) 0xf0) == 0x30; (*skip_len)++)
 559             ;
 560
 561         if (curr_str[2 + *skip_len] != '}')
 562             return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 563
 564         tmp_str = g_strndup (curr_str + 2, *skip_len);
 565         if (tmp_str == NULL)
 566             return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 567
 568         ret = atoi (tmp_str);
 569         g_free (tmp_str);
 570
 571         *skip_len += 3;         /* ${} */
 572         return ret;             /* capture buffer index >= 0 */
 573     }
 574
 575     if (curr_str[0] == '\\' && replace_str->len > current_pos + 1)
 576     {
 577         if (strutils_is_char_escaped (replace_str->str, curr_str))
 578         {
 579             *skip_len = 1;
 580             return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 581         }
 582
 583         if (g_ascii_isdigit (curr_str[1]))
 584         {
 585             ret = g_ascii_digit_value (curr_str[1]);    /* capture buffer index >= 0 */
 586             *skip_len = 2;      /* \\ and one digit */
 587             return ret;
 588         }
 589
 590         if (!mc_search_regex__replace_handle_esc_seq (replace_str, current_pos, skip_len, &ret))
 591             return ret;
 592
 593         ret = REPLACE_PREPARE_T_REPLACE_FLAG;
 594         *skip_len += 2;
 595
 596         switch (curr_str[1])
 597         {
 598         case 'U':
 599             *replace_flags |= REPLACE_T_UPP_TRANSFORM;
 600             *replace_flags &= ~REPLACE_T_LOW_TRANSFORM;
 601             break;
 602         case 'u':
 603             *replace_flags |= REPLACE_T_UPP_TRANSFORM_CHAR;
 604             break;
 605         case 'L':
 606             *replace_flags |= REPLACE_T_LOW_TRANSFORM;
 607             *replace_flags &= ~REPLACE_T_UPP_TRANSFORM;
 608             break;
 609         case 'l':
 610             *replace_flags |= REPLACE_T_LOW_TRANSFORM_CHAR;
 611             break;
 612         case 'E':
 613             *replace_flags = REPLACE_T_NO_TRANSFORM;
 614             break;
 615         default:
 616             ret = REPLACE_PREPARE_T_NOTHING_SPECIAL;
 617             break;
 618         }
 619     }
 620     return ret;
 621 }
 622
 623 /* --------------------------------------------------------------------------------------------- */
 624
 625 static void
 626 mc_search_regex__process_append_str (GString * dest_str, const char *from, gsize len,
 627                                      replace_transform_type_t * replace_flags)
 628 {
 629     gsize loop;
 630     gsize char_len;
 631
 632     if (len == (gsize) (-1))
 633         len = strlen (from);
 634
 635     if (*replace_flags == REPLACE_T_NO_TRANSFORM)
 636     {
 637         g_string_append_len (dest_str, from, len);
 638         return;
 639     }
 640
 641     for (loop = 0; loop < len; loop += char_len)
 642     {
 643         GString *tmp_string = NULL;
 644         GString *s;
 645
 646         s = mc_search__get_one_symbol (NULL, from + loop, len - loop, NULL);
 647         char_len = s->len;
 648
 649         if ((*replace_flags & REPLACE_T_UPP_TRANSFORM_CHAR) != 0)
 650         {
 651             *replace_flags &= ~REPLACE_T_UPP_TRANSFORM_CHAR;
 652             tmp_string = mc_search__toupper_case_str (NULL, s);
 653             g_string_append_len (dest_str, tmp_string->str, tmp_string->len);
 654         }
 655         else if ((*replace_flags & REPLACE_T_LOW_TRANSFORM_CHAR) != 0)
 656         {
 657             *replace_flags &= ~REPLACE_T_LOW_TRANSFORM_CHAR;
 658             tmp_string = mc_search__tolower_case_str (NULL, s);
 659             g_string_append_len (dest_str, tmp_string->str, tmp_string->len);
 660         }
 661         else if ((*replace_flags & REPLACE_T_UPP_TRANSFORM) != 0)
 662         {
 663             tmp_string = mc_search__toupper_case_str (NULL, s);
 664             g_string_append_len (dest_str, tmp_string->str, tmp_string->len);
 665         }
 666         else if ((*replace_flags & REPLACE_T_LOW_TRANSFORM) != 0)
 667         {
 668             tmp_string = mc_search__tolower_case_str (NULL, s);
 669             g_string_append_len (dest_str, tmp_string->str, tmp_string->len);
 670         }
 671
 672         g_string_free (s, TRUE);
 673         if (tmp_string != NULL)
 674             g_string_free (tmp_string, TRUE);
 675     }
 676 }
 677
 678 /* --------------------------------------------------------------------------------------------- */
 679
 680 static void
 681 mc_search_regex__process_escape_sequence (GString * dest_str, const char *from, gsize len,
 682                                           replace_transform_type_t * replace_flags,
 683                                           gboolean is_utf8)
 684 {
 685     gsize i = 0;
 686     unsigned int c = 0;
 687     char b;
 688
 689     if (len == (gsize) (-1))
 690         len = strlen (from);
 691     if (len == 0)
 692         return;
 693
 694     if (from[i] == '{')
 695         i++;
 696     if (i >= len)
 697         return;
 698
 699     if (from[i] == 'x')
 700     {
 701         i++;
 702         if (i < len && from[i] == '{')
 703             i++;
 704         for (; i < len; i++)
 705         {
 706             if (from[i] >= '0' && from[i] <= '9')
 707                 c = c * 16 + from[i] - '0';
 708             else if (from[i] >= 'a' && from[i] <= 'f')
 709                 c = c * 16 + 10 + from[i] - 'a';
 710             else if (from[i] >= 'A' && from[i] <= 'F')
 711                 c = c * 16 + 10 + from[i] - 'A';
 712             else
 713                 break;
 714         }
 715     }
 716     else if (from[i] >= '0' && from[i] <= '7')
 717         for (; i < len && from[i] >= '0' && from[i] <= '7'; i++)
 718             c = c * 8 + from[i] - '0';
 719     else
 720     {
 721         switch (from[i])
 722         {
 723         case 'n':
 724             c = '\n';
 725             break;
 726         case 't':
 727             c = '\t';
 728             break;
 729         case 'v':
 730             c = '\v';
 731             break;
 732         case 'b':
 733             c = '\b';
 734             break;
 735         case 'r':
 736             c = '\r';
 737             break;
 738         case 'f':
 739             c = '\f';
 740             break;
 741         case 'a':
 742             c = '\a';
 743             break;
 744         default:
 745             mc_search_regex__process_append_str (dest_str, from, len, replace_flags);
 746             return;
 747         }
 748     }
 749
 750     if (c < 0x80 || !is_utf8)
 751         g_string_append_c (dest_str, (char) c);
 752     else if (c < 0x800)
 753     {
 754         b = 0xC0 | (c >> 6);
 755         g_string_append_c (dest_str, b);
 756         b = 0x80 | (c & 0x3F);
 757         g_string_append_c (dest_str, b);
 758     }
 759     else if (c < 0x10000)
 760     {
 761         b = 0xE0 | (c >> 12);
 762         g_string_append_c (dest_str, b);
 763         b = 0x80 | ((c >> 6) & 0x3F);
 764         g_string_append_c (dest_str, b);
 765         b = 0x80 | (c & 0x3F);
 766         g_string_append_c (dest_str, b);
 767     }
 768     else if (c < 0x10FFFF)
 769     {
 770         b = 0xF0 | (c >> 16);
 771         g_string_append_c (dest_str, b);
 772         b = 0x80 | ((c >> 12) & 0x3F);
 773         g_string_append_c (dest_str, b);
 774         b = 0x80 | ((c >> 6) & 0x3F);
 775         g_string_append_c (dest_str, b);
 776         b = 0x80 | (c & 0x3F);
 777         g_string_append_c (dest_str, b);
 778     }
 779 }
 780
 781 /* --------------------------------------------------------------------------------------------- */
 782 /*** public functions ****************************************************************************/
 783 /* --------------------------------------------------------------------------------------------- */
 784
 785 void
 786 mc_search__cond_struct_new_init_regex (const char *charset, mc_search_t * lc_mc_search,
 787                                        mc_search_cond_t * mc_search_cond)
 788 {
 789     if (lc_mc_search->whole_words && !lc_mc_search->is_entire_line)
 790     {
 791         /* NOTE: \b as word boundary doesn't allow search
 792          * whole words with non-ASCII symbols.
 793          * Update: Is it still true nowadays? Probably not. #2396, #3524 */
 794         g_string_prepend (mc_search_cond->str, "(?<![\\p{L}\\p{N}_])");
 795         g_string_append (mc_search_cond->str, "(?![\\p{L}\\p{N}_])");
 796     }
 797
 798     {
 799 #ifdef SEARCH_TYPE_GLIB
 800         GError *mcerror = NULL;
 801         GRegexCompileFlags g_regex_options = G_REGEX_OPTIMIZE | G_REGEX_DOTALL;
 802
 803         if (str_isutf8 (charset) && mc_global.utf8_display)
 804         {
 805             if (!lc_mc_search->is_case_sensitive)
 806                 g_regex_options |= G_REGEX_CASELESS;
 807         }
 808         else
 809         {
 810             g_regex_options |= G_REGEX_RAW;
 811
 812             if (!lc_mc_search->is_case_sensitive)
 813             {
 814                 GString *tmp;
 815
 816                 tmp = mc_search_cond->str;
 817                 mc_search_cond->str = mc_search__cond_struct_new_regex_ci_str (charset, tmp);
 818                 g_string_free (tmp, TRUE);
 819             }
 820         }
 821
 822         mc_search_cond->regex_handle =
 823             g_regex_new (mc_search_cond->str->str, g_regex_options, 0, &mcerror);
 824
 825         if (mcerror != NULL)
 826         {
 827             lc_mc_search->error = MC_SEARCH_E_REGEX_COMPILE;
 828             g_free (lc_mc_search->error_str);
 829             lc_mc_search->error_str =
 830                 str_conv_gerror_message (mcerror, _("Regular expression error"));
 831             g_error_free (mcerror);
 832             return;
 833         }
 834 #else /* SEARCH_TYPE_GLIB */
 835         const char *error;
 836         int erroffset;
 837         int pcre_options = PCRE_EXTRA | PCRE_MULTILINE;
 838
 839         if (str_isutf8 (charset) && mc_global.utf8_display)
 840         {
 841             pcre_options |= PCRE_UTF8;
 842             if (!lc_mc_search->is_case_sensitive)
 843                 pcre_options |= PCRE_CASELESS;
 844         }
 845         else if (!lc_mc_search->is_case_sensitive)
 846         {
 847             GString *tmp;
 848
 849             tmp = mc_search_cond->str;
 850             mc_search_cond->str = mc_search__cond_struct_new_regex_ci_str (charset, tmp);
 851             g_string_free (tmp, TRUE);
 852         }
 853
 854         mc_search_cond->regex_handle =
 855             pcre_compile (mc_search_cond->str->str, pcre_options, &error, &erroffset, NULL);
 856         if (mc_search_cond->regex_handle == NULL)
 857         {
 858             mc_search_set_error (lc_mc_search, MC_SEARCH_E_REGEX_COMPILE, "%s", error);
 859             return;
 860         }
 861         lc_mc_search->regex_match_info = pcre_study (mc_search_cond->regex_handle, 0, &error);
 862         if (lc_mc_search->regex_match_info == NULL && error != NULL)
 863         {
 864             mc_search_set_error (lc_mc_search, MC_SEARCH_E_REGEX_COMPILE, "%s", error);
 865             MC_PTR_FREE (mc_search_cond->regex_handle);
 866             return;
 867         }
 868 #endif /* SEARCH_TYPE_GLIB */
 869     }
 870
 871     lc_mc_search->is_utf8 = str_isutf8 (charset);
 872 }
 873
 874 /* --------------------------------------------------------------------------------------------- */
 875
 876 gboolean
 877 mc_search__run_regex (mc_search_t * lc_mc_search, const void *user_data,
 878                       gsize start_search, gsize end_search, gsize * found_len)
 879 {
 880     mc_search_cbret_t ret = MC_SEARCH_CB_NOTFOUND;
 881     gsize current_pos, virtual_pos;
 882     gint start_pos;
 883     gint end_pos;
 884
 885     if (lc_mc_search->regex_buffer != NULL)
 886         g_string_set_size (lc_mc_search->regex_buffer, 0);
 887     else
 888         lc_mc_search->regex_buffer = g_string_sized_new (64);
 889
 890     virtual_pos = current_pos = start_search;
 891     while (virtual_pos <= end_search)
 892     {
 893         g_string_set_size (lc_mc_search->regex_buffer, 0);
 894         lc_mc_search->start_buffer = current_pos;
 895
 896         if (lc_mc_search->search_fn != NULL)
 897         {
 898             while (TRUE)
 899             {
 900                 int current_chr = '\n'; /* stop search symbol */
 901
 902                 ret = lc_mc_search->search_fn (user_data, current_pos, &current_chr);
 903
 904                 if (ret == MC_SEARCH_CB_ABORT)
 905                     break;
 906
 907                 if (ret == MC_SEARCH_CB_INVALID)
 908                     continue;
 909
 910                 current_pos++;
 911
 912                 if (ret == MC_SEARCH_CB_SKIP)
 913                     continue;
 914
 915                 virtual_pos++;
 916
 917                 g_string_append_c (lc_mc_search->regex_buffer, (char) current_chr);
 918
 919                 if ((char) current_chr == '\n' || virtual_pos > end_search)
 920                     break;
 921             }
 922         }
 923         else
 924         {
 925             /* optimization for standard case (for search from file manager)
 926              *  where there is no MC_SEARCH_CB_INVALID or MC_SEARCH_CB_SKIP
 927              *  return codes, so we can copy line at regex buffer all at once
 928              */
 929             while (TRUE)
 930             {
 931                 const char current_chr = ((const char *) user_data)[current_pos];
 932
 933                 if (current_chr == '\0')
 934                     break;
 935
 936                 current_pos++;
 937
 938                 if (current_chr == '\n' || current_pos > end_search)
 939                     break;
 940             }
 941
 942             /* use virtual_pos as index of start of current chunk */
 943             g_string_append_len (lc_mc_search->regex_buffer, (const char *) user_data + virtual_pos,
 944                                  current_pos - virtual_pos);
 945             virtual_pos = current_pos;
 946         }
 947
 948         switch (mc_search__regex_found_cond (lc_mc_search, lc_mc_search->regex_buffer))
 949         {
 950         case COND__FOUND_OK:
 951 #ifdef SEARCH_TYPE_GLIB
 952             g_match_info_fetch_pos (lc_mc_search->regex_match_info, 0, &start_pos, &end_pos);
 953 #else /* SEARCH_TYPE_GLIB */
 954             start_pos = lc_mc_search->iovector[0];
 955             end_pos = lc_mc_search->iovector[1];
 956 #endif /* SEARCH_TYPE_GLIB */
 957             if (found_len != NULL)
 958                 *found_len = end_pos - start_pos;
 959             lc_mc_search->normal_offset = lc_mc_search->start_buffer + start_pos;
 960             return TRUE;
 961         case COND__NOT_ALL_FOUND:
 962             break;
 963         default:
 964             g_string_free (lc_mc_search->regex_buffer, TRUE);
 965             lc_mc_search->regex_buffer = NULL;
 966             return FALSE;
 967         }
 968
 969         if ((lc_mc_search->update_fn != NULL) &&
 970             ((lc_mc_search->update_fn) (user_data, current_pos) == MC_SEARCH_CB_ABORT))
 971             ret = MC_SEARCH_CB_ABORT;
 972
 973         if (ret == MC_SEARCH_CB_ABORT || ret == MC_SEARCH_CB_NOTFOUND)
 974             break;
 975     }
 976
 977     g_string_free (lc_mc_search->regex_buffer, TRUE);
 978     lc_mc_search->regex_buffer = NULL;
 979
 980     MC_PTR_FREE (lc_mc_search->error_str);
 981     lc_mc_search->error = ret == MC_SEARCH_CB_ABORT ? MC_SEARCH_E_ABORT : MC_SEARCH_E_NOTFOUND;
 982
 983     return FALSE;
 984 }
 985
 986 /* --------------------------------------------------------------------------------------------- */
 987
 988 GString *
 989 mc_search_regex_prepare_replace_str (mc_search_t * lc_mc_search, GString * replace_str)
 990 {
 991     GString *ret;
 992
 993     int num_replace_tokens;
 994     gsize loop;
 995     gsize prev = 0;
 996     replace_transform_type_t replace_flags = REPLACE_T_NO_TRANSFORM;
 997
 998     num_replace_tokens =
 999         mc_search_regex__get_max_num_of_replace_tokens (replace_str->str, replace_str->len);
1000
1001     if (lc_mc_search->num_results < 0)
1002         return mc_g_string_dup (replace_str);
1003
1004     if (num_replace_tokens > lc_mc_search->num_results - 1
1005         || num_replace_tokens > MC_SEARCH__NUM_REPLACE_ARGS)
1006     {
1007         mc_search_set_error (lc_mc_search, MC_SEARCH_E_REGEX_REPLACE, "%s",
1008                              _(STR_E_RPL_NOT_EQ_TO_FOUND));
1009         return NULL;
1010     }
1011
1012     ret = g_string_sized_new (64);
1013
1014     for (loop = 0; loop < replace_str->len - 1; loop++)
1015     {
1016         int lc_index;
1017         gchar *tmp_str;
1018         gsize len = 0;
1019
1020         lc_index = mc_search_regex__process_replace_str (replace_str, loop, &len, &replace_flags);
1021
1022         if (lc_index == REPLACE_PREPARE_T_NOTHING_SPECIAL)
1023         {
1024             if (len != 0)
1025             {
1026                 mc_search_regex__process_append_str (ret, replace_str->str + prev, loop - prev,
1027                                                      &replace_flags);
1028                 mc_search_regex__process_append_str (ret, replace_str->str + loop + 1, len - 1,
1029                                                      &replace_flags);
1030                 prev = loop + len;
1031                 loop = prev - 1;        /* prepare to loop++ */
1032             }
1033
1034             continue;
1035         }
1036
1037         if (lc_index == REPLACE_PREPARE_T_REPLACE_FLAG)
1038         {
1039             if (loop != 0)
1040                 mc_search_regex__process_append_str (ret, replace_str->str + prev, loop - prev,
1041                                                      &replace_flags);
1042             prev = loop + len;
1043             loop = prev - 1;    /* prepare to loop++ */
1044             continue;
1045         }
1046
1047         /* escape sequence */
1048         if (lc_index == REPLACE_PREPARE_T_ESCAPE_SEQ)
1049         {
1050             mc_search_regex__process_append_str (ret, replace_str->str + prev, loop - prev,
1051                                                  &replace_flags);
1052             /* call process_escape_sequence without starting '\\' */
1053             mc_search_regex__process_escape_sequence (ret, replace_str->str + loop + 1, len - 1,
1054                                                       &replace_flags, lc_mc_search->is_utf8);
1055             prev = loop + len;
1056             loop = prev - 1;    /* prepare to loop++ */
1057             continue;
1058         }
1059
1060         /* invalid capture buffer number */
1061         if (lc_index > lc_mc_search->num_results)
1062         {
1063             g_string_free (ret, TRUE);
1064             mc_search_set_error (lc_mc_search, MC_SEARCH_E_REGEX_REPLACE,
1065                                  _(STR_E_RPL_INVALID_TOKEN), lc_index);
1066             return NULL;
1067         }
1068
1069         tmp_str = mc_search_regex__get_token_by_num (lc_mc_search, lc_index);
1070
1071         if (loop != 0)
1072             mc_search_regex__process_append_str (ret, replace_str->str + prev, loop - prev,
1073                                                  &replace_flags);
1074
1075         mc_search_regex__process_append_str (ret, tmp_str, -1, &replace_flags);
1076         g_free (tmp_str);
1077
1078         prev = loop + len;
1079         loop = prev - 1;        /* prepare to loop++ */
1080     }
1081
1082     mc_search_regex__process_append_str (ret, replace_str->str + prev, replace_str->len - prev,
1083                                          &replace_flags);
1084
1085     return ret;
1086 }