lib/search/regex.c

   1 /*
   2    Search text engine.
   3    Regex search
   4
   5    Copyright (C) 2009-2016
   6    Free Software Foundation, Inc.
   7
   8    Written by:
   9    Slava Zanko <slavazanko@gmail.com>, 2009, 2010, 2011, 2013
  10    Vitaliy Filippov <vitalif@yourcmc.ru>, 2011
  11    Andrew Borodin <aborodin@vmail.ru>, 2013-2015
  12
  13    This file is part of the Midnight Commander.
  14
  15    The Midnight Commander is free software: you can redistribute it
  16    and/or modify it under the terms of the GNU General Public License as
  17    published by the Free Software Foundation, either version 3 of the License,
  18    or (at your option) any later version.
  19
  20    The Midnight Commander is distributed in the hope that it will be useful,
  21    but WITHOUT ANY WARRANTY; without even the implied warranty of
  22    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  23    GNU General Public License for more details.
  24
  25    You should have received a copy of the GNU General Public License
  26    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  27  */
  28
  29 #include <config.h>
  30
  31 #include <stdlib.h>
  32
  33 #include "lib/global.h"
  34 #include "lib/strutil.h"
  35 #include "lib/search.h"
  36 #include "lib/strescape.h"
  37
  38 #include "internal.h"
  39
  40 /*** global variables ****************************************************************************/
  41
  42 /*** file scope macro definitions ****************************************************************/
  43
  44 #define REPLACE_PREPARE_T_NOTHING_SPECIAL -1
  45 #define REPLACE_PREPARE_T_REPLACE_FLAG    -2
  46 #define REPLACE_PREPARE_T_ESCAPE_SEQ      -3
  47
  48 /*** file scope type declarations ****************************************************************/
  49
  50 typedef enum
  51 {
  52     REPLACE_T_NO_TRANSFORM = 0,
  53     REPLACE_T_UPP_TRANSFORM_CHAR = 1,
  54     REPLACE_T_LOW_TRANSFORM_CHAR = 2,
  55     REPLACE_T_UPP_TRANSFORM = 4,
  56     REPLACE_T_LOW_TRANSFORM = 8
  57 } replace_transform_type_t;
  58
  59
  60 /*** file scope variables ************************************************************************/
  61
  62 /*** file scope functions ************************************************************************/
  63
  64 static gboolean
  65 mc_search__regex_str_append_if_special (GString * copy_to, const GString * regex_str,
  66                                         gsize * offset)
  67 {
  68     const char *special_chars[] = {
  69         "\\s", "\\S",
  70         "\\d", "\\D",
  71         "\\b", "\\B",
  72         "\\w", "\\W",
  73         "\\t", "\\n",
  74         "\\r", "\\f",
  75         "\\a", "\\e",
  76         "\\x", "\\X",
  77         "\\c", "\\C",
  78         "\\l", "\\L",
  79         "\\u", "\\U",
  80         "\\E", "\\Q",
  81         NULL
  82     };
  83
  84     char *tmp_regex_str;
  85     const char **spec_chr;
  86
  87     tmp_regex_str = &(regex_str->str[*offset]);
  88
  89     for (spec_chr = special_chars; *spec_chr != NULL; spec_chr++)
  90     {
  91         gsize spec_chr_len;
  92
  93         spec_chr_len = strlen (*spec_chr);
  94
  95         if (strncmp (tmp_regex_str, *spec_chr, spec_chr_len) == 0
  96             && !strutils_is_char_escaped (regex_str->str, tmp_regex_str))
  97         {
  98             if (strncmp ("\\x", *spec_chr, spec_chr_len) == 0)
  99             {
 100                 if (tmp_regex_str[spec_chr_len] != '{')
 101                     spec_chr_len += 2;
 102                 else
 103                 {
 104                     while ((spec_chr_len < regex_str->len - *offset)
 105                            && tmp_regex_str[spec_chr_len] != '}')
 106                         spec_chr_len++;
 107                     if (tmp_regex_str[spec_chr_len] == '}')
 108                         spec_chr_len++;
 109                 }
 110             }
 111             g_string_append_len (copy_to, tmp_regex_str, spec_chr_len);
 112             *offset += spec_chr_len;
 113             return TRUE;
 114         }
 115     }
 116
 117     return FALSE;
 118 }
 119
 120 /* --------------------------------------------------------------------------------------------- */
 121
 122 static void
 123 mc_search__cond_struct_new_regex_hex_add (const char *charset, GString * str_to,
 124                                           const char *one_char, gsize str_len)
 125 {
 126     GString *upp, *low;
 127     gsize loop;
 128
 129     upp = mc_search__toupper_case_str (charset, one_char, str_len);
 130     low = mc_search__tolower_case_str (charset, one_char, str_len);
 131
 132     for (loop = 0; loop < upp->len; loop++)
 133     {
 134         gchar tmp_str[10 + 1];  /* longest content is "[\\x%02X\\x%02X]" */
 135         gint tmp_len;
 136
 137         if (loop >= low->len || upp->str[loop] == low->str[loop])
 138             tmp_len =
 139                 g_snprintf (tmp_str, sizeof (tmp_str), "\\x%02X", (unsigned char) upp->str[loop]);
 140         else
 141             tmp_len =
 142                 g_snprintf (tmp_str, sizeof (tmp_str), "[\\x%02X\\x%02X]",
 143                             (unsigned char) upp->str[loop], (unsigned char) low->str[loop]);
 144
 145         g_string_append_len (str_to, tmp_str, tmp_len);
 146     }
 147
 148     g_string_free (upp, TRUE);
 149     g_string_free (low, TRUE);
 150 }
 151
 152 /* --------------------------------------------------------------------------------------------- */
 153
 154 static void
 155 mc_search__cond_struct_new_regex_accum_append (const char *charset, GString * str_to,
 156                                                GString * str_from)
 157 {
 158     GString *recoded_part;
 159     gsize loop = 0;
 160
 161     recoded_part = g_string_sized_new (32);
 162
 163     while (loop < str_from->len)
 164     {
 165         gchar *one_char;
 166         gsize one_char_len;
 167         gboolean just_letters;
 168
 169         one_char =
 170             mc_search__get_one_symbol (charset, &(str_from->str[loop]),
 171                                        MIN (str_from->len - loop, 6), &just_letters);
 172         one_char_len = strlen (one_char);
 173
 174         if (one_char_len == 0)
 175             loop++;
 176         else
 177         {
 178             loop += one_char_len;
 179
 180             if (just_letters)
 181                 mc_search__cond_struct_new_regex_hex_add (charset, recoded_part, one_char,
 182                                                           one_char_len);
 183             else
 184                 g_string_append_len (recoded_part, one_char, one_char_len);
 185         }
 186
 187         g_free (one_char);
 188     }
 189
 190     g_string_append_len (str_to, recoded_part->str, recoded_part->len);
 191     g_string_free (recoded_part, TRUE);
 192     g_string_set_size (str_from, 0);
 193 }
 194
 195 /* --------------------------------------------------------------------------------------------- */
 196
 197 static GString *
 198 mc_search__cond_struct_new_regex_ci_str (const char *charset, const GString * astr)
 199 {
 200     GString *accumulator, *spec_char, *ret_str;
 201     gsize loop;
 202
 203     ret_str = g_string_sized_new (64);
 204     accumulator = g_string_sized_new (64);
 205     spec_char = g_string_sized_new (64);
 206     loop = 0;
 207
 208     while (loop <= astr->len)
 209     {
 210         if (mc_search__regex_str_append_if_special (spec_char, astr, &loop))
 211         {
 212             mc_search__cond_struct_new_regex_accum_append (charset, ret_str, accumulator);
 213             g_string_append_len (ret_str, spec_char->str, spec_char->len);
 214             g_string_set_size (spec_char, 0);
 215             continue;
 216         }
 217
 218         if (astr->str[loop] == '[' && !strutils_is_char_escaped (astr->str, &(astr->str[loop])))
 219         {
 220             mc_search__cond_struct_new_regex_accum_append (charset, ret_str, accumulator);
 221
 222             while (loop < astr->len && !(astr->str[loop] == ']'
 223                                          && !strutils_is_char_escaped (astr->str,
 224                                                                        &(astr->str[loop]))))
 225             {
 226                 g_string_append_c (ret_str, astr->str[loop]);
 227                 loop++;
 228             }
 229
 230             g_string_append_c (ret_str, astr->str[loop]);
 231             loop++;
 232             continue;
 233         }
 234         /*
 235            TODO: handle [ and ]
 236          */
 237         g_string_append_c (accumulator, astr->str[loop]);
 238         loop++;
 239     }
 240     mc_search__cond_struct_new_regex_accum_append (charset, ret_str, accumulator);
 241
 242     g_string_free (accumulator, TRUE);
 243     g_string_free (spec_char, TRUE);
 244
 245     return ret_str;
 246 }
 247
 248 /* --------------------------------------------------------------------------------------------- */
 249
 250 #ifdef SEARCH_TYPE_GLIB
 251 /* A thin wrapper above g_regex_match_full that makes sure the string passed
 252  * to it is valid UTF-8 (unless G_REGEX_RAW compile flag was set), as it is a
 253  * requirement by glib and it might crash otherwise. See: mc ticket 3449.
 254  * Be careful: there might be embedded NULs in the strings. */
 255 static gboolean
 256 mc_search__g_regex_match_full_safe (const GRegex * regex,
 257                                     const gchar * string,
 258                                     gssize string_len,
 259                                     gint start_position,
 260                                     GRegexMatchFlags match_options,
 261                                     GMatchInfo ** match_info, GError ** error)
 262 {
 263     char *string_safe, *p, *end;
 264     gboolean ret;
 265
 266     if ((g_regex_get_compile_flags (regex) & G_REGEX_RAW)
 267         || g_utf8_validate (string, string_len, NULL))
 268     {
 269         return g_regex_match_full (regex, string, string_len, start_position, match_options,
 270                                    match_info, error);
 271     }
 272
 273     if (string_len < 0)
 274     {
 275         string_len = strlen (string);
 276     }
 277
 278     /* Correctly handle embedded NULs while copying */
 279     p = string_safe = g_malloc (string_len);
 280     memcpy (string_safe, string, string_len);
 281     end = p + string_len;
 282
 283     while (p < end)
 284     {
 285         gunichar c = g_utf8_get_char_validated (p, -1);
 286         if (c != (gunichar) (-1) && c != (gunichar) (-2))
 287         {
 288             p = g_utf8_next_char (p);
 289         }
 290         else
 291         {
 292             /* U+FFFD would be the proper choice, but then we'd have to
 293                maintain mapping between old and new offsets.
 294                So rather do a byte by byte replacement. */
 295             *p++ = '\0';
 296         }
 297     }
 298
 299     ret =
 300         g_regex_match_full (regex, string_safe, string_len, start_position, match_options,
 301                             match_info, error);
 302     g_free (string_safe);
 303     return ret;
 304 }
 305 #endif /* SEARCH_TYPE_GLIB */
 306
 307 /* --------------------------------------------------------------------------------------------- */
 308
 309 static mc_search__found_cond_t
 310 mc_search__regex_found_cond_one (mc_search_t * lc_mc_search, mc_search_regex_t * regex,
 311                                  GString * search_str)
 312 {
 313 #ifdef SEARCH_TYPE_GLIB
 314     GError *mcerror = NULL;
 315
 316     if (!mc_search__g_regex_match_full_safe
 317         (regex, search_str->str, search_str->len, 0, G_REGEX_MATCH_NEWLINE_ANY,
 318          &lc_mc_search->regex_match_info, &mcerror))
 319     {
 320         g_match_info_free (lc_mc_search->regex_match_info);
 321         lc_mc_search->regex_match_info = NULL;
 322         if (mcerror != NULL)
 323         {
 324             lc_mc_search->error = MC_SEARCH_E_REGEX;
 325             lc_mc_search->error_str =
 326                 str_conv_gerror_message (mcerror, _("Regular expression error"));
 327             g_error_free (mcerror);
 328             return COND__FOUND_ERROR;
 329         }
 330         return COND__NOT_FOUND;
 331     }
 332     lc_mc_search->num_results = g_match_info_get_match_count (lc_mc_search->regex_match_info);
 333 #else /* SEARCH_TYPE_GLIB */
 334     lc_mc_search->num_results = pcre_exec (regex, lc_mc_search->regex_match_info,
 335                                            search_str->str, search_str->len, 0, 0,
 336                                            lc_mc_search->iovector, MC_SEARCH__NUM_REPLACE_ARGS);
 337     if (lc_mc_search->num_results < 0)
 338     {
 339         return COND__NOT_FOUND;
 340     }
 341 #endif /* SEARCH_TYPE_GLIB */
 342     return COND__FOUND_OK;
 343
 344 }
 345
 346 /* --------------------------------------------------------------------------------------------- */
 347
 348 static mc_search__found_cond_t
 349 mc_search__regex_found_cond (mc_search_t * lc_mc_search, GString * search_str)
 350 {
 351     gsize loop1;
 352
 353     for (loop1 = 0; loop1 < lc_mc_search->conditions->len; loop1++)
 354     {
 355         mc_search_cond_t *mc_search_cond;
 356         mc_search__found_cond_t ret;
 357
 358         mc_search_cond = (mc_search_cond_t *) g_ptr_array_index (lc_mc_search->conditions, loop1);
 359
 360         if (!mc_search_cond->regex_handle)
 361             continue;
 362
 363         ret =
 364             mc_search__regex_found_cond_one (lc_mc_search, mc_search_cond->regex_handle,
 365                                              search_str);
 366         if (ret != COND__NOT_FOUND)
 367             return ret;
 368     }
 369     return COND__NOT_ALL_FOUND;
 370 }
 371
 372 /* --------------------------------------------------------------------------------------------- */
 373
 374 static int
 375 mc_search_regex__get_max_num_of_replace_tokens (const gchar * str, gsize len)
 376 {
 377     int max_token = 0;
 378     gsize loop;
 379     for (loop = 0; loop < len - 1; loop++)
 380     {
 381         if (str[loop] == '\\' && g_ascii_isdigit (str[loop + 1]))
 382         {
 383             if (strutils_is_char_escaped (str, &str[loop]))
 384                 continue;
 385             if (max_token < str[loop + 1] - '0')
 386                 max_token = str[loop + 1] - '0';
 387             continue;
 388         }
 389         if (str[loop] == '$' && str[loop + 1] == '{')
 390         {
 391             gsize tmp_len;
 392
 393             if (strutils_is_char_escaped (str, &str[loop]))
 394                 continue;
 395
 396             for (tmp_len = 0;
 397                  loop + tmp_len + 2 < len && (str[loop + 2 + tmp_len] & (char) 0xf0) == 0x30;
 398                  tmp_len++);
 399
 400             if (str[loop + 2 + tmp_len] == '}')
 401             {
 402                 int tmp_token;
 403                 char *tmp_str;
 404
 405                 tmp_str = g_strndup (&str[loop + 2], tmp_len);
 406                 tmp_token = atoi (tmp_str);
 407                 if (max_token < tmp_token)
 408                     max_token = tmp_token;
 409                 g_free (tmp_str);
 410             }
 411         }
 412     }
 413     return max_token;
 414 }
 415
 416 /* --------------------------------------------------------------------------------------------- */
 417
 418 static char *
 419 mc_search_regex__get_token_by_num (const mc_search_t * lc_mc_search, gsize lc_index)
 420 {
 421     int fnd_start = 0, fnd_end = 0;
 422
 423 #ifdef SEARCH_TYPE_GLIB
 424     g_match_info_fetch_pos (lc_mc_search->regex_match_info, lc_index, &fnd_start, &fnd_end);
 425 #else /* SEARCH_TYPE_GLIB */
 426     fnd_start = lc_mc_search->iovector[lc_index * 2 + 0];
 427     fnd_end = lc_mc_search->iovector[lc_index * 2 + 1];
 428 #endif /* SEARCH_TYPE_GLIB */
 429
 430     if (fnd_end == fnd_start)
 431         return g_strdup ("");
 432
 433     return g_strndup (lc_mc_search->regex_buffer->str + fnd_start, fnd_end - fnd_start);
 434
 435 }
 436
 437 /* --------------------------------------------------------------------------------------------- */
 438
 439 static gboolean
 440 mc_search_regex__replace_handle_esc_seq (const GString * replace_str, const gsize current_pos,
 441                                          gsize * skip_len, int *ret)
 442 {
 443     char *curr_str = &(replace_str->str[current_pos]);
 444     char c = curr_str[1];
 445
 446     if (replace_str->len > current_pos + 2)
 447     {
 448         if (c == '{')
 449         {
 450             for (*skip_len = 2; /* \{ */
 451                  current_pos + *skip_len < replace_str->len && curr_str[*skip_len] >= '0'
 452                  && curr_str[*skip_len] <= '7'; (*skip_len)++)
 453                 ;
 454
 455             if (current_pos + *skip_len < replace_str->len && curr_str[*skip_len] == '}')
 456             {
 457                 (*skip_len)++;
 458                 *ret = REPLACE_PREPARE_T_ESCAPE_SEQ;
 459                 return FALSE;
 460             }
 461             else
 462             {
 463                 *ret = REPLACE_PREPARE_T_NOTHING_SPECIAL;
 464                 return TRUE;
 465             }
 466         }
 467
 468         if (c == 'x')
 469         {
 470             *skip_len = 2;      /* \x */
 471             c = curr_str[2];
 472             if (c == '{')
 473             {
 474                 for (*skip_len = 3;     /* \x{ */
 475                      current_pos + *skip_len < replace_str->len
 476                      && g_ascii_isxdigit ((guchar) curr_str[*skip_len]); (*skip_len)++)
 477                     ;
 478
 479                 if (current_pos + *skip_len < replace_str->len && curr_str[*skip_len] == '}')
 480                 {
 481                     (*skip_len)++;
 482                     *ret = REPLACE_PREPARE_T_ESCAPE_SEQ;
 483                     return FALSE;
 484                 }
 485                 else
 486                 {
 487                     *ret = REPLACE_PREPARE_T_NOTHING_SPECIAL;
 488                     return TRUE;
 489                 }
 490             }
 491             else if (!g_ascii_isxdigit ((guchar) c))
 492             {
 493                 *skip_len = 2;  /* \x without number behind */
 494                 *ret = REPLACE_PREPARE_T_NOTHING_SPECIAL;
 495                 return FALSE;
 496             }
 497             else
 498             {
 499                 c = curr_str[3];
 500                 if (!g_ascii_isxdigit ((guchar) c))
 501                     *skip_len = 3;      /* \xH */
 502                 else
 503                     *skip_len = 4;      /* \xHH */
 504                 *ret = REPLACE_PREPARE_T_ESCAPE_SEQ;
 505                 return FALSE;
 506             }
 507         }
 508     }
 509
 510     if (strchr ("ntvbrfa", c) != NULL)
 511     {
 512         *skip_len = 2;
 513         *ret = REPLACE_PREPARE_T_ESCAPE_SEQ;
 514         return FALSE;
 515     }
 516     return TRUE;
 517 }
 518
 519 /* --------------------------------------------------------------------------------------------- */
 520
 521 static int
 522 mc_search_regex__process_replace_str (const GString * replace_str, const gsize current_pos,
 523                                       gsize * skip_len, replace_transform_type_t * replace_flags)
 524 {
 525     int ret = -1;
 526     const char *curr_str = &(replace_str->str[current_pos]);
 527
 528     if (current_pos > replace_str->len)
 529         return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 530
 531     *skip_len = 0;
 532
 533     if (replace_str->len > current_pos + 2 && curr_str[0] == '$' && curr_str[1] == '{'
 534         && (curr_str[2] & (char) 0xf0) == 0x30)
 535     {
 536         char *tmp_str;
 537
 538         if (strutils_is_char_escaped (replace_str->str, curr_str))
 539         {
 540             *skip_len = 1;
 541             return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 542         }
 543
 544         for (*skip_len = 0;
 545              current_pos + *skip_len + 2 < replace_str->len
 546              && (curr_str[2 + *skip_len] & (char) 0xf0) == 0x30; (*skip_len)++)
 547             ;
 548
 549         if (curr_str[2 + *skip_len] != '}')
 550             return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 551
 552         tmp_str = g_strndup (curr_str + 2, *skip_len);
 553         if (tmp_str == NULL)
 554             return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 555
 556         ret = atoi (tmp_str);
 557         g_free (tmp_str);
 558
 559         *skip_len += 3;         /* ${} */
 560         return ret;             /* capture buffer index >= 0 */
 561     }
 562
 563     if (curr_str[0] == '\\' && replace_str->len > current_pos + 1)
 564     {
 565         if (strutils_is_char_escaped (replace_str->str, curr_str))
 566         {
 567             *skip_len = 1;
 568             return REPLACE_PREPARE_T_NOTHING_SPECIAL;
 569         }
 570
 571         if (g_ascii_isdigit (curr_str[1]))
 572         {
 573             ret = g_ascii_digit_value (curr_str[1]);    /* capture buffer index >= 0 */
 574             *skip_len = 2;      /* \\ and one digit */
 575             return ret;
 576         }
 577
 578         if (!mc_search_regex__replace_handle_esc_seq (replace_str, current_pos, skip_len, &ret))
 579             return ret;
 580
 581         ret = REPLACE_PREPARE_T_REPLACE_FLAG;
 582         *skip_len += 2;
 583
 584         switch (curr_str[1])
 585         {
 586         case 'U':
 587             *replace_flags |= REPLACE_T_UPP_TRANSFORM;
 588             *replace_flags &= ~REPLACE_T_LOW_TRANSFORM;
 589             break;
 590         case 'u':
 591             *replace_flags |= REPLACE_T_UPP_TRANSFORM_CHAR;
 592             break;
 593         case 'L':
 594             *replace_flags |= REPLACE_T_LOW_TRANSFORM;
 595             *replace_flags &= ~REPLACE_T_UPP_TRANSFORM;
 596             break;
 597         case 'l':
 598             *replace_flags |= REPLACE_T_LOW_TRANSFORM_CHAR;
 599             break;
 600         case 'E':
 601             *replace_flags = REPLACE_T_NO_TRANSFORM;
 602             break;
 603         default:
 604             ret = REPLACE_PREPARE_T_NOTHING_SPECIAL;
 605             break;
 606         }
 607     }
 608     return ret;
 609 }
 610
 611 /* --------------------------------------------------------------------------------------------- */
 612
 613 static void
 614 mc_search_regex__process_append_str (GString * dest_str, const char *from, gsize len,
 615                                      replace_transform_type_t * replace_flags)
 616 {
 617     gsize loop;
 618     gsize char_len;
 619
 620     if (len == (gsize) (-1))
 621         len = strlen (from);
 622
 623     if (*replace_flags == REPLACE_T_NO_TRANSFORM)
 624     {
 625         g_string_append_len (dest_str, from, len);
 626         return;
 627     }
 628
 629     for (loop = 0; loop < len; loop += char_len)
 630     {
 631         GString *tmp_string = NULL;
 632         char *tmp_str;
 633
 634         tmp_str = mc_search__get_one_symbol (NULL, from + loop, len - loop, NULL);
 635         char_len = strlen (tmp_str);
 636
 637         if ((*replace_flags & REPLACE_T_UPP_TRANSFORM_CHAR) != 0)
 638         {
 639             *replace_flags &= ~REPLACE_T_UPP_TRANSFORM_CHAR;
 640             tmp_string = mc_search__toupper_case_str (NULL, tmp_str, char_len);
 641             g_string_append_len (dest_str, tmp_string->str, tmp_string->len);
 642             g_string_free (tmp_string, TRUE);
 643         }
 644         else if ((*replace_flags & REPLACE_T_LOW_TRANSFORM_CHAR) != 0)
 645         {
 646             *replace_flags &= ~REPLACE_T_LOW_TRANSFORM_CHAR;
 647             tmp_string = mc_search__tolower_case_str (NULL, tmp_str, char_len);
 648             g_string_append_len (dest_str, tmp_string->str, tmp_string->len);
 649             g_string_free (tmp_string, TRUE);
 650         }
 651         else if ((*replace_flags & REPLACE_T_UPP_TRANSFORM) != 0)
 652         {
 653             tmp_string = mc_search__toupper_case_str (NULL, tmp_str, char_len);
 654             g_string_append_len (dest_str, tmp_string->str, tmp_string->len);
 655             g_string_free (tmp_string, TRUE);
 656         }
 657         else if ((*replace_flags & REPLACE_T_LOW_TRANSFORM) != 0)
 658         {
 659             tmp_string = mc_search__tolower_case_str (NULL, tmp_str, char_len);
 660             g_string_append_len (dest_str, tmp_string->str, tmp_string->len);
 661             g_string_free (tmp_string, TRUE);
 662         }
 663
 664         g_free (tmp_str);
 665     }
 666 }
 667
 668 /* --------------------------------------------------------------------------------------------- */
 669
 670 static void
 671 mc_search_regex__process_escape_sequence (GString * dest_str, const char *from, gsize len,
 672                                           replace_transform_type_t * replace_flags,
 673                                           gboolean is_utf8)
 674 {
 675     gsize i = 0;
 676     unsigned int c = 0;
 677     char b;
 678
 679     if (len == (gsize) (-1))
 680         len = strlen (from);
 681     if (len == 0)
 682         return;
 683
 684     if (from[i] == '{')
 685         i++;
 686     if (i >= len)
 687         return;
 688
 689     if (from[i] == 'x')
 690     {
 691         i++;
 692         if (i < len && from[i] == '{')
 693             i++;
 694         for (; i < len; i++)
 695         {
 696             if (from[i] >= '0' && from[i] <= '9')
 697                 c = c * 16 + from[i] - '0';
 698             else if (from[i] >= 'a' && from[i] <= 'f')
 699                 c = c * 16 + 10 + from[i] - 'a';
 700             else if (from[i] >= 'A' && from[i] <= 'F')
 701                 c = c * 16 + 10 + from[i] - 'A';
 702             else
 703                 break;
 704         }
 705     }
 706     else if (from[i] >= '0' && from[i] <= '7')
 707         for (; i < len && from[i] >= '0' && from[i] <= '7'; i++)
 708             c = c * 8 + from[i] - '0';
 709     else
 710     {
 711         switch (from[i])
 712         {
 713         case 'n':
 714             c = '\n';
 715             break;
 716         case 't':
 717             c = '\t';
 718             break;
 719         case 'v':
 720             c = '\v';
 721             break;
 722         case 'b':
 723             c = '\b';
 724             break;
 725         case 'r':
 726             c = '\r';
 727             break;
 728         case 'f':
 729             c = '\f';
 730             break;
 731         case 'a':
 732             c = '\a';
 733             break;
 734         default:
 735             mc_search_regex__process_append_str (dest_str, from, len, replace_flags);
 736             return;
 737         }
 738     }
 739
 740     if (c < 0x80 || !is_utf8)
 741         g_string_append_c (dest_str, (char) c);
 742     else if (c < 0x800)
 743     {
 744         b = 0xC0 | (c >> 6);
 745         g_string_append_c (dest_str, b);
 746         b = 0x80 | (c & 0x3F);
 747         g_string_append_c (dest_str, b);
 748     }
 749     else if (c < 0x10000)
 750     {
 751         b = 0xE0 | (c >> 12);
 752         g_string_append_c (dest_str, b);
 753         b = 0x80 | ((c >> 6) & 0x3F);
 754         g_string_append_c (dest_str, b);
 755         b = 0x80 | (c & 0x3F);
 756         g_string_append_c (dest_str, b);
 757     }
 758     else if (c < 0x10FFFF)
 759     {
 760         b = 0xF0 | (c >> 16);
 761         g_string_append_c (dest_str, b);
 762         b = 0x80 | ((c >> 12) & 0x3F);
 763         g_string_append_c (dest_str, b);
 764         b = 0x80 | ((c >> 6) & 0x3F);
 765         g_string_append_c (dest_str, b);
 766         b = 0x80 | (c & 0x3F);
 767         g_string_append_c (dest_str, b);
 768     }
 769 }
 770
 771 /* --------------------------------------------------------------------------------------------- */
 772 /*** public functions ****************************************************************************/
 773 /* --------------------------------------------------------------------------------------------- */
 774
 775 void
 776 mc_search__cond_struct_new_init_regex (const char *charset, mc_search_t * lc_mc_search,
 777                                        mc_search_cond_t * mc_search_cond)
 778 {
 779     if (lc_mc_search->whole_words && !lc_mc_search->is_entire_line)
 780     {
 781         /* NOTE: \b as word boundary doesn't allow search
 782          * whole words with non-ASCII symbols.
 783          * Update: Is it still true nowadays? Probably not. #2396, #3524 */
 784         g_string_prepend (mc_search_cond->str, "(?<![\\p{L}\\p{N}_])");
 785         g_string_append (mc_search_cond->str, "(?![\\p{L}\\p{N}_])");
 786     }
 787
 788     {
 789 #ifdef SEARCH_TYPE_GLIB
 790         GError *mcerror = NULL;
 791         GRegexCompileFlags g_regex_options = G_REGEX_OPTIMIZE | G_REGEX_DOTALL;
 792
 793         if (str_isutf8 (charset) && mc_global.utf8_display)
 794         {
 795             if (!lc_mc_search->is_case_sensitive)
 796                 g_regex_options |= G_REGEX_CASELESS;
 797         }
 798         else
 799         {
 800             g_regex_options |= G_REGEX_RAW;
 801
 802             if (!lc_mc_search->is_case_sensitive)
 803             {
 804                 GString *tmp;
 805
 806                 tmp = mc_search_cond->str;
 807                 mc_search_cond->str = mc_search__cond_struct_new_regex_ci_str (charset, tmp);
 808                 g_string_free (tmp, TRUE);
 809
 810             }
 811         }
 812
 813         mc_search_cond->regex_handle =
 814             g_regex_new (mc_search_cond->str->str, g_regex_options, 0, &mcerror);
 815
 816         if (mcerror != NULL)
 817         {
 818             lc_mc_search->error = MC_SEARCH_E_REGEX_COMPILE;
 819             lc_mc_search->error_str =
 820                 str_conv_gerror_message (mcerror, _("Regular expression error"));
 821             g_error_free (mcerror);
 822             return;
 823         }
 824 #else /* SEARCH_TYPE_GLIB */
 825         const char *error;
 826         int erroffset;
 827         int pcre_options = PCRE_EXTRA | PCRE_MULTILINE;
 828
 829         if (str_isutf8 (charset) && mc_global.utf8_display)
 830         {
 831             pcre_options |= PCRE_UTF8;
 832             if (!lc_mc_search->is_case_sensitive)
 833                 pcre_options |= PCRE_CASELESS;
 834         }
 835         else
 836         {
 837             if (!lc_mc_search->is_case_sensitive)
 838             {
 839                 GString *tmp;
 840
 841                 tmp = mc_search_cond->str;
 842                 mc_search_cond->str = mc_search__cond_struct_new_regex_ci_str (charset, tmp);
 843                 g_string_free (tmp, TRUE);
 844             }
 845         }
 846
 847         mc_search_cond->regex_handle =
 848             pcre_compile (mc_search_cond->str->str, pcre_options, &error, &erroffset, NULL);
 849         if (mc_search_cond->regex_handle == NULL)
 850         {
 851             lc_mc_search->error = MC_SEARCH_E_REGEX_COMPILE;
 852             lc_mc_search->error_str = g_strdup (error);
 853             return;
 854         }
 855         lc_mc_search->regex_match_info = pcre_study (mc_search_cond->regex_handle, 0, &error);
 856         if (lc_mc_search->regex_match_info == NULL)
 857         {
 858             if (error != NULL)
 859             {
 860                 lc_mc_search->error = MC_SEARCH_E_REGEX_COMPILE;
 861                 lc_mc_search->error_str = g_strdup (error);
 862                 g_free (mc_search_cond->regex_handle);
 863                 mc_search_cond->regex_handle = NULL;
 864                 return;
 865             }
 866         }
 867 #endif /* SEARCH_TYPE_GLIB */
 868     }
 869
 870     lc_mc_search->is_utf8 = str_isutf8 (charset);
 871 }
 872
 873 /* --------------------------------------------------------------------------------------------- */
 874
 875 gboolean
 876 mc_search__run_regex (mc_search_t * lc_mc_search, const void *user_data,
 877                       gsize start_search, gsize end_search, gsize * found_len)
 878 {
 879     mc_search_cbret_t ret = MC_SEARCH_CB_ABORT;
 880     gsize current_pos, virtual_pos;
 881     gint start_pos;
 882     gint end_pos;
 883
 884     if (lc_mc_search->regex_buffer != NULL)
 885         g_string_free (lc_mc_search->regex_buffer, TRUE);
 886
 887     lc_mc_search->regex_buffer = g_string_sized_new (64);
 888
 889     virtual_pos = current_pos = start_search;
 890     while (virtual_pos <= end_search)
 891     {
 892         g_string_set_size (lc_mc_search->regex_buffer, 0);
 893         lc_mc_search->start_buffer = current_pos;
 894
 895         if (lc_mc_search->search_fn != NULL)
 896         {
 897             while (TRUE)
 898             {
 899                 int current_chr = '\n'; /* stop search symbol */
 900
 901                 ret = lc_mc_search->search_fn (user_data, current_pos, &current_chr);
 902
 903                 if (ret == MC_SEARCH_CB_ABORT)
 904                     break;
 905
 906                 if (ret == MC_SEARCH_CB_INVALID)
 907                     continue;
 908
 909                 current_pos++;
 910
 911                 if (ret == MC_SEARCH_CB_SKIP)
 912                     continue;
 913
 914                 virtual_pos++;
 915
 916                 g_string_append_c (lc_mc_search->regex_buffer, (char) current_chr);
 917
 918                 if ((char) current_chr == '\n' || virtual_pos > end_search)
 919                     break;
 920             }
 921         }
 922         else
 923         {
 924             /* optimization for standard case (for search from file manager)
 925              *  where there is no MC_SEARCH_CB_INVALID or MC_SEARCH_CB_SKIP
 926              *  return codes, so we can copy line at regex buffer all at once
 927              */
 928             while (TRUE)
 929             {
 930                 const char current_chr = ((const char *) user_data)[current_pos];
 931
 932                 if (current_chr == '\0')
 933                     break;
 934
 935                 current_pos++;
 936
 937                 if (current_chr == '\n' || current_pos > end_search)
 938                     break;
 939             }
 940
 941             /* use virtual_pos as index of start of current chunk */
 942             g_string_append_len (lc_mc_search->regex_buffer, (const char *) user_data + virtual_pos,
 943                                  current_pos - virtual_pos);
 944             virtual_pos = current_pos;
 945         }
 946
 947         switch (mc_search__regex_found_cond (lc_mc_search, lc_mc_search->regex_buffer))
 948         {
 949         case COND__FOUND_OK:
 950 #ifdef SEARCH_TYPE_GLIB
 951             g_match_info_fetch_pos (lc_mc_search->regex_match_info, 0, &start_pos, &end_pos);
 952 #else /* SEARCH_TYPE_GLIB */
 953             start_pos = lc_mc_search->iovector[0];
 954             end_pos = lc_mc_search->iovector[1];
 955 #endif /* SEARCH_TYPE_GLIB */
 956             if (found_len != NULL)
 957                 *found_len = end_pos - start_pos;
 958             lc_mc_search->normal_offset = lc_mc_search->start_buffer + start_pos;
 959             return TRUE;
 960         case COND__NOT_ALL_FOUND:
 961             break;
 962         default:
 963             g_string_free (lc_mc_search->regex_buffer, TRUE);
 964             lc_mc_search->regex_buffer = NULL;
 965             return FALSE;
 966         }
 967
 968         if ((lc_mc_search->update_fn != NULL) &&
 969             ((lc_mc_search->update_fn) (user_data, current_pos) == MC_SEARCH_CB_ABORT))
 970             ret = MC_SEARCH_CB_ABORT;
 971
 972         if (ret == MC_SEARCH_CB_ABORT)
 973             break;
 974     }
 975
 976     g_string_free (lc_mc_search->regex_buffer, TRUE);
 977     lc_mc_search->regex_buffer = NULL;
 978     lc_mc_search->error = MC_SEARCH_E_NOTFOUND;
 979
 980     if (ret != MC_SEARCH_CB_ABORT)
 981         lc_mc_search->error_str = g_strdup (_(STR_E_NOTFOUND));
 982     else
 983         lc_mc_search->error_str = NULL;
 984
 985     return FALSE;
 986 }
 987
 988 /* --------------------------------------------------------------------------------------------- */
 989
 990 GString *
 991 mc_search_regex_prepare_replace_str (mc_search_t * lc_mc_search, GString * replace_str)
 992 {
 993     GString *ret;
 994
 995     int num_replace_tokens;
 996     gsize loop;
 997     gsize prev = 0;
 998     replace_transform_type_t replace_flags = REPLACE_T_NO_TRANSFORM;
 999
1000     num_replace_tokens =
1001         mc_search_regex__get_max_num_of_replace_tokens (replace_str->str, replace_str->len);
1002
1003     if (lc_mc_search->num_results < 0)
1004         return g_string_new_len (replace_str->str, replace_str->len);
1005
1006     if (num_replace_tokens > lc_mc_search->num_results - 1
1007         || num_replace_tokens > MC_SEARCH__NUM_REPLACE_ARGS)
1008     {
1009         lc_mc_search->error = MC_SEARCH_E_REGEX_REPLACE;
1010         lc_mc_search->error_str = g_strdup (_(STR_E_RPL_NOT_EQ_TO_FOUND));
1011         return NULL;
1012     }
1013
1014     ret = g_string_sized_new (64);
1015
1016     for (loop = 0; loop < replace_str->len - 1; loop++)
1017     {
1018         int lc_index;
1019         gchar *tmp_str;
1020         gsize len = 0;
1021
1022         lc_index = mc_search_regex__process_replace_str (replace_str, loop, &len, &replace_flags);
1023
1024         if (lc_index == REPLACE_PREPARE_T_NOTHING_SPECIAL)
1025         {
1026             if (len != 0)
1027             {
1028                 mc_search_regex__process_append_str (ret, replace_str->str + prev, loop - prev,
1029                                                      &replace_flags);
1030                 mc_search_regex__process_append_str (ret, replace_str->str + loop + 1, len - 1,
1031                                                      &replace_flags);
1032                 prev = loop + len;
1033                 loop = prev - 1;        /* prepare to loop++ */
1034             }
1035
1036             continue;
1037         }
1038
1039         if (lc_index == REPLACE_PREPARE_T_REPLACE_FLAG)
1040         {
1041             if (loop != 0)
1042                 mc_search_regex__process_append_str (ret, replace_str->str + prev, loop - prev,
1043                                                      &replace_flags);
1044             prev = loop + len;
1045             loop = prev - 1;    /* prepare to loop++ */
1046             continue;
1047         }
1048
1049         /* escape sequence */
1050         if (lc_index == REPLACE_PREPARE_T_ESCAPE_SEQ)
1051         {
1052             mc_search_regex__process_append_str (ret, replace_str->str + prev, loop - prev,
1053                                                  &replace_flags);
1054             /* call process_escape_sequence without starting '\\' */
1055             mc_search_regex__process_escape_sequence (ret, replace_str->str + loop + 1, len - 1,
1056                                                       &replace_flags, lc_mc_search->is_utf8);
1057             prev = loop + len;
1058             loop = prev - 1;    /* prepare to loop++ */
1059             continue;
1060         }
1061
1062         /* invalid capture buffer number */
1063         if (lc_index > lc_mc_search->num_results)
1064         {
1065             g_string_free (ret, TRUE);
1066             lc_mc_search->error = MC_SEARCH_E_REGEX_REPLACE;
1067             lc_mc_search->error_str = g_strdup_printf (_(STR_E_RPL_INVALID_TOKEN), lc_index);
1068             return NULL;
1069         }
1070
1071         tmp_str = mc_search_regex__get_token_by_num (lc_mc_search, lc_index);
1072
1073         if (loop != 0)
1074             mc_search_regex__process_append_str (ret, replace_str->str + prev, loop - prev,
1075                                                  &replace_flags);
1076
1077         mc_search_regex__process_append_str (ret, tmp_str, -1, &replace_flags);
1078         g_free (tmp_str);
1079
1080         prev = loop + len;
1081         loop = prev - 1;        /* prepare to loop++ */
1082     }
1083
1084     mc_search_regex__process_append_str (ret, replace_str->str + prev, replace_str->len - prev,
1085                                          &replace_flags);
1086
1087     return ret;
1088 }