?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Report actual error to the caller in encodings_convert_to_utf8_auto()
[geany-mirror.git] / src / encodings.c
blobe75a56de04c500ec26a6ee41ccdbc03702d5a51a
1 /*
2 * encodings.c - this file is part of Geany, a fast and lightweight IDE
3 *
4 * Copyright 2005 The Geany contributors
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19 */
20
21 /*
22 * Encoding conversion and Byte Order Mark (BOM) handling.
23 */
24
25 /*
26 * Modified by the gedit Team, 2002. See the gedit AUTHORS file for a
27 * list of people on the gedit Team.
28 * See the gedit ChangeLog files for a list of changes.
29 */
30 /* Stolen from anjuta */
31
32 #ifdef HAVE_CONFIG_H
33 # include "config.h"
34 #endif
35
36 #include "encodings.h"
37 #include "encodingsprivate.h"
38
39 #include "app.h"
40 #include "callbacks.h"
41 #include "documentprivate.h"
42 #include "support.h"
43 #include "ui_utils.h"
44 #include "utils.h"
45
46 #include <string.h>
47
48
49 /* <meta http-equiv="content-type" content="text/html; charset=UTF-8" /> */
50 #define PATTERN_HTMLMETA "<meta\\s+http-equiv\\s*=\\s*\"?content-type\"?\\s+content\\s*=\\s*\"text/x?html;\\s*charset=([a-z0-9_-]+)\"\\s*/?>"
51 /* " geany_encoding=utf-8 " or " coding: utf-8 " */
52 #define PATTERN_CODING "coding[\t ]*[:=][\t ]*\"?([a-z0-9-]+)\"?[\t ]*"
53
54 /* precompiled regexps */
55 static GRegex *pregs[2];
56 static gboolean pregs_loaded = FALSE;
57
58
59 GeanyEncoding encodings[GEANY_ENCODINGS_MAX];
60
61
62 #define fill(Order, Group, Idx, Charset, Name) \
63 encodings[Idx].idx = Idx; \
64 encodings[Idx].order = Order; \
65 encodings[Idx].group = Group; \
66 encodings[Idx].charset = Charset; \
67 encodings[Idx].name = Name;
68
69 static void init_encodings(void)
70 {
71 fill(0, WESTEUROPEAN, GEANY_ENCODING_ISO_8859_14, "ISO-8859-14", _("Celtic"));
72 fill(1, WESTEUROPEAN, GEANY_ENCODING_ISO_8859_7, "ISO-8859-7", _("Greek"));
73 fill(2, WESTEUROPEAN, GEANY_ENCODING_WINDOWS_1253, "WINDOWS-1253", _("Greek"));
74 fill(3, WESTEUROPEAN, GEANY_ENCODING_ISO_8859_10, "ISO-8859-10", _("Nordic"));
75 fill(4, WESTEUROPEAN, GEANY_ENCODING_ISO_8859_3, "ISO-8859-3", _("South European"));
76 fill(5, WESTEUROPEAN, GEANY_ENCODING_IBM_850, "IBM850", _("Western"));
77 fill(6, WESTEUROPEAN, GEANY_ENCODING_ISO_8859_1, "ISO-8859-1", _("Western"));
78 fill(7, WESTEUROPEAN, GEANY_ENCODING_ISO_8859_15, "ISO-8859-15", _("Western"));
79 fill(8, WESTEUROPEAN, GEANY_ENCODING_WINDOWS_1252, "WINDOWS-1252", _("Western"));
80
81 fill(0, EASTEUROPEAN, GEANY_ENCODING_ISO_8859_4, "ISO-8859-4", _("Baltic"));
82 fill(1, EASTEUROPEAN, GEANY_ENCODING_ISO_8859_13, "ISO-8859-13", _("Baltic"));
83 fill(2, EASTEUROPEAN, GEANY_ENCODING_WINDOWS_1257, "WINDOWS-1257", _("Baltic"));
84 fill(3, EASTEUROPEAN, GEANY_ENCODING_IBM_852, "IBM852", _("Central European"));
85 fill(4, EASTEUROPEAN, GEANY_ENCODING_ISO_8859_2, "ISO-8859-2", _("Central European"));
86 fill(5, EASTEUROPEAN, GEANY_ENCODING_WINDOWS_1250, "WINDOWS-1250", _("Central European"));
87 fill(6, EASTEUROPEAN, GEANY_ENCODING_IBM_855, "IBM855", _("Cyrillic"));
88 fill(7, EASTEUROPEAN, GEANY_ENCODING_ISO_8859_5, "ISO-8859-5", _("Cyrillic"));
89 /* ISO-IR-111 not available on Windows */
90 fill(8, EASTEUROPEAN, GEANY_ENCODING_ISO_IR_111, "ISO-IR-111", _("Cyrillic"));
91 fill(9, EASTEUROPEAN, GEANY_ENCODING_KOI8_R, "KOI8-R", _("Cyrillic"));
92 fill(10, EASTEUROPEAN, GEANY_ENCODING_WINDOWS_1251, "WINDOWS-1251", _("Cyrillic"));
93 fill(11, EASTEUROPEAN, GEANY_ENCODING_CP_866, "CP866", _("Cyrillic/Russian"));
94 fill(12, EASTEUROPEAN, GEANY_ENCODING_KOI8_U, "KOI8-U", _("Cyrillic/Ukrainian"));
95 fill(13, EASTEUROPEAN, GEANY_ENCODING_ISO_8859_16, "ISO-8859-16", _("Romanian"));
96
97 fill(0, MIDDLEEASTERN, GEANY_ENCODING_IBM_864, "IBM864", _("Arabic"));
98 fill(1, MIDDLEEASTERN, GEANY_ENCODING_ISO_8859_6, "ISO-8859-6", _("Arabic"));
99 fill(2, MIDDLEEASTERN, GEANY_ENCODING_WINDOWS_1256, "WINDOWS-1256", _("Arabic"));
100 fill(3, MIDDLEEASTERN, GEANY_ENCODING_IBM_862, "IBM862", _("Hebrew"));
101 /* not available at all, ? */
102 fill(4, MIDDLEEASTERN, GEANY_ENCODING_ISO_8859_8_I, "ISO-8859-8-I", _("Hebrew"));
103 fill(5, MIDDLEEASTERN, GEANY_ENCODING_WINDOWS_1255, "WINDOWS-1255", _("Hebrew"));
104 fill(6, MIDDLEEASTERN, GEANY_ENCODING_ISO_8859_8, "ISO-8859-8", _("Hebrew Visual"));
105
106 fill(0, ASIAN, GEANY_ENCODING_ARMSCII_8, "ARMSCII-8", _("Armenian"));
107 fill(1, ASIAN, GEANY_ENCODING_GEOSTD8, "GEORGIAN-ACADEMY", _("Georgian"));
108 fill(2, ASIAN, GEANY_ENCODING_TIS_620, "TIS-620", _("Thai"));
109 fill(3, ASIAN, GEANY_ENCODING_IBM_857, "IBM857", _("Turkish"));
110 fill(4, ASIAN, GEANY_ENCODING_WINDOWS_1254, "WINDOWS-1254", _("Turkish"));
111 fill(5, ASIAN, GEANY_ENCODING_ISO_8859_9, "ISO-8859-9", _("Turkish"));
112 fill(6, ASIAN, GEANY_ENCODING_TCVN, "TCVN", _("Vietnamese"));
113 fill(7, ASIAN, GEANY_ENCODING_VISCII, "VISCII", _("Vietnamese"));
114 fill(8, ASIAN, GEANY_ENCODING_WINDOWS_1258, "WINDOWS-1258", _("Vietnamese"));
115
116 fill(0, UNICODE, GEANY_ENCODING_UTF_7, "UTF-7", _("Unicode"));
117 fill(1, UNICODE, GEANY_ENCODING_UTF_8, "UTF-8", _("Unicode"));
118 fill(2, UNICODE, GEANY_ENCODING_UTF_16LE, "UTF-16LE", _("Unicode"));
119 fill(3, UNICODE, GEANY_ENCODING_UTF_16BE, "UTF-16BE", _("Unicode"));
120 fill(4, UNICODE, GEANY_ENCODING_UCS_2LE, "UCS-2LE", _("Unicode"));
121 fill(5, UNICODE, GEANY_ENCODING_UCS_2BE, "UCS-2BE", _("Unicode"));
122 fill(6, UNICODE, GEANY_ENCODING_UTF_32LE, "UTF-32LE", _("Unicode"));
123 fill(7, UNICODE, GEANY_ENCODING_UTF_32BE, "UTF-32BE", _("Unicode"));
124
125 fill(0, EASTASIAN, GEANY_ENCODING_GB18030, "GB18030", _("Chinese Simplified"));
126 fill(1, EASTASIAN, GEANY_ENCODING_GB2312, "GB2312", _("Chinese Simplified"));
127 fill(2, EASTASIAN, GEANY_ENCODING_GBK, "GBK", _("Chinese Simplified"));
128 /* maybe not available on Linux */
129 fill(3, EASTASIAN, GEANY_ENCODING_HZ, "HZ", _("Chinese Simplified"));
130 fill(4, EASTASIAN, GEANY_ENCODING_BIG5, "BIG5", _("Chinese Traditional"));
131 fill(5, EASTASIAN, GEANY_ENCODING_BIG5_HKSCS, "BIG5-HKSCS", _("Chinese Traditional"));
132 fill(6, EASTASIAN, GEANY_ENCODING_EUC_TW, "EUC-TW", _("Chinese Traditional"));
133 fill(7, EASTASIAN, GEANY_ENCODING_EUC_JP, "EUC-JP", _("Japanese"));
134 fill(8, EASTASIAN, GEANY_ENCODING_ISO_2022_JP, "ISO-2022-JP", _("Japanese"));
135 fill(9, EASTASIAN, GEANY_ENCODING_SHIFT_JIS, "SHIFT_JIS", _("Japanese"));
136 fill(10, EASTASIAN, GEANY_ENCODING_CP_932, "CP932", _("Japanese"));
137 fill(11, EASTASIAN, GEANY_ENCODING_EUC_KR, "EUC-KR", _("Korean"));
138 fill(12, EASTASIAN, GEANY_ENCODING_ISO_2022_KR, "ISO-2022-KR", _("Korean"));
139 fill(13, EASTASIAN, GEANY_ENCODING_JOHAB, "JOHAB", _("Korean"));
140 fill(14, EASTASIAN, GEANY_ENCODING_UHC, "UHC", _("Korean"));
141
142 fill(0, NONE, GEANY_ENCODING_NONE, "None", _("Without encoding"));
143 }
144
145
146 /* compares two encoding names in a permissive fashion.
147 * e.g. "utf8" matches "UTF-8", "iso8859_1" matches "ISO-8859-1", etc. */
148 static gboolean encodings_charset_equals(const gchar *a, const gchar *b)
149 {
150 gboolean was_alpha = FALSE; /* whether last character of previous word was a letter */
151 gboolean need_sep = FALSE; /* whether we're expecting an implicit separator */
152
153 while (*a && *b)
154 {
155 gboolean is_alpha;
156
157 if (g_ascii_toupper(*a) == g_ascii_toupper(*b) &&
158 ((is_alpha = g_ascii_isalpha(*a)) || g_ascii_isdigit(*a)))
159 {
160 /* either there was a real separator, or we need a implicit one (a chage from alpha to
161 * numeric or so) */
162 if (! need_sep || (was_alpha != is_alpha))
163 {
164 a++;
165 b++;
166 was_alpha = is_alpha;
167 need_sep = FALSE;
168 }
169 else
170 return FALSE;
171 }
172 else
173 {
174 guint n_sep = 0;
175
176 if (! g_ascii_isalnum(*a))
177 {
178 a++;
179 n_sep++;
180 }
181 if (! g_ascii_isalnum(*b))
182 {
183 b++;
184 n_sep++;
185 }
186 if (n_sep < 1)
187 return FALSE;
188 else if (n_sep < 2)
189 need_sep = TRUE;
190 }
191 }
192 return *a == *b;
193 }
194
195
196 GeanyEncodingIndex encodings_get_idx_from_charset(const gchar *charset)
197 {
198 if (charset == NULL)
199 return GEANY_ENCODING_UTF_8;
200
201 for (gint i = 0; i < GEANY_ENCODINGS_MAX; i++)
202 {
203 if (encodings_charset_equals(charset, encodings[i].charset))
204 return i;
205 }
206 return GEANY_ENCODING_UTF_8;
207 }
208
209
210 const GeanyEncoding *encodings_get_from_charset(const gchar *charset)
211 {
212 if (charset == NULL)
213 return &encodings[GEANY_ENCODING_UTF_8];
214
215 for (gint i = 0; i < GEANY_ENCODINGS_MAX; i++)
216 {
217 if (encodings_charset_equals(charset, encodings[i].charset))
218 return &encodings[i];
219 }
220
221 return NULL;
222 }
223
224
225 static const gchar *encodings_normalize_charset(const gchar *charset)
226 {
227 const GeanyEncoding *encoding;
228
229 encoding = encodings_get_from_charset(charset);
230 if (encoding != NULL)
231 return encoding->charset;
232
233 return NULL;
234 }
235
236
237 const GeanyEncoding *encodings_get_from_index(gint idx)
238 {
239 g_return_val_if_fail(idx >= 0 && idx < GEANY_ENCODINGS_MAX, NULL);
240
241 return &encodings[idx];
242 }
243
244
245 /**
246 * Gets the character set name of the specified index e.g. for use with
247 * @ref document_set_encoding().
248 *
249 * @param idx @ref GeanyEncodingIndex to retrieve the corresponding character set.
250 *
251 *
252 * @return @nullable The charset according to idx, or @c NULL if the index is invalid.
253 *
254 * @since 0.13
255 **/
256 GEANY_API_SYMBOL
257 const gchar* encodings_get_charset_from_index(gint idx)
258 {
259 g_return_val_if_fail(idx >= 0 && idx < GEANY_ENCODINGS_MAX, NULL);
260
261 return encodings[idx].charset;
262 }
263
264
265 gchar *encodings_to_string(const GeanyEncoding* enc)
266 {
267 g_return_val_if_fail(enc != NULL, NULL);
268 g_return_val_if_fail(enc->name != NULL, NULL);
269 g_return_val_if_fail(enc->charset != NULL, NULL);
270
271 return g_strdup_printf("%s (%s)", enc->name, enc->charset);
272 }
273
274
275 const gchar *encodings_get_charset(const GeanyEncoding* enc)
276 {
277 g_return_val_if_fail(enc != NULL, NULL);
278 g_return_val_if_fail(enc->charset != NULL, NULL);
279
280 return enc->charset;
281 }
282
283
284 static GtkWidget *radio_items[GEANY_ENCODINGS_MAX];
285
286
287 void encodings_select_radio_item(const gchar *charset)
288 {
289 gint i;
290
291 g_return_if_fail(charset != NULL);
292
293 for (i = 0; i < GEANY_ENCODINGS_MAX; i++)
294 {
295 if (utils_str_equal(charset, encodings[i].charset))
296 break;
297 }
298 if (i == GEANY_ENCODINGS_MAX)
299 i = GEANY_ENCODING_UTF_8; /* fallback to UTF-8 */
300
301 /* ignore_callback has to be set by the caller */
302 gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(radio_items[i]), TRUE);
303 }
304
305
306 /* Regexp detection of file encoding declared in the file itself.
307 * Idea and parts of code taken from Bluefish, thanks.
308 * regex_compile() is used to compile regular expressions on program init and keep it in memory
309 * for faster access when opening a file. Pre-compiled regexps will be freed on program exit.
310 */
311 static GRegex *regex_compile(const gchar *pattern)
312 {
313 GError *error = NULL;
314 GRegex *regex = g_regex_new(pattern, G_REGEX_CASELESS | G_REGEX_RAW, 0, &error);
315
316 if (!regex)
317 {
318 geany_debug("Failed to compile encoding regex (%s)", error->message);
319 g_error_free(error);
320 }
321 return regex;
322 }
323
324
325 static gchar *regex_match(GRegex *preg, const gchar *buffer, gsize size)
326 {
327 gchar *encoding = NULL;
328 GMatchInfo *minfo;
329
330 if (G_UNLIKELY(! pregs_loaded || buffer == NULL))
331 return NULL;
332
333 /* scan only the first 512 characters in the buffer */
334 size = MIN(size, 512);
335
336 if (g_regex_match_full(preg, buffer, size, 0, 0, &minfo, NULL) &&
337 g_match_info_get_match_count(minfo) >= 2)
338 {
339 encoding = g_match_info_fetch(minfo, 1);
340 geany_debug("Detected encoding by regex search: %s", encoding);
341
342 SETPTR(encoding, g_utf8_strup(encoding, -1));
343 }
344 g_match_info_free(minfo);
345 return encoding;
346 }
347
348
349 static void encodings_radio_item_change_cb(GtkCheckMenuItem *menuitem, gpointer user_data)
350 {
351 GeanyDocument *doc = document_get_current();
352 const gchar *charset = user_data;
353
354 if (ignore_callback || doc == NULL || charset == NULL ||
355 ! gtk_check_menu_item_get_active(menuitem) ||
356 utils_str_equal(charset, doc->encoding))
357 return;
358
359 if (doc->readonly)
360 {
361 utils_beep();
362 return;
363 }
364 document_undo_add(doc, UNDO_ENCODING, g_strdup(doc->encoding));
365
366 document_set_encoding(doc, charset);
367 }
368
369 static void encodings_reload_radio_item_change_cb(GtkMenuItem *menuitem, gpointer user_data)
370 {
371 GeanyDocument *doc = document_get_current();
372
373 g_return_if_fail(doc != NULL);
374
375 document_reload_prompt(doc, user_data);
376 }
377
378
379 void encodings_finalize(void)
380 {
381 if (pregs_loaded)
382 {
383 guint i, len;
384 len = G_N_ELEMENTS(pregs);
385 for (i = 0; i < len; i++)
386 {
387 g_regex_unref(pregs[i]);
388 }
389 }
390 }
391
392
393 /* initialization of non-UI parts */
394 void encodings_init_headless(void)
395 {
396 static gboolean initialized = FALSE;
397
398 if (initialized)
399 return;
400
401 init_encodings();
402
403 if (! pregs_loaded)
404 {
405 pregs[0] = regex_compile(PATTERN_HTMLMETA);
406 pregs[1] = regex_compile(PATTERN_CODING);
407 pregs_loaded = TRUE;
408 }
409
410 initialized = TRUE;
411 }
412
413
414 void encodings_init(void)
415 {
416 GtkWidget *menu[2];
417 GCallback cb_func[2];
418 const gchar *const groups[GEANY_ENCODING_GROUPS_MAX] =
419 {
420 [NONE] = NULL,
421 [WESTEUROPEAN] = N_("_West European"),
422 [EASTEUROPEAN] = N_("_East European"),
423 [EASTASIAN] = N_("East _Asian"),
424 [ASIAN] = N_("_SE & SW Asian"),
425 [MIDDLEEASTERN] = N_("_Middle Eastern"),
426 [UNICODE] = N_("_Unicode"),
427 };
428
429 encodings_init_headless();
430
431 /* create encodings submenu in document menu */
432 menu[0] = ui_lookup_widget(main_widgets.window, "set_encoding1_menu");
433 menu[1] = ui_lookup_widget(main_widgets.window, "menu_reload_as1_menu");
434 cb_func[0] = G_CALLBACK(encodings_radio_item_change_cb);
435 cb_func[1] = G_CALLBACK(encodings_reload_radio_item_change_cb);
436
437 for (guint k = 0; k < 2; k++)
438 {
439 GSList *group = NULL;
440 GtkWidget *submenus[GEANY_ENCODING_GROUPS_MAX];
441 gint orders[GEANY_ENCODING_GROUPS_MAX] = { 0 };
442 guint n_added = 0;
443
444 for (guint i = 0; i < GEANY_ENCODING_GROUPS_MAX; i++)
445 {
446 if (! groups[i]) /* NONE */
447 submenus[i] = menu[k];
448 else
449 {
450 GtkWidget *item = gtk_menu_item_new_with_mnemonic(_(groups[i]));
451 submenus[i] = gtk_menu_new();
452 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item), submenus[i]);
453 gtk_container_add(GTK_CONTAINER(menu[k]), item);
454 gtk_widget_show_all(item);
455 }
456 }
457
458 /** TODO can it be optimized? ATM 882 runs at line "if (encodings[i].order ...)" */
459 do
460 {
461 for (guint i = 0; i < G_N_ELEMENTS(encodings); i++)
462 {
463 if (encodings[i].order == orders[encodings[i].group])
464 {
465 GtkWidget *item;
466 gchar *label = encodings_to_string(&encodings[i]);
467
468 if (k == 0) /* Set Encoding menu */
469 {
470 item = gtk_radio_menu_item_new_with_label(group, label);
471 group = gtk_radio_menu_item_get_group(GTK_RADIO_MENU_ITEM(item));
472 radio_items[i] = item;
473 }
474 else
475 item = gtk_menu_item_new_with_label(label);
476 gtk_widget_show(item);
477 gtk_container_add(GTK_CONTAINER(submenus[encodings[i].group]), item);
478 g_signal_connect(item, "activate", cb_func[k],
479 (gpointer) encodings[i].charset);
480 g_free(label);
481
482 orders[encodings[i].group]++;
483 n_added++;
484 }
485 }
486 }
487 while (n_added < G_N_ELEMENTS(encodings));
488 }
489 }
490
491
492 static gint encoding_combo_store_sort_func(GtkTreeModel *model,
493 GtkTreeIter *a,
494 GtkTreeIter *b,
495 gpointer data)
496 {
497 gboolean a_has_child = gtk_tree_model_iter_has_child(model, a);
498 gboolean b_has_child = gtk_tree_model_iter_has_child(model, b);
499 gchar *a_string;
500 gchar *b_string;
501 gint cmp_res;
502
503 if (a_has_child != b_has_child)
504 return a_has_child ? -1 : 1;
505
506 gtk_tree_model_get(model, a, 1, &a_string, -1);
507 gtk_tree_model_get(model, b, 1, &b_string, -1);
508 cmp_res = strcmp(a_string, b_string);
509 g_free(a_string);
510 g_free(b_string);
511 return cmp_res;
512 }
513
514
515 GtkTreeStore *encodings_encoding_store_new(gboolean has_detect)
516 {
517 GtkTreeStore *store;
518 GtkTreeIter iter_current, iter_westeuro, iter_easteuro, iter_eastasian,
519 iter_asian, iter_utf8, iter_middleeast;
520 GtkTreeIter *iter_parent;
521 gint i;
522
523 store = gtk_tree_store_new(2, G_TYPE_INT, G_TYPE_STRING);
524
525 if (has_detect)
526 {
527 gtk_tree_store_append(store, &iter_current, NULL);
528 gtk_tree_store_set(store, &iter_current, 0, GEANY_ENCODINGS_MAX, 1, _("Detect from file"), -1);
529 }
530
531 gtk_tree_store_append(store, &iter_westeuro, NULL);
532 gtk_tree_store_set(store, &iter_westeuro, 0, -1, 1, _("West European"), -1);
533 gtk_tree_store_append(store, &iter_easteuro, NULL);
534 gtk_tree_store_set(store, &iter_easteuro, 0, -1, 1, _("East European"), -1);
535 gtk_tree_store_append(store, &iter_eastasian, NULL);
536 gtk_tree_store_set(store, &iter_eastasian, 0, -1, 1, _("East Asian"), -1);
537 gtk_tree_store_append(store, &iter_asian, NULL);
538 gtk_tree_store_set(store, &iter_asian, 0, -1, 1, _("SE & SW Asian"), -1);
539 gtk_tree_store_append(store, &iter_middleeast, NULL);
540 gtk_tree_store_set(store, &iter_middleeast, 0, -1, 1, _("Middle Eastern"), -1);
541 gtk_tree_store_append(store, &iter_utf8, NULL);
542 gtk_tree_store_set(store, &iter_utf8, 0, -1, 1, _("Unicode"), -1);
543
544 for (i = 0; i < GEANY_ENCODINGS_MAX; i++)
545 {
546 gchar *encoding_string;
547
548 switch (encodings[i].group)
549 {
550 case WESTEUROPEAN: iter_parent = &iter_westeuro; break;
551 case EASTEUROPEAN: iter_parent = &iter_easteuro; break;
552 case EASTASIAN: iter_parent = &iter_eastasian; break;
553 case ASIAN: iter_parent = &iter_asian; break;
554 case MIDDLEEASTERN: iter_parent = &iter_middleeast; break;
555 case UNICODE: iter_parent = &iter_utf8; break;
556 case NONE:
557 default: iter_parent = NULL;
558 }
559 gtk_tree_store_append(store, &iter_current, iter_parent);
560 encoding_string = encodings_to_string(&encodings[i]);
561 gtk_tree_store_set(store, &iter_current, 0, i, 1, encoding_string, -1);
562 g_free(encoding_string);
563 }
564
565 gtk_tree_sortable_set_sort_column_id(GTK_TREE_SORTABLE(store), 1, GTK_SORT_ASCENDING);
566 gtk_tree_sortable_set_sort_func(GTK_TREE_SORTABLE(store), 1, encoding_combo_store_sort_func, NULL, NULL);
567
568 return store;
569 }
570
571
572 gint encodings_encoding_store_get_encoding(GtkTreeStore *store, GtkTreeIter *iter)
573 {
574 gint enc;
575 gtk_tree_model_get(GTK_TREE_MODEL(store), iter, 0, &enc, -1);
576 return enc;
577 }
578
579
580 gboolean encodings_encoding_store_get_iter(GtkTreeStore *store, GtkTreeIter *iter, gint enc)
581 {
582 if (gtk_tree_model_get_iter_first(GTK_TREE_MODEL(store), iter))
583 {
584 do
585 {
586 if (encodings_encoding_store_get_encoding(store, iter) == enc)
587 return TRUE;
588 }
589 while (ui_tree_model_iter_any_next(GTK_TREE_MODEL(store), iter, TRUE));
590 }
591 return FALSE;
592 }
593
594
595 void encodings_encoding_store_cell_data_func(GtkCellLayout *cell_layout,
596 GtkCellRenderer *cell,
597 GtkTreeModel *tree_model,
598 GtkTreeIter *iter,
599 gpointer data)
600 {
601 gboolean sensitive = !gtk_tree_model_iter_has_child(tree_model, iter);
602 gchar *text;
603
604 gtk_tree_model_get(tree_model, iter, 1, &text, -1);
605 g_object_set(cell, "sensitive", sensitive, "text", text, NULL);
606 g_free(text);
607 }
608
609
610 static gchar *convert_to_utf8_from_charset(const gchar *buffer, gssize size,
611 const gchar *charset, gboolean fast,
612 gsize *utf8_size, GError **error)
613 {
614 gchar *utf8_content = NULL;
615 GError *conv_error = NULL;
616 gchar* converted_contents = NULL;
617 gsize bytes_written;
618
619 g_return_val_if_fail(buffer != NULL, NULL);
620 g_return_val_if_fail(charset != NULL, NULL);
621
622 converted_contents = g_convert(buffer, size, "UTF-8", charset, NULL,
623 &bytes_written, &conv_error);
624
625 if (fast)
626 {
627 utf8_content = converted_contents;
628 if (conv_error != NULL) g_propagate_error(error, conv_error);
629 }
630 else if (conv_error != NULL || ! g_utf8_validate(converted_contents, bytes_written, NULL))
631 {
632 if (conv_error != NULL)
633 {
634 geany_debug("Couldn't convert from %s to UTF-8 (%s).", charset, conv_error->message);
635 g_propagate_error(error, conv_error);
636 conv_error = NULL;
637 }
638 else
639 {
640 geany_debug("Couldn't convert from %s to UTF-8.", charset);
641 g_set_error(error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
642 _("Data contains NULs"));
643 }
644
645 utf8_content = NULL;
646 g_free(converted_contents);
647 }
648 else
649 {
650 geany_debug("Converted from %s to UTF-8.", charset);
651 utf8_content = converted_contents;
652 }
653
654 if (utf8_content && utf8_size)
655 *utf8_size = bytes_written;
656
657 return utf8_content;
658 }
659
660
661 /**
662 * Tries to convert @a buffer into UTF-8 encoding from the encoding specified with @a charset.
663 * If @a fast is not set, additional checks to validate the converted string are performed.
664 *
665 * @param buffer The input string to convert.
666 * @param size The length of the string, or -1 if the string is nul-terminated.
667 * @param charset The charset to be used for conversion.
668 * @param fast @c TRUE to only convert the input and skip extended checks on the converted string.
669 *
670 * @return If the conversion was successful, a newly allocated nul-terminated string,
671 * which must be freed with @c g_free(). Otherwise @c NULL.
672 **/
673 GEANY_API_SYMBOL
674 gchar *encodings_convert_to_utf8_from_charset(const gchar *buffer, gssize size,
675 const gchar *charset, gboolean fast)
676 {
677 /* If fast=FALSE, we can safely ignore the size as the output cannot contain NULs.
678 * Otherwise, the caller already agrees on partial data anyway. */
679 return convert_to_utf8_from_charset(buffer, size, charset, fast, NULL, NULL);
680 }
681
682
683 static gchar *encodings_check_regexes(const gchar *buffer, gsize size)
684 {
685 guint i;
686
687 for (i = 0; i < G_N_ELEMENTS(pregs); i++)
688 {
689 gchar *charset;
690
691 if ((charset = regex_match(pregs[i], buffer, size)) != NULL)
692 return charset;
693 }
694 return NULL;
695 }
696
697
698 static gchar *encodings_convert_to_utf8_with_suggestion(const gchar *buffer, gssize size,
699 const gchar *suggested_charset, gchar **used_encoding, gsize *utf8_size, GError **error)
700 {
701 const gchar *locale_charset = NULL;
702 const gchar *charset;
703 gchar *utf8_content;
704 gboolean check_suggestion = suggested_charset != NULL;
705 gboolean check_locale = FALSE;
706 gint i, preferred_charset;
707
708 if (size == -1)
709 {
710 size = strlen(buffer);
711 }
712
713 /* current locale is not UTF-8, we have to check this charset */
714 check_locale = ! g_get_charset(&locale_charset);
715
716 /* First check for preferred charset, if specified */
717 preferred_charset = file_prefs.default_open_encoding;
718
719 if (preferred_charset == (gint) encodings[GEANY_ENCODING_NONE].idx ||
720 preferred_charset < 0 ||
721 preferred_charset >= GEANY_ENCODINGS_MAX)
722 {
723 preferred_charset = -1;
724 }
725
726 /* -1 means "Preferred charset" */
727 for (i = -1; i < GEANY_ENCODINGS_MAX; i++)
728 {
729 if (G_UNLIKELY(i == (gint) encodings[GEANY_ENCODING_NONE].idx))
730 continue;
731
732 if (check_suggestion)
733 {
734 check_suggestion = FALSE;
735 charset = encodings_normalize_charset(suggested_charset);
736 if (charset == NULL) /* we failed at normalizing suggested encoding, try it as is */
737 charset = suggested_charset;
738 i = -2; /* keep i below the start value to have it again at -1 on the next loop run */
739 }
740 else if (check_locale)
741 {
742 check_locale = FALSE;
743 charset = locale_charset;
744 i = -2; /* keep i below the start value to have it again at -1 on the next loop run */
745 }
746 else if (i == -1)
747 {
748 if (preferred_charset >= 0)
749 {
750 charset = encodings[preferred_charset].charset;
751 geany_debug("Using preferred charset: %s", charset);
752 }
753 else
754 continue;
755 }
756 else if (i >= 0)
757 charset = encodings[i].charset;
758 else /* in this case we have i == -2, continue to increase i and go ahead */
759 continue;
760
761 if (G_UNLIKELY(charset == NULL))
762 continue;
763
764 geany_debug("Trying to convert %" G_GSIZE_FORMAT " bytes of data from %s into UTF-8.",
765 size, charset);
766 utf8_content = convert_to_utf8_from_charset(buffer, size, charset, FALSE, utf8_size, NULL);
767
768 if (G_LIKELY(utf8_content != NULL))
769 {
770 if (used_encoding != NULL)
771 {
772 if (G_UNLIKELY(*used_encoding != NULL))
773 {
774 geany_debug("%s:%d", __FILE__, __LINE__);
775 g_free(*used_encoding);
776 }
777 *used_encoding = g_strdup(charset);
778 }
779 return utf8_content;
780 }
781 }
782
783 g_set_error(error, G_CONVERT_ERROR, G_CONVERT_ERROR_FAILED,
784 _("Data contains NULs or the encoding is not supported"));
785
786 return NULL;
787 }
788
789
790 /**
791 * Tries to convert @a buffer into UTF-8 encoding and store the detected original encoding in
792 * @a used_encoding.
793 *
794 * @param buffer the input string to convert.
795 * @param size the length of the string, or -1 if the string is nul-terminated.
796 * @param used_encoding @out @optional return location of the detected encoding of the input string, or @c NULL.
797 *
798 * @return @nullable If the conversion was successful, a newly allocated nul-terminated string,
799 * which must be freed with @c g_free(). Otherwise @c NULL.
800 **/
801 GEANY_API_SYMBOL
802 gchar *encodings_convert_to_utf8(const gchar *buffer, gssize size, gchar **used_encoding)
803 {
804 gchar *regex_charset;
805 gchar *utf8;
806
807 /* first try to read the encoding from the file content */
808 regex_charset = encodings_check_regexes(buffer, size);
809 /* we know this cannot succeed if there are NULs in the output, so ignoring the size is OK */
810 utf8 = encodings_convert_to_utf8_with_suggestion(buffer, size, regex_charset, used_encoding, NULL, NULL);
811 g_free(regex_charset);
812
813 return utf8;
814 }
815
816
817 /* If there's a BOM, return a corresponding GEANY_ENCODING_UTF_* index,
818 * otherwise GEANY_ENCODING_NONE.
819 * */
820 GeanyEncodingIndex encodings_scan_unicode_bom(const gchar *string, gsize len, guint *bom_len)
821 {
822 if (len >= 3)
823 {
824 if (bom_len)
825 *bom_len = 3;
826
827 if ((guchar)string[0] == 0xef && (guchar)string[1] == 0xbb &&
828 (guchar)string[2] == 0xbf)
829 {
830 return GEANY_ENCODING_UTF_8;
831 }
832 }
833 if (len >= 4)
834 {
835 if (bom_len)
836 *bom_len = 4;
837
838 if ((guchar)string[0] == 0x00 && (guchar)string[1] == 0x00 &&
839 (guchar)string[2] == 0xfe && (guchar)string[3] == 0xff)
840 {
841 return GEANY_ENCODING_UTF_32BE; /* Big endian */
842 }
843 if ((guchar)string[0] == 0xff && (guchar)string[1] == 0xfe &&
844 (guchar)string[2] == 0x00 && (guchar)string[3] == 0x00)
845 {
846 return GEANY_ENCODING_UTF_32LE; /* Little endian */
847 }
848 if ((string[0] == 0x2b && string[1] == 0x2f && string[2] == 0x76) &&
849 (string[3] == 0x38 || string[3] == 0x39 || string[3] == 0x2b || string[3] == 0x2f))
850 {
851 return GEANY_ENCODING_UTF_7;
852 }
853 }
854 if (len >= 2)
855 {
856 if (bom_len)
857 *bom_len = 2;
858
859 if ((guchar)string[0] == 0xfe && (guchar)string[1] == 0xff)
860 {
861 return GEANY_ENCODING_UTF_16BE; /* Big endian */
862 }
863 if ((guchar)string[0] == 0xff && (guchar)string[1] == 0xfe)
864 {
865 return GEANY_ENCODING_UTF_16LE; /* Little endian */
866 }
867 }
868 if (bom_len)
869 *bom_len = 0;
870 return GEANY_ENCODING_NONE;
871 }
872
873
874 gboolean encodings_is_unicode_charset(const gchar *string)
875 {
876 if (string != NULL &&
877 (strncmp(string, "UTF", 3) == 0 || strncmp(string, "UCS", 3) == 0))
878 {
879 return TRUE;
880 }
881 return FALSE;
882 }
883
884
885 typedef struct
886 {
887 gchar *data; /* null-terminated data */
888 gsize size; /* actual data size */
889 gchar *enc;
890 gboolean bom;
891 } BufferData;
892
893
894 /* convert data with the specified encoding */
895 static gboolean
896 handle_forced_encoding(BufferData *buffer, const gchar *forced_enc, GError **error)
897 {
898 GeanyEncodingIndex enc_idx;
899
900 if (utils_str_equal(forced_enc, "UTF-8"))
901 {
902 if (! g_utf8_validate(buffer->data, buffer->size, NULL))
903 {
904 g_set_error(error, G_CONVERT_ERROR, G_CONVERT_ERROR_ILLEGAL_SEQUENCE,
905 _("Data contains NULs or is not valid UTF-8"));
906 return FALSE;
907 }
908 }
909 else
910 {
911 gchar *converted_text = convert_to_utf8_from_charset(
912 buffer->data, buffer->size, forced_enc, FALSE, &buffer->size, error);
913 if (converted_text == NULL)
914 {
915 return FALSE;
916 }
917 else
918 {
919 SETPTR(buffer->data, converted_text);
920 }
921 }
922 enc_idx = encodings_scan_unicode_bom(buffer->data, buffer->size, NULL);
923 buffer->bom = (enc_idx == GEANY_ENCODING_UTF_8);
924 buffer->enc = g_strdup(forced_enc);
925 return TRUE;
926 }
927
928
929 /* detect encoding and convert to UTF-8 if necessary */
930 static gboolean
931 handle_encoding(BufferData *buffer, GeanyEncodingIndex enc_idx, GError **error)
932 {
933 g_return_val_if_fail(buffer->enc == NULL, FALSE);
934 g_return_val_if_fail(buffer->bom == FALSE, FALSE);
935
936 if (buffer->size == 0)
937 {
938 /* we have no data so assume UTF-8 */
939 buffer->enc = g_strdup("UTF-8");
940 }
941 else
942 {
943 /* first check for a BOM */
944 if (enc_idx != GEANY_ENCODING_NONE)
945 {
946 buffer->enc = g_strdup(encodings[enc_idx].charset);
947 buffer->bom = TRUE;
948
949 if (enc_idx == GEANY_ENCODING_UTF_8)
950 {
951 if (! g_utf8_validate(buffer->data, buffer->size, NULL))
952 {
953 /* this is not actually valid UTF-8 */
954 SETPTR(buffer->enc, NULL);
955 buffer->bom = FALSE;
956 }
957 }
958 else /* the BOM indicated something else than UTF-8 */
959 {
960 gchar *converted_text = convert_to_utf8_from_charset(
961 buffer->data, buffer->size, buffer->enc, FALSE, &buffer->size, NULL);
962 if (converted_text != NULL)
963 {
964 SETPTR(buffer->data, converted_text);
965 }
966 else
967 {
968 /* there was a problem converting data from BOM encoding type */
969 SETPTR(buffer->enc, NULL);
970 buffer->bom = FALSE;
971 }
972 }
973 }
974
975 if (buffer->enc == NULL) /* either there was no BOM or the BOM encoding failed */
976 {
977 /* first try to read the encoding from the file content */
978 gchar *regex_charset = encodings_check_regexes(buffer->data, buffer->size);
979
980 /* try UTF-8 first */
981 if (encodings_get_idx_from_charset(regex_charset) == GEANY_ENCODING_UTF_8 &&
982 g_utf8_validate(buffer->data, buffer->size, NULL))
983 {
984 buffer->enc = g_strdup("UTF-8");
985 }
986 else
987 {
988 /* detect the encoding */
989 gchar *converted_text = encodings_convert_to_utf8_with_suggestion(buffer->data,
990 buffer->size, regex_charset, &buffer->enc, &buffer->size, error);
991
992 if (converted_text == NULL)
993 {
994 g_free(regex_charset);
995 return FALSE;
996 }
997 SETPTR(buffer->data, converted_text);
998 }
999 g_free(regex_charset);
1000 }
1001 }
1002 return TRUE;
1003 }
1004
1005
1006 static void
1007 handle_bom(BufferData *buffer)
1008 {
1009 guint bom_len;
1010
1011 encodings_scan_unicode_bom(buffer->data, buffer->size, &bom_len);
1012 g_return_if_fail(bom_len != 0);
1013
1014 /* the contents are already converted into UTF-8 here */
1015 buffer->size -= bom_len;
1016 /* overwrite the BOM with the remainder of the file contents, plus the NULL terminator. */
1017 memmove(buffer->data, buffer->data + bom_len, buffer->size + 1);
1018 buffer->data = g_realloc(buffer->data, buffer->size + 1);
1019 }
1020
1021
1022 /* loads textfile data, verifies and converts to forced_enc or UTF-8. Also handles BOM. */
1023 static gboolean handle_buffer(BufferData *buffer, const gchar *forced_enc, GError **error)
1024 {
1025 GeanyEncodingIndex tmp_enc_idx;
1026
1027 /* temporarily retrieve the encoding idx based on the BOM to suppress the following warning
1028 * if we have a BOM */
1029 tmp_enc_idx = encodings_scan_unicode_bom(buffer->data, buffer->size, NULL);
1030
1031 /* Determine character encoding and convert to UTF-8 */
1032 if (forced_enc != NULL)
1033 {
1034 /* the encoding should be ignored(requested by user), so open the file "as it is" */
1035 if (utils_str_equal(forced_enc, encodings[GEANY_ENCODING_NONE].charset))
1036 {
1037 buffer->bom = FALSE;
1038 buffer->enc = g_strdup(encodings[GEANY_ENCODING_NONE].charset);
1039 }
1040 else if (! handle_forced_encoding(buffer, forced_enc, error))
1041 {
1042 return FALSE;
1043 }
1044 }
1045 else if (! handle_encoding(buffer, tmp_enc_idx, error))
1046 {
1047 return FALSE;
1048 }
1049
1050 if (buffer->bom)
1051 handle_bom(buffer);
1052 return TRUE;
1053 }
1054
1055
1056 /*
1057 * Tries to convert @a buffer into UTF-8 encoding. Unlike encodings_convert_to_utf8()
1058 * and encodings_convert_to_utf8_from_charset() it handles the possible BOM in the data.
1059 *
1060 * @param buf a pointer to modifiable null-terminated buffer to convert.
1061 * It may or may not be modified, and should be freed whatever happens.
1062 * @param size a pointer to the size of the buffer (expected to be e.g. the on-disk
1063 * file size). It will be updated to the new size.
1064 * @param forced_enc forced encoding to use, or @c NULL
1065 * @param used_encoding return location for the actually used encoding, or @c NULL
1066 * @param has_bom return location to store whether the data had a BOM, or @c NULL
1067 * @param has_nuls return location to store whether the converted data contains NULs, or @c NULL
1068 *
1069 * @return @C TRUE if the conversion succeeded, @c FALSE otherwise.
1070 */
1071 GEANY_EXPORT_SYMBOL
1072 gboolean encodings_convert_to_utf8_auto(gchar **buf, gsize *size, const gchar *forced_enc,
1073 gchar **used_encoding, gboolean *has_bom, gboolean *has_nuls, GError **error)
1074 {
1075 BufferData buffer;
1076
1077 buffer.data = *buf;
1078 buffer.size = *size;
1079 buffer.enc = NULL;
1080 buffer.bom = FALSE;
1081
1082 if (! handle_buffer(&buffer, forced_enc, error))
1083 return FALSE;
1084
1085 *size = buffer.size;
1086 if (used_encoding)
1087 *used_encoding = buffer.enc;
1088 else
1089 g_free(buffer.enc);
1090 if (has_bom)
1091 *has_bom = buffer.bom;
1092 if (has_nuls)
1093 *has_nuls = strlen(buffer.data) != buffer.size;
1094
1095 *buf = buffer.data;
1096 return TRUE;
1097 }
A fast and lightweight IDE