?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Update HACKING for changed doc generation instructions
[geany-mirror.git] / src / encodings.c
blob4bc2a9acce00639e5d74a4d08a53dd7dd324661f
1 /*
2 * encodings.c - this file is part of Geany, a fast and lightweight IDE
3 *
4 * Copyright 2005-2012 Enrico Tröger <enrico(dot)troeger(at)uvena(dot)de>
5 * Copyright 2006-2012 Nick Treleaven <nick(dot)treleaven(at)btinternet(dot)com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License along
18 * with this program; if not, write to the Free Software Foundation, Inc.,
19 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 */
21
22 /*
23 * Encoding conversion and Byte Order Mark (BOM) handling.
24 */
25
26 /*
27 * Modified by the gedit Team, 2002. See the gedit AUTHORS file for a
28 * list of people on the gedit Team.
29 * See the gedit ChangeLog files for a list of changes.
30 */
31 /* Stolen from anjuta */
32
33 #ifdef HAVE_CONFIG_H
34 # include "config.h"
35 #endif
36
37 #include "encodings.h"
38
39 #include "app.h"
40 #include "callbacks.h"
41 #include "documentprivate.h"
42 #include "support.h"
43 #include "ui_utils.h"
44 #include "utils.h"
45
46 #include <string.h>
47
48
49 /* <meta http-equiv="content-type" content="text/html; charset=UTF-8" /> */
50 #define PATTERN_HTMLMETA "<meta\\s+http-equiv\\s*=\\s*\"?content-type\"?\\s+content\\s*=\\s*\"text/x?html;\\s*charset=([a-z0-9_-]+)\"\\s*/?>"
51 /* " geany_encoding=utf-8 " or " coding: utf-8 " */
52 #define PATTERN_CODING "coding[\t ]*[:=][\t ]*\"?([a-z0-9-]+)\"?[\t ]*"
53
54 /* precompiled regexps */
55 static GRegex *pregs[2];
56 static gboolean pregs_loaded = FALSE;
57
58
59 GeanyEncoding encodings[GEANY_ENCODINGS_MAX];
60
61
62 #define fill(Order, Group, Idx, Charset, Name) \
63 encodings[Idx].idx = Idx; \
64 encodings[Idx].order = Order; \
65 encodings[Idx].group = Group; \
66 encodings[Idx].charset = Charset; \
67 encodings[Idx].name = Name;
68
69 static void init_encodings(void)
70 {
71 fill(0, WESTEUROPEAN, GEANY_ENCODING_ISO_8859_14, "ISO-8859-14", _("Celtic"));
72 fill(1, WESTEUROPEAN, GEANY_ENCODING_ISO_8859_7, "ISO-8859-7", _("Greek"));
73 fill(2, WESTEUROPEAN, GEANY_ENCODING_WINDOWS_1253, "WINDOWS-1253", _("Greek"));
74 fill(3, WESTEUROPEAN, GEANY_ENCODING_ISO_8859_10, "ISO-8859-10", _("Nordic"));
75 fill(4, WESTEUROPEAN, GEANY_ENCODING_ISO_8859_3, "ISO-8859-3", _("South European"));
76 fill(5, WESTEUROPEAN, GEANY_ENCODING_IBM_850, "IBM850", _("Western"));
77 fill(6, WESTEUROPEAN, GEANY_ENCODING_ISO_8859_1, "ISO-8859-1", _("Western"));
78 fill(7, WESTEUROPEAN, GEANY_ENCODING_ISO_8859_15, "ISO-8859-15", _("Western"));
79 fill(8, WESTEUROPEAN, GEANY_ENCODING_WINDOWS_1252, "WINDOWS-1252", _("Western"));
80
81 fill(0, EASTEUROPEAN, GEANY_ENCODING_ISO_8859_4, "ISO-8859-4", _("Baltic"));
82 fill(1, EASTEUROPEAN, GEANY_ENCODING_ISO_8859_13, "ISO-8859-13", _("Baltic"));
83 fill(2, EASTEUROPEAN, GEANY_ENCODING_WINDOWS_1257, "WINDOWS-1257", _("Baltic"));
84 fill(3, EASTEUROPEAN, GEANY_ENCODING_IBM_852, "IBM852", _("Central European"));
85 fill(4, EASTEUROPEAN, GEANY_ENCODING_ISO_8859_2, "ISO-8859-2", _("Central European"));
86 fill(5, EASTEUROPEAN, GEANY_ENCODING_WINDOWS_1250, "WINDOWS-1250", _("Central European"));
87 fill(6, EASTEUROPEAN, GEANY_ENCODING_IBM_855, "IBM855", _("Cyrillic"));
88 fill(7, EASTEUROPEAN, GEANY_ENCODING_ISO_8859_5, "ISO-8859-5", _("Cyrillic"));
89 /* ISO-IR-111 not available on Windows */
90 fill(8, EASTEUROPEAN, GEANY_ENCODING_ISO_IR_111, "ISO-IR-111", _("Cyrillic"));
91 fill(9, EASTEUROPEAN, GEANY_ENCODING_KOI8_R, "KOI8-R", _("Cyrillic"));
92 fill(10, EASTEUROPEAN, GEANY_ENCODING_WINDOWS_1251, "WINDOWS-1251", _("Cyrillic"));
93 fill(11, EASTEUROPEAN, GEANY_ENCODING_CP_866, "CP866", _("Cyrillic/Russian"));
94 fill(12, EASTEUROPEAN, GEANY_ENCODING_KOI8_U, "KOI8-U", _("Cyrillic/Ukrainian"));
95 fill(13, EASTEUROPEAN, GEANY_ENCODING_ISO_8859_16, "ISO-8859-16", _("Romanian"));
96
97 fill(0, MIDDLEEASTERN, GEANY_ENCODING_IBM_864, "IBM864", _("Arabic"));
98 fill(1, MIDDLEEASTERN, GEANY_ENCODING_ISO_8859_6, "ISO-8859-6", _("Arabic"));
99 fill(2, MIDDLEEASTERN, GEANY_ENCODING_WINDOWS_1256, "WINDOWS-1256", _("Arabic"));
100 fill(3, MIDDLEEASTERN, GEANY_ENCODING_IBM_862, "IBM862", _("Hebrew"));
101 /* not available at all, ? */
102 fill(4, MIDDLEEASTERN, GEANY_ENCODING_ISO_8859_8_I, "ISO-8859-8-I", _("Hebrew"));
103 fill(5, MIDDLEEASTERN, GEANY_ENCODING_WINDOWS_1255, "WINDOWS-1255", _("Hebrew"));
104 fill(6, MIDDLEEASTERN, GEANY_ENCODING_ISO_8859_8, "ISO-8859-8", _("Hebrew Visual"));
105
106 fill(0, ASIAN, GEANY_ENCODING_ARMSCII_8, "ARMSCII-8", _("Armenian"));
107 fill(1, ASIAN, GEANY_ENCODING_GEOSTD8, "GEORGIAN-ACADEMY", _("Georgian"));
108 fill(2, ASIAN, GEANY_ENCODING_TIS_620, "TIS-620", _("Thai"));
109 fill(3, ASIAN, GEANY_ENCODING_IBM_857, "IBM857", _("Turkish"));
110 fill(4, ASIAN, GEANY_ENCODING_WINDOWS_1254, "WINDOWS-1254", _("Turkish"));
111 fill(5, ASIAN, GEANY_ENCODING_ISO_8859_9, "ISO-8859-9", _("Turkish"));
112 fill(6, ASIAN, GEANY_ENCODING_TCVN, "TCVN", _("Vietnamese"));
113 fill(7, ASIAN, GEANY_ENCODING_VISCII, "VISCII", _("Vietnamese"));
114 fill(8, ASIAN, GEANY_ENCODING_WINDOWS_1258, "WINDOWS-1258", _("Vietnamese"));
115
116 fill(0, UNICODE, GEANY_ENCODING_UTF_7, "UTF-7", _("Unicode"));
117 fill(1, UNICODE, GEANY_ENCODING_UTF_8, "UTF-8", _("Unicode"));
118 fill(2, UNICODE, GEANY_ENCODING_UTF_16LE, "UTF-16LE", _("Unicode"));
119 fill(3, UNICODE, GEANY_ENCODING_UTF_16BE, "UTF-16BE", _("Unicode"));
120 fill(4, UNICODE, GEANY_ENCODING_UCS_2LE, "UCS-2LE", _("Unicode"));
121 fill(5, UNICODE, GEANY_ENCODING_UCS_2BE, "UCS-2BE", _("Unicode"));
122 fill(6, UNICODE, GEANY_ENCODING_UTF_32LE, "UTF-32LE", _("Unicode"));
123 fill(7, UNICODE, GEANY_ENCODING_UTF_32BE, "UTF-32BE", _("Unicode"));
124
125 fill(0, EASTASIAN, GEANY_ENCODING_GB18030, "GB18030", _("Chinese Simplified"));
126 fill(1, EASTASIAN, GEANY_ENCODING_GB2312, "GB2312", _("Chinese Simplified"));
127 fill(2, EASTASIAN, GEANY_ENCODING_GBK, "GBK", _("Chinese Simplified"));
128 /* maybe not available on Linux */
129 fill(3, EASTASIAN, GEANY_ENCODING_HZ, "HZ", _("Chinese Simplified"));
130 fill(4, EASTASIAN, GEANY_ENCODING_BIG5, "BIG5", _("Chinese Traditional"));
131 fill(5, EASTASIAN, GEANY_ENCODING_BIG5_HKSCS, "BIG5-HKSCS", _("Chinese Traditional"));
132 fill(6, EASTASIAN, GEANY_ENCODING_EUC_TW, "EUC-TW", _("Chinese Traditional"));
133 fill(7, EASTASIAN, GEANY_ENCODING_EUC_JP, "EUC-JP", _("Japanese"));
134 fill(8, EASTASIAN, GEANY_ENCODING_ISO_2022_JP, "ISO-2022-JP", _("Japanese"));
135 fill(9, EASTASIAN, GEANY_ENCODING_SHIFT_JIS, "SHIFT_JIS", _("Japanese"));
136 fill(10, EASTASIAN, GEANY_ENCODING_CP_932, "CP932", _("Japanese"));
137 fill(11, EASTASIAN, GEANY_ENCODING_EUC_KR, "EUC-KR", _("Korean"));
138 fill(12, EASTASIAN, GEANY_ENCODING_ISO_2022_KR, "ISO-2022-KR", _("Korean"));
139 fill(13, EASTASIAN, GEANY_ENCODING_JOHAB, "JOHAB", _("Korean"));
140 fill(14, EASTASIAN, GEANY_ENCODING_UHC, "UHC", _("Korean"));
141
142 fill(0, NONE, GEANY_ENCODING_NONE, "None", _("Without encoding"));
143 }
144
145
146 /* compares two encoding names in a permissive fashion.
147 * e.g. "utf8" matches "UTF-8", "iso8859_1" matches "ISO-8859-1", etc. */
148 static gboolean encodings_charset_equals(const gchar *a, const gchar *b)
149 {
150 gboolean was_alpha = FALSE; /* whether last character of previous word was a letter */
151 gboolean need_sep = FALSE; /* whether we're expecting an implicit separator */
152
153 while (*a && *b)
154 {
155 gboolean is_alpha;
156
157 if (g_ascii_toupper(*a) == g_ascii_toupper(*b) &&
158 ((is_alpha = g_ascii_isalpha(*a)) || g_ascii_isdigit(*a)))
159 {
160 /* either there was a real separator, or we need a implicit one (a chage from alpha to
161 * numeric or so) */
162 if (! need_sep || (was_alpha != is_alpha))
163 {
164 a++;
165 b++;
166 was_alpha = is_alpha;
167 need_sep = FALSE;
168 }
169 else
170 return FALSE;
171 }
172 else
173 {
174 guint n_sep = 0;
175
176 if (! g_ascii_isalnum(*a))
177 {
178 a++;
179 n_sep++;
180 }
181 if (! g_ascii_isalnum(*b))
182 {
183 b++;
184 n_sep++;
185 }
186 if (n_sep < 1)
187 return FALSE;
188 else if (n_sep < 2)
189 need_sep = TRUE;
190 }
191 }
192 return *a == *b;
193 }
194
195
196 GeanyEncodingIndex encodings_get_idx_from_charset(const gchar *charset)
197 {
198 gint i;
199
200 if (charset == NULL)
201 return GEANY_ENCODING_UTF_8;
202
203 i = 0;
204 while (i < GEANY_ENCODINGS_MAX)
205 {
206 if (encodings_charset_equals(charset, encodings[i].charset))
207 return i;
208
209 ++i;
210 }
211 return GEANY_ENCODING_UTF_8;
212 }
213
214
215 const GeanyEncoding *encodings_get_from_charset(const gchar *charset)
216 {
217 gint i;
218
219 if (charset == NULL)
220 return &encodings[GEANY_ENCODING_UTF_8];
221
222 i = 0;
223 while (i < GEANY_ENCODINGS_MAX)
224 {
225 if (encodings_charset_equals(charset, encodings[i].charset))
226 return &encodings[i];
227
228 ++i;
229 }
230
231 return NULL;
232 }
233
234
235 static const gchar *encodings_normalize_charset(const gchar *charset)
236 {
237 const GeanyEncoding *encoding;
238
239 encoding = encodings_get_from_charset(charset);
240 if (encoding != NULL)
241 return encoding->charset;
242
243 return NULL;
244 }
245
246
247 const GeanyEncoding *encodings_get_from_index(gint idx)
248 {
249 g_return_val_if_fail(idx >= 0 && idx < GEANY_ENCODINGS_MAX, NULL);
250
251 return &encodings[idx];
252 }
253
254
255 /**
256 * Gets the character set name of the specified index e.g. for use with
257 * @ref document_set_encoding().
258 *
259 * @param idx @ref GeanyEncodingIndex to retrieve the corresponding character set.
260 *
261 *
262 * @return The charset according to idx, or @c NULL if the index is invalid.
263 *
264 * @since 0.13
265 **/
266 const gchar* encodings_get_charset_from_index(gint idx)
267 {
268 g_return_val_if_fail(idx >= 0 && idx < GEANY_ENCODINGS_MAX, NULL);
269
270 return encodings[idx].charset;
271 }
272
273
274 gchar *encodings_to_string(const GeanyEncoding* enc)
275 {
276 g_return_val_if_fail(enc != NULL, NULL);
277 g_return_val_if_fail(enc->name != NULL, NULL);
278 g_return_val_if_fail(enc->charset != NULL, NULL);
279
280 return g_strdup_printf("%s (%s)", enc->name, enc->charset);
281 }
282
283
284 const gchar *encodings_get_charset(const GeanyEncoding* enc)
285 {
286 g_return_val_if_fail(enc != NULL, NULL);
287 g_return_val_if_fail(enc->charset != NULL, NULL);
288
289 return enc->charset;
290 }
291
292
293 static GtkWidget *radio_items[GEANY_ENCODINGS_MAX];
294
295
296 void encodings_select_radio_item(const gchar *charset)
297 {
298 gint i;
299
300 g_return_if_fail(charset != NULL);
301
302 i = 0;
303 while (i < GEANY_ENCODINGS_MAX)
304 {
305 if (utils_str_equal(charset, encodings[i].charset))
306 break;
307 i++;
308 }
309 if (i == GEANY_ENCODINGS_MAX)
310 i = GEANY_ENCODING_UTF_8; /* fallback to UTF-8 */
311
312 /* ignore_callback has to be set by the caller */
313 gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(radio_items[i]), TRUE);
314 }
315
316
317 /* Regexp detection of file encoding declared in the file itself.
318 * Idea and parts of code taken from Bluefish, thanks.
319 * regex_compile() is used to compile regular expressions on program init and keep it in memory
320 * for faster access when opening a file. Pre-compiled regexps will be freed on program exit.
321 */
322 static GRegex *regex_compile(const gchar *pattern)
323 {
324 GError *error = NULL;
325 GRegex *regex = g_regex_new(pattern, G_REGEX_CASELESS, 0, &error);
326
327 if (!regex)
328 {
329 geany_debug("Failed to compile encoding regex (%s)", error->message);
330 g_error_free(error);
331 }
332 return regex;
333 }
334
335
336 static gchar *regex_match(GRegex *preg, const gchar *buffer, gsize size)
337 {
338 gchar *encoding = NULL;
339 GMatchInfo *minfo;
340
341 if (G_UNLIKELY(! pregs_loaded || buffer == NULL))
342 return NULL;
343
344 /* scan only the first 512 characters in the buffer */
345 size = MIN(size, 512);
346
347 if (g_regex_match_full(preg, buffer, size, 0, 0, &minfo, NULL) &&
348 g_match_info_get_match_count(minfo) >= 2)
349 {
350 encoding = g_match_info_fetch(minfo, 1);
351 geany_debug("Detected encoding by regex search: %s", encoding);
352
353 SETPTR(encoding, g_utf8_strup(encoding, -1));
354 }
355 g_match_info_free(minfo);
356 return encoding;
357 }
358
359
360 static void encodings_radio_item_change_cb(GtkCheckMenuItem *menuitem, gpointer user_data)
361 {
362 GeanyDocument *doc = document_get_current();
363 const gchar *charset = user_data;
364
365 if (ignore_callback || doc == NULL || charset == NULL ||
366 ! gtk_check_menu_item_get_active(menuitem) ||
367 utils_str_equal(charset, doc->encoding))
368 return;
369
370 if (doc->readonly)
371 {
372 utils_beep();
373 return;
374 }
375 document_undo_add(doc, UNDO_ENCODING, g_strdup(doc->encoding));
376
377 document_set_encoding(doc, charset);
378 }
379
380 static void encodings_reload_radio_item_change_cb(GtkMenuItem *menuitem, gpointer user_data)
381 {
382 GeanyDocument *doc = document_get_current();
383
384 g_return_if_fail(doc != NULL);
385
386 document_reload_prompt(doc, user_data);
387 }
388
389
390 void encodings_finalize(void)
391 {
392 if (pregs_loaded)
393 {
394 guint i, len;
395 len = G_N_ELEMENTS(pregs);
396 for (i = 0; i < len; i++)
397 {
398 g_regex_unref(pregs[i]);
399 }
400 }
401 }
402
403
404 void encodings_init(void)
405 {
406 GtkWidget *item, *menu[2], *submenu, *menu_westeuro, *menu_easteuro, *menu_eastasian, *menu_asian,
407 *menu_utf8, *menu_middleeast, *item_westeuro, *item_easteuro, *item_eastasian,
408 *item_asian, *item_utf8, *item_middleeast;
409 GCallback cb_func[2];
410 GSList *group = NULL;
411 gchar *label;
412 gint order, group_size;
413 guint i, j, k;
414
415 init_encodings();
416
417 if (! pregs_loaded)
418 {
419 pregs[0] = regex_compile(PATTERN_HTMLMETA);
420 pregs[1] = regex_compile(PATTERN_CODING);
421 pregs_loaded = TRUE;
422 }
423
424 /* create encodings submenu in document menu */
425 menu[0] = ui_lookup_widget(main_widgets.window, "set_encoding1_menu");
426 menu[1] = ui_lookup_widget(main_widgets.window, "menu_reload_as1_menu");
427 cb_func[0] = G_CALLBACK(encodings_radio_item_change_cb);
428 cb_func[1] = G_CALLBACK(encodings_reload_radio_item_change_cb);
429
430 for (k = 0; k < 2; k++)
431 {
432 menu_westeuro = gtk_menu_new();
433 item_westeuro = gtk_menu_item_new_with_mnemonic(_("_West European"));
434 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item_westeuro), menu_westeuro);
435 gtk_container_add(GTK_CONTAINER(menu[k]), item_westeuro);
436 gtk_widget_show_all(item_westeuro);
437
438 menu_easteuro = gtk_menu_new();
439 item_easteuro = gtk_menu_item_new_with_mnemonic(_("_East European"));
440 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item_easteuro), menu_easteuro);
441 gtk_container_add(GTK_CONTAINER(menu[k]), item_easteuro);
442 gtk_widget_show_all(item_easteuro);
443
444 menu_eastasian = gtk_menu_new();
445 item_eastasian = gtk_menu_item_new_with_mnemonic(_("East _Asian"));
446 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item_eastasian), menu_eastasian);
447 gtk_container_add(GTK_CONTAINER(menu[k]), item_eastasian);
448 gtk_widget_show_all(item_eastasian);
449
450 menu_asian = gtk_menu_new();
451 item_asian = gtk_menu_item_new_with_mnemonic(_("_SE & SW Asian"));
452 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item_asian), menu_asian);
453 gtk_container_add(GTK_CONTAINER(menu[k]), item_asian);
454 gtk_widget_show_all(item_asian);
455
456 menu_middleeast = gtk_menu_new();
457 item_middleeast = gtk_menu_item_new_with_mnemonic(_("_Middle Eastern"));
458 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item_middleeast), menu_middleeast);
459 gtk_container_add(GTK_CONTAINER(menu[k]), item_middleeast);
460 gtk_widget_show_all(item_middleeast);
461
462 menu_utf8 = gtk_menu_new();
463 item_utf8 = gtk_menu_item_new_with_mnemonic(_("_Unicode"));
464 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item_utf8), menu_utf8);
465 gtk_container_add(GTK_CONTAINER(menu[k]), item_utf8);
466 gtk_widget_show_all(item_utf8);
467
468 /** TODO can it be optimized? ATM 3782 runs at line "if (encodings[j].group ...)" */
469 for (i = 0; i < GEANY_ENCODING_GROUPS_MAX; i++)
470 {
471 order = 0;
472 switch (i)
473 {
474 case WESTEUROPEAN: submenu = menu_westeuro; group_size = 9; break;
475 case EASTEUROPEAN: submenu = menu_easteuro; group_size = 14; break;
476 case EASTASIAN: submenu = menu_eastasian; group_size = 14; break;
477 case ASIAN: submenu = menu_asian; group_size = 9; break;
478 case MIDDLEEASTERN: submenu = menu_middleeast; group_size = 7; break;
479 case UNICODE: submenu = menu_utf8; group_size = 8; break;
480 default: submenu = menu[k]; group_size = 1;
481 }
482
483 while (order < group_size) /* the biggest group has 13 elements */
484 {
485 for (j = 0; j < GEANY_ENCODINGS_MAX; j++)
486 {
487 if (encodings[j].group == i && encodings[j].order == order)
488 {
489 label = encodings_to_string(&encodings[j]);
490 if (k == 0)
491 {
492 item = gtk_radio_menu_item_new_with_label(group, label);
493 group = gtk_radio_menu_item_get_group(GTK_RADIO_MENU_ITEM(item));
494 radio_items[j] = item;
495 }
496 else
497 item = gtk_menu_item_new_with_label(label);
498 gtk_widget_show(item);
499 gtk_container_add(GTK_CONTAINER(submenu), item);
500 g_signal_connect(item, "activate", cb_func[k],
501 (gpointer) encodings[j].charset);
502 g_free(label);
503 break;
504 }
505 }
506 order++;
507 }
508 }
509 }
510 }
511
512
513 static gint encoding_combo_store_sort_func(GtkTreeModel *model,
514 GtkTreeIter *a,
515 GtkTreeIter *b,
516 gpointer data)
517 {
518 gboolean a_has_child = gtk_tree_model_iter_has_child(model, a);
519 gboolean b_has_child = gtk_tree_model_iter_has_child(model, b);
520 gchar *a_string;
521 gchar *b_string;
522 gint cmp_res;
523
524 if (a_has_child != b_has_child)
525 return a_has_child ? -1 : 1;
526
527 gtk_tree_model_get(model, a, 1, &a_string, -1);
528 gtk_tree_model_get(model, b, 1, &b_string, -1);
529 cmp_res = strcmp(a_string, b_string);
530 g_free(a_string);
531 g_free(b_string);
532 return cmp_res;
533 }
534
535
536 GtkTreeStore *encodings_encoding_store_new(gboolean has_detect)
537 {
538 GtkTreeStore *store;
539 GtkTreeIter iter_current, iter_westeuro, iter_easteuro, iter_eastasian,
540 iter_asian, iter_utf8, iter_middleeast;
541 GtkTreeIter *iter_parent;
542 gchar *encoding_string;
543 gint i;
544
545 store = gtk_tree_store_new(2, G_TYPE_INT, G_TYPE_STRING);
546
547 if (has_detect)
548 {
549 gtk_tree_store_append(store, &iter_current, NULL);
550 gtk_tree_store_set(store, &iter_current, 0, GEANY_ENCODINGS_MAX, 1, _("Detect from file"), -1);
551 }
552
553 gtk_tree_store_append(store, &iter_westeuro, NULL);
554 gtk_tree_store_set(store, &iter_westeuro, 0, -1, 1, _("West European"), -1);
555 gtk_tree_store_append(store, &iter_easteuro, NULL);
556 gtk_tree_store_set(store, &iter_easteuro, 0, -1, 1, _("East European"), -1);
557 gtk_tree_store_append(store, &iter_eastasian, NULL);
558 gtk_tree_store_set(store, &iter_eastasian, 0, -1, 1, _("East Asian"), -1);
559 gtk_tree_store_append(store, &iter_asian, NULL);
560 gtk_tree_store_set(store, &iter_asian, 0, -1, 1, _("SE & SW Asian"), -1);
561 gtk_tree_store_append(store, &iter_middleeast, NULL);
562 gtk_tree_store_set(store, &iter_middleeast, 0, -1, 1, _("Middle Eastern"), -1);
563 gtk_tree_store_append(store, &iter_utf8, NULL);
564 gtk_tree_store_set(store, &iter_utf8, 0, -1, 1, _("Unicode"), -1);
565
566 for (i = 0; i < GEANY_ENCODINGS_MAX; i++)
567 {
568 switch (encodings[i].group)
569 {
570 case WESTEUROPEAN: iter_parent = &iter_westeuro; break;
571 case EASTEUROPEAN: iter_parent = &iter_easteuro; break;
572 case EASTASIAN: iter_parent = &iter_eastasian; break;
573 case ASIAN: iter_parent = &iter_asian; break;
574 case MIDDLEEASTERN: iter_parent = &iter_middleeast; break;
575 case UNICODE: iter_parent = &iter_utf8; break;
576 case NONE:
577 default: iter_parent = NULL;
578 }
579 gtk_tree_store_append(store, &iter_current, iter_parent);
580 encoding_string = encodings_to_string(&encodings[i]);
581 gtk_tree_store_set(store, &iter_current, 0, i, 1, encoding_string, -1);
582 g_free(encoding_string);
583 }
584
585 gtk_tree_sortable_set_sort_column_id(GTK_TREE_SORTABLE(store), 1, GTK_SORT_ASCENDING);
586 gtk_tree_sortable_set_sort_func(GTK_TREE_SORTABLE(store), 1, encoding_combo_store_sort_func, NULL, NULL);
587
588 return store;
589 }
590
591
592 gint encodings_encoding_store_get_encoding(GtkTreeStore *store, GtkTreeIter *iter)
593 {
594 gint enc;
595 gtk_tree_model_get(GTK_TREE_MODEL(store), iter, 0, &enc, -1);
596 return enc;
597 }
598
599
600 gboolean encodings_encoding_store_get_iter(GtkTreeStore *store, GtkTreeIter *iter, gint enc)
601 {
602 if (gtk_tree_model_get_iter_first(GTK_TREE_MODEL(store), iter))
603 {
604 do
605 {
606 if (encodings_encoding_store_get_encoding(store, iter) == enc)
607 return TRUE;
608 }
609 while (ui_tree_model_iter_any_next(GTK_TREE_MODEL(store), iter, TRUE));
610 }
611 return FALSE;
612 }
613
614
615 void encodings_encoding_store_cell_data_func(GtkCellLayout *cell_layout,
616 GtkCellRenderer *cell,
617 GtkTreeModel *tree_model,
618 GtkTreeIter *iter,
619 gpointer data)
620 {
621 gboolean sensitive = !gtk_tree_model_iter_has_child(tree_model, iter);
622 gchar *text;
623
624 gtk_tree_model_get(tree_model, iter, 1, &text, -1);
625 g_object_set(cell, "sensitive", sensitive, "text", text, NULL);
626 g_free(text);
627 }
628
629
630 /**
631 * Tries to convert @a buffer into UTF-8 encoding from the encoding specified with @a charset.
632 * If @a fast is not set, additional checks to validate the converted string are performed.
633 *
634 * @param buffer The input string to convert.
635 * @param size The length of the string, or -1 if the string is nul-terminated.
636 * @param charset The charset to be used for conversion.
637 * @param fast @c TRUE to only convert the input and skip extended checks on the converted string.
638 *
639 * @return If the conversion was successful, a newly allocated nul-terminated string,
640 * which must be freed with @c g_free(). Otherwise @c NULL.
641 **/
642 gchar *encodings_convert_to_utf8_from_charset(const gchar *buffer, gssize size,
643 const gchar *charset, gboolean fast)
644 {
645 gchar *utf8_content = NULL;
646 GError *conv_error = NULL;
647 gchar* converted_contents = NULL;
648 gsize bytes_written;
649
650 g_return_val_if_fail(buffer != NULL, NULL);
651 g_return_val_if_fail(charset != NULL, NULL);
652
653 converted_contents = g_convert(buffer, size, "UTF-8", charset, NULL,
654 &bytes_written, &conv_error);
655
656 if (fast)
657 {
658 utf8_content = converted_contents;
659 if (conv_error != NULL) g_error_free(conv_error);
660 }
661 else if (conv_error != NULL || ! g_utf8_validate(converted_contents, bytes_written, NULL))
662 {
663 if (conv_error != NULL)
664 {
665 geany_debug("Couldn't convert from %s to UTF-8 (%s).", charset, conv_error->message);
666 g_error_free(conv_error);
667 conv_error = NULL;
668 }
669 else
670 geany_debug("Couldn't convert from %s to UTF-8.", charset);
671
672 utf8_content = NULL;
673 g_free(converted_contents);
674 }
675 else
676 {
677 geany_debug("Converted from %s to UTF-8.", charset);
678 utf8_content = converted_contents;
679 }
680
681 return utf8_content;
682 }
683
684
685 static gchar *encodings_check_regexes(const gchar *buffer, gsize size)
686 {
687 guint i;
688
689 for (i = 0; i < G_N_ELEMENTS(pregs); i++)
690 {
691 gchar *charset;
692
693 if ((charset = regex_match(pregs[i], buffer, size)) != NULL)
694 return charset;
695 }
696 return NULL;
697 }
698
699
700 static gchar *encodings_convert_to_utf8_with_suggestion(const gchar *buffer, gssize size,
701 const gchar *suggested_charset, gchar **used_encoding)
702 {
703 const gchar *locale_charset = NULL;
704 const gchar *charset;
705 gchar *utf8_content;
706 gboolean check_suggestion = suggested_charset != NULL;
707 gboolean check_locale = FALSE;
708 gint i, preferred_charset;
709
710 if (size == -1)
711 {
712 size = strlen(buffer);
713 }
714
715 /* current locale is not UTF-8, we have to check this charset */
716 check_locale = ! g_get_charset(&locale_charset);
717
718 /* First check for preferred charset, if specified */
719 preferred_charset = file_prefs.default_open_encoding;
720
721 if (preferred_charset == encodings[GEANY_ENCODING_NONE].idx ||
722 preferred_charset < 0 ||
723 preferred_charset >= GEANY_ENCODINGS_MAX)
724 {
725 preferred_charset = -1;
726 }
727
728 /* -1 means "Preferred charset" */
729 for (i = -1; i < GEANY_ENCODINGS_MAX; i++)
730 {
731 if (G_UNLIKELY(i == encodings[GEANY_ENCODING_NONE].idx))
732 continue;
733
734 if (check_suggestion)
735 {
736 check_suggestion = FALSE;
737 charset = encodings_normalize_charset(suggested_charset);
738 if (charset == NULL) /* we failed at normalizing suggested encoding, try it as is */
739 charset = suggested_charset;
740 i = -2; /* keep i below the start value to have it again at -1 on the next loop run */
741 }
742 else if (check_locale)
743 {
744 check_locale = FALSE;
745 charset = locale_charset;
746 i = -2; /* keep i below the start value to have it again at -1 on the next loop run */
747 }
748 else if (i == -1)
749 {
750 if (preferred_charset >= 0)
751 {
752 charset = encodings[preferred_charset].charset;
753 geany_debug("Using preferred charset: %s", charset);
754 }
755 else
756 continue;
757 }
758 else if (i >= 0)
759 charset = encodings[i].charset;
760 else /* in this case we have i == -2, continue to increase i and go ahead */
761 continue;
762
763 if (G_UNLIKELY(charset == NULL))
764 continue;
765
766 geany_debug("Trying to convert %" G_GSIZE_FORMAT " bytes of data from %s into UTF-8.",
767 size, charset);
768 utf8_content = encodings_convert_to_utf8_from_charset(buffer, size, charset, FALSE);
769
770 if (G_LIKELY(utf8_content != NULL))
771 {
772 if (used_encoding != NULL)
773 {
774 if (G_UNLIKELY(*used_encoding != NULL))
775 {
776 geany_debug("%s:%d", __FILE__, __LINE__);
777 g_free(*used_encoding);
778 }
779 *used_encoding = g_strdup(charset);
780 }
781 return utf8_content;
782 }
783 }
784
785 return NULL;
786 }
787
788
789 /**
790 * Tries to convert @a buffer into UTF-8 encoding and store the detected original encoding in
791 * @a used_encoding.
792 *
793 * @param buffer the input string to convert.
794 * @param size the length of the string, or -1 if the string is nul-terminated.
795 * @param used_encoding return location of the detected encoding of the input string, or @c NULL.
796 *
797 * @return If the conversion was successful, a newly allocated nul-terminated string,
798 * which must be freed with @c g_free(). Otherwise @c NULL.
799 **/
800 gchar *encodings_convert_to_utf8(const gchar *buffer, gssize size, gchar **used_encoding)
801 {
802 gchar *regex_charset;
803 gchar *utf8;
804
805 /* first try to read the encoding from the file content */
806 regex_charset = encodings_check_regexes(buffer, size);
807 utf8 = encodings_convert_to_utf8_with_suggestion(buffer, size, regex_charset, used_encoding);
808 g_free(regex_charset);
809
810 return utf8;
811 }
812
813
814 /* If there's a BOM, return a corresponding GEANY_ENCODING_UTF_* index,
815 * otherwise GEANY_ENCODING_NONE.
816 * */
817 GeanyEncodingIndex encodings_scan_unicode_bom(const gchar *string, gsize len, guint *bom_len)
818 {
819 if (len >= 3)
820 {
821 if (bom_len)
822 *bom_len = 3;
823
824 if ((guchar)string[0] == 0xef && (guchar)string[1] == 0xbb &&
825 (guchar)string[2] == 0xbf)
826 {
827 return GEANY_ENCODING_UTF_8;
828 }
829 }
830 if (len >= 4)
831 {
832 if (bom_len)
833 *bom_len = 4;
834
835 if ((guchar)string[0] == 0x00 && (guchar)string[1] == 0x00 &&
836 (guchar)string[2] == 0xfe && (guchar)string[3] == 0xff)
837 {
838 return GEANY_ENCODING_UTF_32BE; /* Big endian */
839 }
840 if ((guchar)string[0] == 0xff && (guchar)string[1] == 0xfe &&
841 (guchar)string[2] == 0x00 && (guchar)string[3] == 0x00)
842 {
843 return GEANY_ENCODING_UTF_32LE; /* Little endian */
844 }
845 if ((string[0] == 0x2b && string[1] == 0x2f && string[2] == 0x76) &&
846 (string[3] == 0x38 || string[3] == 0x39 || string[3] == 0x2b || string[3] == 0x2f))
847 {
848 return GEANY_ENCODING_UTF_7;
849 }
850 }
851 if (len >= 2)
852 {
853 if (bom_len)
854 *bom_len = 2;
855
856 if ((guchar)string[0] == 0xfe && (guchar)string[1] == 0xff)
857 {
858 return GEANY_ENCODING_UTF_16BE; /* Big endian */
859 }
860 if ((guchar)string[0] == 0xff && (guchar)string[1] == 0xfe)
861 {
862 return GEANY_ENCODING_UTF_16LE; /* Little endian */
863 }
864 }
865 if (bom_len)
866 *bom_len = 0;
867 return GEANY_ENCODING_NONE;
868 }
869
870
871 gboolean encodings_is_unicode_charset(const gchar *string)
872 {
873 if (string != NULL &&
874 (strncmp(string, "UTF", 3) == 0 || strncmp(string, "UCS", 3) == 0))
875 {
876 return TRUE;
877 }
878 return FALSE;
879 }
880
881
882 typedef struct
883 {
884 gchar *data; /* null-terminated data */
885 gsize size; /* actual data size */
886 gsize len; /* string length of data */
887 gchar *enc;
888 gboolean bom;
889 gboolean partial;
890 } BufferData;
891
892
893 /* convert data with the specified encoding */
894 static gboolean
895 handle_forced_encoding(BufferData *buffer, const gchar *forced_enc)
896 {
897 GeanyEncodingIndex enc_idx;
898
899 if (utils_str_equal(forced_enc, "UTF-8"))
900 {
901 if (! g_utf8_validate(buffer->data, buffer->len, NULL))
902 {
903 return FALSE;
904 }
905 }
906 else
907 {
908 gchar *converted_text = encodings_convert_to_utf8_from_charset(
909 buffer->data, buffer->size, forced_enc, FALSE);
910 if (converted_text == NULL)
911 {
912 return FALSE;
913 }
914 else
915 {
916 SETPTR(buffer->data, converted_text);
917 buffer->len = strlen(converted_text);
918 }
919 }
920 enc_idx = encodings_scan_unicode_bom(buffer->data, buffer->size, NULL);
921 buffer->bom = (enc_idx == GEANY_ENCODING_UTF_8);
922 buffer->enc = g_strdup(forced_enc);
923 return TRUE;
924 }
925
926
927 /* detect encoding and convert to UTF-8 if necessary */
928 static gboolean
929 handle_encoding(BufferData *buffer, GeanyEncodingIndex enc_idx)
930 {
931 g_return_val_if_fail(buffer->enc == NULL, FALSE);
932 g_return_val_if_fail(buffer->bom == FALSE, FALSE);
933
934 if (buffer->size == 0)
935 {
936 /* we have no data so assume UTF-8, buffer->len can be 0 even we have an empty
937 * e.g. UTF32 file with a BOM(so size is 4, len is 0) */
938 buffer->enc = g_strdup("UTF-8");
939 }
940 else
941 {
942 /* first check for a BOM */
943 if (enc_idx != GEANY_ENCODING_NONE)
944 {
945 buffer->enc = g_strdup(encodings[enc_idx].charset);
946 buffer->bom = TRUE;
947
948 if (enc_idx != GEANY_ENCODING_UTF_8) /* the BOM indicated something else than UTF-8 */
949 {
950 gchar *converted_text = encodings_convert_to_utf8_from_charset(
951 buffer->data, buffer->size, buffer->enc, FALSE);
952 if (converted_text != NULL)
953 {
954 SETPTR(buffer->data, converted_text);
955 buffer->len = strlen(converted_text);
956 }
957 else
958 {
959 /* there was a problem converting data from BOM encoding type */
960 SETPTR(buffer->enc, NULL);
961 buffer->bom = FALSE;
962 }
963 }
964 }
965
966 if (buffer->enc == NULL) /* either there was no BOM or the BOM encoding failed */
967 {
968 /* first try to read the encoding from the file content */
969 gchar *regex_charset = encodings_check_regexes(buffer->data, buffer->size);
970
971 /* try UTF-8 first */
972 if (encodings_get_idx_from_charset(regex_charset) == GEANY_ENCODING_UTF_8 &&
973 (buffer->size == buffer->len) && g_utf8_validate(buffer->data, buffer->len, NULL))
974 {
975 buffer->enc = g_strdup("UTF-8");
976 }
977 else
978 {
979 /* detect the encoding */
980 gchar *converted_text = encodings_convert_to_utf8_with_suggestion(buffer->data,
981 buffer->size, regex_charset, &buffer->enc);
982
983 if (converted_text == NULL)
984 {
985 g_free(regex_charset);
986 return FALSE;
987 }
988 SETPTR(buffer->data, converted_text);
989 buffer->len = strlen(converted_text);
990 }
991 g_free(regex_charset);
992 }
993 }
994 return TRUE;
995 }
996
997
998 static void
999 handle_bom(BufferData *buffer)
1000 {
1001 guint bom_len;
1002
1003 encodings_scan_unicode_bom(buffer->data, buffer->size, &bom_len);
1004 g_return_if_fail(bom_len != 0);
1005
1006 /* use filedata->len here because the contents are already converted into UTF-8 */
1007 buffer->len -= bom_len;
1008 /* overwrite the BOM with the remainder of the file contents, plus the NULL terminator. */
1009 g_memmove(buffer->data, buffer->data + bom_len, buffer->len + 1);
1010 buffer->data = g_realloc(buffer->data, buffer->len + 1);
1011 }
1012
1013
1014 /* loads textfile data, verifies and converts to forced_enc or UTF-8. Also handles BOM. */
1015 static gboolean handle_buffer(BufferData *buffer, const gchar *forced_enc)
1016 {
1017 GeanyEncodingIndex tmp_enc_idx;
1018
1019 /* temporarily retrieve the encoding idx based on the BOM to suppress the following warning
1020 * if we have a BOM */
1021 tmp_enc_idx = encodings_scan_unicode_bom(buffer->data, buffer->size, NULL);
1022
1023 /* check whether the size of the loaded data is equal to the size of the file in the
1024 * filesystem file size may be 0 to allow opening files in /proc/ which have typically a
1025 * file size of 0 bytes */
1026 if (buffer->len != buffer->size && buffer->size != 0 && (
1027 tmp_enc_idx == GEANY_ENCODING_UTF_8 || /* tmp_enc_idx can be UTF-7/8/16/32, UCS and None */
1028 tmp_enc_idx == GEANY_ENCODING_UTF_7)) /* filter UTF-7/8 where no NULL bytes are allowed */
1029 {
1030 buffer->partial = TRUE;
1031 }
1032
1033 /* Determine character encoding and convert to UTF-8 */
1034 if (forced_enc != NULL)
1035 {
1036 /* the encoding should be ignored(requested by user), so open the file "as it is" */
1037 if (utils_str_equal(forced_enc, encodings[GEANY_ENCODING_NONE].charset))
1038 {
1039 buffer->bom = FALSE;
1040 buffer->enc = g_strdup(encodings[GEANY_ENCODING_NONE].charset);
1041 }
1042 else if (! handle_forced_encoding(buffer, forced_enc))
1043 {
1044 return FALSE;
1045 }
1046 }
1047 else if (! handle_encoding(buffer, tmp_enc_idx))
1048 {
1049 return FALSE;
1050 }
1051
1052 if (buffer->bom)
1053 handle_bom(buffer);
1054 return TRUE;
1055 }
1056
1057
1058 /*
1059 * Tries to convert @a buffer into UTF-8 encoding. Unlike encodings_convert_to_utf8()
1060 * and encodings_convert_to_utf8_from_charset() it handles the possible BOM in the data.
1061 *
1062 * @param buf a pointer to modifiable null-terminated buffer to convert.
1063 * It may or may not be modified, and should be freed whatever happens.
1064 * @param size a pointer to the size of the buffer (expected to be e.g. the on-disk
1065 * file size). It will be updated to the new size.
1066 * @param forced_enc forced encoding to use, or @c NULL
1067 * @param used_encoding return location for the actually used encoding, or @c NULL
1068 * @param has_bom return location to store whether the data had a BOM, or @c NULL
1069 * @param partial return location to store whether the conversion may be partial, or @c NULL
1070 *
1071 * @return @C TRUE if the conversion succeeded, @c FALSE otherwise.
1072 */
1073 gboolean encodings_convert_to_utf8_auto(gchar **buf, gsize *size, const gchar *forced_enc,
1074 gchar **used_encoding, gboolean *has_bom, gboolean *partial)
1075 {
1076 BufferData buffer;
1077
1078 buffer.data = *buf;
1079 buffer.size = *size;
1080 /* use strlen to check for null chars */
1081 buffer.len = strlen(buffer.data);
1082 buffer.enc = NULL;
1083 buffer.bom = FALSE;
1084 buffer.partial = FALSE;
1085
1086 if (! handle_buffer(&buffer, forced_enc))
1087 return FALSE;
1088
1089 *size = buffer.len;
1090 if (used_encoding)
1091 *used_encoding = buffer.enc;
1092 else
1093 g_free(buffer.enc);
1094 if (has_bom)
1095 *has_bom = buffer.bom;
1096 if (partial)
1097 *partial = buffer.partial;
1098
1099 *buf = buffer.data;
1100 return TRUE;
1101 }
A fast and lightweight IDE