2 * encodings.c - this file is part of Geany, a fast and lightweight IDE
4 * Copyright 2005-2012 Enrico Tröger <enrico(dot)troeger(at)uvena(dot)de>
5 * Copyright 2006-2012 Nick Treleaven <nick(dot)treleaven(at)btinternet(dot)com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License along
18 * with this program; if not, write to the Free Software Foundation, Inc.,
19 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23 * Encoding conversion and Byte Order Mark (BOM) handling.
27 * Modified by the gedit Team, 2002. See the gedit AUTHORS file for a
28 * list of people on the gedit Team.
29 * See the gedit ChangeLog files for a list of changes.
31 /* Stolen from anjuta */
37 #include "encodings.h"
40 #include "callbacks.h"
41 #include "documentprivate.h"
49 /* <meta http-equiv="content-type" content="text/html; charset=UTF-8" /> */
50 #define PATTERN_HTMLMETA "<meta\\s+http-equiv\\s*=\\s*\"?content-type\"?\\s+content\\s*=\\s*\"text/x?html;\\s*charset=([a-z0-9_-]+)\"\\s*/?>"
51 /* " geany_encoding=utf-8 " or " coding: utf-8 " */
52 #define PATTERN_CODING "coding[\t ]*[:=][\t ]*\"?([a-z0-9-]+)\"?[\t ]*"
54 /* precompiled regexps */
55 static GRegex
*pregs
[2];
56 static gboolean pregs_loaded
= FALSE
;
59 GeanyEncoding encodings
[GEANY_ENCODINGS_MAX
];
62 #define fill(Order, Group, Idx, Charset, Name) \
63 encodings[Idx].idx = Idx; \
64 encodings[Idx].order = Order; \
65 encodings[Idx].group = Group; \
66 encodings[Idx].charset = Charset; \
67 encodings[Idx].name = Name;
69 static void init_encodings(void)
71 fill(0, WESTEUROPEAN
, GEANY_ENCODING_ISO_8859_14
, "ISO-8859-14", _("Celtic"));
72 fill(1, WESTEUROPEAN
, GEANY_ENCODING_ISO_8859_7
, "ISO-8859-7", _("Greek"));
73 fill(2, WESTEUROPEAN
, GEANY_ENCODING_WINDOWS_1253
, "WINDOWS-1253", _("Greek"));
74 fill(3, WESTEUROPEAN
, GEANY_ENCODING_ISO_8859_10
, "ISO-8859-10", _("Nordic"));
75 fill(4, WESTEUROPEAN
, GEANY_ENCODING_ISO_8859_3
, "ISO-8859-3", _("South European"));
76 fill(5, WESTEUROPEAN
, GEANY_ENCODING_IBM_850
, "IBM850", _("Western"));
77 fill(6, WESTEUROPEAN
, GEANY_ENCODING_ISO_8859_1
, "ISO-8859-1", _("Western"));
78 fill(7, WESTEUROPEAN
, GEANY_ENCODING_ISO_8859_15
, "ISO-8859-15", _("Western"));
79 fill(8, WESTEUROPEAN
, GEANY_ENCODING_WINDOWS_1252
, "WINDOWS-1252", _("Western"));
81 fill(0, EASTEUROPEAN
, GEANY_ENCODING_ISO_8859_4
, "ISO-8859-4", _("Baltic"));
82 fill(1, EASTEUROPEAN
, GEANY_ENCODING_ISO_8859_13
, "ISO-8859-13", _("Baltic"));
83 fill(2, EASTEUROPEAN
, GEANY_ENCODING_WINDOWS_1257
, "WINDOWS-1257", _("Baltic"));
84 fill(3, EASTEUROPEAN
, GEANY_ENCODING_IBM_852
, "IBM852", _("Central European"));
85 fill(4, EASTEUROPEAN
, GEANY_ENCODING_ISO_8859_2
, "ISO-8859-2", _("Central European"));
86 fill(5, EASTEUROPEAN
, GEANY_ENCODING_WINDOWS_1250
, "WINDOWS-1250", _("Central European"));
87 fill(6, EASTEUROPEAN
, GEANY_ENCODING_IBM_855
, "IBM855", _("Cyrillic"));
88 fill(7, EASTEUROPEAN
, GEANY_ENCODING_ISO_8859_5
, "ISO-8859-5", _("Cyrillic"));
89 /* ISO-IR-111 not available on Windows */
90 fill(8, EASTEUROPEAN
, GEANY_ENCODING_ISO_IR_111
, "ISO-IR-111", _("Cyrillic"));
91 fill(9, EASTEUROPEAN
, GEANY_ENCODING_KOI8_R
, "KOI8-R", _("Cyrillic"));
92 fill(10, EASTEUROPEAN
, GEANY_ENCODING_WINDOWS_1251
, "WINDOWS-1251", _("Cyrillic"));
93 fill(11, EASTEUROPEAN
, GEANY_ENCODING_CP_866
, "CP866", _("Cyrillic/Russian"));
94 fill(12, EASTEUROPEAN
, GEANY_ENCODING_KOI8_U
, "KOI8-U", _("Cyrillic/Ukrainian"));
95 fill(13, EASTEUROPEAN
, GEANY_ENCODING_ISO_8859_16
, "ISO-8859-16", _("Romanian"));
97 fill(0, MIDDLEEASTERN
, GEANY_ENCODING_IBM_864
, "IBM864", _("Arabic"));
98 fill(1, MIDDLEEASTERN
, GEANY_ENCODING_ISO_8859_6
, "ISO-8859-6", _("Arabic"));
99 fill(2, MIDDLEEASTERN
, GEANY_ENCODING_WINDOWS_1256
, "WINDOWS-1256", _("Arabic"));
100 fill(3, MIDDLEEASTERN
, GEANY_ENCODING_IBM_862
, "IBM862", _("Hebrew"));
101 /* not available at all, ? */
102 fill(4, MIDDLEEASTERN
, GEANY_ENCODING_ISO_8859_8_I
, "ISO-8859-8-I", _("Hebrew"));
103 fill(5, MIDDLEEASTERN
, GEANY_ENCODING_WINDOWS_1255
, "WINDOWS-1255", _("Hebrew"));
104 fill(6, MIDDLEEASTERN
, GEANY_ENCODING_ISO_8859_8
, "ISO-8859-8", _("Hebrew Visual"));
106 fill(0, ASIAN
, GEANY_ENCODING_ARMSCII_8
, "ARMSCII-8", _("Armenian"));
107 fill(1, ASIAN
, GEANY_ENCODING_GEOSTD8
, "GEORGIAN-ACADEMY", _("Georgian"));
108 fill(2, ASIAN
, GEANY_ENCODING_TIS_620
, "TIS-620", _("Thai"));
109 fill(3, ASIAN
, GEANY_ENCODING_IBM_857
, "IBM857", _("Turkish"));
110 fill(4, ASIAN
, GEANY_ENCODING_WINDOWS_1254
, "WINDOWS-1254", _("Turkish"));
111 fill(5, ASIAN
, GEANY_ENCODING_ISO_8859_9
, "ISO-8859-9", _("Turkish"));
112 fill(6, ASIAN
, GEANY_ENCODING_TCVN
, "TCVN", _("Vietnamese"));
113 fill(7, ASIAN
, GEANY_ENCODING_VISCII
, "VISCII", _("Vietnamese"));
114 fill(8, ASIAN
, GEANY_ENCODING_WINDOWS_1258
, "WINDOWS-1258", _("Vietnamese"));
116 fill(0, UNICODE
, GEANY_ENCODING_UTF_7
, "UTF-7", _("Unicode"));
117 fill(1, UNICODE
, GEANY_ENCODING_UTF_8
, "UTF-8", _("Unicode"));
118 fill(2, UNICODE
, GEANY_ENCODING_UTF_16LE
, "UTF-16LE", _("Unicode"));
119 fill(3, UNICODE
, GEANY_ENCODING_UTF_16BE
, "UTF-16BE", _("Unicode"));
120 fill(4, UNICODE
, GEANY_ENCODING_UCS_2LE
, "UCS-2LE", _("Unicode"));
121 fill(5, UNICODE
, GEANY_ENCODING_UCS_2BE
, "UCS-2BE", _("Unicode"));
122 fill(6, UNICODE
, GEANY_ENCODING_UTF_32LE
, "UTF-32LE", _("Unicode"));
123 fill(7, UNICODE
, GEANY_ENCODING_UTF_32BE
, "UTF-32BE", _("Unicode"));
125 fill(0, EASTASIAN
, GEANY_ENCODING_GB18030
, "GB18030", _("Chinese Simplified"));
126 fill(1, EASTASIAN
, GEANY_ENCODING_GB2312
, "GB2312", _("Chinese Simplified"));
127 fill(2, EASTASIAN
, GEANY_ENCODING_GBK
, "GBK", _("Chinese Simplified"));
128 /* maybe not available on Linux */
129 fill(3, EASTASIAN
, GEANY_ENCODING_HZ
, "HZ", _("Chinese Simplified"));
130 fill(4, EASTASIAN
, GEANY_ENCODING_BIG5
, "BIG5", _("Chinese Traditional"));
131 fill(5, EASTASIAN
, GEANY_ENCODING_BIG5_HKSCS
, "BIG5-HKSCS", _("Chinese Traditional"));
132 fill(6, EASTASIAN
, GEANY_ENCODING_EUC_TW
, "EUC-TW", _("Chinese Traditional"));
133 fill(7, EASTASIAN
, GEANY_ENCODING_EUC_JP
, "EUC-JP", _("Japanese"));
134 fill(8, EASTASIAN
, GEANY_ENCODING_ISO_2022_JP
, "ISO-2022-JP", _("Japanese"));
135 fill(9, EASTASIAN
, GEANY_ENCODING_SHIFT_JIS
, "SHIFT_JIS", _("Japanese"));
136 fill(10, EASTASIAN
, GEANY_ENCODING_CP_932
, "CP932", _("Japanese"));
137 fill(11, EASTASIAN
, GEANY_ENCODING_EUC_KR
, "EUC-KR", _("Korean"));
138 fill(12, EASTASIAN
, GEANY_ENCODING_ISO_2022_KR
, "ISO-2022-KR", _("Korean"));
139 fill(13, EASTASIAN
, GEANY_ENCODING_JOHAB
, "JOHAB", _("Korean"));
140 fill(14, EASTASIAN
, GEANY_ENCODING_UHC
, "UHC", _("Korean"));
142 fill(0, NONE
, GEANY_ENCODING_NONE
, "None", _("Without encoding"));
146 /* compares two encoding names in a permissive fashion.
147 * e.g. "utf8" matches "UTF-8", "iso8859_1" matches "ISO-8859-1", etc. */
148 static gboolean
encodings_charset_equals(const gchar
*a
, const gchar
*b
)
150 gboolean was_alpha
= FALSE
; /* whether last character of previous word was a letter */
151 gboolean need_sep
= FALSE
; /* whether we're expecting an implicit separator */
157 if (g_ascii_toupper(*a
) == g_ascii_toupper(*b
) &&
158 ((is_alpha
= g_ascii_isalpha(*a
)) || g_ascii_isdigit(*a
)))
160 /* either there was a real separator, or we need a implicit one (a chage from alpha to
162 if (! need_sep
|| (was_alpha
!= is_alpha
))
166 was_alpha
= is_alpha
;
176 if (! g_ascii_isalnum(*a
))
181 if (! g_ascii_isalnum(*b
))
196 GeanyEncodingIndex
encodings_get_idx_from_charset(const gchar
*charset
)
201 return GEANY_ENCODING_UTF_8
;
204 while (i
< GEANY_ENCODINGS_MAX
)
206 if (encodings_charset_equals(charset
, encodings
[i
].charset
))
211 return GEANY_ENCODING_UTF_8
;
215 const GeanyEncoding
*encodings_get_from_charset(const gchar
*charset
)
220 return &encodings
[GEANY_ENCODING_UTF_8
];
223 while (i
< GEANY_ENCODINGS_MAX
)
225 if (encodings_charset_equals(charset
, encodings
[i
].charset
))
226 return &encodings
[i
];
235 static const gchar
*encodings_normalize_charset(const gchar
*charset
)
237 const GeanyEncoding
*encoding
;
239 encoding
= encodings_get_from_charset(charset
);
240 if (encoding
!= NULL
)
241 return encoding
->charset
;
247 const GeanyEncoding
*encodings_get_from_index(gint idx
)
249 g_return_val_if_fail(idx
>= 0 && idx
< GEANY_ENCODINGS_MAX
, NULL
);
251 return &encodings
[idx
];
256 * Gets the character set name of the specified index e.g. for use with
257 * @ref document_set_encoding().
259 * @param idx @ref GeanyEncodingIndex to retrieve the corresponding character set.
262 * @return The charset according to idx, or @c NULL if the index is invalid.
266 const gchar
* encodings_get_charset_from_index(gint idx
)
268 g_return_val_if_fail(idx
>= 0 && idx
< GEANY_ENCODINGS_MAX
, NULL
);
270 return encodings
[idx
].charset
;
274 gchar
*encodings_to_string(const GeanyEncoding
* enc
)
276 g_return_val_if_fail(enc
!= NULL
, NULL
);
277 g_return_val_if_fail(enc
->name
!= NULL
, NULL
);
278 g_return_val_if_fail(enc
->charset
!= NULL
, NULL
);
280 return g_strdup_printf("%s (%s)", enc
->name
, enc
->charset
);
284 const gchar
*encodings_get_charset(const GeanyEncoding
* enc
)
286 g_return_val_if_fail(enc
!= NULL
, NULL
);
287 g_return_val_if_fail(enc
->charset
!= NULL
, NULL
);
293 static GtkWidget
*radio_items
[GEANY_ENCODINGS_MAX
];
296 void encodings_select_radio_item(const gchar
*charset
)
300 g_return_if_fail(charset
!= NULL
);
303 while (i
< GEANY_ENCODINGS_MAX
)
305 if (utils_str_equal(charset
, encodings
[i
].charset
))
309 if (i
== GEANY_ENCODINGS_MAX
)
310 i
= GEANY_ENCODING_UTF_8
; /* fallback to UTF-8 */
312 /* ignore_callback has to be set by the caller */
313 gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(radio_items
[i
]), TRUE
);
317 /* Regexp detection of file encoding declared in the file itself.
318 * Idea and parts of code taken from Bluefish, thanks.
319 * regex_compile() is used to compile regular expressions on program init and keep it in memory
320 * for faster access when opening a file. Pre-compiled regexps will be freed on program exit.
322 static GRegex
*regex_compile(const gchar
*pattern
)
324 GError
*error
= NULL
;
325 GRegex
*regex
= g_regex_new(pattern
, G_REGEX_CASELESS
, 0, &error
);
329 geany_debug("Failed to compile encoding regex (%s)", error
->message
);
336 static gchar
*regex_match(GRegex
*preg
, const gchar
*buffer
, gsize size
)
338 gchar
*encoding
= NULL
;
341 if (G_UNLIKELY(! pregs_loaded
|| buffer
== NULL
))
344 /* scan only the first 512 characters in the buffer */
345 size
= MIN(size
, 512);
347 if (g_regex_match_full(preg
, buffer
, size
, 0, 0, &minfo
, NULL
) &&
348 g_match_info_get_match_count(minfo
) >= 2)
350 encoding
= g_match_info_fetch(minfo
, 1);
351 geany_debug("Detected encoding by regex search: %s", encoding
);
353 SETPTR(encoding
, g_utf8_strup(encoding
, -1));
355 g_match_info_free(minfo
);
360 static void encodings_radio_item_change_cb(GtkCheckMenuItem
*menuitem
, gpointer user_data
)
362 GeanyDocument
*doc
= document_get_current();
363 const gchar
*charset
= user_data
;
365 if (ignore_callback
|| doc
== NULL
|| charset
== NULL
||
366 ! gtk_check_menu_item_get_active(menuitem
) ||
367 utils_str_equal(charset
, doc
->encoding
))
375 document_undo_add(doc
, UNDO_ENCODING
, g_strdup(doc
->encoding
));
377 document_set_encoding(doc
, charset
);
380 static void encodings_reload_radio_item_change_cb(GtkMenuItem
*menuitem
, gpointer user_data
)
382 GeanyDocument
*doc
= document_get_current();
384 g_return_if_fail(doc
!= NULL
);
386 document_reload_prompt(doc
, user_data
);
390 void encodings_finalize(void)
395 len
= G_N_ELEMENTS(pregs
);
396 for (i
= 0; i
< len
; i
++)
398 g_regex_unref(pregs
[i
]);
404 void encodings_init(void)
406 GtkWidget
*item
, *menu
[2], *submenu
, *menu_westeuro
, *menu_easteuro
, *menu_eastasian
, *menu_asian
,
407 *menu_utf8
, *menu_middleeast
, *item_westeuro
, *item_easteuro
, *item_eastasian
,
408 *item_asian
, *item_utf8
, *item_middleeast
;
409 GCallback cb_func
[2];
410 GSList
*group
= NULL
;
412 gint order
, group_size
;
419 pregs
[0] = regex_compile(PATTERN_HTMLMETA
);
420 pregs
[1] = regex_compile(PATTERN_CODING
);
424 /* create encodings submenu in document menu */
425 menu
[0] = ui_lookup_widget(main_widgets
.window
, "set_encoding1_menu");
426 menu
[1] = ui_lookup_widget(main_widgets
.window
, "menu_reload_as1_menu");
427 cb_func
[0] = G_CALLBACK(encodings_radio_item_change_cb
);
428 cb_func
[1] = G_CALLBACK(encodings_reload_radio_item_change_cb
);
430 for (k
= 0; k
< 2; k
++)
432 menu_westeuro
= gtk_menu_new();
433 item_westeuro
= gtk_menu_item_new_with_mnemonic(_("_West European"));
434 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item_westeuro
), menu_westeuro
);
435 gtk_container_add(GTK_CONTAINER(menu
[k
]), item_westeuro
);
436 gtk_widget_show_all(item_westeuro
);
438 menu_easteuro
= gtk_menu_new();
439 item_easteuro
= gtk_menu_item_new_with_mnemonic(_("_East European"));
440 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item_easteuro
), menu_easteuro
);
441 gtk_container_add(GTK_CONTAINER(menu
[k
]), item_easteuro
);
442 gtk_widget_show_all(item_easteuro
);
444 menu_eastasian
= gtk_menu_new();
445 item_eastasian
= gtk_menu_item_new_with_mnemonic(_("East _Asian"));
446 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item_eastasian
), menu_eastasian
);
447 gtk_container_add(GTK_CONTAINER(menu
[k
]), item_eastasian
);
448 gtk_widget_show_all(item_eastasian
);
450 menu_asian
= gtk_menu_new();
451 item_asian
= gtk_menu_item_new_with_mnemonic(_("_SE & SW Asian"));
452 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item_asian
), menu_asian
);
453 gtk_container_add(GTK_CONTAINER(menu
[k
]), item_asian
);
454 gtk_widget_show_all(item_asian
);
456 menu_middleeast
= gtk_menu_new();
457 item_middleeast
= gtk_menu_item_new_with_mnemonic(_("_Middle Eastern"));
458 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item_middleeast
), menu_middleeast
);
459 gtk_container_add(GTK_CONTAINER(menu
[k
]), item_middleeast
);
460 gtk_widget_show_all(item_middleeast
);
462 menu_utf8
= gtk_menu_new();
463 item_utf8
= gtk_menu_item_new_with_mnemonic(_("_Unicode"));
464 gtk_menu_item_set_submenu(GTK_MENU_ITEM(item_utf8
), menu_utf8
);
465 gtk_container_add(GTK_CONTAINER(menu
[k
]), item_utf8
);
466 gtk_widget_show_all(item_utf8
);
468 /** TODO can it be optimized? ATM 3782 runs at line "if (encodings[j].group ...)" */
469 for (i
= 0; i
< GEANY_ENCODING_GROUPS_MAX
; i
++)
474 case WESTEUROPEAN
: submenu
= menu_westeuro
; group_size
= 9; break;
475 case EASTEUROPEAN
: submenu
= menu_easteuro
; group_size
= 14; break;
476 case EASTASIAN
: submenu
= menu_eastasian
; group_size
= 14; break;
477 case ASIAN
: submenu
= menu_asian
; group_size
= 9; break;
478 case MIDDLEEASTERN
: submenu
= menu_middleeast
; group_size
= 7; break;
479 case UNICODE
: submenu
= menu_utf8
; group_size
= 8; break;
480 default: submenu
= menu
[k
]; group_size
= 1;
483 while (order
< group_size
) /* the biggest group has 13 elements */
485 for (j
= 0; j
< GEANY_ENCODINGS_MAX
; j
++)
487 if (encodings
[j
].group
== i
&& encodings
[j
].order
== order
)
489 label
= encodings_to_string(&encodings
[j
]);
492 item
= gtk_radio_menu_item_new_with_label(group
, label
);
493 group
= gtk_radio_menu_item_get_group(GTK_RADIO_MENU_ITEM(item
));
494 radio_items
[j
] = item
;
497 item
= gtk_menu_item_new_with_label(label
);
498 gtk_widget_show(item
);
499 gtk_container_add(GTK_CONTAINER(submenu
), item
);
500 g_signal_connect(item
, "activate", cb_func
[k
],
501 (gpointer
) encodings
[j
].charset
);
513 static gint
encoding_combo_store_sort_func(GtkTreeModel
*model
,
518 gboolean a_has_child
= gtk_tree_model_iter_has_child(model
, a
);
519 gboolean b_has_child
= gtk_tree_model_iter_has_child(model
, b
);
524 if (a_has_child
!= b_has_child
)
525 return a_has_child
? -1 : 1;
527 gtk_tree_model_get(model
, a
, 1, &a_string
, -1);
528 gtk_tree_model_get(model
, b
, 1, &b_string
, -1);
529 cmp_res
= strcmp(a_string
, b_string
);
536 GtkTreeStore
*encodings_encoding_store_new(gboolean has_detect
)
539 GtkTreeIter iter_current
, iter_westeuro
, iter_easteuro
, iter_eastasian
,
540 iter_asian
, iter_utf8
, iter_middleeast
;
541 GtkTreeIter
*iter_parent
;
542 gchar
*encoding_string
;
545 store
= gtk_tree_store_new(2, G_TYPE_INT
, G_TYPE_STRING
);
549 gtk_tree_store_append(store
, &iter_current
, NULL
);
550 gtk_tree_store_set(store
, &iter_current
, 0, GEANY_ENCODINGS_MAX
, 1, _("Detect from file"), -1);
553 gtk_tree_store_append(store
, &iter_westeuro
, NULL
);
554 gtk_tree_store_set(store
, &iter_westeuro
, 0, -1, 1, _("West European"), -1);
555 gtk_tree_store_append(store
, &iter_easteuro
, NULL
);
556 gtk_tree_store_set(store
, &iter_easteuro
, 0, -1, 1, _("East European"), -1);
557 gtk_tree_store_append(store
, &iter_eastasian
, NULL
);
558 gtk_tree_store_set(store
, &iter_eastasian
, 0, -1, 1, _("East Asian"), -1);
559 gtk_tree_store_append(store
, &iter_asian
, NULL
);
560 gtk_tree_store_set(store
, &iter_asian
, 0, -1, 1, _("SE & SW Asian"), -1);
561 gtk_tree_store_append(store
, &iter_middleeast
, NULL
);
562 gtk_tree_store_set(store
, &iter_middleeast
, 0, -1, 1, _("Middle Eastern"), -1);
563 gtk_tree_store_append(store
, &iter_utf8
, NULL
);
564 gtk_tree_store_set(store
, &iter_utf8
, 0, -1, 1, _("Unicode"), -1);
566 for (i
= 0; i
< GEANY_ENCODINGS_MAX
; i
++)
568 switch (encodings
[i
].group
)
570 case WESTEUROPEAN
: iter_parent
= &iter_westeuro
; break;
571 case EASTEUROPEAN
: iter_parent
= &iter_easteuro
; break;
572 case EASTASIAN
: iter_parent
= &iter_eastasian
; break;
573 case ASIAN
: iter_parent
= &iter_asian
; break;
574 case MIDDLEEASTERN
: iter_parent
= &iter_middleeast
; break;
575 case UNICODE
: iter_parent
= &iter_utf8
; break;
577 default: iter_parent
= NULL
;
579 gtk_tree_store_append(store
, &iter_current
, iter_parent
);
580 encoding_string
= encodings_to_string(&encodings
[i
]);
581 gtk_tree_store_set(store
, &iter_current
, 0, i
, 1, encoding_string
, -1);
582 g_free(encoding_string
);
585 gtk_tree_sortable_set_sort_column_id(GTK_TREE_SORTABLE(store
), 1, GTK_SORT_ASCENDING
);
586 gtk_tree_sortable_set_sort_func(GTK_TREE_SORTABLE(store
), 1, encoding_combo_store_sort_func
, NULL
, NULL
);
592 gint
encodings_encoding_store_get_encoding(GtkTreeStore
*store
, GtkTreeIter
*iter
)
595 gtk_tree_model_get(GTK_TREE_MODEL(store
), iter
, 0, &enc
, -1);
600 gboolean
encodings_encoding_store_get_iter(GtkTreeStore
*store
, GtkTreeIter
*iter
, gint enc
)
602 if (gtk_tree_model_get_iter_first(GTK_TREE_MODEL(store
), iter
))
606 if (encodings_encoding_store_get_encoding(store
, iter
) == enc
)
609 while (ui_tree_model_iter_any_next(GTK_TREE_MODEL(store
), iter
, TRUE
));
615 void encodings_encoding_store_cell_data_func(GtkCellLayout
*cell_layout
,
616 GtkCellRenderer
*cell
,
617 GtkTreeModel
*tree_model
,
621 gboolean sensitive
= !gtk_tree_model_iter_has_child(tree_model
, iter
);
624 gtk_tree_model_get(tree_model
, iter
, 1, &text
, -1);
625 g_object_set(cell
, "sensitive", sensitive
, "text", text
, NULL
);
631 * Tries to convert @a buffer into UTF-8 encoding from the encoding specified with @a charset.
632 * If @a fast is not set, additional checks to validate the converted string are performed.
634 * @param buffer The input string to convert.
635 * @param size The length of the string, or -1 if the string is nul-terminated.
636 * @param charset The charset to be used for conversion.
637 * @param fast @c TRUE to only convert the input and skip extended checks on the converted string.
639 * @return If the conversion was successful, a newly allocated nul-terminated string,
640 * which must be freed with @c g_free(). Otherwise @c NULL.
642 gchar
*encodings_convert_to_utf8_from_charset(const gchar
*buffer
, gssize size
,
643 const gchar
*charset
, gboolean fast
)
645 gchar
*utf8_content
= NULL
;
646 GError
*conv_error
= NULL
;
647 gchar
* converted_contents
= NULL
;
650 g_return_val_if_fail(buffer
!= NULL
, NULL
);
651 g_return_val_if_fail(charset
!= NULL
, NULL
);
653 converted_contents
= g_convert(buffer
, size
, "UTF-8", charset
, NULL
,
654 &bytes_written
, &conv_error
);
658 utf8_content
= converted_contents
;
659 if (conv_error
!= NULL
) g_error_free(conv_error
);
661 else if (conv_error
!= NULL
|| ! g_utf8_validate(converted_contents
, bytes_written
, NULL
))
663 if (conv_error
!= NULL
)
665 geany_debug("Couldn't convert from %s to UTF-8 (%s).", charset
, conv_error
->message
);
666 g_error_free(conv_error
);
670 geany_debug("Couldn't convert from %s to UTF-8.", charset
);
673 g_free(converted_contents
);
677 geany_debug("Converted from %s to UTF-8.", charset
);
678 utf8_content
= converted_contents
;
685 static gchar
*encodings_check_regexes(const gchar
*buffer
, gsize size
)
689 for (i
= 0; i
< G_N_ELEMENTS(pregs
); i
++)
693 if ((charset
= regex_match(pregs
[i
], buffer
, size
)) != NULL
)
700 static gchar
*encodings_convert_to_utf8_with_suggestion(const gchar
*buffer
, gssize size
,
701 const gchar
*suggested_charset
, gchar
**used_encoding
)
703 const gchar
*locale_charset
= NULL
;
704 const gchar
*charset
;
706 gboolean check_suggestion
= suggested_charset
!= NULL
;
707 gboolean check_locale
= FALSE
;
708 gint i
, preferred_charset
;
712 size
= strlen(buffer
);
715 /* current locale is not UTF-8, we have to check this charset */
716 check_locale
= ! g_get_charset(&locale_charset
);
718 /* First check for preferred charset, if specified */
719 preferred_charset
= file_prefs
.default_open_encoding
;
721 if (preferred_charset
== encodings
[GEANY_ENCODING_NONE
].idx
||
722 preferred_charset
< 0 ||
723 preferred_charset
>= GEANY_ENCODINGS_MAX
)
725 preferred_charset
= -1;
728 /* -1 means "Preferred charset" */
729 for (i
= -1; i
< GEANY_ENCODINGS_MAX
; i
++)
731 if (G_UNLIKELY(i
== encodings
[GEANY_ENCODING_NONE
].idx
))
734 if (check_suggestion
)
736 check_suggestion
= FALSE
;
737 charset
= encodings_normalize_charset(suggested_charset
);
738 if (charset
== NULL
) /* we failed at normalizing suggested encoding, try it as is */
739 charset
= suggested_charset
;
740 i
= -2; /* keep i below the start value to have it again at -1 on the next loop run */
742 else if (check_locale
)
744 check_locale
= FALSE
;
745 charset
= locale_charset
;
746 i
= -2; /* keep i below the start value to have it again at -1 on the next loop run */
750 if (preferred_charset
>= 0)
752 charset
= encodings
[preferred_charset
].charset
;
753 geany_debug("Using preferred charset: %s", charset
);
759 charset
= encodings
[i
].charset
;
760 else /* in this case we have i == -2, continue to increase i and go ahead */
763 if (G_UNLIKELY(charset
== NULL
))
766 geany_debug("Trying to convert %" G_GSIZE_FORMAT
" bytes of data from %s into UTF-8.",
768 utf8_content
= encodings_convert_to_utf8_from_charset(buffer
, size
, charset
, FALSE
);
770 if (G_LIKELY(utf8_content
!= NULL
))
772 if (used_encoding
!= NULL
)
774 if (G_UNLIKELY(*used_encoding
!= NULL
))
776 geany_debug("%s:%d", __FILE__
, __LINE__
);
777 g_free(*used_encoding
);
779 *used_encoding
= g_strdup(charset
);
790 * Tries to convert @a buffer into UTF-8 encoding and store the detected original encoding in
793 * @param buffer the input string to convert.
794 * @param size the length of the string, or -1 if the string is nul-terminated.
795 * @param used_encoding return location of the detected encoding of the input string, or @c NULL.
797 * @return If the conversion was successful, a newly allocated nul-terminated string,
798 * which must be freed with @c g_free(). Otherwise @c NULL.
800 gchar
*encodings_convert_to_utf8(const gchar
*buffer
, gssize size
, gchar
**used_encoding
)
802 gchar
*regex_charset
;
805 /* first try to read the encoding from the file content */
806 regex_charset
= encodings_check_regexes(buffer
, size
);
807 utf8
= encodings_convert_to_utf8_with_suggestion(buffer
, size
, regex_charset
, used_encoding
);
808 g_free(regex_charset
);
814 /* If there's a BOM, return a corresponding GEANY_ENCODING_UTF_* index,
815 * otherwise GEANY_ENCODING_NONE.
817 GeanyEncodingIndex
encodings_scan_unicode_bom(const gchar
*string
, gsize len
, guint
*bom_len
)
824 if ((guchar
)string
[0] == 0xef && (guchar
)string
[1] == 0xbb &&
825 (guchar
)string
[2] == 0xbf)
827 return GEANY_ENCODING_UTF_8
;
835 if ((guchar
)string
[0] == 0x00 && (guchar
)string
[1] == 0x00 &&
836 (guchar
)string
[2] == 0xfe && (guchar
)string
[3] == 0xff)
838 return GEANY_ENCODING_UTF_32BE
; /* Big endian */
840 if ((guchar
)string
[0] == 0xff && (guchar
)string
[1] == 0xfe &&
841 (guchar
)string
[2] == 0x00 && (guchar
)string
[3] == 0x00)
843 return GEANY_ENCODING_UTF_32LE
; /* Little endian */
845 if ((string
[0] == 0x2b && string
[1] == 0x2f && string
[2] == 0x76) &&
846 (string
[3] == 0x38 || string
[3] == 0x39 || string
[3] == 0x2b || string
[3] == 0x2f))
848 return GEANY_ENCODING_UTF_7
;
856 if ((guchar
)string
[0] == 0xfe && (guchar
)string
[1] == 0xff)
858 return GEANY_ENCODING_UTF_16BE
; /* Big endian */
860 if ((guchar
)string
[0] == 0xff && (guchar
)string
[1] == 0xfe)
862 return GEANY_ENCODING_UTF_16LE
; /* Little endian */
867 return GEANY_ENCODING_NONE
;
871 gboolean
encodings_is_unicode_charset(const gchar
*string
)
873 if (string
!= NULL
&&
874 (strncmp(string
, "UTF", 3) == 0 || strncmp(string
, "UCS", 3) == 0))
884 gchar
*data
; /* null-terminated data */
885 gsize size
; /* actual data size */
886 gsize len
; /* string length of data */
893 /* convert data with the specified encoding */
895 handle_forced_encoding(BufferData
*buffer
, const gchar
*forced_enc
)
897 GeanyEncodingIndex enc_idx
;
899 if (utils_str_equal(forced_enc
, "UTF-8"))
901 if (! g_utf8_validate(buffer
->data
, buffer
->len
, NULL
))
908 gchar
*converted_text
= encodings_convert_to_utf8_from_charset(
909 buffer
->data
, buffer
->size
, forced_enc
, FALSE
);
910 if (converted_text
== NULL
)
916 SETPTR(buffer
->data
, converted_text
);
917 buffer
->len
= strlen(converted_text
);
920 enc_idx
= encodings_scan_unicode_bom(buffer
->data
, buffer
->size
, NULL
);
921 buffer
->bom
= (enc_idx
== GEANY_ENCODING_UTF_8
);
922 buffer
->enc
= g_strdup(forced_enc
);
927 /* detect encoding and convert to UTF-8 if necessary */
929 handle_encoding(BufferData
*buffer
, GeanyEncodingIndex enc_idx
)
931 g_return_val_if_fail(buffer
->enc
== NULL
, FALSE
);
932 g_return_val_if_fail(buffer
->bom
== FALSE
, FALSE
);
934 if (buffer
->size
== 0)
936 /* we have no data so assume UTF-8, buffer->len can be 0 even we have an empty
937 * e.g. UTF32 file with a BOM(so size is 4, len is 0) */
938 buffer
->enc
= g_strdup("UTF-8");
942 /* first check for a BOM */
943 if (enc_idx
!= GEANY_ENCODING_NONE
)
945 buffer
->enc
= g_strdup(encodings
[enc_idx
].charset
);
948 if (enc_idx
!= GEANY_ENCODING_UTF_8
) /* the BOM indicated something else than UTF-8 */
950 gchar
*converted_text
= encodings_convert_to_utf8_from_charset(
951 buffer
->data
, buffer
->size
, buffer
->enc
, FALSE
);
952 if (converted_text
!= NULL
)
954 SETPTR(buffer
->data
, converted_text
);
955 buffer
->len
= strlen(converted_text
);
959 /* there was a problem converting data from BOM encoding type */
960 SETPTR(buffer
->enc
, NULL
);
966 if (buffer
->enc
== NULL
) /* either there was no BOM or the BOM encoding failed */
968 /* first try to read the encoding from the file content */
969 gchar
*regex_charset
= encodings_check_regexes(buffer
->data
, buffer
->size
);
971 /* try UTF-8 first */
972 if (encodings_get_idx_from_charset(regex_charset
) == GEANY_ENCODING_UTF_8
&&
973 (buffer
->size
== buffer
->len
) && g_utf8_validate(buffer
->data
, buffer
->len
, NULL
))
975 buffer
->enc
= g_strdup("UTF-8");
979 /* detect the encoding */
980 gchar
*converted_text
= encodings_convert_to_utf8_with_suggestion(buffer
->data
,
981 buffer
->size
, regex_charset
, &buffer
->enc
);
983 if (converted_text
== NULL
)
985 g_free(regex_charset
);
988 SETPTR(buffer
->data
, converted_text
);
989 buffer
->len
= strlen(converted_text
);
991 g_free(regex_charset
);
999 handle_bom(BufferData
*buffer
)
1003 encodings_scan_unicode_bom(buffer
->data
, buffer
->size
, &bom_len
);
1004 g_return_if_fail(bom_len
!= 0);
1006 /* use filedata->len here because the contents are already converted into UTF-8 */
1007 buffer
->len
-= bom_len
;
1008 /* overwrite the BOM with the remainder of the file contents, plus the NULL terminator. */
1009 g_memmove(buffer
->data
, buffer
->data
+ bom_len
, buffer
->len
+ 1);
1010 buffer
->data
= g_realloc(buffer
->data
, buffer
->len
+ 1);
1014 /* loads textfile data, verifies and converts to forced_enc or UTF-8. Also handles BOM. */
1015 static gboolean
handle_buffer(BufferData
*buffer
, const gchar
*forced_enc
)
1017 GeanyEncodingIndex tmp_enc_idx
;
1019 /* temporarily retrieve the encoding idx based on the BOM to suppress the following warning
1020 * if we have a BOM */
1021 tmp_enc_idx
= encodings_scan_unicode_bom(buffer
->data
, buffer
->size
, NULL
);
1023 /* check whether the size of the loaded data is equal to the size of the file in the
1024 * filesystem file size may be 0 to allow opening files in /proc/ which have typically a
1025 * file size of 0 bytes */
1026 if (buffer
->len
!= buffer
->size
&& buffer
->size
!= 0 && (
1027 tmp_enc_idx
== GEANY_ENCODING_UTF_8
|| /* tmp_enc_idx can be UTF-7/8/16/32, UCS and None */
1028 tmp_enc_idx
== GEANY_ENCODING_UTF_7
)) /* filter UTF-7/8 where no NULL bytes are allowed */
1030 buffer
->partial
= TRUE
;
1033 /* Determine character encoding and convert to UTF-8 */
1034 if (forced_enc
!= NULL
)
1036 /* the encoding should be ignored(requested by user), so open the file "as it is" */
1037 if (utils_str_equal(forced_enc
, encodings
[GEANY_ENCODING_NONE
].charset
))
1039 buffer
->bom
= FALSE
;
1040 buffer
->enc
= g_strdup(encodings
[GEANY_ENCODING_NONE
].charset
);
1042 else if (! handle_forced_encoding(buffer
, forced_enc
))
1047 else if (! handle_encoding(buffer
, tmp_enc_idx
))
1059 * Tries to convert @a buffer into UTF-8 encoding. Unlike encodings_convert_to_utf8()
1060 * and encodings_convert_to_utf8_from_charset() it handles the possible BOM in the data.
1062 * @param buf a pointer to modifiable null-terminated buffer to convert.
1063 * It may or may not be modified, and should be freed whatever happens.
1064 * @param size a pointer to the size of the buffer (expected to be e.g. the on-disk
1065 * file size). It will be updated to the new size.
1066 * @param forced_enc forced encoding to use, or @c NULL
1067 * @param used_encoding return location for the actually used encoding, or @c NULL
1068 * @param has_bom return location to store whether the data had a BOM, or @c NULL
1069 * @param partial return location to store whether the conversion may be partial, or @c NULL
1071 * @return @C TRUE if the conversion succeeded, @c FALSE otherwise.
1073 gboolean
encodings_convert_to_utf8_auto(gchar
**buf
, gsize
*size
, const gchar
*forced_enc
,
1074 gchar
**used_encoding
, gboolean
*has_bom
, gboolean
*partial
)
1079 buffer
.size
= *size
;
1080 /* use strlen to check for null chars */
1081 buffer
.len
= strlen(buffer
.data
);
1084 buffer
.partial
= FALSE
;
1086 if (! handle_buffer(&buffer
, forced_enc
))
1091 *used_encoding
= buffer
.enc
;
1095 *has_bom
= buffer
.bom
;
1097 *partial
= buffer
.partial
;