2 * stf.c : Utilizes the stf-parse engine and the dialog-stf to provide a plug-in for
3 * importing text files with a structure (CSV/fixed width)
5 * Copyright (C) Almer. S. Tigelaar <almer@gnome.org>
6 * Copyright (C) 1999-2009 Morten Welinder (terra@gnome.org)
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, see <https://www.gnu.org/licenses/>.
22 #include <gnumeric-config.h>
23 #include <glib/gi18n-lib.h>
26 #include <stf-export.h>
28 #include <goffice/goffice.h>
31 #include <sheet-view.h>
32 #include <sheet-style.h>
35 #include <command-context.h>
37 #include <workbook-view.h>
39 #include <dialogs/dialog-stf.h>
40 #include <dialogs/dialog-stf-export.h>
44 #include <gnm-format.h>
45 #include <selection.h>
47 #include <clipboard.h>
48 #include <parse-util.h>
53 #include <gsf/gsf-input.h>
55 #include <gsf/gsf-output.h>
56 #include <gsf/gsf-output-memory.h>
57 #include <gsf/gsf-utils.h>
61 stf_warning (GOIOContext
*context
, char const *msg
)
64 * Using go_cmd_context_error_import will destroy the
65 * successfully imported portion. We ought to have a
66 * way to issue a warning.
68 if (GNM_IS_WBC_GTK (context
->impl
))
70 (wbcg_toplevel (WBC_GTK (context
->impl
)),
74 g_warning ("%s", msg
);
80 * @filename: name of the file to open&read
82 * Will open filename, read the file into a g_alloced memory buffer
84 * NOTE : The returned buffer has to be g_freed by the calling routine.
86 * returns : a buffer containing the file contents
89 stf_open_and_read (G_GNUC_UNUSED GOIOContext
*context
, GsfInput
*input
, size_t *readsize
)
93 gsf_off_t size
= gsf_input_size (input
);
95 if (gsf_input_seek (input
, 0, G_SEEK_SET
))
98 *readsize
= (size_t) size
;
99 if ((gsf_off_t
) *readsize
!= size
) /* Check for overflow */
102 allocsize
= (gulong
) size
;
103 if ((gsf_off_t
) allocsize
!= size
) /* Check for overflow */
105 result
= g_try_malloc (allocsize
);
109 *((char *)result
+ *readsize
) = '\0';
111 if (*readsize
> 0 && gsf_input_read (input
, *readsize
, result
) == NULL
) {
112 g_warning ("gsf_input_read failed.");
120 stf_preparse (GOIOContext
*context
, GsfInput
*input
, size_t *data_len
)
124 data
= stf_open_and_read (context
, input
, data_len
);
128 go_cmd_context_error_import (GO_CMD_CONTEXT (context
),
129 _("Error while trying to read file"));
137 stf_store_results (DialogStfResult_t
*dialogresult
,
138 Sheet
*sheet
, int start_col
, int start_row
)
140 return stf_parse_sheet (dialogresult
->parseoptions
,
141 dialogresult
->text
, NULL
, sheet
,
142 start_col
, start_row
);
146 resize_columns (Sheet
*sheet
)
150 if (gnm_debug_flag ("stf"))
151 g_printerr ("Auto-fitting columns...\n");
153 /* If we have lots of rows, auto-fitting will take a very long
154 time. It is probably better to look at only, say, 1000 rows
156 range_init_full_sheet (&r
, sheet
);
157 r
.end
.row
= MIN (r
.end
.row
, 1000);
159 colrow_autofit (sheet
, &r
, TRUE
,
160 TRUE
, /* Ignore strings */
161 TRUE
, /* Don't shrink */
162 TRUE
, /* Don't shrink */
164 if (gnm_debug_flag ("stf"))
165 g_printerr ("Auto-fitting columns... done\n");
167 sheet_queue_respan (sheet
, 0, gnm_sheet_get_last_row (sheet
));
174 * @enc: encoding of file
175 * @context: command context
177 * @input: file to read from+convert
179 * Main routine, handles importing a file including all dialog mumbo-jumbo
182 stf_read_workbook (G_GNUC_UNUSED GOFileOpener
const *fo
, gchar
const *enc
,
183 GOIOContext
*context
, GoView
*view
, GsfInput
*input
)
185 DialogStfResult_t
*dialogresult
= NULL
;
186 char *name
, *nameutf8
= NULL
;
189 WorkbookView
*wbv
= GNM_WORKBOOK_VIEW (view
);
191 if (!GNM_IS_WBC_GTK (context
->impl
)) {
192 go_io_error_string (context
, _("This importer can only be used with a GUI."));
196 name
= g_path_get_basename (gsf_input_name (input
));
197 nameutf8
= g_filename_to_utf8 (name
, -1, NULL
, NULL
, NULL
);
200 g_warning ("Failed to convert filename to UTF-8. This shouldn't happen here.");
204 data
= stf_preparse (context
, input
, &data_len
);
208 dialogresult
= stf_dialog (WBC_GTK (context
->impl
),
209 enc
, FALSE
, NULL
, FALSE
,
210 nameutf8
, data
, data_len
);
211 if (dialogresult
!= NULL
) {
212 Workbook
*book
= wb_view_get_workbook (wbv
);
213 int cols
= dialogresult
->colcount
, rows
= dialogresult
->rowcount
;
216 gnm_sheet_suggest_size (&cols
, &rows
);
217 sheet
= sheet_new (book
, nameutf8
, cols
, rows
);
218 workbook_sheet_attach (book
, sheet
);
219 if (stf_store_results (dialogresult
, sheet
, 0, 0)) {
220 workbook_recalc_all (book
);
221 resize_columns (sheet
);
222 workbook_set_saveinfo
224 GO_FILE_FL_WRITE_ONLY
,
226 ("Gnumeric_stf:stf_assistant"));
228 /* the user has cancelled */
229 /* the caller should notice that we have no sheets */
230 workbook_sheet_delete (sheet
);
237 if (dialogresult
!= NULL
)
238 stf_dialog_result_free (dialogresult
);
242 cb_get_content (GnmCellIter
const *iter
, GsfOutput
*buf
)
246 if (NULL
!= (cell
= iter
->cell
)) {
248 if (gnm_cell_has_expr (cell
))
249 tmp
= gnm_expr_top_as_string (cell
->base
.texpr
,
250 &iter
->pp
, iter
->pp
.sheet
->convs
);
251 else if (VALUE_FMT (cell
->value
) != NULL
)
252 tmp
= format_value (NULL
, cell
->value
, -1,
253 workbook_date_conv (iter
->pp
.wb
));
255 tmp
= value_get_as_string (cell
->value
);
257 gsf_output_write (buf
, strlen (tmp
), tmp
);
260 gsf_output_write (buf
, 1, "\n");
266 * stf_text_to_columns:
267 * @wbc: The control making the request
270 * Main routine, handles importing a file including all dialog mumbo-jumbo
273 stf_text_to_columns (WorkbookControl
*wbc
, GOCmdContext
*cc
)
275 DialogStfResult_t
*dialogresult
= NULL
;
277 Sheet
*src_sheet
, *target_sheet
;
284 sv
= wb_control_cur_sheet_view (wbc
);
285 src_sheet
= sv_sheet (sv
);
286 src
= selection_first_range (sv
, cc
, _("Text to Columns"));
289 if (range_width (src
) > 1) {
290 go_cmd_context_error (cc
, g_error_new (go_error_invalid (), 0,
291 _("Only one column of input data can be parsed at a time")));
295 /* FIXME : how to do this cleanly ? */
296 if (!GNM_IS_WBC_GTK (wbc
))
299 #warning Add UI for this
300 target_sheet
= src_sheet
;
302 range_translate (&target
, target_sheet
, 1, 0);
304 buf
= gsf_output_memory_new ();
305 sheet_foreach_cell_in_range (src_sheet
, CELL_ITER_ALL
, src
,
306 (CellIterFunc
) &cb_get_content
, buf
);
308 gsf_output_close (buf
);
309 data
= gsf_output_memory_get_bytes (GSF_OUTPUT_MEMORY (buf
));
310 data_len
= (size_t)gsf_output_size (buf
);
312 go_cmd_context_error_import (GO_CMD_CONTEXT (cc
),
313 _("There is no data "
316 dialogresult
= stf_dialog (WBC_GTK (wbc
),
317 NULL
, FALSE
, NULL
, FALSE
,
318 _("Text to Columns"),
321 if (dialogresult
!= NULL
) {
322 GnmCellRegion
*cr
= stf_parse_region (dialogresult
->parseoptions
,
323 dialogresult
->text
, NULL
, target_sheet
->workbook
);
325 stf_dialog_result_attach_formats_to_cr (dialogresult
, cr
);
326 target
.end
.col
= target
.start
.col
+ cr
->cols
- 1;
327 target
.end
.row
= target
.start
.row
+ cr
->rows
- 1;
330 cmd_text_to_columns (wbc
, src
, src_sheet
,
331 &target
, target_sheet
, cr
))
332 go_cmd_context_error_import (GO_CMD_CONTEXT (cc
),
333 _("Error while trying to "
334 "parse data into sheet"));
335 stf_dialog_result_free (dialogresult
);
338 g_object_unref (buf
);
342 clear_stray_NULs (GOIOContext
*context
, GString
*utf8data
)
344 char *cpointer
, *endpointer
;
346 char const *valid_end
;
348 cpointer
= utf8data
->str
;
349 endpointer
= utf8data
->str
+ utf8data
->len
;
350 while (*cpointer
!= 0)
352 while (cpointer
!= endpointer
) {
355 while (*cpointer
!= 0)
358 if (null_chars
> 0) {
361 format
= ngettext ("The file contains %d NUL character. "
362 "It has been changed to a space.",
363 "The file contains %d NUL characters. "
364 "They have been changed to spaces.",
366 msg
= g_strdup_printf (format
, null_chars
);
367 stf_warning (context
, msg
);
371 if (!g_utf8_validate (utf8data
->str
, utf8data
->len
, &valid_end
)) {
372 g_string_truncate (utf8data
, valid_end
- utf8data
->str
);
373 stf_warning (context
, _("The file contains invalid UTF-8 encoded characters and has been truncated"));
378 * stf_read_workbook_auto_csvtab:
380 * @enc: optional encoding
381 * @context: command context
383 * @input: file to read from+convert
385 * Attempt to auto-detect CSV or tab-delimited file
388 stf_read_workbook_auto_csvtab (G_GNUC_UNUSED GOFileOpener
const *fo
, gchar
const *enc
,
389 GOIOContext
*context
,
390 GoView
*view
, GsfInput
*input
)
398 StfParseOptions_t
*po
;
401 GStringChunk
*lines_chunk
;
403 WorkbookView
*wbv
= GNM_WORKBOOK_VIEW (view
);
405 g_return_if_fail (context
!= NULL
);
406 g_return_if_fail (wbv
!= NULL
);
408 book
= wb_view_get_workbook (wbv
);
410 data
= stf_preparse (context
, input
, &data_len
);
414 enc
= go_guess_encoding (data
, data_len
, enc
, &utf8data
, NULL
);
418 go_cmd_context_error_import (GO_CMD_CONTEXT (context
),
419 _("That file is not in the given encoding."));
423 clear_stray_NULs (context
, utf8data
);
426 * Try to get the filename we're reading from. This is not a
429 gsfname
= gsf_input_name (input
);
432 const char *ext
= gsf_extension_pointer (gsfname
);
433 gboolean iscsv
= ext
&& strcasecmp (ext
, "csv") == 0;
435 po
= stf_parse_options_guess_csv (utf8data
->str
);
437 po
= stf_parse_options_guess (utf8data
->str
);
440 lines_chunk
= g_string_chunk_new (100 * 1024);
441 lines
= stf_parse_general (po
, lines_chunk
,
442 utf8data
->str
, utf8data
->str
+ utf8data
->len
);
445 for (i
= 0; i
< rows
; i
++) {
446 GPtrArray
*line
= g_ptr_array_index (lines
, i
);
447 cols
= MAX (cols
, (int)line
->len
);
449 gnm_sheet_suggest_size (&cols
, &rows
);
450 stf_parse_general_free (lines
);
451 g_string_chunk_free (lines_chunk
);
453 name
= g_path_get_basename (gsfname
);
454 sheet
= sheet_new (book
, name
, cols
, rows
);
456 workbook_sheet_attach (book
, sheet
);
458 if (stf_parse_sheet (po
, utf8data
->str
, NULL
, sheet
, 0, 0)) {
460 workbook_recalc_all (book
);
461 resize_columns (sheet
);
462 if (po
->cols_exceeded
|| po
->rows_exceeded
) {
463 stf_warning (context
,
464 _("Some data did not fit on the "
465 "sheet and was dropped."));
467 is_csv
= po
->sep
.chr
&& po
->sep
.chr
[0] == ',';
468 workbook_set_saveinfo
470 GO_FILE_FL_WRITE_ONLY
,
472 (is_csv
? "Gnumeric_stf:stf_csv" : "Gnumeric_stf:stf_assistant"));
474 workbook_sheet_delete (sheet
);
475 go_cmd_context_error_import (GO_CMD_CONTEXT (context
),
476 _("Parse error while trying to parse data into sheet"));
480 stf_parse_options_free (po
);
481 g_string_free (utf8data
, TRUE
);
484 /***********************************************************************************/
487 stf_write_csv (GOFileSaver
const *fs
, GOIOContext
*context
,
488 GoView
const *view
, GsfOutput
*output
)
491 WorkbookView
*wbv
= GNM_WORKBOOK_VIEW (view
);
493 GnmStfExport
*config
= g_object_new
494 (GNM_STF_EXPORT_TYPE
,
496 "quoting-triggers", ", \t\n\"",
499 sheet
= gnm_file_saver_get_sheet (fs
, wbv
);
501 gnm_stf_export_options_sheet_list_add (config
, sheet
);
503 if (gnm_stf_export (config
) == FALSE
)
504 go_cmd_context_error_import (GO_CMD_CONTEXT (context
),
505 _("Error while trying to write CSV file"));
507 g_object_unref (config
);
511 csv_tsv_probe (GOFileOpener
const *fo
, GsfInput
*input
, GOFileProbeLevel pl
)
513 /* Rough and ready heuristic. If the first N bytes have no
514 * unprintable characters this may be text */
515 const gsf_off_t N
= 512;
517 if (pl
== GO_FILE_PROBE_CONTENT
) {
518 guint8
const *header
;
520 char const *enc
= NULL
;
521 GString
*header_utf8
;
525 if (gsf_input_seek (input
, 0, G_SEEK_SET
))
527 i
= gsf_input_remaining (input
);
529 /* If someone ships us an empty file, accept it only if
530 it has a proper name. */
532 return csv_tsv_probe (fo
, input
, GO_FILE_PROBE_FILE_NAME
);
535 if (NULL
== (header
= gsf_input_read (input
, i
, NULL
)))
538 enc
= go_guess_encoding (header
, i
, NULL
, &header_utf8
, NULL
);
542 for (p
= header_utf8
->str
; *p
; p
= g_utf8_next_char (p
)) {
543 gunichar uc
= g_utf8_get_char (p
);
544 /* isprint might not be true for these: */
545 if (uc
== '\n' || uc
== '\t' || uc
== '\r')
547 /* Also, ignore a byte-order mark which may be used to
548 * indicate UTF-8; see
549 * http://en.wikipedia.org/wiki/Byte_Order_Mark for
552 if (p
== header_utf8
->str
&& uc
== 0x0000FEFF) {
555 if (!g_unichar_isprint (uc
)) {
561 g_string_free (header_utf8
, TRUE
);
564 char const *name
= gsf_input_name (input
);
567 name
= gsf_extension_pointer (name
);
568 return (name
!= NULL
&&
569 (g_ascii_strcasecmp (name
, "csv") == 0 ||
570 g_ascii_strcasecmp (name
, "tsv") == 0 ||
571 g_ascii_strcasecmp (name
, "txt") == 0));
581 GSList
*suffixes
= go_slist_create (
586 GSList
*mimes
= go_slist_create (
587 g_strdup ("application/tab-separated-values"),
588 g_strdup ("text/comma-separated-values"),
589 g_strdup ("text/csv"),
590 g_strdup ("text/x-csv"),
591 g_strdup ("text/spreadsheet"),
592 g_strdup ("text/tab-separated-values"),
594 GSList
*mimes_txt
= go_slist_create (
595 g_strdup ("text/plain"),
596 g_strdup ("text/csv"),
597 g_strdup ("text/x-csv"),
598 g_strdup ("text/comma-separated-values"),
599 g_strdup ("text/tab-separated-values"),
602 GOFileOpener
*opener
;
604 opener
= go_file_opener_new_with_enc (
605 "Gnumeric_stf:stf_csvtab",
606 _("Comma or tab separated values (CSV/TSV)"),
608 csv_tsv_probe
, stf_read_workbook_auto_csvtab
);
609 go_file_opener_register (opener
, 0);
610 g_object_unref (opener
);
612 opener
= go_file_opener_new_with_enc (
613 "Gnumeric_stf:stf_assistant",
614 _("Text import (configurable)"),
616 NULL
, stf_read_workbook
);
617 g_object_set (G_OBJECT (opener
), "interactive-only", TRUE
, NULL
);
618 go_file_opener_register (opener
, 0);
619 g_object_unref (opener
);
621 saver
= gnm_stf_file_saver_create ("Gnumeric_stf:stf_assistant");
622 /* Unlike the opener, the saver doesn't require interaction. */
623 go_file_saver_register (saver
);
624 g_object_unref (saver
);
626 saver
= go_file_saver_new (
627 "Gnumeric_stf:stf_csv", "csv",
628 _("Comma separated values (CSV)"),
629 GO_FILE_FL_MANUAL_REMEMBER
, stf_write_csv
);
630 go_file_saver_set_save_scope (saver
, GO_FILE_SAVE_SHEET
);
631 g_object_set (G_OBJECT (saver
), "sheet-selection", TRUE
, NULL
);
632 go_file_saver_register (saver
);
633 g_object_unref (saver
);
637 * stf_shutdown: (skip)
642 go_file_saver_unregister
643 (go_file_saver_for_id ("Gnumeric_stf:stf_assistant"));
644 go_file_saver_unregister
645 (go_file_saver_for_id ("Gnumeric_stf:stf_csv"));
647 go_file_opener_unregister
648 (go_file_opener_for_id ("Gnumeric_stf:stf_csvtab"));
649 go_file_opener_unregister
650 (go_file_opener_for_id ("Gnumeric_stf:stf_assistant"));