utilities/pspp-output.c

   1  /* PSPP - a program for statistical analysis.
   2    Copyright (C) 2017, 2018 Free Software Foundation, Inc.
   3
   4    This program is free software: you can redistribute it and/or modify
   5    it under the terms of the GNU General Public License as published by
   6    the Free Software Foundation, either version 3 of the License, or
   7    (at your option) any later version.
   8
   9    This program is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12    GNU General Public License for more details.
  13
  14    You should have received a copy of the GNU General Public License
  15    along with this program.  If not, see <http://www.gnu.org/licenses/>. */
  16
  17 #include <config.h>
  18
  19 #include <cairo.h>
  20 #include <getopt.h>
  21 #include <limits.h>
  22 #include <stdlib.h>
  23 #include <unistd.h>
  24 #include <unistr.h>
  25
  26 #include "data/file-handle-def.h"
  27 #include "data/settings.h"
  28 #include "libpspp/encoding-guesser.h"
  29 #include "libpspp/i18n.h"
  30 #include "libpspp/message.h"
  31 #include "libpspp/string-map.h"
  32 #include "libpspp/string-set.h"
  33 #include "libpspp/zip-reader.h"
  34 #include "output/driver.h"
  35 #include "output/output-item.h"
  36 #include "output/pivot-table.h"
  37 #include "output/page-setup.h"
  38 #include "output/select.h"
  39 #include "output/spv/light-binary-parser.h"
  40 #include "output/spv/spv-legacy-data.h"
  41 #include "output/spv/spv-light-decoder.h"
  42 #include "output/spv/spv-table-look.h"
  43 #include "output/spv/spv.h"
  44
  45 #include "gl/c-ctype.h"
  46 #include "gl/error.h"
  47 #include "gl/progname.h"
  48 #include "gl/version-etc.h"
  49 #include "gl/xalloc.h"
  50
  51 #include <libxml/tree.h>
  52 #include <libxml/xpath.h>
  53 #include <libxml/xpathInternals.h>
  54
  55 #include "gettext.h"
  56 #define _(msgid) gettext (msgid)
  57
  58 /* -O key=value: Output driver options. */
  59 static struct string_map output_options
  60     = STRING_MAP_INITIALIZER (output_options);
  61
  62 /* --member-names: Include .zip member name in "dir" output. */
  63 static bool show_member_names;
  64
  65 /* --show-hidden, --select, --commands, ...: Selection criteria. */
  66 static struct output_criteria *criteria;
  67 static size_t n_criteria, allocated_criteria;
  68
  69 /* --or: Add new element to 'criteria' array. */
  70 static bool new_criteria;
  71
  72 /* --sort: Sort members under dump-light-table, to make comparisons easier. */
  73 static bool sort;
  74
  75 /* --raw: Dump raw binary data in "dump-light-table"; dump all strings in
  76      "strings". */
  77 static bool raw;
  78
  79 /* --no-ascii-only: Drop all-ASCII strings in "strings". */
  80 static bool exclude_ascii_only;
  81
  82 /* --utf8-only: Only print strings that have UTF-8 multibyte sequences in
  83  * "strings". */
  84 static bool include_utf8_only;
  85
  86 /* -f, --force: Keep output file even on error. */
  87 static bool force;
  88
  89 /* --table-look: TableLook to replace table style for conversion. */
  90 static struct pivot_table_look *table_look;
  91
  92 /* Number of warnings issued. */
  93 static size_t n_warnings;
  94
  95 static void usage (void);
  96 static void developer_usage (void);
  97 static void parse_options (int argc, char **argv);
  98
  99 static struct output_item *
 100 annotate_member_names (const struct output_item *in)
 101 {
 102   if (in->type == OUTPUT_ITEM_GROUP)
 103     {
 104       struct output_item *out = group_item_clone_empty (in);
 105       for (size_t i = 0; i < in->group.n_children; i++)
 106         {
 107           const struct output_item *item = in->group.children[i];
 108           const char *members[4];
 109           size_t n = spv_info_get_members (item->spv_info, members,
 110                                            sizeof members / sizeof *members);
 111           if (n)
 112             {
 113               struct string s = DS_EMPTY_INITIALIZER;
 114               ds_put_cstr (&s, members[0]);
 115               for (size_t i = 1; i < n; i++)
 116                 ds_put_format (&s, " and %s", members[i]);
 117               group_item_add_child (out, text_item_create_nocopy (
 118                                       TEXT_ITEM_TITLE, ds_steal_cstr (&s),
 119                                       xstrdup ("Member Names")));
 120             }
 121
 122           group_item_add_child (out, output_item_ref (item));
 123         }
 124       return out;
 125     }
 126   else
 127     return output_item_ref (in);
 128 }
 129
 130 static void
 131 print_item_directory (const struct output_item *item, int level)
 132 {
 133   for (int i = 0; i < level; i++)
 134     printf ("    ");
 135
 136   printf ("- %s", output_item_type_to_string (item->type));
 137
 138   const char *label = output_item_get_label (item);
 139   if (label)
 140     printf (" \"%s\"", label);
 141
 142   if (item->type == OUTPUT_ITEM_TABLE)
 143     {
 144       char *title = pivot_value_to_string (item->table->title, item->table);
 145       if (!label || strcmp (title, label))
 146         printf (" title \"%s\"", title);
 147       free (title);
 148     }
 149
 150   if (item->command_name)
 151     printf (" command \"%s\"", item->command_name);
 152
 153   char *subtype = output_item_get_subtype (item);
 154   if (subtype)
 155     {
 156       if (!label || strcmp (label, subtype))
 157         printf (" subtype \"%s\"", subtype);
 158       free (subtype);
 159     }
 160
 161   if (!item->show)
 162     printf (" (%s)", item->type == OUTPUT_ITEM_GROUP ? "collapsed" : "hidden");
 163
 164   if (show_member_names)
 165     {
 166       const char *members[4];
 167       size_t n = spv_info_get_members (item->spv_info, members,
 168                                        sizeof members / sizeof *members);
 169
 170       for (size_t i = 0; i < n; i++)
 171           printf (" %s %s", i == 0 ? "in" : "and", members[i]);
 172     }
 173   putchar ('\n');
 174
 175   if (item->type == OUTPUT_ITEM_GROUP)
 176     for (size_t i = 0; i < item->group.n_children; i++)
 177       print_item_directory (item->group.children[i], level + 1);
 178 }
 179
 180 static void
 181 run_detect (int argc UNUSED, char **argv)
 182 {
 183   char *err = spv_detect (argv[1]);
 184   if (err)
 185     error (1, 0, "%s", err);
 186 }
 187
 188 static struct output_item *
 189 read_and_filter_spv (const char *name, struct page_setup **psp)
 190 {
 191   struct output_item *root;
 192   char *err = spv_read (name, &root, psp);
 193   if (err)
 194     error (1, 0, "%s", err);
 195   return output_select (root, criteria, n_criteria);
 196 }
 197
 198 static void
 199 run_directory (int argc UNUSED, char **argv)
 200 {
 201   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 202   for (size_t i = 0; i < root->group.n_children; i++)
 203     print_item_directory (root->group.children[i], 0);
 204   output_item_unref (root);
 205 }
 206
 207 static void
 208 set_table_look_recursively (struct output_item *item,
 209                             const struct pivot_table_look *look)
 210 {
 211   if (item->type == OUTPUT_ITEM_TABLE)
 212     pivot_table_set_look (item->table, look);
 213   else if (item->type == OUTPUT_ITEM_GROUP)
 214     for (size_t i = 0; i < item->group.n_children; i++)
 215       set_table_look_recursively (item->group.children[i], look);
 216 }
 217
 218 static void
 219 run_convert (int argc UNUSED, char **argv)
 220 {
 221   struct page_setup *ps;
 222   struct output_item *root = read_and_filter_spv (argv[1], &ps);
 223   if (table_look)
 224     set_table_look_recursively (root, table_look);
 225   if (show_member_names)
 226     {
 227       struct output_item *new_root = annotate_member_names (root);
 228       output_item_unref (root);
 229       root = new_root;
 230     }
 231
 232   output_engine_push ();
 233   output_set_filename (argv[1]);
 234   string_map_replace (&output_options, "output-file", argv[2]);
 235   struct output_driver *driver = output_driver_create (&output_options);
 236   if (!driver)
 237     exit (EXIT_FAILURE);
 238   output_driver_register (driver);
 239
 240   if (ps)
 241     {
 242       output_set_page_setup (ps);
 243       page_setup_destroy (ps);
 244     }
 245   output_item_submit_children (root);
 246
 247   output_engine_pop ();
 248   fh_done ();
 249
 250   if (n_warnings && !force)
 251     {
 252       /* XXX There could be other files to unlink, e.g. the ascii driver can
 253          produce additional files with the charts. */
 254       unlink (argv[2]);
 255     }
 256 }
 257
 258 static const struct pivot_table *
 259 get_first_table (const struct output_item *item)
 260 {
 261   if (item->type == OUTPUT_ITEM_TABLE)
 262     return item->table;
 263   else if (item->type == OUTPUT_ITEM_GROUP)
 264     for (size_t i = 0; i < item->group.n_children; i++)
 265       {
 266         const struct pivot_table *table
 267           = get_first_table (item->group.children[i]);
 268         if (table)
 269           return table;
 270       }
 271
 272   return NULL;
 273 }
 274
 275 static void
 276 run_get_table_look (int argc UNUSED, char **argv)
 277 {
 278   struct pivot_table_look *look;
 279   if (strcmp (argv[1], "-"))
 280     {
 281       struct output_item *root = read_and_filter_spv (argv[1], NULL);
 282       const struct pivot_table *table = get_first_table (root);
 283       if (!table)
 284         error (1, 0, "%s: no tables found", argv[1]);
 285
 286       look = pivot_table_look_ref (pivot_table_get_look (table));
 287
 288       output_item_unref (root);
 289     }
 290   else
 291     look = pivot_table_look_ref (pivot_table_look_builtin_default ());
 292
 293   char *err = spv_table_look_write (argv[2], look);
 294   if (err)
 295     error (1, 0, "%s", err);
 296
 297   pivot_table_look_unref (look);
 298 }
 299
 300 static void
 301 run_convert_table_look (int argc UNUSED, char **argv)
 302 {
 303   struct pivot_table_look *look;
 304   char *err = spv_table_look_read (argv[1], &look);
 305   if (err)
 306     error (1, 0, "%s", err);
 307
 308   err = spv_table_look_write (argv[2], look);
 309   if (err)
 310     error (1, 0, "%s", err);
 311
 312   pivot_table_look_unref (look);
 313   free (look);
 314 }
 315
 316 static void
 317 run_dump (int argc UNUSED, char **argv)
 318 {
 319   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 320   output_item_dump (root, 0);
 321   output_item_unref (root);
 322 }
 323
 324 static int
 325 compare_borders (const void *a_, const void *b_)
 326 {
 327   const struct spvlb_border *const *ap = a_;
 328   const struct spvlb_border *const *bp = b_;
 329   uint32_t a = (*ap)->border_type;
 330   uint32_t b = (*bp)->border_type;
 331
 332   return a < b ? -1 : a > b;
 333 }
 334
 335 static int
 336 compare_cells (const void *a_, const void *b_)
 337 {
 338   const struct spvlb_cell *const *ap = a_;
 339   const struct spvlb_cell *const *bp = b_;
 340   uint64_t a = (*ap)->index;
 341   uint64_t b = (*bp)->index;
 342
 343   return a < b ? -1 : a > b;
 344 }
 345
 346 static char * WARN_UNUSED_RESULT
 347 dump_raw (struct zip_reader *zr, const char *member_name)
 348 {
 349   void *data;
 350   size_t size;
 351   char *error = zip_member_read_all (zr, member_name, &data, &size);
 352   if (!error)
 353     {
 354       fwrite (data, size, 1, stdout);
 355       free (data);
 356     }
 357   return error;
 358 }
 359
 360 static void
 361 dump_light_table (const struct output_item *item)
 362 {
 363   char *error;
 364   if (raw)
 365     error = dump_raw (item->spv_info->zip_reader,
 366                       item->spv_info->bin_member);
 367   else
 368     {
 369       struct spvlb_table *table;
 370       error = spv_read_light_table (item->spv_info->zip_reader,
 371                                     item->spv_info->bin_member, &table);
 372       if (!error)
 373         {
 374           if (sort)
 375             {
 376               qsort (table->borders->borders, table->borders->n_borders,
 377                      sizeof *table->borders->borders, compare_borders);
 378               qsort (table->cells->cells, table->cells->n_cells,
 379                      sizeof *table->cells->cells, compare_cells);
 380             }
 381           spvlb_print_table (item->spv_info->bin_member, 0, table);
 382           spvlb_free_table (table);
 383         }
 384     }
 385   if (error)
 386     {
 387       msg (ME, "%s", error);
 388       free (error);
 389     }
 390 }
 391
 392 static void
 393 run_dump_light_table (int argc UNUSED, char **argv)
 394 {
 395   if (raw && isatty (STDOUT_FILENO))
 396     error (1, 0, "not writing binary data to tty");
 397
 398   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 399   struct output_iterator iter;
 400   OUTPUT_ITEM_FOR_EACH (&iter, root)
 401     if (iter.cur->type == OUTPUT_ITEM_TABLE && !iter.cur->spv_info->xml_member)
 402       dump_light_table (iter.cur);
 403   output_item_unref (root);
 404 }
 405
 406 static void
 407 dump_legacy_data (const struct output_item *item)
 408 {
 409   char *error;
 410   if (raw)
 411     error = dump_raw (item->spv_info->zip_reader,
 412                       item->spv_info->bin_member);
 413   else
 414     {
 415       struct spv_data data;
 416       error = spv_read_legacy_data (item->spv_info->zip_reader,
 417                                     item->spv_info->bin_member, &data);
 418       if (!error)
 419         {
 420           printf ("%s:\n", item->spv_info->bin_member);
 421           spv_data_dump (&data, stdout);
 422           spv_data_uninit (&data);
 423           printf ("\n");
 424         }
 425     }
 426
 427   if (error)
 428     {
 429       msg (ME, "%s", error);
 430       free (error);
 431     }
 432 }
 433
 434 static void
 435 run_dump_legacy_data (int argc UNUSED, char **argv)
 436 {
 437   if (raw && isatty (STDOUT_FILENO))
 438     error (1, 0, "not writing binary data to tty");
 439
 440   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 441   struct output_iterator iter;
 442   OUTPUT_ITEM_FOR_EACH (&iter, root)
 443     if (iter.cur->type == OUTPUT_ITEM_TABLE
 444         && iter.cur->spv_info->xml_member
 445         && iter.cur->spv_info->bin_member)
 446       dump_legacy_data (iter.cur);
 447   output_item_unref (root);
 448 }
 449
 450 /* This is really bogus.
 451
 452    XPath doesn't have any notion of a default XML namespace, but all of the
 453    elements in the documents we're interested in have a namespace.  Thus, we'd
 454    need to require the XPath expressions to have a namespace on every single
 455    element: vis:sourceVariable, vis:graph, and so on.  That's a pain.  So,
 456    instead, we remove the default namespace from everyplace it occurs.  XPath
 457    does support the null namespace, so this allows sourceVariable, graph,
 458    etc. to work.
 459
 460    See http://plasmasturm.org/log/259/ and
 461    https://mail.gnome.org/archives/xml/2003-April/msg00144.html for more
 462    information.*/
 463 static void
 464 remove_default_xml_namespace (xmlNode *node)
 465 {
 466   if (node->ns && !node->ns->prefix)
 467     node->ns = NULL;
 468
 469   for (xmlNode *child = node->children; child; child = child->next)
 470     remove_default_xml_namespace (child);
 471 }
 472
 473 static void
 474 register_ns (xmlXPathContext *ctx, const char *prefix, const char *uri)
 475 {
 476   xmlXPathRegisterNs (ctx, CHAR_CAST (xmlChar *, prefix),
 477                       CHAR_CAST (xmlChar *, uri));
 478 }
 479
 480 static xmlXPathContext *
 481 create_xpath_context (xmlDoc *doc)
 482 {
 483   xmlXPathContext *ctx = xmlXPathNewContext (doc);
 484   register_ns (ctx, "vgr", "http://xml.spss.com/spss/viewer/viewer-graph");
 485   register_ns (ctx, "vizml", "http://xml.spss.com/visualization");
 486   register_ns (ctx, "vmd", "http://xml.spss.com/spss/viewer/viewer-model");
 487   register_ns (ctx, "vps", "http://xml.spss.com/spss/viewer/viewer-pagesetup");
 488   register_ns (ctx, "vst", "http://xml.spss.com/spss/viewer/viewer-style");
 489   register_ns (ctx, "vtb", "http://xml.spss.com/spss/viewer/viewer-table");
 490   register_ns (ctx, "vtl", "http://xml.spss.com/spss/viewer/table-looks");
 491   register_ns (ctx, "vtt", "http://xml.spss.com/spss/viewer/viewer-treemodel");
 492   register_ns (ctx, "vtx", "http://xml.spss.com/spss/viewer/viewer-text");
 493   register_ns (ctx, "xsi", "http://www.w3.org/2001/XMLSchema-instance");
 494   return ctx;
 495 }
 496
 497 static void
 498 dump_xml (int argc, char **argv, const char *member_name,
 499           char *error_s, xmlDoc *doc)
 500 {
 501   if (!error_s)
 502     {
 503       if (argc == 2)
 504         {
 505           printf ("<!-- %s -->\n", member_name);
 506           xmlElemDump (stdout, NULL, xmlDocGetRootElement (doc));
 507           putchar ('\n');
 508         }
 509       else
 510         {
 511           bool any_results = false;
 512
 513           remove_default_xml_namespace (xmlDocGetRootElement (doc));
 514           for (int i = 2; i < argc; i++)
 515             {
 516               xmlXPathContext *xpath_ctx = create_xpath_context (doc);
 517               xmlXPathSetContextNode (xmlDocGetRootElement (doc),
 518                                       xpath_ctx);
 519               xmlXPathObject *xpath_obj = xmlXPathEvalExpression(
 520                 CHAR_CAST (xmlChar *, argv[i]), xpath_ctx);
 521               if (!xpath_obj)
 522                 error (1, 0, _("%s: invalid XPath expression"), argv[i]);
 523
 524               const xmlNodeSet *nodes = xpath_obj->nodesetval;
 525               if (nodes && nodes->nodeNr > 0)
 526                 {
 527                   if (!any_results)
 528                     {
 529                       printf ("<!-- %s -->\n", member_name);
 530                       any_results = true;
 531                     }
 532                   for (size_t j = 0; j < nodes->nodeNr; j++)
 533                     {
 534                       xmlElemDump (stdout, doc, nodes->nodeTab[j]);
 535                       putchar ('\n');
 536                     }
 537                 }
 538
 539               xmlXPathFreeObject (xpath_obj);
 540               xmlXPathFreeContext (xpath_ctx);
 541             }
 542           if (any_results)
 543             putchar ('\n');
 544         }
 545       xmlFreeDoc (doc);
 546     }
 547   else
 548     {
 549       printf ("<!-- %s -->\n", member_name);
 550       msg (ME, "%s", error_s);
 551       free (error_s);
 552     }
 553 }
 554
 555 static void
 556 dump_legacy_table (int argc, char **argv, const struct output_item *item)
 557 {
 558   xmlDoc *doc;
 559   char *error_s = spv_read_xml_member (item->spv_info->zip_reader,
 560                                        item->spv_info->xml_member,
 561                                        false, "visualization", &doc);
 562   dump_xml (argc, argv, item->spv_info->xml_member, error_s, doc);
 563 }
 564
 565 static void
 566 run_dump_legacy_table (int argc, char **argv)
 567 {
 568   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 569   struct output_iterator iter;
 570   OUTPUT_ITEM_FOR_EACH (&iter, root)
 571     if (iter.cur->type == OUTPUT_ITEM_TABLE
 572         && iter.cur->spv_info->xml_member)
 573       dump_legacy_table (argc, argv, iter.cur);
 574   output_item_unref (root);
 575 }
 576
 577 static void
 578 dump_structure (int argc, char **argv, const struct output_item *item)
 579 {
 580   xmlDoc *doc;
 581   char *error_s = spv_read_xml_member (item->spv_info->zip_reader,
 582                                        item->spv_info->structure_member,
 583                                        true, "heading", &doc);
 584   dump_xml (argc, argv, item->spv_info->structure_member, error_s, doc);
 585 }
 586
 587 static void
 588 run_dump_structure (int argc, char **argv)
 589 {
 590   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 591
 592   const char *last_structure_member = NULL;
 593   struct output_iterator iter;
 594   OUTPUT_ITEM_FOR_EACH (&iter, root)
 595     {
 596       const struct output_item *item = iter.cur;
 597       if (item->spv_info->structure_member
 598           && (!last_structure_member
 599               || strcmp (item->spv_info->structure_member,
 600                          last_structure_member)))
 601         {
 602           last_structure_member = item->spv_info->structure_member;
 603           dump_structure (argc, argv, item);
 604         }
 605     }
 606   output_item_unref (root);
 607 }
 608
 609 static bool
 610 is_any_legacy (const struct output_item *item)
 611 {
 612   if (item->type == OUTPUT_ITEM_TABLE)
 613     return item->spv_info->xml_member != NULL;
 614   else if (item->type == OUTPUT_ITEM_GROUP)
 615     for (size_t i = 0; i < item->group.n_children; i++)
 616       if (is_any_legacy (item->group.children[i]))
 617         return true;
 618
 619   return false;
 620 }
 621
 622 static void
 623 run_is_legacy (int argc UNUSED, char **argv)
 624 {
 625   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 626   bool is_legacy = is_any_legacy (root);
 627   output_item_unref (root);
 628
 629   exit (is_legacy ? EXIT_SUCCESS : EXIT_FAILURE);
 630 }
 631
 632 static bool
 633 is_all_ascii (const char *s)
 634 {
 635   for (; *s; s++)
 636     if (!encoding_guess_is_ascii_text (*s))
 637       return false;
 638
 639   return true;
 640 }
 641
 642 static void
 643 dump_strings (const char *encoding, struct string_array *strings)
 644 {
 645   string_array_sort (strings);
 646   string_array_uniq (strings);
 647
 648   if (raw)
 649     {
 650       if (exclude_ascii_only || include_utf8_only)
 651         {
 652           size_t i = 0;
 653           for (size_t j = 0; j < strings->n; j++)
 654             {
 655               char *s = strings->strings[j];
 656               bool is_ascii = is_all_ascii (s);
 657               bool is_utf8 = !u8_check (CHAR_CAST (uint8_t *, s), strlen (s));
 658               if (!is_ascii && (!include_utf8_only || is_utf8))
 659                 strings->strings[i++] = s;
 660               else
 661                 free (s);
 662             }
 663           strings->n = i;
 664         }
 665       for (size_t i = 0; i < strings->n; i++)
 666         puts (strings->strings[i]);
 667     }
 668   else
 669     {
 670       size_t n_nonascii = 0;
 671       size_t n_utf8 = 0;
 672       for (size_t i = 0; i < strings->n; i++)
 673         {
 674           const char *s = strings->strings[i];
 675           if (!is_all_ascii (s))
 676             {
 677               n_nonascii++;
 678               if (!u8_check (CHAR_CAST (uint8_t *, s), strlen (s)))
 679                 n_utf8++;
 680             }
 681         }
 682       printf ("%s: %zu unique strings, %zu non-ASCII, %zu UTF-8.\n",
 683               encoding, strings->n, n_nonascii, n_utf8);
 684     }
 685 }
 686
 687 struct encoded_strings
 688   {
 689     char *encoding;
 690     struct string_array strings;
 691   };
 692
 693 struct encoded_strings_table
 694   {
 695     struct encoded_strings *es;
 696     size_t n, allocated;
 697   };
 698
 699 static void
 700 collect_strings (const struct output_item *item,
 701                  struct encoded_strings_table *t)
 702 {
 703   char *error;
 704   struct spvlb_table *table;
 705   error = spv_read_light_table (item->spv_info->zip_reader,
 706                                 item->spv_info->bin_member, &table);
 707   if (error)
 708     {
 709       msg (ME, "%s", error);
 710       free (error);
 711       return;
 712     }
 713
 714   const char *table_encoding = spvlb_table_get_encoding (table);
 715   size_t j = 0;
 716   for (j = 0; j < t->n; j++)
 717     if (!strcmp (t->es[j].encoding, table_encoding))
 718       break;
 719   if (j >= t->n)
 720     {
 721       if (t->n >= t->allocated)
 722         t->es = x2nrealloc (t->es, &t->allocated, sizeof *t->es);
 723       t->es[t->n++] = (struct encoded_strings) {
 724         .encoding = xstrdup (table_encoding),
 725         .strings = STRING_ARRAY_INITIALIZER,
 726       };
 727     }
 728   collect_spvlb_strings (table, &t->es[j].strings);
 729 }
 730
 731 static void
 732 run_strings (int argc UNUSED, char **argv)
 733 {
 734   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 735
 736   struct encoded_strings_table t = { .es = NULL };
 737   struct output_iterator iter;
 738   OUTPUT_ITEM_FOR_EACH (&iter, root)
 739     {
 740       const struct output_item *item = iter.cur;
 741       if (item->type == OUTPUT_ITEM_TABLE
 742           && !item->spv_info->xml_member
 743           && item->spv_info->bin_member)
 744         collect_strings (item, &t);
 745     }
 746
 747   for (size_t i = 0; i < t.n; i++)
 748     {
 749       dump_strings (t.es[i].encoding, &t.es[i].strings);
 750       free (t.es[i].encoding);
 751       string_array_destroy (&t.es[i].strings);
 752     }
 753   free (t.es);
 754
 755   output_item_unref (root);
 756 }
 757
 758 struct command
 759   {
 760     const char *name;
 761     int min_args, max_args;
 762     void (*run) (int argc, char **argv);
 763   };
 764
 765 static const struct command commands[] =
 766   {
 767     { "detect", 1, 1, run_detect },
 768     { "dir", 1, 1, run_directory },
 769     { "convert", 2, 2, run_convert },
 770     { "get-table-look", 2, 2, run_get_table_look },
 771     { "convert-table-look", 2, 2, run_convert_table_look },
 772
 773     /* Undocumented commands. */
 774     { "dump", 1, 1, run_dump },
 775     { "dump-light-table", 1, 1, run_dump_light_table },
 776     { "dump-legacy-data", 1, 1, run_dump_legacy_data },
 777     { "dump-legacy-table", 1, INT_MAX, run_dump_legacy_table },
 778     { "dump-structure", 1, INT_MAX, run_dump_structure },
 779     { "is-legacy", 1, 1, run_is_legacy },
 780     { "strings", 1, 1, run_strings },
 781   };
 782 static const int n_commands = sizeof commands / sizeof *commands;
 783
 784 static const struct command *
 785 find_command (const char *name)
 786 {
 787   for (size_t i = 0; i < n_commands; i++)
 788     {
 789       const struct command *c = &commands[i];
 790       if (!strcmp (name, c->name))
 791         return c;
 792     }
 793   return NULL;
 794 }
 795
 796 static void
 797 emit_msg (const struct msg *m, void *aux UNUSED)
 798 {
 799   if (m->severity == MSG_S_ERROR || m->severity == MSG_S_WARNING)
 800     n_warnings++;
 801
 802   char *s = msg_to_string (m);
 803   fprintf (stderr, "%s\n", s);
 804   free (s);
 805 }
 806
 807 int
 808 main (int argc, char **argv)
 809 {
 810   set_program_name (argv[0]);
 811   msg_set_handler (emit_msg, NULL);
 812   settings_init ();
 813   i18n_init ();
 814
 815   parse_options (argc, argv);
 816
 817   argc -= optind;
 818   argv += optind;
 819
 820   if (argc < 1)
 821     error (1, 0, _("missing command name (use --help for help)"));
 822
 823   const struct command *c = find_command (argv[0]);
 824   if (!c)
 825     error (1, 0, _("unknown command \"%s\" (use --help for help)"), argv[0]);
 826
 827   int n_args = argc - 1;
 828   if (n_args < c->min_args || n_args > c->max_args)
 829     {
 830       if (c->min_args == c->max_args)
 831         {
 832           error (1, 0,
 833                  ngettext ("\"%s\" command takes exactly %d argument",
 834                            "\"%s\" command takes exactly %d arguments",
 835                            c->min_args), c->name, c->min_args);
 836         }
 837       else if (c->max_args == INT_MAX)
 838         {
 839           error (1, 0,
 840                  ngettext ("\"%s\" command requires at least %d argument",
 841                            "\"%s\" command requires at least %d arguments",
 842                            c->min_args), c->name, c->min_args);
 843         }
 844       else
 845         {
 846           error (1, 0,
 847                  _("\"%s\" command requires between %d and %d arguments"),
 848                  c->name, c->min_args, c->max_args);
 849         }
 850     }
 851
 852   c->run (argc, argv);
 853
 854   pivot_table_look_unref (table_look);
 855   i18n_done ();
 856
 857   return n_warnings ? EXIT_FAILURE : EXIT_SUCCESS;
 858 }
 859
 860 static struct output_criteria *
 861 get_criteria (void)
 862 {
 863   if (!n_criteria || new_criteria)
 864     {
 865       new_criteria = false;
 866       if (n_criteria >= allocated_criteria)
 867         criteria = x2nrealloc (criteria, &allocated_criteria,
 868                                sizeof *criteria);
 869       criteria[n_criteria++]
 870         = (struct output_criteria) OUTPUT_CRITERIA_INITIALIZER;
 871     }
 872
 873   return &criteria[n_criteria - 1];
 874 }
 875
 876 static void
 877 parse_select (char *arg)
 878 {
 879   bool invert = arg[0] == '^';
 880   arg += invert;
 881
 882   unsigned classes = 0;
 883   for (char *token = strtok (arg, ","); token; token = strtok (NULL, ","))
 884     {
 885       if (!strcmp (arg, "all"))
 886         classes = OUTPUT_ALL_CLASSES;
 887       else if (!strcmp (arg, "help"))
 888         {
 889           puts (_("The following object classes are supported:"));
 890           for (int class = 0; class < OUTPUT_N_CLASSES; class++)
 891             printf ("- %s\n", output_item_class_to_string (class));
 892           exit (0);
 893         }
 894       else
 895         {
 896           int class = output_item_class_from_string (token);
 897           if (class == OUTPUT_N_CLASSES)
 898             error (1, 0, _("unknown object class \"%s\" (use --select=help "
 899                            "for help)"), arg);
 900           classes |= 1u << class;
 901         }
 902     }
 903
 904   struct output_criteria *c = get_criteria ();
 905   c->classes = invert ? classes ^ OUTPUT_ALL_CLASSES : classes;
 906 }
 907
 908 static struct output_criteria_match *
 909 get_criteria_match (const char **arg)
 910 {
 911   struct output_criteria *c = get_criteria ();
 912   if ((*arg)[0] == '^')
 913     {
 914       (*arg)++;
 915       return &c->exclude;
 916     }
 917   else
 918     return &c->include;
 919 }
 920
 921 static void
 922 parse_commands (const char *arg)
 923 {
 924   struct output_criteria_match *cm = get_criteria_match (&arg);
 925   string_array_parse (&cm->commands, ss_cstr (arg), ss_cstr (","));
 926 }
 927
 928 static void
 929 parse_subtypes (const char *arg)
 930 {
 931   struct output_criteria_match *cm = get_criteria_match (&arg);
 932   string_array_parse (&cm->subtypes, ss_cstr (arg), ss_cstr (","));
 933 }
 934
 935 static void
 936 parse_labels (const char *arg)
 937 {
 938   struct output_criteria_match *cm = get_criteria_match (&arg);
 939   string_array_parse (&cm->labels, ss_cstr (arg), ss_cstr (","));
 940 }
 941
 942 static void
 943 parse_instances (char *arg)
 944 {
 945   struct output_criteria *c = get_criteria ();
 946   size_t allocated_instances = c->n_instances;
 947
 948   for (char *token = strtok (arg, ","); token; token = strtok (NULL, ","))
 949     {
 950       if (c->n_instances >= allocated_instances)
 951         c->instances = x2nrealloc (c->instances, &allocated_instances,
 952                                    sizeof *c->instances);
 953
 954       c->instances[c->n_instances++] = (!strcmp (token, "last") ? -1
 955                                         : atoi (token));
 956     }
 957 }
 958
 959 static void
 960 parse_nth_commands (char *arg)
 961 {
 962   struct output_criteria *c = get_criteria ();
 963   size_t allocated_commands = c->n_commands;
 964
 965   for (char *token = strtok (arg, ","); token; token = strtok (NULL, ","))
 966     {
 967       if (c->n_commands >= allocated_commands)
 968         c->commands = x2nrealloc (c->commands, &allocated_commands,
 969                                    sizeof *c->commands);
 970
 971       c->commands[c->n_commands++] = atoi (token);
 972     }
 973 }
 974
 975 static void
 976 parse_members (const char *arg)
 977 {
 978   struct output_criteria *cm = get_criteria ();
 979   string_array_parse (&cm->members, ss_cstr (arg), ss_cstr (","));
 980 }
 981
 982 static void
 983 parse_table_look (const char *arg)
 984 {
 985   pivot_table_look_unref (table_look);
 986
 987   char *error_s = pivot_table_look_read (arg, &table_look);
 988   if (error_s)
 989     error (1, 0, "%s", error_s);
 990 }
 991
 992 static void
 993 parse_options (int argc, char *argv[])
 994 {
 995   for (;;)
 996     {
 997       enum
 998         {
 999           OPT_MEMBER_NAMES = UCHAR_MAX + 1,
1000           OPT_SHOW_HIDDEN,
1001           OPT_SELECT,
1002           OPT_COMMANDS,
1003           OPT_NTH_COMMANDS,
1004           OPT_SUBTYPES,
1005           OPT_LABELS,
1006           OPT_INSTANCES,
1007           OPT_MEMBERS,
1008           OPT_ERRORS,
1009           OPT_OR,
1010           OPT_SORT,
1011           OPT_RAW,
1012           OPT_NO_ASCII_ONLY,
1013           OPT_UTF8_ONLY,
1014           OPT_TABLE_LOOK,
1015           OPT_HELP_DEVELOPER,
1016         };
1017       static const struct option long_options[] =
1018         {
1019           /* Input selection options. */
1020           { "show-hidden", no_argument, NULL, OPT_SHOW_HIDDEN },
1021           { "select", required_argument, NULL, OPT_SELECT },
1022           { "commands", required_argument, NULL, OPT_COMMANDS },
1023           { "nth-commands", required_argument, NULL, OPT_NTH_COMMANDS },
1024           { "subtypes", required_argument, NULL, OPT_SUBTYPES },
1025           { "labels", required_argument, NULL, OPT_LABELS },
1026           { "instances", required_argument, NULL, OPT_INSTANCES },
1027           { "members", required_argument, NULL, OPT_MEMBERS },
1028           { "errors", no_argument, NULL, OPT_ERRORS },
1029           { "or", no_argument, NULL, OPT_OR },
1030
1031           /* "dir" command options. */
1032           { "member-names", no_argument, NULL, OPT_MEMBER_NAMES },
1033
1034           /* "convert" command options. */
1035           { "force", no_argument, NULL, 'f' },
1036           { "table-look", required_argument, NULL, OPT_TABLE_LOOK },
1037
1038           /* "dump-light-table" command options. */
1039           { "sort", no_argument, NULL, OPT_SORT },
1040           { "raw", no_argument, NULL, OPT_RAW },
1041
1042           /* "strings" command options. */
1043           { "no-ascii-only", no_argument, NULL, OPT_NO_ASCII_ONLY },
1044           { "utf8-only", no_argument, NULL, OPT_UTF8_ONLY },
1045
1046           { "help", no_argument, NULL, 'h' },
1047           { "help-developer", no_argument, NULL, OPT_HELP_DEVELOPER },
1048           { "version", no_argument, NULL, 'v' },
1049
1050           { NULL, 0, NULL, 0 },
1051         };
1052
1053       int c;
1054
1055       c = getopt_long (argc, argv, "O:hvf", long_options, NULL);
1056       if (c == -1)
1057         break;
1058
1059       switch (c)
1060         {
1061         case 'O':
1062           output_driver_parse_option (optarg, &output_options);
1063           break;
1064
1065         case OPT_MEMBER_NAMES:
1066           show_member_names = true;
1067           break;
1068
1069         case OPT_SHOW_HIDDEN:
1070           get_criteria ()->include_hidden = true;
1071           break;
1072
1073         case OPT_SELECT:
1074           parse_select (optarg);
1075           break;
1076
1077         case OPT_COMMANDS:
1078           parse_commands (optarg);
1079           break;
1080
1081         case OPT_NTH_COMMANDS:
1082           parse_nth_commands (optarg);
1083           break;
1084
1085         case OPT_SUBTYPES:
1086           parse_subtypes (optarg);
1087           break;
1088
1089         case OPT_LABELS:
1090           parse_labels (optarg);
1091           break;
1092
1093         case OPT_INSTANCES:
1094           parse_instances (optarg);
1095           break;
1096
1097         case OPT_MEMBERS:
1098           parse_members (optarg);
1099           break;
1100
1101         case OPT_ERRORS:
1102           get_criteria ()->error = true;
1103           break;
1104
1105         case OPT_OR:
1106           new_criteria = true;
1107           break;
1108
1109         case OPT_SORT:
1110           sort = true;
1111           break;
1112
1113         case OPT_RAW:
1114           raw = true;
1115           break;
1116
1117         case OPT_TABLE_LOOK:
1118           parse_table_look (optarg);
1119           break;
1120
1121         case OPT_NO_ASCII_ONLY:
1122           exclude_ascii_only = true;
1123           break;
1124
1125         case OPT_UTF8_ONLY:
1126           include_utf8_only = true;
1127           break;
1128
1129         case 'f':
1130           force = true;
1131           break;
1132
1133         case 'v':
1134           version_etc (stdout, "pspp-output", PACKAGE_NAME, PACKAGE_VERSION,
1135                        "Ben Pfaff", "John Darrington", NULL_SENTINEL);
1136           exit (EXIT_SUCCESS);
1137
1138         case 'h':
1139           usage ();
1140           exit (EXIT_SUCCESS);
1141
1142         case OPT_HELP_DEVELOPER:
1143           developer_usage ();
1144           exit (EXIT_SUCCESS);
1145
1146         default:
1147           exit (EXIT_FAILURE);
1148         }
1149     }
1150 }
1151
1152 static void
1153 usage (void)
1154 {
1155   struct string s = DS_EMPTY_INITIALIZER;
1156   struct string_set formats = STRING_SET_INITIALIZER(formats);
1157   output_get_supported_formats (&formats);
1158   const char *format;
1159   const struct string_set_node *node;
1160   STRING_SET_FOR_EACH (format, node, &formats)
1161     {
1162       if (!ds_is_empty (&s))
1163         ds_put_byte (&s, ' ');
1164       ds_put_cstr (&s, format);
1165     }
1166   string_set_destroy (&formats);
1167
1168   printf ("\
1169 %s, a utility for working with SPSS viewer (.spv) files.\n\
1170 Usage: %s [OPTION]... COMMAND ARG...\n\
1171 \n\
1172 The following commands are available:\n\
1173   detect FILE            Detect whether FILE is an SPV file.\n\
1174   dir FILE               List tables and other items in FILE.\n\
1175   convert SOURCE DEST    Convert .spv SOURCE to DEST.\n\
1176   get-table-look SOURCE DEST  Copies first selected TableLook into DEST\n\
1177   convert-table-look SOURCE DEST  Copies .tlo or .stt SOURCE into DEST\n\
1178 \n\
1179 Input selection options for \"dir\" and \"convert\":\n\
1180   --select=CLASS...   include only some kinds of objects\n\
1181   --select=help       print known object classes\n\
1182   --commands=COMMAND...  include only specified COMMANDs\n\
1183   --nth-commands=N...  include only the Nth instance of selected commands\n\
1184   --subtypes=SUBTYPE...  include only specified SUBTYPEs of output\n\
1185   --labels=LABEL...   include only output objects with the given LABELs\n\
1186   --instances=INSTANCE...  include only the given object INSTANCEs\n\
1187   --show-hidden       include hidden output objects\n\
1188   --or                separate two sets of selection options\n\
1189 \n\
1190 \"convert\" by default infers the destination's format from its extension.\n\
1191 The known extensions are: %s\n\
1192 The following options override \"convert\" behavior:\n\
1193   -O format=FORMAT          set destination format to FORMAT\n\
1194   -O OPTION=VALUE           set output option\n\
1195   -f, --force               keep output file even given errors\n\
1196   --table-look=FILE         override tables' style with TableLook from FILE\n\
1197 Other options:\n\
1198   --help              display this help and exit\n\
1199   --help-developer    display help for developer commands and exit\n\
1200   --version           output version information and exit\n",
1201           program_name, program_name, ds_cstr (&s));
1202   ds_destroy (&s);
1203 }
1204
1205 static void
1206 developer_usage (void)
1207 {
1208   printf ("\
1209 The following developer commands are available:\n\
1210   dump FILE              Dump pivot table structure\n\
1211   [--raw | --sort] dump-light-table FILE  Dump light tables\n\
1212   [--raw] dump-legacy-data FILE  Dump legacy table data\n\
1213   dump-legacy-table FILE [XPATH]...  Dump legacy table XML\n\
1214   dump-structure FILE [XPATH]...  Dump structure XML\n\
1215   is-legacy FILE         Exit with status 0 if any legacy table selected\n\
1216   strings FILE           Dump analysis of strings\n\
1217 \n\
1218 Additional input selection options:\n\
1219   --members=MEMBER...    include only objects with these Zip member names\n\
1220   --errors               include only objects that cannot be loaded\n\
1221 \n\
1222 Additional options for \"dir\" command:\n\
1223   --member-names         show Zip member names with objects\n\
1224 \n\
1225 Options for the \"strings\" command:\n\
1226   --raw                  Dump all (unique) strings\n\
1227   --raw --no-ascii-only  Dump all strings that contain non-ASCII characters\n\
1228   --raw --utf8-only      Dump all non-ASCII strings that are valid UTF-8\n\
1229 \n\
1230 Other options:\n\
1231   --raw                  print raw binary data instead of a parsed version\n\
1232   --sort                 sort borders and areas for shorter \"diff\" output\n");
1233 }