utilities/pspp-output.c

   1  /* PSPP - a program for statistical analysis.
   2    Copyright (C) 2017, 2018 Free Software Foundation, Inc.
   3
   4    This program is free software: you can redistribute it and/or modify
   5    it under the terms of the GNU General Public License as published by
   6    the Free Software Foundation, either version 3 of the License, or
   7    (at your option) any later version.
   8
   9    This program is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12    GNU General Public License for more details.
  13
  14    You should have received a copy of the GNU General Public License
  15    along with this program.  If not, see <http://www.gnu.org/licenses/>. */
  16
  17 #include <config.h>
  18
  19 #include <cairo.h>
  20 #include <getopt.h>
  21 #include <limits.h>
  22 #include <stdlib.h>
  23 #include <unistd.h>
  24 #include <unistr.h>
  25
  26 #include "data/file-handle-def.h"
  27 #include "data/settings.h"
  28 #include "libpspp/encoding-guesser.h"
  29 #include "libpspp/i18n.h"
  30 #include "libpspp/message.h"
  31 #include "libpspp/string-map.h"
  32 #include "libpspp/string-set.h"
  33 #include "libpspp/zip-reader.h"
  34 #include "output/driver.h"
  35 #include "output/output-item.h"
  36 #include "output/pivot-table.h"
  37 #include "output/page-setup.h"
  38 #include "output/select.h"
  39 #include "output/spv/light-binary-parser.h"
  40 #include "output/spv/spv-legacy-data.h"
  41 #include "output/spv/spv-light-decoder.h"
  42 #include "output/spv/spv-table-look.h"
  43 #include "output/spv/spv.h"
  44
  45 #include "gl/c-ctype.h"
  46 #include "gl/error.h"
  47 #include "gl/progname.h"
  48 #include "gl/version-etc.h"
  49 #include "gl/xalloc.h"
  50
  51 #include <libxml/tree.h>
  52 #include <libxml/xpath.h>
  53 #include <libxml/xpathInternals.h>
  54
  55 #include "gettext.h"
  56 #define _(msgid) gettext (msgid)
  57
  58 /* -O key=value: Output driver options. */
  59 static struct string_map output_options
  60     = STRING_MAP_INITIALIZER (output_options);
  61
  62 /* --member-names: Include .zip member name in "dir" output. */
  63 static bool show_member_names;
  64
  65 /* --show-hidden, --select, --commands, ...: Selection criteria. */
  66 static struct output_criteria *criteria;
  67 static size_t n_criteria, allocated_criteria;
  68
  69 /* --or: Add new element to 'criteria' array. */
  70 static bool new_criteria;
  71
  72 /* --sort: Sort members under dump-light-table, to make comparisons easier. */
  73 static bool sort;
  74
  75 /* --raw: Dump raw binary data in "dump-light-table"; dump all strings in
  76      "strings". */
  77 static bool raw;
  78
  79 /* --no-ascii-only: Drop all-ASCII strings in "strings". */
  80 static bool exclude_ascii_only;
  81
  82 /* --utf8-only: Only print strings that have UTF-8 multibyte sequences in
  83  * "strings". */
  84 static bool include_utf8_only;
  85
  86 /* -f, --force: Keep output file even on error. */
  87 static bool force;
  88
  89 /* --table-look: TableLook to replace table style for conversion. */
  90 static struct pivot_table_look *table_look;
  91
  92 /* --use-page-setup: Use page setup from .spv file rather than command line. */
  93 static bool use_page_setup;
  94
  95 /* Number of warnings issued. */
  96 static size_t n_warnings;
  97
  98 static void usage (void);
  99 static void developer_usage (void);
 100 static void parse_options (int argc, char **argv);
 101
 102 static struct output_item *
 103 annotate_member_names (const struct output_item *in)
 104 {
 105   if (in->type == OUTPUT_ITEM_GROUP)
 106     {
 107       struct output_item *out = group_item_clone_empty (in);
 108       for (size_t i = 0; i < in->group.n_children; i++)
 109         {
 110           const struct output_item *item = in->group.children[i];
 111           const char *members[4];
 112           size_t n = spv_info_get_members (item->spv_info, members,
 113                                            sizeof members / sizeof *members);
 114           if (n)
 115             {
 116               struct string s = DS_EMPTY_INITIALIZER;
 117               ds_put_cstr (&s, members[0]);
 118               for (size_t i = 1; i < n; i++)
 119                 ds_put_format (&s, " and %s", members[i]);
 120               group_item_add_child (out, text_item_create_nocopy (
 121                                       TEXT_ITEM_TITLE, ds_steal_cstr (&s),
 122                                       xstrdup ("Member Names")));
 123             }
 124
 125           group_item_add_child (out, output_item_ref (item));
 126         }
 127       return out;
 128     }
 129   else
 130     return output_item_ref (in);
 131 }
 132
 133 static void
 134 print_item_directory (const struct output_item *item, int level)
 135 {
 136   for (int i = 0; i < level; i++)
 137     printf ("    ");
 138
 139   printf ("- %s", output_item_type_to_string (item->type));
 140
 141   const char *label = output_item_get_label (item);
 142   if (label)
 143     printf (" \"%s\"", label);
 144
 145   if (item->type == OUTPUT_ITEM_TABLE)
 146     {
 147       char *title = pivot_value_to_string (item->table->title, item->table);
 148       if (!label || strcmp (title, label))
 149         printf (" title \"%s\"", title);
 150       free (title);
 151     }
 152
 153   if (item->command_name)
 154     printf (" command \"%s\"", item->command_name);
 155
 156   char *subtype = output_item_get_subtype (item);
 157   if (subtype)
 158     {
 159       if (!label || strcmp (label, subtype))
 160         printf (" subtype \"%s\"", subtype);
 161       free (subtype);
 162     }
 163
 164   if (!item->show)
 165     printf (" (%s)", item->type == OUTPUT_ITEM_GROUP ? "collapsed" : "hidden");
 166
 167   if (show_member_names)
 168     {
 169       const char *members[4];
 170       size_t n = spv_info_get_members (item->spv_info, members,
 171                                        sizeof members / sizeof *members);
 172
 173       for (size_t i = 0; i < n; i++)
 174           printf (" %s %s", i == 0 ? "in" : "and", members[i]);
 175     }
 176   putchar ('\n');
 177
 178   if (item->type == OUTPUT_ITEM_GROUP)
 179     for (size_t i = 0; i < item->group.n_children; i++)
 180       print_item_directory (item->group.children[i], level + 1);
 181 }
 182
 183 static void
 184 run_detect (int argc UNUSED, char **argv)
 185 {
 186   char *err = spv_detect (argv[1]);
 187   if (err)
 188     error (1, 0, "%s", err);
 189 }
 190
 191 static struct output_item *
 192 read_and_filter_spv (const char *name, struct page_setup **psp)
 193 {
 194   struct output_item *root;
 195   char *err = spv_read (name, &root, psp);
 196   if (err)
 197     error (1, 0, "%s", err);
 198   return output_select (root, criteria, n_criteria);
 199 }
 200
 201 static void
 202 run_directory (int argc UNUSED, char **argv)
 203 {
 204   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 205   for (size_t i = 0; i < root->group.n_children; i++)
 206     print_item_directory (root->group.children[i], 0);
 207   output_item_unref (root);
 208 }
 209
 210 static void
 211 set_table_look_recursively (struct output_item *item,
 212                             const struct pivot_table_look *look)
 213 {
 214   if (item->type == OUTPUT_ITEM_TABLE)
 215     pivot_table_set_look (item->table, look);
 216   else if (item->type == OUTPUT_ITEM_GROUP)
 217     for (size_t i = 0; i < item->group.n_children; i++)
 218       set_table_look_recursively (item->group.children[i], look);
 219 }
 220
 221 static void
 222 run_convert (int argc UNUSED, char **argv)
 223 {
 224   struct page_setup *ps;
 225   struct output_item *root = read_and_filter_spv (argv[1], &ps);
 226   if (table_look)
 227     set_table_look_recursively (root, table_look);
 228   if (show_member_names)
 229     {
 230       struct output_item *new_root = annotate_member_names (root);
 231       output_item_unref (root);
 232       root = new_root;
 233     }
 234
 235   output_engine_push ();
 236   output_set_filename (argv[1]);
 237   string_map_replace (&output_options, "output-file", argv[2]);
 238   struct output_driver *driver = output_driver_create (&output_options);
 239   if (!driver)
 240     exit (EXIT_FAILURE);
 241   output_driver_register (driver);
 242
 243   if (ps)
 244     {
 245       if (use_page_setup)
 246         output_set_page_setup (ps);
 247       page_setup_destroy (ps);
 248     }
 249   output_item_submit_children (root);
 250
 251   output_engine_pop ();
 252   fh_done ();
 253
 254   if (n_warnings && !force)
 255     {
 256       /* XXX There could be other files to unlink, e.g. the ascii driver can
 257          produce additional files with the charts. */
 258       unlink (argv[2]);
 259     }
 260 }
 261
 262 static const struct pivot_table *
 263 get_first_table (const struct output_item *item)
 264 {
 265   if (item->type == OUTPUT_ITEM_TABLE)
 266     return item->table;
 267   else if (item->type == OUTPUT_ITEM_GROUP)
 268     for (size_t i = 0; i < item->group.n_children; i++)
 269       {
 270         const struct pivot_table *table
 271           = get_first_table (item->group.children[i]);
 272         if (table)
 273           return table;
 274       }
 275
 276   return NULL;
 277 }
 278
 279 static void
 280 run_get_table_look (int argc UNUSED, char **argv)
 281 {
 282   struct pivot_table_look *look;
 283   if (strcmp (argv[1], "-"))
 284     {
 285       struct output_item *root = read_and_filter_spv (argv[1], NULL);
 286       const struct pivot_table *table = get_first_table (root);
 287       if (!table)
 288         error (1, 0, "%s: no tables found", argv[1]);
 289
 290       look = pivot_table_look_ref (pivot_table_get_look (table));
 291
 292       output_item_unref (root);
 293     }
 294   else
 295     look = pivot_table_look_ref (pivot_table_look_builtin_default ());
 296
 297   char *err = spv_table_look_write (argv[2], look);
 298   if (err)
 299     error (1, 0, "%s", err);
 300
 301   pivot_table_look_unref (look);
 302 }
 303
 304 static void
 305 run_convert_table_look (int argc UNUSED, char **argv)
 306 {
 307   struct pivot_table_look *look;
 308   char *err = spv_table_look_read (argv[1], &look);
 309   if (err)
 310     error (1, 0, "%s", err);
 311
 312   err = spv_table_look_write (argv[2], look);
 313   if (err)
 314     error (1, 0, "%s", err);
 315
 316   pivot_table_look_unref (look);
 317   free (look);
 318 }
 319
 320 static void
 321 run_dump (int argc UNUSED, char **argv)
 322 {
 323   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 324   output_item_dump (root, 0);
 325   output_item_unref (root);
 326 }
 327
 328 static int
 329 compare_borders (const void *a_, const void *b_)
 330 {
 331   const struct spvlb_border *const *ap = a_;
 332   const struct spvlb_border *const *bp = b_;
 333   uint32_t a = (*ap)->border_type;
 334   uint32_t b = (*bp)->border_type;
 335
 336   return a < b ? -1 : a > b;
 337 }
 338
 339 static int
 340 compare_cells (const void *a_, const void *b_)
 341 {
 342   const struct spvlb_cell *const *ap = a_;
 343   const struct spvlb_cell *const *bp = b_;
 344   uint64_t a = (*ap)->index;
 345   uint64_t b = (*bp)->index;
 346
 347   return a < b ? -1 : a > b;
 348 }
 349
 350 static char * WARN_UNUSED_RESULT
 351 dump_raw (struct zip_reader *zr, const char *member_name)
 352 {
 353   void *data;
 354   size_t size;
 355   char *error = zip_member_read_all (zr, member_name, &data, &size);
 356   if (!error)
 357     {
 358       fwrite (data, size, 1, stdout);
 359       free (data);
 360     }
 361   return error;
 362 }
 363
 364 static void
 365 dump_light_table (const struct output_item *item)
 366 {
 367   char *error;
 368   if (raw)
 369     error = dump_raw (item->spv_info->zip_reader,
 370                       item->spv_info->bin_member);
 371   else
 372     {
 373       struct spvlb_table *table;
 374       error = spv_read_light_table (item->spv_info->zip_reader,
 375                                     item->spv_info->bin_member, &table);
 376       if (!error)
 377         {
 378           if (sort)
 379             {
 380               qsort (table->borders->borders, table->borders->n_borders,
 381                      sizeof *table->borders->borders, compare_borders);
 382               qsort (table->cells->cells, table->cells->n_cells,
 383                      sizeof *table->cells->cells, compare_cells);
 384             }
 385           spvlb_print_table (item->spv_info->bin_member, 0, table);
 386           spvlb_free_table (table);
 387         }
 388     }
 389   if (error)
 390     {
 391       msg (ME, "%s", error);
 392       free (error);
 393     }
 394 }
 395
 396 static void
 397 run_dump_light_table (int argc UNUSED, char **argv)
 398 {
 399   if (raw && isatty (STDOUT_FILENO))
 400     error (1, 0, "not writing binary data to tty");
 401
 402   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 403   struct output_iterator iter;
 404   OUTPUT_ITEM_FOR_EACH (&iter, root)
 405     if (iter.cur->type == OUTPUT_ITEM_TABLE && !iter.cur->spv_info->xml_member)
 406       dump_light_table (iter.cur);
 407   output_item_unref (root);
 408 }
 409
 410 static void
 411 dump_legacy_data (const struct output_item *item)
 412 {
 413   char *error;
 414   if (raw)
 415     error = dump_raw (item->spv_info->zip_reader,
 416                       item->spv_info->bin_member);
 417   else
 418     {
 419       struct spv_data data;
 420       error = spv_read_legacy_data (item->spv_info->zip_reader,
 421                                     item->spv_info->bin_member, &data);
 422       if (!error)
 423         {
 424           printf ("%s:\n", item->spv_info->bin_member);
 425           spv_data_dump (&data, stdout);
 426           spv_data_uninit (&data);
 427           printf ("\n");
 428         }
 429     }
 430
 431   if (error)
 432     {
 433       msg (ME, "%s", error);
 434       free (error);
 435     }
 436 }
 437
 438 static void
 439 run_dump_legacy_data (int argc UNUSED, char **argv)
 440 {
 441   if (raw && isatty (STDOUT_FILENO))
 442     error (1, 0, "not writing binary data to tty");
 443
 444   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 445   struct output_iterator iter;
 446   OUTPUT_ITEM_FOR_EACH (&iter, root)
 447     if (iter.cur->type == OUTPUT_ITEM_TABLE
 448         && iter.cur->spv_info->xml_member
 449         && iter.cur->spv_info->bin_member)
 450       dump_legacy_data (iter.cur);
 451   output_item_unref (root);
 452 }
 453
 454 /* This is really bogus.
 455
 456    XPath doesn't have any notion of a default XML namespace, but all of the
 457    elements in the documents we're interested in have a namespace.  Thus, we'd
 458    need to require the XPath expressions to have a namespace on every single
 459    element: vis:sourceVariable, vis:graph, and so on.  That's a pain.  So,
 460    instead, we remove the default namespace from everyplace it occurs.  XPath
 461    does support the null namespace, so this allows sourceVariable, graph,
 462    etc. to work.
 463
 464    See http://plasmasturm.org/log/259/ and
 465    https://mail.gnome.org/archives/xml/2003-April/msg00144.html for more
 466    information.*/
 467 static void
 468 remove_default_xml_namespace (xmlNode *node)
 469 {
 470   if (node->ns && !node->ns->prefix)
 471     node->ns = NULL;
 472
 473   for (xmlNode *child = node->children; child; child = child->next)
 474     remove_default_xml_namespace (child);
 475 }
 476
 477 static void
 478 register_ns (xmlXPathContext *ctx, const char *prefix, const char *uri)
 479 {
 480   xmlXPathRegisterNs (ctx, CHAR_CAST (xmlChar *, prefix),
 481                       CHAR_CAST (xmlChar *, uri));
 482 }
 483
 484 static xmlXPathContext *
 485 create_xpath_context (xmlDoc *doc)
 486 {
 487   xmlXPathContext *ctx = xmlXPathNewContext (doc);
 488   register_ns (ctx, "vgr", "http://xml.spss.com/spss/viewer/viewer-graph");
 489   register_ns (ctx, "vizml", "http://xml.spss.com/visualization");
 490   register_ns (ctx, "vmd", "http://xml.spss.com/spss/viewer/viewer-model");
 491   register_ns (ctx, "vps", "http://xml.spss.com/spss/viewer/viewer-pagesetup");
 492   register_ns (ctx, "vst", "http://xml.spss.com/spss/viewer/viewer-style");
 493   register_ns (ctx, "vtb", "http://xml.spss.com/spss/viewer/viewer-table");
 494   register_ns (ctx, "vtl", "http://xml.spss.com/spss/viewer/table-looks");
 495   register_ns (ctx, "vtt", "http://xml.spss.com/spss/viewer/viewer-treemodel");
 496   register_ns (ctx, "vtx", "http://xml.spss.com/spss/viewer/viewer-text");
 497   register_ns (ctx, "xsi", "http://www.w3.org/2001/XMLSchema-instance");
 498   return ctx;
 499 }
 500
 501 static void
 502 dump_xml (int argc, char **argv, const char *member_name,
 503           char *error_s, xmlDoc *doc)
 504 {
 505   if (!error_s)
 506     {
 507       if (argc == 2)
 508         {
 509           printf ("<!-- %s -->\n", member_name);
 510           xmlElemDump (stdout, NULL, xmlDocGetRootElement (doc));
 511           putchar ('\n');
 512         }
 513       else
 514         {
 515           bool any_results = false;
 516
 517           remove_default_xml_namespace (xmlDocGetRootElement (doc));
 518           for (int i = 2; i < argc; i++)
 519             {
 520               xmlXPathContext *xpath_ctx = create_xpath_context (doc);
 521               xmlXPathSetContextNode (xmlDocGetRootElement (doc),
 522                                       xpath_ctx);
 523               xmlXPathObject *xpath_obj = xmlXPathEvalExpression(
 524                 CHAR_CAST (xmlChar *, argv[i]), xpath_ctx);
 525               if (!xpath_obj)
 526                 error (1, 0, _("%s: invalid XPath expression"), argv[i]);
 527
 528               const xmlNodeSet *nodes = xpath_obj->nodesetval;
 529               if (nodes && nodes->nodeNr > 0)
 530                 {
 531                   if (!any_results)
 532                     {
 533                       printf ("<!-- %s -->\n", member_name);
 534                       any_results = true;
 535                     }
 536                   for (size_t j = 0; j < nodes->nodeNr; j++)
 537                     {
 538                       xmlElemDump (stdout, doc, nodes->nodeTab[j]);
 539                       putchar ('\n');
 540                     }
 541                 }
 542
 543               xmlXPathFreeObject (xpath_obj);
 544               xmlXPathFreeContext (xpath_ctx);
 545             }
 546           if (any_results)
 547             putchar ('\n');
 548         }
 549       xmlFreeDoc (doc);
 550     }
 551   else
 552     {
 553       printf ("<!-- %s -->\n", member_name);
 554       msg (ME, "%s", error_s);
 555       free (error_s);
 556     }
 557 }
 558
 559 static void
 560 dump_legacy_table (int argc, char **argv, const struct output_item *item)
 561 {
 562   xmlDoc *doc;
 563   char *error_s = spv_read_xml_member (item->spv_info->zip_reader,
 564                                        item->spv_info->xml_member,
 565                                        false, "visualization", &doc);
 566   dump_xml (argc, argv, item->spv_info->xml_member, error_s, doc);
 567 }
 568
 569 static void
 570 run_dump_legacy_table (int argc, char **argv)
 571 {
 572   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 573   struct output_iterator iter;
 574   OUTPUT_ITEM_FOR_EACH (&iter, root)
 575     if (iter.cur->type == OUTPUT_ITEM_TABLE
 576         && iter.cur->spv_info->xml_member)
 577       dump_legacy_table (argc, argv, iter.cur);
 578   output_item_unref (root);
 579 }
 580
 581 static void
 582 dump_structure (int argc, char **argv, const struct output_item *item)
 583 {
 584   xmlDoc *doc;
 585   char *error_s = spv_read_xml_member (item->spv_info->zip_reader,
 586                                        item->spv_info->structure_member,
 587                                        true, "heading", &doc);
 588   dump_xml (argc, argv, item->spv_info->structure_member, error_s, doc);
 589 }
 590
 591 static void
 592 run_dump_structure (int argc, char **argv)
 593 {
 594   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 595
 596   const char *last_structure_member = NULL;
 597   struct output_iterator iter;
 598   OUTPUT_ITEM_FOR_EACH (&iter, root)
 599     {
 600       const struct output_item *item = iter.cur;
 601       if (item->spv_info->structure_member
 602           && (!last_structure_member
 603               || strcmp (item->spv_info->structure_member,
 604                          last_structure_member)))
 605         {
 606           last_structure_member = item->spv_info->structure_member;
 607           dump_structure (argc, argv, item);
 608         }
 609     }
 610   output_item_unref (root);
 611 }
 612
 613 static bool
 614 is_any_legacy (const struct output_item *item)
 615 {
 616   if (item->type == OUTPUT_ITEM_TABLE)
 617     return item->spv_info->xml_member != NULL;
 618   else if (item->type == OUTPUT_ITEM_GROUP)
 619     for (size_t i = 0; i < item->group.n_children; i++)
 620       if (is_any_legacy (item->group.children[i]))
 621         return true;
 622
 623   return false;
 624 }
 625
 626 static void
 627 run_is_legacy (int argc UNUSED, char **argv)
 628 {
 629   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 630   bool is_legacy = is_any_legacy (root);
 631   output_item_unref (root);
 632
 633   exit (is_legacy ? EXIT_SUCCESS : EXIT_FAILURE);
 634 }
 635
 636 static bool
 637 is_all_ascii (const char *s)
 638 {
 639   for (; *s; s++)
 640     if (!encoding_guess_is_ascii_text (*s))
 641       return false;
 642
 643   return true;
 644 }
 645
 646 static void
 647 dump_strings (const char *encoding, struct string_array *strings)
 648 {
 649   string_array_sort (strings);
 650   string_array_uniq (strings);
 651
 652   if (raw)
 653     {
 654       if (exclude_ascii_only || include_utf8_only)
 655         {
 656           size_t i = 0;
 657           for (size_t j = 0; j < strings->n; j++)
 658             {
 659               char *s = strings->strings[j];
 660               bool is_ascii = is_all_ascii (s);
 661               bool is_utf8 = !u8_check (CHAR_CAST (uint8_t *, s), strlen (s));
 662               if (!is_ascii && (!include_utf8_only || is_utf8))
 663                 strings->strings[i++] = s;
 664               else
 665                 free (s);
 666             }
 667           strings->n = i;
 668         }
 669       for (size_t i = 0; i < strings->n; i++)
 670         puts (strings->strings[i]);
 671     }
 672   else
 673     {
 674       size_t n_nonascii = 0;
 675       size_t n_utf8 = 0;
 676       for (size_t i = 0; i < strings->n; i++)
 677         {
 678           const char *s = strings->strings[i];
 679           if (!is_all_ascii (s))
 680             {
 681               n_nonascii++;
 682               if (!u8_check (CHAR_CAST (uint8_t *, s), strlen (s)))
 683                 n_utf8++;
 684             }
 685         }
 686       printf ("%s: %zu unique strings, %zu non-ASCII, %zu UTF-8.\n",
 687               encoding, strings->n, n_nonascii, n_utf8);
 688     }
 689 }
 690
 691 struct encoded_strings
 692   {
 693     char *encoding;
 694     struct string_array strings;
 695   };
 696
 697 struct encoded_strings_table
 698   {
 699     struct encoded_strings *es;
 700     size_t n, allocated;
 701   };
 702
 703 static void
 704 collect_strings (const struct output_item *item,
 705                  struct encoded_strings_table *t)
 706 {
 707   char *error;
 708   struct spvlb_table *table;
 709   error = spv_read_light_table (item->spv_info->zip_reader,
 710                                 item->spv_info->bin_member, &table);
 711   if (error)
 712     {
 713       msg (ME, "%s", error);
 714       free (error);
 715       return;
 716     }
 717
 718   const char *table_encoding = spvlb_table_get_encoding (table);
 719   size_t j = 0;
 720   for (j = 0; j < t->n; j++)
 721     if (!strcmp (t->es[j].encoding, table_encoding))
 722       break;
 723   if (j >= t->n)
 724     {
 725       if (t->n >= t->allocated)
 726         t->es = x2nrealloc (t->es, &t->allocated, sizeof *t->es);
 727       t->es[t->n++] = (struct encoded_strings) {
 728         .encoding = xstrdup (table_encoding),
 729         .strings = STRING_ARRAY_INITIALIZER,
 730       };
 731     }
 732   collect_spvlb_strings (table, &t->es[j].strings);
 733 }
 734
 735 static void
 736 run_strings (int argc UNUSED, char **argv)
 737 {
 738   struct output_item *root = read_and_filter_spv (argv[1], NULL);
 739
 740   struct encoded_strings_table t = { .es = NULL };
 741   struct output_iterator iter;
 742   OUTPUT_ITEM_FOR_EACH (&iter, root)
 743     {
 744       const struct output_item *item = iter.cur;
 745       if (item->type == OUTPUT_ITEM_TABLE
 746           && !item->spv_info->xml_member
 747           && item->spv_info->bin_member)
 748         collect_strings (item, &t);
 749     }
 750
 751   for (size_t i = 0; i < t.n; i++)
 752     {
 753       dump_strings (t.es[i].encoding, &t.es[i].strings);
 754       free (t.es[i].encoding);
 755       string_array_destroy (&t.es[i].strings);
 756     }
 757   free (t.es);
 758
 759   output_item_unref (root);
 760 }
 761
 762 struct command
 763   {
 764     const char *name;
 765     int min_args, max_args;
 766     void (*run) (int argc, char **argv);
 767   };
 768
 769 static const struct command commands[] =
 770   {
 771     { "detect", 1, 1, run_detect },
 772     { "dir", 1, 1, run_directory },
 773     { "convert", 2, 2, run_convert },
 774     { "get-table-look", 2, 2, run_get_table_look },
 775     { "convert-table-look", 2, 2, run_convert_table_look },
 776
 777     /* Undocumented commands. */
 778     { "dump", 1, 1, run_dump },
 779     { "dump-light-table", 1, 1, run_dump_light_table },
 780     { "dump-legacy-data", 1, 1, run_dump_legacy_data },
 781     { "dump-legacy-table", 1, INT_MAX, run_dump_legacy_table },
 782     { "dump-structure", 1, INT_MAX, run_dump_structure },
 783     { "is-legacy", 1, 1, run_is_legacy },
 784     { "strings", 1, 1, run_strings },
 785   };
 786 static const int n_commands = sizeof commands / sizeof *commands;
 787
 788 static const struct command *
 789 find_command (const char *name)
 790 {
 791   for (size_t i = 0; i < n_commands; i++)
 792     {
 793       const struct command *c = &commands[i];
 794       if (!strcmp (name, c->name))
 795         return c;
 796     }
 797   return NULL;
 798 }
 799
 800 static void
 801 emit_msg (const struct msg *m, void *aux UNUSED)
 802 {
 803   if (m->severity == MSG_S_ERROR || m->severity == MSG_S_WARNING)
 804     n_warnings++;
 805
 806   char *s = msg_to_string (m);
 807   fprintf (stderr, "%s\n", s);
 808   free (s);
 809 }
 810
 811 int
 812 main (int argc, char **argv)
 813 {
 814   set_program_name (argv[0]);
 815   msg_set_handler (&(struct msg_handler) { .output_msg = emit_msg });
 816   settings_init ();
 817   i18n_init ();
 818
 819   parse_options (argc, argv);
 820
 821   argc -= optind;
 822   argv += optind;
 823
 824   if (argc < 1)
 825     error (1, 0, _("missing command name (use --help for help)"));
 826
 827   const struct command *c = find_command (argv[0]);
 828   if (!c)
 829     error (1, 0, _("unknown command \"%s\" (use --help for help)"), argv[0]);
 830
 831   int n_args = argc - 1;
 832   if (n_args < c->min_args || n_args > c->max_args)
 833     {
 834       if (c->min_args == c->max_args)
 835         {
 836           error (1, 0,
 837                  ngettext ("\"%s\" command takes exactly %d argument",
 838                            "\"%s\" command takes exactly %d arguments",
 839                            c->min_args), c->name, c->min_args);
 840         }
 841       else if (c->max_args == INT_MAX)
 842         {
 843           error (1, 0,
 844                  ngettext ("\"%s\" command requires at least %d argument",
 845                            "\"%s\" command requires at least %d arguments",
 846                            c->min_args), c->name, c->min_args);
 847         }
 848       else
 849         {
 850           error (1, 0,
 851                  _("\"%s\" command requires between %d and %d arguments"),
 852                  c->name, c->min_args, c->max_args);
 853         }
 854     }
 855
 856   c->run (argc, argv);
 857
 858   pivot_table_look_unref (table_look);
 859   i18n_done ();
 860
 861   return n_warnings ? EXIT_FAILURE : EXIT_SUCCESS;
 862 }
 863
 864 static struct output_criteria *
 865 get_criteria (void)
 866 {
 867   if (!n_criteria || new_criteria)
 868     {
 869       new_criteria = false;
 870       if (n_criteria >= allocated_criteria)
 871         criteria = x2nrealloc (criteria, &allocated_criteria,
 872                                sizeof *criteria);
 873       criteria[n_criteria++]
 874         = (struct output_criteria) OUTPUT_CRITERIA_INITIALIZER;
 875     }
 876
 877   return &criteria[n_criteria - 1];
 878 }
 879
 880 static void
 881 parse_select (char *arg)
 882 {
 883   bool invert = arg[0] == '^';
 884   arg += invert;
 885
 886   unsigned classes = 0;
 887   for (char *token = strtok (arg, ","); token; token = strtok (NULL, ","))
 888     {
 889       if (!strcmp (arg, "all"))
 890         classes = OUTPUT_ALL_CLASSES;
 891       else if (!strcmp (arg, "help"))
 892         {
 893           puts (_("The following object classes are supported:"));
 894           for (int class = 0; class < OUTPUT_N_CLASSES; class++)
 895             printf ("- %s\n", output_item_class_to_string (class));
 896           exit (0);
 897         }
 898       else
 899         {
 900           int class = output_item_class_from_string (token);
 901           if (class == OUTPUT_N_CLASSES)
 902             error (1, 0, _("unknown object class \"%s\" (use --select=help "
 903                            "for help)"), arg);
 904           classes |= 1u << class;
 905         }
 906     }
 907
 908   struct output_criteria *c = get_criteria ();
 909   c->classes = invert ? classes ^ OUTPUT_ALL_CLASSES : classes;
 910 }
 911
 912 static struct output_criteria_match *
 913 get_criteria_match (const char **arg)
 914 {
 915   struct output_criteria *c = get_criteria ();
 916   if ((*arg)[0] == '^')
 917     {
 918       (*arg)++;
 919       return &c->exclude;
 920     }
 921   else
 922     return &c->include;
 923 }
 924
 925 static void
 926 parse_commands (const char *arg)
 927 {
 928   struct output_criteria_match *cm = get_criteria_match (&arg);
 929   string_array_parse (&cm->commands, ss_cstr (arg), ss_cstr (","));
 930 }
 931
 932 static void
 933 parse_subtypes (const char *arg)
 934 {
 935   struct output_criteria_match *cm = get_criteria_match (&arg);
 936   string_array_parse (&cm->subtypes, ss_cstr (arg), ss_cstr (","));
 937 }
 938
 939 static void
 940 parse_labels (const char *arg)
 941 {
 942   struct output_criteria_match *cm = get_criteria_match (&arg);
 943   string_array_parse (&cm->labels, ss_cstr (arg), ss_cstr (","));
 944 }
 945
 946 static void
 947 parse_instances (char *arg)
 948 {
 949   struct output_criteria *c = get_criteria ();
 950   size_t allocated_instances = c->n_instances;
 951
 952   for (char *token = strtok (arg, ","); token; token = strtok (NULL, ","))
 953     {
 954       if (c->n_instances >= allocated_instances)
 955         c->instances = x2nrealloc (c->instances, &allocated_instances,
 956                                    sizeof *c->instances);
 957
 958       c->instances[c->n_instances++] = (!strcmp (token, "last") ? -1
 959                                         : atoi (token));
 960     }
 961 }
 962
 963 static void
 964 parse_nth_commands (char *arg)
 965 {
 966   struct output_criteria *c = get_criteria ();
 967   size_t allocated_commands = c->n_commands;
 968
 969   for (char *token = strtok (arg, ","); token; token = strtok (NULL, ","))
 970     {
 971       if (c->n_commands >= allocated_commands)
 972         c->commands = x2nrealloc (c->commands, &allocated_commands,
 973                                    sizeof *c->commands);
 974
 975       c->commands[c->n_commands++] = atoi (token);
 976     }
 977 }
 978
 979 static void
 980 parse_members (const char *arg)
 981 {
 982   struct output_criteria *cm = get_criteria ();
 983   string_array_parse (&cm->members, ss_cstr (arg), ss_cstr (","));
 984 }
 985
 986 static void
 987 parse_table_look (const char *arg)
 988 {
 989   pivot_table_look_unref (table_look);
 990
 991   char *error_s = pivot_table_look_read (arg, &table_look);
 992   if (error_s)
 993     error (1, 0, "%s", error_s);
 994 }
 995
 996 static void
 997 parse_options (int argc, char *argv[])
 998 {
 999   for (;;)
1000     {
1001       enum
1002         {
1003           OPT_MEMBER_NAMES = UCHAR_MAX + 1,
1004           OPT_SHOW_HIDDEN,
1005           OPT_SELECT,
1006           OPT_COMMANDS,
1007           OPT_NTH_COMMANDS,
1008           OPT_SUBTYPES,
1009           OPT_LABELS,
1010           OPT_INSTANCES,
1011           OPT_MEMBERS,
1012           OPT_ERRORS,
1013           OPT_OR,
1014           OPT_SORT,
1015           OPT_RAW,
1016           OPT_NO_ASCII_ONLY,
1017           OPT_UTF8_ONLY,
1018           OPT_TABLE_LOOK,
1019           OPT_USE_PAGE_SETUP,
1020           OPT_HELP_DEVELOPER,
1021         };
1022       static const struct option long_options[] =
1023         {
1024           /* Input selection options. */
1025           { "show-hidden", no_argument, NULL, OPT_SHOW_HIDDEN },
1026           { "select", required_argument, NULL, OPT_SELECT },
1027           { "commands", required_argument, NULL, OPT_COMMANDS },
1028           { "nth-commands", required_argument, NULL, OPT_NTH_COMMANDS },
1029           { "subtypes", required_argument, NULL, OPT_SUBTYPES },
1030           { "labels", required_argument, NULL, OPT_LABELS },
1031           { "instances", required_argument, NULL, OPT_INSTANCES },
1032           { "members", required_argument, NULL, OPT_MEMBERS },
1033           { "errors", no_argument, NULL, OPT_ERRORS },
1034           { "or", no_argument, NULL, OPT_OR },
1035
1036           /* "dir" command options. */
1037           { "member-names", no_argument, NULL, OPT_MEMBER_NAMES },
1038
1039           /* "convert" command options. */
1040           { "force", no_argument, NULL, 'f' },
1041           { "table-look", required_argument, NULL, OPT_TABLE_LOOK },
1042           { "use-page-setup", no_argument, NULL, OPT_USE_PAGE_SETUP },
1043
1044           /* "dump-light-table" command options. */
1045           { "sort", no_argument, NULL, OPT_SORT },
1046           { "raw", no_argument, NULL, OPT_RAW },
1047
1048           /* "strings" command options. */
1049           { "no-ascii-only", no_argument, NULL, OPT_NO_ASCII_ONLY },
1050           { "utf8-only", no_argument, NULL, OPT_UTF8_ONLY },
1051
1052           { "help", no_argument, NULL, 'h' },
1053           { "help-developer", no_argument, NULL, OPT_HELP_DEVELOPER },
1054           { "version", no_argument, NULL, 'v' },
1055
1056           { NULL, 0, NULL, 0 },
1057         };
1058
1059       int c;
1060
1061       c = getopt_long (argc, argv, "O:hvf", long_options, NULL);
1062       if (c == -1)
1063         break;
1064
1065       switch (c)
1066         {
1067         case 'O':
1068           output_driver_parse_option (optarg, &output_options);
1069           break;
1070
1071         case OPT_MEMBER_NAMES:
1072           show_member_names = true;
1073           break;
1074
1075         case OPT_SHOW_HIDDEN:
1076           get_criteria ()->include_hidden = true;
1077           break;
1078
1079         case OPT_SELECT:
1080           parse_select (optarg);
1081           break;
1082
1083         case OPT_COMMANDS:
1084           parse_commands (optarg);
1085           break;
1086
1087         case OPT_NTH_COMMANDS:
1088           parse_nth_commands (optarg);
1089           break;
1090
1091         case OPT_SUBTYPES:
1092           parse_subtypes (optarg);
1093           break;
1094
1095         case OPT_LABELS:
1096           parse_labels (optarg);
1097           break;
1098
1099         case OPT_INSTANCES:
1100           parse_instances (optarg);
1101           break;
1102
1103         case OPT_MEMBERS:
1104           parse_members (optarg);
1105           break;
1106
1107         case OPT_ERRORS:
1108           get_criteria ()->error = true;
1109           break;
1110
1111         case OPT_OR:
1112           new_criteria = true;
1113           break;
1114
1115         case OPT_SORT:
1116           sort = true;
1117           break;
1118
1119         case OPT_RAW:
1120           raw = true;
1121           break;
1122
1123         case OPT_TABLE_LOOK:
1124           parse_table_look (optarg);
1125           break;
1126
1127         case OPT_USE_PAGE_SETUP:
1128           use_page_setup = true;
1129           break;
1130
1131         case OPT_NO_ASCII_ONLY:
1132           exclude_ascii_only = true;
1133           break;
1134
1135         case OPT_UTF8_ONLY:
1136           include_utf8_only = true;
1137           break;
1138
1139         case 'f':
1140           force = true;
1141           break;
1142
1143         case 'v':
1144           version_etc (stdout, "pspp-output", PACKAGE_NAME, PACKAGE_VERSION,
1145                        "Ben Pfaff", "John Darrington", NULL_SENTINEL);
1146           exit (EXIT_SUCCESS);
1147
1148         case 'h':
1149           usage ();
1150           exit (EXIT_SUCCESS);
1151
1152         case OPT_HELP_DEVELOPER:
1153           developer_usage ();
1154           exit (EXIT_SUCCESS);
1155
1156         default:
1157           exit (EXIT_FAILURE);
1158         }
1159     }
1160 }
1161
1162 static void
1163 usage (void)
1164 {
1165   struct string s = DS_EMPTY_INITIALIZER;
1166   struct string_set formats = STRING_SET_INITIALIZER(formats);
1167   output_get_supported_formats (&formats);
1168   const char *format;
1169   const struct string_set_node *node;
1170   STRING_SET_FOR_EACH (format, node, &formats)
1171     {
1172       if (!ds_is_empty (&s))
1173         ds_put_byte (&s, ' ');
1174       ds_put_cstr (&s, format);
1175     }
1176   string_set_destroy (&formats);
1177
1178   printf ("\
1179 %s, a utility for working with SPSS viewer (.spv) files.\n\
1180 Usage: %s [OPTION]... COMMAND ARG...\n\
1181 \n\
1182 The following commands are available:\n\
1183   detect FILE            Detect whether FILE is an SPV file.\n\
1184   dir FILE               List tables and other items in FILE.\n\
1185   convert SOURCE DEST    Convert .spv SOURCE to DEST.\n\
1186   get-table-look SOURCE DEST  Copies first selected TableLook into DEST\n\
1187   convert-table-look SOURCE DEST  Copies .tlo or .stt SOURCE into .stt DEST\n\
1188 \n\
1189 Input selection options for \"dir\" and \"convert\":\n\
1190   --select=CLASS...   include only some kinds of objects\n\
1191   --select=help       print known object classes\n\
1192   --commands=COMMAND...  include only specified COMMANDs\n\
1193   --nth-commands=N...  include only the Nth instance of selected commands\n\
1194   --subtypes=SUBTYPE...  include only specified SUBTYPEs of output\n\
1195   --labels=LABEL...   include only output objects with the given LABELs\n\
1196   --instances=INSTANCE...  include only the given object INSTANCEs\n\
1197   --show-hidden       include hidden output objects\n\
1198   --or                separate two sets of selection options\n\
1199 \n\
1200 \"convert\" by default infers the destination's format from its extension.\n\
1201 The known extensions are: %s\n\
1202 The following options override \"convert\" behavior:\n\
1203   -O format=FORMAT          set destination format to FORMAT\n\
1204   -O OPTION=VALUE           set output option\n\
1205   -f, --force               keep output file even given errors\n\
1206   --table-look=FILE         override tables' style with TableLook from FILE\n\
1207   --use-page-setup          use page setup from SOURCE\n\
1208 Other options:\n\
1209   --help              display this help and exit\n\
1210   --help-developer    display help for developer commands and exit\n\
1211   --version           output version information and exit\n",
1212           program_name, program_name, ds_cstr (&s));
1213   ds_destroy (&s);
1214 }
1215
1216 static void
1217 developer_usage (void)
1218 {
1219   printf ("\
1220 The following developer commands are available:\n\
1221   dump FILE              Dump pivot table structure\n\
1222   [--raw | --sort] dump-light-table FILE  Dump light tables\n\
1223   [--raw] dump-legacy-data FILE  Dump legacy table data\n\
1224   dump-legacy-table FILE [XPATH]...  Dump legacy table XML\n\
1225   dump-structure FILE [XPATH]...  Dump structure XML\n\
1226   is-legacy FILE         Exit with status 0 if any legacy table selected\n\
1227   strings FILE           Dump analysis of strings\n\
1228 \n\
1229 Additional input selection options:\n\
1230   --members=MEMBER...    include only objects with these Zip member names\n\
1231   --errors               include only objects that cannot be loaded\n\
1232 \n\
1233 Additional options for \"dir\" command:\n\
1234   --member-names         show Zip member names with objects\n\
1235 \n\
1236 Options for the \"strings\" command:\n\
1237   --raw                  Dump all (unique) strings\n\
1238   --raw --no-ascii-only  Dump all strings that contain non-ASCII characters\n\
1239   --raw --utf8-only      Dump all non-ASCII strings that are valid UTF-8\n\
1240 \n\
1241 Other options:\n\
1242   --raw                  print raw binary data instead of a parsed version\n\
1243   --sort                 sort borders and areas for shorter \"diff\" output\n");
1244 }