1 /* PSPP - a program for statistical analysis.
2 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
28 #include "data/val-type.h"
29 #include "libpspp/cast.h"
30 #include "libpspp/compiler.h"
31 #include "libpspp/float-format.h"
32 #include "libpspp/integer-format.h"
33 #include "libpspp/misc.h"
36 #include "gl/minmax.h"
37 #include "gl/progname.h"
38 #include "gl/version-etc.h"
39 #include "gl/xalloc.h"
53 const char *file_name
;
56 int n_variable_records
, n_variables
;
59 size_t n_var_widths
, allocated_var_widths
;
61 enum integer_format integer_format
;
62 enum float_format float_format
;
64 enum compression compression
;
68 static void read_header (struct sfm_reader
*);
69 static void read_variable_record (struct sfm_reader
*);
70 static void read_value_label_record (struct sfm_reader
*);
71 static void read_document_record (struct sfm_reader
*);
72 static void read_extension_record (struct sfm_reader
*);
73 static void read_machine_integer_info (struct sfm_reader
*,
74 size_t size
, size_t count
);
75 static void read_machine_float_info (struct sfm_reader
*,
76 size_t size
, size_t count
);
77 static void read_extra_product_info (struct sfm_reader
*,
78 size_t size
, size_t count
);
79 static void read_variable_sets (struct sfm_reader
*, size_t size
, size_t count
);
80 static void read_mrsets (struct sfm_reader
*, size_t size
, size_t count
);
81 static void read_display_parameters (struct sfm_reader
*,
82 size_t size
, size_t count
);
83 static void read_long_var_name_map (struct sfm_reader
*r
,
84 size_t size
, size_t count
);
85 static void read_long_string_map (struct sfm_reader
*r
,
86 size_t size
, size_t count
);
87 static void read_datafile_attributes (struct sfm_reader
*r
,
88 size_t size
, size_t count
);
89 static void read_variable_attributes (struct sfm_reader
*r
,
90 size_t size
, size_t count
);
91 static void read_ncases64 (struct sfm_reader
*, size_t size
, size_t count
);
92 static void read_character_encoding (struct sfm_reader
*r
,
93 size_t size
, size_t count
);
94 static void read_long_string_value_labels (struct sfm_reader
*r
,
95 size_t size
, size_t count
);
96 static void read_long_string_missing_values (struct sfm_reader
*r
,
97 size_t size
, size_t count
);
98 static void read_unknown_extension (struct sfm_reader
*,
99 size_t size
, size_t count
);
100 static void read_simple_compressed_data (struct sfm_reader
*, int max_cases
);
101 static void read_zlib_compressed_data (struct sfm_reader
*);
103 static struct text_record
*open_text_record (
104 struct sfm_reader
*, size_t size
, size_t count
);
105 static void close_text_record (struct text_record
*);
106 static bool read_variable_to_value_pair (struct text_record
*,
107 char **key
, char **value
);
108 static char *text_tokenize (struct text_record
*, int delimiter
);
109 static bool text_match (struct text_record
*text
, int c
);
110 static const char *text_parse_counted_string (struct text_record
*);
111 static size_t text_pos (const struct text_record
*);
112 static const char *text_get_all (const struct text_record
*);
114 static void usage (void);
115 static void sys_warn (struct sfm_reader
*, const char *, ...)
116 PRINTF_FORMAT (2, 3);
117 static void sys_error (struct sfm_reader
*, const char *, ...)
121 static void read_bytes (struct sfm_reader
*, void *, size_t);
122 static bool try_read_bytes (struct sfm_reader
*, void *, size_t);
123 static int read_int (struct sfm_reader
*);
124 static int64_t read_int64 (struct sfm_reader
*);
125 static double read_float (struct sfm_reader
*);
126 static void read_string (struct sfm_reader
*, char *, size_t);
127 static void skip_bytes (struct sfm_reader
*, size_t);
128 static void trim_spaces (char *);
130 static void print_string (const char *s
, size_t len
);
133 main (int argc
, char *argv
[])
139 set_program_name (argv
[0]);
143 static const struct option long_options
[] =
145 { "data", optional_argument
, NULL
, 'd' },
146 { "help", no_argument
, NULL
, 'h' },
147 { "version", no_argument
, NULL
, 'v' },
148 { NULL
, 0, NULL
, 0 },
153 c
= getopt_long (argc
, argv
, "d::hv", long_options
, NULL
);
160 max_cases
= optarg
? atoi (optarg
) : INT_MAX
;
164 version_etc (stdout
, "pspp-dump-sav", PACKAGE_NAME
, PACKAGE_VERSION
,
165 "Ben Pfaff", "John Darrington", NULL_SENTINEL
);
178 error (1, 0, "at least one non-option argument is required; "
179 "use --help for help");
181 for (i
= optind
; i
< argc
; i
++)
185 r
.file_name
= argv
[i
];
186 r
.file
= fopen (r
.file_name
, "rb");
188 error (EXIT_FAILURE
, errno
, "error opening `%s'", r
.file_name
);
189 r
.n_variable_records
= 0;
192 r
.allocated_var_widths
= 0;
194 r
.compression
= COMP_NONE
;
196 if (argc
- optind
> 1)
197 printf ("Reading \"%s\":\n", r
.file_name
);
200 while ((rec_type
= read_int (&r
)) != 999)
205 read_variable_record (&r
);
209 read_value_label_record (&r
);
213 sys_error (&r
, "Misplaced type 4 record.");
216 read_document_record (&r
);
220 read_extension_record (&r
);
224 sys_error (&r
, "Unrecognized record type %d.", rec_type
);
227 printf ("%08llx: end-of-dictionary record "
228 "(first byte of data at %08llx)\n",
229 (long long int) ftello (r
.file
),
230 (long long int) ftello (r
.file
) + 4);
232 if (r
.compression
== COMP_SIMPLE
)
235 read_simple_compressed_data (&r
, max_cases
);
237 else if (r
.compression
== COMP_ZLIB
)
238 read_zlib_compressed_data (&r
);
249 read_header (struct sfm_reader
*r
)
252 char eye_catcher
[61];
253 uint8_t raw_layout_code
[4];
256 int32_t weight_index
;
259 char creation_date
[10];
260 char creation_time
[9];
264 read_string (r
, rec_type
, sizeof rec_type
);
265 read_string (r
, eye_catcher
, sizeof eye_catcher
);
267 if (!strcmp ("$FL2", rec_type
))
269 else if (!strcmp ("$FL3", rec_type
))
272 sys_error (r
, "This is not an SPSS system file.");
274 /* Identify integer format. */
275 read_bytes (r
, raw_layout_code
, sizeof raw_layout_code
);
276 if ((!integer_identify (2, raw_layout_code
, sizeof raw_layout_code
,
278 && !integer_identify (3, raw_layout_code
, sizeof raw_layout_code
,
280 || (r
->integer_format
!= INTEGER_MSB_FIRST
281 && r
->integer_format
!= INTEGER_LSB_FIRST
))
282 sys_error (r
, "This is not an SPSS system file.");
283 layout_code
= integer_get (r
->integer_format
,
284 raw_layout_code
, sizeof raw_layout_code
);
286 read_int (r
); /* Nominal case size (not actually useful). */
287 compressed
= read_int (r
);
288 weight_index
= read_int (r
);
289 ncases
= read_int (r
);
294 r
->compression
= COMP_NONE
;
295 else if (compressed
== 1)
296 r
->compression
= COMP_SIMPLE
;
297 else if (compressed
!= 0)
298 sys_error (r
, "SAV file header has invalid compression value "
299 "%"PRId32
".", compressed
);
304 r
->compression
= COMP_ZLIB
;
306 sys_error (r
, "ZSAV file header has invalid compression value "
307 "%"PRId32
".", compressed
);
310 /* Identify floating-point format and obtain compression bias. */
311 read_bytes (r
, raw_bias
, sizeof raw_bias
);
312 if (float_identify (100.0, raw_bias
, sizeof raw_bias
, &r
->float_format
) == 0)
314 sys_warn (r
, "Compression bias is not the usual value of 100, or system "
315 "file uses unrecognized floating-point format.");
316 if (r
->integer_format
== INTEGER_MSB_FIRST
)
317 r
->float_format
= FLOAT_IEEE_DOUBLE_BE
;
319 r
->float_format
= FLOAT_IEEE_DOUBLE_LE
;
321 r
->bias
= float_get_double (r
->float_format
, raw_bias
);
323 read_string (r
, creation_date
, sizeof creation_date
);
324 read_string (r
, creation_time
, sizeof creation_time
);
325 read_string (r
, file_label
, sizeof file_label
);
326 trim_spaces (file_label
);
329 printf ("File header record:\n");
330 printf ("\t%17s: %s\n", "Product name", eye_catcher
);
331 printf ("\t%17s: %"PRId32
"\n", "Layout code", layout_code
);
332 printf ("\t%17s: %"PRId32
" (%s)\n", "Compressed",
334 r
->compression
== COMP_NONE
? "no compression"
335 : r
->compression
== COMP_SIMPLE
? "simple compression"
336 : r
->compression
== COMP_ZLIB
? "ZLIB compression"
338 printf ("\t%17s: %"PRId32
"\n", "Weight index", weight_index
);
339 printf ("\t%17s: %"PRId32
"\n", "Number of cases", ncases
);
340 printf ("\t%17s: %.*g\n", "Compression bias", DBL_DIG
+ 1, r
->bias
);
341 printf ("\t%17s: %s\n", "Creation date", creation_date
);
342 printf ("\t%17s: %s\n", "Creation time", creation_time
);
343 printf ("\t%17s: \"%s\"\n", "File label", file_label
);
347 format_name (int format
)
349 switch ((format
>> 16) & 0xff)
352 case 2: return "AHEX";
353 case 3: return "COMMA";
354 case 4: return "DOLLAR";
357 case 7: return "PIBHEX";
359 case 9: return "PIB";
360 case 10: return "PK";
361 case 11: return "RB";
362 case 12: return "RBHEX";
366 case 20: return "DATE";
367 case 21: return "TIME";
368 case 22: return "DATETIME";
369 case 23: return "ADATE";
370 case 24: return "JDATE";
371 case 25: return "DTIME";
372 case 26: return "WKDAY";
373 case 27: return "MONTH";
374 case 28: return "MOYR";
375 case 29: return "QYR";
376 case 30: return "WKYR";
377 case 31: return "PCT";
378 case 32: return "DOT";
379 case 33: return "CCA";
380 case 34: return "CCB";
381 case 35: return "CCC";
382 case 36: return "CCD";
383 case 37: return "CCE";
384 case 38: return "EDATE";
385 case 39: return "SDATE";
386 case 40: return "MTIME";
387 case 41: return "YMDHMS";
388 default: return "invalid";
392 /* Reads a variable (type 2) record from R and adds the
393 corresponding variable to DICT.
394 Also skips past additional variable records for long string
397 read_variable_record (struct sfm_reader
*r
)
400 int has_variable_label
;
401 int missing_value_code
;
406 printf ("%08llx: variable record #%d\n",
407 (long long int) ftello (r
->file
), ++r
->n_variable_records
);
409 width
= read_int (r
);
410 has_variable_label
= read_int (r
);
411 missing_value_code
= read_int (r
);
412 print_format
= read_int (r
);
413 write_format
= read_int (r
);
414 read_string (r
, name
, sizeof name
);
415 name
[strcspn (name
, " ")] = '\0';
420 if (r
->n_var_widths
>= r
->allocated_var_widths
)
421 r
->var_widths
= x2nrealloc (r
->var_widths
, &r
->allocated_var_widths
,
422 sizeof *r
->var_widths
);
423 r
->var_widths
[r
->n_var_widths
++] = width
;
425 printf ("\tWidth: %d (%s)\n",
428 : width
== 0 ? "numeric"
429 : "long string continuation record");
430 printf ("\tVariable label: %d\n", has_variable_label
);
431 printf ("\tMissing values code: %d (%s)\n", missing_value_code
,
432 (missing_value_code
== 0 ? "no missing values"
433 : missing_value_code
== 1 ? "one missing value"
434 : missing_value_code
== 2 ? "two missing values"
435 : missing_value_code
== 3 ? "three missing values"
436 : missing_value_code
== -2 ? "one missing value range"
437 : missing_value_code
== -3 ? "one missing value, one range"
439 printf ("\tPrint format: %06x (%s%d.%d)\n",
440 print_format
, format_name (print_format
),
441 (print_format
>> 8) & 0xff, print_format
& 0xff);
442 printf ("\tWrite format: %06x (%s%d.%d)\n",
443 write_format
, format_name (write_format
),
444 (write_format
>> 8) & 0xff, write_format
& 0xff);
445 printf ("\tName: %s\n", name
);
447 /* Get variable label, if any. */
448 if (has_variable_label
!= 0 && has_variable_label
!= 1)
449 sys_error (r
, "Variable label indicator field is not 0 or 1.");
450 if (has_variable_label
== 1)
452 long long int offset
= ftello (r
->file
);
453 enum { MAX_LABEL_LEN
= 65536 };
455 size_t len
= read_int (r
);
456 size_t read_len
= MIN (MAX_LABEL_LEN
, len
);
457 char *label
= xmalloc (read_len
+ 1);
458 read_string (r
, label
, read_len
+ 1);
459 printf("\t%08llx Variable label: \"%s\"\n", offset
, label
);
462 /* Skip label padding up to multiple of 4 bytes. */
463 skip_bytes (r
, ROUND_UP (len
, 4) - len
);
466 /* Set missing values. */
467 if (missing_value_code
!= 0)
471 printf ("\t%08llx Missing values:", (long long int) ftello (r
->file
));
474 if (missing_value_code
< -3 || missing_value_code
> 3
475 || missing_value_code
== -1)
476 sys_error (r
, "Numeric missing value indicator field is not "
477 "-3, -2, 0, 1, 2, or 3.");
478 if (missing_value_code
< 0)
480 double low
= read_float (r
);
481 double high
= read_float (r
);
482 printf (" %.*g...%.*g", DBL_DIG
+ 1, low
, DBL_DIG
+ 1, high
);
483 missing_value_code
= -missing_value_code
- 2;
485 for (i
= 0; i
< missing_value_code
; i
++)
486 printf (" %.*g", DBL_DIG
+ 1, read_float (r
));
490 if (missing_value_code
< 1 || missing_value_code
> 3)
491 sys_error (r
, "String missing value indicator field is not "
493 for (i
= 0; i
< missing_value_code
; i
++)
496 read_string (r
, string
, sizeof string
);
497 printf (" \"%s\"", string
);
505 print_untyped_value (struct sfm_reader
*r
, char raw_value
[8])
510 value
= float_get_double (r
->float_format
, raw_value
);
511 for (n_printable
= 0; n_printable
< 8; n_printable
++)
512 if (!isprint (raw_value
[n_printable
]))
515 printf ("%.*g/\"%.*s\"", DBL_DIG
+ 1, value
, n_printable
, raw_value
);
518 /* Reads value labels from sysfile R and inserts them into the
519 associated dictionary. */
521 read_value_label_record (struct sfm_reader
*r
)
523 int n_labels
, n_vars
;
526 printf ("%08llx: value labels record\n", (long long int) ftello (r
->file
));
528 /* Read number of labels. */
529 n_labels
= read_int (r
);
530 for (i
= 0; i
< n_labels
; i
++)
533 unsigned char label_len
;
537 read_bytes (r
, raw_value
, sizeof raw_value
);
539 /* Read label length. */
540 read_bytes (r
, &label_len
, sizeof label_len
);
541 padded_len
= ROUND_UP (label_len
+ 1, 8);
543 /* Read label, padding. */
544 read_bytes (r
, label
, padded_len
- 1);
545 label
[label_len
] = 0;
548 print_untyped_value (r
, raw_value
);
549 printf (": \"%s\"\n", label
);
552 /* Now, read the type 4 record that has the list of variables
553 to which the value labels are to be applied. */
555 /* Read record type of type 4 record. */
556 if (read_int (r
) != 4)
557 sys_error (r
, "Variable index record (type 4) does not immediately "
558 "follow value label record (type 3) as it should.");
560 /* Read number of variables associated with value label from type 4
562 printf ("\t%08llx: apply to variables", (long long int) ftello (r
->file
));
563 n_vars
= read_int (r
);
564 for (i
= 0; i
< n_vars
; i
++)
565 printf (" #%d", read_int (r
));
570 read_document_record (struct sfm_reader
*r
)
575 printf ("%08llx: document record\n", (long long int) ftello (r
->file
));
576 n_lines
= read_int (r
);
577 printf ("\t%d lines of documents\n", n_lines
);
579 for (i
= 0; i
< n_lines
; i
++)
582 printf ("\t%08llx: ", (long long int) ftello (r
->file
));
583 read_string (r
, line
, sizeof line
);
585 printf ("line %d: \"%s\"\n", i
, line
);
590 read_extension_record (struct sfm_reader
*r
)
592 long long int offset
= ftello (r
->file
);
593 int subtype
= read_int (r
);
594 size_t size
= read_int (r
);
595 size_t count
= read_int (r
);
596 size_t bytes
= size
* count
;
598 printf ("%08llx: Record 7, subtype %d, size=%zu, count=%zu\n",
599 offset
, subtype
, size
, count
);
604 read_machine_integer_info (r
, size
, count
);
608 read_machine_float_info (r
, size
, count
);
612 read_variable_sets (r
, size
, count
);
616 /* DATE variable information. We don't use it yet, but we
622 read_mrsets (r
, size
, count
);
626 read_extra_product_info (r
, size
, count
);
630 read_display_parameters (r
, size
, count
);
634 read_long_var_name_map (r
, size
, count
);
638 read_long_string_map (r
, size
, count
);
642 read_ncases64 (r
, size
, count
);
646 read_datafile_attributes (r
, size
, count
);
650 read_variable_attributes (r
, size
, count
);
654 read_character_encoding (r
, size
, count
);
658 read_long_string_value_labels (r
, size
, count
);
662 read_long_string_missing_values (r
, size
, count
);
666 sys_warn (r
, "Unrecognized record type 7, subtype %d.", subtype
);
667 read_unknown_extension (r
, size
, count
);
671 skip_bytes (r
, bytes
);
675 read_machine_integer_info (struct sfm_reader
*r
, size_t size
, size_t count
)
677 long long int offset
= ftello (r
->file
);
678 int version_major
= read_int (r
);
679 int version_minor
= read_int (r
);
680 int version_revision
= read_int (r
);
681 int machine_code
= read_int (r
);
682 int float_representation
= read_int (r
);
683 int compression_code
= read_int (r
);
684 int integer_representation
= read_int (r
);
685 int character_code
= read_int (r
);
687 printf ("%08llx: machine integer info\n", offset
);
688 if (size
!= 4 || count
!= 8)
689 sys_error (r
, "Bad size (%zu) or count (%zu) field on record type 7, "
690 "subtype 3.", size
, count
);
692 printf ("\tVersion: %d.%d.%d\n",
693 version_major
, version_minor
, version_revision
);
694 printf ("\tMachine code: %d\n", machine_code
);
695 printf ("\tFloating point representation: %d (%s)\n",
696 float_representation
,
697 float_representation
== 1 ? "IEEE 754"
698 : float_representation
== 2 ? "IBM 370"
699 : float_representation
== 3 ? "DEC VAX"
701 printf ("\tCompression code: %d\n", compression_code
);
702 printf ("\tEndianness: %d (%s)\n", integer_representation
,
703 integer_representation
== 1 ? "big"
704 : integer_representation
== 2 ? "little" : "unknown");
705 printf ("\tCharacter code: %d\n", character_code
);
708 /* Read record type 7, subtype 4. */
710 read_machine_float_info (struct sfm_reader
*r
, size_t size
, size_t count
)
712 long long int offset
= ftello (r
->file
);
713 double sysmis
= read_float (r
);
714 double highest
= read_float (r
);
715 double lowest
= read_float (r
);
717 printf ("%08llx: machine float info\n", offset
);
718 if (size
!= 8 || count
!= 3)
719 sys_error (r
, "Bad size (%zu) or count (%zu) on extension 4.",
722 printf ("\tsysmis: %.*g (%a)\n", DBL_DIG
+ 1, sysmis
, sysmis
);
723 if (sysmis
!= SYSMIS
)
724 sys_warn (r
, "File specifies unexpected value %.*g (%a) as %s.",
725 DBL_DIG
+ 1, sysmis
, sysmis
, "SYSMIS");
727 printf ("\thighest: %.*g (%a)\n", DBL_DIG
+ 1, highest
, highest
);
728 if (highest
!= HIGHEST
)
729 sys_warn (r
, "File specifies unexpected value %.*g (%a) as %s.",
730 DBL_DIG
+ 1, highest
, highest
, "HIGHEST");
732 printf ("\tlowest: %.*g (%a)\n", DBL_DIG
+ 1, lowest
, lowest
);
733 if (lowest
!= LOWEST
&& lowest
!= SYSMIS
)
734 sys_warn (r
, "File specifies unexpected value %.*g (%a) as %s.",
735 DBL_DIG
+ 1, lowest
, lowest
, "LOWEST");
738 /* Read record type 7, subtype 5. */
740 read_variable_sets (struct sfm_reader
*r
, size_t size
, size_t count
)
742 printf ("%08llx: variable sets\n", (long long int) ftello (r
->file
));
743 struct text_record
*text
= open_text_record (r
, size
, count
);
746 while (text_match (text
, '\n'))
749 const char *set
= text_tokenize (text
, '=');
753 /* Always present even for an empty set. */
754 text_match (text
, ' ');
756 char *variables
= text_tokenize (text
, '\n');
758 printf ("\tset \"%s\" is empty\n", set
);
761 size_t length
= strlen (variables
);
762 if (variables
[length
- 1] == '\r')
763 variables
[length
- 1] = '\0';
764 printf ("\tset \"%s\" contains \"%s\"\n", set
, variables
);
767 close_text_record (text
);
771 read_extra_product_info (struct sfm_reader
*r
,
772 size_t size
, size_t count
)
774 struct text_record
*text
;
777 printf ("%08llx: extra product info\n", (long long int) ftello (r
->file
));
778 text
= open_text_record (r
, size
, count
);
779 s
= text_get_all (text
);
780 print_string (s
, strlen (s
));
781 close_text_record (text
);
784 /* Read record type 7, subtype 7. */
786 read_mrsets (struct sfm_reader
*r
, size_t size
, size_t count
)
788 struct text_record
*text
;
790 printf ("%08llx: multiple response sets\n",
791 (long long int) ftello (r
->file
));
792 text
= open_text_record (r
, size
, count
);
796 enum { MRSET_MC
, MRSET_MD
} type
;
797 bool cat_label_from_counted_values
= false;
798 bool label_from_var_label
= false;
801 const char *variables
;
803 while (text_match (text
, '\n'))
806 name
= text_tokenize (text
, '=');
810 if (text_match (text
, 'C'))
814 if (!text_match (text
, ' '))
816 sys_warn (r
, "missing space following 'C' at offset %zu "
817 "in mrsets record", text_pos (text
));
821 else if (text_match (text
, 'D'))
825 else if (text_match (text
, 'E'))
830 cat_label_from_counted_values
= true;
832 if (!text_match (text
, ' '))
834 sys_warn (r
, "Missing space following `%c' at offset %zu "
835 "in MRSETS record", 'E', text_pos (text
));
839 number
= text_tokenize (text
, ' ');
841 sys_warn (r
, "Missing label source value "
842 "following `E' at offset %zu in MRSETS record",
844 else if (!strcmp (number
, "11"))
845 label_from_var_label
= true;
846 else if (strcmp (number
, "1"))
847 sys_warn (r
, "Unexpected label source value `%s' "
848 "following `E' at offset %zu in MRSETS record",
849 number
, text_pos (text
));
854 sys_warn (r
, "missing `C', `D', or `E' at offset %zu "
855 "in mrsets record", text_pos (text
));
859 if (type
== MRSET_MD
)
861 counted
= text_parse_counted_string (text
);
866 label
= text_parse_counted_string (text
);
870 variables
= text_tokenize (text
, '\n');
872 printf ("\t\"%s\": multiple %s set",
873 name
, type
== MRSET_MC
? "category" : "dichotomy");
875 printf (", counted value \"%s\"", counted
);
876 if (cat_label_from_counted_values
)
877 printf (", category labels from counted values");
878 if (label
[0] != '\0')
879 printf (", label \"%s\"", label
);
880 if (label_from_var_label
)
881 printf (", label from variable label");
882 if (variables
!= NULL
)
883 printf(", variables \"%s\"\n", variables
);
885 printf(", no variables\n");
887 close_text_record (text
);
890 /* Read record type 7, subtype 11. */
892 read_display_parameters (struct sfm_reader
*r
, size_t size
, size_t count
)
898 printf ("%08llx: variable display parameters\n",
899 (long long int) ftello (r
->file
));
902 sys_warn (r
, "Bad size %zu on extension 11.", size
);
903 skip_bytes (r
, size
* count
);
907 n_vars
= r
->n_variables
;
908 if (count
== 3 * n_vars
)
909 includes_width
= true;
910 else if (count
== 2 * n_vars
)
911 includes_width
= false;
914 sys_warn (r
, "Extension 11 has bad count %zu (for %zu variables.",
916 skip_bytes (r
, size
* count
);
920 for (i
= 0; i
< n_vars
; ++i
)
922 int measure
= read_int (r
);
923 int width
= includes_width
? read_int (r
) : 0;
924 int align
= read_int (r
);
926 printf ("\tVar #%zu: measure=%d (%s)", i
, measure
,
927 (measure
== 1 ? "nominal"
928 : measure
== 2 ? "ordinal"
929 : measure
== 3 ? "scale"
932 printf (", width=%d", width
);
933 printf (", align=%d (%s)\n", align
,
935 : align
== 1 ? "right"
936 : align
== 2 ? "centre"
941 /* Reads record type 7, subtype 13, which gives the long name
942 that corresponds to each short name. */
944 read_long_var_name_map (struct sfm_reader
*r
, size_t size
, size_t count
)
946 struct text_record
*text
;
950 printf ("%08llx: long variable names (short => long)\n",
951 (long long int) ftello (r
->file
));
952 text
= open_text_record (r
, size
, count
);
953 while (read_variable_to_value_pair (text
, &var
, &long_name
))
954 printf ("\t%s => %s\n", var
, long_name
);
955 close_text_record (text
);
958 /* Reads record type 7, subtype 14, which gives the real length
959 of each very long string. Rearranges DICT accordingly. */
961 read_long_string_map (struct sfm_reader
*r
, size_t size
, size_t count
)
963 struct text_record
*text
;
967 printf ("%08llx: very long strings (variable => length)\n",
968 (long long int) ftello (r
->file
));
969 text
= open_text_record (r
, size
, count
);
970 while (read_variable_to_value_pair (text
, &var
, &length_s
))
971 printf ("\t%s => %d\n", var
, atoi (length_s
));
972 close_text_record (text
);
976 read_attributes (struct sfm_reader
*r
, struct text_record
*text
,
977 const char *variable
)
984 key
= text_tokenize (text
, '(');
988 for (index
= 1; ; index
++)
990 /* Parse the value. */
991 const char *value
= text_tokenize (text
, '\n');
994 sys_warn (r
, "%s: Error parsing attribute value %s[%d]",
995 variable
, key
, index
);
998 if (strlen (value
) < 2
999 || value
[0] != '\'' || value
[strlen (value
) - 1] != '\'')
1000 sys_warn (r
, "%s: Attribute value %s[%d] is not quoted: %s",
1001 variable
, key
, index
, value
);
1003 printf ("\t%s: %s[%d] = \"%.*s\"\n",
1004 variable
, key
, index
, (int) strlen (value
) - 2, value
+ 1);
1006 /* Was this the last value for this attribute? */
1007 if (text_match (text
, ')'))
1011 if (text_match (text
, '/'))
1016 /* Read extended number of cases record. */
1018 read_ncases64 (struct sfm_reader
*r
, size_t size
, size_t count
)
1020 int64_t unknown
, ncases64
;
1024 sys_warn (r
, "Bad size %zu for extended number of cases.", size
);
1025 skip_bytes (r
, size
* count
);
1030 sys_warn (r
, "Bad count %zu for extended number of cases.", size
);
1031 skip_bytes (r
, size
* count
);
1034 unknown
= read_int64 (r
);
1035 ncases64
= read_int64 (r
);
1036 printf ("%08llx: extended number of cases: "
1037 "unknown=%"PRId64
", ncases64=%"PRId64
"\n",
1038 (long long int) ftello (r
->file
), unknown
, ncases64
);
1042 read_datafile_attributes (struct sfm_reader
*r
, size_t size
, size_t count
)
1044 struct text_record
*text
;
1046 printf ("%08llx: datafile attributes\n", (long long int) ftello (r
->file
));
1047 text
= open_text_record (r
, size
, count
);
1048 read_attributes (r
, text
, "datafile");
1049 close_text_record (text
);
1053 read_character_encoding (struct sfm_reader
*r
, size_t size
, size_t count
)
1055 long long int posn
= ftello (r
->file
);
1056 char *encoding
= xcalloc (size
, count
+ 1);
1057 read_string (r
, encoding
, count
+ 1);
1059 printf ("%08llx: Character Encoding: %s\n", posn
, encoding
);
1065 read_long_string_value_labels (struct sfm_reader
*r
, size_t size
, size_t count
)
1067 long long int start
= ftello (r
->file
);
1069 printf ("%08llx: long string value labels\n", start
);
1070 while (ftello (r
->file
) - start
< size
* count
)
1072 long long posn
= ftello (r
->file
);
1073 char var_name
[ID_MAX_LEN
+ 1];
1079 /* Read variable name. */
1080 var_name_len
= read_int (r
);
1081 if (var_name_len
> ID_MAX_LEN
)
1082 sys_error (r
, "Variable name length in long string value label "
1083 "record (%d) exceeds %d-byte limit.",
1084 var_name_len
, ID_MAX_LEN
);
1085 read_string (r
, var_name
, var_name_len
+ 1);
1087 /* Read width, number of values. */
1088 width
= read_int (r
);
1089 n_values
= read_int (r
);
1091 printf ("\t%08llx: %s, width %d, %d values\n",
1092 posn
, var_name
, width
, n_values
);
1095 for (i
= 0; i
< n_values
; i
++)
1103 posn
= ftello (r
->file
);
1106 value_length
= read_int (r
);
1107 value
= xmalloc (value_length
+ 1);
1108 read_string (r
, value
, value_length
+ 1);
1111 label_length
= read_int (r
);
1112 label
= xmalloc (label_length
+ 1);
1113 read_string (r
, label
, label_length
+ 1);
1115 printf ("\t\t%08llx: \"%s\" (%d bytes) => \"%s\" (%d bytes)\n",
1116 posn
, value
, value_length
, label
, label_length
);
1125 read_long_string_missing_values (struct sfm_reader
*r
,
1126 size_t size
, size_t count
)
1128 long long int start
= ftello (r
->file
);
1130 printf ("%08llx: long string missing values\n", start
);
1131 while (ftello (r
->file
) - start
< size
* count
)
1133 long long posn
= ftello (r
->file
);
1135 /* Read variable name. */
1136 int var_name_len
= read_int (r
);
1137 if (var_name_len
> ID_MAX_LEN
)
1138 sys_error (r
, "Variable name length in long string value label "
1139 "record (%d) exceeds %d-byte limit.",
1140 var_name_len
, ID_MAX_LEN
);
1141 char var_name
[ID_MAX_LEN
+ 1];
1142 read_string (r
, var_name
, var_name_len
+ 1);
1144 /* Read number of values. */
1145 uint8_t n_missing_values
;
1146 read_bytes (r
, &n_missing_values
, 1);
1148 /* Read value length. */
1149 int value_length
= read_int (r
);
1151 printf ("\t%08llx: %s, %d missing values, each %d bytes:",
1152 posn
, var_name
, n_missing_values
, value_length
);
1155 for (int i
= 0; i
< n_missing_values
; i
++)
1157 posn
= ftello (r
->file
);
1160 char *value
= xmalloc (value_length
+ 1);
1161 read_string (r
, value
, value_length
+ 1);
1163 printf (" \"%s\"", value
);
1172 hex_dump (size_t offset
, const void *buffer_
, size_t buffer_size
)
1174 const uint8_t *buffer
= buffer_
;
1176 while (buffer_size
> 0)
1178 size_t n
= MIN (buffer_size
, 16);
1181 printf ("%04zx", offset
);
1182 for (i
= 0; i
< 16; i
++)
1185 printf ("%c%02x", i
== 8 ? '-' : ' ', buffer
[i
]);
1191 for (i
= 0; i
< 16; i
++)
1193 unsigned char c
= i
< n
? buffer
[i
] : ' ';
1194 putchar (isprint (c
) ? c
: '.');
1204 /* Reads and prints any type 7 record that we don't understand. */
1206 read_unknown_extension (struct sfm_reader
*r
, size_t size
, size_t count
)
1208 unsigned char *buffer
;
1211 if (size
== 0 || count
> 65536 / size
)
1212 skip_bytes (r
, size
* count
);
1215 buffer
= xmalloc (size
);
1216 for (i
= 0; i
< count
; i
++)
1218 read_bytes (r
, buffer
, size
);
1219 hex_dump (i
* size
, buffer
, size
);
1225 buffer
= xmalloc (count
);
1226 read_bytes (r
, buffer
, count
);
1227 print_string (CHAR_CAST (char *, buffer
), count
);
1233 read_variable_attributes (struct sfm_reader
*r
, size_t size
, size_t count
)
1235 struct text_record
*text
;
1237 printf ("%08llx: variable attributes\n", (long long int) ftello (r
->file
));
1238 text
= open_text_record (r
, size
, count
);
1241 const char *variable
= text_tokenize (text
, ':');
1242 if (variable
== NULL
|| !read_attributes (r
, text
, variable
))
1245 close_text_record (text
);
1249 read_simple_compressed_data (struct sfm_reader
*r
, int max_cases
)
1251 enum { N_OPCODES
= 8 };
1252 uint8_t opcodes
[N_OPCODES
];
1253 long long int opcode_ofs
;
1259 printf ("\n%08llx: compressed data:\n", (long long int) ftello (r
->file
));
1261 opcode_idx
= N_OPCODES
;
1264 for (case_num
= 0; case_num
< max_cases
; case_num
++)
1266 printf ("%08llx: case %d's uncompressible data begins\n",
1267 (long long int) ftello (r
->file
), case_num
);
1268 for (i
= 0; i
< r
->n_var_widths
;)
1270 int width
= r
->var_widths
[i
];
1274 if (opcode_idx
>= N_OPCODES
)
1276 opcode_ofs
= ftello (r
->file
);
1279 if (!try_read_bytes (r
, opcodes
, 8))
1283 read_bytes (r
, opcodes
, 8);
1286 opcode
= opcodes
[opcode_idx
];
1287 printf ("%08llx: variable %d: opcode %d: ",
1288 opcode_ofs
+ opcode_idx
, i
, opcode
);
1293 printf ("%.*g", DBL_DIG
+ 1, opcode
- r
->bias
);
1295 printf (", but this is a string variable (width=%d)", width
);
1301 printf ("ignored padding\n");
1305 printf ("end of data\n");
1309 read_bytes (r
, raw_value
, 8);
1310 printf ("uncompressible data: ");
1311 print_untyped_value (r
, raw_value
);
1319 printf (", but this is a numeric variable");
1327 printf (", but this is a string variable (width=%d)", width
);
1339 read_zlib_compressed_data (struct sfm_reader
*r
)
1342 long long int this_ofs
, next_ofs
, next_len
;
1343 long long int bias
, zero
;
1344 long long int expected_uncmp_ofs
, expected_cmp_ofs
;
1345 unsigned int block_size
, n_blocks
;
1349 ofs
= ftello (r
->file
);
1350 printf ("\n%08llx: ZLIB compressed data header:\n", ofs
);
1352 this_ofs
= read_int64 (r
);
1353 next_ofs
= read_int64 (r
);
1354 next_len
= read_int64 (r
);
1356 printf ("\tzheader_ofs: 0x%llx\n", this_ofs
);
1357 if (this_ofs
!= ofs
)
1358 printf ("\t\t(Expected 0x%llx.)\n", ofs
);
1359 printf ("\tztrailer_ofs: 0x%llx\n", next_ofs
);
1360 printf ("\tztrailer_len: %lld\n", next_len
);
1361 if (next_len
< 24 || next_len
% 24)
1362 printf ("\t\t(Trailer length is not a positive multiple of 24.)\n");
1364 printf ("\n%08llx: 0x%llx bytes of ZLIB compressed data\n",
1365 ofs
+ 8 * 3, next_ofs
- (ofs
+ 8 * 3));
1367 skip_bytes (r
, next_ofs
- (ofs
+ 8 * 3));
1369 printf ("\n%08llx: ZLIB trailer fixed header:\n", next_ofs
);
1370 bias
= read_int64 (r
);
1371 zero
= read_int64 (r
);
1372 block_size
= read_int (r
);
1373 n_blocks
= read_int (r
);
1374 printf ("\tbias: %lld\n", bias
);
1375 printf ("\tzero: 0x%llx\n", zero
);
1377 printf ("\t\t(Expected 0.)\n");
1378 printf ("\tblock_size: 0x%x\n", block_size
);
1379 if (block_size
!= 0x3ff000)
1380 printf ("\t\t(Expected 0x3ff000.)\n");
1381 printf ("\tn_blocks: %u\n", n_blocks
);
1382 if (n_blocks
!= next_len
/ 24 - 1)
1383 printf ("\t\t(Expected %llu.)\n", next_len
/ 24 - 1);
1385 expected_uncmp_ofs
= ofs
;
1386 expected_cmp_ofs
= ofs
+ 24;
1387 for (i
= 0; i
< n_blocks
; i
++)
1389 long long int blockinfo_ofs
= ftello (r
->file
);
1390 unsigned long long int uncompressed_ofs
= read_int64 (r
);
1391 unsigned long long int compressed_ofs
= read_int64 (r
);
1392 unsigned int uncompressed_size
= read_int (r
);
1393 unsigned int compressed_size
= read_int (r
);
1395 printf ("\n%08llx: ZLIB block descriptor %d\n", blockinfo_ofs
, i
+ 1);
1397 printf ("\tuncompressed_ofs: 0x%llx\n", uncompressed_ofs
);
1398 if (uncompressed_ofs
!= expected_uncmp_ofs
)
1399 printf ("\t\t(Expected 0x%llx.)\n", ofs
);
1401 printf ("\tcompressed_ofs: 0x%llx\n", compressed_ofs
);
1402 if (compressed_ofs
!= expected_cmp_ofs
)
1403 printf ("\t\t(Expected 0x%llx.)\n", ofs
+ 24);
1405 printf ("\tuncompressed_size: 0x%x\n", uncompressed_size
);
1406 if (i
< n_blocks
- 1 && uncompressed_size
!= block_size
)
1407 printf ("\t\t(Expected 0x%x.)\n", block_size
);
1409 printf ("\tcompressed_size: 0x%x\n", compressed_size
);
1410 if (i
== n_blocks
- 1 && compressed_ofs
+ compressed_size
!= next_ofs
)
1411 printf ("\t\t(This was expected to be 0x%llx.)\n",
1412 next_ofs
- compressed_size
);
1414 expected_uncmp_ofs
+= uncompressed_size
;
1415 expected_cmp_ofs
+= compressed_size
;
1419 /* Helpers for reading records that consist of structured text
1425 struct sfm_reader
*reader
; /* Reader. */
1426 char *buffer
; /* Record contents. */
1427 size_t size
; /* Size of buffer. */
1428 size_t pos
; /* Current position in buffer. */
1431 /* Reads SIZE * COUNT bytes into a text record for R,
1432 and returns the new text record. */
1433 static struct text_record
*
1434 open_text_record (struct sfm_reader
*r
, size_t size
, size_t count
)
1436 struct text_record
*text
= xmalloc (sizeof *text
);
1438 if (size_overflow_p (xsum (1, xtimes (size
, count
))))
1439 sys_error (r
, "Extension record too large.");
1441 size_t n_bytes
= size
* count
;
1442 char *buffer
= xmalloc (n_bytes
+ 1);
1443 read_bytes (r
, buffer
, n_bytes
);
1444 buffer
[n_bytes
] = '\0';
1446 text
->buffer
= buffer
;
1447 text
->size
= n_bytes
;
1452 /* Closes TEXT and frees its storage.
1453 Not really needed, because the pool will free the text record anyway,
1454 but can be used to free it earlier. */
1456 close_text_record (struct text_record
*text
)
1458 free (text
->buffer
);
1463 text_tokenize (struct text_record
*text
, int delimiter
)
1465 size_t start
= text
->pos
;
1466 while (text
->pos
< text
->size
1467 && text
->buffer
[text
->pos
] != delimiter
1468 && text
->buffer
[text
->pos
] != '\0')
1470 if (start
== text
->pos
)
1472 text
->buffer
[text
->pos
++] = '\0';
1473 return &text
->buffer
[start
];
1477 text_match (struct text_record
*text
, int c
)
1479 if (text
->pos
< text
->size
&& text
->buffer
[text
->pos
] == c
)
1488 /* Reads an integer value expressed in decimal, then a space, then a string that
1489 consists of exactly as many bytes as specified by the integer, then a space,
1490 from TEXT. Returns the string, null-terminated, as a subset of TEXT's
1491 buffer (so the caller should not free the string). */
1493 text_parse_counted_string (struct text_record
*text
)
1501 while (isdigit ((unsigned char) text
->buffer
[text
->pos
]))
1502 n
= (n
* 10) + (text
->buffer
[text
->pos
++] - '0');
1503 if (start
== text
->pos
)
1505 sys_error (text
->reader
, "expecting digit at offset %zu in record",
1510 if (!text_match (text
, ' '))
1512 sys_error (text
->reader
, "expecting space at offset %zu in record",
1517 if (text
->pos
+ n
> text
->size
)
1519 sys_error (text
->reader
, "%zu-byte string starting at offset %zu "
1520 "exceeds record length %zu", n
, text
->pos
, text
->size
);
1524 s
= &text
->buffer
[text
->pos
];
1527 sys_error (text
->reader
, "expecting space at offset %zu following "
1528 "%zu-byte string", text
->pos
+ n
, n
);
1536 /* Reads a variable=value pair from TEXT.
1537 Looks up the variable in DICT and stores it into *VAR.
1538 Stores a null-terminated value into *VALUE. */
1540 read_variable_to_value_pair (struct text_record
*text
,
1541 char **key
, char **value
)
1543 *key
= text_tokenize (text
, '=');
1544 *value
= text_tokenize (text
, '\t');
1545 if (!*key
|| !*value
)
1548 while (text
->pos
< text
->size
1549 && (text
->buffer
[text
->pos
] == '\t'
1550 || text
->buffer
[text
->pos
] == '\0'))
1555 /* Returns the current byte offset inside the TEXT's string. */
1557 text_pos (const struct text_record
*text
)
1563 text_get_all (const struct text_record
*text
)
1565 return text
->buffer
;
1572 %s, a utility for dissecting system files.\n\
1573 Usage: %s [OPTION]... SYSFILE...\n\
1574 where each SYSFILE is the name of a system file.\n\
1577 --data[=MAXCASES] print (up to MAXCASES cases of) compressed data\n\
1578 --help display this help and exit\n\
1579 --version output version information and exit\n",
1580 program_name
, program_name
);
1583 /* Displays a corruption message. */
1585 sys_msg (struct sfm_reader
*r
, const char *format
, va_list args
)
1587 printf ("\"%s\" near offset 0x%llx: ",
1588 r
->file_name
, (long long int) ftello (r
->file
));
1589 vprintf (format
, args
);
1593 /* Displays a warning for the current file position. */
1595 sys_warn (struct sfm_reader
*r
, const char *format
, ...)
1599 va_start (args
, format
);
1600 sys_msg (r
, format
, args
);
1604 /* Displays an error for the current file position,
1605 marks it as in an error state,
1606 and aborts reading it using longjmp. */
1608 sys_error (struct sfm_reader
*r
, const char *format
, ...)
1612 va_start (args
, format
);
1613 sys_msg (r
, format
, args
);
1616 exit (EXIT_FAILURE
);
1619 /* Reads BYTE_CNT bytes into BUF.
1620 Returns true if exactly BYTE_CNT bytes are successfully read.
1621 Aborts if an I/O error or a partial read occurs.
1622 If EOF_IS_OK, then an immediate end-of-file causes false to be
1623 returned; otherwise, immediate end-of-file causes an abort
1626 read_bytes_internal (struct sfm_reader
*r
, bool eof_is_ok
,
1627 void *buf
, size_t n_bytes
)
1629 size_t bytes_read
= fread (buf
, 1, n_bytes
, r
->file
);
1630 if (bytes_read
== n_bytes
)
1632 else if (ferror (r
->file
))
1633 sys_error (r
, "System error: %s.", strerror (errno
));
1634 else if (!eof_is_ok
|| bytes_read
!= 0)
1635 sys_error (r
, "Unexpected end of file.");
1640 /* Reads BYTE_CNT into BUF.
1641 Aborts upon I/O error or if end-of-file is encountered. */
1643 read_bytes (struct sfm_reader
*r
, void *buf
, size_t n_bytes
)
1645 read_bytes_internal (r
, false, buf
, n_bytes
);
1648 /* Reads BYTE_CNT bytes into BUF.
1649 Returns true if exactly BYTE_CNT bytes are successfully read.
1650 Returns false if an immediate end-of-file is encountered.
1651 Aborts if an I/O error or a partial read occurs. */
1653 try_read_bytes (struct sfm_reader
*r
, void *buf
, size_t n_bytes
)
1655 return read_bytes_internal (r
, true, buf
, n_bytes
);
1658 /* Reads a 32-bit signed integer from R and returns its value in
1661 read_int (struct sfm_reader
*r
)
1664 read_bytes (r
, integer
, sizeof integer
);
1665 return integer_get (r
->integer_format
, integer
, sizeof integer
);
1668 /* Reads a 64-bit signed integer from R and returns its value in
1671 read_int64 (struct sfm_reader
*r
)
1674 read_bytes (r
, integer
, sizeof integer
);
1675 return integer_get (r
->integer_format
, integer
, sizeof integer
);
1678 /* Reads a 64-bit floating-point number from R and returns its
1679 value in host format. */
1681 read_float (struct sfm_reader
*r
)
1684 read_bytes (r
, number
, sizeof number
);
1685 return float_get_double (r
->float_format
, number
);
1688 /* Reads exactly SIZE - 1 bytes into BUFFER
1689 and stores a null byte into BUFFER[SIZE - 1]. */
1691 read_string (struct sfm_reader
*r
, char *buffer
, size_t size
)
1694 read_bytes (r
, buffer
, size
- 1);
1695 buffer
[size
- 1] = '\0';
1698 /* Skips BYTES bytes forward in R. */
1700 skip_bytes (struct sfm_reader
*r
, size_t bytes
)
1705 size_t chunk
= MIN (sizeof buffer
, bytes
);
1706 read_bytes (r
, buffer
, chunk
);
1712 trim_spaces (char *s
)
1714 char *end
= strchr (s
, '\0');
1715 while (end
> s
&& end
[-1] == ' ')
1721 print_string (const char *s
, size_t len
)
1723 if (memchr (s
, 0, len
) == 0)
1727 for (i
= 0; i
< len
; i
++)
1729 unsigned char c
= s
[i
];
1733 else if (c
== '\n' || isprint (c
))
1736 printf ("\\%02x", c
);
1741 hex_dump (0, s
, len
);