1 /* uniq -- remove duplicate lines from a sorted file
2 Copyright (C) 1986-2013 Free Software Foundation, Inc.
4 This program is free software: you can redistribute it and/or modify
5 it under the terms of the GNU General Public License as published by
6 the Free Software Foundation, either version 3 of the License, or
7 (at your option) any later version.
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
14 You should have received a copy of the GNU General Public License
15 along with this program. If not, see <http://www.gnu.org/licenses/>. */
17 /* Written by Richard M. Stallman and David MacKenzie. */
22 #include <sys/types.h>
26 #include "linebuffer.h"
29 #include "hard-locale.h"
35 #include "memcasecmp.h"
37 /* The official name of this program (e.g., no 'g' prefix). */
38 #define PROGRAM_NAME "uniq"
41 proper_name ("Richard M. Stallman"), \
42 proper_name ("David MacKenzie")
44 #define SWAP_LINES(A, B) \
47 struct linebuffer *_tmp; \
54 /* True if the LC_COLLATE locale is hard. */
55 static bool hard_LC_COLLATE
;
57 /* Number of fields to skip on each line when doing comparisons. */
58 static size_t skip_fields
;
60 /* Number of chars to skip after skipping any fields. */
61 static size_t skip_chars
;
63 /* Number of chars to compare. */
64 static size_t check_chars
;
68 count_occurrences
, /* -c Print count before output lines. */
69 count_none
/* Default. Do not print counts. */
72 /* Whether and how to precede the output lines with a count of the number of
73 times they occurred in the input. */
74 static enum countmode countmode
;
76 /* Which lines to output: unique lines, the first of a group of
77 repeated lines, and the second and subsequented of a group of
79 static bool output_unique
;
80 static bool output_first_repeated
;
81 static bool output_later_repeated
;
83 /* If true, ignore case when comparing. */
84 static bool ignore_case
;
88 /* No delimiters output. --all-repeated[=none] */
91 /* Delimiter precedes all groups. --all-repeated=prepend */
94 /* Delimit all groups. --all-repeated=separate */
98 static char const *const delimit_method_string
[] =
100 "none", "prepend", "separate", NULL
103 static enum delimit_method
const delimit_method_map
[] =
105 DM_NONE
, DM_PREPEND
, DM_SEPARATE
108 /* Select whether/how to delimit groups of duplicate lines. */
109 static enum delimit_method delimit_groups
;
111 static struct option
const longopts
[] =
113 {"count", no_argument
, NULL
, 'c'},
114 {"repeated", no_argument
, NULL
, 'd'},
115 {"all-repeated", optional_argument
, NULL
, 'D'},
116 {"ignore-case", no_argument
, NULL
, 'i'},
117 {"unique", no_argument
, NULL
, 'u'},
118 {"skip-fields", required_argument
, NULL
, 'f'},
119 {"skip-chars", required_argument
, NULL
, 's'},
120 {"check-chars", required_argument
, NULL
, 'w'},
121 {"zero-terminated", no_argument
, NULL
, 'z'},
122 {GETOPT_HELP_OPTION_DECL
},
123 {GETOPT_VERSION_OPTION_DECL
},
130 if (status
!= EXIT_SUCCESS
)
135 Usage: %s [OPTION]... [INPUT [OUTPUT]]\n\
139 Filter adjacent matching lines from INPUT (or standard input),\n\
140 writing to OUTPUT (or standard output).\n\
142 With no options, matching lines are merged to the first occurrence.\n\
145 emit_mandatory_arg_note ();
148 -c, --count prefix lines by the number of occurrences\n\
149 -d, --repeated only print duplicate lines\n\
152 -D, --all-repeated[=delimit-method] print all duplicate lines\n\
153 delimit-method={none(default),prepend,separate}\n\
154 Delimiting is done with blank lines\n\
155 -f, --skip-fields=N avoid comparing the first N fields\n\
156 -i, --ignore-case ignore differences in case when comparing\n\
157 -s, --skip-chars=N avoid comparing the first N characters\n\
158 -u, --unique only print unique lines\n\
159 -z, --zero-terminated end lines with 0 byte, not newline\n\
162 -w, --check-chars=N compare no more than N characters in lines\n\
164 fputs (HELP_OPTION_DESCRIPTION
, stdout
);
165 fputs (VERSION_OPTION_DESCRIPTION
, stdout
);
168 A field is a run of blanks (usually spaces and/or TABs), then non-blank\n\
169 characters. Fields are skipped before chars.\n\
173 Note: 'uniq' does not detect repeated lines unless they are adjacent.\n\
174 You may want to sort the input first, or use 'sort -u' without 'uniq'.\n\
175 Also, comparisons honor the rules specified by 'LC_COLLATE'.\n\
177 emit_ancillary_info ();
182 /* Convert OPT to size_t, reporting an error using MSGID if OPT is
183 invalid. Silently convert too-large values to SIZE_MAX. */
186 size_opt (char const *opt
, char const *msgid
)
188 unsigned long int size
;
189 verify (SIZE_MAX
<= ULONG_MAX
);
191 switch (xstrtoul (opt
, NULL
, 10, &size
, ""))
194 case LONGINT_OVERFLOW
:
198 error (EXIT_FAILURE
, 0, "%s: %s", opt
, _(msgid
));
201 return MIN (size
, SIZE_MAX
);
204 /* Given a linebuffer LINE,
205 return a pointer to the beginning of the line's field to be compared. */
207 static char * _GL_ATTRIBUTE_PURE
208 find_field (struct linebuffer
const *line
)
211 char const *lp
= line
->buffer
;
212 size_t size
= line
->length
- 1;
215 for (count
= 0; count
< skip_fields
&& i
< size
; count
++)
217 while (i
< size
&& isblank (to_uchar (lp
[i
])))
219 while (i
< size
&& !isblank (to_uchar (lp
[i
])))
223 i
+= MIN (skip_chars
, size
- i
);
225 return line
->buffer
+ i
;
228 /* Return false if two strings OLD and NEW match, true if not.
229 OLD and NEW point not to the beginnings of the lines
230 but rather to the beginnings of the fields to compare.
231 OLDLEN and NEWLEN are their lengths. */
234 different (char *old
, char *new, size_t oldlen
, size_t newlen
)
236 if (check_chars
< oldlen
)
237 oldlen
= check_chars
;
238 if (check_chars
< newlen
)
239 newlen
= check_chars
;
243 /* FIXME: This should invoke strcoll somehow. */
244 return oldlen
!= newlen
|| memcasecmp (old
, new, oldlen
);
246 else if (hard_LC_COLLATE
)
247 return xmemcoll (old
, oldlen
, new, newlen
) != 0;
249 return oldlen
!= newlen
|| memcmp (old
, new, oldlen
);
252 /* Output the line in linebuffer LINE to standard output
253 provided that the switches say it should be output.
254 MATCH is true if the line matches the previous line.
255 If requested, print the number of times it occurred, as well;
256 LINECOUNT + 1 is the number of times that the line occurred. */
259 writeline (struct linebuffer
const *line
,
260 bool match
, uintmax_t linecount
)
262 if (! (linecount
== 0 ? output_unique
263 : !match
? output_first_repeated
264 : output_later_repeated
))
267 if (countmode
== count_occurrences
)
268 printf ("%7" PRIuMAX
" ", linecount
+ 1);
270 fwrite (line
->buffer
, sizeof (char), line
->length
, stdout
);
273 /* Process input file INFILE with output to OUTFILE.
274 If either is "-", use the standard I/O stream for it instead. */
277 check_file (const char *infile
, const char *outfile
, char delimiter
)
279 struct linebuffer lb1
, lb2
;
280 struct linebuffer
*thisline
, *prevline
;
282 if (! (STREQ (infile
, "-") || freopen (infile
, "r", stdin
)))
283 error (EXIT_FAILURE
, errno
, "%s", infile
);
284 if (! (STREQ (outfile
, "-") || freopen (outfile
, "w", stdout
)))
285 error (EXIT_FAILURE
, errno
, "%s", outfile
);
287 fadvise (stdin
, FADVISE_SEQUENTIAL
);
292 initbuffer (thisline
);
293 initbuffer (prevline
);
295 /* The duplication in the following 'if' and 'else' blocks is an
296 optimization to distinguish the common case (in which none of
297 the following options has been specified: --count, -repeated,
298 --all-repeated, --unique) from the others. In the common case,
299 this optimization lets uniq output each different line right away,
300 without waiting to see if the next one is different. */
302 if (output_unique
&& output_first_repeated
&& countmode
== count_none
)
304 char *prevfield
IF_LINT ( = NULL
);
305 size_t prevlen
IF_LINT ( = 0);
307 while (!feof (stdin
))
311 if (readlinebuffer_delim (thisline
, stdin
, delimiter
) == 0)
313 thisfield
= find_field (thisline
);
314 thislen
= thisline
->length
- 1 - (thisfield
- thisline
->buffer
);
315 if (prevline
->length
== 0
316 || different (thisfield
, prevfield
, thislen
, prevlen
))
318 fwrite (thisline
->buffer
, sizeof (char),
319 thisline
->length
, stdout
);
321 SWAP_LINES (prevline
, thisline
);
322 prevfield
= thisfield
;
331 uintmax_t match_count
= 0;
332 bool first_delimiter
= true;
334 if (readlinebuffer_delim (prevline
, stdin
, delimiter
) == 0)
336 prevfield
= find_field (prevline
);
337 prevlen
= prevline
->length
- 1 - (prevfield
- prevline
->buffer
);
339 while (!feof (stdin
))
344 if (readlinebuffer_delim (thisline
, stdin
, delimiter
) == 0)
350 thisfield
= find_field (thisline
);
351 thislen
= thisline
->length
- 1 - (thisfield
- thisline
->buffer
);
352 match
= !different (thisfield
, prevfield
, thislen
, prevlen
);
353 match_count
+= match
;
355 if (match_count
== UINTMAX_MAX
)
357 if (count_occurrences
)
358 error (EXIT_FAILURE
, 0, _("too many repeated lines"));
362 if (delimit_groups
!= DM_NONE
)
366 if (match_count
) /* a previous match */
367 first_delimiter
= false; /* Only used when DM_SEPARATE */
369 else if (match_count
== 1)
371 if ((delimit_groups
== DM_PREPEND
)
372 || (delimit_groups
== DM_SEPARATE
373 && !first_delimiter
))
378 if (!match
|| output_later_repeated
)
380 writeline (prevline
, match
, match_count
);
381 SWAP_LINES (prevline
, thisline
);
382 prevfield
= thisfield
;
389 writeline (prevline
, false, match_count
);
393 if (ferror (stdin
) || fclose (stdin
) != 0)
394 error (EXIT_FAILURE
, 0, _("error reading %s"), infile
);
396 /* stdout is handled via the atexit-invoked close_stdout function. */
402 enum Skip_field_option_type
410 main (int argc
, char **argv
)
413 bool posixly_correct
= (getenv ("POSIXLY_CORRECT") != NULL
);
414 enum Skip_field_option_type skip_field_option_type
= SFO_NONE
;
417 char delimiter
= '\n'; /* change with --zero-terminated, -z */
419 file
[0] = file
[1] = "-";
420 initialize_main (&argc
, &argv
);
421 set_program_name (argv
[0]);
422 setlocale (LC_ALL
, "");
423 bindtextdomain (PACKAGE
, LOCALEDIR
);
424 textdomain (PACKAGE
);
425 hard_LC_COLLATE
= hard_locale (LC_COLLATE
);
427 atexit (close_stdout
);
431 check_chars
= SIZE_MAX
;
432 output_unique
= output_first_repeated
= true;
433 output_later_repeated
= false;
434 countmode
= count_none
;
435 delimit_groups
= DM_NONE
;
439 /* Parse an operand with leading "+" as a file after "--" was
440 seen; or if pedantic and a file was seen; or if not
444 || (posixly_correct
&& nfiles
!= 0)
445 || ((optc
= getopt_long (argc
, argv
,
446 "-0123456789Dcdf:is:uw:z", longopts
, NULL
))
453 error (0, 0, _("extra operand %s"), quote (argv
[optind
]));
454 usage (EXIT_FAILURE
);
456 file
[nfiles
++] = argv
[optind
++];
462 unsigned long int size
;
464 && posix2_version () < 200112
465 && xstrtoul (optarg
, NULL
, 10, &size
, "") == LONGINT_OK
468 else if (nfiles
== 2)
470 error (0, 0, _("extra operand %s"), quote (optarg
));
471 usage (EXIT_FAILURE
);
474 file
[nfiles
++] = optarg
;
489 if (skip_field_option_type
== SFO_NEW
)
492 if (!DECIMAL_DIGIT_ACCUMULATE (skip_fields
, optc
- '0', size_t))
493 skip_fields
= SIZE_MAX
;
495 skip_field_option_type
= SFO_OBSOLETE
;
500 countmode
= count_occurrences
;
504 output_unique
= false;
508 output_unique
= false;
509 output_later_repeated
= true;
511 delimit_groups
= DM_NONE
;
513 delimit_groups
= XARGMATCH ("--all-repeated", optarg
,
514 delimit_method_string
,
519 skip_field_option_type
= SFO_NEW
;
520 skip_fields
= size_opt (optarg
,
521 N_("invalid number of fields to skip"));
529 skip_chars
= size_opt (optarg
,
530 N_("invalid number of bytes to skip"));
534 output_first_repeated
= false;
538 check_chars
= size_opt (optarg
,
539 N_("invalid number of bytes to compare"));
546 case_GETOPT_HELP_CHAR
;
548 case_GETOPT_VERSION_CHAR (PROGRAM_NAME
, AUTHORS
);
551 usage (EXIT_FAILURE
);
555 if (countmode
== count_occurrences
&& output_later_repeated
)
558 _("printing all duplicated lines and repeat counts is meaningless"));
559 usage (EXIT_FAILURE
);
562 check_file (file
[0], file
[1], delimiter
);