usr.bin/ktrdump/ktrdump.c

   1 /*-
   2  * Copyright (c) 2002 Jake Burkholder
   3  * Copyright (c) 2004 Robert Watson
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  * $FreeBSD: src/usr.bin/ktrdump/ktrdump.c,v 1.10 2005/05/21 09:55:06 ru Exp $
  28  * $DragonFly: src/usr.bin/ktrdump/ktrdump.c,v 1.7 2007/06/08 18:24:22 dillon Exp $
  29  */
  30
  31 #include <sys/cdefs.h>
  32
  33 #include <sys/types.h>
  34 #include <sys/ktr.h>
  35 #include <sys/mman.h>
  36 #include <sys/stat.h>
  37 #include <sys/queue.h>
  38
  39 #include <err.h>
  40 #include <fcntl.h>
  41 #include <kvm.h>
  42 #include <limits.h>
  43 #include <nlist.h>
  44 #include <stdint.h>
  45 #include <stdio.h>
  46 #include <stdlib.h>
  47 #include <string.h>
  48 #include <unistd.h>
  49
  50 #define SBUFLEN 128
  51
  52 extern char *optarg;
  53 extern int optind;
  54
  55 struct ktr_buffer {
  56         struct ktr_entry *ents;
  57         int modified;
  58         int reset;
  59         int beg_idx;            /* Beginning index */
  60         int end_idx;            /* Ending index */
  61 };
  62
  63 static struct nlist nl[] = {
  64         { "_ktr_version" },
  65         { "_ktr_entries" },
  66         { "_ktr_idx" },
  67         { "_ktr_buf" },
  68         { "_ncpus" },
  69         { "_tsc_frequency" },
  70         { NULL }
  71 };
  72
  73 static int cflag;
  74 static int fflag;
  75 static int iflag;
  76 static int lflag;
  77 static int nflag;
  78 static int qflag;
  79 static int rflag;
  80 static int sflag;
  81 static int tflag;
  82 static int xflag;
  83 static int pflag;
  84 static int Mflag;
  85 static int Nflag;
  86 static double tsc_frequency;
  87 static double correction_factor = 0.0;
  88
  89 static char corefile[PATH_MAX];
  90 static char execfile[PATH_MAX];
  91
  92 static char errbuf[_POSIX2_LINE_MAX];
  93 static int ncpus;
  94 static kvm_t *kd;
  95 static int entries_per_buf;
  96 static int fifo_mask;
  97
  98 static void usage(void);
  99 static int earliest_ts(struct ktr_buffer *buf);
 100 static void print_header(FILE *fo, int row);
 101 static void print_entry(FILE *fo, kvm_t *kd, int n, int i, struct ktr_entry *entry, u_int64_t *last_timestamp);
 102 static struct ktr_info *kvm_ktrinfo(kvm_t *kd, void *kptr);
 103 static const char *kvm_string(kvm_t *kd, const char *kptr);
 104 static const char *trunc_path(const char *str, int maxlen);
 105 static void read_symbols(const char *execfile);
 106 static const char *address_to_symbol(void *kptr);
 107 static struct ktr_buffer *ktr_bufs_init(int);
 108 static void load_bufs(struct ktr_buffer *, struct ktr_entry **);
 109 static void print_buf(FILE *, struct ktr_buffer *, int, u_int64_t *);
 110 static void print_bufs_timesorted(FILE *, struct ktr_buffer *, u_int64_t *);
 111
 112
 113 /*
 114  * Reads the ktr trace buffer from kernel memory and prints the trace entries.
 115  */
 116 int
 117 main(int ac, char **av)
 118 {
 119         struct ktr_buffer *ktr_bufs;
 120         struct ktr_entry **ktr_kbuf;
 121         FILE *fo;
 122         int64_t tts;
 123         int *ktr_start_index;
 124         int version;
 125         int c;
 126         int n;
 127
 128         /*
 129          * Parse commandline arguments.
 130          */
 131         fo = stdout;
 132         while ((c = getopt(ac, av, "acfinqrtxpslA:N:M:o:")) != -1) {
 133                 switch (c) {
 134                 case 'a':
 135                         cflag = 1;
 136                         iflag = 1;
 137                         tflag = 1;
 138                         xflag = 1;
 139                         fflag = 1;
 140                         pflag = 1;
 141                         break;
 142                 case 'c':
 143                         cflag = 1;
 144                         break;
 145                 case 'N':
 146                         if (strlcpy(execfile, optarg, sizeof(execfile))
 147                             >= sizeof(execfile))
 148                                 errx(1, "%s: File name too long", optarg);
 149                         Nflag = 1;
 150                         break;
 151                 case 'f':
 152                         fflag = 1;
 153                         break;
 154                 case 'l':
 155                         lflag = 1;
 156                         break;
 157                 case 'i':
 158                         iflag = 1;
 159                         break;
 160                 case 'A':
 161                         correction_factor = strtod(optarg, NULL);
 162                         break;
 163                 case 'M':
 164                         if (strlcpy(corefile, optarg, sizeof(corefile))
 165                             >= sizeof(corefile))
 166                                 errx(1, "%s: File name too long", optarg);
 167                         Mflag = 1;
 168                         break;
 169                 case 'n':
 170                         nflag = 1;
 171                         break;
 172                 case 'o':
 173                         if ((fo = fopen(optarg, "w")) == NULL)
 174                                 err(1, "%s", optarg);
 175                         break;
 176                 case 'p':
 177                         pflag++;
 178                         break;
 179                 case 'q':
 180                         qflag++;
 181                         break;
 182                 case 'r':
 183                         rflag = 1;
 184                         break;
 185                 case 's':
 186                         sflag = 1;      /* sort across the cpus */
 187                         break;
 188                 case 't':
 189                         tflag = 1;
 190                         break;
 191                 case 'x':
 192                         xflag = 1;
 193                         break;
 194                 case '?':
 195                 default:
 196                         usage();
 197                 }
 198         }
 199         if (cflag + iflag + tflag + xflag + fflag + pflag == 0) {
 200                 cflag = 1;
 201                 iflag = 1;
 202                 tflag = 1;
 203                 pflag = 1;
 204         }
 205         if (correction_factor != 0.0 && (rflag == 0 || nflag)) {
 206                 fprintf(stderr, "Correction factor can only be applied with -r and without -n\n");
 207                 exit(1);
 208         }
 209         ac -= optind;
 210         av += optind;
 211         if (ac != 0)
 212                 usage();
 213
 214         /*
 215          * Open our execfile and corefile, resolve needed symbols and read in
 216          * the trace buffer.
 217          */
 218         if ((kd = kvm_openfiles(Nflag ? execfile : NULL,
 219             Mflag ? corefile : NULL, NULL, O_RDONLY, errbuf)) == NULL)
 220                 errx(1, "%s", errbuf);
 221         if (kvm_nlist(kd, nl) != 0)
 222                 errx(1, "%s", kvm_geterr(kd));
 223         if (kvm_read(kd, nl[0].n_value, &version, sizeof(version)) == -1)
 224                 errx(1, "%s", kvm_geterr(kd));
 225         if (kvm_read(kd, nl[4].n_value, &ncpus, sizeof(ncpus)) == -1)
 226                 errx(1, "%s", kvm_geterr(kd));
 227         ktr_start_index = malloc(sizeof(*ktr_start_index) * ncpus);
 228         if (version >= 3) {
 229                 if (kvm_read(kd, nl[5].n_value, &tts, sizeof(tts)) == -1)
 230                         errx(1, "%s", kvm_geterr(kd));
 231                 tsc_frequency = (double)tts;
 232         }
 233         if (version > KTR_VERSION)
 234                 errx(1, "ktr version too high for us to handle");
 235         if (kvm_read(kd, nl[1].n_value, &entries_per_buf,
 236                                 sizeof(entries_per_buf)) == -1)
 237                 errx(1, "%s", kvm_geterr(kd));
 238         fifo_mask = entries_per_buf - 1;
 239
 240         printf("TSC frequency is %6.3f MHz\n", tsc_frequency / 1000000.0);
 241
 242         ktr_kbuf = malloc(sizeof(*ktr_kbuf) * ncpus);
 243
 244         if (nflag == 0)
 245                 read_symbols(Nflag ? execfile : NULL);
 246
 247         if (kvm_read(kd, nl[3].n_value, ktr_kbuf, sizeof(*ktr_kbuf) * ncpus) == -1)
 248                 errx(1, "%s", kvm_geterr(kd));
 249
 250         ktr_bufs = ktr_bufs_init(ncpus);
 251
 252         if (sflag) {
 253                 u_int64_t last_timestamp = 0;
 254                 do {
 255                         load_bufs(ktr_bufs, ktr_kbuf);
 256                         print_bufs_timesorted(fo, ktr_bufs, &last_timestamp);
 257                         if (lflag)
 258                                 usleep(1000000 / 10);
 259                 } while (lflag);
 260         } else {
 261                 u_int64_t *last_timestamp = calloc(sizeof(u_int64_t), ncpus);
 262                 do {
 263                         load_bufs(ktr_bufs, ktr_kbuf);
 264                         for (n = 0; n < ncpus; ++n)
 265                                 print_buf(fo, ktr_bufs, n, &last_timestamp[n]);
 266                         if (lflag)
 267                                 usleep(1000000 / 10);
 268                 } while (lflag);
 269         }
 270         return (0);
 271 }
 272
 273 static void
 274 print_header(FILE *fo, int row)
 275 {
 276         if (qflag == 0 && (u_int32_t)row % 20 == 0) {
 277                 fprintf(fo, "%-6s ", "index");
 278                 if (cflag)
 279                         fprintf(fo, "%-3s ", "cpu");
 280                 if (tflag || rflag)
 281                         fprintf(fo, "%-16s ", "timestamp");
 282                 if (xflag) {
 283                         if (nflag)
 284                             fprintf(fo, "%-10s %-10s", "caller2", "caller1");
 285                         else
 286                             fprintf(fo, "%-20s %-20s", "caller2", "caller1");
 287                 }
 288                 if (iflag)
 289                         fprintf(fo, "%-20s ", "ID");
 290                 if (fflag)
 291                         fprintf(fo, "%10s%-30s ", "", "file and line");
 292                 if (pflag)
 293                         fprintf(fo, "%s", "trace");
 294                 fprintf(fo, "\n");
 295         }
 296 }
 297
 298 static void
 299 print_entry(FILE *fo, kvm_t *kd, int n, int row, struct ktr_entry *entry,
 300             u_int64_t *last_timestamp)
 301 {
 302         struct ktr_info *info = NULL;
 303
 304         fprintf(fo, " %06x ", row & 0x00FFFFFF);
 305         if (cflag)
 306                 fprintf(fo, "%-3d ", n);
 307         if (tflag || rflag) {
 308                 if (rflag && !nflag && tsc_frequency != 0.0) {
 309                         fprintf(fo, "%13.3f uS ",
 310                                 (double)(entry->ktr_timestamp - *last_timestamp) * 1000000.0 / tsc_frequency - correction_factor);
 311                 } else if (rflag) {
 312                         fprintf(fo, "%-16lld ", entry->ktr_timestamp -
 313                                                 *last_timestamp);
 314                 } else {
 315                         fprintf(fo, "%-16lld ", entry->ktr_timestamp);
 316                 }
 317         }
 318         if (xflag) {
 319                 if (nflag) {
 320                     fprintf(fo, "%p %p ",
 321                             entry->ktr_caller2, entry->ktr_caller1);
 322                 } else {
 323                     fprintf(fo, "%-20s ",
 324                             address_to_symbol(entry->ktr_caller2));
 325                     fprintf(fo, "%-20s ",
 326                             address_to_symbol(entry->ktr_caller1));
 327                 }
 328         }
 329         if (iflag) {
 330                 info = kvm_ktrinfo(kd, entry->ktr_info);
 331                 if (info)
 332                         fprintf(fo, "%-20s ", kvm_string(kd, info->kf_name));
 333                 else
 334                         fprintf(fo, "%-20s ", "<empty>");
 335         }
 336         if (fflag)
 337                 fprintf(fo, "%34s:%-4d ", trunc_path(kvm_string(kd, entry->ktr_file), 34), entry->ktr_line);
 338         if (pflag) {
 339                 if (info == NULL)
 340                         info = kvm_ktrinfo(kd, entry->ktr_info);
 341                 if (info) {
 342                         fprintf(fo, kvm_string(kd, info->kf_format),
 343                                 entry->ktr_data[0], entry->ktr_data[1],
 344                                 entry->ktr_data[2], entry->ktr_data[3],
 345                                 entry->ktr_data[4], entry->ktr_data[5],
 346                                 entry->ktr_data[6], entry->ktr_data[7],
 347                                 entry->ktr_data[8], entry->ktr_data[9]);
 348                 } else {
 349                         fprintf(fo, "");
 350                 }
 351         }
 352         fprintf(fo, "\n");
 353         *last_timestamp = entry->ktr_timestamp;
 354 }
 355
 356 static
 357 struct ktr_info *
 358 kvm_ktrinfo(kvm_t *kd, void *kptr)
 359 {
 360         static struct ktr_info save_info;
 361         static void *save_kptr;
 362
 363         if (kptr == NULL)
 364                 return(NULL);
 365         if (save_kptr != kptr) {
 366                 if (kvm_read(kd, (uintptr_t)kptr, &save_info, sizeof(save_info)) == -1) {
 367                         bzero(&save_info, sizeof(save_info));
 368                 } else {
 369                         save_kptr = kptr;
 370                 }
 371         }
 372         return(&save_info);
 373 }
 374
 375 static
 376 const char *
 377 kvm_string(kvm_t *kd, const char *kptr)
 378 {
 379         static char save_str[128];
 380         static const char *save_kptr;
 381         int l;
 382         int n;
 383
 384         if (kptr == NULL)
 385                 return("?");
 386         if (save_kptr != kptr) {
 387                 save_kptr = kptr;
 388                 l = 0;
 389                 while (l < sizeof(save_str) - 1) {
 390                         n = 256 - ((intptr_t)(kptr + l) & 255);
 391                         if (n > sizeof(save_str) - l - 1)
 392                                 n = sizeof(save_str) - l - 1;
 393                         if (kvm_read(kd, (uintptr_t)(kptr + l), save_str + l, n) < 0)
 394                                 break;
 395                         while (l < sizeof(save_str) && n) {
 396                             if (save_str[l] == 0)
 397                                     break;
 398                             --n;
 399                             ++l;
 400                         }
 401                         if (n)
 402                             break;
 403                 }
 404                 save_str[l] = 0;
 405         }
 406         return(save_str);
 407 }
 408
 409 static
 410 const char *
 411 trunc_path(const char *str, int maxlen)
 412 {
 413         int len = strlen(str);
 414
 415         if (len > maxlen)
 416                 return(str + len - maxlen);
 417         else
 418                 return(str);
 419 }
 420
 421 struct symdata {
 422         TAILQ_ENTRY(symdata) link;
 423         const char *symname;
 424         char *symaddr;
 425         char symtype;
 426 };
 427
 428 static TAILQ_HEAD(symlist, symdata) symlist;
 429 static struct symdata *symcache;
 430 static char *symbegin;
 431 static char *symend;
 432
 433 static
 434 void
 435 read_symbols(const char *execfile)
 436 {
 437         char buf[256];
 438         char cmd[256];
 439         int buflen = sizeof(buf);
 440         FILE *fp;
 441         struct symdata *sym;
 442         char *s1;
 443         char *s2;
 444         char *s3;
 445
 446         TAILQ_INIT(&symlist);
 447
 448         if (execfile == NULL) {
 449                 if (sysctlbyname("kern.bootfile", buf, &buflen, NULL, 0) < 0)
 450                         execfile = "/kernel";
 451                 else
 452                         execfile = buf;
 453         }
 454         snprintf(cmd, sizeof(cmd), "nm -n %s", execfile);
 455         if ((fp = popen(cmd, "r")) != NULL) {
 456                 while (fgets(buf, sizeof(buf), fp) != NULL) {
 457                     s1 = strtok(buf, " \t\n");
 458                     s2 = strtok(NULL, " \t\n");
 459                     s3 = strtok(NULL, " \t\n");
 460                     if (s1 && s2 && s3) {
 461                         sym = malloc(sizeof(struct symdata));
 462                         sym->symaddr = (char *)strtoul(s1, NULL, 16);
 463                         sym->symtype = s2[0];
 464                         sym->symname = strdup(s3);
 465                         if (strcmp(s3, "kernbase") == 0)
 466                                 symbegin = sym->symaddr;
 467                         if (strcmp(s3, "end") == 0)
 468                                 symend = sym->symaddr;
 469                         TAILQ_INSERT_TAIL(&symlist, sym, link);
 470                     }
 471                 }
 472                 pclose(fp);
 473         }
 474         symcache = TAILQ_FIRST(&symlist);
 475 }
 476
 477 static
 478 const char *
 479 address_to_symbol(void *kptr)
 480 {
 481         static char buf[64];
 482
 483         if (symcache == NULL ||
 484            (char *)kptr < symbegin || (char *)kptr >= symend
 485         ) {
 486                 snprintf(buf, sizeof(buf), "%p", kptr);
 487                 return(buf);
 488         }
 489         while ((char *)symcache->symaddr < (char *)kptr) {
 490                 if (TAILQ_NEXT(symcache, link) == NULL)
 491                         break;
 492                 symcache = TAILQ_NEXT(symcache, link);
 493         }
 494         while ((char *)symcache->symaddr > (char *)kptr) {
 495                 if (symcache != TAILQ_FIRST(&symlist))
 496                         symcache = TAILQ_PREV(symcache, symlist, link);
 497         }
 498         snprintf(buf, sizeof(buf), "%s+%d", symcache->symname,
 499                 (int)((char *)kptr - symcache->symaddr));
 500         return(buf);
 501 }
 502
 503 static
 504 struct ktr_buffer *
 505 ktr_bufs_init(int ncpus)
 506 {
 507         struct ktr_buffer *ktr_bufs, *it;
 508         int i;
 509
 510         ktr_bufs = malloc(sizeof(*ktr_bufs) * ncpus);
 511         if (!ktr_bufs)
 512                 err(1, "can't allocate data structures\n");
 513         for (i = 0; i < ncpus; ++i) {
 514                 it = ktr_bufs + i;
 515                 it->ents = malloc(sizeof(struct ktr_entry) * entries_per_buf);
 516                 if (it->ents == NULL)
 517                         err(1, "can't allocate data structures\n");
 518                 it->reset = 1;
 519                 it->beg_idx = -1;
 520                 it->end_idx = -1;
 521         }
 522         return ktr_bufs;
 523 }
 524
 525 static
 526 void
 527 get_indices(kvm_t *kd, int *idx)
 528 {
 529         if (kvm_read(kd, nl[2].n_value, idx, sizeof(*idx) * ncpus) == -1)
 530                 errx(1, "%s", kvm_geterr(kd));
 531 }
 532
 533 /*
 534  * Get the trace buffer data from the kernel
 535  */
 536 static
 537 void
 538 load_bufs(struct ktr_buffer *ktr_bufs, struct ktr_entry **kbufs)
 539 {
 540         static int *kern_idx;
 541         struct ktr_buffer *kbuf;
 542         int i;
 543
 544         if (!kern_idx) {
 545                 kern_idx = malloc(sizeof(*kern_idx) * ncpus);
 546                 if (!kern_idx) {
 547                         err(1, "can't allocate data structures\n");
 548                 }
 549         }
 550
 551         get_indices(kd, kern_idx);
 552         for (i = 0; i < ncpus; ++i) {
 553                 kbuf = &ktr_bufs[i];
 554                 if (kern_idx[i] == kbuf->end_idx)
 555                         continue;
 556                 kbuf->end_idx = kern_idx[i];
 557
 558                 /*
 559                  * If we do not have a notion of the beginning index, assume
 560                  * it is entries_per_buf before the ending index.  Don't
 561                  * worry about underflows/negative numbers, the indices will
 562                  * be masked.
 563                  */
 564                 if (kbuf->reset) {
 565                         kbuf->beg_idx = kbuf->end_idx - entries_per_buf + 1;
 566                         kbuf->reset = 0;
 567                 }
 568                 if (kvm_read(kd, (uintptr_t)kbufs[i], ktr_bufs[i].ents,
 569                                 sizeof(struct ktr_entry) * entries_per_buf)
 570                                                                         == -1)
 571                         errx(1, "%s", kvm_geterr(kd));
 572                 kbuf->modified = 1;
 573                 kbuf->beg_idx = earliest_ts(kbuf);
 574         }
 575
 576 }
 577
 578 /*
 579  * Locate the earliest timestamp iterating backwards from end_idx, but
 580  * not going further back then beg_idx.  We have to do this because
 581  * the kernel uses a circulating buffer.
 582  */
 583 static
 584 int
 585 earliest_ts(struct ktr_buffer *buf)
 586 {
 587         struct ktr_entry *save;
 588         int count, scan, i, earliest;
 589
 590         count = 0;
 591         earliest = buf->end_idx - 1;
 592         save = &buf->ents[earliest & fifo_mask];
 593         for (scan = buf->end_idx - 1; scan != buf->beg_idx -1; --scan) {
 594                 i = scan & fifo_mask;
 595                 if (buf->ents[i].ktr_timestamp < save->ktr_timestamp)
 596                         earliest = scan;
 597                 /*
 598                  * We may have gotten so far behind that beg_idx wrapped
 599                  * more then once around the buffer.  Just stop
 600                  */
 601                 if (++count == entries_per_buf)
 602                         break;
 603         }
 604         return earliest;
 605 }
 606
 607 static
 608 void
 609 print_buf(FILE *fo, struct ktr_buffer *ktr_bufs, int cpu,
 610           u_int64_t *last_timestamp)
 611 {
 612         int first, last;
 613         struct ktr_buffer *buf = ktr_bufs + cpu;
 614
 615         if (buf->modified == 0)
 616                 return;
 617         if (*last_timestamp == 0) {
 618                 *last_timestamp =
 619                         buf->ents[buf->beg_idx & fifo_mask].ktr_timestamp;
 620         }
 621         while (buf->beg_idx != buf->end_idx) {
 622                 print_header(fo, buf->beg_idx);
 623                 print_entry(fo, kd, cpu, buf->beg_idx,
 624                             &buf->ents[buf->beg_idx & fifo_mask],
 625                             last_timestamp);
 626                 ++buf->beg_idx;
 627         }
 628         buf->modified = 0;
 629 }
 630
 631 static
 632 void
 633 print_bufs_timesorted(FILE *fo, struct ktr_buffer *ktr_bufs,
 634                       u_int64_t *last_timestamp)
 635 {
 636         struct ktr_entry *ent;
 637         struct ktr_buffer *buf;
 638         int n, bestn;
 639         u_int64_t ts;
 640         static int row = 0;
 641
 642         for (;;) {
 643                 ts = 0;
 644                 bestn = -1;
 645                 for (n = 0; n < ncpus; ++n) {
 646                         buf = ktr_bufs + n;
 647                         if (buf->beg_idx == buf->end_idx)
 648                                 continue;
 649                         ent = &buf->ents[buf->beg_idx & fifo_mask];
 650                         if (ts == 0 || (ts >= ent->ktr_timestamp)) {
 651                                 ts = ent->ktr_timestamp;
 652                                 bestn = n;
 653                         }
 654                 }
 655                 if ((bestn < 0) || (ts < *last_timestamp))
 656                         break;
 657                 buf = ktr_bufs + bestn;
 658                 print_header(fo, row);
 659                 print_entry(fo, kd, bestn, row,
 660                             &buf->ents[buf->beg_idx & fifo_mask],
 661                             last_timestamp);
 662                 ++buf->beg_idx;
 663                 *last_timestamp = ts;
 664                 ++row;
 665         }
 666 }
 667
 668 static void
 669 usage(void)
 670 {
 671         fprintf(stderr, "usage: ktrdump [-acfinpqrtx] [-N execfile] "
 672                         "[-M corefile] [-o outfile]\n");
 673         exit(1);
 674 }