5 #include "util/cache.h"
6 #include "util/symbol.h"
7 #include "util/thread.h"
8 #include "util/header.h"
10 #include "util/parse-options.h"
11 #include "util/trace-event.h"
13 #include "util/debug.h"
14 #include "util/session.h"
16 #include <sys/types.h>
17 #include <sys/prctl.h>
18 #include <semaphore.h>
23 #include <linux/list.h>
24 #include <linux/hash.h>
26 static struct perf_session
*session
;
28 /* based on kernel/lockdep.c */
29 #define LOCKHASH_BITS 12
30 #define LOCKHASH_SIZE (1UL << LOCKHASH_BITS)
32 static struct list_head lockhash_table
[LOCKHASH_SIZE
];
34 #define __lockhashfn(key) hash_long((unsigned long)key, LOCKHASH_BITS)
35 #define lockhashentry(key) (lockhash_table + __lockhashfn((key)))
38 struct list_head hash_entry
;
39 struct rb_node rb
; /* used for sorting */
41 void *addr
; /* address of lockdep_map, used as ID */
42 char *name
; /* for strcpy(), we cannot use const */
44 unsigned int nr_acquire
;
45 unsigned int nr_acquired
;
46 unsigned int nr_contended
;
47 unsigned int nr_release
;
49 unsigned int nr_readlock
;
50 unsigned int nr_trylock
;
51 /* these times are in nano sec. */
56 int discard
; /* flag of blacklist */
60 * States of lock_seq_stat
62 * UNINITIALIZED is required for detecting first event of acquire.
63 * As the nature of lock events, there is no guarantee
64 * that the first event for the locks are acquire,
65 * it can be acquired, contended or release.
67 #define SEQ_STATE_UNINITIALIZED 0 /* initial state */
68 #define SEQ_STATE_RELEASED 1
69 #define SEQ_STATE_ACQUIRING 2
70 #define SEQ_STATE_ACQUIRED 3
71 #define SEQ_STATE_READ_ACQUIRED 4
72 #define SEQ_STATE_CONTENDED 5
76 * Imported from include/linux/sched.h.
77 * Should this be synchronized?
79 #define MAX_LOCK_DEPTH 48
82 * struct lock_seq_stat:
83 * Place to put on state of one lock sequence
84 * 1) acquire -> acquired -> release
85 * 2) acquire -> contended -> acquired -> release
86 * 3) acquire (with read or try) -> release
87 * 4) Are there other patterns?
89 struct lock_seq_stat
{
90 struct list_head list
;
102 struct list_head seq_list
;
105 static struct rb_root thread_stats
;
107 static struct thread_stat
*thread_stat_find(u32 tid
)
109 struct rb_node
*node
;
110 struct thread_stat
*st
;
112 node
= thread_stats
.rb_node
;
114 st
= container_of(node
, struct thread_stat
, rb
);
117 else if (tid
< st
->tid
)
118 node
= node
->rb_left
;
120 node
= node
->rb_right
;
126 static void thread_stat_insert(struct thread_stat
*new)
128 struct rb_node
**rb
= &thread_stats
.rb_node
;
129 struct rb_node
*parent
= NULL
;
130 struct thread_stat
*p
;
133 p
= container_of(*rb
, struct thread_stat
, rb
);
136 if (new->tid
< p
->tid
)
137 rb
= &(*rb
)->rb_left
;
138 else if (new->tid
> p
->tid
)
139 rb
= &(*rb
)->rb_right
;
141 BUG_ON("inserting invalid thread_stat\n");
144 rb_link_node(&new->rb
, parent
, rb
);
145 rb_insert_color(&new->rb
, &thread_stats
);
148 static struct thread_stat
*thread_stat_findnew_after_first(u32 tid
)
150 struct thread_stat
*st
;
152 st
= thread_stat_find(tid
);
156 st
= zalloc(sizeof(struct thread_stat
));
158 die("memory allocation failed\n");
161 INIT_LIST_HEAD(&st
->seq_list
);
163 thread_stat_insert(st
);
168 static struct thread_stat
*thread_stat_findnew_first(u32 tid
);
169 static struct thread_stat
*(*thread_stat_findnew
)(u32 tid
) =
170 thread_stat_findnew_first
;
172 static struct thread_stat
*thread_stat_findnew_first(u32 tid
)
174 struct thread_stat
*st
;
176 st
= zalloc(sizeof(struct thread_stat
));
178 die("memory allocation failed\n");
180 INIT_LIST_HEAD(&st
->seq_list
);
182 rb_link_node(&st
->rb
, NULL
, &thread_stats
.rb_node
);
183 rb_insert_color(&st
->rb
, &thread_stats
);
185 thread_stat_findnew
= thread_stat_findnew_after_first
;
189 /* build simple key function one is bigger than two */
190 #define SINGLE_KEY(member) \
191 static int lock_stat_key_ ## member(struct lock_stat *one, \
192 struct lock_stat *two) \
194 return one->member > two->member; \
197 SINGLE_KEY(nr_acquired
)
198 SINGLE_KEY(nr_contended
)
199 SINGLE_KEY(wait_time_total
)
200 SINGLE_KEY(wait_time_min
)
201 SINGLE_KEY(wait_time_max
)
205 * name: the value for specify by user
206 * this should be simpler than raw name of member
207 * e.g. nr_acquired -> acquired, wait_time_total -> wait_total
210 int (*key
)(struct lock_stat
*, struct lock_stat
*);
213 static const char *sort_key
= "acquired";
215 static int (*compare
)(struct lock_stat
*, struct lock_stat
*);
217 static struct rb_root result
; /* place to store sorted data */
219 #define DEF_KEY_LOCK(name, fn_suffix) \
220 { #name, lock_stat_key_ ## fn_suffix }
221 struct lock_key keys
[] = {
222 DEF_KEY_LOCK(acquired
, nr_acquired
),
223 DEF_KEY_LOCK(contended
, nr_contended
),
224 DEF_KEY_LOCK(wait_total
, wait_time_total
),
225 DEF_KEY_LOCK(wait_min
, wait_time_min
),
226 DEF_KEY_LOCK(wait_max
, wait_time_max
),
228 /* extra comparisons much complicated should be here */
233 static void select_key(void)
237 for (i
= 0; keys
[i
].name
; i
++) {
238 if (!strcmp(keys
[i
].name
, sort_key
)) {
239 compare
= keys
[i
].key
;
244 die("Unknown compare key:%s\n", sort_key
);
247 static void insert_to_result(struct lock_stat
*st
,
248 int (*bigger
)(struct lock_stat
*, struct lock_stat
*))
250 struct rb_node
**rb
= &result
.rb_node
;
251 struct rb_node
*parent
= NULL
;
255 p
= container_of(*rb
, struct lock_stat
, rb
);
259 rb
= &(*rb
)->rb_left
;
261 rb
= &(*rb
)->rb_right
;
264 rb_link_node(&st
->rb
, parent
, rb
);
265 rb_insert_color(&st
->rb
, &result
);
268 /* returns left most element of result, and erase it */
269 static struct lock_stat
*pop_from_result(void)
271 struct rb_node
*node
= result
.rb_node
;
276 while (node
->rb_left
)
277 node
= node
->rb_left
;
279 rb_erase(node
, &result
);
280 return container_of(node
, struct lock_stat
, rb
);
283 static struct lock_stat
*lock_stat_findnew(void *addr
, const char *name
)
285 struct list_head
*entry
= lockhashentry(addr
);
286 struct lock_stat
*ret
, *new;
288 list_for_each_entry(ret
, entry
, hash_entry
) {
289 if (ret
->addr
== addr
)
293 new = zalloc(sizeof(struct lock_stat
));
298 new->name
= zalloc(sizeof(char) * strlen(name
) + 1);
301 strcpy(new->name
, name
);
303 new->wait_time_min
= ULLONG_MAX
;
305 list_add(&new->hash_entry
, entry
);
309 die("memory allocation failed\n");
312 static char const *input_name
= "perf.data";
314 struct raw_event_sample
{
319 struct trace_acquire_event
{
325 struct trace_acquired_event
{
330 struct trace_contended_event
{
335 struct trace_release_event
{
340 struct trace_lock_handler
{
341 void (*acquire_event
)(struct trace_acquire_event
*,
345 struct thread
*thread
);
347 void (*acquired_event
)(struct trace_acquired_event
*,
351 struct thread
*thread
);
353 void (*contended_event
)(struct trace_contended_event
*,
357 struct thread
*thread
);
359 void (*release_event
)(struct trace_release_event
*,
363 struct thread
*thread
);
366 static struct lock_seq_stat
*get_seq(struct thread_stat
*ts
, void *addr
)
368 struct lock_seq_stat
*seq
;
370 list_for_each_entry(seq
, &ts
->seq_list
, list
) {
371 if (seq
->addr
== addr
)
375 seq
= zalloc(sizeof(struct lock_seq_stat
));
377 die("Not enough memory\n");
378 seq
->state
= SEQ_STATE_UNINITIALIZED
;
381 list_add(&seq
->list
, &ts
->seq_list
);
393 static int bad_hist
[BROKEN_MAX
];
401 report_lock_acquire_event(struct trace_acquire_event
*acquire_event
,
402 struct event
*__event __used
,
404 u64 timestamp __used
,
405 struct thread
*thread __used
)
407 struct lock_stat
*ls
;
408 struct thread_stat
*ts
;
409 struct lock_seq_stat
*seq
;
411 ls
= lock_stat_findnew(acquire_event
->addr
, acquire_event
->name
);
415 ts
= thread_stat_findnew(thread
->pid
);
416 seq
= get_seq(ts
, acquire_event
->addr
);
418 switch (seq
->state
) {
419 case SEQ_STATE_UNINITIALIZED
:
420 case SEQ_STATE_RELEASED
:
421 if (!acquire_event
->flag
) {
422 seq
->state
= SEQ_STATE_ACQUIRING
;
424 if (acquire_event
->flag
& TRY_LOCK
)
426 if (acquire_event
->flag
& READ_LOCK
)
428 seq
->state
= SEQ_STATE_READ_ACQUIRED
;
433 case SEQ_STATE_READ_ACQUIRED
:
434 if (acquire_event
->flag
& READ_LOCK
) {
442 case SEQ_STATE_ACQUIRED
:
443 case SEQ_STATE_ACQUIRING
:
444 case SEQ_STATE_CONTENDED
:
446 /* broken lock sequence, discard it */
448 bad_hist
[BROKEN_ACQUIRE
]++;
449 list_del(&seq
->list
);
454 BUG_ON("Unknown state of lock sequence found!\n");
459 seq
->prev_event_time
= timestamp
;
465 report_lock_acquired_event(struct trace_acquired_event
*acquired_event
,
466 struct event
*__event __used
,
468 u64 timestamp __used
,
469 struct thread
*thread __used
)
471 struct lock_stat
*ls
;
472 struct thread_stat
*ts
;
473 struct lock_seq_stat
*seq
;
476 ls
= lock_stat_findnew(acquired_event
->addr
, acquired_event
->name
);
480 ts
= thread_stat_findnew(thread
->pid
);
481 seq
= get_seq(ts
, acquired_event
->addr
);
483 switch (seq
->state
) {
484 case SEQ_STATE_UNINITIALIZED
:
485 /* orphan event, do nothing */
487 case SEQ_STATE_ACQUIRING
:
489 case SEQ_STATE_CONTENDED
:
490 contended_term
= timestamp
- seq
->prev_event_time
;
491 ls
->wait_time_total
+= contended_term
;
492 if (contended_term
< ls
->wait_time_min
)
493 ls
->wait_time_min
= contended_term
;
494 if (ls
->wait_time_max
< contended_term
)
495 ls
->wait_time_max
= contended_term
;
497 case SEQ_STATE_RELEASED
:
498 case SEQ_STATE_ACQUIRED
:
499 case SEQ_STATE_READ_ACQUIRED
:
500 /* broken lock sequence, discard it */
502 bad_hist
[BROKEN_ACQUIRED
]++;
503 list_del(&seq
->list
);
509 BUG_ON("Unknown state of lock sequence found!\n");
513 seq
->state
= SEQ_STATE_ACQUIRED
;
515 seq
->prev_event_time
= timestamp
;
521 report_lock_contended_event(struct trace_contended_event
*contended_event
,
522 struct event
*__event __used
,
524 u64 timestamp __used
,
525 struct thread
*thread __used
)
527 struct lock_stat
*ls
;
528 struct thread_stat
*ts
;
529 struct lock_seq_stat
*seq
;
531 ls
= lock_stat_findnew(contended_event
->addr
, contended_event
->name
);
535 ts
= thread_stat_findnew(thread
->pid
);
536 seq
= get_seq(ts
, contended_event
->addr
);
538 switch (seq
->state
) {
539 case SEQ_STATE_UNINITIALIZED
:
540 /* orphan event, do nothing */
542 case SEQ_STATE_ACQUIRING
:
544 case SEQ_STATE_RELEASED
:
545 case SEQ_STATE_ACQUIRED
:
546 case SEQ_STATE_READ_ACQUIRED
:
547 case SEQ_STATE_CONTENDED
:
548 /* broken lock sequence, discard it */
550 bad_hist
[BROKEN_CONTENDED
]++;
551 list_del(&seq
->list
);
556 BUG_ON("Unknown state of lock sequence found!\n");
560 seq
->state
= SEQ_STATE_CONTENDED
;
562 seq
->prev_event_time
= timestamp
;
568 report_lock_release_event(struct trace_release_event
*release_event
,
569 struct event
*__event __used
,
571 u64 timestamp __used
,
572 struct thread
*thread __used
)
574 struct lock_stat
*ls
;
575 struct thread_stat
*ts
;
576 struct lock_seq_stat
*seq
;
578 ls
= lock_stat_findnew(release_event
->addr
, release_event
->name
);
582 ts
= thread_stat_findnew(thread
->pid
);
583 seq
= get_seq(ts
, release_event
->addr
);
585 switch (seq
->state
) {
586 case SEQ_STATE_UNINITIALIZED
:
589 case SEQ_STATE_ACQUIRED
:
591 case SEQ_STATE_READ_ACQUIRED
:
593 BUG_ON(seq
->read_count
< 0);
594 if (!seq
->read_count
) {
599 case SEQ_STATE_ACQUIRING
:
600 case SEQ_STATE_CONTENDED
:
601 case SEQ_STATE_RELEASED
:
602 /* broken lock sequence, discard it */
604 bad_hist
[BROKEN_RELEASE
]++;
608 BUG_ON("Unknown state of lock sequence found!\n");
614 list_del(&seq
->list
);
620 /* lock oriented handlers */
621 /* TODO: handlers for CPU oriented, thread oriented */
622 static struct trace_lock_handler report_lock_ops
= {
623 .acquire_event
= report_lock_acquire_event
,
624 .acquired_event
= report_lock_acquired_event
,
625 .contended_event
= report_lock_contended_event
,
626 .release_event
= report_lock_release_event
,
629 static struct trace_lock_handler
*trace_handler
;
632 process_lock_acquire_event(void *data
,
633 struct event
*event __used
,
635 u64 timestamp __used
,
636 struct thread
*thread __used
)
638 struct trace_acquire_event acquire_event
;
639 u64 tmp
; /* this is required for casting... */
641 tmp
= raw_field_value(event
, "lockdep_addr", data
);
642 memcpy(&acquire_event
.addr
, &tmp
, sizeof(void *));
643 acquire_event
.name
= (char *)raw_field_ptr(event
, "name", data
);
644 acquire_event
.flag
= (int)raw_field_value(event
, "flag", data
);
646 if (trace_handler
->acquire_event
)
647 trace_handler
->acquire_event(&acquire_event
, event
, cpu
, timestamp
, thread
);
651 process_lock_acquired_event(void *data
,
652 struct event
*event __used
,
654 u64 timestamp __used
,
655 struct thread
*thread __used
)
657 struct trace_acquired_event acquired_event
;
658 u64 tmp
; /* this is required for casting... */
660 tmp
= raw_field_value(event
, "lockdep_addr", data
);
661 memcpy(&acquired_event
.addr
, &tmp
, sizeof(void *));
662 acquired_event
.name
= (char *)raw_field_ptr(event
, "name", data
);
664 if (trace_handler
->acquire_event
)
665 trace_handler
->acquired_event(&acquired_event
, event
, cpu
, timestamp
, thread
);
669 process_lock_contended_event(void *data
,
670 struct event
*event __used
,
672 u64 timestamp __used
,
673 struct thread
*thread __used
)
675 struct trace_contended_event contended_event
;
676 u64 tmp
; /* this is required for casting... */
678 tmp
= raw_field_value(event
, "lockdep_addr", data
);
679 memcpy(&contended_event
.addr
, &tmp
, sizeof(void *));
680 contended_event
.name
= (char *)raw_field_ptr(event
, "name", data
);
682 if (trace_handler
->acquire_event
)
683 trace_handler
->contended_event(&contended_event
, event
, cpu
, timestamp
, thread
);
687 process_lock_release_event(void *data
,
688 struct event
*event __used
,
690 u64 timestamp __used
,
691 struct thread
*thread __used
)
693 struct trace_release_event release_event
;
694 u64 tmp
; /* this is required for casting... */
696 tmp
= raw_field_value(event
, "lockdep_addr", data
);
697 memcpy(&release_event
.addr
, &tmp
, sizeof(void *));
698 release_event
.name
= (char *)raw_field_ptr(event
, "name", data
);
700 if (trace_handler
->acquire_event
)
701 trace_handler
->release_event(&release_event
, event
, cpu
, timestamp
, thread
);
705 process_raw_event(void *data
, int cpu
, u64 timestamp
, struct thread
*thread
)
710 type
= trace_parse_common_type(data
);
711 event
= trace_find_event(type
);
713 if (!strcmp(event
->name
, "lock_acquire"))
714 process_lock_acquire_event(data
, event
, cpu
, timestamp
, thread
);
715 if (!strcmp(event
->name
, "lock_acquired"))
716 process_lock_acquired_event(data
, event
, cpu
, timestamp
, thread
);
717 if (!strcmp(event
->name
, "lock_contended"))
718 process_lock_contended_event(data
, event
, cpu
, timestamp
, thread
);
719 if (!strcmp(event
->name
, "lock_release"))
720 process_lock_release_event(data
, event
, cpu
, timestamp
, thread
);
723 static void print_bad_events(int bad
, int total
)
725 /* Output for debug, this have to be removed */
727 const char *name
[4] =
728 { "acquire", "acquired", "contended", "release" };
730 pr_info("\n=== output for debug===\n\n");
731 pr_info("bad: %d, total: %d\n", bad
, total
);
732 pr_info("bad rate: %f %%\n", (double)bad
/ (double)total
* 100);
733 pr_info("histogram of events caused bad sequence\n");
734 for (i
= 0; i
< BROKEN_MAX
; i
++)
735 pr_info(" %10s: %d\n", name
[i
], bad_hist
[i
]);
738 /* TODO: various way to print, coloring, nano or milli sec */
739 static void print_result(void)
741 struct lock_stat
*st
;
745 pr_info("%20s ", "Name");
746 pr_info("%10s ", "acquired");
747 pr_info("%10s ", "contended");
749 pr_info("%15s ", "total wait (ns)");
750 pr_info("%15s ", "max wait (ns)");
751 pr_info("%15s ", "min wait (ns)");
756 while ((st
= pop_from_result())) {
764 if (strlen(st
->name
) < 16) {
765 /* output raw name */
766 pr_info("%20s ", st
->name
);
768 strncpy(cut_name
, st
->name
, 16);
773 /* cut off name for saving output style */
774 pr_info("%20s ", cut_name
);
777 pr_info("%10u ", st
->nr_acquired
);
778 pr_info("%10u ", st
->nr_contended
);
780 pr_info("%15llu ", st
->wait_time_total
);
781 pr_info("%15llu ", st
->wait_time_max
);
782 pr_info("%15llu ", st
->wait_time_min
== ULLONG_MAX
?
783 0 : st
->wait_time_min
);
787 print_bad_events(bad
, total
);
790 static bool info_threads
, info_map
;
792 static void dump_threads(void)
794 struct thread_stat
*st
;
795 struct rb_node
*node
;
798 pr_info("%10s: comm\n", "Thread ID");
800 node
= rb_first(&thread_stats
);
802 st
= container_of(node
, struct thread_stat
, rb
);
803 t
= perf_session__findnew(session
, st
->tid
);
804 pr_info("%10d: %s\n", st
->tid
, t
->comm
);
805 node
= rb_next(node
);
809 static void dump_map(void)
812 struct lock_stat
*st
;
814 pr_info("Address of instance: name of class\n");
815 for (i
= 0; i
< LOCKHASH_SIZE
; i
++) {
816 list_for_each_entry(st
, &lockhash_table
[i
], hash_entry
) {
817 pr_info(" %p: %s\n", st
->addr
, st
->name
);
822 static void dump_info(void)
829 die("Unknown type of information\n");
832 static int process_sample_event(event_t
*self
, struct perf_session
*s
)
834 struct sample_data data
;
835 struct thread
*thread
;
837 bzero(&data
, sizeof(data
));
838 event__parse_sample(self
, s
->sample_type
, &data
);
840 thread
= perf_session__findnew(s
, data
.tid
);
841 if (thread
== NULL
) {
842 pr_debug("problem processing %d event, skipping it.\n",
847 process_raw_event(data
.raw_data
, data
.cpu
, data
.time
, thread
);
852 static struct perf_event_ops eops
= {
853 .sample
= process_sample_event
,
854 .comm
= event__process_comm
,
855 .ordered_samples
= true,
858 static int read_events(void)
860 session
= perf_session__new(input_name
, O_RDONLY
, 0, false);
862 die("Initializing perf session failed\n");
864 return perf_session__process_events(session
, &eops
);
867 static void sort_result(void)
870 struct lock_stat
*st
;
872 for (i
= 0; i
< LOCKHASH_SIZE
; i
++) {
873 list_for_each_entry(st
, &lockhash_table
[i
], hash_entry
) {
874 insert_to_result(st
, compare
);
879 static void __cmd_report(void)
888 static const char * const report_usage
[] = {
889 "perf lock report [<options>]",
893 static const struct option report_options
[] = {
894 OPT_STRING('k', "key", &sort_key
, "acquired",
900 static const char * const info_usage
[] = {
901 "perf lock info [<options>]",
905 static const struct option info_options
[] = {
906 OPT_BOOLEAN('t', "threads", &info_threads
,
907 "dump thread list in perf.data"),
908 OPT_BOOLEAN('m', "map", &info_map
,
909 "map of lock instances (name:address table)"),
913 static const char * const lock_usage
[] = {
914 "perf lock [<options>] {record|trace|report}",
918 static const struct option lock_options
[] = {
919 OPT_STRING('i', "input", &input_name
, "file", "input file name"),
920 OPT_INCR('v', "verbose", &verbose
, "be more verbose (show symbol address, etc)"),
921 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace
, "dump raw trace in ASCII"),
925 static const char *record_args
[] = {
931 "-e", "lock:lock_acquire:r",
932 "-e", "lock:lock_acquired:r",
933 "-e", "lock:lock_contended:r",
934 "-e", "lock:lock_release:r",
937 static int __cmd_record(int argc
, const char **argv
)
939 unsigned int rec_argc
, i
, j
;
940 const char **rec_argv
;
942 rec_argc
= ARRAY_SIZE(record_args
) + argc
- 1;
943 rec_argv
= calloc(rec_argc
+ 1, sizeof(char *));
945 for (i
= 0; i
< ARRAY_SIZE(record_args
); i
++)
946 rec_argv
[i
] = strdup(record_args
[i
]);
948 for (j
= 1; j
< (unsigned int)argc
; j
++, i
++)
949 rec_argv
[i
] = argv
[j
];
951 BUG_ON(i
!= rec_argc
);
953 return cmd_record(i
, rec_argv
, NULL
);
956 int cmd_lock(int argc
, const char **argv
, const char *prefix __used
)
961 for (i
= 0; i
< LOCKHASH_SIZE
; i
++)
962 INIT_LIST_HEAD(lockhash_table
+ i
);
964 argc
= parse_options(argc
, argv
, lock_options
, lock_usage
,
965 PARSE_OPT_STOP_AT_NON_OPTION
);
967 usage_with_options(lock_usage
, lock_options
);
969 if (!strncmp(argv
[0], "rec", 3)) {
970 return __cmd_record(argc
, argv
);
971 } else if (!strncmp(argv
[0], "report", 6)) {
972 trace_handler
= &report_lock_ops
;
974 argc
= parse_options(argc
, argv
,
975 report_options
, report_usage
, 0);
977 usage_with_options(report_usage
, report_options
);
980 } else if (!strcmp(argv
[0], "trace")) {
981 /* Aliased to 'perf trace' */
982 return cmd_trace(argc
, argv
, prefix
);
983 } else if (!strcmp(argv
[0], "info")) {
985 argc
= parse_options(argc
, argv
,
986 info_options
, info_usage
, 0);
988 usage_with_options(info_usage
, info_options
);
990 /* recycling report_lock_ops */
991 trace_handler
= &report_lock_ops
;
996 usage_with_options(lock_usage
, lock_options
);