?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
pahole: Describe expected use of 'default' in the man page
[dwarves.git] / btf_encoder.c
blob19e9d90f999591d2ad15c5e765d17ee5f97d3cd2
1 /*
2 SPDX-License-Identifier: GPL-2.0-only
3
4 Copyright (C) 2019 Facebook
5
6 Derived from ctf_encoder.c, which is:
7
8 Copyright (C) Arnaldo Carvalho de Melo <acme@redhat.com>
9 Copyright (C) Red Hat Inc
10 */
11
12 #include <linux/btf.h>
13 #include "dwarves.h"
14 #include "elf_symtab.h"
15 #include "btf_encoder.h"
16 #include "gobuffer.h"
17
18 #include <bpf/btf.h>
19 #include <bpf/libbpf.h>
20 #include <ctype.h> /* for isalpha() and isalnum() */
21 #include <stdlib.h> /* for qsort() and bsearch() */
22 #include <inttypes.h>
23 #include <limits.h>
24
25 #include <sys/types.h>
26 #include <sys/stat.h>
27 #include <fcntl.h>
28
29 #include <unistd.h>
30
31 #include <errno.h>
32 #include <stdint.h>
33 #include <search.h> /* for tsearch(), tfind() and tdestroy() */
34 #include <pthread.h>
35
36 #define BTF_ENCODER_MAX_PROTO 512
37
38 /* state used to do later encoding of saved functions */
39 struct btf_encoder_state {
40 uint32_t type_id_off;
41 bool got_proto;
42 char proto[BTF_ENCODER_MAX_PROTO];
43 };
44
45 struct elf_function {
46 const char *name;
47 bool generated;
48 size_t prefixlen;
49 struct function *function;
50 struct btf_encoder_state state;
51 };
52
53 struct var_info {
54 uint64_t addr;
55 const char *name;
56 uint32_t sz;
57 };
58
59 /*
60 * cu: cu being processed.
61 */
62 struct btf_encoder {
63 struct list_head node;
64 struct btf *btf;
65 struct cu *cu;
66 struct gobuffer percpu_secinfo;
67 const char *filename;
68 struct elf_symtab *symtab;
69 uint32_t type_id_off;
70 int saved_func_cnt;
71 bool has_index_type,
72 need_index_type,
73 skip_encoding_vars,
74 raw_output,
75 verbose,
76 force,
77 gen_floats,
78 is_rel;
79 uint32_t array_index_id;
80 struct {
81 struct var_info *vars;
82 int var_cnt;
83 int allocated;
84 uint32_t shndx;
85 uint64_t base_addr;
86 uint64_t sec_sz;
87 } percpu;
88 struct {
89 struct elf_function *entries;
90 int allocated;
91 int cnt;
92 int suffix_cnt; /* number of .isra, .part etc */
93 } functions;
94 };
95
96 static LIST_HEAD(encoders);
97 static pthread_mutex_t encoders__lock = PTHREAD_MUTEX_INITIALIZER;
98
99 static void btf_encoder__add_saved_funcs(struct btf_encoder *encoder);
100
101 /* mutex only needed for add/delete, as this can happen in multiple encoding
102 * threads. Traversal of the list is currently confined to thread collection.
103 */
104
105 #define btf_encoders__for_each_encoder(encoder) \
106 list_for_each_entry(encoder, &encoders, node)
107
108 static void btf_encoders__add(struct btf_encoder *encoder)
109 {
110 pthread_mutex_lock(&encoders__lock);
111 list_add_tail(&encoder->node, &encoders);
112 pthread_mutex_unlock(&encoders__lock);
113 }
114
115 static void btf_encoders__delete(struct btf_encoder *encoder)
116 {
117 struct btf_encoder *existing = NULL;
118
119 pthread_mutex_lock(&encoders__lock);
120 /* encoder may not have been added to list yet; check. */
121 btf_encoders__for_each_encoder(existing) {
122 if (encoder == existing)
123 break;
124 }
125 if (encoder == existing)
126 list_del(&encoder->node);
127 pthread_mutex_unlock(&encoders__lock);
128 }
129
130 #define PERCPU_SECTION ".data..percpu"
131
132 /*
133 * This depends on the GNU extension to eliminate the stray comma in the zero
134 * arguments case.
135 *
136 * The difference between elf_errmsg(-1) and elf_errmsg(elf_errno()) is that the
137 * latter clears the current error.
138 */
139 #define elf_error(fmt, ...) \
140 fprintf(stderr, "%s: " fmt ": %s.\n", __func__, ##__VA_ARGS__, elf_errmsg(-1))
141
142 /*
143 * This depends on the GNU extension to eliminate the stray comma in the zero
144 * arguments case.
145 *
146 * The difference between elf_errmsg(-1) and elf_errmsg(elf_errno()) is that the
147 * latter clears the current error.
148 */
149 #define elf_error(fmt, ...) \
150 fprintf(stderr, "%s: " fmt ": %s.\n", __func__, ##__VA_ARGS__, elf_errmsg(-1))
151
152 static int btf_var_secinfo_cmp(const void *a, const void *b)
153 {
154 const struct btf_var_secinfo *av = a;
155 const struct btf_var_secinfo *bv = b;
156
157 return av->offset - bv->offset;
158 }
159
160 #define BITS_PER_BYTE 8
161 #define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1)
162 #define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK)
163 #define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3)
164 #define BITS_ROUNDUP_BYTES(bits) (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits))
165
166 static const char * const btf_kind_str[] = {
167 [BTF_KIND_UNKN] = "UNKNOWN",
168 [BTF_KIND_INT] = "INT",
169 [BTF_KIND_PTR] = "PTR",
170 [BTF_KIND_ARRAY] = "ARRAY",
171 [BTF_KIND_STRUCT] = "STRUCT",
172 [BTF_KIND_UNION] = "UNION",
173 [BTF_KIND_ENUM] = "ENUM",
174 [BTF_KIND_FWD] = "FWD",
175 [BTF_KIND_TYPEDEF] = "TYPEDEF",
176 [BTF_KIND_VOLATILE] = "VOLATILE",
177 [BTF_KIND_CONST] = "CONST",
178 [BTF_KIND_RESTRICT] = "RESTRICT",
179 [BTF_KIND_FUNC] = "FUNC",
180 [BTF_KIND_FUNC_PROTO] = "FUNC_PROTO",
181 [BTF_KIND_VAR] = "VAR",
182 [BTF_KIND_DATASEC] = "DATASEC",
183 [BTF_KIND_FLOAT] = "FLOAT",
184 [BTF_KIND_DECL_TAG] = "DECL_TAG",
185 [BTF_KIND_TYPE_TAG] = "TYPE_TAG",
186 [BTF_KIND_ENUM64] = "ENUM64",
187 };
188
189 static const char *btf__printable_name(const struct btf *btf, uint32_t offset)
190 {
191 if (!offset)
192 return "(anon)";
193 else
194 return btf__str_by_offset(btf, offset);
195 }
196
197 static const char * btf__int_encoding_str(uint8_t encoding)
198 {
199 if (encoding == 0)
200 return "(none)";
201 else if (encoding == BTF_INT_SIGNED)
202 return "SIGNED";
203 else if (encoding == BTF_INT_CHAR)
204 return "CHAR";
205 else if (encoding == BTF_INT_BOOL)
206 return "BOOL";
207 else
208 return "UNKN";
209 }
210
211 __attribute ((format (printf, 5, 6)))
212 static void btf__log_err(const struct btf *btf, int kind, const char *name,
213 bool output_cr, const char *fmt, ...)
214 {
215 fprintf(stderr, "[%u] %s %s", btf__type_cnt(btf),
216 btf_kind_str[kind], name ?: "(anon)");
217
218 if (fmt && *fmt) {
219 va_list ap;
220
221 fprintf(stderr, " ");
222 va_start(ap, fmt);
223 vfprintf(stderr, fmt, ap);
224 va_end(ap);
225 }
226
227 if (output_cr)
228 fprintf(stderr, "\n");
229 }
230
231 __attribute ((format (printf, 5, 6)))
232 static void btf_encoder__log_type(const struct btf_encoder *encoder, const struct btf_type *t,
233 bool err, bool output_cr, const char *fmt, ...)
234 {
235 const struct btf *btf = encoder->btf;
236 uint8_t kind;
237 FILE *out;
238
239 if (!encoder->verbose && !err)
240 return;
241
242 kind = BTF_INFO_KIND(t->info);
243 out = err ? stderr : stdout;
244
245 fprintf(out, "[%u] %s %s",
246 btf__type_cnt(btf) - 1, btf_kind_str[kind],
247 btf__printable_name(btf, t->name_off));
248
249 if (fmt && *fmt) {
250 va_list ap;
251
252 fprintf(out, " ");
253 va_start(ap, fmt);
254 vfprintf(out, fmt, ap);
255 va_end(ap);
256 }
257
258 if (output_cr)
259 fprintf(out, "\n");
260 }
261
262 __attribute ((format (printf, 5, 6)))
263 static void btf_encoder__log_member(const struct btf_encoder *encoder, const struct btf_type *t,
264 const struct btf_member *member, bool err, const char *fmt, ...)
265 {
266 const struct btf *btf = encoder->btf;
267 FILE *out;
268
269 if (!encoder->verbose && !err)
270 return;
271
272 out = err ? stderr : stdout;
273
274 if (btf_kflag(t))
275 fprintf(out, "\t%s type_id=%u bitfield_size=%u bits_offset=%u",
276 btf__printable_name(btf, member->name_off),
277 member->type,
278 BTF_MEMBER_BITFIELD_SIZE(member->offset),
279 BTF_MEMBER_BIT_OFFSET(member->offset));
280 else
281 fprintf(out, "\t%s type_id=%u bits_offset=%u",
282 btf__printable_name(btf, member->name_off),
283 member->type,
284 member->offset);
285
286 if (fmt && *fmt) {
287 va_list ap;
288
289 fprintf(out, " ");
290 va_start(ap, fmt);
291 vfprintf(out, fmt, ap);
292 va_end(ap);
293 }
294
295 fprintf(out, "\n");
296 }
297
298 __attribute ((format (printf, 6, 7)))
299 static void btf_encoder__log_func_param(struct btf_encoder *encoder, const char *name, uint32_t type,
300 bool err, bool is_last_param, const char *fmt, ...)
301 {
302 FILE *out;
303
304 if (!encoder->verbose && !err)
305 return;
306
307 out = err ? stderr : stdout;
308
309 if (is_last_param && !type)
310 fprintf(out, "vararg)\n");
311 else
312 fprintf(out, "%u %s%s", type, name, is_last_param ? ")\n" : ", ");
313
314 if (fmt && *fmt) {
315 va_list ap;
316
317 fprintf(out, " ");
318 va_start(ap, fmt);
319 vfprintf(out, fmt, ap);
320 va_end(ap);
321 }
322 }
323
324 static int32_t btf_encoder__add_float(struct btf_encoder *encoder, const struct base_type *bt, const char *name)
325 {
326 int32_t id = btf__add_float(encoder->btf, name, BITS_ROUNDUP_BYTES(bt->bit_size));
327
328 if (id < 0) {
329 btf__log_err(encoder->btf, BTF_KIND_FLOAT, name, true, "Error emitting BTF type");
330 } else {
331 const struct btf_type *t;
332
333 t = btf__type_by_id(encoder->btf, id);
334 btf_encoder__log_type(encoder, t, false, true, "size=%u nr_bits=%u", t->size, bt->bit_size);
335 }
336
337 return id;
338 }
339
340 static int32_t btf_encoder__add_base_type(struct btf_encoder *encoder, const struct base_type *bt, const char *name)
341 {
342 const struct btf_type *t;
343 uint8_t encoding = 0;
344 uint16_t byte_sz;
345 int32_t id;
346
347 if (bt->is_signed) {
348 encoding = BTF_INT_SIGNED;
349 } else if (bt->is_bool) {
350 encoding = BTF_INT_BOOL;
351 } else if (bt->float_type && encoder->gen_floats) {
352 /*
353 * Encode floats as BTF_KIND_FLOAT if allowed, otherwise (in
354 * compatibility mode) encode them as BTF_KIND_INT - that's not
355 * fully correct, but that's what it used to be.
356 */
357 if (bt->float_type == BT_FP_SINGLE ||
358 bt->float_type == BT_FP_DOUBLE ||
359 bt->float_type == BT_FP_LDBL)
360 return btf_encoder__add_float(encoder, bt, name);
361 fprintf(stderr, "Complex, interval and imaginary float types are not supported\n");
362 return -1;
363 }
364
365 /* dwarf5 may emit DW_ATE_[un]signed_{num} base types where
366 * {num} is not power of 2 and may exceed 128. Such attributes
367 * are mostly used to record operation for an actual parameter
368 * or variable.
369 * For example,
370 * DW_AT_location (indexed (0x3c) loclist = 0x00008fb0:
371 * [0xffffffff82808812, 0xffffffff82808817):
372 * DW_OP_breg0 RAX+0,
373 * DW_OP_convert (0x000e97d5) "DW_ATE_unsigned_64",
374 * DW_OP_convert (0x000e97df) "DW_ATE_unsigned_8",
375 * DW_OP_stack_value,
376 * DW_OP_piece 0x1,
377 * DW_OP_breg0 RAX+0,
378 * DW_OP_convert (0x000e97d5) "DW_ATE_unsigned_64",
379 * DW_OP_convert (0x000e97da) "DW_ATE_unsigned_32",
380 * DW_OP_lit8,
381 * DW_OP_shr,
382 * DW_OP_convert (0x000e97da) "DW_ATE_unsigned_32",
383 * DW_OP_convert (0x000e97e4) "DW_ATE_unsigned_24",
384 * DW_OP_stack_value, DW_OP_piece 0x3
385 * DW_AT_name ("ebx")
386 * DW_AT_decl_file ("/linux/arch/x86/events/intel/core.c")
387 *
388 * In the above example, at some point, one unsigned_32 value
389 * is right shifted by 8 and the result is converted to unsigned_32
390 * and then unsigned_24.
391 *
392 * BTF does not need such DW_OP_* information so let us sanitize
393 * these non-regular int types to avoid libbpf/kernel complaints.
394 */
395 byte_sz = BITS_ROUNDUP_BYTES(bt->bit_size);
396 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16) {
397 name = "__SANITIZED_FAKE_INT__";
398 byte_sz = 4;
399 }
400
401 id = btf__add_int(encoder->btf, name, byte_sz, encoding);
402 if (id < 0) {
403 btf__log_err(encoder->btf, BTF_KIND_INT, name, true, "Error emitting BTF type");
404 } else {
405 t = btf__type_by_id(encoder->btf, id);
406 btf_encoder__log_type(encoder, t, false, true, "size=%u nr_bits=%u encoding=%s%s",
407 t->size, bt->bit_size, btf__int_encoding_str(encoding),
408 id < 0 ? " Error in emitting BTF" : "" );
409 }
410
411 return id;
412 }
413
414 static int32_t btf_encoder__add_ref_type(struct btf_encoder *encoder, uint16_t kind, uint32_t type,
415 const char *name, bool kind_flag)
416 {
417 struct btf *btf = encoder->btf;
418 const struct btf_type *t;
419 int32_t id;
420
421 switch (kind) {
422 case BTF_KIND_PTR:
423 id = btf__add_ptr(btf, type);
424 break;
425 case BTF_KIND_VOLATILE:
426 id = btf__add_volatile(btf, type);
427 break;
428 case BTF_KIND_CONST:
429 id = btf__add_const(btf, type);
430 break;
431 case BTF_KIND_RESTRICT:
432 id = btf__add_restrict(btf, type);
433 break;
434 case BTF_KIND_TYPEDEF:
435 id = btf__add_typedef(btf, name, type);
436 break;
437 case BTF_KIND_TYPE_TAG:
438 id = btf__add_type_tag(btf, name, type);
439 break;
440 case BTF_KIND_FWD:
441 id = btf__add_fwd(btf, name, kind_flag);
442 break;
443 case BTF_KIND_FUNC:
444 id = btf__add_func(btf, name, BTF_FUNC_STATIC, type);
445 break;
446 default:
447 btf__log_err(btf, kind, name, true, "Unexpected kind for reference");
448 return -1;
449 }
450
451 if (id > 0) {
452 t = btf__type_by_id(btf, id);
453 if (kind == BTF_KIND_FWD)
454 btf_encoder__log_type(encoder, t, false, true, "%s", kind_flag ? "union" : "struct");
455 else
456 btf_encoder__log_type(encoder, t, false, true, "type_id=%u", t->type);
457 } else {
458 btf__log_err(btf, kind, name, true, "Error emitting BTF type");
459 }
460 return id;
461 }
462
463 static int32_t btf_encoder__add_array(struct btf_encoder *encoder, uint32_t type, uint32_t index_type, uint32_t nelems)
464 {
465 struct btf *btf = encoder->btf;
466 const struct btf_type *t;
467 const struct btf_array *array;
468 int32_t id;
469
470 id = btf__add_array(btf, index_type, type, nelems);
471 if (id > 0) {
472 t = btf__type_by_id(btf, id);
473 array = btf_array(t);
474 btf_encoder__log_type(encoder, t, false, true, "type_id=%u index_type_id=%u nr_elems=%u",
475 array->type, array->index_type, array->nelems);
476 } else {
477 btf__log_err(btf, BTF_KIND_ARRAY, NULL, true,
478 "type_id=%u index_type_id=%u nr_elems=%u Error emitting BTF type",
479 type, index_type, nelems);
480 }
481 return id;
482 }
483
484 static int btf_encoder__add_field(struct btf_encoder *encoder, const char *name, uint32_t type, uint32_t bitfield_size, uint32_t offset)
485 {
486 struct btf *btf = encoder->btf;
487 const struct btf_type *t;
488 const struct btf_member *m;
489 int err;
490
491 err = btf__add_field(btf, name, type, offset, bitfield_size);
492 t = btf__type_by_id(btf, btf__type_cnt(btf) - 1);
493 if (err) {
494 fprintf(stderr, "[%u] %s %s's field '%s' offset=%u bit_size=%u type=%u Error emitting field\n",
495 btf__type_cnt(btf) - 1, btf_kind_str[btf_kind(t)],
496 btf__printable_name(btf, t->name_off),
497 name, offset, bitfield_size, type);
498 } else {
499 m = &btf_members(t)[btf_vlen(t) - 1];
500 btf_encoder__log_member(encoder, t, m, false, NULL);
501 }
502 return err;
503 }
504
505 static int32_t btf_encoder__add_struct(struct btf_encoder *encoder, uint8_t kind, const char *name, uint32_t size)
506 {
507 struct btf *btf = encoder->btf;
508 const struct btf_type *t;
509 int32_t id;
510
511 switch (kind) {
512 case BTF_KIND_STRUCT:
513 id = btf__add_struct(btf, name, size);
514 break;
515 case BTF_KIND_UNION:
516 id = btf__add_union(btf, name, size);
517 break;
518 default:
519 btf__log_err(btf, kind, name, true, "Unexpected kind of struct");
520 return -1;
521 }
522
523 if (id < 0) {
524 btf__log_err(btf, kind, name, true, "Error emitting BTF type");
525 } else {
526 t = btf__type_by_id(btf, id);
527 btf_encoder__log_type(encoder, t, false, true, "size=%u", t->size);
528 }
529
530 return id;
531 }
532
533 #if LIBBPF_MAJOR_VERSION < 1
534 static inline int libbpf_err(int ret)
535 {
536 if (ret < 0)
537 errno = -ret;
538 return ret;
539 }
540
541 static
542 int btf__add_enum64(struct btf *btf __maybe_unused, const char *name __maybe_unused,
543 __u32 byte_sz __maybe_unused, bool is_signed __maybe_unused)
544 {
545 return libbpf_err(-ENOTSUP);
546 }
547
548 static
549 int btf__add_enum64_value(struct btf *btf __maybe_unused, const char *name __maybe_unused,
550 __u64 value __maybe_unused)
551 {
552 return libbpf_err(-ENOTSUP);
553 }
554 #endif
555
556 static int32_t btf_encoder__add_enum(struct btf_encoder *encoder, const char *name, struct type *etype,
557 struct conf_load *conf_load)
558 {
559 struct btf *btf = encoder->btf;
560 const struct btf_type *t;
561 int32_t id, size;
562 bool is_enum32;
563
564 size = BITS_ROUNDUP_BYTES(etype->size);
565 is_enum32 = size <= 4 || conf_load->skip_encoding_btf_enum64;
566 if (is_enum32)
567 id = btf__add_enum(btf, name, size);
568 else
569 id = btf__add_enum64(btf, name, size, etype->is_signed_enum);
570 if (id > 0) {
571 t = btf__type_by_id(btf, id);
572 btf_encoder__log_type(encoder, t, false, true, "size=%u", t->size);
573 } else {
574 btf__log_err(btf, is_enum32 ? BTF_KIND_ENUM : BTF_KIND_ENUM64, name, true,
575 "size=%u Error emitting BTF type", size);
576 }
577 return id;
578 }
579
580 static int btf_encoder__add_enum_val(struct btf_encoder *encoder, const char *name, int64_t value,
581 struct type *etype, struct conf_load *conf_load)
582 {
583 const char *fmt_str;
584 int err;
585
586 /* If enum64 is not allowed, generate enum32 with unsigned int value. In enum64-supported
587 * libbpf library, btf__add_enum_value() will set the kflag (sign bit) in common_type
588 * if the value is negative.
589 */
590 if (conf_load->skip_encoding_btf_enum64)
591 err = btf__add_enum_value(encoder->btf, name, (uint32_t)value);
592 else if (etype->size > 32)
593 err = btf__add_enum64_value(encoder->btf, name, value);
594 else
595 err = btf__add_enum_value(encoder->btf, name, value);
596
597 if (!err) {
598 if (encoder->verbose) {
599 if (conf_load->skip_encoding_btf_enum64) {
600 printf("\t%s val=%u\n", name, (uint32_t)value);
601 } else {
602 fmt_str = etype->is_signed_enum ? "\t%s val=%lld\n" : "\t%s val=%llu\n";
603 printf(fmt_str, name, (unsigned long long)value);
604 }
605 }
606 } else {
607 if (conf_load->skip_encoding_btf_enum64) {
608 fprintf(stderr, "\t%s val=%u Error emitting BTF enum value\n", name, (uint32_t)value);
609 } else {
610 fmt_str = etype->is_signed_enum ? "\t%s val=%lld Error emitting BTF enum value\n"
611 : "\t%s val=%llu Error emitting BTF enum value\n";
612 fprintf(stderr, fmt_str, name, (unsigned long long)value);
613 }
614 }
615 return err;
616 }
617
618 static int32_t btf_encoder__add_func_param(struct btf_encoder *encoder, const char *name, uint32_t type, bool is_last_param)
619 {
620 int err = btf__add_func_param(encoder->btf, name, type);
621
622 if (!err) {
623 btf_encoder__log_func_param(encoder, name, type, false, is_last_param, NULL);
624 return 0;
625 } else {
626 btf_encoder__log_func_param(encoder, name, type, true, is_last_param, "Error adding func param");
627 return -1;
628 }
629 }
630
631 static int32_t btf_encoder__tag_type(struct btf_encoder *encoder, uint32_t tag_type)
632 {
633 if (tag_type == 0)
634 return 0;
635
636 return encoder->type_id_off + tag_type;
637 }
638
639 static int32_t btf_encoder__add_func_proto(struct btf_encoder *encoder, struct ftype *ftype)
640 {
641 struct btf *btf = encoder->btf;
642 const struct btf_type *t;
643 struct parameter *param;
644 uint16_t nr_params, param_idx;
645 int32_t id, type_id;
646
647 /* add btf_type for func_proto */
648 nr_params = ftype->nr_parms + (ftype->unspec_parms ? 1 : 0);
649 type_id = btf_encoder__tag_type(encoder, ftype->tag.type);
650
651 id = btf__add_func_proto(btf, type_id);
652 if (id > 0) {
653 t = btf__type_by_id(btf, id);
654 btf_encoder__log_type(encoder, t, false, false, "return=%u args=(%s", t->type, !nr_params ? "void)\n" : "");
655 } else {
656 btf__log_err(btf, BTF_KIND_FUNC_PROTO, NULL, true,
657 "return=%u vlen=%u Error emitting BTF type",
658 type_id, nr_params);
659 return id;
660 }
661
662 /* add parameters */
663 param_idx = 0;
664 ftype__for_each_parameter(ftype, param) {
665 const char *name = parameter__name(param);
666
667 type_id = param->tag.type == 0 ? 0 : encoder->type_id_off + param->tag.type;
668 ++param_idx;
669 if (btf_encoder__add_func_param(encoder, name, type_id, param_idx == nr_params))
670 return -1;
671 }
672
673 ++param_idx;
674 if (ftype->unspec_parms)
675 if (btf_encoder__add_func_param(encoder, NULL, 0, param_idx == nr_params))
676 return -1;
677
678 return id;
679 }
680
681 static int32_t btf_encoder__add_var(struct btf_encoder *encoder, uint32_t type, const char *name, uint32_t linkage)
682 {
683 struct btf *btf = encoder->btf;
684 const struct btf_type *t;
685 int32_t id;
686
687 id = btf__add_var(btf, name, linkage, type);
688 if (id > 0) {
689 t = btf__type_by_id(btf, id);
690 btf_encoder__log_type(encoder, t, false, true, "type=%u linkage=%u", t->type, btf_var(t)->linkage);
691 } else {
692 btf__log_err(btf, BTF_KIND_VAR, name, true,
693 "type=%u linkage=%u Error emitting BTF type",
694 type, linkage);
695 }
696 return id;
697 }
698
699 static int32_t btf_encoder__add_var_secinfo(struct btf_encoder *encoder, uint32_t type,
700 uint32_t offset, uint32_t size)
701 {
702 struct btf_var_secinfo si = {
703 .type = type,
704 .offset = offset,
705 .size = size,
706 };
707 return gobuffer__add(&encoder->percpu_secinfo, &si, sizeof(si));
708 }
709
710 int32_t btf_encoder__add_encoder(struct btf_encoder *encoder, struct btf_encoder *other)
711 {
712 struct gobuffer *var_secinfo_buf = &other->percpu_secinfo;
713 size_t sz = gobuffer__size(var_secinfo_buf);
714 uint16_t nr_var_secinfo = sz / sizeof(struct btf_var_secinfo);
715 uint32_t type_id;
716 uint32_t next_type_id = btf__type_cnt(encoder->btf);
717 int32_t i, id;
718 struct btf_var_secinfo *vsi;
719
720 if (encoder == other)
721 return 0;
722
723 btf_encoder__add_saved_funcs(other);
724
725 for (i = 0; i < nr_var_secinfo; i++) {
726 vsi = (struct btf_var_secinfo *)var_secinfo_buf->entries + i;
727 type_id = next_type_id + vsi->type - 1; /* Type ID starts from 1 */
728 id = btf_encoder__add_var_secinfo(encoder, type_id, vsi->offset, vsi->size);
729 if (id < 0)
730 return id;
731 }
732
733 return btf__add_btf(encoder->btf, other->btf);
734 }
735
736 static int32_t btf_encoder__add_datasec(struct btf_encoder *encoder, const char *section_name)
737 {
738 struct gobuffer *var_secinfo_buf = &encoder->percpu_secinfo;
739 struct btf *btf = encoder->btf;
740 size_t sz = gobuffer__size(var_secinfo_buf);
741 uint16_t nr_var_secinfo = sz / sizeof(struct btf_var_secinfo);
742 struct btf_var_secinfo *last_vsi, *vsi;
743 const struct btf_type *t;
744 uint32_t datasec_sz;
745 int32_t err, id, i;
746
747 qsort(var_secinfo_buf->entries, nr_var_secinfo,
748 sizeof(struct btf_var_secinfo), btf_var_secinfo_cmp);
749
750 last_vsi = (struct btf_var_secinfo *)var_secinfo_buf->entries + nr_var_secinfo - 1;
751 datasec_sz = last_vsi->offset + last_vsi->size;
752
753 id = btf__add_datasec(btf, section_name, datasec_sz);
754 if (id < 0) {
755 btf__log_err(btf, BTF_KIND_DATASEC, section_name, true,
756 "size=%u vlen=%u Error emitting BTF type",
757 datasec_sz, nr_var_secinfo);
758 } else {
759 t = btf__type_by_id(btf, id);
760 btf_encoder__log_type(encoder, t, false, true, "size=%u vlen=%u", t->size, nr_var_secinfo);
761 }
762
763 for (i = 0; i < nr_var_secinfo; i++) {
764 vsi = (struct btf_var_secinfo *)var_secinfo_buf->entries + i;
765 err = btf__add_datasec_var_info(btf, vsi->type, vsi->offset, vsi->size);
766 if (!err) {
767 if (encoder->verbose)
768 printf("\ttype=%u offset=%u size=%u\n",
769 vsi->type, vsi->offset, vsi->size);
770 } else {
771 fprintf(stderr, "\ttype=%u offset=%u size=%u Error emitting BTF datasec var info\n",
772 vsi->type, vsi->offset, vsi->size);
773 return -1;
774 }
775 }
776
777 return id;
778 }
779
780 static int32_t btf_encoder__add_decl_tag(struct btf_encoder *encoder, const char *value, uint32_t type,
781 int component_idx)
782 {
783 struct btf *btf = encoder->btf;
784 const struct btf_type *t;
785 int32_t id;
786
787 id = btf__add_decl_tag(btf, value, type, component_idx);
788 if (id > 0) {
789 t = btf__type_by_id(btf, id);
790 btf_encoder__log_type(encoder, t, false, true, "type_id=%u component_idx=%d",
791 t->type, component_idx);
792 } else {
793 btf__log_err(btf, BTF_KIND_DECL_TAG, value, true, "component_idx=%d Error emitting BTF type",
794 component_idx);
795 }
796
797 return id;
798 }
799
800 static bool proto__get(struct function *func, char *proto, size_t len)
801 {
802 const struct conf_fprintf conf = {
803 .name_spacing = 23,
804 .type_spacing = 26,
805 .emit_stats = 0,
806 .no_parm_names = 1,
807 .skip_emitting_errors = 1,
808 .skip_emitting_modifier = 1,
809 };
810
811 return function__prototype_conf(func, func->priv, &conf, proto, len) != NULL;
812 }
813
814 static bool funcs__match(struct btf_encoder *encoder, struct elf_function *func, struct function *f2)
815 {
816 char proto[BTF_ENCODER_MAX_PROTO];
817 struct function *f1 = func->function;
818 const char *name;
819
820 if (!f1)
821 return false;
822
823 name = function__name(f1);
824
825 if (f1->proto.nr_parms != f2->proto.nr_parms) {
826 if (encoder->verbose)
827 printf("function mismatch for '%s'(%s): %d params != %d params\n",
828 name, f1->alias ?: name,
829 f1->proto.nr_parms, f2->proto.nr_parms);
830 return false;
831 }
832 if (f1->proto.nr_parms == 0)
833 return true;
834
835 if (f1->proto.tag.type == f2->proto.tag.type)
836 return true;
837
838 if (!func->state.got_proto)
839 func->state.got_proto = proto__get(f1, func->state.proto, sizeof(func->state.proto));
840
841 if (proto__get(f2, proto, sizeof(proto))) {
842 if (strcmp(func->state.proto, proto) != 0) {
843 if (encoder->verbose)
844 printf("function mismatch for '%s'('%s'): '%s' != '%s'\n",
845 name, f1->alias ?: name,
846 func->state.proto, proto);
847 return false;
848 }
849 }
850 return true;
851 }
852
853 static int32_t btf_encoder__save_func(struct btf_encoder *encoder, struct function *fn, struct elf_function *func)
854 {
855 fn->priv = encoder->cu;
856 if (func->function) {
857 struct function *existing = func->function;
858
859 /* If saving and we find an existing entry, we want to merge
860 * observations across both functions, checking that the
861 * "seen optimized parameters", "inconsistent prototype"
862 * and "unexpected register" status is reflected in the
863 * the func entry.
864 * If the entry is new, record encoder state required
865 * to add the local function later (encoder + type_id_off)
866 * such that we can add the function later.
867 */
868 existing->proto.optimized_parms |= fn->proto.optimized_parms;
869 existing->proto.unexpected_reg |= fn->proto.unexpected_reg;
870 if (!existing->proto.unexpected_reg && !existing->proto.inconsistent_proto &&
871 !funcs__match(encoder, func, fn))
872 existing->proto.inconsistent_proto = 1;
873 } else {
874 func->state.type_id_off = encoder->type_id_off;
875 func->function = fn;
876 encoder->saved_func_cnt++;
877 }
878 return 0;
879 }
880
881 static int32_t btf_encoder__add_func(struct btf_encoder *encoder, struct function *fn)
882 {
883 int btf_fnproto_id, btf_fn_id, tag_type_id;
884 struct llvm_annotation *annot;
885 const char *name;
886
887 btf_fnproto_id = btf_encoder__add_func_proto(encoder, &fn->proto);
888 name = function__name(fn);
889 btf_fn_id = btf_encoder__add_ref_type(encoder, BTF_KIND_FUNC, btf_fnproto_id, name, false);
890 if (btf_fnproto_id < 0 || btf_fn_id < 0) {
891 printf("error: failed to encode function '%s'\n", function__name(fn));
892 return -1;
893 }
894 list_for_each_entry(annot, &fn->annots, node) {
895 tag_type_id = btf_encoder__add_decl_tag(encoder, annot->value, btf_fn_id,
896 annot->component_idx);
897 if (tag_type_id < 0) {
898 fprintf(stderr, "error: failed to encode tag '%s' to func %s with component_idx %d\n",
899 annot->value, name, annot->component_idx);
900 return -1;
901 }
902 }
903 return 0;
904 }
905
906 static void btf_encoder__add_saved_funcs(struct btf_encoder *encoder)
907 {
908 int i;
909
910 for (i = 0; i < encoder->functions.cnt; i++) {
911 struct elf_function *func = &encoder->functions.entries[i];
912 struct function *fn = func->function;
913 struct btf_encoder *other_encoder;
914
915 if (!fn || fn->proto.processed)
916 continue;
917
918 /* merge optimized-out status across encoders; since each
919 * encoder has the same elf symbol table we can use the
920 * same index to access the same elf symbol.
921 */
922 btf_encoders__for_each_encoder(other_encoder) {
923 struct function *other_fn;
924
925 if (other_encoder == encoder)
926 continue;
927
928 other_fn = other_encoder->functions.entries[i].function;
929 if (!other_fn)
930 continue;
931 fn->proto.optimized_parms |= other_fn->proto.optimized_parms;
932 fn->proto.unexpected_reg |= other_fn->proto.unexpected_reg;
933 if (other_fn->proto.inconsistent_proto)
934 fn->proto.inconsistent_proto = 1;
935 if (!fn->proto.unexpected_reg && !fn->proto.inconsistent_proto &&
936 !funcs__match(encoder, func, other_fn))
937 fn->proto.inconsistent_proto = 1;
938 other_fn->proto.processed = 1;
939 }
940 /* do not exclude functions with optimized-out parameters; they
941 * may still be _called_ with the right parameter values, they
942 * just do not _use_ them. Only exclude functions with
943 * unexpected register use or multiple inconsistent prototypes.
944 */
945 if (fn->proto.unexpected_reg || fn->proto.inconsistent_proto) {
946 if (encoder->verbose) {
947 const char *name = function__name(fn);
948
949 printf("skipping addition of '%s'(%s) due to %s\n",
950 name, fn->alias ?: name,
951 fn->proto.unexpected_reg ? "unexpected register used for parameter" :
952 "multiple inconsistent function prototypes");
953 }
954 } else {
955 encoder->type_id_off = func->state.type_id_off;
956 btf_encoder__add_func(encoder, fn);
957 }
958 fn->proto.processed = 1;
959 }
960 }
961
962 /*
963 * This corresponds to the same macro defined in
964 * include/linux/kallsyms.h
965 */
966 #define KSYM_NAME_LEN 128
967
968 static int functions_cmp(const void *_a, const void *_b)
969 {
970 const struct elf_function *a = _a;
971 const struct elf_function *b = _b;
972
973 /* if search key allows prefix match, verify target has matching
974 * prefix len and prefix matches.
975 */
976 if (a->prefixlen && a->prefixlen == b->prefixlen)
977 return strncmp(a->name, b->name, b->prefixlen);
978 return strcmp(a->name, b->name);
979 }
980
981 #ifndef max
982 #define max(x, y) ((x) < (y) ? (y) : (x))
983 #endif
984
985 static void *reallocarray_grow(void *ptr, int *nmemb, size_t size)
986 {
987 int new_nmemb = max(1000, *nmemb * 3 / 2);
988 void *new = realloc(ptr, new_nmemb * size);
989
990 if (new)
991 *nmemb = new_nmemb;
992 return new;
993 }
994
995 static int btf_encoder__collect_function(struct btf_encoder *encoder, GElf_Sym *sym)
996 {
997 struct elf_function *new;
998 const char *name;
999
1000 if (elf_sym__type(sym) != STT_FUNC)
1001 return 0;
1002 name = elf_sym__name(sym, encoder->symtab);
1003 if (!name)
1004 return 0;
1005
1006 if (encoder->functions.cnt == encoder->functions.allocated) {
1007 new = reallocarray_grow(encoder->functions.entries,
1008 &encoder->functions.allocated,
1009 sizeof(*encoder->functions.entries));
1010 if (!new) {
1011 /*
1012 * The cleanup - delete_functions is called
1013 * in btf_encoder__encode_cu error path.
1014 */
1015 return -1;
1016 }
1017 encoder->functions.entries = new;
1018 }
1019
1020 encoder->functions.entries[encoder->functions.cnt].name = name;
1021 if (strchr(name, '.')) {
1022 const char *suffix = strchr(name, '.');
1023
1024 encoder->functions.suffix_cnt++;
1025 encoder->functions.entries[encoder->functions.cnt].prefixlen = suffix - name;
1026 }
1027 encoder->functions.entries[encoder->functions.cnt].generated = false;
1028 encoder->functions.entries[encoder->functions.cnt].function = NULL;
1029 encoder->functions.entries[encoder->functions.cnt].state.got_proto = false;
1030 encoder->functions.entries[encoder->functions.cnt].state.proto[0] = '\0';
1031 encoder->functions.entries[encoder->functions.cnt].state.type_id_off = 0;
1032 encoder->functions.cnt++;
1033 return 0;
1034 }
1035
1036 static struct elf_function *btf_encoder__find_function(const struct btf_encoder *encoder,
1037 const char *name, size_t prefixlen)
1038 {
1039 struct elf_function key = { .name = name, .prefixlen = prefixlen };
1040
1041 return bsearch(&key, encoder->functions.entries, encoder->functions.cnt, sizeof(key), functions_cmp);
1042 }
1043
1044 static bool btf_name_char_ok(char c, bool first)
1045 {
1046 if (c == '_' || c == '.')
1047 return true;
1048
1049 return first ? isalpha(c) : isalnum(c);
1050 }
1051
1052 /* Check whether the given name is valid in vmlinux btf. */
1053 static bool btf_name_valid(const char *p)
1054 {
1055 const char *limit;
1056
1057 if (!btf_name_char_ok(*p, true))
1058 return false;
1059
1060 /* set a limit on identifier length */
1061 limit = p + KSYM_NAME_LEN;
1062 p++;
1063 while (*p && p < limit) {
1064 if (!btf_name_char_ok(*p, false))
1065 return false;
1066 p++;
1067 }
1068
1069 return !*p;
1070 }
1071
1072 static void dump_invalid_symbol(const char *msg, const char *sym,
1073 int verbose, bool force)
1074 {
1075 if (force) {
1076 if (verbose)
1077 fprintf(stderr, "PAHOLE: Warning: %s, ignored (sym: '%s').\n",
1078 msg, sym);
1079 return;
1080 }
1081
1082 fprintf(stderr, "PAHOLE: Error: %s (sym: '%s').\n", msg, sym);
1083 fprintf(stderr, "PAHOLE: Error: Use '--btf_encode_force' to ignore such symbols and force emit the btf.\n");
1084 }
1085
1086 static int tag__check_id_drift(struct btf_encoder *encoder, const struct tag *tag,
1087 uint32_t core_id, uint32_t btf_type_id)
1088 {
1089 if (btf_type_id != (core_id + encoder->type_id_off)) {
1090 fprintf(stderr,
1091 "%s: %s id drift, core_id: %u, btf_type_id: %u, type_id_off: %u\n",
1092 __func__, dwarf_tag_name(tag->tag),
1093 core_id, btf_type_id, encoder->type_id_off);
1094 return -1;
1095 }
1096
1097 return 0;
1098 }
1099
1100 static int32_t btf_encoder__add_struct_type(struct btf_encoder *encoder, struct tag *tag)
1101 {
1102 struct type *type = tag__type(tag);
1103 struct class_member *pos;
1104 const char *name = type__name(type);
1105 int32_t type_id;
1106 uint8_t kind;
1107
1108 kind = (tag->tag == DW_TAG_union_type) ?
1109 BTF_KIND_UNION : BTF_KIND_STRUCT;
1110
1111 type_id = btf_encoder__add_struct(encoder, kind, name, type->size);
1112 if (type_id < 0)
1113 return type_id;
1114
1115 type__for_each_data_member(type, pos) {
1116 /*
1117 * dwarf_loader uses DWARF's recommended bit offset addressing
1118 * scheme, which conforms to BTF requirement, so no conversion
1119 * is required.
1120 */
1121 name = class_member__name(pos);
1122 if (btf_encoder__add_field(encoder, name, encoder->type_id_off + pos->tag.type,
1123 pos->bitfield_size, pos->bit_offset))
1124 return -1;
1125 }
1126
1127 return type_id;
1128 }
1129
1130 static uint32_t array_type__nelems(struct tag *tag)
1131 {
1132 int i;
1133 uint32_t nelem = 1;
1134 struct array_type *array = tag__array_type(tag);
1135
1136 for (i = array->dimensions - 1; i >= 0; --i)
1137 nelem *= array->nr_entries[i];
1138
1139 return nelem;
1140 }
1141
1142 static int32_t btf_encoder__add_enum_type(struct btf_encoder *encoder, struct tag *tag,
1143 struct conf_load *conf_load)
1144 {
1145 struct type *etype = tag__type(tag);
1146 struct enumerator *pos;
1147 const char *name = type__name(etype);
1148 int32_t type_id;
1149
1150 type_id = btf_encoder__add_enum(encoder, name, etype, conf_load);
1151 if (type_id < 0)
1152 return type_id;
1153
1154 type__for_each_enumerator(etype, pos) {
1155 name = enumerator__name(pos);
1156 if (btf_encoder__add_enum_val(encoder, name, pos->value, etype, conf_load))
1157 return -1;
1158 }
1159
1160 return type_id;
1161 }
1162
1163 static int btf_encoder__encode_tag(struct btf_encoder *encoder, struct tag *tag,
1164 struct conf_load *conf_load)
1165 {
1166 /* single out type 0 as it represents special type "void" */
1167 uint32_t ref_type_id = tag->type == 0 ? 0 : encoder->type_id_off + tag->type;
1168 struct base_type *bt;
1169 const char *name;
1170
1171 switch (tag->tag) {
1172 case DW_TAG_base_type:
1173 bt = tag__base_type(tag);
1174 name = __base_type__name(bt);
1175 return btf_encoder__add_base_type(encoder, bt, name);
1176 case DW_TAG_const_type:
1177 return btf_encoder__add_ref_type(encoder, BTF_KIND_CONST, ref_type_id, NULL, false);
1178 case DW_TAG_pointer_type:
1179 return btf_encoder__add_ref_type(encoder, BTF_KIND_PTR, ref_type_id, NULL, false);
1180 case DW_TAG_restrict_type:
1181 return btf_encoder__add_ref_type(encoder, BTF_KIND_RESTRICT, ref_type_id, NULL, false);
1182 case DW_TAG_volatile_type:
1183 return btf_encoder__add_ref_type(encoder, BTF_KIND_VOLATILE, ref_type_id, NULL, false);
1184 case DW_TAG_typedef:
1185 name = namespace__name(tag__namespace(tag));
1186 return btf_encoder__add_ref_type(encoder, BTF_KIND_TYPEDEF, ref_type_id, name, false);
1187 case DW_TAG_LLVM_annotation:
1188 name = tag__btf_type_tag(tag)->value;
1189 return btf_encoder__add_ref_type(encoder, BTF_KIND_TYPE_TAG, ref_type_id, name, false);
1190 case DW_TAG_structure_type:
1191 case DW_TAG_union_type:
1192 case DW_TAG_class_type:
1193 name = namespace__name(tag__namespace(tag));
1194 if (tag__type(tag)->declaration)
1195 return btf_encoder__add_ref_type(encoder, BTF_KIND_FWD, 0, name, tag->tag == DW_TAG_union_type);
1196 else
1197 return btf_encoder__add_struct_type(encoder, tag);
1198 case DW_TAG_array_type:
1199 /* TODO: Encode one dimension at a time. */
1200 encoder->need_index_type = true;
1201 return btf_encoder__add_array(encoder, ref_type_id, encoder->array_index_id, array_type__nelems(tag));
1202 case DW_TAG_enumeration_type:
1203 return btf_encoder__add_enum_type(encoder, tag, conf_load);
1204 case DW_TAG_subroutine_type:
1205 return btf_encoder__add_func_proto(encoder, tag__ftype(tag));
1206 case DW_TAG_unspecified_type:
1207 /* Just don't encode this for now, converting anything with this type to void (0) instead.
1208 *
1209 * If we end up needing to encode this, one possible hack is to do as follows, as "const void".
1210 *
1211 * Returning zero means we skipped encoding a DWARF type.
1212 */
1213 // btf_encoder__add_ref_type(encoder, BTF_KIND_CONST, 0, NULL, false);
1214 return 0;
1215 default:
1216 fprintf(stderr, "Unsupported DW_TAG_%s(0x%x): type: 0x%x\n",
1217 dwarf_tag_name(tag->tag), tag->tag, ref_type_id);
1218 return -1;
1219 }
1220 }
1221
1222 static int btf_encoder__write_raw_file(struct btf_encoder *encoder)
1223 {
1224 const char *filename = encoder->filename;
1225 uint32_t raw_btf_size;
1226 const void *raw_btf_data;
1227 int fd, err;
1228
1229 raw_btf_data = btf__raw_data(encoder->btf, &raw_btf_size);
1230 if (raw_btf_data == NULL) {
1231 fprintf(stderr, "%s: btf__raw_data failed!\n", __func__);
1232 return -1;
1233 }
1234
1235 fd = open(filename, O_WRONLY | O_CREAT, 0640);
1236 if (fd < 0) {
1237 fprintf(stderr, "%s: Couldn't open %s for writing the raw BTF info: %s\n", __func__, filename, strerror(errno));
1238 return -1;
1239 }
1240 err = write(fd, raw_btf_data, raw_btf_size);
1241 if (err < 0)
1242 fprintf(stderr, "%s: Couldn't write the raw BTF info to %s: %s\n", __func__, filename, strerror(errno));
1243
1244 close(fd);
1245
1246 if ((uint32_t)err != raw_btf_size) {
1247 fprintf(stderr, "%s: Could only write %d bytes to %s of raw BTF info out of %d, aborting\n", __func__, err, filename, raw_btf_size);
1248 unlink(filename);
1249 err = -1;
1250 } else {
1251 /* go from bytes written == raw_btf_size to an indication that all went fine */
1252 err = 0;
1253 }
1254
1255 return err;
1256 }
1257
1258 static int btf_encoder__write_elf(struct btf_encoder *encoder)
1259 {
1260 struct btf *btf = encoder->btf;
1261 const char *filename = encoder->filename;
1262 GElf_Shdr shdr_mem, *shdr;
1263 Elf_Data *btf_data = NULL;
1264 Elf_Scn *scn = NULL;
1265 Elf *elf = NULL;
1266 const void *raw_btf_data;
1267 uint32_t raw_btf_size;
1268 int fd, err = -1;
1269 size_t strndx;
1270
1271 fd = open(filename, O_RDWR);
1272 if (fd < 0) {
1273 fprintf(stderr, "Cannot open %s\n", filename);
1274 return -1;
1275 }
1276
1277 if (elf_version(EV_CURRENT) == EV_NONE) {
1278 elf_error("Cannot set libelf version");
1279 goto out;
1280 }
1281
1282 elf = elf_begin(fd, ELF_C_RDWR, NULL);
1283 if (elf == NULL) {
1284 elf_error("Cannot update ELF file");
1285 goto out;
1286 }
1287
1288 elf_flagelf(elf, ELF_C_SET, ELF_F_DIRTY);
1289
1290 /*
1291 * First we look if there was already a .BTF section to overwrite.
1292 */
1293
1294 elf_getshdrstrndx(elf, &strndx);
1295 while ((scn = elf_nextscn(elf, scn)) != NULL) {
1296 shdr = gelf_getshdr(scn, &shdr_mem);
1297 if (shdr == NULL)
1298 continue;
1299 char *secname = elf_strptr(elf, strndx, shdr->sh_name);
1300 if (strcmp(secname, ".BTF") == 0) {
1301 btf_data = elf_getdata(scn, btf_data);
1302 break;
1303 }
1304 }
1305
1306 raw_btf_data = btf__raw_data(btf, &raw_btf_size);
1307
1308 if (btf_data) {
1309 /* Existing .BTF section found */
1310 btf_data->d_buf = (void *)raw_btf_data;
1311 btf_data->d_size = raw_btf_size;
1312 elf_flagdata(btf_data, ELF_C_SET, ELF_F_DIRTY);
1313
1314 if (elf_update(elf, ELF_C_NULL) >= 0 &&
1315 elf_update(elf, ELF_C_WRITE) >= 0)
1316 err = 0;
1317 else
1318 elf_error("elf_update failed");
1319 } else {
1320 const char *llvm_objcopy;
1321 char tmp_fn[PATH_MAX];
1322 char cmd[PATH_MAX * 2];
1323
1324 llvm_objcopy = getenv("LLVM_OBJCOPY");
1325 if (!llvm_objcopy)
1326 llvm_objcopy = "llvm-objcopy";
1327
1328 /* Use objcopy to add a .BTF section */
1329 snprintf(tmp_fn, sizeof(tmp_fn), "%s.btf", filename);
1330 close(fd);
1331 fd = creat(tmp_fn, S_IRUSR | S_IWUSR);
1332 if (fd == -1) {
1333 fprintf(stderr, "%s: open(%s) failed!\n", __func__,
1334 tmp_fn);
1335 goto out;
1336 }
1337
1338 if (write(fd, raw_btf_data, raw_btf_size) != raw_btf_size) {
1339 fprintf(stderr, "%s: write of %d bytes to '%s' failed: %d!\n",
1340 __func__, raw_btf_size, tmp_fn, errno);
1341 goto unlink;
1342 }
1343
1344 snprintf(cmd, sizeof(cmd), "%s --add-section .BTF=%s %s",
1345 llvm_objcopy, tmp_fn, filename);
1346 if (system(cmd)) {
1347 fprintf(stderr, "%s: failed to add .BTF section to '%s': %d!\n",
1348 __func__, filename, errno);
1349 goto unlink;
1350 }
1351
1352 err = 0;
1353 unlink:
1354 unlink(tmp_fn);
1355 }
1356
1357 out:
1358 if (fd != -1)
1359 close(fd);
1360 if (elf)
1361 elf_end(elf);
1362 return err;
1363 }
1364
1365 int btf_encoder__encode(struct btf_encoder *encoder)
1366 {
1367 int err;
1368
1369 /* for single-threaded case, saved funcs are added here */
1370 btf_encoder__add_saved_funcs(encoder);
1371
1372 if (gobuffer__size(&encoder->percpu_secinfo) != 0)
1373 btf_encoder__add_datasec(encoder, PERCPU_SECTION);
1374
1375 /* Empty file, nothing to do, so... done! */
1376 if (btf__type_cnt(encoder->btf) == 1)
1377 return 0;
1378
1379 if (btf__dedup(encoder->btf, NULL)) {
1380 fprintf(stderr, "%s: btf__dedup failed!\n", __func__);
1381 return -1;
1382 }
1383
1384 if (encoder->raw_output)
1385 err = btf_encoder__write_raw_file(encoder);
1386 else
1387 err = btf_encoder__write_elf(encoder);
1388
1389 return err;
1390 }
1391
1392 static int percpu_var_cmp(const void *_a, const void *_b)
1393 {
1394 const struct var_info *a = _a;
1395 const struct var_info *b = _b;
1396
1397 if (a->addr == b->addr)
1398 return 0;
1399 return a->addr < b->addr ? -1 : 1;
1400 }
1401
1402 static bool btf_encoder__percpu_var_exists(struct btf_encoder *encoder, uint64_t addr, uint32_t *sz, const char **name)
1403 {
1404 struct var_info key = { .addr = addr };
1405 const struct var_info *p = bsearch(&key, encoder->percpu.vars, encoder->percpu.var_cnt,
1406 sizeof(encoder->percpu.vars[0]), percpu_var_cmp);
1407 if (!p)
1408 return false;
1409
1410 *sz = p->sz;
1411 *name = p->name;
1412 return true;
1413 }
1414
1415 static int btf_encoder__collect_percpu_var(struct btf_encoder *encoder, GElf_Sym *sym, size_t sym_sec_idx)
1416 {
1417 const char *sym_name;
1418 uint64_t addr;
1419 uint32_t size;
1420
1421 /* compare a symbol's shndx to determine if it's a percpu variable */
1422 if (sym_sec_idx != encoder->percpu.shndx)
1423 return 0;
1424 if (elf_sym__type(sym) != STT_OBJECT)
1425 return 0;
1426
1427 addr = elf_sym__value(sym);
1428
1429 size = elf_sym__size(sym);
1430 if (!size)
1431 return 0; /* ignore zero-sized symbols */
1432
1433 sym_name = elf_sym__name(sym, encoder->symtab);
1434 if (!btf_name_valid(sym_name)) {
1435 dump_invalid_symbol("Found symbol of invalid name when encoding btf",
1436 sym_name, encoder->verbose, encoder->force);
1437 if (encoder->force)
1438 return 0;
1439 return -1;
1440 }
1441
1442 if (encoder->verbose)
1443 printf("Found per-CPU symbol '%s' at address 0x%" PRIx64 "\n", sym_name, addr);
1444
1445 /* Make sure addr is section-relative. For kernel modules (which are
1446 * ET_REL files) this is already the case. For vmlinux (which is an
1447 * ET_EXEC file) we need to subtract the section address.
1448 */
1449 if (!encoder->is_rel)
1450 addr -= encoder->percpu.base_addr;
1451
1452 if (encoder->percpu.var_cnt == encoder->percpu.allocated) {
1453 struct var_info *new;
1454
1455 new = reallocarray_grow(encoder->percpu.vars,
1456 &encoder->percpu.allocated,
1457 sizeof(*encoder->percpu.vars));
1458 if (!new) {
1459 fprintf(stderr, "Failed to allocate memory for variables\n");
1460 return -1;
1461 }
1462 encoder->percpu.vars = new;
1463 }
1464 encoder->percpu.vars[encoder->percpu.var_cnt].addr = addr;
1465 encoder->percpu.vars[encoder->percpu.var_cnt].sz = size;
1466 encoder->percpu.vars[encoder->percpu.var_cnt].name = sym_name;
1467 encoder->percpu.var_cnt++;
1468
1469 return 0;
1470 }
1471
1472 static int btf_encoder__collect_symbols(struct btf_encoder *encoder, bool collect_percpu_vars)
1473 {
1474 Elf32_Word sym_sec_idx;
1475 uint32_t core_id;
1476 GElf_Sym sym;
1477
1478 /* cache variables' addresses, preparing for searching in symtab. */
1479 encoder->percpu.var_cnt = 0;
1480
1481 /* search within symtab for percpu variables */
1482 elf_symtab__for_each_symbol_index(encoder->symtab, core_id, sym, sym_sec_idx) {
1483 if (collect_percpu_vars && btf_encoder__collect_percpu_var(encoder, &sym, sym_sec_idx))
1484 return -1;
1485 if (btf_encoder__collect_function(encoder, &sym))
1486 return -1;
1487 }
1488
1489 if (collect_percpu_vars) {
1490 if (encoder->percpu.var_cnt)
1491 qsort(encoder->percpu.vars, encoder->percpu.var_cnt, sizeof(encoder->percpu.vars[0]), percpu_var_cmp);
1492
1493 if (encoder->verbose)
1494 printf("Found %d per-CPU variables!\n", encoder->percpu.var_cnt);
1495 }
1496
1497 if (encoder->functions.cnt) {
1498 qsort(encoder->functions.entries, encoder->functions.cnt, sizeof(encoder->functions.entries[0]),
1499 functions_cmp);
1500 if (encoder->verbose)
1501 printf("Found %d functions!\n", encoder->functions.cnt);
1502 }
1503
1504 return 0;
1505 }
1506
1507 static bool ftype__has_arg_names(const struct ftype *ftype)
1508 {
1509 struct parameter *param;
1510
1511 ftype__for_each_parameter(ftype, param) {
1512 if (parameter__name(param) == NULL)
1513 return false;
1514 }
1515 return true;
1516 }
1517
1518 static int btf_encoder__encode_cu_variables(struct btf_encoder *encoder)
1519 {
1520 struct cu *cu = encoder->cu;
1521 uint32_t core_id;
1522 struct tag *pos;
1523 int err = -1;
1524
1525 if (encoder->percpu.shndx == 0 || !encoder->symtab)
1526 return 0;
1527
1528 if (encoder->verbose)
1529 printf("search cu '%s' for percpu global variables.\n", cu->name);
1530
1531 cu__for_each_variable(cu, core_id, pos) {
1532 struct variable *var = tag__variable(pos);
1533 uint32_t size, type, linkage;
1534 const char *name, *dwarf_name;
1535 struct llvm_annotation *annot;
1536 const struct tag *tag;
1537 uint64_t addr;
1538 int id;
1539
1540 if (var->declaration && !var->spec)
1541 continue;
1542
1543 /* percpu variables are allocated in global space */
1544 if (variable__scope(var) != VSCOPE_GLOBAL && !var->spec)
1545 continue;
1546
1547 /* addr has to be recorded before we follow spec */
1548 addr = var->ip.addr;
1549 dwarf_name = variable__name(var);
1550
1551 /* Make sure addr is section-relative. DWARF, unlike ELF,
1552 * always contains virtual symbol addresses, so subtract
1553 * the section address unconditionally.
1554 */
1555 if (addr < encoder->percpu.base_addr || addr >= encoder->percpu.base_addr + encoder->percpu.sec_sz)
1556 continue;
1557 addr -= encoder->percpu.base_addr;
1558
1559 if (!btf_encoder__percpu_var_exists(encoder, addr, &size, &name))
1560 continue; /* not a per-CPU variable */
1561
1562 /* A lot of "special" DWARF variables (e.g, __UNIQUE_ID___xxx)
1563 * have addr == 0, which is the same as, say, valid
1564 * fixed_percpu_data per-CPU variable. To distinguish between
1565 * them, additionally compare DWARF and ELF symbol names. If
1566 * DWARF doesn't provide proper name, pessimistically assume
1567 * bad variable.
1568 *
1569 * Examples of such special variables are:
1570 *
1571 * 1. __ADDRESSABLE(sym), which are forcely emitted as symbols.
1572 * 2. __UNIQUE_ID(prefix), which are introduced to generate unique ids.
1573 * 3. __exitcall(fn), functions which are labeled as exit calls.
1574 *
1575 * This is relevant only for vmlinux image, as for kernel
1576 * modules per-CPU data section has non-zero offset so all
1577 * per-CPU symbols have non-zero values.
1578 */
1579 if (var->ip.addr == 0) {
1580 if (!dwarf_name || strcmp(dwarf_name, name))
1581 continue;
1582 }
1583
1584 if (var->spec)
1585 var = var->spec;
1586
1587 if (var->ip.tag.type == 0) {
1588 fprintf(stderr, "error: found variable '%s' in CU '%s' that has void type\n",
1589 name, cu->name);
1590 if (encoder->force)
1591 continue;
1592 err = -1;
1593 break;
1594 }
1595
1596 tag = cu__type(cu, var->ip.tag.type);
1597 if (tag__size(tag, cu) == 0) {
1598 if (encoder->verbose)
1599 fprintf(stderr, "Ignoring zero-sized per-CPU variable '%s'...\n", dwarf_name ?: "<missing name>");
1600 continue;
1601 }
1602
1603 type = var->ip.tag.type + encoder->type_id_off;
1604 linkage = var->external ? BTF_VAR_GLOBAL_ALLOCATED : BTF_VAR_STATIC;
1605
1606 if (encoder->verbose) {
1607 printf("Variable '%s' from CU '%s' at address 0x%" PRIx64 " encoded\n",
1608 name, cu->name, addr);
1609 }
1610
1611 /* add a BTF_KIND_VAR in encoder->types */
1612 id = btf_encoder__add_var(encoder, type, name, linkage);
1613 if (id < 0) {
1614 fprintf(stderr, "error: failed to encode variable '%s' at addr 0x%" PRIx64 "\n",
1615 name, addr);
1616 goto out;
1617 }
1618
1619 list_for_each_entry(annot, &var->annots, node) {
1620 int tag_type_id = btf_encoder__add_decl_tag(encoder, annot->value, id, annot->component_idx);
1621 if (tag_type_id < 0) {
1622 fprintf(stderr, "error: failed to encode tag '%s' to variable '%s' with component_idx %d\n",
1623 annot->value, name, annot->component_idx);
1624 goto out;
1625 }
1626 }
1627
1628 /*
1629 * add a BTF_VAR_SECINFO in encoder->percpu_secinfo, which will be added into
1630 * encoder->types later when we add BTF_VAR_DATASEC.
1631 */
1632 id = btf_encoder__add_var_secinfo(encoder, id, addr, size);
1633 if (id < 0) {
1634 fprintf(stderr, "error: failed to encode section info for variable '%s' at addr 0x%" PRIx64 "\n",
1635 name, addr);
1636 goto out;
1637 }
1638 }
1639
1640 err = 0;
1641 out:
1642 return err;
1643 }
1644
1645 struct btf_encoder *btf_encoder__new(struct cu *cu, const char *detached_filename, struct btf *base_btf, bool verbose, struct conf_load *conf_load)
1646 {
1647 struct btf_encoder *encoder = zalloc(sizeof(*encoder));
1648
1649 if (encoder) {
1650 encoder->raw_output = detached_filename != NULL;
1651 encoder->filename = strdup(encoder->raw_output ? detached_filename : cu->filename);
1652 if (encoder->filename == NULL)
1653 goto out_delete;
1654
1655 encoder->btf = btf__new_empty_split(base_btf);
1656 if (encoder->btf == NULL)
1657 goto out_delete;
1658
1659 encoder->force = conf_load->btf_encode_force;
1660 encoder->gen_floats = conf_load->btf_gen_floats;
1661 encoder->skip_encoding_vars = conf_load->skip_encoding_btf_vars;
1662 encoder->verbose = verbose;
1663 encoder->has_index_type = false;
1664 encoder->need_index_type = false;
1665 encoder->array_index_id = 0;
1666
1667 GElf_Ehdr ehdr;
1668
1669 if (gelf_getehdr(cu->elf, &ehdr) == NULL) {
1670 if (encoder->verbose)
1671 elf_error("cannot get ELF header");
1672 goto out_delete;
1673 }
1674
1675 encoder->is_rel = ehdr.e_type == ET_REL;
1676
1677 switch (ehdr.e_ident[EI_DATA]) {
1678 case ELFDATA2LSB:
1679 btf__set_endianness(encoder->btf, BTF_LITTLE_ENDIAN);
1680 break;
1681 case ELFDATA2MSB:
1682 btf__set_endianness(encoder->btf, BTF_BIG_ENDIAN);
1683 break;
1684 default:
1685 fprintf(stderr, "%s: unknown ELF endianness.\n", __func__);
1686 goto out_delete;
1687 }
1688
1689 encoder->symtab = elf_symtab__new(NULL, cu->elf);
1690 if (!encoder->symtab) {
1691 if (encoder->verbose)
1692 printf("%s: '%s' doesn't have symtab.\n", __func__, cu->filename);
1693 goto out;
1694 }
1695
1696 /* find percpu section's shndx */
1697
1698 GElf_Shdr shdr;
1699 Elf_Scn *sec = elf_section_by_name(cu->elf, &shdr, PERCPU_SECTION, NULL);
1700
1701 if (!sec) {
1702 if (encoder->verbose)
1703 printf("%s: '%s' doesn't have '%s' section\n", __func__, cu->filename, PERCPU_SECTION);
1704 } else {
1705 encoder->percpu.shndx = elf_ndxscn(sec);
1706 encoder->percpu.base_addr = shdr.sh_addr;
1707 encoder->percpu.sec_sz = shdr.sh_size;
1708 }
1709
1710 if (btf_encoder__collect_symbols(encoder, !encoder->skip_encoding_vars))
1711 goto out_delete;
1712
1713 if (encoder->verbose)
1714 printf("File %s:\n", cu->filename);
1715 btf_encoders__add(encoder);
1716 }
1717 out:
1718 return encoder;
1719
1720 out_delete:
1721 btf_encoder__delete(encoder);
1722 return NULL;
1723 }
1724
1725 void btf_encoder__delete(struct btf_encoder *encoder)
1726 {
1727 if (encoder == NULL)
1728 return;
1729
1730 btf_encoders__delete(encoder);
1731 __gobuffer__delete(&encoder->percpu_secinfo);
1732 zfree(&encoder->filename);
1733 btf__free(encoder->btf);
1734 encoder->btf = NULL;
1735 elf_symtab__delete(encoder->symtab);
1736
1737 encoder->functions.allocated = encoder->functions.cnt = 0;
1738 free(encoder->functions.entries);
1739 encoder->functions.entries = NULL;
1740 encoder->percpu.allocated = encoder->percpu.var_cnt = 0;
1741 free(encoder->percpu.vars);
1742 encoder->percpu.vars = NULL;
1743
1744 free(encoder);
1745 }
1746
1747 int btf_encoder__encode_cu(struct btf_encoder *encoder, struct cu *cu, struct conf_load *conf_load)
1748 {
1749 struct llvm_annotation *annot;
1750 int btf_type_id, tag_type_id, skipped_types = 0;
1751 uint32_t core_id;
1752 struct function *fn;
1753 struct tag *pos;
1754 int err = 0;
1755
1756 encoder->cu = cu;
1757 encoder->type_id_off = btf__type_cnt(encoder->btf) - 1;
1758
1759 if (!encoder->has_index_type) {
1760 /* cu__find_base_type_by_name() takes "type_id_t *id" */
1761 type_id_t id;
1762 if (cu__find_base_type_by_name(cu, "int", &id)) {
1763 encoder->has_index_type = true;
1764 encoder->array_index_id = encoder->type_id_off + id;
1765 } else {
1766 encoder->has_index_type = false;
1767 encoder->array_index_id = encoder->type_id_off + cu->types_table.nr_entries;
1768 }
1769 }
1770
1771 cu__for_each_type(cu, core_id, pos) {
1772 btf_type_id = btf_encoder__encode_tag(encoder, pos, conf_load);
1773
1774 if (btf_type_id == 0) {
1775 ++skipped_types;
1776 continue;
1777 }
1778
1779 if (btf_type_id < 0 ||
1780 tag__check_id_drift(encoder, pos, core_id, btf_type_id + skipped_types)) {
1781 err = -1;
1782 goto out;
1783 }
1784 }
1785
1786 if (encoder->need_index_type && !encoder->has_index_type) {
1787 struct base_type bt = {};
1788
1789 bt.name = 0;
1790 bt.bit_size = 32;
1791 bt.is_signed = true;
1792 btf_encoder__add_base_type(encoder, &bt, "int");
1793 encoder->has_index_type = true;
1794 }
1795
1796 cu__for_each_type(cu, core_id, pos) {
1797 struct namespace *ns;
1798 const char *tag_name;
1799
1800 switch (pos->tag) {
1801 case DW_TAG_structure_type:
1802 tag_name = "struct";
1803 break;
1804 case DW_TAG_union_type:
1805 tag_name = "union";
1806 break;
1807 case DW_TAG_typedef:
1808 tag_name = "typedef";
1809 break;
1810 default:
1811 continue;
1812 }
1813
1814 btf_type_id = encoder->type_id_off + core_id;
1815 ns = tag__namespace(pos);
1816 list_for_each_entry(annot, &ns->annots, node) {
1817 tag_type_id = btf_encoder__add_decl_tag(encoder, annot->value, btf_type_id, annot->component_idx);
1818 if (tag_type_id < 0) {
1819 fprintf(stderr, "error: failed to encode tag '%s' to %s '%s' with component_idx %d\n",
1820 annot->value, tag_name, namespace__name(ns), annot->component_idx);
1821 goto out;
1822 }
1823 }
1824 }
1825
1826 cu__for_each_function(cu, core_id, fn) {
1827 struct elf_function *func = NULL;
1828 bool save = false;
1829
1830 /*
1831 * Skip functions that:
1832 * - are marked as declarations
1833 * - do not have full argument names
1834 * - are not in ftrace list (if it's available)
1835 * - are not external (in case ftrace filter is not available)
1836 */
1837 if (fn->declaration)
1838 continue;
1839 if (!ftype__has_arg_names(&fn->proto))
1840 continue;
1841 if (encoder->functions.cnt) {
1842 const char *name;
1843
1844 name = function__name(fn);
1845 if (!name)
1846 continue;
1847
1848 /* prefer exact function name match... */
1849 func = btf_encoder__find_function(encoder, name, 0);
1850 if (func) {
1851 if (func->generated)
1852 continue;
1853 if (conf_load->skip_encoding_btf_inconsistent_proto)
1854 save = true;
1855 else
1856 func->generated = true;
1857 } else if (encoder->functions.suffix_cnt &&
1858 conf_load->btf_gen_optimized) {
1859 /* falling back to name.isra.0 match if no exact
1860 * match is found; only bother if we found any
1861 * .suffix function names. The function
1862 * will be saved and added once we ensure
1863 * it does not have optimized-out parameters
1864 * in any cu.
1865 */
1866 func = btf_encoder__find_function(encoder, name,
1867 strlen(name));
1868 if (func) {
1869 save = true;
1870 if (encoder->verbose)
1871 printf("matched function '%s' with '%s'%s\n",
1872 name, func->name,
1873 fn->proto.optimized_parms ?
1874 ", has optimized-out parameters" :
1875 fn->proto.unexpected_reg ? ", has unexpected register use by params" :
1876 "");
1877 fn->alias = func->name;
1878 }
1879 }
1880 if (!func)
1881 continue;
1882 } else {
1883 if (!fn->external)
1884 continue;
1885 }
1886
1887 if (save)
1888 err = btf_encoder__save_func(encoder, fn, func);
1889 else
1890 err = btf_encoder__add_func(encoder, fn);
1891 if (err)
1892 goto out;
1893 }
1894
1895 if (!encoder->skip_encoding_vars)
1896 err = btf_encoder__encode_cu_variables(encoder);
1897
1898 /* It is only safe to delete this CU if we have not stashed any static
1899 * functions for later addition.
1900 */
1901 if (!err)
1902 err = encoder->saved_func_cnt > 0 ? LSK__KEEPIT : LSK__DELETE;
1903 out:
1904 encoder->cu = NULL;
1905 return err;
1906 }
1907
1908 struct btf *btf_encoder__btf(struct btf_encoder *encoder)
1909 {
1910 return encoder->btf;
1911 }
Pahole and other DWARF utils