1
2
3
4#include <byteswap.h>
5#include <endian.h>
6#include <stdio.h>
7#include <stdlib.h>
8#include <string.h>
9#include <fcntl.h>
10#include <unistd.h>
11#include <errno.h>
12#include <sys/utsname.h>
13#include <sys/param.h>
14#include <sys/stat.h>
15#include <linux/kernel.h>
16#include <linux/err.h>
17#include <linux/btf.h>
18#include <gelf.h>
19#include "btf.h"
20#include "bpf.h"
21#include "libbpf.h"
22#include "libbpf_internal.h"
23#include "hashmap.h"
24#include "strset.h"
25
26#define BTF_MAX_NR_TYPES 0x7fffffffU
27#define BTF_MAX_STR_OFFSET 0x7fffffffU
28
29static struct btf_type btf_void;
30
31struct btf {
32
33 void *raw_data;
34
35 void *raw_data_swapped;
36 __u32 raw_size;
37
38 bool swapped_endian;
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77 struct btf_header *hdr;
78
79 void *types_data;
80 size_t types_data_cap;
81
82
83
84
85
86
87 __u32 *type_offs;
88 size_t type_offs_cap;
89
90
91
92
93 __u32 nr_types;
94
95
96
97 struct btf *base_btf;
98
99
100
101
102 int start_id;
103
104
105
106
107 int start_str_off;
108
109
110
111
112
113 void *strs_data;
114
115 struct strset *strs_set;
116
117 bool strs_deduped;
118
119
120 int fd;
121
122
123 int ptr_sz;
124};
125
126static inline __u64 ptr_to_u64(const void *ptr)
127{
128 return (__u64) (unsigned long) ptr;
129}
130
131
132
133
134
135
136
137
138
139
140
141void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz,
142 size_t cur_cnt, size_t max_cnt, size_t add_cnt)
143{
144 size_t new_cnt;
145 void *new_data;
146
147 if (cur_cnt + add_cnt <= *cap_cnt)
148 return *data + cur_cnt * elem_sz;
149
150
151 if (cur_cnt + add_cnt > max_cnt)
152 return NULL;
153
154 new_cnt = *cap_cnt;
155 new_cnt += new_cnt / 4;
156 if (new_cnt < 16)
157 new_cnt = 16;
158 if (new_cnt > max_cnt)
159 new_cnt = max_cnt;
160 if (new_cnt < cur_cnt + add_cnt)
161 new_cnt = cur_cnt + add_cnt;
162
163 new_data = libbpf_reallocarray(*data, new_cnt, elem_sz);
164 if (!new_data)
165 return NULL;
166
167
168 memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz);
169
170 *data = new_data;
171 *cap_cnt = new_cnt;
172 return new_data + cur_cnt * elem_sz;
173}
174
175
176
177
178int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt)
179{
180 void *p;
181
182 if (need_cnt <= *cap_cnt)
183 return 0;
184
185 p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt);
186 if (!p)
187 return -ENOMEM;
188
189 return 0;
190}
191
192static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off)
193{
194 __u32 *p;
195
196 p = libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32),
197 btf->nr_types, BTF_MAX_NR_TYPES, 1);
198 if (!p)
199 return -ENOMEM;
200
201 *p = type_off;
202 return 0;
203}
204
205static void btf_bswap_hdr(struct btf_header *h)
206{
207 h->magic = bswap_16(h->magic);
208 h->hdr_len = bswap_32(h->hdr_len);
209 h->type_off = bswap_32(h->type_off);
210 h->type_len = bswap_32(h->type_len);
211 h->str_off = bswap_32(h->str_off);
212 h->str_len = bswap_32(h->str_len);
213}
214
215static int btf_parse_hdr(struct btf *btf)
216{
217 struct btf_header *hdr = btf->hdr;
218 __u32 meta_left;
219
220 if (btf->raw_size < sizeof(struct btf_header)) {
221 pr_debug("BTF header not found\n");
222 return -EINVAL;
223 }
224
225 if (hdr->magic == bswap_16(BTF_MAGIC)) {
226 btf->swapped_endian = true;
227 if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) {
228 pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n",
229 bswap_32(hdr->hdr_len));
230 return -ENOTSUP;
231 }
232 btf_bswap_hdr(hdr);
233 } else if (hdr->magic != BTF_MAGIC) {
234 pr_debug("Invalid BTF magic:%x\n", hdr->magic);
235 return -EINVAL;
236 }
237
238 meta_left = btf->raw_size - sizeof(*hdr);
239 if (meta_left < hdr->str_off + hdr->str_len) {
240 pr_debug("Invalid BTF total size:%u\n", btf->raw_size);
241 return -EINVAL;
242 }
243
244 if (hdr->type_off + hdr->type_len > hdr->str_off) {
245 pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n",
246 hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len);
247 return -EINVAL;
248 }
249
250 if (hdr->type_off % 4) {
251 pr_debug("BTF type section is not aligned to 4 bytes\n");
252 return -EINVAL;
253 }
254
255 return 0;
256}
257
258static int btf_parse_str_sec(struct btf *btf)
259{
260 const struct btf_header *hdr = btf->hdr;
261 const char *start = btf->strs_data;
262 const char *end = start + btf->hdr->str_len;
263
264 if (btf->base_btf && hdr->str_len == 0)
265 return 0;
266 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) {
267 pr_debug("Invalid BTF string section\n");
268 return -EINVAL;
269 }
270 if (!btf->base_btf && start[0]) {
271 pr_debug("Invalid BTF string section\n");
272 return -EINVAL;
273 }
274 return 0;
275}
276
277static int btf_type_size(const struct btf_type *t)
278{
279 const int base_size = sizeof(struct btf_type);
280 __u16 vlen = btf_vlen(t);
281
282 switch (btf_kind(t)) {
283 case BTF_KIND_FWD:
284 case BTF_KIND_CONST:
285 case BTF_KIND_VOLATILE:
286 case BTF_KIND_RESTRICT:
287 case BTF_KIND_PTR:
288 case BTF_KIND_TYPEDEF:
289 case BTF_KIND_FUNC:
290 case BTF_KIND_FLOAT:
291 return base_size;
292 case BTF_KIND_INT:
293 return base_size + sizeof(__u32);
294 case BTF_KIND_ENUM:
295 return base_size + vlen * sizeof(struct btf_enum);
296 case BTF_KIND_ARRAY:
297 return base_size + sizeof(struct btf_array);
298 case BTF_KIND_STRUCT:
299 case BTF_KIND_UNION:
300 return base_size + vlen * sizeof(struct btf_member);
301 case BTF_KIND_FUNC_PROTO:
302 return base_size + vlen * sizeof(struct btf_param);
303 case BTF_KIND_VAR:
304 return base_size + sizeof(struct btf_var);
305 case BTF_KIND_DATASEC:
306 return base_size + vlen * sizeof(struct btf_var_secinfo);
307 default:
308 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));
309 return -EINVAL;
310 }
311}
312
313static void btf_bswap_type_base(struct btf_type *t)
314{
315 t->name_off = bswap_32(t->name_off);
316 t->info = bswap_32(t->info);
317 t->type = bswap_32(t->type);
318}
319
320static int btf_bswap_type_rest(struct btf_type *t)
321{
322 struct btf_var_secinfo *v;
323 struct btf_member *m;
324 struct btf_array *a;
325 struct btf_param *p;
326 struct btf_enum *e;
327 __u16 vlen = btf_vlen(t);
328 int i;
329
330 switch (btf_kind(t)) {
331 case BTF_KIND_FWD:
332 case BTF_KIND_CONST:
333 case BTF_KIND_VOLATILE:
334 case BTF_KIND_RESTRICT:
335 case BTF_KIND_PTR:
336 case BTF_KIND_TYPEDEF:
337 case BTF_KIND_FUNC:
338 case BTF_KIND_FLOAT:
339 return 0;
340 case BTF_KIND_INT:
341 *(__u32 *)(t + 1) = bswap_32(*(__u32 *)(t + 1));
342 return 0;
343 case BTF_KIND_ENUM:
344 for (i = 0, e = btf_enum(t); i < vlen; i++, e++) {
345 e->name_off = bswap_32(e->name_off);
346 e->val = bswap_32(e->val);
347 }
348 return 0;
349 case BTF_KIND_ARRAY:
350 a = btf_array(t);
351 a->type = bswap_32(a->type);
352 a->index_type = bswap_32(a->index_type);
353 a->nelems = bswap_32(a->nelems);
354 return 0;
355 case BTF_KIND_STRUCT:
356 case BTF_KIND_UNION:
357 for (i = 0, m = btf_members(t); i < vlen; i++, m++) {
358 m->name_off = bswap_32(m->name_off);
359 m->type = bswap_32(m->type);
360 m->offset = bswap_32(m->offset);
361 }
362 return 0;
363 case BTF_KIND_FUNC_PROTO:
364 for (i = 0, p = btf_params(t); i < vlen; i++, p++) {
365 p->name_off = bswap_32(p->name_off);
366 p->type = bswap_32(p->type);
367 }
368 return 0;
369 case BTF_KIND_VAR:
370 btf_var(t)->linkage = bswap_32(btf_var(t)->linkage);
371 return 0;
372 case BTF_KIND_DATASEC:
373 for (i = 0, v = btf_var_secinfos(t); i < vlen; i++, v++) {
374 v->type = bswap_32(v->type);
375 v->offset = bswap_32(v->offset);
376 v->size = bswap_32(v->size);
377 }
378 return 0;
379 default:
380 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));
381 return -EINVAL;
382 }
383}
384
385static int btf_parse_type_sec(struct btf *btf)
386{
387 struct btf_header *hdr = btf->hdr;
388 void *next_type = btf->types_data;
389 void *end_type = next_type + hdr->type_len;
390 int err, type_size;
391
392 while (next_type + sizeof(struct btf_type) <= end_type) {
393 if (btf->swapped_endian)
394 btf_bswap_type_base(next_type);
395
396 type_size = btf_type_size(next_type);
397 if (type_size < 0)
398 return type_size;
399 if (next_type + type_size > end_type) {
400 pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types);
401 return -EINVAL;
402 }
403
404 if (btf->swapped_endian && btf_bswap_type_rest(next_type))
405 return -EINVAL;
406
407 err = btf_add_type_idx_entry(btf, next_type - btf->types_data);
408 if (err)
409 return err;
410
411 next_type += type_size;
412 btf->nr_types++;
413 }
414
415 if (next_type != end_type) {
416 pr_warn("BTF types data is malformed\n");
417 return -EINVAL;
418 }
419
420 return 0;
421}
422
423__u32 btf__get_nr_types(const struct btf *btf)
424{
425 return btf->start_id + btf->nr_types - 1;
426}
427
428const struct btf *btf__base_btf(const struct btf *btf)
429{
430 return btf->base_btf;
431}
432
433
434struct btf_type *btf_type_by_id(struct btf *btf, __u32 type_id)
435{
436 if (type_id == 0)
437 return &btf_void;
438 if (type_id < btf->start_id)
439 return btf_type_by_id(btf->base_btf, type_id);
440 return btf->types_data + btf->type_offs[type_id - btf->start_id];
441}
442
443const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id)
444{
445 if (type_id >= btf->start_id + btf->nr_types)
446 return errno = EINVAL, NULL;
447 return btf_type_by_id((struct btf *)btf, type_id);
448}
449
450static int determine_ptr_size(const struct btf *btf)
451{
452 const struct btf_type *t;
453 const char *name;
454 int i, n;
455
456 if (btf->base_btf && btf->base_btf->ptr_sz > 0)
457 return btf->base_btf->ptr_sz;
458
459 n = btf__get_nr_types(btf);
460 for (i = 1; i <= n; i++) {
461 t = btf__type_by_id(btf, i);
462 if (!btf_is_int(t))
463 continue;
464
465 name = btf__name_by_offset(btf, t->name_off);
466 if (!name)
467 continue;
468
469 if (strcmp(name, "long int") == 0 ||
470 strcmp(name, "long unsigned int") == 0) {
471 if (t->size != 4 && t->size != 8)
472 continue;
473 return t->size;
474 }
475 }
476
477 return -1;
478}
479
480static size_t btf_ptr_sz(const struct btf *btf)
481{
482 if (!btf->ptr_sz)
483 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf);
484 return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz;
485}
486
487
488
489
490
491
492
493
494
495size_t btf__pointer_size(const struct btf *btf)
496{
497 if (!btf->ptr_sz)
498 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf);
499
500 if (btf->ptr_sz < 0)
501
502 return 0;
503
504 return btf->ptr_sz;
505}
506
507
508
509
510int btf__set_pointer_size(struct btf *btf, size_t ptr_sz)
511{
512 if (ptr_sz != 4 && ptr_sz != 8)
513 return libbpf_err(-EINVAL);
514 btf->ptr_sz = ptr_sz;
515 return 0;
516}
517
518static bool is_host_big_endian(void)
519{
520#if __BYTE_ORDER == __LITTLE_ENDIAN
521 return false;
522#elif __BYTE_ORDER == __BIG_ENDIAN
523 return true;
524#else
525# error "Unrecognized __BYTE_ORDER__"
526#endif
527}
528
529enum btf_endianness btf__endianness(const struct btf *btf)
530{
531 if (is_host_big_endian())
532 return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN;
533 else
534 return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
535}
536
537int btf__set_endianness(struct btf *btf, enum btf_endianness endian)
538{
539 if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN)
540 return libbpf_err(-EINVAL);
541
542 btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN);
543 if (!btf->swapped_endian) {
544 free(btf->raw_data_swapped);
545 btf->raw_data_swapped = NULL;
546 }
547 return 0;
548}
549
550static bool btf_type_is_void(const struct btf_type *t)
551{
552 return t == &btf_void || btf_is_fwd(t);
553}
554
555static bool btf_type_is_void_or_null(const struct btf_type *t)
556{
557 return !t || btf_type_is_void(t);
558}
559
560#define MAX_RESOLVE_DEPTH 32
561
562__s64 btf__resolve_size(const struct btf *btf, __u32 type_id)
563{
564 const struct btf_array *array;
565 const struct btf_type *t;
566 __u32 nelems = 1;
567 __s64 size = -1;
568 int i;
569
570 t = btf__type_by_id(btf, type_id);
571 for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t); i++) {
572 switch (btf_kind(t)) {
573 case BTF_KIND_INT:
574 case BTF_KIND_STRUCT:
575 case BTF_KIND_UNION:
576 case BTF_KIND_ENUM:
577 case BTF_KIND_DATASEC:
578 case BTF_KIND_FLOAT:
579 size = t->size;
580 goto done;
581 case BTF_KIND_PTR:
582 size = btf_ptr_sz(btf);
583 goto done;
584 case BTF_KIND_TYPEDEF:
585 case BTF_KIND_VOLATILE:
586 case BTF_KIND_CONST:
587 case BTF_KIND_RESTRICT:
588 case BTF_KIND_VAR:
589 type_id = t->type;
590 break;
591 case BTF_KIND_ARRAY:
592 array = btf_array(t);
593 if (nelems && array->nelems > UINT32_MAX / nelems)
594 return libbpf_err(-E2BIG);
595 nelems *= array->nelems;
596 type_id = array->type;
597 break;
598 default:
599 return libbpf_err(-EINVAL);
600 }
601
602 t = btf__type_by_id(btf, type_id);
603 }
604
605done:
606 if (size < 0)
607 return libbpf_err(-EINVAL);
608 if (nelems && size > UINT32_MAX / nelems)
609 return libbpf_err(-E2BIG);
610
611 return nelems * size;
612}
613
614int btf__align_of(const struct btf *btf, __u32 id)
615{
616 const struct btf_type *t = btf__type_by_id(btf, id);
617 __u16 kind = btf_kind(t);
618
619 switch (kind) {
620 case BTF_KIND_INT:
621 case BTF_KIND_ENUM:
622 case BTF_KIND_FLOAT:
623 return min(btf_ptr_sz(btf), (size_t)t->size);
624 case BTF_KIND_PTR:
625 return btf_ptr_sz(btf);
626 case BTF_KIND_TYPEDEF:
627 case BTF_KIND_VOLATILE:
628 case BTF_KIND_CONST:
629 case BTF_KIND_RESTRICT:
630 return btf__align_of(btf, t->type);
631 case BTF_KIND_ARRAY:
632 return btf__align_of(btf, btf_array(t)->type);
633 case BTF_KIND_STRUCT:
634 case BTF_KIND_UNION: {
635 const struct btf_member *m = btf_members(t);
636 __u16 vlen = btf_vlen(t);
637 int i, max_align = 1, align;
638
639 for (i = 0; i < vlen; i++, m++) {
640 align = btf__align_of(btf, m->type);
641 if (align <= 0)
642 return libbpf_err(align);
643 max_align = max(max_align, align);
644 }
645
646 return max_align;
647 }
648 default:
649 pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t));
650 return errno = EINVAL, 0;
651 }
652}
653
654int btf__resolve_type(const struct btf *btf, __u32 type_id)
655{
656 const struct btf_type *t;
657 int depth = 0;
658
659 t = btf__type_by_id(btf, type_id);
660 while (depth < MAX_RESOLVE_DEPTH &&
661 !btf_type_is_void_or_null(t) &&
662 (btf_is_mod(t) || btf_is_typedef(t) || btf_is_var(t))) {
663 type_id = t->type;
664 t = btf__type_by_id(btf, type_id);
665 depth++;
666 }
667
668 if (depth == MAX_RESOLVE_DEPTH || btf_type_is_void_or_null(t))
669 return libbpf_err(-EINVAL);
670
671 return type_id;
672}
673
674__s32 btf__find_by_name(const struct btf *btf, const char *type_name)
675{
676 __u32 i, nr_types = btf__get_nr_types(btf);
677
678 if (!strcmp(type_name, "void"))
679 return 0;
680
681 for (i = 1; i <= nr_types; i++) {
682 const struct btf_type *t = btf__type_by_id(btf, i);
683 const char *name = btf__name_by_offset(btf, t->name_off);
684
685 if (name && !strcmp(type_name, name))
686 return i;
687 }
688
689 return libbpf_err(-ENOENT);
690}
691
692__s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name,
693 __u32 kind)
694{
695 __u32 i, nr_types = btf__get_nr_types(btf);
696
697 if (kind == BTF_KIND_UNKN || !strcmp(type_name, "void"))
698 return 0;
699
700 for (i = 1; i <= nr_types; i++) {
701 const struct btf_type *t = btf__type_by_id(btf, i);
702 const char *name;
703
704 if (btf_kind(t) != kind)
705 continue;
706 name = btf__name_by_offset(btf, t->name_off);
707 if (name && !strcmp(type_name, name))
708 return i;
709 }
710
711 return libbpf_err(-ENOENT);
712}
713
714static bool btf_is_modifiable(const struct btf *btf)
715{
716 return (void *)btf->hdr != btf->raw_data;
717}
718
719void btf__free(struct btf *btf)
720{
721 if (IS_ERR_OR_NULL(btf))
722 return;
723
724 if (btf->fd >= 0)
725 close(btf->fd);
726
727 if (btf_is_modifiable(btf)) {
728
729
730
731
732
733
734 free(btf->hdr);
735 free(btf->types_data);
736 strset__free(btf->strs_set);
737 }
738 free(btf->raw_data);
739 free(btf->raw_data_swapped);
740 free(btf->type_offs);
741 free(btf);
742}
743
744static struct btf *btf_new_empty(struct btf *base_btf)
745{
746 struct btf *btf;
747
748 btf = calloc(1, sizeof(*btf));
749 if (!btf)
750 return ERR_PTR(-ENOMEM);
751
752 btf->nr_types = 0;
753 btf->start_id = 1;
754 btf->start_str_off = 0;
755 btf->fd = -1;
756 btf->ptr_sz = sizeof(void *);
757 btf->swapped_endian = false;
758
759 if (base_btf) {
760 btf->base_btf = base_btf;
761 btf->start_id = btf__get_nr_types(base_btf) + 1;
762 btf->start_str_off = base_btf->hdr->str_len;
763 }
764
765
766 btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1);
767 btf->raw_data = calloc(1, btf->raw_size);
768 if (!btf->raw_data) {
769 free(btf);
770 return ERR_PTR(-ENOMEM);
771 }
772
773 btf->hdr = btf->raw_data;
774 btf->hdr->hdr_len = sizeof(struct btf_header);
775 btf->hdr->magic = BTF_MAGIC;
776 btf->hdr->version = BTF_VERSION;
777
778 btf->types_data = btf->raw_data + btf->hdr->hdr_len;
779 btf->strs_data = btf->raw_data + btf->hdr->hdr_len;
780 btf->hdr->str_len = base_btf ? 0 : 1;
781
782 return btf;
783}
784
785struct btf *btf__new_empty(void)
786{
787 return libbpf_ptr(btf_new_empty(NULL));
788}
789
790struct btf *btf__new_empty_split(struct btf *base_btf)
791{
792 return libbpf_ptr(btf_new_empty(base_btf));
793}
794
795static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf)
796{
797 struct btf *btf;
798 int err;
799
800 btf = calloc(1, sizeof(struct btf));
801 if (!btf)
802 return ERR_PTR(-ENOMEM);
803
804 btf->nr_types = 0;
805 btf->start_id = 1;
806 btf->start_str_off = 0;
807 btf->fd = -1;
808
809 if (base_btf) {
810 btf->base_btf = base_btf;
811 btf->start_id = btf__get_nr_types(base_btf) + 1;
812 btf->start_str_off = base_btf->hdr->str_len;
813 }
814
815 btf->raw_data = malloc(size);
816 if (!btf->raw_data) {
817 err = -ENOMEM;
818 goto done;
819 }
820 memcpy(btf->raw_data, data, size);
821 btf->raw_size = size;
822
823 btf->hdr = btf->raw_data;
824 err = btf_parse_hdr(btf);
825 if (err)
826 goto done;
827
828 btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off;
829 btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off;
830
831 err = btf_parse_str_sec(btf);
832 err = err ?: btf_parse_type_sec(btf);
833 if (err)
834 goto done;
835
836done:
837 if (err) {
838 btf__free(btf);
839 return ERR_PTR(err);
840 }
841
842 return btf;
843}
844
845struct btf *btf__new(const void *data, __u32 size)
846{
847 return libbpf_ptr(btf_new(data, size, NULL));
848}
849
850static struct btf *btf_parse_elf(const char *path, struct btf *base_btf,
851 struct btf_ext **btf_ext)
852{
853 Elf_Data *btf_data = NULL, *btf_ext_data = NULL;
854 int err = 0, fd = -1, idx = 0;
855 struct btf *btf = NULL;
856 Elf_Scn *scn = NULL;
857 Elf *elf = NULL;
858 GElf_Ehdr ehdr;
859 size_t shstrndx;
860
861 if (elf_version(EV_CURRENT) == EV_NONE) {
862 pr_warn("failed to init libelf for %s\n", path);
863 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
864 }
865
866 fd = open(path, O_RDONLY);
867 if (fd < 0) {
868 err = -errno;
869 pr_warn("failed to open %s: %s\n", path, strerror(errno));
870 return ERR_PTR(err);
871 }
872
873 err = -LIBBPF_ERRNO__FORMAT;
874
875 elf = elf_begin(fd, ELF_C_READ, NULL);
876 if (!elf) {
877 pr_warn("failed to open %s as ELF file\n", path);
878 goto done;
879 }
880 if (!gelf_getehdr(elf, &ehdr)) {
881 pr_warn("failed to get EHDR from %s\n", path);
882 goto done;
883 }
884
885 if (elf_getshdrstrndx(elf, &shstrndx)) {
886 pr_warn("failed to get section names section index for %s\n",
887 path);
888 goto done;
889 }
890
891 if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) {
892 pr_warn("failed to get e_shstrndx from %s\n", path);
893 goto done;
894 }
895
896 while ((scn = elf_nextscn(elf, scn)) != NULL) {
897 GElf_Shdr sh;
898 char *name;
899
900 idx++;
901 if (gelf_getshdr(scn, &sh) != &sh) {
902 pr_warn("failed to get section(%d) header from %s\n",
903 idx, path);
904 goto done;
905 }
906 name = elf_strptr(elf, shstrndx, sh.sh_name);
907 if (!name) {
908 pr_warn("failed to get section(%d) name from %s\n",
909 idx, path);
910 goto done;
911 }
912 if (strcmp(name, BTF_ELF_SEC) == 0) {
913 btf_data = elf_getdata(scn, 0);
914 if (!btf_data) {
915 pr_warn("failed to get section(%d, %s) data from %s\n",
916 idx, name, path);
917 goto done;
918 }
919 continue;
920 } else if (btf_ext && strcmp(name, BTF_EXT_ELF_SEC) == 0) {
921 btf_ext_data = elf_getdata(scn, 0);
922 if (!btf_ext_data) {
923 pr_warn("failed to get section(%d, %s) data from %s\n",
924 idx, name, path);
925 goto done;
926 }
927 continue;
928 }
929 }
930
931 err = 0;
932
933 if (!btf_data) {
934 err = -ENOENT;
935 goto done;
936 }
937 btf = btf_new(btf_data->d_buf, btf_data->d_size, base_btf);
938 err = libbpf_get_error(btf);
939 if (err)
940 goto done;
941
942 switch (gelf_getclass(elf)) {
943 case ELFCLASS32:
944 btf__set_pointer_size(btf, 4);
945 break;
946 case ELFCLASS64:
947 btf__set_pointer_size(btf, 8);
948 break;
949 default:
950 pr_warn("failed to get ELF class (bitness) for %s\n", path);
951 break;
952 }
953
954 if (btf_ext && btf_ext_data) {
955 *btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
956 err = libbpf_get_error(*btf_ext);
957 if (err)
958 goto done;
959 } else if (btf_ext) {
960 *btf_ext = NULL;
961 }
962done:
963 if (elf)
964 elf_end(elf);
965 close(fd);
966
967 if (!err)
968 return btf;
969
970 if (btf_ext)
971 btf_ext__free(*btf_ext);
972 btf__free(btf);
973
974 return ERR_PTR(err);
975}
976
977struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
978{
979 return libbpf_ptr(btf_parse_elf(path, NULL, btf_ext));
980}
981
982struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf)
983{
984 return libbpf_ptr(btf_parse_elf(path, base_btf, NULL));
985}
986
987static struct btf *btf_parse_raw(const char *path, struct btf *base_btf)
988{
989 struct btf *btf = NULL;
990 void *data = NULL;
991 FILE *f = NULL;
992 __u16 magic;
993 int err = 0;
994 long sz;
995
996 f = fopen(path, "rb");
997 if (!f) {
998 err = -errno;
999 goto err_out;
1000 }
1001
1002
1003 if (fread(&magic, 1, sizeof(magic), f) < sizeof(magic)) {
1004 err = -EIO;
1005 goto err_out;
1006 }
1007 if (magic != BTF_MAGIC && magic != bswap_16(BTF_MAGIC)) {
1008
1009 err = -EPROTO;
1010 goto err_out;
1011 }
1012
1013
1014 if (fseek(f, 0, SEEK_END)) {
1015 err = -errno;
1016 goto err_out;
1017 }
1018 sz = ftell(f);
1019 if (sz < 0) {
1020 err = -errno;
1021 goto err_out;
1022 }
1023
1024 if (fseek(f, 0, SEEK_SET)) {
1025 err = -errno;
1026 goto err_out;
1027 }
1028
1029
1030 data = malloc(sz);
1031 if (!data) {
1032 err = -ENOMEM;
1033 goto err_out;
1034 }
1035 if (fread(data, 1, sz, f) < sz) {
1036 err = -EIO;
1037 goto err_out;
1038 }
1039
1040
1041 btf = btf_new(data, sz, base_btf);
1042
1043err_out:
1044 free(data);
1045 if (f)
1046 fclose(f);
1047 return err ? ERR_PTR(err) : btf;
1048}
1049
1050struct btf *btf__parse_raw(const char *path)
1051{
1052 return libbpf_ptr(btf_parse_raw(path, NULL));
1053}
1054
1055struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf)
1056{
1057 return libbpf_ptr(btf_parse_raw(path, base_btf));
1058}
1059
1060static struct btf *btf_parse(const char *path, struct btf *base_btf, struct btf_ext **btf_ext)
1061{
1062 struct btf *btf;
1063 int err;
1064
1065 if (btf_ext)
1066 *btf_ext = NULL;
1067
1068 btf = btf_parse_raw(path, base_btf);
1069 err = libbpf_get_error(btf);
1070 if (!err)
1071 return btf;
1072 if (err != -EPROTO)
1073 return ERR_PTR(err);
1074 return btf_parse_elf(path, base_btf, btf_ext);
1075}
1076
1077struct btf *btf__parse(const char *path, struct btf_ext **btf_ext)
1078{
1079 return libbpf_ptr(btf_parse(path, NULL, btf_ext));
1080}
1081
1082struct btf *btf__parse_split(const char *path, struct btf *base_btf)
1083{
1084 return libbpf_ptr(btf_parse(path, base_btf, NULL));
1085}
1086
1087static int compare_vsi_off(const void *_a, const void *_b)
1088{
1089 const struct btf_var_secinfo *a = _a;
1090 const struct btf_var_secinfo *b = _b;
1091
1092 return a->offset - b->offset;
1093}
1094
1095static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
1096 struct btf_type *t)
1097{
1098 __u32 size = 0, off = 0, i, vars = btf_vlen(t);
1099 const char *name = btf__name_by_offset(btf, t->name_off);
1100 const struct btf_type *t_var;
1101 struct btf_var_secinfo *vsi;
1102 const struct btf_var *var;
1103 int ret;
1104
1105 if (!name) {
1106 pr_debug("No name found in string section for DATASEC kind.\n");
1107 return -ENOENT;
1108 }
1109
1110
1111
1112
1113 if (t->size)
1114 goto sort_vars;
1115
1116 ret = bpf_object__section_size(obj, name, &size);
1117 if (ret || !size || (t->size && t->size != size)) {
1118 pr_debug("Invalid size for section %s: %u bytes\n", name, size);
1119 return -ENOENT;
1120 }
1121
1122 t->size = size;
1123
1124 for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {
1125 t_var = btf__type_by_id(btf, vsi->type);
1126 var = btf_var(t_var);
1127
1128 if (!btf_is_var(t_var)) {
1129 pr_debug("Non-VAR type seen in section %s\n", name);
1130 return -EINVAL;
1131 }
1132
1133 if (var->linkage == BTF_VAR_STATIC)
1134 continue;
1135
1136 name = btf__name_by_offset(btf, t_var->name_off);
1137 if (!name) {
1138 pr_debug("No name found in string section for VAR kind\n");
1139 return -ENOENT;
1140 }
1141
1142 ret = bpf_object__variable_offset(obj, name, &off);
1143 if (ret) {
1144 pr_debug("No offset found in symbol table for VAR %s\n",
1145 name);
1146 return -ENOENT;
1147 }
1148
1149 vsi->offset = off;
1150 }
1151
1152sort_vars:
1153 qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
1154 return 0;
1155}
1156
1157int btf__finalize_data(struct bpf_object *obj, struct btf *btf)
1158{
1159 int err = 0;
1160 __u32 i;
1161
1162 for (i = 1; i <= btf->nr_types; i++) {
1163 struct btf_type *t = btf_type_by_id(btf, i);
1164
1165
1166
1167
1168
1169
1170 if (btf_is_datasec(t)) {
1171 err = btf_fixup_datasec(obj, btf, t);
1172 if (err)
1173 break;
1174 }
1175 }
1176
1177 return libbpf_err(err);
1178}
1179
1180static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian);
1181
1182int btf__load(struct btf *btf)
1183{
1184 __u32 log_buf_size = 0, raw_size;
1185 char *log_buf = NULL;
1186 void *raw_data;
1187 int err = 0;
1188
1189 if (btf->fd >= 0)
1190 return libbpf_err(-EEXIST);
1191
1192retry_load:
1193 if (log_buf_size) {
1194 log_buf = malloc(log_buf_size);
1195 if (!log_buf)
1196 return libbpf_err(-ENOMEM);
1197
1198 *log_buf = 0;
1199 }
1200
1201 raw_data = btf_get_raw_data(btf, &raw_size, false);
1202 if (!raw_data) {
1203 err = -ENOMEM;
1204 goto done;
1205 }
1206
1207 btf->raw_size = raw_size;
1208 btf->raw_data = raw_data;
1209
1210 btf->fd = bpf_load_btf(raw_data, raw_size, log_buf, log_buf_size, false);
1211 if (btf->fd < 0) {
1212 if (!log_buf || errno == ENOSPC) {
1213 log_buf_size = max((__u32)BPF_LOG_BUF_SIZE,
1214 log_buf_size << 1);
1215 free(log_buf);
1216 goto retry_load;
1217 }
1218
1219 err = -errno;
1220 pr_warn("Error loading BTF: %s(%d)\n", strerror(errno), errno);
1221 if (*log_buf)
1222 pr_warn("%s\n", log_buf);
1223 goto done;
1224 }
1225
1226done:
1227 free(log_buf);
1228 return libbpf_err(err);
1229}
1230
1231int btf__fd(const struct btf *btf)
1232{
1233 return btf->fd;
1234}
1235
1236void btf__set_fd(struct btf *btf, int fd)
1237{
1238 btf->fd = fd;
1239}
1240
1241static const void *btf_strs_data(const struct btf *btf)
1242{
1243 return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set);
1244}
1245
1246static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian)
1247{
1248 struct btf_header *hdr = btf->hdr;
1249 struct btf_type *t;
1250 void *data, *p;
1251 __u32 data_sz;
1252 int i;
1253
1254 data = swap_endian ? btf->raw_data_swapped : btf->raw_data;
1255 if (data) {
1256 *size = btf->raw_size;
1257 return data;
1258 }
1259
1260 data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len;
1261 data = calloc(1, data_sz);
1262 if (!data)
1263 return NULL;
1264 p = data;
1265
1266 memcpy(p, hdr, hdr->hdr_len);
1267 if (swap_endian)
1268 btf_bswap_hdr(p);
1269 p += hdr->hdr_len;
1270
1271 memcpy(p, btf->types_data, hdr->type_len);
1272 if (swap_endian) {
1273 for (i = 0; i < btf->nr_types; i++) {
1274 t = p + btf->type_offs[i];
1275
1276
1277
1278
1279 if (btf_bswap_type_rest(t))
1280 goto err_out;
1281 btf_bswap_type_base(t);
1282 }
1283 }
1284 p += hdr->type_len;
1285
1286 memcpy(p, btf_strs_data(btf), hdr->str_len);
1287 p += hdr->str_len;
1288
1289 *size = data_sz;
1290 return data;
1291err_out:
1292 free(data);
1293 return NULL;
1294}
1295
1296const void *btf__get_raw_data(const struct btf *btf_ro, __u32 *size)
1297{
1298 struct btf *btf = (struct btf *)btf_ro;
1299 __u32 data_sz;
1300 void *data;
1301
1302 data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian);
1303 if (!data)
1304 return errno = -ENOMEM, NULL;
1305
1306 btf->raw_size = data_sz;
1307 if (btf->swapped_endian)
1308 btf->raw_data_swapped = data;
1309 else
1310 btf->raw_data = data;
1311 *size = data_sz;
1312 return data;
1313}
1314
1315const char *btf__str_by_offset(const struct btf *btf, __u32 offset)
1316{
1317 if (offset < btf->start_str_off)
1318 return btf__str_by_offset(btf->base_btf, offset);
1319 else if (offset - btf->start_str_off < btf->hdr->str_len)
1320 return btf_strs_data(btf) + (offset - btf->start_str_off);
1321 else
1322 return errno = EINVAL, NULL;
1323}
1324
1325const char *btf__name_by_offset(const struct btf *btf, __u32 offset)
1326{
1327 return btf__str_by_offset(btf, offset);
1328}
1329
1330struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf)
1331{
1332 struct bpf_btf_info btf_info;
1333 __u32 len = sizeof(btf_info);
1334 __u32 last_size;
1335 struct btf *btf;
1336 void *ptr;
1337 int err;
1338
1339
1340
1341
1342
1343 last_size = 4096;
1344 ptr = malloc(last_size);
1345 if (!ptr)
1346 return ERR_PTR(-ENOMEM);
1347
1348 memset(&btf_info, 0, sizeof(btf_info));
1349 btf_info.btf = ptr_to_u64(ptr);
1350 btf_info.btf_size = last_size;
1351 err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
1352
1353 if (!err && btf_info.btf_size > last_size) {
1354 void *temp_ptr;
1355
1356 last_size = btf_info.btf_size;
1357 temp_ptr = realloc(ptr, last_size);
1358 if (!temp_ptr) {
1359 btf = ERR_PTR(-ENOMEM);
1360 goto exit_free;
1361 }
1362 ptr = temp_ptr;
1363
1364 len = sizeof(btf_info);
1365 memset(&btf_info, 0, sizeof(btf_info));
1366 btf_info.btf = ptr_to_u64(ptr);
1367 btf_info.btf_size = last_size;
1368
1369 err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
1370 }
1371
1372 if (err || btf_info.btf_size > last_size) {
1373 btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG);
1374 goto exit_free;
1375 }
1376
1377 btf = btf_new(ptr, btf_info.btf_size, base_btf);
1378
1379exit_free:
1380 free(ptr);
1381 return btf;
1382}
1383
1384int btf__get_from_id(__u32 id, struct btf **btf)
1385{
1386 struct btf *res;
1387 int err, btf_fd;
1388
1389 *btf = NULL;
1390 btf_fd = bpf_btf_get_fd_by_id(id);
1391 if (btf_fd < 0)
1392 return libbpf_err(-errno);
1393
1394 res = btf_get_from_fd(btf_fd, NULL);
1395 err = libbpf_get_error(res);
1396
1397 close(btf_fd);
1398
1399 if (err)
1400 return libbpf_err(err);
1401
1402 *btf = res;
1403 return 0;
1404}
1405
1406int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,
1407 __u32 expected_key_size, __u32 expected_value_size,
1408 __u32 *key_type_id, __u32 *value_type_id)
1409{
1410 const struct btf_type *container_type;
1411 const struct btf_member *key, *value;
1412 const size_t max_name = 256;
1413 char container_name[max_name];
1414 __s64 key_size, value_size;
1415 __s32 container_id;
1416
1417 if (snprintf(container_name, max_name, "____btf_map_%s", map_name) == max_name) {
1418 pr_warn("map:%s length of '____btf_map_%s' is too long\n",
1419 map_name, map_name);
1420 return libbpf_err(-EINVAL);
1421 }
1422
1423 container_id = btf__find_by_name(btf, container_name);
1424 if (container_id < 0) {
1425 pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n",
1426 map_name, container_name);
1427 return libbpf_err(container_id);
1428 }
1429
1430 container_type = btf__type_by_id(btf, container_id);
1431 if (!container_type) {
1432 pr_warn("map:%s cannot find BTF type for container_id:%u\n",
1433 map_name, container_id);
1434 return libbpf_err(-EINVAL);
1435 }
1436
1437 if (!btf_is_struct(container_type) || btf_vlen(container_type) < 2) {
1438 pr_warn("map:%s container_name:%s is an invalid container struct\n",
1439 map_name, container_name);
1440 return libbpf_err(-EINVAL);
1441 }
1442
1443 key = btf_members(container_type);
1444 value = key + 1;
1445
1446 key_size = btf__resolve_size(btf, key->type);
1447 if (key_size < 0) {
1448 pr_warn("map:%s invalid BTF key_type_size\n", map_name);
1449 return libbpf_err(key_size);
1450 }
1451
1452 if (expected_key_size != key_size) {
1453 pr_warn("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
1454 map_name, (__u32)key_size, expected_key_size);
1455 return libbpf_err(-EINVAL);
1456 }
1457
1458 value_size = btf__resolve_size(btf, value->type);
1459 if (value_size < 0) {
1460 pr_warn("map:%s invalid BTF value_type_size\n", map_name);
1461 return libbpf_err(value_size);
1462 }
1463
1464 if (expected_value_size != value_size) {
1465 pr_warn("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
1466 map_name, (__u32)value_size, expected_value_size);
1467 return libbpf_err(-EINVAL);
1468 }
1469
1470 *key_type_id = key->type;
1471 *value_type_id = value->type;
1472
1473 return 0;
1474}
1475
1476static void btf_invalidate_raw_data(struct btf *btf)
1477{
1478 if (btf->raw_data) {
1479 free(btf->raw_data);
1480 btf->raw_data = NULL;
1481 }
1482 if (btf->raw_data_swapped) {
1483 free(btf->raw_data_swapped);
1484 btf->raw_data_swapped = NULL;
1485 }
1486}
1487
1488
1489
1490
1491
1492static int btf_ensure_modifiable(struct btf *btf)
1493{
1494 void *hdr, *types;
1495 struct strset *set = NULL;
1496 int err = -ENOMEM;
1497
1498 if (btf_is_modifiable(btf)) {
1499
1500 btf_invalidate_raw_data(btf);
1501 return 0;
1502 }
1503
1504
1505 hdr = malloc(btf->hdr->hdr_len);
1506 types = malloc(btf->hdr->type_len);
1507 if (!hdr || !types)
1508 goto err_out;
1509
1510 memcpy(hdr, btf->hdr, btf->hdr->hdr_len);
1511 memcpy(types, btf->types_data, btf->hdr->type_len);
1512
1513
1514 set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len);
1515 if (IS_ERR(set)) {
1516 err = PTR_ERR(set);
1517 goto err_out;
1518 }
1519
1520
1521 btf->hdr = hdr;
1522 btf->types_data = types;
1523 btf->types_data_cap = btf->hdr->type_len;
1524 btf->strs_data = NULL;
1525 btf->strs_set = set;
1526
1527
1528
1529 if (btf->hdr->str_len == 0)
1530 btf->strs_deduped = true;
1531 if (!btf->base_btf && btf->hdr->str_len == 1)
1532 btf->strs_deduped = true;
1533
1534
1535 btf_invalidate_raw_data(btf);
1536
1537 return 0;
1538
1539err_out:
1540 strset__free(set);
1541 free(hdr);
1542 free(types);
1543 return err;
1544}
1545
1546
1547
1548
1549
1550
1551
1552int btf__find_str(struct btf *btf, const char *s)
1553{
1554 int off;
1555
1556 if (btf->base_btf) {
1557 off = btf__find_str(btf->base_btf, s);
1558 if (off != -ENOENT)
1559 return off;
1560 }
1561
1562
1563 if (btf_ensure_modifiable(btf))
1564 return libbpf_err(-ENOMEM);
1565
1566 off = strset__find_str(btf->strs_set, s);
1567 if (off < 0)
1568 return libbpf_err(off);
1569
1570 return btf->start_str_off + off;
1571}
1572
1573
1574
1575
1576
1577
1578int btf__add_str(struct btf *btf, const char *s)
1579{
1580 int off;
1581
1582 if (btf->base_btf) {
1583 off = btf__find_str(btf->base_btf, s);
1584 if (off != -ENOENT)
1585 return off;
1586 }
1587
1588 if (btf_ensure_modifiable(btf))
1589 return libbpf_err(-ENOMEM);
1590
1591 off = strset__add_str(btf->strs_set, s);
1592 if (off < 0)
1593 return libbpf_err(off);
1594
1595 btf->hdr->str_len = strset__data_size(btf->strs_set);
1596
1597 return btf->start_str_off + off;
1598}
1599
1600static void *btf_add_type_mem(struct btf *btf, size_t add_sz)
1601{
1602 return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1,
1603 btf->hdr->type_len, UINT_MAX, add_sz);
1604}
1605
1606static void btf_type_inc_vlen(struct btf_type *t)
1607{
1608 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t));
1609}
1610
1611static int btf_commit_type(struct btf *btf, int data_sz)
1612{
1613 int err;
1614
1615 err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
1616 if (err)
1617 return libbpf_err(err);
1618
1619 btf->hdr->type_len += data_sz;
1620 btf->hdr->str_off += data_sz;
1621 btf->nr_types++;
1622 return btf->start_id + btf->nr_types - 1;
1623}
1624
1625struct btf_pipe {
1626 const struct btf *src;
1627 struct btf *dst;
1628};
1629
1630static int btf_rewrite_str(__u32 *str_off, void *ctx)
1631{
1632 struct btf_pipe *p = ctx;
1633 int off;
1634
1635 if (!*str_off)
1636 return 0;
1637
1638 off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off));
1639 if (off < 0)
1640 return off;
1641
1642 *str_off = off;
1643 return 0;
1644}
1645
1646int btf__add_type(struct btf *btf, const struct btf *src_btf, const struct btf_type *src_type)
1647{
1648 struct btf_pipe p = { .src = src_btf, .dst = btf };
1649 struct btf_type *t;
1650 int sz, err;
1651
1652 sz = btf_type_size(src_type);
1653 if (sz < 0)
1654 return libbpf_err(sz);
1655
1656
1657 if (btf_ensure_modifiable(btf))
1658 return libbpf_err(-ENOMEM);
1659
1660 t = btf_add_type_mem(btf, sz);
1661 if (!t)
1662 return libbpf_err(-ENOMEM);
1663
1664 memcpy(t, src_type, sz);
1665
1666 err = btf_type_visit_str_offs(t, btf_rewrite_str, &p);
1667 if (err)
1668 return libbpf_err(err);
1669
1670 return btf_commit_type(btf, sz);
1671}
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding)
1683{
1684 struct btf_type *t;
1685 int sz, name_off;
1686
1687
1688 if (!name || !name[0])
1689 return libbpf_err(-EINVAL);
1690
1691 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16)
1692 return libbpf_err(-EINVAL);
1693 if (encoding & ~(BTF_INT_SIGNED | BTF_INT_CHAR | BTF_INT_BOOL))
1694 return libbpf_err(-EINVAL);
1695
1696
1697 if (btf_ensure_modifiable(btf))
1698 return libbpf_err(-ENOMEM);
1699
1700 sz = sizeof(struct btf_type) + sizeof(int);
1701 t = btf_add_type_mem(btf, sz);
1702 if (!t)
1703 return libbpf_err(-ENOMEM);
1704
1705
1706
1707
1708
1709 name_off = btf__add_str(btf, name);
1710 if (name_off < 0)
1711 return name_off;
1712
1713 t->name_off = name_off;
1714 t->info = btf_type_info(BTF_KIND_INT, 0, 0);
1715 t->size = byte_sz;
1716
1717 *(__u32 *)(t + 1) = (encoding << 24) | (byte_sz * 8);
1718
1719 return btf_commit_type(btf, sz);
1720}
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730int btf__add_float(struct btf *btf, const char *name, size_t byte_sz)
1731{
1732 struct btf_type *t;
1733 int sz, name_off;
1734
1735
1736 if (!name || !name[0])
1737 return libbpf_err(-EINVAL);
1738
1739
1740 if (byte_sz != 2 && byte_sz != 4 && byte_sz != 8 && byte_sz != 12 &&
1741 byte_sz != 16)
1742 return libbpf_err(-EINVAL);
1743
1744 if (btf_ensure_modifiable(btf))
1745 return libbpf_err(-ENOMEM);
1746
1747 sz = sizeof(struct btf_type);
1748 t = btf_add_type_mem(btf, sz);
1749 if (!t)
1750 return libbpf_err(-ENOMEM);
1751
1752 name_off = btf__add_str(btf, name);
1753 if (name_off < 0)
1754 return name_off;
1755
1756 t->name_off = name_off;
1757 t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0);
1758 t->size = byte_sz;
1759
1760 return btf_commit_type(btf, sz);
1761}
1762
1763
1764
1765
1766
1767static int validate_type_id(int id)
1768{
1769 if (id < 0 || id > BTF_MAX_NR_TYPES)
1770 return -EINVAL;
1771 return 0;
1772}
1773
1774
1775static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id)
1776{
1777 struct btf_type *t;
1778 int sz, name_off = 0;
1779
1780 if (validate_type_id(ref_type_id))
1781 return libbpf_err(-EINVAL);
1782
1783 if (btf_ensure_modifiable(btf))
1784 return libbpf_err(-ENOMEM);
1785
1786 sz = sizeof(struct btf_type);
1787 t = btf_add_type_mem(btf, sz);
1788 if (!t)
1789 return libbpf_err(-ENOMEM);
1790
1791 if (name && name[0]) {
1792 name_off = btf__add_str(btf, name);
1793 if (name_off < 0)
1794 return name_off;
1795 }
1796
1797 t->name_off = name_off;
1798 t->info = btf_type_info(kind, 0, 0);
1799 t->type = ref_type_id;
1800
1801 return btf_commit_type(btf, sz);
1802}
1803
1804
1805
1806
1807
1808
1809
1810
1811int btf__add_ptr(struct btf *btf, int ref_type_id)
1812{
1813 return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id);
1814}
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 nr_elems)
1826{
1827 struct btf_type *t;
1828 struct btf_array *a;
1829 int sz;
1830
1831 if (validate_type_id(index_type_id) || validate_type_id(elem_type_id))
1832 return libbpf_err(-EINVAL);
1833
1834 if (btf_ensure_modifiable(btf))
1835 return libbpf_err(-ENOMEM);
1836
1837 sz = sizeof(struct btf_type) + sizeof(struct btf_array);
1838 t = btf_add_type_mem(btf, sz);
1839 if (!t)
1840 return libbpf_err(-ENOMEM);
1841
1842 t->name_off = 0;
1843 t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0);
1844 t->size = 0;
1845
1846 a = btf_array(t);
1847 a->type = elem_type_id;
1848 a->index_type = index_type_id;
1849 a->nelems = nr_elems;
1850
1851 return btf_commit_type(btf, sz);
1852}
1853
1854
1855static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 bytes_sz)
1856{
1857 struct btf_type *t;
1858 int sz, name_off = 0;
1859
1860 if (btf_ensure_modifiable(btf))
1861 return libbpf_err(-ENOMEM);
1862
1863 sz = sizeof(struct btf_type);
1864 t = btf_add_type_mem(btf, sz);
1865 if (!t)
1866 return libbpf_err(-ENOMEM);
1867
1868 if (name && name[0]) {
1869 name_off = btf__add_str(btf, name);
1870 if (name_off < 0)
1871 return name_off;
1872 }
1873
1874
1875
1876
1877 t->name_off = name_off;
1878 t->info = btf_type_info(kind, 0, 0);
1879 t->size = bytes_sz;
1880
1881 return btf_commit_type(btf, sz);
1882}
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896int btf__add_struct(struct btf *btf, const char *name, __u32 byte_sz)
1897{
1898 return btf_add_composite(btf, BTF_KIND_STRUCT, name, byte_sz);
1899}
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914int btf__add_union(struct btf *btf, const char *name, __u32 byte_sz)
1915{
1916 return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz);
1917}
1918
1919static struct btf_type *btf_last_type(struct btf *btf)
1920{
1921 return btf_type_by_id(btf, btf__get_nr_types(btf));
1922}
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934int btf__add_field(struct btf *btf, const char *name, int type_id,
1935 __u32 bit_offset, __u32 bit_size)
1936{
1937 struct btf_type *t;
1938 struct btf_member *m;
1939 bool is_bitfield;
1940 int sz, name_off = 0;
1941
1942
1943 if (btf->nr_types == 0)
1944 return libbpf_err(-EINVAL);
1945 t = btf_last_type(btf);
1946 if (!btf_is_composite(t))
1947 return libbpf_err(-EINVAL);
1948
1949 if (validate_type_id(type_id))
1950 return libbpf_err(-EINVAL);
1951
1952 is_bitfield = bit_size || (bit_offset % 8 != 0);
1953 if (is_bitfield && (bit_size == 0 || bit_size > 255 || bit_offset > 0xffffff))
1954 return libbpf_err(-EINVAL);
1955
1956
1957 if (btf_is_union(t) && bit_offset)
1958 return libbpf_err(-EINVAL);
1959
1960
1961 if (btf_ensure_modifiable(btf))
1962 return libbpf_err(-ENOMEM);
1963
1964 sz = sizeof(struct btf_member);
1965 m = btf_add_type_mem(btf, sz);
1966 if (!m)
1967 return libbpf_err(-ENOMEM);
1968
1969 if (name && name[0]) {
1970 name_off = btf__add_str(btf, name);
1971 if (name_off < 0)
1972 return name_off;
1973 }
1974
1975 m->name_off = name_off;
1976 m->type = type_id;
1977 m->offset = bit_offset | (bit_size << 24);
1978
1979
1980 t = btf_last_type(btf);
1981
1982 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t));
1983
1984 btf->hdr->type_len += sz;
1985 btf->hdr->str_off += sz;
1986 return 0;
1987}
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz)
2003{
2004 struct btf_type *t;
2005 int sz, name_off = 0;
2006
2007
2008 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8)
2009 return libbpf_err(-EINVAL);
2010
2011 if (btf_ensure_modifiable(btf))
2012 return libbpf_err(-ENOMEM);
2013
2014 sz = sizeof(struct btf_type);
2015 t = btf_add_type_mem(btf, sz);
2016 if (!t)
2017 return libbpf_err(-ENOMEM);
2018
2019 if (name && name[0]) {
2020 name_off = btf__add_str(btf, name);
2021 if (name_off < 0)
2022 return name_off;
2023 }
2024
2025
2026 t->name_off = name_off;
2027 t->info = btf_type_info(BTF_KIND_ENUM, 0, 0);
2028 t->size = byte_sz;
2029
2030 return btf_commit_type(btf, sz);
2031}
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041int btf__add_enum_value(struct btf *btf, const char *name, __s64 value)
2042{
2043 struct btf_type *t;
2044 struct btf_enum *v;
2045 int sz, name_off;
2046
2047
2048 if (btf->nr_types == 0)
2049 return libbpf_err(-EINVAL);
2050 t = btf_last_type(btf);
2051 if (!btf_is_enum(t))
2052 return libbpf_err(-EINVAL);
2053
2054
2055 if (!name || !name[0])
2056 return libbpf_err(-EINVAL);
2057 if (value < INT_MIN || value > UINT_MAX)
2058 return libbpf_err(-E2BIG);
2059
2060
2061 if (btf_ensure_modifiable(btf))
2062 return libbpf_err(-ENOMEM);
2063
2064 sz = sizeof(struct btf_enum);
2065 v = btf_add_type_mem(btf, sz);
2066 if (!v)
2067 return libbpf_err(-ENOMEM);
2068
2069 name_off = btf__add_str(btf, name);
2070 if (name_off < 0)
2071 return name_off;
2072
2073 v->name_off = name_off;
2074 v->val = value;
2075
2076
2077 t = btf_last_type(btf);
2078 btf_type_inc_vlen(t);
2079
2080 btf->hdr->type_len += sz;
2081 btf->hdr->str_off += sz;
2082 return 0;
2083}
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind)
2095{
2096 if (!name || !name[0])
2097 return libbpf_err(-EINVAL);
2098
2099 switch (fwd_kind) {
2100 case BTF_FWD_STRUCT:
2101 case BTF_FWD_UNION: {
2102 struct btf_type *t;
2103 int id;
2104
2105 id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0);
2106 if (id <= 0)
2107 return id;
2108 t = btf_type_by_id(btf, id);
2109 t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION);
2110 return id;
2111 }
2112 case BTF_FWD_ENUM:
2113
2114
2115
2116 return btf__add_enum(btf, name, sizeof(int));
2117 default:
2118 return libbpf_err(-EINVAL);
2119 }
2120}
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id)
2131{
2132 if (!name || !name[0])
2133 return libbpf_err(-EINVAL);
2134
2135 return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id);
2136}
2137
2138
2139
2140
2141
2142
2143
2144
2145int btf__add_volatile(struct btf *btf, int ref_type_id)
2146{
2147 return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id);
2148}
2149
2150
2151
2152
2153
2154
2155
2156
2157int btf__add_const(struct btf *btf, int ref_type_id)
2158{
2159 return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id);
2160}
2161
2162
2163
2164
2165
2166
2167
2168
2169int btf__add_restrict(struct btf *btf, int ref_type_id)
2170{
2171 return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id);
2172}
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182int btf__add_func(struct btf *btf, const char *name,
2183 enum btf_func_linkage linkage, int proto_type_id)
2184{
2185 int id;
2186
2187 if (!name || !name[0])
2188 return libbpf_err(-EINVAL);
2189 if (linkage != BTF_FUNC_STATIC && linkage != BTF_FUNC_GLOBAL &&
2190 linkage != BTF_FUNC_EXTERN)
2191 return libbpf_err(-EINVAL);
2192
2193 id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id);
2194 if (id > 0) {
2195 struct btf_type *t = btf_type_by_id(btf, id);
2196
2197 t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0);
2198 }
2199 return libbpf_err(id);
2200}
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214int btf__add_func_proto(struct btf *btf, int ret_type_id)
2215{
2216 struct btf_type *t;
2217 int sz;
2218
2219 if (validate_type_id(ret_type_id))
2220 return libbpf_err(-EINVAL);
2221
2222 if (btf_ensure_modifiable(btf))
2223 return libbpf_err(-ENOMEM);
2224
2225 sz = sizeof(struct btf_type);
2226 t = btf_add_type_mem(btf, sz);
2227 if (!t)
2228 return libbpf_err(-ENOMEM);
2229
2230
2231
2232
2233 t->name_off = 0;
2234 t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0);
2235 t->type = ret_type_id;
2236
2237 return btf_commit_type(btf, sz);
2238}
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248int btf__add_func_param(struct btf *btf, const char *name, int type_id)
2249{
2250 struct btf_type *t;
2251 struct btf_param *p;
2252 int sz, name_off = 0;
2253
2254 if (validate_type_id(type_id))
2255 return libbpf_err(-EINVAL);
2256
2257
2258 if (btf->nr_types == 0)
2259 return libbpf_err(-EINVAL);
2260 t = btf_last_type(btf);
2261 if (!btf_is_func_proto(t))
2262 return libbpf_err(-EINVAL);
2263
2264
2265 if (btf_ensure_modifiable(btf))
2266 return libbpf_err(-ENOMEM);
2267
2268 sz = sizeof(struct btf_param);
2269 p = btf_add_type_mem(btf, sz);
2270 if (!p)
2271 return libbpf_err(-ENOMEM);
2272
2273 if (name && name[0]) {
2274 name_off = btf__add_str(btf, name);
2275 if (name_off < 0)
2276 return name_off;
2277 }
2278
2279 p->name_off = name_off;
2280 p->type = type_id;
2281
2282
2283 t = btf_last_type(btf);
2284 btf_type_inc_vlen(t);
2285
2286 btf->hdr->type_len += sz;
2287 btf->hdr->str_off += sz;
2288 return 0;
2289}
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id)
2302{
2303 struct btf_type *t;
2304 struct btf_var *v;
2305 int sz, name_off;
2306
2307
2308 if (!name || !name[0])
2309 return libbpf_err(-EINVAL);
2310 if (linkage != BTF_VAR_STATIC && linkage != BTF_VAR_GLOBAL_ALLOCATED &&
2311 linkage != BTF_VAR_GLOBAL_EXTERN)
2312 return libbpf_err(-EINVAL);
2313 if (validate_type_id(type_id))
2314 return libbpf_err(-EINVAL);
2315
2316
2317 if (btf_ensure_modifiable(btf))
2318 return libbpf_err(-ENOMEM);
2319
2320 sz = sizeof(struct btf_type) + sizeof(struct btf_var);
2321 t = btf_add_type_mem(btf, sz);
2322 if (!t)
2323 return libbpf_err(-ENOMEM);
2324
2325 name_off = btf__add_str(btf, name);
2326 if (name_off < 0)
2327 return name_off;
2328
2329 t->name_off = name_off;
2330 t->info = btf_type_info(BTF_KIND_VAR, 0, 0);
2331 t->type = type_id;
2332
2333 v = btf_var(t);
2334 v->linkage = linkage;
2335
2336 return btf_commit_type(btf, sz);
2337}
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz)
2352{
2353 struct btf_type *t;
2354 int sz, name_off;
2355
2356
2357 if (!name || !name[0])
2358 return libbpf_err(-EINVAL);
2359
2360 if (btf_ensure_modifiable(btf))
2361 return libbpf_err(-ENOMEM);
2362
2363 sz = sizeof(struct btf_type);
2364 t = btf_add_type_mem(btf, sz);
2365 if (!t)
2366 return libbpf_err(-ENOMEM);
2367
2368 name_off = btf__add_str(btf, name);
2369 if (name_off < 0)
2370 return name_off;
2371
2372
2373 t->name_off = name_off;
2374 t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0);
2375 t->size = byte_sz;
2376
2377 return btf_commit_type(btf, sz);
2378}
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __u32 byte_sz)
2391{
2392 struct btf_type *t;
2393 struct btf_var_secinfo *v;
2394 int sz;
2395
2396
2397 if (btf->nr_types == 0)
2398 return libbpf_err(-EINVAL);
2399 t = btf_last_type(btf);
2400 if (!btf_is_datasec(t))
2401 return libbpf_err(-EINVAL);
2402
2403 if (validate_type_id(var_type_id))
2404 return libbpf_err(-EINVAL);
2405
2406
2407 if (btf_ensure_modifiable(btf))
2408 return libbpf_err(-ENOMEM);
2409
2410 sz = sizeof(struct btf_var_secinfo);
2411 v = btf_add_type_mem(btf, sz);
2412 if (!v)
2413 return libbpf_err(-ENOMEM);
2414
2415 v->type = var_type_id;
2416 v->offset = offset;
2417 v->size = byte_sz;
2418
2419
2420 t = btf_last_type(btf);
2421 btf_type_inc_vlen(t);
2422
2423 btf->hdr->type_len += sz;
2424 btf->hdr->str_off += sz;
2425 return 0;
2426}
2427
2428struct btf_ext_sec_setup_param {
2429 __u32 off;
2430 __u32 len;
2431 __u32 min_rec_size;
2432 struct btf_ext_info *ext_info;
2433 const char *desc;
2434};
2435
2436static int btf_ext_setup_info(struct btf_ext *btf_ext,
2437 struct btf_ext_sec_setup_param *ext_sec)
2438{
2439 const struct btf_ext_info_sec *sinfo;
2440 struct btf_ext_info *ext_info;
2441 __u32 info_left, record_size;
2442
2443 void *info;
2444
2445 if (ext_sec->len == 0)
2446 return 0;
2447
2448 if (ext_sec->off & 0x03) {
2449 pr_debug(".BTF.ext %s section is not aligned to 4 bytes\n",
2450 ext_sec->desc);
2451 return -EINVAL;
2452 }
2453
2454 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off;
2455 info_left = ext_sec->len;
2456
2457 if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) {
2458 pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n",
2459 ext_sec->desc, ext_sec->off, ext_sec->len);
2460 return -EINVAL;
2461 }
2462
2463
2464 if (info_left < sizeof(__u32)) {
2465 pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc);
2466 return -EINVAL;
2467 }
2468
2469
2470 record_size = *(__u32 *)info;
2471 if (record_size < ext_sec->min_rec_size ||
2472 record_size & 0x03) {
2473 pr_debug("%s section in .BTF.ext has invalid record size %u\n",
2474 ext_sec->desc, record_size);
2475 return -EINVAL;
2476 }
2477
2478 sinfo = info + sizeof(__u32);
2479 info_left -= sizeof(__u32);
2480
2481
2482 if (!info_left) {
2483 pr_debug("%s section in .BTF.ext has no records", ext_sec->desc);
2484 return -EINVAL;
2485 }
2486
2487 while (info_left) {
2488 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec);
2489 __u64 total_record_size;
2490 __u32 num_records;
2491
2492 if (info_left < sec_hdrlen) {
2493 pr_debug("%s section header is not found in .BTF.ext\n",
2494 ext_sec->desc);
2495 return -EINVAL;
2496 }
2497
2498 num_records = sinfo->num_info;
2499 if (num_records == 0) {
2500 pr_debug("%s section has incorrect num_records in .BTF.ext\n",
2501 ext_sec->desc);
2502 return -EINVAL;
2503 }
2504
2505 total_record_size = sec_hdrlen +
2506 (__u64)num_records * record_size;
2507 if (info_left < total_record_size) {
2508 pr_debug("%s section has incorrect num_records in .BTF.ext\n",
2509 ext_sec->desc);
2510 return -EINVAL;
2511 }
2512
2513 info_left -= total_record_size;
2514 sinfo = (void *)sinfo + total_record_size;
2515 }
2516
2517 ext_info = ext_sec->ext_info;
2518 ext_info->len = ext_sec->len - sizeof(__u32);
2519 ext_info->rec_size = record_size;
2520 ext_info->info = info + sizeof(__u32);
2521
2522 return 0;
2523}
2524
2525static int btf_ext_setup_func_info(struct btf_ext *btf_ext)
2526{
2527 struct btf_ext_sec_setup_param param = {
2528 .off = btf_ext->hdr->func_info_off,
2529 .len = btf_ext->hdr->func_info_len,
2530 .min_rec_size = sizeof(struct bpf_func_info_min),
2531 .ext_info = &btf_ext->func_info,
2532 .desc = "func_info"
2533 };
2534
2535 return btf_ext_setup_info(btf_ext, ¶m);
2536}
2537
2538static int btf_ext_setup_line_info(struct btf_ext *btf_ext)
2539{
2540 struct btf_ext_sec_setup_param param = {
2541 .off = btf_ext->hdr->line_info_off,
2542 .len = btf_ext->hdr->line_info_len,
2543 .min_rec_size = sizeof(struct bpf_line_info_min),
2544 .ext_info = &btf_ext->line_info,
2545 .desc = "line_info",
2546 };
2547
2548 return btf_ext_setup_info(btf_ext, ¶m);
2549}
2550
2551static int btf_ext_setup_core_relos(struct btf_ext *btf_ext)
2552{
2553 struct btf_ext_sec_setup_param param = {
2554 .off = btf_ext->hdr->core_relo_off,
2555 .len = btf_ext->hdr->core_relo_len,
2556 .min_rec_size = sizeof(struct bpf_core_relo),
2557 .ext_info = &btf_ext->core_relo_info,
2558 .desc = "core_relo",
2559 };
2560
2561 return btf_ext_setup_info(btf_ext, ¶m);
2562}
2563
2564static int btf_ext_parse_hdr(__u8 *data, __u32 data_size)
2565{
2566 const struct btf_ext_header *hdr = (struct btf_ext_header *)data;
2567
2568 if (data_size < offsetofend(struct btf_ext_header, hdr_len) ||
2569 data_size < hdr->hdr_len) {
2570 pr_debug("BTF.ext header not found");
2571 return -EINVAL;
2572 }
2573
2574 if (hdr->magic == bswap_16(BTF_MAGIC)) {
2575 pr_warn("BTF.ext in non-native endianness is not supported\n");
2576 return -ENOTSUP;
2577 } else if (hdr->magic != BTF_MAGIC) {
2578 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic);
2579 return -EINVAL;
2580 }
2581
2582 if (hdr->version != BTF_VERSION) {
2583 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version);
2584 return -ENOTSUP;
2585 }
2586
2587 if (hdr->flags) {
2588 pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags);
2589 return -ENOTSUP;
2590 }
2591
2592 if (data_size == hdr->hdr_len) {
2593 pr_debug("BTF.ext has no data\n");
2594 return -EINVAL;
2595 }
2596
2597 return 0;
2598}
2599
2600void btf_ext__free(struct btf_ext *btf_ext)
2601{
2602 if (IS_ERR_OR_NULL(btf_ext))
2603 return;
2604 free(btf_ext->data);
2605 free(btf_ext);
2606}
2607
2608struct btf_ext *btf_ext__new(__u8 *data, __u32 size)
2609{
2610 struct btf_ext *btf_ext;
2611 int err;
2612
2613 err = btf_ext_parse_hdr(data, size);
2614 if (err)
2615 return libbpf_err_ptr(err);
2616
2617 btf_ext = calloc(1, sizeof(struct btf_ext));
2618 if (!btf_ext)
2619 return libbpf_err_ptr(-ENOMEM);
2620
2621 btf_ext->data_size = size;
2622 btf_ext->data = malloc(size);
2623 if (!btf_ext->data) {
2624 err = -ENOMEM;
2625 goto done;
2626 }
2627 memcpy(btf_ext->data, data, size);
2628
2629 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, line_info_len)) {
2630 err = -EINVAL;
2631 goto done;
2632 }
2633
2634 err = btf_ext_setup_func_info(btf_ext);
2635 if (err)
2636 goto done;
2637
2638 err = btf_ext_setup_line_info(btf_ext);
2639 if (err)
2640 goto done;
2641
2642 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) {
2643 err = -EINVAL;
2644 goto done;
2645 }
2646
2647 err = btf_ext_setup_core_relos(btf_ext);
2648 if (err)
2649 goto done;
2650
2651done:
2652 if (err) {
2653 btf_ext__free(btf_ext);
2654 return libbpf_err_ptr(err);
2655 }
2656
2657 return btf_ext;
2658}
2659
2660const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size)
2661{
2662 *size = btf_ext->data_size;
2663 return btf_ext->data;
2664}
2665
2666static int btf_ext_reloc_info(const struct btf *btf,
2667 const struct btf_ext_info *ext_info,
2668 const char *sec_name, __u32 insns_cnt,
2669 void **info, __u32 *cnt)
2670{
2671 __u32 sec_hdrlen = sizeof(struct btf_ext_info_sec);
2672 __u32 i, record_size, existing_len, records_len;
2673 struct btf_ext_info_sec *sinfo;
2674 const char *info_sec_name;
2675 __u64 remain_len;
2676 void *data;
2677
2678 record_size = ext_info->rec_size;
2679 sinfo = ext_info->info;
2680 remain_len = ext_info->len;
2681 while (remain_len > 0) {
2682 records_len = sinfo->num_info * record_size;
2683 info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off);
2684 if (strcmp(info_sec_name, sec_name)) {
2685 remain_len -= sec_hdrlen + records_len;
2686 sinfo = (void *)sinfo + sec_hdrlen + records_len;
2687 continue;
2688 }
2689
2690 existing_len = (*cnt) * record_size;
2691 data = realloc(*info, existing_len + records_len);
2692 if (!data)
2693 return libbpf_err(-ENOMEM);
2694
2695 memcpy(data + existing_len, sinfo->data, records_len);
2696
2697
2698
2699 for (i = 0; i < sinfo->num_info; i++) {
2700 __u32 *insn_off;
2701
2702 insn_off = data + existing_len + (i * record_size);
2703 *insn_off = *insn_off / sizeof(struct bpf_insn) + insns_cnt;
2704 }
2705 *info = data;
2706 *cnt += sinfo->num_info;
2707 return 0;
2708 }
2709
2710 return libbpf_err(-ENOENT);
2711}
2712
2713int btf_ext__reloc_func_info(const struct btf *btf,
2714 const struct btf_ext *btf_ext,
2715 const char *sec_name, __u32 insns_cnt,
2716 void **func_info, __u32 *cnt)
2717{
2718 return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name,
2719 insns_cnt, func_info, cnt);
2720}
2721
2722int btf_ext__reloc_line_info(const struct btf *btf,
2723 const struct btf_ext *btf_ext,
2724 const char *sec_name, __u32 insns_cnt,
2725 void **line_info, __u32 *cnt)
2726{
2727 return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name,
2728 insns_cnt, line_info, cnt);
2729}
2730
2731__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext)
2732{
2733 return btf_ext->func_info.rec_size;
2734}
2735
2736__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext)
2737{
2738 return btf_ext->line_info.rec_size;
2739}
2740
2741struct btf_dedup;
2742
2743static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
2744 const struct btf_dedup_opts *opts);
2745static void btf_dedup_free(struct btf_dedup *d);
2746static int btf_dedup_prep(struct btf_dedup *d);
2747static int btf_dedup_strings(struct btf_dedup *d);
2748static int btf_dedup_prim_types(struct btf_dedup *d);
2749static int btf_dedup_struct_types(struct btf_dedup *d);
2750static int btf_dedup_ref_types(struct btf_dedup *d);
2751static int btf_dedup_compact_types(struct btf_dedup *d);
2752static int btf_dedup_remap_types(struct btf_dedup *d);
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891int btf__dedup(struct btf *btf, struct btf_ext *btf_ext,
2892 const struct btf_dedup_opts *opts)
2893{
2894 struct btf_dedup *d = btf_dedup_new(btf, btf_ext, opts);
2895 int err;
2896
2897 if (IS_ERR(d)) {
2898 pr_debug("btf_dedup_new failed: %ld", PTR_ERR(d));
2899 return libbpf_err(-EINVAL);
2900 }
2901
2902 if (btf_ensure_modifiable(btf))
2903 return libbpf_err(-ENOMEM);
2904
2905 err = btf_dedup_prep(d);
2906 if (err) {
2907 pr_debug("btf_dedup_prep failed:%d\n", err);
2908 goto done;
2909 }
2910 err = btf_dedup_strings(d);
2911 if (err < 0) {
2912 pr_debug("btf_dedup_strings failed:%d\n", err);
2913 goto done;
2914 }
2915 err = btf_dedup_prim_types(d);
2916 if (err < 0) {
2917 pr_debug("btf_dedup_prim_types failed:%d\n", err);
2918 goto done;
2919 }
2920 err = btf_dedup_struct_types(d);
2921 if (err < 0) {
2922 pr_debug("btf_dedup_struct_types failed:%d\n", err);
2923 goto done;
2924 }
2925 err = btf_dedup_ref_types(d);
2926 if (err < 0) {
2927 pr_debug("btf_dedup_ref_types failed:%d\n", err);
2928 goto done;
2929 }
2930 err = btf_dedup_compact_types(d);
2931 if (err < 0) {
2932 pr_debug("btf_dedup_compact_types failed:%d\n", err);
2933 goto done;
2934 }
2935 err = btf_dedup_remap_types(d);
2936 if (err < 0) {
2937 pr_debug("btf_dedup_remap_types failed:%d\n", err);
2938 goto done;
2939 }
2940
2941done:
2942 btf_dedup_free(d);
2943 return libbpf_err(err);
2944}
2945
2946#define BTF_UNPROCESSED_ID ((__u32)-1)
2947#define BTF_IN_PROGRESS_ID ((__u32)-2)
2948
2949struct btf_dedup {
2950
2951 struct btf *btf;
2952
2953
2954
2955
2956 struct btf_ext *btf_ext;
2957
2958
2959
2960
2961
2962
2963
2964 struct hashmap *dedup_table;
2965
2966 __u32 *map;
2967
2968 __u32 *hypot_map;
2969 __u32 *hypot_list;
2970 size_t hypot_cnt;
2971 size_t hypot_cap;
2972
2973
2974
2975
2976
2977
2978 bool hypot_adjust_canon;
2979
2980 struct btf_dedup_opts opts;
2981
2982 struct strset *strs_set;
2983};
2984
2985static long hash_combine(long h, long value)
2986{
2987 return h * 31 + value;
2988}
2989
2990#define for_each_dedup_cand(d, node, hash) \
2991 hashmap__for_each_key_entry(d->dedup_table, node, (void *)hash)
2992
2993static int btf_dedup_table_add(struct btf_dedup *d, long hash, __u32 type_id)
2994{
2995 return hashmap__append(d->dedup_table,
2996 (void *)hash, (void *)(long)type_id);
2997}
2998
2999static int btf_dedup_hypot_map_add(struct btf_dedup *d,
3000 __u32 from_id, __u32 to_id)
3001{
3002 if (d->hypot_cnt == d->hypot_cap) {
3003 __u32 *new_list;
3004
3005 d->hypot_cap += max((size_t)16, d->hypot_cap / 2);
3006 new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32));
3007 if (!new_list)
3008 return -ENOMEM;
3009 d->hypot_list = new_list;
3010 }
3011 d->hypot_list[d->hypot_cnt++] = from_id;
3012 d->hypot_map[from_id] = to_id;
3013 return 0;
3014}
3015
3016static void btf_dedup_clear_hypot_map(struct btf_dedup *d)
3017{
3018 int i;
3019
3020 for (i = 0; i < d->hypot_cnt; i++)
3021 d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID;
3022 d->hypot_cnt = 0;
3023 d->hypot_adjust_canon = false;
3024}
3025
3026static void btf_dedup_free(struct btf_dedup *d)
3027{
3028 hashmap__free(d->dedup_table);
3029 d->dedup_table = NULL;
3030
3031 free(d->map);
3032 d->map = NULL;
3033
3034 free(d->hypot_map);
3035 d->hypot_map = NULL;
3036
3037 free(d->hypot_list);
3038 d->hypot_list = NULL;
3039
3040 free(d);
3041}
3042
3043static size_t btf_dedup_identity_hash_fn(const void *key, void *ctx)
3044{
3045 return (size_t)key;
3046}
3047
3048static size_t btf_dedup_collision_hash_fn(const void *key, void *ctx)
3049{
3050 return 0;
3051}
3052
3053static bool btf_dedup_equal_fn(const void *k1, const void *k2, void *ctx)
3054{
3055 return k1 == k2;
3056}
3057
3058static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext,
3059 const struct btf_dedup_opts *opts)
3060{
3061 struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup));
3062 hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn;
3063 int i, err = 0, type_cnt;
3064
3065 if (!d)
3066 return ERR_PTR(-ENOMEM);
3067
3068 d->opts.dont_resolve_fwds = opts && opts->dont_resolve_fwds;
3069
3070 if (opts && opts->dedup_table_size == 1)
3071 hash_fn = btf_dedup_collision_hash_fn;
3072
3073 d->btf = btf;
3074 d->btf_ext = btf_ext;
3075
3076 d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL);
3077 if (IS_ERR(d->dedup_table)) {
3078 err = PTR_ERR(d->dedup_table);
3079 d->dedup_table = NULL;
3080 goto done;
3081 }
3082
3083 type_cnt = btf__get_nr_types(btf) + 1;
3084 d->map = malloc(sizeof(__u32) * type_cnt);
3085 if (!d->map) {
3086 err = -ENOMEM;
3087 goto done;
3088 }
3089
3090 d->map[0] = 0;
3091 for (i = 1; i < type_cnt; i++) {
3092 struct btf_type *t = btf_type_by_id(d->btf, i);
3093
3094
3095 if (btf_is_var(t) || btf_is_datasec(t))
3096 d->map[i] = i;
3097 else
3098 d->map[i] = BTF_UNPROCESSED_ID;
3099 }
3100
3101 d->hypot_map = malloc(sizeof(__u32) * type_cnt);
3102 if (!d->hypot_map) {
3103 err = -ENOMEM;
3104 goto done;
3105 }
3106 for (i = 0; i < type_cnt; i++)
3107 d->hypot_map[i] = BTF_UNPROCESSED_ID;
3108
3109done:
3110 if (err) {
3111 btf_dedup_free(d);
3112 return ERR_PTR(err);
3113 }
3114
3115 return d;
3116}
3117
3118
3119
3120
3121
3122static int btf_for_each_str_off(struct btf_dedup *d, str_off_visit_fn fn, void *ctx)
3123{
3124 int i, r;
3125
3126 for (i = 0; i < d->btf->nr_types; i++) {
3127 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i);
3128
3129 r = btf_type_visit_str_offs(t, fn, ctx);
3130 if (r)
3131 return r;
3132 }
3133
3134 if (!d->btf_ext)
3135 return 0;
3136
3137 r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx);
3138 if (r)
3139 return r;
3140
3141 return 0;
3142}
3143
3144static int strs_dedup_remap_str_off(__u32 *str_off_ptr, void *ctx)
3145{
3146 struct btf_dedup *d = ctx;
3147 __u32 str_off = *str_off_ptr;
3148 const char *s;
3149 int off, err;
3150
3151
3152 if (str_off == 0 || str_off < d->btf->start_str_off)
3153 return 0;
3154
3155 s = btf__str_by_offset(d->btf, str_off);
3156 if (d->btf->base_btf) {
3157 err = btf__find_str(d->btf->base_btf, s);
3158 if (err >= 0) {
3159 *str_off_ptr = err;
3160 return 0;
3161 }
3162 if (err != -ENOENT)
3163 return err;
3164 }
3165
3166 off = strset__add_str(d->strs_set, s);
3167 if (off < 0)
3168 return off;
3169
3170 *str_off_ptr = d->btf->start_str_off + off;
3171 return 0;
3172}
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185static int btf_dedup_strings(struct btf_dedup *d)
3186{
3187 int err;
3188
3189 if (d->btf->strs_deduped)
3190 return 0;
3191
3192 d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0);
3193 if (IS_ERR(d->strs_set)) {
3194 err = PTR_ERR(d->strs_set);
3195 goto err_out;
3196 }
3197
3198 if (!d->btf->base_btf) {
3199
3200
3201
3202 err = strset__add_str(d->strs_set, "");
3203 if (err < 0)
3204 goto err_out;
3205 }
3206
3207
3208 err = btf_for_each_str_off(d, strs_dedup_remap_str_off, d);
3209 if (err)
3210 goto err_out;
3211
3212
3213 strset__free(d->btf->strs_set);
3214 d->btf->hdr->str_len = strset__data_size(d->strs_set);
3215 d->btf->strs_set = d->strs_set;
3216 d->strs_set = NULL;
3217 d->btf->strs_deduped = true;
3218 return 0;
3219
3220err_out:
3221 strset__free(d->strs_set);
3222 d->strs_set = NULL;
3223
3224 return err;
3225}
3226
3227static long btf_hash_common(struct btf_type *t)
3228{
3229 long h;
3230
3231 h = hash_combine(0, t->name_off);
3232 h = hash_combine(h, t->info);
3233 h = hash_combine(h, t->size);
3234 return h;
3235}
3236
3237static bool btf_equal_common(struct btf_type *t1, struct btf_type *t2)
3238{
3239 return t1->name_off == t2->name_off &&
3240 t1->info == t2->info &&
3241 t1->size == t2->size;
3242}
3243
3244
3245static long btf_hash_int(struct btf_type *t)
3246{
3247 __u32 info = *(__u32 *)(t + 1);
3248 long h;
3249
3250 h = btf_hash_common(t);
3251 h = hash_combine(h, info);
3252 return h;
3253}
3254
3255
3256static bool btf_equal_int(struct btf_type *t1, struct btf_type *t2)
3257{
3258 __u32 info1, info2;
3259
3260 if (!btf_equal_common(t1, t2))
3261 return false;
3262 info1 = *(__u32 *)(t1 + 1);
3263 info2 = *(__u32 *)(t2 + 1);
3264 return info1 == info2;
3265}
3266
3267
3268static long btf_hash_enum(struct btf_type *t)
3269{
3270 long h;
3271
3272
3273 h = hash_combine(0, t->name_off);
3274 h = hash_combine(h, t->info & ~0xffff);
3275 h = hash_combine(h, t->size);
3276 return h;
3277}
3278
3279
3280static bool btf_equal_enum(struct btf_type *t1, struct btf_type *t2)
3281{
3282 const struct btf_enum *m1, *m2;
3283 __u16 vlen;
3284 int i;
3285
3286 if (!btf_equal_common(t1, t2))
3287 return false;
3288
3289 vlen = btf_vlen(t1);
3290 m1 = btf_enum(t1);
3291 m2 = btf_enum(t2);
3292 for (i = 0; i < vlen; i++) {
3293 if (m1->name_off != m2->name_off || m1->val != m2->val)
3294 return false;
3295 m1++;
3296 m2++;
3297 }
3298 return true;
3299}
3300
3301static inline bool btf_is_enum_fwd(struct btf_type *t)
3302{
3303 return btf_is_enum(t) && btf_vlen(t) == 0;
3304}
3305
3306static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2)
3307{
3308 if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2))
3309 return btf_equal_enum(t1, t2);
3310
3311 return t1->name_off == t2->name_off &&
3312 (t1->info & ~0xffff) == (t2->info & ~0xffff) &&
3313 t1->size == t2->size;
3314}
3315
3316
3317
3318
3319
3320
3321static long btf_hash_struct(struct btf_type *t)
3322{
3323 const struct btf_member *member = btf_members(t);
3324 __u32 vlen = btf_vlen(t);
3325 long h = btf_hash_common(t);
3326 int i;
3327
3328 for (i = 0; i < vlen; i++) {
3329 h = hash_combine(h, member->name_off);
3330 h = hash_combine(h, member->offset);
3331
3332 member++;
3333 }
3334 return h;
3335}
3336
3337
3338
3339
3340
3341
3342static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2)
3343{
3344 const struct btf_member *m1, *m2;
3345 __u16 vlen;
3346 int i;
3347
3348 if (!btf_equal_common(t1, t2))
3349 return false;
3350
3351 vlen = btf_vlen(t1);
3352 m1 = btf_members(t1);
3353 m2 = btf_members(t2);
3354 for (i = 0; i < vlen; i++) {
3355 if (m1->name_off != m2->name_off || m1->offset != m2->offset)
3356 return false;
3357 m1++;
3358 m2++;
3359 }
3360 return true;
3361}
3362
3363
3364
3365
3366
3367
3368static long btf_hash_array(struct btf_type *t)
3369{
3370 const struct btf_array *info = btf_array(t);
3371 long h = btf_hash_common(t);
3372
3373 h = hash_combine(h, info->type);
3374 h = hash_combine(h, info->index_type);
3375 h = hash_combine(h, info->nelems);
3376 return h;
3377}
3378
3379
3380
3381
3382
3383
3384
3385
3386static bool btf_equal_array(struct btf_type *t1, struct btf_type *t2)
3387{
3388 const struct btf_array *info1, *info2;
3389
3390 if (!btf_equal_common(t1, t2))
3391 return false;
3392
3393 info1 = btf_array(t1);
3394 info2 = btf_array(t2);
3395 return info1->type == info2->type &&
3396 info1->index_type == info2->index_type &&
3397 info1->nelems == info2->nelems;
3398}
3399
3400
3401
3402
3403
3404
3405static bool btf_compat_array(struct btf_type *t1, struct btf_type *t2)
3406{
3407 if (!btf_equal_common(t1, t2))
3408 return false;
3409
3410 return btf_array(t1)->nelems == btf_array(t2)->nelems;
3411}
3412
3413
3414
3415
3416
3417
3418static long btf_hash_fnproto(struct btf_type *t)
3419{
3420 const struct btf_param *member = btf_params(t);
3421 __u16 vlen = btf_vlen(t);
3422 long h = btf_hash_common(t);
3423 int i;
3424
3425 for (i = 0; i < vlen; i++) {
3426 h = hash_combine(h, member->name_off);
3427 h = hash_combine(h, member->type);
3428 member++;
3429 }
3430 return h;
3431}
3432
3433
3434
3435
3436
3437
3438
3439
3440static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2)
3441{
3442 const struct btf_param *m1, *m2;
3443 __u16 vlen;
3444 int i;
3445
3446 if (!btf_equal_common(t1, t2))
3447 return false;
3448
3449 vlen = btf_vlen(t1);
3450 m1 = btf_params(t1);
3451 m2 = btf_params(t2);
3452 for (i = 0; i < vlen; i++) {
3453 if (m1->name_off != m2->name_off || m1->type != m2->type)
3454 return false;
3455 m1++;
3456 m2++;
3457 }
3458 return true;
3459}
3460
3461
3462
3463
3464
3465
3466static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2)
3467{
3468 const struct btf_param *m1, *m2;
3469 __u16 vlen;
3470 int i;
3471
3472
3473 if (t1->name_off != t2->name_off || t1->info != t2->info)
3474 return false;
3475
3476 vlen = btf_vlen(t1);
3477 m1 = btf_params(t1);
3478 m2 = btf_params(t2);
3479 for (i = 0; i < vlen; i++) {
3480 if (m1->name_off != m2->name_off)
3481 return false;
3482 m1++;
3483 m2++;
3484 }
3485 return true;
3486}
3487
3488
3489
3490
3491
3492static int btf_dedup_prep(struct btf_dedup *d)
3493{
3494 struct btf_type *t;
3495 int type_id;
3496 long h;
3497
3498 if (!d->btf->base_btf)
3499 return 0;
3500
3501 for (type_id = 1; type_id < d->btf->start_id; type_id++) {
3502 t = btf_type_by_id(d->btf, type_id);
3503
3504
3505 d->map[type_id] = type_id;
3506
3507 switch (btf_kind(t)) {
3508 case BTF_KIND_VAR:
3509 case BTF_KIND_DATASEC:
3510
3511 continue;
3512 case BTF_KIND_CONST:
3513 case BTF_KIND_VOLATILE:
3514 case BTF_KIND_RESTRICT:
3515 case BTF_KIND_PTR:
3516 case BTF_KIND_FWD:
3517 case BTF_KIND_TYPEDEF:
3518 case BTF_KIND_FUNC:
3519 case BTF_KIND_FLOAT:
3520 h = btf_hash_common(t);
3521 break;
3522 case BTF_KIND_INT:
3523 h = btf_hash_int(t);
3524 break;
3525 case BTF_KIND_ENUM:
3526 h = btf_hash_enum(t);
3527 break;
3528 case BTF_KIND_STRUCT:
3529 case BTF_KIND_UNION:
3530 h = btf_hash_struct(t);
3531 break;
3532 case BTF_KIND_ARRAY:
3533 h = btf_hash_array(t);
3534 break;
3535 case BTF_KIND_FUNC_PROTO:
3536 h = btf_hash_fnproto(t);
3537 break;
3538 default:
3539 pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id);
3540 return -EINVAL;
3541 }
3542 if (btf_dedup_table_add(d, h, type_id))
3543 return -ENOMEM;
3544 }
3545
3546 return 0;
3547}
3548
3549
3550
3551
3552
3553
3554
3555static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id)
3556{
3557 struct btf_type *t = btf_type_by_id(d->btf, type_id);
3558 struct hashmap_entry *hash_entry;
3559 struct btf_type *cand;
3560
3561 __u32 new_id = type_id;
3562 __u32 cand_id;
3563 long h;
3564
3565 switch (btf_kind(t)) {
3566 case BTF_KIND_CONST:
3567 case BTF_KIND_VOLATILE:
3568 case BTF_KIND_RESTRICT:
3569 case BTF_KIND_PTR:
3570 case BTF_KIND_TYPEDEF:
3571 case BTF_KIND_ARRAY:
3572 case BTF_KIND_STRUCT:
3573 case BTF_KIND_UNION:
3574 case BTF_KIND_FUNC:
3575 case BTF_KIND_FUNC_PROTO:
3576 case BTF_KIND_VAR:
3577 case BTF_KIND_DATASEC:
3578 return 0;
3579
3580 case BTF_KIND_INT:
3581 h = btf_hash_int(t);
3582 for_each_dedup_cand(d, hash_entry, h) {
3583 cand_id = (__u32)(long)hash_entry->value;
3584 cand = btf_type_by_id(d->btf, cand_id);
3585 if (btf_equal_int(t, cand)) {
3586 new_id = cand_id;
3587 break;
3588 }
3589 }
3590 break;
3591
3592 case BTF_KIND_ENUM:
3593 h = btf_hash_enum(t);
3594 for_each_dedup_cand(d, hash_entry, h) {
3595 cand_id = (__u32)(long)hash_entry->value;
3596 cand = btf_type_by_id(d->btf, cand_id);
3597 if (btf_equal_enum(t, cand)) {
3598 new_id = cand_id;
3599 break;
3600 }
3601 if (d->opts.dont_resolve_fwds)
3602 continue;
3603 if (btf_compat_enum(t, cand)) {
3604 if (btf_is_enum_fwd(t)) {
3605
3606 new_id = cand_id;
3607 break;
3608 }
3609
3610 d->map[cand_id] = type_id;
3611 }
3612 }
3613 break;
3614
3615 case BTF_KIND_FWD:
3616 case BTF_KIND_FLOAT:
3617 h = btf_hash_common(t);
3618 for_each_dedup_cand(d, hash_entry, h) {
3619 cand_id = (__u32)(long)hash_entry->value;
3620 cand = btf_type_by_id(d->btf, cand_id);
3621 if (btf_equal_common(t, cand)) {
3622 new_id = cand_id;
3623 break;
3624 }
3625 }
3626 break;
3627
3628 default:
3629 return -EINVAL;
3630 }
3631
3632 d->map[type_id] = new_id;
3633 if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
3634 return -ENOMEM;
3635
3636 return 0;
3637}
3638
3639static int btf_dedup_prim_types(struct btf_dedup *d)
3640{
3641 int i, err;
3642
3643 for (i = 0; i < d->btf->nr_types; i++) {
3644 err = btf_dedup_prim_type(d, d->btf->start_id + i);
3645 if (err)
3646 return err;
3647 }
3648 return 0;
3649}
3650
3651
3652
3653
3654static inline bool is_type_mapped(struct btf_dedup *d, uint32_t type_id)
3655{
3656 return d->map[type_id] <= BTF_MAX_NR_TYPES;
3657}
3658
3659
3660
3661
3662
3663
3664static inline __u32 resolve_type_id(struct btf_dedup *d, __u32 type_id)
3665{
3666 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)
3667 type_id = d->map[type_id];
3668 return type_id;
3669}
3670
3671
3672
3673
3674
3675static uint32_t resolve_fwd_id(struct btf_dedup *d, uint32_t type_id)
3676{
3677 __u32 orig_type_id = type_id;
3678
3679 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id)))
3680 return type_id;
3681
3682 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)
3683 type_id = d->map[type_id];
3684
3685 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id)))
3686 return type_id;
3687
3688 return orig_type_id;
3689}
3690
3691
3692static inline __u16 btf_fwd_kind(struct btf_type *t)
3693{
3694 return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT;
3695}
3696
3697
3698static int btf_dedup_identical_arrays(struct btf_dedup *d, __u32 id1, __u32 id2)
3699{
3700 struct btf_type *t1, *t2;
3701
3702 t1 = btf_type_by_id(d->btf, id1);
3703 t2 = btf_type_by_id(d->btf, id2);
3704 if (!btf_is_array(t1) || !btf_is_array(t2))
3705 return 0;
3706
3707 return btf_equal_array(t1, t2);
3708}
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
3804 __u32 canon_id)
3805{
3806 struct btf_type *cand_type;
3807 struct btf_type *canon_type;
3808 __u32 hypot_type_id;
3809 __u16 cand_kind;
3810 __u16 canon_kind;
3811 int i, eq;
3812
3813
3814 if (resolve_type_id(d, cand_id) == resolve_type_id(d, canon_id))
3815 return 1;
3816
3817 canon_id = resolve_fwd_id(d, canon_id);
3818
3819 hypot_type_id = d->hypot_map[canon_id];
3820 if (hypot_type_id <= BTF_MAX_NR_TYPES) {
3821
3822
3823
3824
3825
3826
3827
3828
3829 return hypot_type_id == cand_id ||
3830 btf_dedup_identical_arrays(d, hypot_type_id, cand_id);
3831 }
3832
3833 if (btf_dedup_hypot_map_add(d, canon_id, cand_id))
3834 return -ENOMEM;
3835
3836 cand_type = btf_type_by_id(d->btf, cand_id);
3837 canon_type = btf_type_by_id(d->btf, canon_id);
3838 cand_kind = btf_kind(cand_type);
3839 canon_kind = btf_kind(canon_type);
3840
3841 if (cand_type->name_off != canon_type->name_off)
3842 return 0;
3843
3844
3845 if (!d->opts.dont_resolve_fwds
3846 && (cand_kind == BTF_KIND_FWD || canon_kind == BTF_KIND_FWD)
3847 && cand_kind != canon_kind) {
3848 __u16 real_kind;
3849 __u16 fwd_kind;
3850
3851 if (cand_kind == BTF_KIND_FWD) {
3852 real_kind = canon_kind;
3853 fwd_kind = btf_fwd_kind(cand_type);
3854 } else {
3855 real_kind = cand_kind;
3856 fwd_kind = btf_fwd_kind(canon_type);
3857
3858 if (fwd_kind == real_kind && canon_id < d->btf->start_id)
3859 d->hypot_adjust_canon = true;
3860 }
3861 return fwd_kind == real_kind;
3862 }
3863
3864 if (cand_kind != canon_kind)
3865 return 0;
3866
3867 switch (cand_kind) {
3868 case BTF_KIND_INT:
3869 return btf_equal_int(cand_type, canon_type);
3870
3871 case BTF_KIND_ENUM:
3872 if (d->opts.dont_resolve_fwds)
3873 return btf_equal_enum(cand_type, canon_type);
3874 else
3875 return btf_compat_enum(cand_type, canon_type);
3876
3877 case BTF_KIND_FWD:
3878 case BTF_KIND_FLOAT:
3879 return btf_equal_common(cand_type, canon_type);
3880
3881 case BTF_KIND_CONST:
3882 case BTF_KIND_VOLATILE:
3883 case BTF_KIND_RESTRICT:
3884 case BTF_KIND_PTR:
3885 case BTF_KIND_TYPEDEF:
3886 case BTF_KIND_FUNC:
3887 if (cand_type->info != canon_type->info)
3888 return 0;
3889 return btf_dedup_is_equiv(d, cand_type->type, canon_type->type);
3890
3891 case BTF_KIND_ARRAY: {
3892 const struct btf_array *cand_arr, *canon_arr;
3893
3894 if (!btf_compat_array(cand_type, canon_type))
3895 return 0;
3896 cand_arr = btf_array(cand_type);
3897 canon_arr = btf_array(canon_type);
3898 eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type);
3899 if (eq <= 0)
3900 return eq;
3901 return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type);
3902 }
3903
3904 case BTF_KIND_STRUCT:
3905 case BTF_KIND_UNION: {
3906 const struct btf_member *cand_m, *canon_m;
3907 __u16 vlen;
3908
3909 if (!btf_shallow_equal_struct(cand_type, canon_type))
3910 return 0;
3911 vlen = btf_vlen(cand_type);
3912 cand_m = btf_members(cand_type);
3913 canon_m = btf_members(canon_type);
3914 for (i = 0; i < vlen; i++) {
3915 eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type);
3916 if (eq <= 0)
3917 return eq;
3918 cand_m++;
3919 canon_m++;
3920 }
3921
3922 return 1;
3923 }
3924
3925 case BTF_KIND_FUNC_PROTO: {
3926 const struct btf_param *cand_p, *canon_p;
3927 __u16 vlen;
3928
3929 if (!btf_compat_fnproto(cand_type, canon_type))
3930 return 0;
3931 eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type);
3932 if (eq <= 0)
3933 return eq;
3934 vlen = btf_vlen(cand_type);
3935 cand_p = btf_params(cand_type);
3936 canon_p = btf_params(canon_type);
3937 for (i = 0; i < vlen; i++) {
3938 eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type);
3939 if (eq <= 0)
3940 return eq;
3941 cand_p++;
3942 canon_p++;
3943 }
3944 return 1;
3945 }
3946
3947 default:
3948 return -EINVAL;
3949 }
3950 return 0;
3951}
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979static void btf_dedup_merge_hypot_map(struct btf_dedup *d)
3980{
3981 __u32 canon_type_id, targ_type_id;
3982 __u16 t_kind, c_kind;
3983 __u32 t_id, c_id;
3984 int i;
3985
3986 for (i = 0; i < d->hypot_cnt; i++) {
3987 canon_type_id = d->hypot_list[i];
3988 targ_type_id = d->hypot_map[canon_type_id];
3989 t_id = resolve_type_id(d, targ_type_id);
3990 c_id = resolve_type_id(d, canon_type_id);
3991 t_kind = btf_kind(btf__type_by_id(d->btf, t_id));
3992 c_kind = btf_kind(btf__type_by_id(d->btf, c_id));
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013 if (t_kind != BTF_KIND_FWD && c_kind == BTF_KIND_FWD)
4014 d->map[c_id] = t_id;
4015
4016
4017
4018
4019
4020
4021 if (d->hypot_adjust_canon)
4022 continue;
4023
4024 if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD)
4025 d->map[t_id] = c_id;
4026
4027 if ((t_kind == BTF_KIND_STRUCT || t_kind == BTF_KIND_UNION) &&
4028 c_kind != BTF_KIND_FWD &&
4029 is_type_mapped(d, c_id) &&
4030 !is_type_mapped(d, t_id)) {
4031
4032
4033
4034
4035
4036
4037 d->map[t_id] = c_id;
4038 }
4039 }
4040}
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id)
4065{
4066 struct btf_type *cand_type, *t;
4067 struct hashmap_entry *hash_entry;
4068
4069 __u32 new_id = type_id;
4070 __u16 kind;
4071 long h;
4072
4073
4074 if (d->map[type_id] <= BTF_MAX_NR_TYPES)
4075 return 0;
4076
4077 t = btf_type_by_id(d->btf, type_id);
4078 kind = btf_kind(t);
4079
4080 if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION)
4081 return 0;
4082
4083 h = btf_hash_struct(t);
4084 for_each_dedup_cand(d, hash_entry, h) {
4085 __u32 cand_id = (__u32)(long)hash_entry->value;
4086 int eq;
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098 cand_type = btf_type_by_id(d->btf, cand_id);
4099 if (!btf_shallow_equal_struct(t, cand_type))
4100 continue;
4101
4102 btf_dedup_clear_hypot_map(d);
4103 eq = btf_dedup_is_equiv(d, type_id, cand_id);
4104 if (eq < 0)
4105 return eq;
4106 if (!eq)
4107 continue;
4108 btf_dedup_merge_hypot_map(d);
4109 if (d->hypot_adjust_canon)
4110 continue;
4111 new_id = cand_id;
4112 break;
4113 }
4114
4115 d->map[type_id] = new_id;
4116 if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
4117 return -ENOMEM;
4118
4119 return 0;
4120}
4121
4122static int btf_dedup_struct_types(struct btf_dedup *d)
4123{
4124 int i, err;
4125
4126 for (i = 0; i < d->btf->nr_types; i++) {
4127 err = btf_dedup_struct_type(d, d->btf->start_id + i);
4128 if (err)
4129 return err;
4130 }
4131 return 0;
4132}
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id)
4159{
4160 struct hashmap_entry *hash_entry;
4161 __u32 new_id = type_id, cand_id;
4162 struct btf_type *t, *cand;
4163
4164 int ref_type_id;
4165 long h;
4166
4167 if (d->map[type_id] == BTF_IN_PROGRESS_ID)
4168 return -ELOOP;
4169 if (d->map[type_id] <= BTF_MAX_NR_TYPES)
4170 return resolve_type_id(d, type_id);
4171
4172 t = btf_type_by_id(d->btf, type_id);
4173 d->map[type_id] = BTF_IN_PROGRESS_ID;
4174
4175 switch (btf_kind(t)) {
4176 case BTF_KIND_CONST:
4177 case BTF_KIND_VOLATILE:
4178 case BTF_KIND_RESTRICT:
4179 case BTF_KIND_PTR:
4180 case BTF_KIND_TYPEDEF:
4181 case BTF_KIND_FUNC:
4182 ref_type_id = btf_dedup_ref_type(d, t->type);
4183 if (ref_type_id < 0)
4184 return ref_type_id;
4185 t->type = ref_type_id;
4186
4187 h = btf_hash_common(t);
4188 for_each_dedup_cand(d, hash_entry, h) {
4189 cand_id = (__u32)(long)hash_entry->value;
4190 cand = btf_type_by_id(d->btf, cand_id);
4191 if (btf_equal_common(t, cand)) {
4192 new_id = cand_id;
4193 break;
4194 }
4195 }
4196 break;
4197
4198 case BTF_KIND_ARRAY: {
4199 struct btf_array *info = btf_array(t);
4200
4201 ref_type_id = btf_dedup_ref_type(d, info->type);
4202 if (ref_type_id < 0)
4203 return ref_type_id;
4204 info->type = ref_type_id;
4205
4206 ref_type_id = btf_dedup_ref_type(d, info->index_type);
4207 if (ref_type_id < 0)
4208 return ref_type_id;
4209 info->index_type = ref_type_id;
4210
4211 h = btf_hash_array(t);
4212 for_each_dedup_cand(d, hash_entry, h) {
4213 cand_id = (__u32)(long)hash_entry->value;
4214 cand = btf_type_by_id(d->btf, cand_id);
4215 if (btf_equal_array(t, cand)) {
4216 new_id = cand_id;
4217 break;
4218 }
4219 }
4220 break;
4221 }
4222
4223 case BTF_KIND_FUNC_PROTO: {
4224 struct btf_param *param;
4225 __u16 vlen;
4226 int i;
4227
4228 ref_type_id = btf_dedup_ref_type(d, t->type);
4229 if (ref_type_id < 0)
4230 return ref_type_id;
4231 t->type = ref_type_id;
4232
4233 vlen = btf_vlen(t);
4234 param = btf_params(t);
4235 for (i = 0; i < vlen; i++) {
4236 ref_type_id = btf_dedup_ref_type(d, param->type);
4237 if (ref_type_id < 0)
4238 return ref_type_id;
4239 param->type = ref_type_id;
4240 param++;
4241 }
4242
4243 h = btf_hash_fnproto(t);
4244 for_each_dedup_cand(d, hash_entry, h) {
4245 cand_id = (__u32)(long)hash_entry->value;
4246 cand = btf_type_by_id(d->btf, cand_id);
4247 if (btf_equal_fnproto(t, cand)) {
4248 new_id = cand_id;
4249 break;
4250 }
4251 }
4252 break;
4253 }
4254
4255 default:
4256 return -EINVAL;
4257 }
4258
4259 d->map[type_id] = new_id;
4260 if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
4261 return -ENOMEM;
4262
4263 return new_id;
4264}
4265
4266static int btf_dedup_ref_types(struct btf_dedup *d)
4267{
4268 int i, err;
4269
4270 for (i = 0; i < d->btf->nr_types; i++) {
4271 err = btf_dedup_ref_type(d, d->btf->start_id + i);
4272 if (err < 0)
4273 return err;
4274 }
4275
4276 hashmap__free(d->dedup_table);
4277 d->dedup_table = NULL;
4278 return 0;
4279}
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292static int btf_dedup_compact_types(struct btf_dedup *d)
4293{
4294 __u32 *new_offs;
4295 __u32 next_type_id = d->btf->start_id;
4296 const struct btf_type *t;
4297 void *p;
4298 int i, id, len;
4299
4300
4301 d->hypot_map[0] = 0;
4302
4303 for (id = 1; id < d->btf->start_id; id++)
4304 d->hypot_map[id] = id;
4305 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++)
4306 d->hypot_map[id] = BTF_UNPROCESSED_ID;
4307
4308 p = d->btf->types_data;
4309
4310 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) {
4311 if (d->map[id] != id)
4312 continue;
4313
4314 t = btf__type_by_id(d->btf, id);
4315 len = btf_type_size(t);
4316 if (len < 0)
4317 return len;
4318
4319 memmove(p, t, len);
4320 d->hypot_map[id] = next_type_id;
4321 d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data;
4322 p += len;
4323 next_type_id++;
4324 }
4325
4326
4327 d->btf->nr_types = next_type_id - d->btf->start_id;
4328 d->btf->type_offs_cap = d->btf->nr_types;
4329 d->btf->hdr->type_len = p - d->btf->types_data;
4330 new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap,
4331 sizeof(*new_offs));
4332 if (d->btf->type_offs_cap && !new_offs)
4333 return -ENOMEM;
4334 d->btf->type_offs = new_offs;
4335 d->btf->hdr->str_off = d->btf->hdr->type_len;
4336 d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len;
4337 return 0;
4338}
4339
4340
4341
4342
4343
4344
4345
4346static int btf_dedup_remap_type_id(__u32 *type_id, void *ctx)
4347{
4348 struct btf_dedup *d = ctx;
4349 __u32 resolved_type_id, new_type_id;
4350
4351 resolved_type_id = resolve_type_id(d, *type_id);
4352 new_type_id = d->hypot_map[resolved_type_id];
4353 if (new_type_id > BTF_MAX_NR_TYPES)
4354 return -EINVAL;
4355
4356 *type_id = new_type_id;
4357 return 0;
4358}
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370static int btf_dedup_remap_types(struct btf_dedup *d)
4371{
4372 int i, r;
4373
4374 for (i = 0; i < d->btf->nr_types; i++) {
4375 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i);
4376
4377 r = btf_type_visit_type_ids(t, btf_dedup_remap_type_id, d);
4378 if (r)
4379 return r;
4380 }
4381
4382 if (!d->btf_ext)
4383 return 0;
4384
4385 r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d);
4386 if (r)
4387 return r;
4388
4389 return 0;
4390}
4391
4392
4393
4394
4395
4396struct btf *libbpf_find_kernel_btf(void)
4397{
4398 struct {
4399 const char *path_fmt;
4400 bool raw_btf;
4401 } locations[] = {
4402
4403 { "/sys/kernel/btf/vmlinux", true },
4404
4405 { "/boot/vmlinux-%1$s" },
4406 { "/lib/modules/%1$s/vmlinux-%1$s" },
4407 { "/lib/modules/%1$s/build/vmlinux" },
4408 { "/usr/lib/modules/%1$s/kernel/vmlinux" },
4409 { "/usr/lib/debug/boot/vmlinux-%1$s" },
4410 { "/usr/lib/debug/boot/vmlinux-%1$s.debug" },
4411 { "/usr/lib/debug/lib/modules/%1$s/vmlinux" },
4412 };
4413 char path[PATH_MAX + 1];
4414 struct utsname buf;
4415 struct btf *btf;
4416 int i, err;
4417
4418 uname(&buf);
4419
4420 for (i = 0; i < ARRAY_SIZE(locations); i++) {
4421 snprintf(path, PATH_MAX, locations[i].path_fmt, buf.release);
4422
4423 if (access(path, R_OK))
4424 continue;
4425
4426 if (locations[i].raw_btf)
4427 btf = btf__parse_raw(path);
4428 else
4429 btf = btf__parse_elf(path, NULL);
4430 err = libbpf_get_error(btf);
4431 pr_debug("loading kernel BTF '%s': %d\n", path, err);
4432 if (err)
4433 continue;
4434
4435 return btf;
4436 }
4437
4438 pr_warn("failed to find valid kernel BTF\n");
4439 return libbpf_err_ptr(-ESRCH);
4440}
4441
4442int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ctx)
4443{
4444 int i, n, err;
4445
4446 switch (btf_kind(t)) {
4447 case BTF_KIND_INT:
4448 case BTF_KIND_FLOAT:
4449 case BTF_KIND_ENUM:
4450 return 0;
4451
4452 case BTF_KIND_FWD:
4453 case BTF_KIND_CONST:
4454 case BTF_KIND_VOLATILE:
4455 case BTF_KIND_RESTRICT:
4456 case BTF_KIND_PTR:
4457 case BTF_KIND_TYPEDEF:
4458 case BTF_KIND_FUNC:
4459 case BTF_KIND_VAR:
4460 return visit(&t->type, ctx);
4461
4462 case BTF_KIND_ARRAY: {
4463 struct btf_array *a = btf_array(t);
4464
4465 err = visit(&a->type, ctx);
4466 err = err ?: visit(&a->index_type, ctx);
4467 return err;
4468 }
4469
4470 case BTF_KIND_STRUCT:
4471 case BTF_KIND_UNION: {
4472 struct btf_member *m = btf_members(t);
4473
4474 for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
4475 err = visit(&m->type, ctx);
4476 if (err)
4477 return err;
4478 }
4479 return 0;
4480 }
4481
4482 case BTF_KIND_FUNC_PROTO: {
4483 struct btf_param *m = btf_params(t);
4484
4485 err = visit(&t->type, ctx);
4486 if (err)
4487 return err;
4488 for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
4489 err = visit(&m->type, ctx);
4490 if (err)
4491 return err;
4492 }
4493 return 0;
4494 }
4495
4496 case BTF_KIND_DATASEC: {
4497 struct btf_var_secinfo *m = btf_var_secinfos(t);
4498
4499 for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
4500 err = visit(&m->type, ctx);
4501 if (err)
4502 return err;
4503 }
4504 return 0;
4505 }
4506
4507 default:
4508 return -EINVAL;
4509 }
4510}
4511
4512int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ctx)
4513{
4514 int i, n, err;
4515
4516 err = visit(&t->name_off, ctx);
4517 if (err)
4518 return err;
4519
4520 switch (btf_kind(t)) {
4521 case BTF_KIND_STRUCT:
4522 case BTF_KIND_UNION: {
4523 struct btf_member *m = btf_members(t);
4524
4525 for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
4526 err = visit(&m->name_off, ctx);
4527 if (err)
4528 return err;
4529 }
4530 break;
4531 }
4532 case BTF_KIND_ENUM: {
4533 struct btf_enum *m = btf_enum(t);
4534
4535 for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
4536 err = visit(&m->name_off, ctx);
4537 if (err)
4538 return err;
4539 }
4540 break;
4541 }
4542 case BTF_KIND_FUNC_PROTO: {
4543 struct btf_param *m = btf_params(t);
4544
4545 for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
4546 err = visit(&m->name_off, ctx);
4547 if (err)
4548 return err;
4549 }
4550 break;
4551 }
4552 default:
4553 break;
4554 }
4555
4556 return 0;
4557}
4558
4559int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx)
4560{
4561 const struct btf_ext_info *seg;
4562 struct btf_ext_info_sec *sec;
4563 int i, err;
4564
4565 seg = &btf_ext->func_info;
4566 for_each_btf_ext_sec(seg, sec) {
4567 struct bpf_func_info_min *rec;
4568
4569 for_each_btf_ext_rec(seg, sec, i, rec) {
4570 err = visit(&rec->type_id, ctx);
4571 if (err < 0)
4572 return err;
4573 }
4574 }
4575
4576 seg = &btf_ext->core_relo_info;
4577 for_each_btf_ext_sec(seg, sec) {
4578 struct bpf_core_relo *rec;
4579
4580 for_each_btf_ext_rec(seg, sec, i, rec) {
4581 err = visit(&rec->type_id, ctx);
4582 if (err < 0)
4583 return err;
4584 }
4585 }
4586
4587 return 0;
4588}
4589
4590int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx)
4591{
4592 const struct btf_ext_info *seg;
4593 struct btf_ext_info_sec *sec;
4594 int i, err;
4595
4596 seg = &btf_ext->func_info;
4597 for_each_btf_ext_sec(seg, sec) {
4598 err = visit(&sec->sec_name_off, ctx);
4599 if (err)
4600 return err;
4601 }
4602
4603 seg = &btf_ext->line_info;
4604 for_each_btf_ext_sec(seg, sec) {
4605 struct bpf_line_info_min *rec;
4606
4607 err = visit(&sec->sec_name_off, ctx);
4608 if (err)
4609 return err;
4610
4611 for_each_btf_ext_rec(seg, sec, i, rec) {
4612 err = visit(&rec->file_name_off, ctx);
4613 if (err)
4614 return err;
4615 err = visit(&rec->line_off, ctx);
4616 if (err)
4617 return err;
4618 }
4619 }
4620
4621 seg = &btf_ext->core_relo_info;
4622 for_each_btf_ext_sec(seg, sec) {
4623 struct bpf_core_relo *rec;
4624
4625 err = visit(&sec->sec_name_off, ctx);
4626 if (err)
4627 return err;
4628
4629 for_each_btf_ext_rec(seg, sec, i, rec) {
4630 err = visit(&rec->access_str_off, ctx);
4631 if (err)
4632 return err;
4633 }
4634 }
4635
4636 return 0;
4637}
4638