1
2
3
4
5
6
7
8
9
10
11
12
13#ifndef _GNU_SOURCE
14#define _GNU_SOURCE
15#endif
16#include <stdlib.h>
17#include <stdio.h>
18#include <stdarg.h>
19#include <libgen.h>
20#include <inttypes.h>
21#include <limits.h>
22#include <string.h>
23#include <unistd.h>
24#include <endian.h>
25#include <fcntl.h>
26#include <errno.h>
27#include <ctype.h>
28#include <asm/unistd.h>
29#include <linux/err.h>
30#include <linux/kernel.h>
31#include <linux/bpf.h>
32#include <linux/btf.h>
33#include <linux/filter.h>
34#include <linux/list.h>
35#include <linux/limits.h>
36#include <linux/perf_event.h>
37#include <linux/ring_buffer.h>
38#include <linux/version.h>
39#include <sys/epoll.h>
40#include <sys/ioctl.h>
41#include <sys/mman.h>
42#include <sys/stat.h>
43#include <sys/types.h>
44#include <sys/vfs.h>
45#include <sys/utsname.h>
46#include <sys/resource.h>
47#include <tools/libc_compat.h>
48#include <libelf.h>
49#include <gelf.h>
50#include <zlib.h>
51
52#include "libbpf.h"
53#include "bpf.h"
54#include "btf.h"
55#include "str_error.h"
56#include "libbpf_internal.h"
57#include "hashmap.h"
58
59
60#pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64
61
62#ifndef EM_BPF
63#define EM_BPF 247
64#endif
65
66#ifndef BPF_FS_MAGIC
67#define BPF_FS_MAGIC 0xcafe4a11
68#endif
69
70
71
72
73#pragma GCC diagnostic ignored "-Wformat-nonliteral"
74
75#define __printf(a, b) __attribute__((format(printf, a, b)))
76
77static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
78static struct bpf_program *bpf_object__find_prog_by_idx(struct bpf_object *obj,
79 int idx);
80static const struct btf_type *
81skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
82
83static int __base_pr(enum libbpf_print_level level, const char *format,
84 va_list args)
85{
86 if (level == LIBBPF_DEBUG)
87 return 0;
88
89 return vfprintf(stderr, format, args);
90}
91
92static libbpf_print_fn_t __libbpf_pr = __base_pr;
93
94libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
95{
96 libbpf_print_fn_t old_print_fn = __libbpf_pr;
97
98 __libbpf_pr = fn;
99 return old_print_fn;
100}
101
102__printf(2, 3)
103void libbpf_print(enum libbpf_print_level level, const char *format, ...)
104{
105 va_list args;
106
107 if (!__libbpf_pr)
108 return;
109
110 va_start(args, format);
111 __libbpf_pr(level, format, args);
112 va_end(args);
113}
114
115static void pr_perm_msg(int err)
116{
117 struct rlimit limit;
118 char buf[100];
119
120 if (err != -EPERM || geteuid() != 0)
121 return;
122
123 err = getrlimit(RLIMIT_MEMLOCK, &limit);
124 if (err)
125 return;
126
127 if (limit.rlim_cur == RLIM_INFINITY)
128 return;
129
130 if (limit.rlim_cur < 1024)
131 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
132 else if (limit.rlim_cur < 1024*1024)
133 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
134 else
135 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
136
137 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
138 buf);
139}
140
141#define STRERR_BUFSIZE 128
142
143
144#ifndef zfree
145# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
146#endif
147
148#ifndef zclose
149# define zclose(fd) ({ \
150 int ___err = 0; \
151 if ((fd) >= 0) \
152 ___err = close((fd)); \
153 fd = -1; \
154 ___err; })
155#endif
156
157#ifdef HAVE_LIBELF_MMAP_SUPPORT
158# define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
159#else
160# define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
161#endif
162
163static inline __u64 ptr_to_u64(const void *ptr)
164{
165 return (__u64) (unsigned long) ptr;
166}
167
168struct bpf_capabilities {
169
170 __u32 name:1;
171
172 __u32 global_data:1;
173
174 __u32 btf_func:1;
175
176 __u32 btf_datasec:1;
177
178 __u32 array_mmap:1;
179
180 __u32 btf_func_global:1;
181
182 __u32 exp_attach_type:1;
183};
184
185enum reloc_type {
186 RELO_LD64,
187 RELO_CALL,
188 RELO_DATA,
189 RELO_EXTERN,
190};
191
192struct reloc_desc {
193 enum reloc_type type;
194 int insn_idx;
195 int map_idx;
196 int sym_off;
197};
198
199struct bpf_sec_def;
200
201typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec,
202 struct bpf_program *prog);
203
204struct bpf_sec_def {
205 const char *sec;
206 size_t len;
207 enum bpf_prog_type prog_type;
208 enum bpf_attach_type expected_attach_type;
209 bool is_exp_attach_type_optional;
210 bool is_attachable;
211 bool is_attach_btf;
212 attach_fn_t attach_fn;
213};
214
215
216
217
218
219struct bpf_program {
220
221 int idx;
222 char *name;
223 int prog_ifindex;
224 char *section_name;
225 const struct bpf_sec_def *sec_def;
226
227
228
229 char *pin_name;
230 struct bpf_insn *insns;
231 size_t insns_cnt, main_prog_cnt;
232 enum bpf_prog_type type;
233 bool load;
234
235 struct reloc_desc *reloc_desc;
236 int nr_reloc;
237 int log_level;
238
239 struct {
240 int nr;
241 int *fds;
242 } instances;
243 bpf_program_prep_t preprocessor;
244
245 struct bpf_object *obj;
246 void *priv;
247 bpf_program_clear_priv_t clear_priv;
248
249 enum bpf_attach_type expected_attach_type;
250 __u32 attach_btf_id;
251 __u32 attach_prog_fd;
252 void *func_info;
253 __u32 func_info_rec_size;
254 __u32 func_info_cnt;
255
256 struct bpf_capabilities *caps;
257
258 void *line_info;
259 __u32 line_info_rec_size;
260 __u32 line_info_cnt;
261 __u32 prog_flags;
262};
263
264struct bpf_struct_ops {
265 const char *tname;
266 const struct btf_type *type;
267 struct bpf_program **progs;
268 __u32 *kern_func_off;
269
270 void *data;
271
272
273
274
275
276
277
278
279
280
281 void *kern_vdata;
282 __u32 type_id;
283};
284
285#define DATA_SEC ".data"
286#define BSS_SEC ".bss"
287#define RODATA_SEC ".rodata"
288#define KCONFIG_SEC ".kconfig"
289#define KSYMS_SEC ".ksyms"
290#define STRUCT_OPS_SEC ".struct_ops"
291
292enum libbpf_map_type {
293 LIBBPF_MAP_UNSPEC,
294 LIBBPF_MAP_DATA,
295 LIBBPF_MAP_BSS,
296 LIBBPF_MAP_RODATA,
297 LIBBPF_MAP_KCONFIG,
298};
299
300static const char * const libbpf_type_to_btf_name[] = {
301 [LIBBPF_MAP_DATA] = DATA_SEC,
302 [LIBBPF_MAP_BSS] = BSS_SEC,
303 [LIBBPF_MAP_RODATA] = RODATA_SEC,
304 [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC,
305};
306
307struct bpf_map {
308 char *name;
309 int fd;
310 int sec_idx;
311 size_t sec_offset;
312 int map_ifindex;
313 int inner_map_fd;
314 struct bpf_map_def def;
315 __u32 numa_node;
316 __u32 btf_var_idx;
317 __u32 btf_key_type_id;
318 __u32 btf_value_type_id;
319 __u32 btf_vmlinux_value_type_id;
320 void *priv;
321 bpf_map_clear_priv_t clear_priv;
322 enum libbpf_map_type libbpf_type;
323 void *mmaped;
324 struct bpf_struct_ops *st_ops;
325 struct bpf_map *inner_map;
326 void **init_slots;
327 int init_slots_sz;
328 char *pin_path;
329 bool pinned;
330 bool reused;
331};
332
333enum extern_type {
334 EXT_UNKNOWN,
335 EXT_KCFG,
336 EXT_KSYM,
337};
338
339enum kcfg_type {
340 KCFG_UNKNOWN,
341 KCFG_CHAR,
342 KCFG_BOOL,
343 KCFG_INT,
344 KCFG_TRISTATE,
345 KCFG_CHAR_ARR,
346};
347
348struct extern_desc {
349 enum extern_type type;
350 int sym_idx;
351 int btf_id;
352 int sec_btf_id;
353 const char *name;
354 bool is_set;
355 bool is_weak;
356 union {
357 struct {
358 enum kcfg_type type;
359 int sz;
360 int align;
361 int data_off;
362 bool is_signed;
363 } kcfg;
364 struct {
365 unsigned long long addr;
366 } ksym;
367 };
368};
369
370static LIST_HEAD(bpf_objects_list);
371
372struct bpf_object {
373 char name[BPF_OBJ_NAME_LEN];
374 char license[64];
375 __u32 kern_version;
376
377 struct bpf_program *programs;
378 size_t nr_programs;
379 struct bpf_map *maps;
380 size_t nr_maps;
381 size_t maps_cap;
382
383 char *kconfig;
384 struct extern_desc *externs;
385 int nr_extern;
386 int kconfig_map_idx;
387
388 bool loaded;
389 bool has_pseudo_calls;
390
391
392
393
394
395 struct {
396 int fd;
397 const void *obj_buf;
398 size_t obj_buf_sz;
399 Elf *elf;
400 GElf_Ehdr ehdr;
401 Elf_Data *symbols;
402 Elf_Data *data;
403 Elf_Data *rodata;
404 Elf_Data *bss;
405 Elf_Data *st_ops_data;
406 size_t strtabidx;
407 struct {
408 GElf_Shdr shdr;
409 Elf_Data *data;
410 } *reloc_sects;
411 int nr_reloc_sects;
412 int maps_shndx;
413 int btf_maps_shndx;
414 __u32 btf_maps_sec_btf_id;
415 int text_shndx;
416 int symbols_shndx;
417 int data_shndx;
418 int rodata_shndx;
419 int bss_shndx;
420 int st_ops_shndx;
421 } efile;
422
423
424
425
426
427 struct list_head list;
428
429 struct btf *btf;
430
431
432
433 struct btf *btf_vmlinux;
434 struct btf_ext *btf_ext;
435
436 void *priv;
437 bpf_object_clear_priv_t clear_priv;
438
439 struct bpf_capabilities caps;
440
441 char path[];
442};
443#define obj_elf_valid(o) ((o)->efile.elf)
444
445void bpf_program__unload(struct bpf_program *prog)
446{
447 int i;
448
449 if (!prog)
450 return;
451
452
453
454
455
456 if (prog->instances.nr > 0) {
457 for (i = 0; i < prog->instances.nr; i++)
458 zclose(prog->instances.fds[i]);
459 } else if (prog->instances.nr != -1) {
460 pr_warn("Internal error: instances.nr is %d\n",
461 prog->instances.nr);
462 }
463
464 prog->instances.nr = -1;
465 zfree(&prog->instances.fds);
466
467 zfree(&prog->func_info);
468 zfree(&prog->line_info);
469}
470
471static void bpf_program__exit(struct bpf_program *prog)
472{
473 if (!prog)
474 return;
475
476 if (prog->clear_priv)
477 prog->clear_priv(prog, prog->priv);
478
479 prog->priv = NULL;
480 prog->clear_priv = NULL;
481
482 bpf_program__unload(prog);
483 zfree(&prog->name);
484 zfree(&prog->section_name);
485 zfree(&prog->pin_name);
486 zfree(&prog->insns);
487 zfree(&prog->reloc_desc);
488
489 prog->nr_reloc = 0;
490 prog->insns_cnt = 0;
491 prog->idx = -1;
492}
493
494static char *__bpf_program__pin_name(struct bpf_program *prog)
495{
496 char *name, *p;
497
498 name = p = strdup(prog->section_name);
499 while ((p = strchr(p, '/')))
500 *p = '_';
501
502 return name;
503}
504
505static int
506bpf_program__init(void *data, size_t size, char *section_name, int idx,
507 struct bpf_program *prog)
508{
509 const size_t bpf_insn_sz = sizeof(struct bpf_insn);
510
511 if (size == 0 || size % bpf_insn_sz) {
512 pr_warn("corrupted section '%s', size: %zu\n",
513 section_name, size);
514 return -EINVAL;
515 }
516
517 memset(prog, 0, sizeof(*prog));
518
519 prog->section_name = strdup(section_name);
520 if (!prog->section_name) {
521 pr_warn("failed to alloc name for prog under section(%d) %s\n",
522 idx, section_name);
523 goto errout;
524 }
525
526 prog->pin_name = __bpf_program__pin_name(prog);
527 if (!prog->pin_name) {
528 pr_warn("failed to alloc pin name for prog under section(%d) %s\n",
529 idx, section_name);
530 goto errout;
531 }
532
533 prog->insns = malloc(size);
534 if (!prog->insns) {
535 pr_warn("failed to alloc insns for prog under section %s\n",
536 section_name);
537 goto errout;
538 }
539 prog->insns_cnt = size / bpf_insn_sz;
540 memcpy(prog->insns, data, size);
541 prog->idx = idx;
542 prog->instances.fds = NULL;
543 prog->instances.nr = -1;
544 prog->type = BPF_PROG_TYPE_UNSPEC;
545 prog->load = true;
546
547 return 0;
548errout:
549 bpf_program__exit(prog);
550 return -ENOMEM;
551}
552
553static int
554bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
555 char *section_name, int idx)
556{
557 struct bpf_program prog, *progs;
558 int nr_progs, err;
559
560 err = bpf_program__init(data, size, section_name, idx, &prog);
561 if (err)
562 return err;
563
564 prog.caps = &obj->caps;
565 progs = obj->programs;
566 nr_progs = obj->nr_programs;
567
568 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0]));
569 if (!progs) {
570
571
572
573
574
575 pr_warn("failed to alloc a new program under section '%s'\n",
576 section_name);
577 bpf_program__exit(&prog);
578 return -ENOMEM;
579 }
580
581 pr_debug("found program %s\n", prog.section_name);
582 obj->programs = progs;
583 obj->nr_programs = nr_progs + 1;
584 prog.obj = obj;
585 progs[nr_progs] = prog;
586 return 0;
587}
588
589static int
590bpf_object__init_prog_names(struct bpf_object *obj)
591{
592 Elf_Data *symbols = obj->efile.symbols;
593 struct bpf_program *prog;
594 size_t pi, si;
595
596 for (pi = 0; pi < obj->nr_programs; pi++) {
597 const char *name = NULL;
598
599 prog = &obj->programs[pi];
600
601 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
602 si++) {
603 GElf_Sym sym;
604
605 if (!gelf_getsym(symbols, si, &sym))
606 continue;
607 if (sym.st_shndx != prog->idx)
608 continue;
609 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
610 continue;
611
612 name = elf_strptr(obj->efile.elf,
613 obj->efile.strtabidx,
614 sym.st_name);
615 if (!name) {
616 pr_warn("failed to get sym name string for prog %s\n",
617 prog->section_name);
618 return -LIBBPF_ERRNO__LIBELF;
619 }
620 }
621
622 if (!name && prog->idx == obj->efile.text_shndx)
623 name = ".text";
624
625 if (!name) {
626 pr_warn("failed to find sym for prog %s\n",
627 prog->section_name);
628 return -EINVAL;
629 }
630
631 prog->name = strdup(name);
632 if (!prog->name) {
633 pr_warn("failed to allocate memory for prog sym %s\n",
634 name);
635 return -ENOMEM;
636 }
637 }
638
639 return 0;
640}
641
642static __u32 get_kernel_version(void)
643{
644 __u32 major, minor, patch;
645 struct utsname info;
646
647 uname(&info);
648 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
649 return 0;
650 return KERNEL_VERSION(major, minor, patch);
651}
652
653static const struct btf_member *
654find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
655{
656 struct btf_member *m;
657 int i;
658
659 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
660 if (btf_member_bit_offset(t, i) == bit_offset)
661 return m;
662 }
663
664 return NULL;
665}
666
667static const struct btf_member *
668find_member_by_name(const struct btf *btf, const struct btf_type *t,
669 const char *name)
670{
671 struct btf_member *m;
672 int i;
673
674 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
675 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
676 return m;
677 }
678
679 return NULL;
680}
681
682#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
683static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
684 const char *name, __u32 kind);
685
686static int
687find_struct_ops_kern_types(const struct btf *btf, const char *tname,
688 const struct btf_type **type, __u32 *type_id,
689 const struct btf_type **vtype, __u32 *vtype_id,
690 const struct btf_member **data_member)
691{
692 const struct btf_type *kern_type, *kern_vtype;
693 const struct btf_member *kern_data_member;
694 __s32 kern_vtype_id, kern_type_id;
695 __u32 i;
696
697 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
698 if (kern_type_id < 0) {
699 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
700 tname);
701 return kern_type_id;
702 }
703 kern_type = btf__type_by_id(btf, kern_type_id);
704
705
706
707
708
709
710 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
711 tname, BTF_KIND_STRUCT);
712 if (kern_vtype_id < 0) {
713 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
714 STRUCT_OPS_VALUE_PREFIX, tname);
715 return kern_vtype_id;
716 }
717 kern_vtype = btf__type_by_id(btf, kern_vtype_id);
718
719
720
721
722
723
724
725 kern_data_member = btf_members(kern_vtype);
726 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
727 if (kern_data_member->type == kern_type_id)
728 break;
729 }
730 if (i == btf_vlen(kern_vtype)) {
731 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
732 tname, STRUCT_OPS_VALUE_PREFIX, tname);
733 return -EINVAL;
734 }
735
736 *type = kern_type;
737 *type_id = kern_type_id;
738 *vtype = kern_vtype;
739 *vtype_id = kern_vtype_id;
740 *data_member = kern_data_member;
741
742 return 0;
743}
744
745static bool bpf_map__is_struct_ops(const struct bpf_map *map)
746{
747 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
748}
749
750
751static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
752 const struct btf *btf,
753 const struct btf *kern_btf)
754{
755 const struct btf_member *member, *kern_member, *kern_data_member;
756 const struct btf_type *type, *kern_type, *kern_vtype;
757 __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
758 struct bpf_struct_ops *st_ops;
759 void *data, *kern_data;
760 const char *tname;
761 int err;
762
763 st_ops = map->st_ops;
764 type = st_ops->type;
765 tname = st_ops->tname;
766 err = find_struct_ops_kern_types(kern_btf, tname,
767 &kern_type, &kern_type_id,
768 &kern_vtype, &kern_vtype_id,
769 &kern_data_member);
770 if (err)
771 return err;
772
773 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
774 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
775
776 map->def.value_size = kern_vtype->size;
777 map->btf_vmlinux_value_type_id = kern_vtype_id;
778
779 st_ops->kern_vdata = calloc(1, kern_vtype->size);
780 if (!st_ops->kern_vdata)
781 return -ENOMEM;
782
783 data = st_ops->data;
784 kern_data_off = kern_data_member->offset / 8;
785 kern_data = st_ops->kern_vdata + kern_data_off;
786
787 member = btf_members(type);
788 for (i = 0; i < btf_vlen(type); i++, member++) {
789 const struct btf_type *mtype, *kern_mtype;
790 __u32 mtype_id, kern_mtype_id;
791 void *mdata, *kern_mdata;
792 __s64 msize, kern_msize;
793 __u32 moff, kern_moff;
794 __u32 kern_member_idx;
795 const char *mname;
796
797 mname = btf__name_by_offset(btf, member->name_off);
798 kern_member = find_member_by_name(kern_btf, kern_type, mname);
799 if (!kern_member) {
800 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
801 map->name, mname);
802 return -ENOTSUP;
803 }
804
805 kern_member_idx = kern_member - btf_members(kern_type);
806 if (btf_member_bitfield_size(type, i) ||
807 btf_member_bitfield_size(kern_type, kern_member_idx)) {
808 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
809 map->name, mname);
810 return -ENOTSUP;
811 }
812
813 moff = member->offset / 8;
814 kern_moff = kern_member->offset / 8;
815
816 mdata = data + moff;
817 kern_mdata = kern_data + kern_moff;
818
819 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
820 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
821 &kern_mtype_id);
822 if (BTF_INFO_KIND(mtype->info) !=
823 BTF_INFO_KIND(kern_mtype->info)) {
824 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
825 map->name, mname, BTF_INFO_KIND(mtype->info),
826 BTF_INFO_KIND(kern_mtype->info));
827 return -ENOTSUP;
828 }
829
830 if (btf_is_ptr(mtype)) {
831 struct bpf_program *prog;
832
833 mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id);
834 kern_mtype = skip_mods_and_typedefs(kern_btf,
835 kern_mtype->type,
836 &kern_mtype_id);
837 if (!btf_is_func_proto(mtype) ||
838 !btf_is_func_proto(kern_mtype)) {
839 pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n",
840 map->name, mname);
841 return -ENOTSUP;
842 }
843
844 prog = st_ops->progs[i];
845 if (!prog) {
846 pr_debug("struct_ops init_kern %s: func ptr %s is not set\n",
847 map->name, mname);
848 continue;
849 }
850
851 prog->attach_btf_id = kern_type_id;
852 prog->expected_attach_type = kern_member_idx;
853
854 st_ops->kern_func_off[i] = kern_data_off + kern_moff;
855
856 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
857 map->name, mname, prog->name, moff,
858 kern_moff);
859
860 continue;
861 }
862
863 msize = btf__resolve_size(btf, mtype_id);
864 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
865 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
866 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
867 map->name, mname, (ssize_t)msize,
868 (ssize_t)kern_msize);
869 return -ENOTSUP;
870 }
871
872 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
873 map->name, mname, (unsigned int)msize,
874 moff, kern_moff);
875 memcpy(kern_mdata, mdata, msize);
876 }
877
878 return 0;
879}
880
881static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
882{
883 struct bpf_map *map;
884 size_t i;
885 int err;
886
887 for (i = 0; i < obj->nr_maps; i++) {
888 map = &obj->maps[i];
889
890 if (!bpf_map__is_struct_ops(map))
891 continue;
892
893 err = bpf_map__init_kern_struct_ops(map, obj->btf,
894 obj->btf_vmlinux);
895 if (err)
896 return err;
897 }
898
899 return 0;
900}
901
902static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
903{
904 const struct btf_type *type, *datasec;
905 const struct btf_var_secinfo *vsi;
906 struct bpf_struct_ops *st_ops;
907 const char *tname, *var_name;
908 __s32 type_id, datasec_id;
909 const struct btf *btf;
910 struct bpf_map *map;
911 __u32 i;
912
913 if (obj->efile.st_ops_shndx == -1)
914 return 0;
915
916 btf = obj->btf;
917 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
918 BTF_KIND_DATASEC);
919 if (datasec_id < 0) {
920 pr_warn("struct_ops init: DATASEC %s not found\n",
921 STRUCT_OPS_SEC);
922 return -EINVAL;
923 }
924
925 datasec = btf__type_by_id(btf, datasec_id);
926 vsi = btf_var_secinfos(datasec);
927 for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
928 type = btf__type_by_id(obj->btf, vsi->type);
929 var_name = btf__name_by_offset(obj->btf, type->name_off);
930
931 type_id = btf__resolve_type(obj->btf, vsi->type);
932 if (type_id < 0) {
933 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
934 vsi->type, STRUCT_OPS_SEC);
935 return -EINVAL;
936 }
937
938 type = btf__type_by_id(obj->btf, type_id);
939 tname = btf__name_by_offset(obj->btf, type->name_off);
940 if (!tname[0]) {
941 pr_warn("struct_ops init: anonymous type is not supported\n");
942 return -ENOTSUP;
943 }
944 if (!btf_is_struct(type)) {
945 pr_warn("struct_ops init: %s is not a struct\n", tname);
946 return -EINVAL;
947 }
948
949 map = bpf_object__add_map(obj);
950 if (IS_ERR(map))
951 return PTR_ERR(map);
952
953 map->sec_idx = obj->efile.st_ops_shndx;
954 map->sec_offset = vsi->offset;
955 map->name = strdup(var_name);
956 if (!map->name)
957 return -ENOMEM;
958
959 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
960 map->def.key_size = sizeof(int);
961 map->def.value_size = type->size;
962 map->def.max_entries = 1;
963
964 map->st_ops = calloc(1, sizeof(*map->st_ops));
965 if (!map->st_ops)
966 return -ENOMEM;
967 st_ops = map->st_ops;
968 st_ops->data = malloc(type->size);
969 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
970 st_ops->kern_func_off = malloc(btf_vlen(type) *
971 sizeof(*st_ops->kern_func_off));
972 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
973 return -ENOMEM;
974
975 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
976 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
977 var_name, STRUCT_OPS_SEC);
978 return -EINVAL;
979 }
980
981 memcpy(st_ops->data,
982 obj->efile.st_ops_data->d_buf + vsi->offset,
983 type->size);
984 st_ops->tname = tname;
985 st_ops->type = type;
986 st_ops->type_id = type_id;
987
988 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
989 tname, type_id, var_name, vsi->offset);
990 }
991
992 return 0;
993}
994
995static struct bpf_object *bpf_object__new(const char *path,
996 const void *obj_buf,
997 size_t obj_buf_sz,
998 const char *obj_name)
999{
1000 struct bpf_object *obj;
1001 char *end;
1002
1003 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1004 if (!obj) {
1005 pr_warn("alloc memory failed for %s\n", path);
1006 return ERR_PTR(-ENOMEM);
1007 }
1008
1009 strcpy(obj->path, path);
1010 if (obj_name) {
1011 strncpy(obj->name, obj_name, sizeof(obj->name) - 1);
1012 obj->name[sizeof(obj->name) - 1] = 0;
1013 } else {
1014
1015 strncpy(obj->name, basename((void *)path),
1016 sizeof(obj->name) - 1);
1017 end = strchr(obj->name, '.');
1018 if (end)
1019 *end = 0;
1020 }
1021
1022 obj->efile.fd = -1;
1023
1024
1025
1026
1027
1028
1029 obj->efile.obj_buf = obj_buf;
1030 obj->efile.obj_buf_sz = obj_buf_sz;
1031 obj->efile.maps_shndx = -1;
1032 obj->efile.btf_maps_shndx = -1;
1033 obj->efile.data_shndx = -1;
1034 obj->efile.rodata_shndx = -1;
1035 obj->efile.bss_shndx = -1;
1036 obj->efile.st_ops_shndx = -1;
1037 obj->kconfig_map_idx = -1;
1038
1039 obj->kern_version = get_kernel_version();
1040 obj->loaded = false;
1041
1042 INIT_LIST_HEAD(&obj->list);
1043 list_add(&obj->list, &bpf_objects_list);
1044 return obj;
1045}
1046
1047static void bpf_object__elf_finish(struct bpf_object *obj)
1048{
1049 if (!obj_elf_valid(obj))
1050 return;
1051
1052 if (obj->efile.elf) {
1053 elf_end(obj->efile.elf);
1054 obj->efile.elf = NULL;
1055 }
1056 obj->efile.symbols = NULL;
1057 obj->efile.data = NULL;
1058 obj->efile.rodata = NULL;
1059 obj->efile.bss = NULL;
1060 obj->efile.st_ops_data = NULL;
1061
1062 zfree(&obj->efile.reloc_sects);
1063 obj->efile.nr_reloc_sects = 0;
1064 zclose(obj->efile.fd);
1065 obj->efile.obj_buf = NULL;
1066 obj->efile.obj_buf_sz = 0;
1067}
1068
1069static int bpf_object__elf_init(struct bpf_object *obj)
1070{
1071 int err = 0;
1072 GElf_Ehdr *ep;
1073
1074 if (obj_elf_valid(obj)) {
1075 pr_warn("elf init: internal error\n");
1076 return -LIBBPF_ERRNO__LIBELF;
1077 }
1078
1079 if (obj->efile.obj_buf_sz > 0) {
1080
1081
1082
1083
1084 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf,
1085 obj->efile.obj_buf_sz);
1086 } else {
1087 obj->efile.fd = open(obj->path, O_RDONLY);
1088 if (obj->efile.fd < 0) {
1089 char errmsg[STRERR_BUFSIZE], *cp;
1090
1091 err = -errno;
1092 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1093 pr_warn("failed to open %s: %s\n", obj->path, cp);
1094 return err;
1095 }
1096
1097 obj->efile.elf = elf_begin(obj->efile.fd,
1098 LIBBPF_ELF_C_READ_MMAP, NULL);
1099 }
1100
1101 if (!obj->efile.elf) {
1102 pr_warn("failed to open %s as ELF file\n", obj->path);
1103 err = -LIBBPF_ERRNO__LIBELF;
1104 goto errout;
1105 }
1106
1107 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
1108 pr_warn("failed to get EHDR from %s\n", obj->path);
1109 err = -LIBBPF_ERRNO__FORMAT;
1110 goto errout;
1111 }
1112 ep = &obj->efile.ehdr;
1113
1114
1115 if (ep->e_type != ET_REL ||
1116 (ep->e_machine && ep->e_machine != EM_BPF)) {
1117 pr_warn("%s is not an eBPF object file\n", obj->path);
1118 err = -LIBBPF_ERRNO__FORMAT;
1119 goto errout;
1120 }
1121
1122 return 0;
1123errout:
1124 bpf_object__elf_finish(obj);
1125 return err;
1126}
1127
1128static int bpf_object__check_endianness(struct bpf_object *obj)
1129{
1130#if __BYTE_ORDER == __LITTLE_ENDIAN
1131 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB)
1132 return 0;
1133#elif __BYTE_ORDER == __BIG_ENDIAN
1134 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB)
1135 return 0;
1136#else
1137# error "Unrecognized __BYTE_ORDER__"
1138#endif
1139 pr_warn("endianness mismatch.\n");
1140 return -LIBBPF_ERRNO__ENDIAN;
1141}
1142
1143static int
1144bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1145{
1146 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1));
1147 pr_debug("license of %s is %s\n", obj->path, obj->license);
1148 return 0;
1149}
1150
1151static int
1152bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1153{
1154 __u32 kver;
1155
1156 if (size != sizeof(kver)) {
1157 pr_warn("invalid kver section in %s\n", obj->path);
1158 return -LIBBPF_ERRNO__FORMAT;
1159 }
1160 memcpy(&kver, data, sizeof(kver));
1161 obj->kern_version = kver;
1162 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1163 return 0;
1164}
1165
1166static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1167{
1168 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1169 type == BPF_MAP_TYPE_HASH_OF_MAPS)
1170 return true;
1171 return false;
1172}
1173
1174static int bpf_object_search_section_size(const struct bpf_object *obj,
1175 const char *name, size_t *d_size)
1176{
1177 const GElf_Ehdr *ep = &obj->efile.ehdr;
1178 Elf *elf = obj->efile.elf;
1179 Elf_Scn *scn = NULL;
1180 int idx = 0;
1181
1182 while ((scn = elf_nextscn(elf, scn)) != NULL) {
1183 const char *sec_name;
1184 Elf_Data *data;
1185 GElf_Shdr sh;
1186
1187 idx++;
1188 if (gelf_getshdr(scn, &sh) != &sh) {
1189 pr_warn("failed to get section(%d) header from %s\n",
1190 idx, obj->path);
1191 return -EIO;
1192 }
1193
1194 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
1195 if (!sec_name) {
1196 pr_warn("failed to get section(%d) name from %s\n",
1197 idx, obj->path);
1198 return -EIO;
1199 }
1200
1201 if (strcmp(name, sec_name))
1202 continue;
1203
1204 data = elf_getdata(scn, 0);
1205 if (!data) {
1206 pr_warn("failed to get section(%d) data from %s(%s)\n",
1207 idx, name, obj->path);
1208 return -EIO;
1209 }
1210
1211 *d_size = data->d_size;
1212 return 0;
1213 }
1214
1215 return -ENOENT;
1216}
1217
1218int bpf_object__section_size(const struct bpf_object *obj, const char *name,
1219 __u32 *size)
1220{
1221 int ret = -ENOENT;
1222 size_t d_size;
1223
1224 *size = 0;
1225 if (!name) {
1226 return -EINVAL;
1227 } else if (!strcmp(name, DATA_SEC)) {
1228 if (obj->efile.data)
1229 *size = obj->efile.data->d_size;
1230 } else if (!strcmp(name, BSS_SEC)) {
1231 if (obj->efile.bss)
1232 *size = obj->efile.bss->d_size;
1233 } else if (!strcmp(name, RODATA_SEC)) {
1234 if (obj->efile.rodata)
1235 *size = obj->efile.rodata->d_size;
1236 } else if (!strcmp(name, STRUCT_OPS_SEC)) {
1237 if (obj->efile.st_ops_data)
1238 *size = obj->efile.st_ops_data->d_size;
1239 } else {
1240 ret = bpf_object_search_section_size(obj, name, &d_size);
1241 if (!ret)
1242 *size = d_size;
1243 }
1244
1245 return *size ? 0 : ret;
1246}
1247
1248int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
1249 __u32 *off)
1250{
1251 Elf_Data *symbols = obj->efile.symbols;
1252 const char *sname;
1253 size_t si;
1254
1255 if (!name || !off)
1256 return -EINVAL;
1257
1258 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) {
1259 GElf_Sym sym;
1260
1261 if (!gelf_getsym(symbols, si, &sym))
1262 continue;
1263 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
1264 GELF_ST_TYPE(sym.st_info) != STT_OBJECT)
1265 continue;
1266
1267 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
1268 sym.st_name);
1269 if (!sname) {
1270 pr_warn("failed to get sym name string for var %s\n",
1271 name);
1272 return -EIO;
1273 }
1274 if (strcmp(name, sname) == 0) {
1275 *off = sym.st_value;
1276 return 0;
1277 }
1278 }
1279
1280 return -ENOENT;
1281}
1282
1283static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1284{
1285 struct bpf_map *new_maps;
1286 size_t new_cap;
1287 int i;
1288
1289 if (obj->nr_maps < obj->maps_cap)
1290 return &obj->maps[obj->nr_maps++];
1291
1292 new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
1293 new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps));
1294 if (!new_maps) {
1295 pr_warn("alloc maps for object failed\n");
1296 return ERR_PTR(-ENOMEM);
1297 }
1298
1299 obj->maps_cap = new_cap;
1300 obj->maps = new_maps;
1301
1302
1303 memset(obj->maps + obj->nr_maps, 0,
1304 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps));
1305
1306
1307
1308
1309 for (i = obj->nr_maps; i < obj->maps_cap; i++) {
1310 obj->maps[i].fd = -1;
1311 obj->maps[i].inner_map_fd = -1;
1312 }
1313
1314 return &obj->maps[obj->nr_maps++];
1315}
1316
1317static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1318{
1319 long page_sz = sysconf(_SC_PAGE_SIZE);
1320 size_t map_sz;
1321
1322 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1323 map_sz = roundup(map_sz, page_sz);
1324 return map_sz;
1325}
1326
1327static char *internal_map_name(struct bpf_object *obj,
1328 enum libbpf_map_type type)
1329{
1330 char map_name[BPF_OBJ_NAME_LEN], *p;
1331 const char *sfx = libbpf_type_to_btf_name[type];
1332 int sfx_len = max((size_t)7, strlen(sfx));
1333 int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1,
1334 strlen(obj->name));
1335
1336 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1337 sfx_len, libbpf_type_to_btf_name[type]);
1338
1339
1340 for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1341 if (!isalnum(*p) && *p != '_' && *p != '.')
1342 *p = '_';
1343
1344 return strdup(map_name);
1345}
1346
1347static int
1348bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1349 int sec_idx, void *data, size_t data_sz)
1350{
1351 struct bpf_map_def *def;
1352 struct bpf_map *map;
1353 int err;
1354
1355 map = bpf_object__add_map(obj);
1356 if (IS_ERR(map))
1357 return PTR_ERR(map);
1358
1359 map->libbpf_type = type;
1360 map->sec_idx = sec_idx;
1361 map->sec_offset = 0;
1362 map->name = internal_map_name(obj, type);
1363 if (!map->name) {
1364 pr_warn("failed to alloc map name\n");
1365 return -ENOMEM;
1366 }
1367
1368 def = &map->def;
1369 def->type = BPF_MAP_TYPE_ARRAY;
1370 def->key_size = sizeof(int);
1371 def->value_size = data_sz;
1372 def->max_entries = 1;
1373 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1374 ? BPF_F_RDONLY_PROG : 0;
1375 def->map_flags |= BPF_F_MMAPABLE;
1376
1377 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1378 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1379
1380 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1381 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1382 if (map->mmaped == MAP_FAILED) {
1383 err = -errno;
1384 map->mmaped = NULL;
1385 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1386 map->name, err);
1387 zfree(&map->name);
1388 return err;
1389 }
1390
1391 if (data)
1392 memcpy(map->mmaped, data, data_sz);
1393
1394 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1395 return 0;
1396}
1397
1398static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1399{
1400 int err;
1401
1402
1403
1404
1405 if (obj->efile.data_shndx >= 0) {
1406 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1407 obj->efile.data_shndx,
1408 obj->efile.data->d_buf,
1409 obj->efile.data->d_size);
1410 if (err)
1411 return err;
1412 }
1413 if (obj->efile.rodata_shndx >= 0) {
1414 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1415 obj->efile.rodata_shndx,
1416 obj->efile.rodata->d_buf,
1417 obj->efile.rodata->d_size);
1418 if (err)
1419 return err;
1420 }
1421 if (obj->efile.bss_shndx >= 0) {
1422 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1423 obj->efile.bss_shndx,
1424 NULL,
1425 obj->efile.bss->d_size);
1426 if (err)
1427 return err;
1428 }
1429 return 0;
1430}
1431
1432
1433static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1434 const void *name)
1435{
1436 int i;
1437
1438 for (i = 0; i < obj->nr_extern; i++) {
1439 if (strcmp(obj->externs[i].name, name) == 0)
1440 return &obj->externs[i];
1441 }
1442 return NULL;
1443}
1444
1445static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
1446 char value)
1447{
1448 switch (ext->kcfg.type) {
1449 case KCFG_BOOL:
1450 if (value == 'm') {
1451 pr_warn("extern (kcfg) %s=%c should be tristate or char\n",
1452 ext->name, value);
1453 return -EINVAL;
1454 }
1455 *(bool *)ext_val = value == 'y' ? true : false;
1456 break;
1457 case KCFG_TRISTATE:
1458 if (value == 'y')
1459 *(enum libbpf_tristate *)ext_val = TRI_YES;
1460 else if (value == 'm')
1461 *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1462 else
1463 *(enum libbpf_tristate *)ext_val = TRI_NO;
1464 break;
1465 case KCFG_CHAR:
1466 *(char *)ext_val = value;
1467 break;
1468 case KCFG_UNKNOWN:
1469 case KCFG_INT:
1470 case KCFG_CHAR_ARR:
1471 default:
1472 pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n",
1473 ext->name, value);
1474 return -EINVAL;
1475 }
1476 ext->is_set = true;
1477 return 0;
1478}
1479
1480static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
1481 const char *value)
1482{
1483 size_t len;
1484
1485 if (ext->kcfg.type != KCFG_CHAR_ARR) {
1486 pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value);
1487 return -EINVAL;
1488 }
1489
1490 len = strlen(value);
1491 if (value[len - 1] != '"') {
1492 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
1493 ext->name, value);
1494 return -EINVAL;
1495 }
1496
1497
1498 len -= 2;
1499 if (len >= ext->kcfg.sz) {
1500 pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n",
1501 ext->name, value, len, ext->kcfg.sz - 1);
1502 len = ext->kcfg.sz - 1;
1503 }
1504 memcpy(ext_val, value + 1, len);
1505 ext_val[len] = '\0';
1506 ext->is_set = true;
1507 return 0;
1508}
1509
1510static int parse_u64(const char *value, __u64 *res)
1511{
1512 char *value_end;
1513 int err;
1514
1515 errno = 0;
1516 *res = strtoull(value, &value_end, 0);
1517 if (errno) {
1518 err = -errno;
1519 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1520 return err;
1521 }
1522 if (*value_end) {
1523 pr_warn("failed to parse '%s' as integer completely\n", value);
1524 return -EINVAL;
1525 }
1526 return 0;
1527}
1528
1529static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
1530{
1531 int bit_sz = ext->kcfg.sz * 8;
1532
1533 if (ext->kcfg.sz == 8)
1534 return true;
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548 if (ext->kcfg.is_signed)
1549 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1550 else
1551 return (v >> bit_sz) == 0;
1552}
1553
1554static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
1555 __u64 value)
1556{
1557 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
1558 pr_warn("extern (kcfg) %s=%llu should be integer\n",
1559 ext->name, (unsigned long long)value);
1560 return -EINVAL;
1561 }
1562 if (!is_kcfg_value_in_range(ext, value)) {
1563 pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n",
1564 ext->name, (unsigned long long)value, ext->kcfg.sz);
1565 return -ERANGE;
1566 }
1567 switch (ext->kcfg.sz) {
1568 case 1: *(__u8 *)ext_val = value; break;
1569 case 2: *(__u16 *)ext_val = value; break;
1570 case 4: *(__u32 *)ext_val = value; break;
1571 case 8: *(__u64 *)ext_val = value; break;
1572 default:
1573 return -EINVAL;
1574 }
1575 ext->is_set = true;
1576 return 0;
1577}
1578
1579static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1580 char *buf, void *data)
1581{
1582 struct extern_desc *ext;
1583 char *sep, *value;
1584 int len, err = 0;
1585 void *ext_val;
1586 __u64 num;
1587
1588 if (strncmp(buf, "CONFIG_", 7))
1589 return 0;
1590
1591 sep = strchr(buf, '=');
1592 if (!sep) {
1593 pr_warn("failed to parse '%s': no separator\n", buf);
1594 return -EINVAL;
1595 }
1596
1597
1598 len = strlen(buf);
1599 if (buf[len - 1] == '\n')
1600 buf[len - 1] = '\0';
1601
1602 *sep = '\0';
1603 if (!sep[1]) {
1604 *sep = '=';
1605 pr_warn("failed to parse '%s': no value\n", buf);
1606 return -EINVAL;
1607 }
1608
1609 ext = find_extern_by_name(obj, buf);
1610 if (!ext || ext->is_set)
1611 return 0;
1612
1613 ext_val = data + ext->kcfg.data_off;
1614 value = sep + 1;
1615
1616 switch (*value) {
1617 case 'y': case 'n': case 'm':
1618 err = set_kcfg_value_tri(ext, ext_val, *value);
1619 break;
1620 case '"':
1621 err = set_kcfg_value_str(ext, ext_val, value);
1622 break;
1623 default:
1624
1625 err = parse_u64(value, &num);
1626 if (err) {
1627 pr_warn("extern (kcfg) %s=%s should be integer\n",
1628 ext->name, value);
1629 return err;
1630 }
1631 err = set_kcfg_value_num(ext, ext_val, num);
1632 break;
1633 }
1634 if (err)
1635 return err;
1636 pr_debug("extern (kcfg) %s=%s\n", ext->name, value);
1637 return 0;
1638}
1639
1640static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1641{
1642 char buf[PATH_MAX];
1643 struct utsname uts;
1644 int len, err = 0;
1645 gzFile file;
1646
1647 uname(&uts);
1648 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1649 if (len < 0)
1650 return -EINVAL;
1651 else if (len >= PATH_MAX)
1652 return -ENAMETOOLONG;
1653
1654
1655 file = gzopen(buf, "r");
1656 if (!file)
1657 file = gzopen("/proc/config.gz", "r");
1658
1659 if (!file) {
1660 pr_warn("failed to open system Kconfig\n");
1661 return -ENOENT;
1662 }
1663
1664 while (gzgets(file, buf, sizeof(buf))) {
1665 err = bpf_object__process_kconfig_line(obj, buf, data);
1666 if (err) {
1667 pr_warn("error parsing system Kconfig line '%s': %d\n",
1668 buf, err);
1669 goto out;
1670 }
1671 }
1672
1673out:
1674 gzclose(file);
1675 return err;
1676}
1677
1678static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1679 const char *config, void *data)
1680{
1681 char buf[PATH_MAX];
1682 int err = 0;
1683 FILE *file;
1684
1685 file = fmemopen((void *)config, strlen(config), "r");
1686 if (!file) {
1687 err = -errno;
1688 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1689 return err;
1690 }
1691
1692 while (fgets(buf, sizeof(buf), file)) {
1693 err = bpf_object__process_kconfig_line(obj, buf, data);
1694 if (err) {
1695 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
1696 buf, err);
1697 break;
1698 }
1699 }
1700
1701 fclose(file);
1702 return err;
1703}
1704
1705static int bpf_object__init_kconfig_map(struct bpf_object *obj)
1706{
1707 struct extern_desc *last_ext = NULL, *ext;
1708 size_t map_sz;
1709 int i, err;
1710
1711 for (i = 0; i < obj->nr_extern; i++) {
1712 ext = &obj->externs[i];
1713 if (ext->type == EXT_KCFG)
1714 last_ext = ext;
1715 }
1716
1717 if (!last_ext)
1718 return 0;
1719
1720 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
1721 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
1722 obj->efile.symbols_shndx,
1723 NULL, map_sz);
1724 if (err)
1725 return err;
1726
1727 obj->kconfig_map_idx = obj->nr_maps - 1;
1728
1729 return 0;
1730}
1731
1732static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
1733{
1734 Elf_Data *symbols = obj->efile.symbols;
1735 int i, map_def_sz = 0, nr_maps = 0, nr_syms;
1736 Elf_Data *data = NULL;
1737 Elf_Scn *scn;
1738
1739 if (obj->efile.maps_shndx < 0)
1740 return 0;
1741
1742 if (!symbols)
1743 return -EINVAL;
1744
1745 scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
1746 if (scn)
1747 data = elf_getdata(scn, NULL);
1748 if (!scn || !data) {
1749 pr_warn("failed to get Elf_Data from map section %d\n",
1750 obj->efile.maps_shndx);
1751 return -EINVAL;
1752 }
1753
1754
1755
1756
1757
1758
1759
1760
1761 nr_syms = symbols->d_size / sizeof(GElf_Sym);
1762 for (i = 0; i < nr_syms; i++) {
1763 GElf_Sym sym;
1764
1765 if (!gelf_getsym(symbols, i, &sym))
1766 continue;
1767 if (sym.st_shndx != obj->efile.maps_shndx)
1768 continue;
1769 nr_maps++;
1770 }
1771
1772 pr_debug("maps in %s: %d maps in %zd bytes\n",
1773 obj->path, nr_maps, data->d_size);
1774
1775 if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {
1776 pr_warn("unable to determine map definition size section %s, %d maps in %zd bytes\n",
1777 obj->path, nr_maps, data->d_size);
1778 return -EINVAL;
1779 }
1780 map_def_sz = data->d_size / nr_maps;
1781
1782
1783 for (i = 0; i < nr_syms; i++) {
1784 GElf_Sym sym;
1785 const char *map_name;
1786 struct bpf_map_def *def;
1787 struct bpf_map *map;
1788
1789 if (!gelf_getsym(symbols, i, &sym))
1790 continue;
1791 if (sym.st_shndx != obj->efile.maps_shndx)
1792 continue;
1793
1794 map = bpf_object__add_map(obj);
1795 if (IS_ERR(map))
1796 return PTR_ERR(map);
1797
1798 map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
1799 sym.st_name);
1800 if (!map_name) {
1801 pr_warn("failed to get map #%d name sym string for obj %s\n",
1802 i, obj->path);
1803 return -LIBBPF_ERRNO__FORMAT;
1804 }
1805
1806 map->libbpf_type = LIBBPF_MAP_UNSPEC;
1807 map->sec_idx = sym.st_shndx;
1808 map->sec_offset = sym.st_value;
1809 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
1810 map_name, map->sec_idx, map->sec_offset);
1811 if (sym.st_value + map_def_sz > data->d_size) {
1812 pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
1813 obj->path, map_name);
1814 return -EINVAL;
1815 }
1816
1817 map->name = strdup(map_name);
1818 if (!map->name) {
1819 pr_warn("failed to alloc map name\n");
1820 return -ENOMEM;
1821 }
1822 pr_debug("map %d is \"%s\"\n", i, map->name);
1823 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
1824
1825
1826
1827
1828
1829
1830 if (map_def_sz <= sizeof(struct bpf_map_def)) {
1831 memcpy(&map->def, def, map_def_sz);
1832 } else {
1833
1834
1835
1836
1837
1838
1839 char *b;
1840
1841 for (b = ((char *)def) + sizeof(struct bpf_map_def);
1842 b < ((char *)def) + map_def_sz; b++) {
1843 if (*b != 0) {
1844 pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",
1845 obj->path, map_name);
1846 if (strict)
1847 return -EINVAL;
1848 }
1849 }
1850 memcpy(&map->def, def, sizeof(struct bpf_map_def));
1851 }
1852 }
1853 return 0;
1854}
1855
1856static const struct btf_type *
1857skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
1858{
1859 const struct btf_type *t = btf__type_by_id(btf, id);
1860
1861 if (res_id)
1862 *res_id = id;
1863
1864 while (btf_is_mod(t) || btf_is_typedef(t)) {
1865 if (res_id)
1866 *res_id = t->type;
1867 t = btf__type_by_id(btf, t->type);
1868 }
1869
1870 return t;
1871}
1872
1873static const struct btf_type *
1874resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
1875{
1876 const struct btf_type *t;
1877
1878 t = skip_mods_and_typedefs(btf, id, NULL);
1879 if (!btf_is_ptr(t))
1880 return NULL;
1881
1882 t = skip_mods_and_typedefs(btf, t->type, res_id);
1883
1884 return btf_is_func_proto(t) ? t : NULL;
1885}
1886
1887
1888
1889
1890
1891
1892
1893
1894static bool get_map_field_int(const char *map_name, const struct btf *btf,
1895 const struct btf_member *m, __u32 *res)
1896{
1897 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
1898 const char *name = btf__name_by_offset(btf, m->name_off);
1899 const struct btf_array *arr_info;
1900 const struct btf_type *arr_t;
1901
1902 if (!btf_is_ptr(t)) {
1903 pr_warn("map '%s': attr '%s': expected PTR, got %u.\n",
1904 map_name, name, btf_kind(t));
1905 return false;
1906 }
1907
1908 arr_t = btf__type_by_id(btf, t->type);
1909 if (!arr_t) {
1910 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
1911 map_name, name, t->type);
1912 return false;
1913 }
1914 if (!btf_is_array(arr_t)) {
1915 pr_warn("map '%s': attr '%s': expected ARRAY, got %u.\n",
1916 map_name, name, btf_kind(arr_t));
1917 return false;
1918 }
1919 arr_info = btf_array(arr_t);
1920 *res = arr_info->nelems;
1921 return true;
1922}
1923
1924static int build_map_pin_path(struct bpf_map *map, const char *path)
1925{
1926 char buf[PATH_MAX];
1927 int err, len;
1928
1929 if (!path)
1930 path = "/sys/fs/bpf";
1931
1932 len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
1933 if (len < 0)
1934 return -EINVAL;
1935 else if (len >= PATH_MAX)
1936 return -ENAMETOOLONG;
1937
1938 err = bpf_map__set_pin_path(map, buf);
1939 if (err)
1940 return err;
1941
1942 return 0;
1943}
1944
1945
1946static int parse_btf_map_def(struct bpf_object *obj,
1947 struct bpf_map *map,
1948 const struct btf_type *def,
1949 bool strict, bool is_inner,
1950 const char *pin_root_path)
1951{
1952 const struct btf_type *t;
1953 const struct btf_member *m;
1954 int vlen, i;
1955
1956 vlen = btf_vlen(def);
1957 m = btf_members(def);
1958 for (i = 0; i < vlen; i++, m++) {
1959 const char *name = btf__name_by_offset(obj->btf, m->name_off);
1960
1961 if (!name) {
1962 pr_warn("map '%s': invalid field #%d.\n", map->name, i);
1963 return -EINVAL;
1964 }
1965 if (strcmp(name, "type") == 0) {
1966 if (!get_map_field_int(map->name, obj->btf, m,
1967 &map->def.type))
1968 return -EINVAL;
1969 pr_debug("map '%s': found type = %u.\n",
1970 map->name, map->def.type);
1971 } else if (strcmp(name, "max_entries") == 0) {
1972 if (!get_map_field_int(map->name, obj->btf, m,
1973 &map->def.max_entries))
1974 return -EINVAL;
1975 pr_debug("map '%s': found max_entries = %u.\n",
1976 map->name, map->def.max_entries);
1977 } else if (strcmp(name, "map_flags") == 0) {
1978 if (!get_map_field_int(map->name, obj->btf, m,
1979 &map->def.map_flags))
1980 return -EINVAL;
1981 pr_debug("map '%s': found map_flags = %u.\n",
1982 map->name, map->def.map_flags);
1983 } else if (strcmp(name, "numa_node") == 0) {
1984 if (!get_map_field_int(map->name, obj->btf, m, &map->numa_node))
1985 return -EINVAL;
1986 pr_debug("map '%s': found numa_node = %u.\n", map->name, map->numa_node);
1987 } else if (strcmp(name, "key_size") == 0) {
1988 __u32 sz;
1989
1990 if (!get_map_field_int(map->name, obj->btf, m, &sz))
1991 return -EINVAL;
1992 pr_debug("map '%s': found key_size = %u.\n",
1993 map->name, sz);
1994 if (map->def.key_size && map->def.key_size != sz) {
1995 pr_warn("map '%s': conflicting key size %u != %u.\n",
1996 map->name, map->def.key_size, sz);
1997 return -EINVAL;
1998 }
1999 map->def.key_size = sz;
2000 } else if (strcmp(name, "key") == 0) {
2001 __s64 sz;
2002
2003 t = btf__type_by_id(obj->btf, m->type);
2004 if (!t) {
2005 pr_warn("map '%s': key type [%d] not found.\n",
2006 map->name, m->type);
2007 return -EINVAL;
2008 }
2009 if (!btf_is_ptr(t)) {
2010 pr_warn("map '%s': key spec is not PTR: %u.\n",
2011 map->name, btf_kind(t));
2012 return -EINVAL;
2013 }
2014 sz = btf__resolve_size(obj->btf, t->type);
2015 if (sz < 0) {
2016 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2017 map->name, t->type, (ssize_t)sz);
2018 return sz;
2019 }
2020 pr_debug("map '%s': found key [%u], sz = %zd.\n",
2021 map->name, t->type, (ssize_t)sz);
2022 if (map->def.key_size && map->def.key_size != sz) {
2023 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2024 map->name, map->def.key_size, (ssize_t)sz);
2025 return -EINVAL;
2026 }
2027 map->def.key_size = sz;
2028 map->btf_key_type_id = t->type;
2029 } else if (strcmp(name, "value_size") == 0) {
2030 __u32 sz;
2031
2032 if (!get_map_field_int(map->name, obj->btf, m, &sz))
2033 return -EINVAL;
2034 pr_debug("map '%s': found value_size = %u.\n",
2035 map->name, sz);
2036 if (map->def.value_size && map->def.value_size != sz) {
2037 pr_warn("map '%s': conflicting value size %u != %u.\n",
2038 map->name, map->def.value_size, sz);
2039 return -EINVAL;
2040 }
2041 map->def.value_size = sz;
2042 } else if (strcmp(name, "value") == 0) {
2043 __s64 sz;
2044
2045 t = btf__type_by_id(obj->btf, m->type);
2046 if (!t) {
2047 pr_warn("map '%s': value type [%d] not found.\n",
2048 map->name, m->type);
2049 return -EINVAL;
2050 }
2051 if (!btf_is_ptr(t)) {
2052 pr_warn("map '%s': value spec is not PTR: %u.\n",
2053 map->name, btf_kind(t));
2054 return -EINVAL;
2055 }
2056 sz = btf__resolve_size(obj->btf, t->type);
2057 if (sz < 0) {
2058 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2059 map->name, t->type, (ssize_t)sz);
2060 return sz;
2061 }
2062 pr_debug("map '%s': found value [%u], sz = %zd.\n",
2063 map->name, t->type, (ssize_t)sz);
2064 if (map->def.value_size && map->def.value_size != sz) {
2065 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2066 map->name, map->def.value_size, (ssize_t)sz);
2067 return -EINVAL;
2068 }
2069 map->def.value_size = sz;
2070 map->btf_value_type_id = t->type;
2071 }
2072 else if (strcmp(name, "values") == 0) {
2073 int err;
2074
2075 if (is_inner) {
2076 pr_warn("map '%s': multi-level inner maps not supported.\n",
2077 map->name);
2078 return -ENOTSUP;
2079 }
2080 if (i != vlen - 1) {
2081 pr_warn("map '%s': '%s' member should be last.\n",
2082 map->name, name);
2083 return -EINVAL;
2084 }
2085 if (!bpf_map_type__is_map_in_map(map->def.type)) {
2086 pr_warn("map '%s': should be map-in-map.\n",
2087 map->name);
2088 return -ENOTSUP;
2089 }
2090 if (map->def.value_size && map->def.value_size != 4) {
2091 pr_warn("map '%s': conflicting value size %u != 4.\n",
2092 map->name, map->def.value_size);
2093 return -EINVAL;
2094 }
2095 map->def.value_size = 4;
2096 t = btf__type_by_id(obj->btf, m->type);
2097 if (!t) {
2098 pr_warn("map '%s': map-in-map inner type [%d] not found.\n",
2099 map->name, m->type);
2100 return -EINVAL;
2101 }
2102 if (!btf_is_array(t) || btf_array(t)->nelems) {
2103 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n",
2104 map->name);
2105 return -EINVAL;
2106 }
2107 t = skip_mods_and_typedefs(obj->btf, btf_array(t)->type,
2108 NULL);
2109 if (!btf_is_ptr(t)) {
2110 pr_warn("map '%s': map-in-map inner def is of unexpected kind %u.\n",
2111 map->name, btf_kind(t));
2112 return -EINVAL;
2113 }
2114 t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
2115 if (!btf_is_struct(t)) {
2116 pr_warn("map '%s': map-in-map inner def is of unexpected kind %u.\n",
2117 map->name, btf_kind(t));
2118 return -EINVAL;
2119 }
2120
2121 map->inner_map = calloc(1, sizeof(*map->inner_map));
2122 if (!map->inner_map)
2123 return -ENOMEM;
2124 map->inner_map->sec_idx = obj->efile.btf_maps_shndx;
2125 map->inner_map->name = malloc(strlen(map->name) +
2126 sizeof(".inner") + 1);
2127 if (!map->inner_map->name)
2128 return -ENOMEM;
2129 sprintf(map->inner_map->name, "%s.inner", map->name);
2130
2131 err = parse_btf_map_def(obj, map->inner_map, t, strict,
2132 true , NULL);
2133 if (err)
2134 return err;
2135 } else if (strcmp(name, "pinning") == 0) {
2136 __u32 val;
2137 int err;
2138
2139 if (is_inner) {
2140 pr_debug("map '%s': inner def can't be pinned.\n",
2141 map->name);
2142 return -EINVAL;
2143 }
2144 if (!get_map_field_int(map->name, obj->btf, m, &val))
2145 return -EINVAL;
2146 pr_debug("map '%s': found pinning = %u.\n",
2147 map->name, val);
2148
2149 if (val != LIBBPF_PIN_NONE &&
2150 val != LIBBPF_PIN_BY_NAME) {
2151 pr_warn("map '%s': invalid pinning value %u.\n",
2152 map->name, val);
2153 return -EINVAL;
2154 }
2155 if (val == LIBBPF_PIN_BY_NAME) {
2156 err = build_map_pin_path(map, pin_root_path);
2157 if (err) {
2158 pr_warn("map '%s': couldn't build pin path.\n",
2159 map->name);
2160 return err;
2161 }
2162 }
2163 } else {
2164 if (strict) {
2165 pr_warn("map '%s': unknown field '%s'.\n",
2166 map->name, name);
2167 return -ENOTSUP;
2168 }
2169 pr_debug("map '%s': ignoring unknown field '%s'.\n",
2170 map->name, name);
2171 }
2172 }
2173
2174 if (map->def.type == BPF_MAP_TYPE_UNSPEC) {
2175 pr_warn("map '%s': map type isn't specified.\n", map->name);
2176 return -EINVAL;
2177 }
2178
2179 return 0;
2180}
2181
2182static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2183 const struct btf_type *sec,
2184 int var_idx, int sec_idx,
2185 const Elf_Data *data, bool strict,
2186 const char *pin_root_path)
2187{
2188 const struct btf_type *var, *def;
2189 const struct btf_var_secinfo *vi;
2190 const struct btf_var *var_extra;
2191 const char *map_name;
2192 struct bpf_map *map;
2193
2194 vi = btf_var_secinfos(sec) + var_idx;
2195 var = btf__type_by_id(obj->btf, vi->type);
2196 var_extra = btf_var(var);
2197 map_name = btf__name_by_offset(obj->btf, var->name_off);
2198
2199 if (map_name == NULL || map_name[0] == '\0') {
2200 pr_warn("map #%d: empty name.\n", var_idx);
2201 return -EINVAL;
2202 }
2203 if ((__u64)vi->offset + vi->size > data->d_size) {
2204 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2205 return -EINVAL;
2206 }
2207 if (!btf_is_var(var)) {
2208 pr_warn("map '%s': unexpected var kind %u.\n",
2209 map_name, btf_kind(var));
2210 return -EINVAL;
2211 }
2212 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&
2213 var_extra->linkage != BTF_VAR_STATIC) {
2214 pr_warn("map '%s': unsupported var linkage %u.\n",
2215 map_name, var_extra->linkage);
2216 return -EOPNOTSUPP;
2217 }
2218
2219 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2220 if (!btf_is_struct(def)) {
2221 pr_warn("map '%s': unexpected def kind %u.\n",
2222 map_name, btf_kind(var));
2223 return -EINVAL;
2224 }
2225 if (def->size > vi->size) {
2226 pr_warn("map '%s': invalid def size.\n", map_name);
2227 return -EINVAL;
2228 }
2229
2230 map = bpf_object__add_map(obj);
2231 if (IS_ERR(map))
2232 return PTR_ERR(map);
2233 map->name = strdup(map_name);
2234 if (!map->name) {
2235 pr_warn("map '%s': failed to alloc map name.\n", map_name);
2236 return -ENOMEM;
2237 }
2238 map->libbpf_type = LIBBPF_MAP_UNSPEC;
2239 map->def.type = BPF_MAP_TYPE_UNSPEC;
2240 map->sec_idx = sec_idx;
2241 map->sec_offset = vi->offset;
2242 map->btf_var_idx = var_idx;
2243 pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2244 map_name, map->sec_idx, map->sec_offset);
2245
2246 return parse_btf_map_def(obj, map, def, strict, false, pin_root_path);
2247}
2248
2249static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2250 const char *pin_root_path)
2251{
2252 const struct btf_type *sec = NULL;
2253 int nr_types, i, vlen, err;
2254 const struct btf_type *t;
2255 const char *name;
2256 Elf_Data *data;
2257 Elf_Scn *scn;
2258
2259 if (obj->efile.btf_maps_shndx < 0)
2260 return 0;
2261
2262 scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx);
2263 if (scn)
2264 data = elf_getdata(scn, NULL);
2265 if (!scn || !data) {
2266 pr_warn("failed to get Elf_Data from map section %d (%s)\n",
2267 obj->efile.btf_maps_shndx, MAPS_ELF_SEC);
2268 return -EINVAL;
2269 }
2270
2271 nr_types = btf__get_nr_types(obj->btf);
2272 for (i = 1; i <= nr_types; i++) {
2273 t = btf__type_by_id(obj->btf, i);
2274 if (!btf_is_datasec(t))
2275 continue;
2276 name = btf__name_by_offset(obj->btf, t->name_off);
2277 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2278 sec = t;
2279 obj->efile.btf_maps_sec_btf_id = i;
2280 break;
2281 }
2282 }
2283
2284 if (!sec) {
2285 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2286 return -ENOENT;
2287 }
2288
2289 vlen = btf_vlen(sec);
2290 for (i = 0; i < vlen; i++) {
2291 err = bpf_object__init_user_btf_map(obj, sec, i,
2292 obj->efile.btf_maps_shndx,
2293 data, strict,
2294 pin_root_path);
2295 if (err)
2296 return err;
2297 }
2298
2299 return 0;
2300}
2301
2302static int bpf_object__init_maps(struct bpf_object *obj,
2303 const struct bpf_object_open_opts *opts)
2304{
2305 const char *pin_root_path;
2306 bool strict;
2307 int err;
2308
2309 strict = !OPTS_GET(opts, relaxed_maps, false);
2310 pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2311
2312 err = bpf_object__init_user_maps(obj, strict);
2313 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2314 err = err ?: bpf_object__init_global_data_maps(obj);
2315 err = err ?: bpf_object__init_kconfig_map(obj);
2316 err = err ?: bpf_object__init_struct_ops_maps(obj);
2317 if (err)
2318 return err;
2319
2320 return 0;
2321}
2322
2323static bool section_have_execinstr(struct bpf_object *obj, int idx)
2324{
2325 Elf_Scn *scn;
2326 GElf_Shdr sh;
2327
2328 scn = elf_getscn(obj->efile.elf, idx);
2329 if (!scn)
2330 return false;
2331
2332 if (gelf_getshdr(scn, &sh) != &sh)
2333 return false;
2334
2335 if (sh.sh_flags & SHF_EXECINSTR)
2336 return true;
2337
2338 return false;
2339}
2340
2341static bool btf_needs_sanitization(struct bpf_object *obj)
2342{
2343 bool has_func_global = obj->caps.btf_func_global;
2344 bool has_datasec = obj->caps.btf_datasec;
2345 bool has_func = obj->caps.btf_func;
2346
2347 return !has_func || !has_datasec || !has_func_global;
2348}
2349
2350static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2351{
2352 bool has_func_global = obj->caps.btf_func_global;
2353 bool has_datasec = obj->caps.btf_datasec;
2354 bool has_func = obj->caps.btf_func;
2355 struct btf_type *t;
2356 int i, j, vlen;
2357
2358 for (i = 1; i <= btf__get_nr_types(btf); i++) {
2359 t = (struct btf_type *)btf__type_by_id(btf, i);
2360
2361 if (!has_datasec && btf_is_var(t)) {
2362
2363 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2364
2365
2366
2367
2368
2369 t->size = 1;
2370 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2371 } else if (!has_datasec && btf_is_datasec(t)) {
2372
2373 const struct btf_var_secinfo *v = btf_var_secinfos(t);
2374 struct btf_member *m = btf_members(t);
2375 struct btf_type *vt;
2376 char *name;
2377
2378 name = (char *)btf__name_by_offset(btf, t->name_off);
2379 while (*name) {
2380 if (*name == '.')
2381 *name = '_';
2382 name++;
2383 }
2384
2385 vlen = btf_vlen(t);
2386 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2387 for (j = 0; j < vlen; j++, v++, m++) {
2388
2389 m->offset = v->offset * 8;
2390 m->type = v->type;
2391
2392 vt = (void *)btf__type_by_id(btf, v->type);
2393 m->name_off = vt->name_off;
2394 }
2395 } else if (!has_func && btf_is_func_proto(t)) {
2396
2397 vlen = btf_vlen(t);
2398 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2399 t->size = sizeof(__u32);
2400 } else if (!has_func && btf_is_func(t)) {
2401
2402 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2403 } else if (!has_func_global && btf_is_func(t)) {
2404
2405 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2406 }
2407 }
2408}
2409
2410static bool libbpf_needs_btf(const struct bpf_object *obj)
2411{
2412 return obj->efile.btf_maps_shndx >= 0 ||
2413 obj->efile.st_ops_shndx >= 0 ||
2414 obj->nr_extern > 0;
2415}
2416
2417static bool kernel_needs_btf(const struct bpf_object *obj)
2418{
2419 return obj->efile.st_ops_shndx >= 0;
2420}
2421
2422static int bpf_object__init_btf(struct bpf_object *obj,
2423 Elf_Data *btf_data,
2424 Elf_Data *btf_ext_data)
2425{
2426 int err = -ENOENT;
2427
2428 if (btf_data) {
2429 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2430 if (IS_ERR(obj->btf)) {
2431 err = PTR_ERR(obj->btf);
2432 obj->btf = NULL;
2433 pr_warn("Error loading ELF section %s: %d.\n",
2434 BTF_ELF_SEC, err);
2435 goto out;
2436 }
2437
2438 btf__set_pointer_size(obj->btf, 8);
2439 err = 0;
2440 }
2441 if (btf_ext_data) {
2442 if (!obj->btf) {
2443 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2444 BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2445 goto out;
2446 }
2447 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
2448 btf_ext_data->d_size);
2449 if (IS_ERR(obj->btf_ext)) {
2450 pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n",
2451 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));
2452 obj->btf_ext = NULL;
2453 goto out;
2454 }
2455 }
2456out:
2457 if (err && libbpf_needs_btf(obj)) {
2458 pr_warn("BTF is required, but is missing or corrupted.\n");
2459 return err;
2460 }
2461 return 0;
2462}
2463
2464static int bpf_object__finalize_btf(struct bpf_object *obj)
2465{
2466 int err;
2467
2468 if (!obj->btf)
2469 return 0;
2470
2471 err = btf__finalize_data(obj, obj->btf);
2472 if (err) {
2473 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
2474 return err;
2475 }
2476
2477 return 0;
2478}
2479
2480static inline bool libbpf_prog_needs_vmlinux_btf(struct bpf_program *prog)
2481{
2482 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
2483 prog->type == BPF_PROG_TYPE_LSM)
2484 return true;
2485
2486
2487
2488
2489 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
2490 return true;
2491
2492 return false;
2493}
2494
2495static int bpf_object__load_vmlinux_btf(struct bpf_object *obj)
2496{
2497 bool need_vmlinux_btf = false;
2498 struct bpf_program *prog;
2499 int err;
2500
2501
2502 if (obj->btf_ext && obj->btf_ext->field_reloc_info.len)
2503 need_vmlinux_btf = true;
2504
2505 bpf_object__for_each_program(prog, obj) {
2506 if (!prog->load)
2507 continue;
2508 if (libbpf_prog_needs_vmlinux_btf(prog)) {
2509 need_vmlinux_btf = true;
2510 break;
2511 }
2512 }
2513
2514 if (!need_vmlinux_btf)
2515 return 0;
2516
2517 obj->btf_vmlinux = libbpf_find_kernel_btf();
2518 if (IS_ERR(obj->btf_vmlinux)) {
2519 err = PTR_ERR(obj->btf_vmlinux);
2520 pr_warn("Error loading vmlinux BTF: %d\n", err);
2521 obj->btf_vmlinux = NULL;
2522 return err;
2523 }
2524 return 0;
2525}
2526
2527static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
2528{
2529 struct btf *kern_btf = obj->btf;
2530 bool btf_mandatory, sanitize;
2531 int err = 0;
2532
2533 if (!obj->btf)
2534 return 0;
2535
2536 sanitize = btf_needs_sanitization(obj);
2537 if (sanitize) {
2538 const void *raw_data;
2539 __u32 sz;
2540
2541
2542 raw_data = btf__get_raw_data(obj->btf, &sz);
2543 kern_btf = btf__new(raw_data, sz);
2544 if (IS_ERR(kern_btf))
2545 return PTR_ERR(kern_btf);
2546
2547
2548 btf__set_pointer_size(obj->btf, 8);
2549 bpf_object__sanitize_btf(obj, kern_btf);
2550 }
2551
2552 err = btf__load(kern_btf);
2553 if (sanitize) {
2554 if (!err) {
2555
2556 btf__set_fd(obj->btf, btf__fd(kern_btf));
2557 btf__set_fd(kern_btf, -1);
2558 }
2559 btf__free(kern_btf);
2560 }
2561 if (err) {
2562 btf_mandatory = kernel_needs_btf(obj);
2563 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
2564 btf_mandatory ? "BTF is mandatory, can't proceed."
2565 : "BTF is optional, ignoring.");
2566 if (!btf_mandatory)
2567 err = 0;
2568 }
2569 return err;
2570}
2571
2572static int bpf_object__elf_collect(struct bpf_object *obj)
2573{
2574 Elf *elf = obj->efile.elf;
2575 GElf_Ehdr *ep = &obj->efile.ehdr;
2576 Elf_Data *btf_ext_data = NULL;
2577 Elf_Data *btf_data = NULL;
2578 Elf_Scn *scn = NULL;
2579 int idx = 0, err = 0;
2580
2581
2582 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
2583 pr_warn("failed to get e_shstrndx from %s\n", obj->path);
2584 return -LIBBPF_ERRNO__FORMAT;
2585 }
2586
2587 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2588 char *name;
2589 GElf_Shdr sh;
2590 Elf_Data *data;
2591
2592 idx++;
2593 if (gelf_getshdr(scn, &sh) != &sh) {
2594 pr_warn("failed to get section(%d) header from %s\n",
2595 idx, obj->path);
2596 return -LIBBPF_ERRNO__FORMAT;
2597 }
2598
2599 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
2600 if (!name) {
2601 pr_warn("failed to get section(%d) name from %s\n",
2602 idx, obj->path);
2603 return -LIBBPF_ERRNO__FORMAT;
2604 }
2605
2606 data = elf_getdata(scn, 0);
2607 if (!data) {
2608 pr_warn("failed to get section(%d) data from %s(%s)\n",
2609 idx, name, obj->path);
2610 return -LIBBPF_ERRNO__FORMAT;
2611 }
2612 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
2613 idx, name, (unsigned long)data->d_size,
2614 (int)sh.sh_link, (unsigned long)sh.sh_flags,
2615 (int)sh.sh_type);
2616
2617 if (strcmp(name, "license") == 0) {
2618 err = bpf_object__init_license(obj,
2619 data->d_buf,
2620 data->d_size);
2621 if (err)
2622 return err;
2623 } else if (strcmp(name, "version") == 0) {
2624 err = bpf_object__init_kversion(obj,
2625 data->d_buf,
2626 data->d_size);
2627 if (err)
2628 return err;
2629 } else if (strcmp(name, "maps") == 0) {
2630 obj->efile.maps_shndx = idx;
2631 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
2632 obj->efile.btf_maps_shndx = idx;
2633 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
2634 btf_data = data;
2635 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
2636 btf_ext_data = data;
2637 } else if (sh.sh_type == SHT_SYMTAB) {
2638 if (obj->efile.symbols) {
2639 pr_warn("bpf: multiple SYMTAB in %s\n",
2640 obj->path);
2641 return -LIBBPF_ERRNO__FORMAT;
2642 }
2643 obj->efile.symbols = data;
2644 obj->efile.symbols_shndx = idx;
2645 obj->efile.strtabidx = sh.sh_link;
2646 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
2647 if (sh.sh_flags & SHF_EXECINSTR) {
2648 if (strcmp(name, ".text") == 0)
2649 obj->efile.text_shndx = idx;
2650 err = bpf_object__add_program(obj, data->d_buf,
2651 data->d_size,
2652 name, idx);
2653 if (err) {
2654 char errmsg[STRERR_BUFSIZE];
2655 char *cp;
2656
2657 cp = libbpf_strerror_r(-err, errmsg,
2658 sizeof(errmsg));
2659 pr_warn("failed to alloc program %s (%s): %s",
2660 name, obj->path, cp);
2661 return err;
2662 }
2663 } else if (strcmp(name, DATA_SEC) == 0) {
2664 obj->efile.data = data;
2665 obj->efile.data_shndx = idx;
2666 } else if (strcmp(name, RODATA_SEC) == 0) {
2667 obj->efile.rodata = data;
2668 obj->efile.rodata_shndx = idx;
2669 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
2670 obj->efile.st_ops_data = data;
2671 obj->efile.st_ops_shndx = idx;
2672 } else {
2673 pr_debug("skip section(%d) %s\n", idx, name);
2674 }
2675 } else if (sh.sh_type == SHT_REL) {
2676 int nr_sects = obj->efile.nr_reloc_sects;
2677 void *sects = obj->efile.reloc_sects;
2678 int sec = sh.sh_info;
2679
2680
2681 if (!section_have_execinstr(obj, sec) &&
2682 strcmp(name, ".rel" STRUCT_OPS_SEC) &&
2683 strcmp(name, ".rel" MAPS_ELF_SEC)) {
2684 pr_debug("skip relo %s(%d) for section(%d)\n",
2685 name, idx, sec);
2686 continue;
2687 }
2688
2689 sects = reallocarray(sects, nr_sects + 1,
2690 sizeof(*obj->efile.reloc_sects));
2691 if (!sects) {
2692 pr_warn("reloc_sects realloc failed\n");
2693 return -ENOMEM;
2694 }
2695
2696 obj->efile.reloc_sects = sects;
2697 obj->efile.nr_reloc_sects++;
2698
2699 obj->efile.reloc_sects[nr_sects].shdr = sh;
2700 obj->efile.reloc_sects[nr_sects].data = data;
2701 } else if (sh.sh_type == SHT_NOBITS &&
2702 strcmp(name, BSS_SEC) == 0) {
2703 obj->efile.bss = data;
2704 obj->efile.bss_shndx = idx;
2705 } else {
2706 pr_debug("skip section(%d) %s\n", idx, name);
2707 }
2708 }
2709
2710 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
2711 pr_warn("Corrupted ELF file: index of strtab invalid\n");
2712 return -LIBBPF_ERRNO__FORMAT;
2713 }
2714 return bpf_object__init_btf(obj, btf_data, btf_ext_data);
2715}
2716
2717static bool sym_is_extern(const GElf_Sym *sym)
2718{
2719 int bind = GELF_ST_BIND(sym->st_info);
2720
2721 return sym->st_shndx == SHN_UNDEF &&
2722 (bind == STB_GLOBAL || bind == STB_WEAK) &&
2723 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE;
2724}
2725
2726static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
2727{
2728 const struct btf_type *t;
2729 const char *var_name;
2730 int i, n;
2731
2732 if (!btf)
2733 return -ESRCH;
2734
2735 n = btf__get_nr_types(btf);
2736 for (i = 1; i <= n; i++) {
2737 t = btf__type_by_id(btf, i);
2738
2739 if (!btf_is_var(t))
2740 continue;
2741
2742 var_name = btf__name_by_offset(btf, t->name_off);
2743 if (strcmp(var_name, ext_name))
2744 continue;
2745
2746 if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
2747 return -EINVAL;
2748
2749 return i;
2750 }
2751
2752 return -ENOENT;
2753}
2754
2755static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
2756 const struct btf_var_secinfo *vs;
2757 const struct btf_type *t;
2758 int i, j, n;
2759
2760 if (!btf)
2761 return -ESRCH;
2762
2763 n = btf__get_nr_types(btf);
2764 for (i = 1; i <= n; i++) {
2765 t = btf__type_by_id(btf, i);
2766
2767 if (!btf_is_datasec(t))
2768 continue;
2769
2770 vs = btf_var_secinfos(t);
2771 for (j = 0; j < btf_vlen(t); j++, vs++) {
2772 if (vs->type == ext_btf_id)
2773 return i;
2774 }
2775 }
2776
2777 return -ENOENT;
2778}
2779
2780static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
2781 bool *is_signed)
2782{
2783 const struct btf_type *t;
2784 const char *name;
2785
2786 t = skip_mods_and_typedefs(btf, id, NULL);
2787 name = btf__name_by_offset(btf, t->name_off);
2788
2789 if (is_signed)
2790 *is_signed = false;
2791 switch (btf_kind(t)) {
2792 case BTF_KIND_INT: {
2793 int enc = btf_int_encoding(t);
2794
2795 if (enc & BTF_INT_BOOL)
2796 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
2797 if (is_signed)
2798 *is_signed = enc & BTF_INT_SIGNED;
2799 if (t->size == 1)
2800 return KCFG_CHAR;
2801 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
2802 return KCFG_UNKNOWN;
2803 return KCFG_INT;
2804 }
2805 case BTF_KIND_ENUM:
2806 if (t->size != 4)
2807 return KCFG_UNKNOWN;
2808 if (strcmp(name, "libbpf_tristate"))
2809 return KCFG_UNKNOWN;
2810 return KCFG_TRISTATE;
2811 case BTF_KIND_ARRAY:
2812 if (btf_array(t)->nelems == 0)
2813 return KCFG_UNKNOWN;
2814 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
2815 return KCFG_UNKNOWN;
2816 return KCFG_CHAR_ARR;
2817 default:
2818 return KCFG_UNKNOWN;
2819 }
2820}
2821
2822static int cmp_externs(const void *_a, const void *_b)
2823{
2824 const struct extern_desc *a = _a;
2825 const struct extern_desc *b = _b;
2826
2827 if (a->type != b->type)
2828 return a->type < b->type ? -1 : 1;
2829
2830 if (a->type == EXT_KCFG) {
2831
2832 if (a->kcfg.align != b->kcfg.align)
2833 return a->kcfg.align > b->kcfg.align ? -1 : 1;
2834
2835 if (a->kcfg.sz != b->kcfg.sz)
2836 return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
2837 }
2838
2839
2840 return strcmp(a->name, b->name);
2841}
2842
2843static int find_int_btf_id(const struct btf *btf)
2844{
2845 const struct btf_type *t;
2846 int i, n;
2847
2848 n = btf__get_nr_types(btf);
2849 for (i = 1; i <= n; i++) {
2850 t = btf__type_by_id(btf, i);
2851
2852 if (btf_is_int(t) && btf_int_bits(t) == 32)
2853 return i;
2854 }
2855
2856 return 0;
2857}
2858
2859static int bpf_object__collect_externs(struct bpf_object *obj)
2860{
2861 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
2862 const struct btf_type *t;
2863 struct extern_desc *ext;
2864 int i, n, off;
2865 const char *ext_name, *sec_name;
2866 Elf_Scn *scn;
2867 GElf_Shdr sh;
2868
2869 if (!obj->efile.symbols)
2870 return 0;
2871
2872 scn = elf_getscn(obj->efile.elf, obj->efile.symbols_shndx);
2873 if (!scn)
2874 return -LIBBPF_ERRNO__FORMAT;
2875 if (gelf_getshdr(scn, &sh) != &sh)
2876 return -LIBBPF_ERRNO__FORMAT;
2877 n = sh.sh_size / sh.sh_entsize;
2878
2879 pr_debug("looking for externs among %d symbols...\n", n);
2880 for (i = 0; i < n; i++) {
2881 GElf_Sym sym;
2882
2883 if (!gelf_getsym(obj->efile.symbols, i, &sym))
2884 return -LIBBPF_ERRNO__FORMAT;
2885 if (!sym_is_extern(&sym))
2886 continue;
2887 ext_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
2888 sym.st_name);
2889 if (!ext_name || !ext_name[0])
2890 continue;
2891
2892 ext = obj->externs;
2893 ext = reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
2894 if (!ext)
2895 return -ENOMEM;
2896 obj->externs = ext;
2897 ext = &ext[obj->nr_extern];
2898 memset(ext, 0, sizeof(*ext));
2899 obj->nr_extern++;
2900
2901 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
2902 if (ext->btf_id <= 0) {
2903 pr_warn("failed to find BTF for extern '%s': %d\n",
2904 ext_name, ext->btf_id);
2905 return ext->btf_id;
2906 }
2907 t = btf__type_by_id(obj->btf, ext->btf_id);
2908 ext->name = btf__name_by_offset(obj->btf, t->name_off);
2909 ext->sym_idx = i;
2910 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK;
2911
2912 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
2913 if (ext->sec_btf_id <= 0) {
2914 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
2915 ext_name, ext->btf_id, ext->sec_btf_id);
2916 return ext->sec_btf_id;
2917 }
2918 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
2919 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
2920
2921 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
2922 kcfg_sec = sec;
2923 ext->type = EXT_KCFG;
2924 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
2925 if (ext->kcfg.sz <= 0) {
2926 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
2927 ext_name, ext->kcfg.sz);
2928 return ext->kcfg.sz;
2929 }
2930 ext->kcfg.align = btf__align_of(obj->btf, t->type);
2931 if (ext->kcfg.align <= 0) {
2932 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
2933 ext_name, ext->kcfg.align);
2934 return -EINVAL;
2935 }
2936 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
2937 &ext->kcfg.is_signed);
2938 if (ext->kcfg.type == KCFG_UNKNOWN) {
2939 pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name);
2940 return -ENOTSUP;
2941 }
2942 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
2943 const struct btf_type *vt;
2944
2945 ksym_sec = sec;
2946 ext->type = EXT_KSYM;
2947
2948 vt = skip_mods_and_typedefs(obj->btf, t->type, NULL);
2949 if (!btf_is_void(vt)) {
2950 pr_warn("extern (ksym) '%s' is not typeless (void)\n", ext_name);
2951 return -ENOTSUP;
2952 }
2953 } else {
2954 pr_warn("unrecognized extern section '%s'\n", sec_name);
2955 return -ENOTSUP;
2956 }
2957 }
2958 pr_debug("collected %d externs total\n", obj->nr_extern);
2959
2960 if (!obj->nr_extern)
2961 return 0;
2962
2963
2964 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
2965
2966
2967
2968
2969
2970 if (ksym_sec) {
2971
2972
2973
2974 int int_btf_id = find_int_btf_id(obj->btf);
2975
2976 for (i = 0; i < obj->nr_extern; i++) {
2977 ext = &obj->externs[i];
2978 if (ext->type != EXT_KSYM)
2979 continue;
2980 pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
2981 i, ext->sym_idx, ext->name);
2982 }
2983
2984 sec = ksym_sec;
2985 n = btf_vlen(sec);
2986 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
2987 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
2988 struct btf_type *vt;
2989
2990 vt = (void *)btf__type_by_id(obj->btf, vs->type);
2991 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
2992 ext = find_extern_by_name(obj, ext_name);
2993 if (!ext) {
2994 pr_warn("failed to find extern definition for BTF var '%s'\n",
2995 ext_name);
2996 return -ESRCH;
2997 }
2998 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
2999 vt->type = int_btf_id;
3000 vs->offset = off;
3001 vs->size = sizeof(int);
3002 }
3003 sec->size = off;
3004 }
3005
3006 if (kcfg_sec) {
3007 sec = kcfg_sec;
3008
3009 off = 0;
3010 for (i = 0; i < obj->nr_extern; i++) {
3011 ext = &obj->externs[i];
3012 if (ext->type != EXT_KCFG)
3013 continue;
3014
3015 ext->kcfg.data_off = roundup(off, ext->kcfg.align);
3016 off = ext->kcfg.data_off + ext->kcfg.sz;
3017 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
3018 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
3019 }
3020 sec->size = off;
3021 n = btf_vlen(sec);
3022 for (i = 0; i < n; i++) {
3023 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3024
3025 t = btf__type_by_id(obj->btf, vs->type);
3026 ext_name = btf__name_by_offset(obj->btf, t->name_off);
3027 ext = find_extern_by_name(obj, ext_name);
3028 if (!ext) {
3029 pr_warn("failed to find extern definition for BTF var '%s'\n",
3030 ext_name);
3031 return -ESRCH;
3032 }
3033 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3034 vs->offset = ext->kcfg.data_off;
3035 }
3036 }
3037 return 0;
3038}
3039
3040static struct bpf_program *
3041bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
3042{
3043 struct bpf_program *prog;
3044 size_t i;
3045
3046 for (i = 0; i < obj->nr_programs; i++) {
3047 prog = &obj->programs[i];
3048 if (prog->idx == idx)
3049 return prog;
3050 }
3051 return NULL;
3052}
3053
3054struct bpf_program *
3055bpf_object__find_program_by_title(const struct bpf_object *obj,
3056 const char *title)
3057{
3058 struct bpf_program *pos;
3059
3060 bpf_object__for_each_program(pos, obj) {
3061 if (pos->section_name && !strcmp(pos->section_name, title))
3062 return pos;
3063 }
3064 return NULL;
3065}
3066
3067struct bpf_program *
3068bpf_object__find_program_by_name(const struct bpf_object *obj,
3069 const char *name)
3070{
3071 struct bpf_program *prog;
3072
3073 bpf_object__for_each_program(prog, obj) {
3074 if (!strcmp(prog->name, name))
3075 return prog;
3076 }
3077 return NULL;
3078}
3079
3080static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
3081 int shndx)
3082{
3083 return shndx == obj->efile.data_shndx ||
3084 shndx == obj->efile.bss_shndx ||
3085 shndx == obj->efile.rodata_shndx;
3086}
3087
3088static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
3089 int shndx)
3090{
3091 return shndx == obj->efile.maps_shndx ||
3092 shndx == obj->efile.btf_maps_shndx;
3093}
3094
3095static enum libbpf_map_type
3096bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
3097{
3098 if (shndx == obj->efile.data_shndx)
3099 return LIBBPF_MAP_DATA;
3100 else if (shndx == obj->efile.bss_shndx)
3101 return LIBBPF_MAP_BSS;
3102 else if (shndx == obj->efile.rodata_shndx)
3103 return LIBBPF_MAP_RODATA;
3104 else if (shndx == obj->efile.symbols_shndx)
3105 return LIBBPF_MAP_KCONFIG;
3106 else
3107 return LIBBPF_MAP_UNSPEC;
3108}
3109
3110static int bpf_program__record_reloc(struct bpf_program *prog,
3111 struct reloc_desc *reloc_desc,
3112 __u32 insn_idx, const char *name,
3113 const GElf_Sym *sym, const GElf_Rel *rel)
3114{
3115 struct bpf_insn *insn = &prog->insns[insn_idx];
3116 size_t map_idx, nr_maps = prog->obj->nr_maps;
3117 struct bpf_object *obj = prog->obj;
3118 __u32 shdr_idx = sym->st_shndx;
3119 enum libbpf_map_type type;
3120 struct bpf_map *map;
3121
3122
3123 if (insn->code == (BPF_JMP | BPF_CALL)) {
3124 if (insn->src_reg != BPF_PSEUDO_CALL) {
3125 pr_warn("incorrect bpf_call opcode\n");
3126 return -LIBBPF_ERRNO__RELOC;
3127 }
3128
3129 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
3130 pr_warn("bad call relo against section %u\n", shdr_idx);
3131 return -LIBBPF_ERRNO__RELOC;
3132 }
3133 if (sym->st_value % 8) {
3134 pr_warn("bad call relo offset: %zu\n",
3135 (size_t)sym->st_value);
3136 return -LIBBPF_ERRNO__RELOC;
3137 }
3138 reloc_desc->type = RELO_CALL;
3139 reloc_desc->insn_idx = insn_idx;
3140 reloc_desc->sym_off = sym->st_value;
3141 obj->has_pseudo_calls = true;
3142 return 0;
3143 }
3144
3145 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) {
3146 pr_warn("invalid relo for insns[%d].code 0x%x\n",
3147 insn_idx, insn->code);
3148 return -LIBBPF_ERRNO__RELOC;
3149 }
3150
3151 if (sym_is_extern(sym)) {
3152 int sym_idx = GELF_R_SYM(rel->r_info);
3153 int i, n = obj->nr_extern;
3154 struct extern_desc *ext;
3155
3156 for (i = 0; i < n; i++) {
3157 ext = &obj->externs[i];
3158 if (ext->sym_idx == sym_idx)
3159 break;
3160 }
3161 if (i >= n) {
3162 pr_warn("extern relo failed to find extern for sym %d\n",
3163 sym_idx);
3164 return -LIBBPF_ERRNO__RELOC;
3165 }
3166 pr_debug("found extern #%d '%s' (sym %d) for insn %u\n",
3167 i, ext->name, ext->sym_idx, insn_idx);
3168 reloc_desc->type = RELO_EXTERN;
3169 reloc_desc->insn_idx = insn_idx;
3170 reloc_desc->sym_off = i;
3171 return 0;
3172 }
3173
3174 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
3175 pr_warn("invalid relo for \'%s\' in special section 0x%x; forgot to initialize global var?..\n",
3176 name, shdr_idx);
3177 return -LIBBPF_ERRNO__RELOC;
3178 }
3179
3180 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
3181
3182
3183 if (type == LIBBPF_MAP_UNSPEC) {
3184 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
3185 pr_warn("bad map relo against section %u\n",
3186 shdr_idx);
3187 return -LIBBPF_ERRNO__RELOC;
3188 }
3189 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3190 map = &obj->maps[map_idx];
3191 if (map->libbpf_type != type ||
3192 map->sec_idx != sym->st_shndx ||
3193 map->sec_offset != sym->st_value)
3194 continue;
3195 pr_debug("found map %zd (%s, sec %d, off %zu) for insn %u\n",
3196 map_idx, map->name, map->sec_idx,
3197 map->sec_offset, insn_idx);
3198 break;
3199 }
3200 if (map_idx >= nr_maps) {
3201 pr_warn("map relo failed to find map for sec %u, off %zu\n",
3202 shdr_idx, (size_t)sym->st_value);
3203 return -LIBBPF_ERRNO__RELOC;
3204 }
3205 reloc_desc->type = RELO_LD64;
3206 reloc_desc->insn_idx = insn_idx;
3207 reloc_desc->map_idx = map_idx;
3208 reloc_desc->sym_off = 0;
3209 return 0;
3210 }
3211
3212
3213 if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
3214 pr_warn("bad data relo against section %u\n", shdr_idx);
3215 return -LIBBPF_ERRNO__RELOC;
3216 }
3217 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3218 map = &obj->maps[map_idx];
3219 if (map->libbpf_type != type)
3220 continue;
3221 pr_debug("found data map %zd (%s, sec %d, off %zu) for insn %u\n",
3222 map_idx, map->name, map->sec_idx, map->sec_offset,
3223 insn_idx);
3224 break;
3225 }
3226 if (map_idx >= nr_maps) {
3227 pr_warn("data relo failed to find map for sec %u\n",
3228 shdr_idx);
3229 return -LIBBPF_ERRNO__RELOC;
3230 }
3231
3232 reloc_desc->type = RELO_DATA;
3233 reloc_desc->insn_idx = insn_idx;
3234 reloc_desc->map_idx = map_idx;
3235 reloc_desc->sym_off = sym->st_value;
3236 return 0;
3237}
3238
3239static int
3240bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
3241 Elf_Data *data, struct bpf_object *obj)
3242{
3243 Elf_Data *symbols = obj->efile.symbols;
3244 int err, i, nrels;
3245
3246 pr_debug("collecting relocating info for: '%s'\n", prog->section_name);
3247 nrels = shdr->sh_size / shdr->sh_entsize;
3248
3249 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
3250 if (!prog->reloc_desc) {
3251 pr_warn("failed to alloc memory in relocation\n");
3252 return -ENOMEM;
3253 }
3254 prog->nr_reloc = nrels;
3255
3256 for (i = 0; i < nrels; i++) {
3257 const char *name;
3258 __u32 insn_idx;
3259 GElf_Sym sym;
3260 GElf_Rel rel;
3261
3262 if (!gelf_getrel(data, i, &rel)) {
3263 pr_warn("relocation: failed to get %d reloc\n", i);
3264 return -LIBBPF_ERRNO__FORMAT;
3265 }
3266 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
3267 pr_warn("relocation: symbol %"PRIx64" not found\n",
3268 GELF_R_SYM(rel.r_info));
3269 return -LIBBPF_ERRNO__FORMAT;
3270 }
3271 if (rel.r_offset % sizeof(struct bpf_insn))
3272 return -LIBBPF_ERRNO__FORMAT;
3273
3274 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
3275 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
3276 sym.st_name) ? : "<?>";
3277
3278 pr_debug("relo for shdr %u, symb %zu, value %zu, type %d, bind %d, name %d (\'%s\'), insn %u\n",
3279 (__u32)sym.st_shndx, (size_t)GELF_R_SYM(rel.r_info),
3280 (size_t)sym.st_value, GELF_ST_TYPE(sym.st_info),
3281 GELF_ST_BIND(sym.st_info), sym.st_name, name,
3282 insn_idx);
3283
3284 err = bpf_program__record_reloc(prog, &prog->reloc_desc[i],
3285 insn_idx, name, &sym, &rel);
3286 if (err)
3287 return err;
3288 }
3289 return 0;
3290}
3291
3292static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
3293{
3294 struct bpf_map_def *def = &map->def;
3295 __u32 key_type_id = 0, value_type_id = 0;
3296 int ret;
3297
3298
3299
3300
3301
3302 if (map->sec_idx == obj->efile.btf_maps_shndx ||
3303 bpf_map__is_struct_ops(map))
3304 return 0;
3305
3306 if (!bpf_map__is_internal(map)) {
3307 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size,
3308 def->value_size, &key_type_id,
3309 &value_type_id);
3310 } else {
3311
3312
3313
3314
3315 ret = btf__find_by_name(obj->btf,
3316 libbpf_type_to_btf_name[map->libbpf_type]);
3317 }
3318 if (ret < 0)
3319 return ret;
3320
3321 map->btf_key_type_id = key_type_id;
3322 map->btf_value_type_id = bpf_map__is_internal(map) ?
3323 ret : value_type_id;
3324 return 0;
3325}
3326
3327int bpf_map__reuse_fd(struct bpf_map *map, int fd)
3328{
3329 struct bpf_map_info info = {};
3330 __u32 len = sizeof(info);
3331 int new_fd, err;
3332 char *new_name;
3333
3334 err = bpf_obj_get_info_by_fd(fd, &info, &len);
3335 if (err)
3336 return err;
3337
3338 new_name = strdup(info.name);
3339 if (!new_name)
3340 return -errno;
3341
3342 new_fd = open("/", O_RDONLY | O_CLOEXEC);
3343 if (new_fd < 0) {
3344 err = -errno;
3345 goto err_free_new_name;
3346 }
3347
3348 new_fd = dup3(fd, new_fd, O_CLOEXEC);
3349 if (new_fd < 0) {
3350 err = -errno;
3351 goto err_close_new_fd;
3352 }
3353
3354 err = zclose(map->fd);
3355 if (err) {
3356 err = -errno;
3357 goto err_close_new_fd;
3358 }
3359 free(map->name);
3360
3361 map->fd = new_fd;
3362 map->name = new_name;
3363 map->def.type = info.type;
3364 map->def.key_size = info.key_size;
3365 map->def.value_size = info.value_size;
3366 map->def.max_entries = info.max_entries;
3367 map->def.map_flags = info.map_flags;
3368 map->btf_key_type_id = info.btf_key_type_id;
3369 map->btf_value_type_id = info.btf_value_type_id;
3370 map->reused = true;
3371
3372 return 0;
3373
3374err_close_new_fd:
3375 close(new_fd);
3376err_free_new_name:
3377 free(new_name);
3378 return err;
3379}
3380
3381__u32 bpf_map__max_entries(const struct bpf_map *map)
3382{
3383 return map->def.max_entries;
3384}
3385
3386int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
3387{
3388 if (map->fd >= 0)
3389 return -EBUSY;
3390 map->def.max_entries = max_entries;
3391 return 0;
3392}
3393
3394int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
3395{
3396 if (!map || !max_entries)
3397 return -EINVAL;
3398
3399 return bpf_map__set_max_entries(map, max_entries);
3400}
3401
3402static int
3403bpf_object__probe_loading(struct bpf_object *obj)
3404{
3405 struct bpf_load_program_attr attr;
3406 char *cp, errmsg[STRERR_BUFSIZE];
3407 struct bpf_insn insns[] = {
3408 BPF_MOV64_IMM(BPF_REG_0, 0),
3409 BPF_EXIT_INSN(),
3410 };
3411 int ret;
3412
3413
3414
3415 memset(&attr, 0, sizeof(attr));
3416 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3417 attr.insns = insns;
3418 attr.insns_cnt = ARRAY_SIZE(insns);
3419 attr.license = "GPL";
3420
3421 ret = bpf_load_program_xattr(&attr, NULL, 0);
3422 if (ret < 0) {
3423 ret = errno;
3424 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3425 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
3426 "program. Make sure your kernel supports BPF "
3427 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
3428 "set to big enough value.\n", __func__, cp, ret);
3429 return -ret;
3430 }
3431 close(ret);
3432
3433 return 0;
3434}
3435
3436static int
3437bpf_object__probe_name(struct bpf_object *obj)
3438{
3439 struct bpf_load_program_attr attr;
3440 struct bpf_insn insns[] = {
3441 BPF_MOV64_IMM(BPF_REG_0, 0),
3442 BPF_EXIT_INSN(),
3443 };
3444 int ret;
3445
3446
3447
3448 memset(&attr, 0, sizeof(attr));
3449 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3450 attr.insns = insns;
3451 attr.insns_cnt = ARRAY_SIZE(insns);
3452 attr.license = "GPL";
3453 attr.name = "test";
3454 ret = bpf_load_program_xattr(&attr, NULL, 0);
3455 if (ret >= 0) {
3456 obj->caps.name = 1;
3457 close(ret);
3458 }
3459
3460 return 0;
3461}
3462
3463static int
3464bpf_object__probe_global_data(struct bpf_object *obj)
3465{
3466 struct bpf_load_program_attr prg_attr;
3467 struct bpf_create_map_attr map_attr;
3468 char *cp, errmsg[STRERR_BUFSIZE];
3469 struct bpf_insn insns[] = {
3470 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
3471 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
3472 BPF_MOV64_IMM(BPF_REG_0, 0),
3473 BPF_EXIT_INSN(),
3474 };
3475 int ret, map;
3476
3477 memset(&map_attr, 0, sizeof(map_attr));
3478 map_attr.map_type = BPF_MAP_TYPE_ARRAY;
3479 map_attr.key_size = sizeof(int);
3480 map_attr.value_size = 32;
3481 map_attr.max_entries = 1;
3482
3483 map = bpf_create_map_xattr(&map_attr);
3484 if (map < 0) {
3485 ret = -errno;
3486 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3487 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
3488 __func__, cp, -ret);
3489 return ret;
3490 }
3491
3492 insns[0].imm = map;
3493
3494 memset(&prg_attr, 0, sizeof(prg_attr));
3495 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3496 prg_attr.insns = insns;
3497 prg_attr.insns_cnt = ARRAY_SIZE(insns);
3498 prg_attr.license = "GPL";
3499
3500 ret = bpf_load_program_xattr(&prg_attr, NULL, 0);
3501 if (ret >= 0) {
3502 obj->caps.global_data = 1;
3503 close(ret);
3504 }
3505
3506 close(map);
3507 return 0;
3508}
3509
3510static int bpf_object__probe_btf_func(struct bpf_object *obj)
3511{
3512 static const char strs[] = "\0int\0x\0a";
3513
3514 __u32 types[] = {
3515
3516 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
3517
3518 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3519 BTF_PARAM_ENC(7, 1),
3520
3521 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
3522 };
3523 int btf_fd;
3524
3525 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types),
3526 strs, sizeof(strs));
3527 if (btf_fd >= 0) {
3528 obj->caps.btf_func = 1;
3529 close(btf_fd);
3530 return 1;
3531 }
3532
3533 return 0;
3534}
3535
3536static int bpf_object__probe_btf_func_global(struct bpf_object *obj)
3537{
3538 static const char strs[] = "\0int\0x\0a";
3539
3540 __u32 types[] = {
3541
3542 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
3543
3544 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3545 BTF_PARAM_ENC(7, 1),
3546
3547 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
3548 };
3549 int btf_fd;
3550
3551 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types),
3552 strs, sizeof(strs));
3553 if (btf_fd >= 0) {
3554 obj->caps.btf_func_global = 1;
3555 close(btf_fd);
3556 return 1;
3557 }
3558
3559 return 0;
3560}
3561
3562static int bpf_object__probe_btf_datasec(struct bpf_object *obj)
3563{
3564 static const char strs[] = "\0x\0.data";
3565
3566 __u32 types[] = {
3567
3568 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
3569
3570 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
3571 BTF_VAR_STATIC,
3572
3573 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
3574 BTF_VAR_SECINFO_ENC(2, 0, 4),
3575 };
3576 int btf_fd;
3577
3578 btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types),
3579 strs, sizeof(strs));
3580 if (btf_fd >= 0) {
3581 obj->caps.btf_datasec = 1;
3582 close(btf_fd);
3583 return 1;
3584 }
3585
3586 return 0;
3587}
3588
3589static int bpf_object__probe_array_mmap(struct bpf_object *obj)
3590{
3591 struct bpf_create_map_attr attr = {
3592 .map_type = BPF_MAP_TYPE_ARRAY,
3593 .map_flags = BPF_F_MMAPABLE,
3594 .key_size = sizeof(int),
3595 .value_size = sizeof(int),
3596 .max_entries = 1,
3597 };
3598 int fd;
3599
3600 fd = bpf_create_map_xattr(&attr);
3601 if (fd >= 0) {
3602 obj->caps.array_mmap = 1;
3603 close(fd);
3604 return 1;
3605 }
3606
3607 return 0;
3608}
3609
3610static int
3611bpf_object__probe_exp_attach_type(struct bpf_object *obj)
3612{
3613 struct bpf_load_program_attr attr;
3614 struct bpf_insn insns[] = {
3615 BPF_MOV64_IMM(BPF_REG_0, 0),
3616 BPF_EXIT_INSN(),
3617 };
3618 int fd;
3619
3620 memset(&attr, 0, sizeof(attr));
3621
3622
3623
3624
3625
3626 attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK;
3627 attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE;
3628 attr.insns = insns;
3629 attr.insns_cnt = ARRAY_SIZE(insns);
3630 attr.license = "GPL";
3631
3632 fd = bpf_load_program_xattr(&attr, NULL, 0);
3633 if (fd >= 0) {
3634 obj->caps.exp_attach_type = 1;
3635 close(fd);
3636 return 1;
3637 }
3638 return 0;
3639}
3640
3641static int
3642bpf_object__probe_caps(struct bpf_object *obj)
3643{
3644 int (*probe_fn[])(struct bpf_object *obj) = {
3645 bpf_object__probe_name,
3646 bpf_object__probe_global_data,
3647 bpf_object__probe_btf_func,
3648 bpf_object__probe_btf_func_global,
3649 bpf_object__probe_btf_datasec,
3650 bpf_object__probe_array_mmap,
3651 bpf_object__probe_exp_attach_type,
3652 };
3653 int i, ret;
3654
3655 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) {
3656 ret = probe_fn[i](obj);
3657 if (ret < 0)
3658 pr_debug("Probe #%d failed with %d.\n", i, ret);
3659 }
3660
3661 return 0;
3662}
3663
3664static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
3665{
3666 struct bpf_map_info map_info = {};
3667 char msg[STRERR_BUFSIZE];
3668 __u32 map_info_len;
3669
3670 map_info_len = sizeof(map_info);
3671
3672 if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) {
3673 pr_warn("failed to get map info for map FD %d: %s\n",
3674 map_fd, libbpf_strerror_r(errno, msg, sizeof(msg)));
3675 return false;
3676 }
3677
3678 return (map_info.type == map->def.type &&
3679 map_info.key_size == map->def.key_size &&
3680 map_info.value_size == map->def.value_size &&
3681 map_info.max_entries == map->def.max_entries &&
3682 map_info.map_flags == map->def.map_flags);
3683}
3684
3685static int
3686bpf_object__reuse_map(struct bpf_map *map)
3687{
3688 char *cp, errmsg[STRERR_BUFSIZE];
3689 int err, pin_fd;
3690
3691 pin_fd = bpf_obj_get(map->pin_path);
3692 if (pin_fd < 0) {
3693 err = -errno;
3694 if (err == -ENOENT) {
3695 pr_debug("found no pinned map to reuse at '%s'\n",
3696 map->pin_path);
3697 return 0;
3698 }
3699
3700 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
3701 pr_warn("couldn't retrieve pinned map '%s': %s\n",
3702 map->pin_path, cp);
3703 return err;
3704 }
3705
3706 if (!map_is_reuse_compat(map, pin_fd)) {
3707 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
3708 map->pin_path);
3709 close(pin_fd);
3710 return -EINVAL;
3711 }
3712
3713 err = bpf_map__reuse_fd(map, pin_fd);
3714 if (err) {
3715 close(pin_fd);
3716 return err;
3717 }
3718 map->pinned = true;
3719 pr_debug("reused pinned map at '%s'\n", map->pin_path);
3720
3721 return 0;
3722}
3723
3724static int
3725bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
3726{
3727 enum libbpf_map_type map_type = map->libbpf_type;
3728 char *cp, errmsg[STRERR_BUFSIZE];
3729 int err, zero = 0;
3730
3731 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
3732 if (err) {
3733 err = -errno;
3734 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
3735 pr_warn("Error setting initial map(%s) contents: %s\n",
3736 map->name, cp);
3737 return err;
3738 }
3739
3740
3741 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
3742 err = bpf_map_freeze(map->fd);
3743 if (err) {
3744 err = -errno;
3745 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
3746 pr_warn("Error freezing map(%s) as read-only: %s\n",
3747 map->name, cp);
3748 return err;
3749 }
3750 }
3751 return 0;
3752}
3753
3754static void bpf_map__destroy(struct bpf_map *map);
3755
3756static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map)
3757{
3758 struct bpf_create_map_attr create_attr;
3759 struct bpf_map_def *def = &map->def;
3760
3761 memset(&create_attr, 0, sizeof(create_attr));
3762
3763 if (obj->caps.name)
3764 create_attr.name = map->name;
3765 create_attr.map_ifindex = map->map_ifindex;
3766 create_attr.map_type = def->type;
3767 create_attr.map_flags = def->map_flags;
3768 create_attr.key_size = def->key_size;
3769 create_attr.value_size = def->value_size;
3770 create_attr.numa_node = map->numa_node;
3771
3772 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) {
3773 int nr_cpus;
3774
3775 nr_cpus = libbpf_num_possible_cpus();
3776 if (nr_cpus < 0) {
3777 pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
3778 map->name, nr_cpus);
3779 return nr_cpus;
3780 }
3781 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
3782 create_attr.max_entries = nr_cpus;
3783 } else {
3784 create_attr.max_entries = def->max_entries;
3785 }
3786
3787 if (bpf_map__is_struct_ops(map))
3788 create_attr.btf_vmlinux_value_type_id =
3789 map->btf_vmlinux_value_type_id;
3790
3791 create_attr.btf_fd = 0;
3792 create_attr.btf_key_type_id = 0;
3793 create_attr.btf_value_type_id = 0;
3794 if (obj->btf && btf__fd(obj->btf) >= 0 && !bpf_map_find_btf_info(obj, map)) {
3795 create_attr.btf_fd = btf__fd(obj->btf);
3796 create_attr.btf_key_type_id = map->btf_key_type_id;
3797 create_attr.btf_value_type_id = map->btf_value_type_id;
3798 }
3799
3800 if (bpf_map_type__is_map_in_map(def->type)) {
3801 if (map->inner_map) {
3802 int err;
3803
3804 err = bpf_object__create_map(obj, map->inner_map);
3805 if (err) {
3806 pr_warn("map '%s': failed to create inner map: %d\n",
3807 map->name, err);
3808 return err;
3809 }
3810 map->inner_map_fd = bpf_map__fd(map->inner_map);
3811 }
3812 if (map->inner_map_fd >= 0)
3813 create_attr.inner_map_fd = map->inner_map_fd;
3814 }
3815
3816 map->fd = bpf_create_map_xattr(&create_attr);
3817 if (map->fd < 0 && (create_attr.btf_key_type_id ||
3818 create_attr.btf_value_type_id)) {
3819 char *cp, errmsg[STRERR_BUFSIZE];
3820 int err = -errno;
3821
3822 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
3823 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
3824 map->name, cp, err);
3825 create_attr.btf_fd = 0;
3826 create_attr.btf_key_type_id = 0;
3827 create_attr.btf_value_type_id = 0;
3828 map->btf_key_type_id = 0;
3829 map->btf_value_type_id = 0;
3830 map->fd = bpf_create_map_xattr(&create_attr);
3831 }
3832
3833 if (map->fd < 0)
3834 return -errno;
3835
3836 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
3837 bpf_map__destroy(map->inner_map);
3838 zfree(&map->inner_map);
3839 }
3840
3841 return 0;
3842}
3843
3844static int
3845bpf_object__create_maps(struct bpf_object *obj)
3846{
3847 struct bpf_map *map;
3848 char *cp, errmsg[STRERR_BUFSIZE];
3849 unsigned int i, j;
3850 int err;
3851
3852 for (i = 0; i < obj->nr_maps; i++) {
3853 map = &obj->maps[i];
3854
3855 if (map->pin_path) {
3856 err = bpf_object__reuse_map(map);
3857 if (err) {
3858 pr_warn("map '%s': error reusing pinned map\n",
3859 map->name);
3860 goto err_out;
3861 }
3862 }
3863
3864 if (map->fd >= 0) {
3865 pr_debug("map '%s': skipping creation (preset fd=%d)\n",
3866 map->name, map->fd);
3867 continue;
3868 }
3869
3870 err = bpf_object__create_map(obj, map);
3871 if (err)
3872 goto err_out;
3873
3874 pr_debug("map '%s': created successfully, fd=%d\n", map->name,
3875 map->fd);
3876
3877 if (bpf_map__is_internal(map)) {
3878 err = bpf_object__populate_internal_map(obj, map);
3879 if (err < 0) {
3880 zclose(map->fd);
3881 goto err_out;
3882 }
3883 }
3884
3885 if (map->init_slots_sz) {
3886 for (j = 0; j < map->init_slots_sz; j++) {
3887 const struct bpf_map *targ_map;
3888 int fd;
3889
3890 if (!map->init_slots[j])
3891 continue;
3892
3893 targ_map = map->init_slots[j];
3894 fd = bpf_map__fd(targ_map);
3895 err = bpf_map_update_elem(map->fd, &j, &fd, 0);
3896 if (err) {
3897 err = -errno;
3898 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
3899 map->name, j, targ_map->name,
3900 fd, err);
3901 goto err_out;
3902 }
3903 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
3904 map->name, j, targ_map->name, fd);
3905 }
3906 zfree(&map->init_slots);
3907 map->init_slots_sz = 0;
3908 }
3909
3910 if (map->pin_path && !map->pinned) {
3911 err = bpf_map__pin(map, NULL);
3912 if (err) {
3913 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
3914 map->name, map->pin_path, err);
3915 zclose(map->fd);
3916 goto err_out;
3917 }
3918 }
3919 }
3920
3921 return 0;
3922
3923err_out:
3924 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
3925 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
3926 pr_perm_msg(err);
3927 for (j = 0; j < i; j++)
3928 zclose(obj->maps[j].fd);
3929 return err;
3930}
3931
3932static int
3933check_btf_ext_reloc_err(struct bpf_program *prog, int err,
3934 void *btf_prog_info, const char *info_name)
3935{
3936 if (err != -ENOENT) {
3937 pr_warn("Error in loading %s for sec %s.\n",
3938 info_name, prog->section_name);
3939 return err;
3940 }
3941
3942
3943
3944 if (btf_prog_info) {
3945
3946
3947
3948
3949 pr_warn("Error in relocating %s for sec %s.\n",
3950 info_name, prog->section_name);
3951 return err;
3952 }
3953
3954
3955 pr_warn("Cannot find %s for main program sec %s. Ignore all %s.\n",
3956 info_name, prog->section_name, info_name);
3957 return 0;
3958}
3959
3960static int
3961bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj,
3962 const char *section_name, __u32 insn_offset)
3963{
3964 int err;
3965
3966 if (!insn_offset || prog->func_info) {
3967
3968
3969
3970
3971
3972
3973 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext,
3974 section_name, insn_offset,
3975 &prog->func_info,
3976 &prog->func_info_cnt);
3977 if (err)
3978 return check_btf_ext_reloc_err(prog, err,
3979 prog->func_info,
3980 "bpf_func_info");
3981
3982 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext);
3983 }
3984
3985 if (!insn_offset || prog->line_info) {
3986 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext,
3987 section_name, insn_offset,
3988 &prog->line_info,
3989 &prog->line_info_cnt);
3990 if (err)
3991 return check_btf_ext_reloc_err(prog, err,
3992 prog->line_info,
3993 "bpf_line_info");
3994
3995 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext);
3996 }
3997
3998 return 0;
3999}
4000
4001#define BPF_CORE_SPEC_MAX_LEN 64
4002
4003
4004struct bpf_core_accessor {
4005 __u32 type_id;
4006 __u32 idx;
4007 const char *name;
4008};
4009
4010struct bpf_core_spec {
4011 const struct btf *btf;
4012
4013 struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
4014
4015 int len;
4016
4017 int raw_spec[BPF_CORE_SPEC_MAX_LEN];
4018
4019 int raw_len;
4020
4021 __u32 bit_offset;
4022};
4023
4024static bool str_is_empty(const char *s)
4025{
4026 return !s || !s[0];
4027}
4028
4029static bool is_flex_arr(const struct btf *btf,
4030 const struct bpf_core_accessor *acc,
4031 const struct btf_array *arr)
4032{
4033 const struct btf_type *t;
4034
4035
4036 if (!acc->name || arr->nelems > 0)
4037 return false;
4038
4039
4040 t = btf__type_by_id(btf, acc->type_id);
4041 return acc->idx == btf_vlen(t) - 1;
4042}
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075static int bpf_core_spec_parse(const struct btf *btf,
4076 __u32 type_id,
4077 const char *spec_str,
4078 struct bpf_core_spec *spec)
4079{
4080 int access_idx, parsed_len, i;
4081 struct bpf_core_accessor *acc;
4082 const struct btf_type *t;
4083 const char *name;
4084 __u32 id;
4085 __s64 sz;
4086
4087 if (str_is_empty(spec_str) || *spec_str == ':')
4088 return -EINVAL;
4089
4090 memset(spec, 0, sizeof(*spec));
4091 spec->btf = btf;
4092
4093
4094 while (*spec_str) {
4095 if (*spec_str == ':')
4096 ++spec_str;
4097 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)
4098 return -EINVAL;
4099 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4100 return -E2BIG;
4101 spec_str += parsed_len;
4102 spec->raw_spec[spec->raw_len++] = access_idx;
4103 }
4104
4105 if (spec->raw_len == 0)
4106 return -EINVAL;
4107
4108
4109 t = skip_mods_and_typedefs(btf, type_id, &id);
4110 if (!t)
4111 return -EINVAL;
4112
4113 access_idx = spec->raw_spec[0];
4114 spec->spec[0].type_id = id;
4115 spec->spec[0].idx = access_idx;
4116 spec->len++;
4117
4118 sz = btf__resolve_size(btf, id);
4119 if (sz < 0)
4120 return sz;
4121 spec->bit_offset = access_idx * sz * 8;
4122
4123 for (i = 1; i < spec->raw_len; i++) {
4124 t = skip_mods_and_typedefs(btf, id, &id);
4125 if (!t)
4126 return -EINVAL;
4127
4128 access_idx = spec->raw_spec[i];
4129 acc = &spec->spec[spec->len];
4130
4131 if (btf_is_composite(t)) {
4132 const struct btf_member *m;
4133 __u32 bit_offset;
4134
4135 if (access_idx >= btf_vlen(t))
4136 return -EINVAL;
4137
4138 bit_offset = btf_member_bit_offset(t, access_idx);
4139 spec->bit_offset += bit_offset;
4140
4141 m = btf_members(t) + access_idx;
4142 if (m->name_off) {
4143 name = btf__name_by_offset(btf, m->name_off);
4144 if (str_is_empty(name))
4145 return -EINVAL;
4146
4147 acc->type_id = id;
4148 acc->idx = access_idx;
4149 acc->name = name;
4150 spec->len++;
4151 }
4152
4153 id = m->type;
4154 } else if (btf_is_array(t)) {
4155 const struct btf_array *a = btf_array(t);
4156 bool flex;
4157
4158 t = skip_mods_and_typedefs(btf, a->type, &id);
4159 if (!t)
4160 return -EINVAL;
4161
4162 flex = is_flex_arr(btf, acc - 1, a);
4163 if (!flex && access_idx >= a->nelems)
4164 return -EINVAL;
4165
4166 spec->spec[spec->len].type_id = id;
4167 spec->spec[spec->len].idx = access_idx;
4168 spec->len++;
4169
4170 sz = btf__resolve_size(btf, id);
4171 if (sz < 0)
4172 return sz;
4173 spec->bit_offset += access_idx * sz * 8;
4174 } else {
4175 pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n",
4176 type_id, spec_str, i, id, btf_kind(t));
4177 return -EINVAL;
4178 }
4179 }
4180
4181 return 0;
4182}
4183
4184static bool bpf_core_is_flavor_sep(const char *s)
4185{
4186
4187 return s[0] != '_' &&
4188 s[1] == '_' && s[2] == '_' && s[3] == '_' &&
4189 s[4] != '_';
4190}
4191
4192
4193
4194
4195
4196static size_t bpf_core_essential_name_len(const char *name)
4197{
4198 size_t n = strlen(name);
4199 int i;
4200
4201 for (i = n - 5; i >= 0; i--) {
4202 if (bpf_core_is_flavor_sep(name + i))
4203 return i + 1;
4204 }
4205 return n;
4206}
4207
4208
4209struct ids_vec {
4210 __u32 *data;
4211 int len;
4212};
4213
4214static void bpf_core_free_cands(struct ids_vec *cand_ids)
4215{
4216 free(cand_ids->data);
4217 free(cand_ids);
4218}
4219
4220static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf,
4221 __u32 local_type_id,
4222 const struct btf *targ_btf)
4223{
4224 size_t local_essent_len, targ_essent_len;
4225 const char *local_name, *targ_name;
4226 const struct btf_type *t;
4227 struct ids_vec *cand_ids;
4228 __u32 *new_ids;
4229 int i, err, n;
4230
4231 t = btf__type_by_id(local_btf, local_type_id);
4232 if (!t)
4233 return ERR_PTR(-EINVAL);
4234
4235 local_name = btf__name_by_offset(local_btf, t->name_off);
4236 if (str_is_empty(local_name))
4237 return ERR_PTR(-EINVAL);
4238 local_essent_len = bpf_core_essential_name_len(local_name);
4239
4240 cand_ids = calloc(1, sizeof(*cand_ids));
4241 if (!cand_ids)
4242 return ERR_PTR(-ENOMEM);
4243
4244 n = btf__get_nr_types(targ_btf);
4245 for (i = 1; i <= n; i++) {
4246 t = btf__type_by_id(targ_btf, i);
4247 targ_name = btf__name_by_offset(targ_btf, t->name_off);
4248 if (str_is_empty(targ_name))
4249 continue;
4250
4251 t = skip_mods_and_typedefs(targ_btf, i, NULL);
4252 if (!btf_is_composite(t) && !btf_is_array(t))
4253 continue;
4254
4255 targ_essent_len = bpf_core_essential_name_len(targ_name);
4256 if (targ_essent_len != local_essent_len)
4257 continue;
4258
4259 if (strncmp(local_name, targ_name, local_essent_len) == 0) {
4260 pr_debug("[%d] %s: found candidate [%d] %s\n",
4261 local_type_id, local_name, i, targ_name);
4262 new_ids = reallocarray(cand_ids->data,
4263 cand_ids->len + 1,
4264 sizeof(*cand_ids->data));
4265 if (!new_ids) {
4266 err = -ENOMEM;
4267 goto err_out;
4268 }
4269 cand_ids->data = new_ids;
4270 cand_ids->data[cand_ids->len++] = i;
4271 }
4272 }
4273 return cand_ids;
4274err_out:
4275 bpf_core_free_cands(cand_ids);
4276 return ERR_PTR(err);
4277}
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294static int bpf_core_fields_are_compat(const struct btf *local_btf,
4295 __u32 local_id,
4296 const struct btf *targ_btf,
4297 __u32 targ_id)
4298{
4299 const struct btf_type *local_type, *targ_type;
4300
4301recur:
4302 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
4303 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
4304 if (!local_type || !targ_type)
4305 return -EINVAL;
4306
4307 if (btf_is_composite(local_type) && btf_is_composite(targ_type))
4308 return 1;
4309 if (btf_kind(local_type) != btf_kind(targ_type))
4310 return 0;
4311
4312 switch (btf_kind(local_type)) {
4313 case BTF_KIND_PTR:
4314 return 1;
4315 case BTF_KIND_FWD:
4316 case BTF_KIND_ENUM: {
4317 const char *local_name, *targ_name;
4318 size_t local_len, targ_len;
4319
4320 local_name = btf__name_by_offset(local_btf,
4321 local_type->name_off);
4322 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off);
4323 local_len = bpf_core_essential_name_len(local_name);
4324 targ_len = bpf_core_essential_name_len(targ_name);
4325
4326 return local_len == 0 || targ_len == 0 ||
4327 (local_len == targ_len &&
4328 strncmp(local_name, targ_name, local_len) == 0);
4329 }
4330 case BTF_KIND_INT:
4331
4332
4333
4334 return btf_int_offset(local_type) == 0 &&
4335 btf_int_offset(targ_type) == 0;
4336 case BTF_KIND_ARRAY:
4337 local_id = btf_array(local_type)->type;
4338 targ_id = btf_array(targ_type)->type;
4339 goto recur;
4340 default:
4341 pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n",
4342 btf_kind(local_type), local_id, targ_id);
4343 return 0;
4344 }
4345}
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363static int bpf_core_match_member(const struct btf *local_btf,
4364 const struct bpf_core_accessor *local_acc,
4365 const struct btf *targ_btf,
4366 __u32 targ_id,
4367 struct bpf_core_spec *spec,
4368 __u32 *next_targ_id)
4369{
4370 const struct btf_type *local_type, *targ_type;
4371 const struct btf_member *local_member, *m;
4372 const char *local_name, *targ_name;
4373 __u32 local_id;
4374 int i, n, found;
4375
4376 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
4377 if (!targ_type)
4378 return -EINVAL;
4379 if (!btf_is_composite(targ_type))
4380 return 0;
4381
4382 local_id = local_acc->type_id;
4383 local_type = btf__type_by_id(local_btf, local_id);
4384 local_member = btf_members(local_type) + local_acc->idx;
4385 local_name = btf__name_by_offset(local_btf, local_member->name_off);
4386
4387 n = btf_vlen(targ_type);
4388 m = btf_members(targ_type);
4389 for (i = 0; i < n; i++, m++) {
4390 __u32 bit_offset;
4391
4392 bit_offset = btf_member_bit_offset(targ_type, i);
4393
4394
4395 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4396 return -E2BIG;
4397
4398
4399 spec->bit_offset += bit_offset;
4400 spec->raw_spec[spec->raw_len++] = i;
4401
4402 targ_name = btf__name_by_offset(targ_btf, m->name_off);
4403 if (str_is_empty(targ_name)) {
4404
4405 found = bpf_core_match_member(local_btf, local_acc,
4406 targ_btf, m->type,
4407 spec, next_targ_id);
4408 if (found)
4409 return found;
4410 } else if (strcmp(local_name, targ_name) == 0) {
4411
4412 struct bpf_core_accessor *targ_acc;
4413
4414 targ_acc = &spec->spec[spec->len++];
4415 targ_acc->type_id = targ_id;
4416 targ_acc->idx = i;
4417 targ_acc->name = targ_name;
4418
4419 *next_targ_id = m->type;
4420 found = bpf_core_fields_are_compat(local_btf,
4421 local_member->type,
4422 targ_btf, m->type);
4423 if (!found)
4424 spec->len--;
4425 return found;
4426 }
4427
4428 spec->bit_offset -= bit_offset;
4429 spec->raw_len--;
4430 }
4431
4432 return 0;
4433}
4434
4435
4436
4437
4438
4439static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
4440 const struct btf *targ_btf, __u32 targ_id,
4441 struct bpf_core_spec *targ_spec)
4442{
4443 const struct btf_type *targ_type;
4444 const struct bpf_core_accessor *local_acc;
4445 struct bpf_core_accessor *targ_acc;
4446 int i, sz, matched;
4447
4448 memset(targ_spec, 0, sizeof(*targ_spec));
4449 targ_spec->btf = targ_btf;
4450
4451 local_acc = &local_spec->spec[0];
4452 targ_acc = &targ_spec->spec[0];
4453
4454 for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {
4455 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,
4456 &targ_id);
4457 if (!targ_type)
4458 return -EINVAL;
4459
4460 if (local_acc->name) {
4461 matched = bpf_core_match_member(local_spec->btf,
4462 local_acc,
4463 targ_btf, targ_id,
4464 targ_spec, &targ_id);
4465 if (matched <= 0)
4466 return matched;
4467 } else {
4468
4469
4470
4471
4472 if (i > 0) {
4473 const struct btf_array *a;
4474 bool flex;
4475
4476 if (!btf_is_array(targ_type))
4477 return 0;
4478
4479 a = btf_array(targ_type);
4480 flex = is_flex_arr(targ_btf, targ_acc - 1, a);
4481 if (!flex && local_acc->idx >= a->nelems)
4482 return 0;
4483 if (!skip_mods_and_typedefs(targ_btf, a->type,
4484 &targ_id))
4485 return -EINVAL;
4486 }
4487
4488
4489 if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4490 return -E2BIG;
4491
4492 targ_acc->type_id = targ_id;
4493 targ_acc->idx = local_acc->idx;
4494 targ_acc->name = NULL;
4495 targ_spec->len++;
4496 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
4497 targ_spec->raw_len++;
4498
4499 sz = btf__resolve_size(targ_btf, targ_id);
4500 if (sz < 0)
4501 return sz;
4502 targ_spec->bit_offset += local_acc->idx * sz * 8;
4503 }
4504 }
4505
4506 return 1;
4507}
4508
4509static int bpf_core_calc_field_relo(const struct bpf_program *prog,
4510 const struct bpf_field_reloc *relo,
4511 const struct bpf_core_spec *spec,
4512 __u32 *val, bool *validate)
4513{
4514 const struct bpf_core_accessor *acc = &spec->spec[spec->len - 1];
4515 const struct btf_type *t = btf__type_by_id(spec->btf, acc->type_id);
4516 __u32 byte_off, byte_sz, bit_off, bit_sz;
4517 const struct btf_member *m;
4518 const struct btf_type *mt;
4519 bool bitfield;
4520 __s64 sz;
4521
4522
4523 if (!acc->name) {
4524 if (relo->kind == BPF_FIELD_BYTE_OFFSET) {
4525 *val = spec->bit_offset / 8;
4526 } else if (relo->kind == BPF_FIELD_BYTE_SIZE) {
4527 sz = btf__resolve_size(spec->btf, acc->type_id);
4528 if (sz < 0)
4529 return -EINVAL;
4530 *val = sz;
4531 } else {
4532 pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",
4533 bpf_program__title(prog, false),
4534 relo->kind, relo->insn_off / 8);
4535 return -EINVAL;
4536 }
4537 if (validate)
4538 *validate = true;
4539 return 0;
4540 }
4541
4542 m = btf_members(t) + acc->idx;
4543 mt = skip_mods_and_typedefs(spec->btf, m->type, NULL);
4544 bit_off = spec->bit_offset;
4545 bit_sz = btf_member_bitfield_size(t, acc->idx);
4546
4547 bitfield = bit_sz > 0;
4548 if (bitfield) {
4549 byte_sz = mt->size;
4550 byte_off = bit_off / 8 / byte_sz * byte_sz;
4551
4552 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {
4553 if (byte_sz >= 8) {
4554
4555 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",
4556 bpf_program__title(prog, false),
4557 relo->kind, relo->insn_off / 8);
4558 return -E2BIG;
4559 }
4560 byte_sz *= 2;
4561 byte_off = bit_off / 8 / byte_sz * byte_sz;
4562 }
4563 } else {
4564 sz = btf__resolve_size(spec->btf, m->type);
4565 if (sz < 0)
4566 return -EINVAL;
4567 byte_sz = sz;
4568 byte_off = spec->bit_offset / 8;
4569 bit_sz = byte_sz * 8;
4570 }
4571
4572
4573
4574
4575
4576 if (validate)
4577 *validate = !bitfield;
4578
4579 switch (relo->kind) {
4580 case BPF_FIELD_BYTE_OFFSET:
4581 *val = byte_off;
4582 break;
4583 case BPF_FIELD_BYTE_SIZE:
4584 *val = byte_sz;
4585 break;
4586 case BPF_FIELD_SIGNED:
4587
4588 *val = btf_is_enum(mt) ||
4589 (btf_int_encoding(mt) & BTF_INT_SIGNED);
4590 if (validate)
4591 *validate = true;
4592 break;
4593 case BPF_FIELD_LSHIFT_U64:
4594#if __BYTE_ORDER == __LITTLE_ENDIAN
4595 *val = 64 - (bit_off + bit_sz - byte_off * 8);
4596#else
4597 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8);
4598#endif
4599 break;
4600 case BPF_FIELD_RSHIFT_U64:
4601 *val = 64 - bit_sz;
4602 if (validate)
4603 *validate = true;
4604 break;
4605 case BPF_FIELD_EXISTS:
4606 default:
4607 pr_warn("prog '%s': unknown relo %d at insn #%d\n",
4608 bpf_program__title(prog, false),
4609 relo->kind, relo->insn_off / 8);
4610 return -EINVAL;
4611 }
4612
4613 return 0;
4614}
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630static int bpf_core_reloc_insn(struct bpf_program *prog,
4631 const struct bpf_field_reloc *relo,
4632 int relo_idx,
4633 const struct bpf_core_spec *local_spec,
4634 const struct bpf_core_spec *targ_spec)
4635{
4636 __u32 orig_val, new_val;
4637 struct bpf_insn *insn;
4638 bool validate = true;
4639 int insn_idx, err;
4640 __u8 class;
4641
4642 if (relo->insn_off % sizeof(struct bpf_insn))
4643 return -EINVAL;
4644 insn_idx = relo->insn_off / sizeof(struct bpf_insn);
4645 insn = &prog->insns[insn_idx];
4646 class = BPF_CLASS(insn->code);
4647
4648 if (relo->kind == BPF_FIELD_EXISTS) {
4649 orig_val = 1;
4650 new_val = targ_spec ? 1 : 0;
4651 } else if (!targ_spec) {
4652 pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n",
4653 bpf_program__title(prog, false), relo_idx, insn_idx);
4654 insn->code = BPF_JMP | BPF_CALL;
4655 insn->dst_reg = 0;
4656 insn->src_reg = 0;
4657 insn->off = 0;
4658
4659
4660
4661
4662 insn->imm = 195896080;
4663 return 0;
4664 } else {
4665 err = bpf_core_calc_field_relo(prog, relo, local_spec,
4666 &orig_val, &validate);
4667 if (err)
4668 return err;
4669 err = bpf_core_calc_field_relo(prog, relo, targ_spec,
4670 &new_val, NULL);
4671 if (err)
4672 return err;
4673 }
4674
4675 switch (class) {
4676 case BPF_ALU:
4677 case BPF_ALU64:
4678 if (BPF_SRC(insn->code) != BPF_K)
4679 return -EINVAL;
4680 if (validate && insn->imm != orig_val) {
4681 pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n",
4682 bpf_program__title(prog, false), relo_idx,
4683 insn_idx, insn->imm, orig_val, new_val);
4684 return -EINVAL;
4685 }
4686 orig_val = insn->imm;
4687 insn->imm = new_val;
4688 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n",
4689 bpf_program__title(prog, false), relo_idx, insn_idx,
4690 orig_val, new_val);
4691 break;
4692 case BPF_LDX:
4693 case BPF_ST:
4694 case BPF_STX:
4695 if (validate && insn->off != orig_val) {
4696 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LD/LDX/ST/STX) value: got %u, exp %u -> %u\n",
4697 bpf_program__title(prog, false), relo_idx,
4698 insn_idx, insn->off, orig_val, new_val);
4699 return -EINVAL;
4700 }
4701 if (new_val > SHRT_MAX) {
4702 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n",
4703 bpf_program__title(prog, false), relo_idx,
4704 insn_idx, new_val);
4705 return -ERANGE;
4706 }
4707 orig_val = insn->off;
4708 insn->off = new_val;
4709 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",
4710 bpf_program__title(prog, false), relo_idx, insn_idx,
4711 orig_val, new_val);
4712 break;
4713 default:
4714 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n",
4715 bpf_program__title(prog, false), relo_idx,
4716 insn_idx, insn->code, insn->src_reg, insn->dst_reg,
4717 insn->off, insn->imm);
4718 return -EINVAL;
4719 }
4720
4721 return 0;
4722}
4723
4724
4725
4726
4727
4728static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec)
4729{
4730 const struct btf_type *t;
4731 const char *s;
4732 __u32 type_id;
4733 int i;
4734
4735 type_id = spec->spec[0].type_id;
4736 t = btf__type_by_id(spec->btf, type_id);
4737 s = btf__name_by_offset(spec->btf, t->name_off);
4738 libbpf_print(level, "[%u] %s + ", type_id, s);
4739
4740 for (i = 0; i < spec->raw_len; i++)
4741 libbpf_print(level, "%d%s", spec->raw_spec[i],
4742 i == spec->raw_len - 1 ? " => " : ":");
4743
4744 libbpf_print(level, "%u.%u @ &x",
4745 spec->bit_offset / 8, spec->bit_offset % 8);
4746
4747 for (i = 0; i < spec->len; i++) {
4748 if (spec->spec[i].name)
4749 libbpf_print(level, ".%s", spec->spec[i].name);
4750 else
4751 libbpf_print(level, "[%u]", spec->spec[i].idx);
4752 }
4753
4754}
4755
4756static size_t bpf_core_hash_fn(const void *key, void *ctx)
4757{
4758 return (size_t)key;
4759}
4760
4761static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
4762{
4763 return k1 == k2;
4764}
4765
4766static void *u32_as_hash_key(__u32 x)
4767{
4768 return (void *)(uintptr_t)x;
4769}
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821static int bpf_core_reloc_field(struct bpf_program *prog,
4822 const struct bpf_field_reloc *relo,
4823 int relo_idx,
4824 const struct btf *local_btf,
4825 const struct btf *targ_btf,
4826 struct hashmap *cand_cache)
4827{
4828 const char *prog_name = bpf_program__title(prog, false);
4829 struct bpf_core_spec local_spec, cand_spec, targ_spec;
4830 const void *type_key = u32_as_hash_key(relo->type_id);
4831 const struct btf_type *local_type, *cand_type;
4832 const char *local_name, *cand_name;
4833 struct ids_vec *cand_ids;
4834 __u32 local_id, cand_id;
4835 const char *spec_str;
4836 int i, j, err;
4837
4838 local_id = relo->type_id;
4839 local_type = btf__type_by_id(local_btf, local_id);
4840 if (!local_type)
4841 return -EINVAL;
4842
4843 local_name = btf__name_by_offset(local_btf, local_type->name_off);
4844 if (str_is_empty(local_name))
4845 return -EINVAL;
4846
4847 spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
4848 if (str_is_empty(spec_str))
4849 return -EINVAL;
4850
4851 err = bpf_core_spec_parse(local_btf, local_id, spec_str, &local_spec);
4852 if (err) {
4853 pr_warn("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n",
4854 prog_name, relo_idx, local_id, local_name, spec_str,
4855 err);
4856 return -EINVAL;
4857 }
4858
4859 pr_debug("prog '%s': relo #%d: kind %d, spec is ", prog_name, relo_idx,
4860 relo->kind);
4861 bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec);
4862 libbpf_print(LIBBPF_DEBUG, "\n");
4863
4864 if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) {
4865 cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf);
4866 if (IS_ERR(cand_ids)) {
4867 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld",
4868 prog_name, relo_idx, local_id, local_name,
4869 PTR_ERR(cand_ids));
4870 return PTR_ERR(cand_ids);
4871 }
4872 err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL);
4873 if (err) {
4874 bpf_core_free_cands(cand_ids);
4875 return err;
4876 }
4877 }
4878
4879 for (i = 0, j = 0; i < cand_ids->len; i++) {
4880 cand_id = cand_ids->data[i];
4881 cand_type = btf__type_by_id(targ_btf, cand_id);
4882 cand_name = btf__name_by_offset(targ_btf, cand_type->name_off);
4883
4884 err = bpf_core_spec_match(&local_spec, targ_btf,
4885 cand_id, &cand_spec);
4886 pr_debug("prog '%s': relo #%d: matching candidate #%d %s against spec ",
4887 prog_name, relo_idx, i, cand_name);
4888 bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec);
4889 libbpf_print(LIBBPF_DEBUG, ": %d\n", err);
4890 if (err < 0) {
4891 pr_warn("prog '%s': relo #%d: matching error: %d\n",
4892 prog_name, relo_idx, err);
4893 return err;
4894 }
4895 if (err == 0)
4896 continue;
4897
4898 if (j == 0) {
4899 targ_spec = cand_spec;
4900 } else if (cand_spec.bit_offset != targ_spec.bit_offset) {
4901
4902
4903
4904 pr_warn("prog '%s': relo #%d: offset ambiguity: %u != %u\n",
4905 prog_name, relo_idx, cand_spec.bit_offset,
4906 targ_spec.bit_offset);
4907 return -EINVAL;
4908 }
4909
4910 cand_ids->data[j++] = cand_spec.spec[0].type_id;
4911 }
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921 if (j > 0)
4922 cand_ids->len = j;
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935 if (j == 0)
4936 pr_debug("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n",
4937 prog_name, relo_idx, local_id, local_name, spec_str);
4938
4939
4940 err = bpf_core_reloc_insn(prog, relo, relo_idx, &local_spec,
4941 j ? &targ_spec : NULL);
4942 if (err) {
4943 pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",
4944 prog_name, relo_idx, relo->insn_off, err);
4945 return -EINVAL;
4946 }
4947
4948 return 0;
4949}
4950
4951static int
4952bpf_core_reloc_fields(struct bpf_object *obj, const char *targ_btf_path)
4953{
4954 const struct btf_ext_info_sec *sec;
4955 const struct bpf_field_reloc *rec;
4956 const struct btf_ext_info *seg;
4957 struct hashmap_entry *entry;
4958 struct hashmap *cand_cache = NULL;
4959 struct bpf_program *prog;
4960 struct btf *targ_btf;
4961 const char *sec_name;
4962 int i, err = 0;
4963
4964 if (targ_btf_path)
4965 targ_btf = btf__parse_elf(targ_btf_path, NULL);
4966 else
4967 targ_btf = obj->btf_vmlinux;
4968 if (IS_ERR_OR_NULL(targ_btf)) {
4969 pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf));
4970 return PTR_ERR(targ_btf);
4971 }
4972
4973 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
4974 if (IS_ERR(cand_cache)) {
4975 err = PTR_ERR(cand_cache);
4976 goto out;
4977 }
4978
4979 seg = &obj->btf_ext->field_reloc_info;
4980 for_each_btf_ext_sec(seg, sec) {
4981 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
4982 if (str_is_empty(sec_name)) {
4983 err = -EINVAL;
4984 goto out;
4985 }
4986 prog = NULL;
4987 for (i = 0; i < obj->nr_programs; i++) {
4988 if (!strcmp(obj->programs[i].section_name, sec_name)) {
4989 prog = &obj->programs[i];
4990 break;
4991 }
4992 }
4993 if (!prog) {
4994 pr_warn("failed to find program '%s' for CO-RE offset relocation\n",
4995 sec_name);
4996 err = -EINVAL;
4997 goto out;
4998 }
4999
5000 pr_debug("prog '%s': performing %d CO-RE offset relocs\n",
5001 sec_name, sec->num_info);
5002
5003 for_each_btf_ext_rec(seg, sec, i, rec) {
5004 err = bpf_core_reloc_field(prog, rec, i, obj->btf,
5005 targ_btf, cand_cache);
5006 if (err) {
5007 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
5008 sec_name, i, err);
5009 goto out;
5010 }
5011 }
5012 }
5013
5014out:
5015
5016 if (targ_btf != obj->btf_vmlinux)
5017 btf__free(targ_btf);
5018 if (!IS_ERR_OR_NULL(cand_cache)) {
5019 hashmap__for_each_entry(cand_cache, entry, i) {
5020 bpf_core_free_cands(entry->value);
5021 }
5022 hashmap__free(cand_cache);
5023 }
5024 return err;
5025}
5026
5027static int
5028bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
5029{
5030 int err = 0;
5031
5032 if (obj->btf_ext->field_reloc_info.len)
5033 err = bpf_core_reloc_fields(obj, targ_btf_path);
5034
5035 return err;
5036}
5037
5038static int
5039bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
5040 struct reloc_desc *relo)
5041{
5042 struct bpf_insn *insn, *new_insn;
5043 struct bpf_program *text;
5044 size_t new_cnt;
5045 int err;
5046
5047 if (prog->idx != obj->efile.text_shndx && prog->main_prog_cnt == 0) {
5048 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
5049 if (!text) {
5050 pr_warn("no .text section found yet relo into text exist\n");
5051 return -LIBBPF_ERRNO__RELOC;
5052 }
5053 new_cnt = prog->insns_cnt + text->insns_cnt;
5054 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));
5055 if (!new_insn) {
5056 pr_warn("oom in prog realloc\n");
5057 return -ENOMEM;
5058 }
5059 prog->insns = new_insn;
5060
5061 if (obj->btf_ext) {
5062 err = bpf_program_reloc_btf_ext(prog, obj,
5063 text->section_name,
5064 prog->insns_cnt);
5065 if (err)
5066 return err;
5067 }
5068
5069 memcpy(new_insn + prog->insns_cnt, text->insns,
5070 text->insns_cnt * sizeof(*insn));
5071 prog->main_prog_cnt = prog->insns_cnt;
5072 prog->insns_cnt = new_cnt;
5073 pr_debug("added %zd insn from %s to prog %s\n",
5074 text->insns_cnt, text->section_name,
5075 prog->section_name);
5076 }
5077
5078 insn = &prog->insns[relo->insn_idx];
5079 insn->imm += relo->sym_off / 8 + prog->main_prog_cnt - relo->insn_idx;
5080 return 0;
5081}
5082
5083static int
5084bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
5085{
5086 int i, err;
5087
5088 if (!prog)
5089 return 0;
5090
5091 if (obj->btf_ext) {
5092 err = bpf_program_reloc_btf_ext(prog, obj,
5093 prog->section_name, 0);
5094 if (err)
5095 return err;
5096 }
5097
5098 if (!prog->reloc_desc)
5099 return 0;
5100
5101 for (i = 0; i < prog->nr_reloc; i++) {
5102 struct reloc_desc *relo = &prog->reloc_desc[i];
5103 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
5104 struct extern_desc *ext;
5105
5106 if (relo->insn_idx + 1 >= (int)prog->insns_cnt) {
5107 pr_warn("relocation out of range: '%s'\n",
5108 prog->section_name);
5109 return -LIBBPF_ERRNO__RELOC;
5110 }
5111
5112 switch (relo->type) {
5113 case RELO_LD64:
5114 insn[0].src_reg = BPF_PSEUDO_MAP_FD;
5115 insn[0].imm = obj->maps[relo->map_idx].fd;
5116 break;
5117 case RELO_DATA:
5118 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
5119 insn[1].imm = insn[0].imm + relo->sym_off;
5120 insn[0].imm = obj->maps[relo->map_idx].fd;
5121 break;
5122 case RELO_EXTERN:
5123 ext = &obj->externs[relo->sym_off];
5124 if (ext->type == EXT_KCFG) {
5125 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
5126 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
5127 insn[1].imm = ext->kcfg.data_off;
5128 } else {
5129 insn[0].imm = (__u32)ext->ksym.addr;
5130 insn[1].imm = ext->ksym.addr >> 32;
5131 }
5132 break;
5133 case RELO_CALL:
5134 err = bpf_program__reloc_text(prog, obj, relo);
5135 if (err)
5136 return err;
5137 break;
5138 default:
5139 pr_warn("relo #%d: bad relo type %d\n", i, relo->type);
5140 return -EINVAL;
5141 }
5142 }
5143
5144 zfree(&prog->reloc_desc);
5145 prog->nr_reloc = 0;
5146 return 0;
5147}
5148
5149static int
5150bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
5151{
5152 struct bpf_program *prog;
5153 size_t i;
5154 int err;
5155
5156 if (obj->btf_ext) {
5157 err = bpf_object__relocate_core(obj, targ_btf_path);
5158 if (err) {
5159 pr_warn("failed to perform CO-RE relocations: %d\n",
5160 err);
5161 return err;
5162 }
5163 }
5164
5165
5166
5167 for (i = 0; i < obj->nr_programs; i++) {
5168 prog = &obj->programs[i];
5169 if (prog->idx != obj->efile.text_shndx)
5170 continue;
5171
5172 err = bpf_program__relocate(prog, obj);
5173 if (err) {
5174 pr_warn("failed to relocate '%s'\n", prog->section_name);
5175 return err;
5176 }
5177 break;
5178 }
5179
5180
5181
5182 for (i = 0; i < obj->nr_programs; i++) {
5183 prog = &obj->programs[i];
5184 if (prog->idx == obj->efile.text_shndx)
5185 continue;
5186
5187 err = bpf_program__relocate(prog, obj);
5188 if (err) {
5189 pr_warn("failed to relocate '%s'\n", prog->section_name);
5190 return err;
5191 }
5192 }
5193 return 0;
5194}
5195
5196static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
5197 GElf_Shdr *shdr, Elf_Data *data);
5198
5199static int bpf_object__collect_map_relos(struct bpf_object *obj,
5200 GElf_Shdr *shdr, Elf_Data *data)
5201{
5202 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
5203 int i, j, nrels, new_sz;
5204 const struct btf_var_secinfo *vi = NULL;
5205 const struct btf_type *sec, *var, *def;
5206 struct bpf_map *map = NULL, *targ_map;
5207 const struct btf_member *member;
5208 const char *name, *mname;
5209 Elf_Data *symbols;
5210 unsigned int moff;
5211 GElf_Sym sym;
5212 GElf_Rel rel;
5213 void *tmp;
5214
5215 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
5216 return -EINVAL;
5217 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
5218 if (!sec)
5219 return -EINVAL;
5220
5221 symbols = obj->efile.symbols;
5222 nrels = shdr->sh_size / shdr->sh_entsize;
5223 for (i = 0; i < nrels; i++) {
5224 if (!gelf_getrel(data, i, &rel)) {
5225 pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
5226 return -LIBBPF_ERRNO__FORMAT;
5227 }
5228 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
5229 pr_warn(".maps relo #%d: symbol %zx not found\n",
5230 i, (size_t)GELF_R_SYM(rel.r_info));
5231 return -LIBBPF_ERRNO__FORMAT;
5232 }
5233 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
5234 sym.st_name) ? : "<?>";
5235 if (sym.st_shndx != obj->efile.btf_maps_shndx) {
5236 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
5237 i, name);
5238 return -LIBBPF_ERRNO__RELOC;
5239 }
5240
5241 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n",
5242 i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value,
5243 (size_t)rel.r_offset, sym.st_name, name);
5244
5245 for (j = 0; j < obj->nr_maps; j++) {
5246 map = &obj->maps[j];
5247 if (map->sec_idx != obj->efile.btf_maps_shndx)
5248 continue;
5249
5250 vi = btf_var_secinfos(sec) + map->btf_var_idx;
5251 if (vi->offset <= rel.r_offset &&
5252 rel.r_offset + bpf_ptr_sz <= vi->offset + vi->size)
5253 break;
5254 }
5255 if (j == obj->nr_maps) {
5256 pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n",
5257 i, name, (size_t)rel.r_offset);
5258 return -EINVAL;
5259 }
5260
5261 if (!bpf_map_type__is_map_in_map(map->def.type))
5262 return -EINVAL;
5263 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
5264 map->def.key_size != sizeof(int)) {
5265 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
5266 i, map->name, sizeof(int));
5267 return -EINVAL;
5268 }
5269
5270 targ_map = bpf_object__find_map_by_name(obj, name);
5271 if (!targ_map)
5272 return -ESRCH;
5273
5274 var = btf__type_by_id(obj->btf, vi->type);
5275 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
5276 if (btf_vlen(def) == 0)
5277 return -EINVAL;
5278 member = btf_members(def) + btf_vlen(def) - 1;
5279 mname = btf__name_by_offset(obj->btf, member->name_off);
5280 if (strcmp(mname, "values"))
5281 return -EINVAL;
5282
5283 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
5284 if (rel.r_offset - vi->offset < moff)
5285 return -EINVAL;
5286
5287 moff = rel.r_offset - vi->offset - moff;
5288
5289
5290
5291 if (moff % bpf_ptr_sz)
5292 return -EINVAL;
5293 moff /= bpf_ptr_sz;
5294 if (moff >= map->init_slots_sz) {
5295 new_sz = moff + 1;
5296 tmp = realloc(map->init_slots, new_sz * host_ptr_sz);
5297 if (!tmp)
5298 return -ENOMEM;
5299 map->init_slots = tmp;
5300 memset(map->init_slots + map->init_slots_sz, 0,
5301 (new_sz - map->init_slots_sz) * host_ptr_sz);
5302 map->init_slots_sz = new_sz;
5303 }
5304 map->init_slots[moff] = targ_map;
5305
5306 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n",
5307 i, map->name, moff, name);
5308 }
5309
5310 return 0;
5311}
5312
5313static int bpf_object__collect_reloc(struct bpf_object *obj)
5314{
5315 int i, err;
5316
5317 if (!obj_elf_valid(obj)) {
5318 pr_warn("Internal error: elf object is closed\n");
5319 return -LIBBPF_ERRNO__INTERNAL;
5320 }
5321
5322 for (i = 0; i < obj->efile.nr_reloc_sects; i++) {
5323 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr;
5324 Elf_Data *data = obj->efile.reloc_sects[i].data;
5325 int idx = shdr->sh_info;
5326 struct bpf_program *prog;
5327
5328 if (shdr->sh_type != SHT_REL) {
5329 pr_warn("internal error at %d\n", __LINE__);
5330 return -LIBBPF_ERRNO__INTERNAL;
5331 }
5332
5333 if (idx == obj->efile.st_ops_shndx) {
5334 err = bpf_object__collect_st_ops_relos(obj, shdr, data);
5335 } else if (idx == obj->efile.btf_maps_shndx) {
5336 err = bpf_object__collect_map_relos(obj, shdr, data);
5337 } else {
5338 prog = bpf_object__find_prog_by_idx(obj, idx);
5339 if (!prog) {
5340 pr_warn("relocation failed: no prog in section(%d)\n", idx);
5341 return -LIBBPF_ERRNO__RELOC;
5342 }
5343 err = bpf_program__collect_reloc(prog, shdr, data, obj);
5344 }
5345 if (err)
5346 return err;
5347 }
5348 return 0;
5349}
5350
5351static int
5352load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
5353 char *license, __u32 kern_version, int *pfd)
5354{
5355 struct bpf_load_program_attr load_attr;
5356 char *cp, errmsg[STRERR_BUFSIZE];
5357 size_t log_buf_size = 0;
5358 char *log_buf = NULL;
5359 int btf_fd, ret;
5360
5361 if (!insns || !insns_cnt)
5362 return -EINVAL;
5363
5364 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
5365 load_attr.prog_type = prog->type;
5366
5367 if (!prog->caps->exp_attach_type && prog->sec_def &&
5368 prog->sec_def->is_exp_attach_type_optional)
5369 load_attr.expected_attach_type = 0;
5370 else
5371 load_attr.expected_attach_type = prog->expected_attach_type;
5372 if (prog->caps->name)
5373 load_attr.name = prog->name;
5374 load_attr.insns = insns;
5375 load_attr.insns_cnt = insns_cnt;
5376 load_attr.license = license;
5377 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
5378 prog->type == BPF_PROG_TYPE_LSM) {
5379 load_attr.attach_btf_id = prog->attach_btf_id;
5380 } else if (prog->type == BPF_PROG_TYPE_TRACING ||
5381 prog->type == BPF_PROG_TYPE_EXT) {
5382 load_attr.attach_prog_fd = prog->attach_prog_fd;
5383 load_attr.attach_btf_id = prog->attach_btf_id;
5384 } else {
5385 load_attr.kern_version = kern_version;
5386 load_attr.prog_ifindex = prog->prog_ifindex;
5387 }
5388
5389 btf_fd = bpf_object__btf_fd(prog->obj);
5390 if (btf_fd >= 0 && prog->obj->caps.btf_func) {
5391 load_attr.prog_btf_fd = btf_fd;
5392 load_attr.func_info = prog->func_info;
5393 load_attr.func_info_rec_size = prog->func_info_rec_size;
5394 load_attr.func_info_cnt = prog->func_info_cnt;
5395 load_attr.line_info = prog->line_info;
5396 load_attr.line_info_rec_size = prog->line_info_rec_size;
5397 load_attr.line_info_cnt = prog->line_info_cnt;
5398 }
5399 load_attr.log_level = prog->log_level;
5400 load_attr.prog_flags = prog->prog_flags;
5401
5402retry_load:
5403 if (log_buf_size) {
5404 log_buf = malloc(log_buf_size);
5405 if (!log_buf)
5406 return -ENOMEM;
5407
5408 *log_buf = 0;
5409 }
5410
5411 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size);
5412
5413 if (ret >= 0) {
5414 if (log_buf && load_attr.log_level)
5415 pr_debug("verifier log:\n%s", log_buf);
5416 *pfd = ret;
5417 ret = 0;
5418 goto out;
5419 }
5420
5421 if (!log_buf || errno == ENOSPC) {
5422 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE,
5423 log_buf_size << 1);
5424
5425 free(log_buf);
5426 goto retry_load;
5427 }
5428 ret = -errno;
5429 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
5430 pr_warn("load bpf program failed: %s\n", cp);
5431 pr_perm_msg(ret);
5432
5433 if (log_buf && log_buf[0] != '\0') {
5434 ret = -LIBBPF_ERRNO__VERIFY;
5435 pr_warn("-- BEGIN DUMP LOG ---\n");
5436 pr_warn("\n%s\n", log_buf);
5437 pr_warn("-- END LOG --\n");
5438 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
5439 pr_warn("Program too large (%zu insns), at most %d insns\n",
5440 load_attr.insns_cnt, BPF_MAXINSNS);
5441 ret = -LIBBPF_ERRNO__PROG2BIG;
5442 } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
5443
5444 int fd;
5445
5446 load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
5447 load_attr.expected_attach_type = 0;
5448 fd = bpf_load_program_xattr(&load_attr, NULL, 0);
5449 if (fd >= 0) {
5450 close(fd);
5451 ret = -LIBBPF_ERRNO__PROGTYPE;
5452 goto out;
5453 }
5454 }
5455
5456out:
5457 free(log_buf);
5458 return ret;
5459}
5460
5461static int libbpf_find_attach_btf_id(struct bpf_program *prog);
5462
5463int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
5464{
5465 int err = 0, fd, i, btf_id;
5466
5467 if (prog->obj->loaded) {
5468 pr_warn("prog '%s'('%s'): can't load after object was loaded\n",
5469 prog->name, prog->section_name);
5470 return -EINVAL;
5471 }
5472
5473 if ((prog->type == BPF_PROG_TYPE_TRACING ||
5474 prog->type == BPF_PROG_TYPE_LSM ||
5475 prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) {
5476 btf_id = libbpf_find_attach_btf_id(prog);
5477 if (btf_id <= 0)
5478 return btf_id;
5479 prog->attach_btf_id = btf_id;
5480 }
5481
5482 if (prog->instances.nr < 0 || !prog->instances.fds) {
5483 if (prog->preprocessor) {
5484 pr_warn("Internal error: can't load program '%s'\n",
5485 prog->section_name);
5486 return -LIBBPF_ERRNO__INTERNAL;
5487 }
5488
5489 prog->instances.fds = malloc(sizeof(int));
5490 if (!prog->instances.fds) {
5491 pr_warn("Not enough memory for BPF fds\n");
5492 return -ENOMEM;
5493 }
5494 prog->instances.nr = 1;
5495 prog->instances.fds[0] = -1;
5496 }
5497
5498 if (!prog->preprocessor) {
5499 if (prog->instances.nr != 1) {
5500 pr_warn("Program '%s' is inconsistent: nr(%d) != 1\n",
5501 prog->section_name, prog->instances.nr);
5502 }
5503 err = load_program(prog, prog->insns, prog->insns_cnt,
5504 license, kern_ver, &fd);
5505 if (!err)
5506 prog->instances.fds[0] = fd;
5507 goto out;
5508 }
5509
5510 for (i = 0; i < prog->instances.nr; i++) {
5511 struct bpf_prog_prep_result result;
5512 bpf_program_prep_t preprocessor = prog->preprocessor;
5513
5514 memset(&result, 0, sizeof(result));
5515 err = preprocessor(prog, i, prog->insns,
5516 prog->insns_cnt, &result);
5517 if (err) {
5518 pr_warn("Preprocessing the %dth instance of program '%s' failed\n",
5519 i, prog->section_name);
5520 goto out;
5521 }
5522
5523 if (!result.new_insn_ptr || !result.new_insn_cnt) {
5524 pr_debug("Skip loading the %dth instance of program '%s'\n",
5525 i, prog->section_name);
5526 prog->instances.fds[i] = -1;
5527 if (result.pfd)
5528 *result.pfd = -1;
5529 continue;
5530 }
5531
5532 err = load_program(prog, result.new_insn_ptr,
5533 result.new_insn_cnt, license, kern_ver, &fd);
5534 if (err) {
5535 pr_warn("Loading the %dth instance of program '%s' failed\n",
5536 i, prog->section_name);
5537 goto out;
5538 }
5539
5540 if (result.pfd)
5541 *result.pfd = fd;
5542 prog->instances.fds[i] = fd;
5543 }
5544out:
5545 if (err)
5546 pr_warn("failed to load program '%s'\n", prog->section_name);
5547 zfree(&prog->insns);
5548 prog->insns_cnt = 0;
5549 return err;
5550}
5551
5552static bool bpf_program__is_function_storage(const struct bpf_program *prog,
5553 const struct bpf_object *obj)
5554{
5555 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls;
5556}
5557
5558static int
5559bpf_object__load_progs(struct bpf_object *obj, int log_level)
5560{
5561 struct bpf_program *prog;
5562 size_t i;
5563 int err;
5564
5565 for (i = 0; i < obj->nr_programs; i++) {
5566 prog = &obj->programs[i];
5567 if (bpf_program__is_function_storage(prog, obj))
5568 continue;
5569 if (!prog->load) {
5570 pr_debug("prog '%s'('%s'): skipped loading\n",
5571 prog->name, prog->section_name);
5572 continue;
5573 }
5574 prog->log_level |= log_level;
5575 err = bpf_program__load(prog, obj->license, obj->kern_version);
5576 if (err)
5577 return err;
5578 }
5579 return 0;
5580}
5581
5582static const struct bpf_sec_def *find_sec_def(const char *sec_name);
5583
5584static struct bpf_object *
5585__bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,
5586 const struct bpf_object_open_opts *opts)
5587{
5588 const char *obj_name, *kconfig;
5589 struct bpf_program *prog;
5590 struct bpf_object *obj;
5591 char tmp_name[64];
5592 int err;
5593
5594 if (elf_version(EV_CURRENT) == EV_NONE) {
5595 pr_warn("failed to init libelf for %s\n",
5596 path ? : "(mem buf)");
5597 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
5598 }
5599
5600 if (!OPTS_VALID(opts, bpf_object_open_opts))
5601 return ERR_PTR(-EINVAL);
5602
5603 obj_name = OPTS_GET(opts, object_name, NULL);
5604 if (obj_buf) {
5605 if (!obj_name) {
5606 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
5607 (unsigned long)obj_buf,
5608 (unsigned long)obj_buf_sz);
5609 obj_name = tmp_name;
5610 }
5611 path = obj_name;
5612 pr_debug("loading object '%s' from buffer\n", obj_name);
5613 }
5614
5615 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
5616 if (IS_ERR(obj))
5617 return obj;
5618
5619 kconfig = OPTS_GET(opts, kconfig, NULL);
5620 if (kconfig) {
5621 obj->kconfig = strdup(kconfig);
5622 if (!obj->kconfig)
5623 return ERR_PTR(-ENOMEM);
5624 }
5625
5626 err = bpf_object__elf_init(obj);
5627 err = err ? : bpf_object__check_endianness(obj);
5628 err = err ? : bpf_object__elf_collect(obj);
5629 err = err ? : bpf_object__collect_externs(obj);
5630 err = err ? : bpf_object__finalize_btf(obj);
5631 err = err ? : bpf_object__init_maps(obj, opts);
5632 err = err ? : bpf_object__init_prog_names(obj);
5633 err = err ? : bpf_object__collect_reloc(obj);
5634 if (err)
5635 goto out;
5636 bpf_object__elf_finish(obj);
5637
5638 bpf_object__for_each_program(prog, obj) {
5639 prog->sec_def = find_sec_def(prog->section_name);
5640 if (!prog->sec_def)
5641
5642 continue;
5643
5644 bpf_program__set_type(prog, prog->sec_def->prog_type);
5645 bpf_program__set_expected_attach_type(prog,
5646 prog->sec_def->expected_attach_type);
5647
5648 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING ||
5649 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT)
5650 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
5651 }
5652
5653 return obj;
5654out:
5655 bpf_object__close(obj);
5656 return ERR_PTR(err);
5657}
5658
5659static struct bpf_object *
5660__bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)
5661{
5662 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
5663 .relaxed_maps = flags & MAPS_RELAX_COMPAT,
5664 );
5665
5666
5667 if (!attr->file)
5668 return NULL;
5669
5670 pr_debug("loading %s\n", attr->file);
5671 return __bpf_object__open(attr->file, NULL, 0, &opts);
5672}
5673
5674struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
5675{
5676 return __bpf_object__open_xattr(attr, 0);
5677}
5678
5679struct bpf_object *bpf_object__open(const char *path)
5680{
5681 struct bpf_object_open_attr attr = {
5682 .file = path,
5683 .prog_type = BPF_PROG_TYPE_UNSPEC,
5684 };
5685
5686 return bpf_object__open_xattr(&attr);
5687}
5688
5689struct bpf_object *
5690bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
5691{
5692 if (!path)
5693 return ERR_PTR(-EINVAL);
5694
5695 pr_debug("loading %s\n", path);
5696
5697 return __bpf_object__open(path, NULL, 0, opts);
5698}
5699
5700struct bpf_object *
5701bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
5702 const struct bpf_object_open_opts *opts)
5703{
5704 if (!obj_buf || obj_buf_sz == 0)
5705 return ERR_PTR(-EINVAL);
5706
5707 return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts);
5708}
5709
5710struct bpf_object *
5711bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
5712 const char *name)
5713{
5714 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
5715 .object_name = name,
5716
5717 .relaxed_maps = true,
5718 );
5719
5720
5721 if (!obj_buf || obj_buf_sz == 0)
5722 return NULL;
5723
5724 return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts);
5725}
5726
5727int bpf_object__unload(struct bpf_object *obj)
5728{
5729 size_t i;
5730
5731 if (!obj)
5732 return -EINVAL;
5733
5734 for (i = 0; i < obj->nr_maps; i++) {
5735 zclose(obj->maps[i].fd);
5736 if (obj->maps[i].st_ops)
5737 zfree(&obj->maps[i].st_ops->kern_vdata);
5738 }
5739
5740 for (i = 0; i < obj->nr_programs; i++)
5741 bpf_program__unload(&obj->programs[i]);
5742
5743 return 0;
5744}
5745
5746static int bpf_object__sanitize_maps(struct bpf_object *obj)
5747{
5748 struct bpf_map *m;
5749
5750 bpf_object__for_each_map(m, obj) {
5751 if (!bpf_map__is_internal(m))
5752 continue;
5753 if (!obj->caps.global_data) {
5754 pr_warn("kernel doesn't support global data\n");
5755 return -ENOTSUP;
5756 }
5757 if (!obj->caps.array_mmap)
5758 m->def.map_flags ^= BPF_F_MMAPABLE;
5759 }
5760
5761 return 0;
5762}
5763
5764static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
5765{
5766 char sym_type, sym_name[500];
5767 unsigned long long sym_addr;
5768 struct extern_desc *ext;
5769 int ret, err = 0;
5770 FILE *f;
5771
5772 f = fopen("/proc/kallsyms", "r");
5773 if (!f) {
5774 err = -errno;
5775 pr_warn("failed to open /proc/kallsyms: %d\n", err);
5776 return err;
5777 }
5778
5779 while (true) {
5780 ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
5781 &sym_addr, &sym_type, sym_name);
5782 if (ret == EOF && feof(f))
5783 break;
5784 if (ret != 3) {
5785 pr_warn("failed to read kallsyms entry: %d\n", ret);
5786 err = -EINVAL;
5787 goto out;
5788 }
5789
5790 ext = find_extern_by_name(obj, sym_name);
5791 if (!ext || ext->type != EXT_KSYM)
5792 continue;
5793
5794 if (ext->is_set && ext->ksym.addr != sym_addr) {
5795 pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n",
5796 sym_name, ext->ksym.addr, sym_addr);
5797 err = -EINVAL;
5798 goto out;
5799 }
5800 if (!ext->is_set) {
5801 ext->is_set = true;
5802 ext->ksym.addr = sym_addr;
5803 pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr);
5804 }
5805 }
5806
5807out:
5808 fclose(f);
5809 return err;
5810}
5811
5812static int bpf_object__resolve_externs(struct bpf_object *obj,
5813 const char *extra_kconfig)
5814{
5815 bool need_config = false, need_kallsyms = false;
5816 struct extern_desc *ext;
5817 void *kcfg_data = NULL;
5818 int err, i;
5819
5820 if (obj->nr_extern == 0)
5821 return 0;
5822
5823 if (obj->kconfig_map_idx >= 0)
5824 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
5825
5826 for (i = 0; i < obj->nr_extern; i++) {
5827 ext = &obj->externs[i];
5828
5829 if (ext->type == EXT_KCFG &&
5830 strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
5831 void *ext_val = kcfg_data + ext->kcfg.data_off;
5832 __u32 kver = get_kernel_version();
5833
5834 if (!kver) {
5835 pr_warn("failed to get kernel version\n");
5836 return -EINVAL;
5837 }
5838 err = set_kcfg_value_num(ext, ext_val, kver);
5839 if (err)
5840 return err;
5841 pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver);
5842 } else if (ext->type == EXT_KCFG &&
5843 strncmp(ext->name, "CONFIG_", 7) == 0) {
5844 need_config = true;
5845 } else if (ext->type == EXT_KSYM) {
5846 need_kallsyms = true;
5847 } else {
5848 pr_warn("unrecognized extern '%s'\n", ext->name);
5849 return -EINVAL;
5850 }
5851 }
5852 if (need_config && extra_kconfig) {
5853 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
5854 if (err)
5855 return -EINVAL;
5856 need_config = false;
5857 for (i = 0; i < obj->nr_extern; i++) {
5858 ext = &obj->externs[i];
5859 if (ext->type == EXT_KCFG && !ext->is_set) {
5860 need_config = true;
5861 break;
5862 }
5863 }
5864 }
5865 if (need_config) {
5866 err = bpf_object__read_kconfig_file(obj, kcfg_data);
5867 if (err)
5868 return -EINVAL;
5869 }
5870 if (need_kallsyms) {
5871 err = bpf_object__read_kallsyms_file(obj);
5872 if (err)
5873 return -EINVAL;
5874 }
5875 for (i = 0; i < obj->nr_extern; i++) {
5876 ext = &obj->externs[i];
5877
5878 if (!ext->is_set && !ext->is_weak) {
5879 pr_warn("extern %s (strong) not resolved\n", ext->name);
5880 return -ESRCH;
5881 } else if (!ext->is_set) {
5882 pr_debug("extern %s (weak) not resolved, defaulting to zero\n",
5883 ext->name);
5884 }
5885 }
5886
5887 return 0;
5888}
5889
5890int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
5891{
5892 struct bpf_object *obj;
5893 int err, i;
5894
5895 if (!attr)
5896 return -EINVAL;
5897 obj = attr->obj;
5898 if (!obj)
5899 return -EINVAL;
5900
5901 if (obj->loaded) {
5902 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
5903 return -EINVAL;
5904 }
5905
5906 err = bpf_object__probe_loading(obj);
5907 err = err ? : bpf_object__probe_caps(obj);
5908 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
5909 err = err ? : bpf_object__sanitize_and_load_btf(obj);
5910 err = err ? : bpf_object__sanitize_maps(obj);
5911 err = err ? : bpf_object__load_vmlinux_btf(obj);
5912 err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
5913 err = err ? : bpf_object__create_maps(obj);
5914 err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
5915 err = err ? : bpf_object__load_progs(obj, attr->log_level);
5916
5917 btf__free(obj->btf_vmlinux);
5918 obj->btf_vmlinux = NULL;
5919
5920 obj->loaded = true;
5921
5922 if (err)
5923 goto out;
5924
5925 return 0;
5926out:
5927
5928 for (i = 0; i < obj->nr_maps; i++)
5929 if (obj->maps[i].pinned && !obj->maps[i].reused)
5930 bpf_map__unpin(&obj->maps[i], NULL);
5931
5932 bpf_object__unload(obj);
5933 pr_warn("failed to load object '%s'\n", obj->path);
5934 return err;
5935}
5936
5937int bpf_object__load(struct bpf_object *obj)
5938{
5939 struct bpf_object_load_attr attr = {
5940 .obj = obj,
5941 };
5942
5943 return bpf_object__load_xattr(&attr);
5944}
5945
5946static int make_parent_dir(const char *path)
5947{
5948 char *cp, errmsg[STRERR_BUFSIZE];
5949 char *dname, *dir;
5950 int err = 0;
5951
5952 dname = strdup(path);
5953 if (dname == NULL)
5954 return -ENOMEM;
5955
5956 dir = dirname(dname);
5957 if (mkdir(dir, 0700) && errno != EEXIST)
5958 err = -errno;
5959
5960 free(dname);
5961 if (err) {
5962 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
5963 pr_warn("failed to mkdir %s: %s\n", path, cp);
5964 }
5965 return err;
5966}
5967
5968static int check_path(const char *path)
5969{
5970 char *cp, errmsg[STRERR_BUFSIZE];
5971 struct statfs st_fs;
5972 char *dname, *dir;
5973 int err = 0;
5974
5975 if (path == NULL)
5976 return -EINVAL;
5977
5978 dname = strdup(path);
5979 if (dname == NULL)
5980 return -ENOMEM;
5981
5982 dir = dirname(dname);
5983 if (statfs(dir, &st_fs)) {
5984 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
5985 pr_warn("failed to statfs %s: %s\n", dir, cp);
5986 err = -errno;
5987 }
5988 free(dname);
5989
5990 if (!err && st_fs.f_type != BPF_FS_MAGIC) {
5991 pr_warn("specified path %s is not on BPF FS\n", path);
5992 err = -EINVAL;
5993 }
5994
5995 return err;
5996}
5997
5998int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
5999 int instance)
6000{
6001 char *cp, errmsg[STRERR_BUFSIZE];
6002 int err;
6003
6004 err = make_parent_dir(path);
6005 if (err)
6006 return err;
6007
6008 err = check_path(path);
6009 if (err)
6010 return err;
6011
6012 if (prog == NULL) {
6013 pr_warn("invalid program pointer\n");
6014 return -EINVAL;
6015 }
6016
6017 if (instance < 0 || instance >= prog->instances.nr) {
6018 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
6019 instance, prog->section_name, prog->instances.nr);
6020 return -EINVAL;
6021 }
6022
6023 if (bpf_obj_pin(prog->instances.fds[instance], path)) {
6024 err = -errno;
6025 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
6026 pr_warn("failed to pin program: %s\n", cp);
6027 return err;
6028 }
6029 pr_debug("pinned program '%s'\n", path);
6030
6031 return 0;
6032}
6033
6034int bpf_program__unpin_instance(struct bpf_program *prog, const char *path,
6035 int instance)
6036{
6037 int err;
6038
6039 err = check_path(path);
6040 if (err)
6041 return err;
6042
6043 if (prog == NULL) {
6044 pr_warn("invalid program pointer\n");
6045 return -EINVAL;
6046 }
6047
6048 if (instance < 0 || instance >= prog->instances.nr) {
6049 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
6050 instance, prog->section_name, prog->instances.nr);
6051 return -EINVAL;
6052 }
6053
6054 err = unlink(path);
6055 if (err != 0)
6056 return -errno;
6057 pr_debug("unpinned program '%s'\n", path);
6058
6059 return 0;
6060}
6061
6062int bpf_program__pin(struct bpf_program *prog, const char *path)
6063{
6064 int i, err;
6065
6066 err = make_parent_dir(path);
6067 if (err)
6068 return err;
6069
6070 err = check_path(path);
6071 if (err)
6072 return err;
6073
6074 if (prog == NULL) {
6075 pr_warn("invalid program pointer\n");
6076 return -EINVAL;
6077 }
6078
6079 if (prog->instances.nr <= 0) {
6080 pr_warn("no instances of prog %s to pin\n",
6081 prog->section_name);
6082 return -EINVAL;
6083 }
6084
6085 if (prog->instances.nr == 1) {
6086
6087 return bpf_program__pin_instance(prog, path, 0);
6088 }
6089
6090 for (i = 0; i < prog->instances.nr; i++) {
6091 char buf[PATH_MAX];
6092 int len;
6093
6094 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
6095 if (len < 0) {
6096 err = -EINVAL;
6097 goto err_unpin;
6098 } else if (len >= PATH_MAX) {
6099 err = -ENAMETOOLONG;
6100 goto err_unpin;
6101 }
6102
6103 err = bpf_program__pin_instance(prog, buf, i);
6104 if (err)
6105 goto err_unpin;
6106 }
6107
6108 return 0;
6109
6110err_unpin:
6111 for (i = i - 1; i >= 0; i--) {
6112 char buf[PATH_MAX];
6113 int len;
6114
6115 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
6116 if (len < 0)
6117 continue;
6118 else if (len >= PATH_MAX)
6119 continue;
6120
6121 bpf_program__unpin_instance(prog, buf, i);
6122 }
6123
6124 rmdir(path);
6125
6126 return err;
6127}
6128
6129int bpf_program__unpin(struct bpf_program *prog, const char *path)
6130{
6131 int i, err;
6132
6133 err = check_path(path);
6134 if (err)
6135 return err;
6136
6137 if (prog == NULL) {
6138 pr_warn("invalid program pointer\n");
6139 return -EINVAL;
6140 }
6141
6142 if (prog->instances.nr <= 0) {
6143 pr_warn("no instances of prog %s to pin\n",
6144 prog->section_name);
6145 return -EINVAL;
6146 }
6147
6148 if (prog->instances.nr == 1) {
6149
6150 return bpf_program__unpin_instance(prog, path, 0);
6151 }
6152
6153 for (i = 0; i < prog->instances.nr; i++) {
6154 char buf[PATH_MAX];
6155 int len;
6156
6157 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
6158 if (len < 0)
6159 return -EINVAL;
6160 else if (len >= PATH_MAX)
6161 return -ENAMETOOLONG;
6162
6163 err = bpf_program__unpin_instance(prog, buf, i);
6164 if (err)
6165 return err;
6166 }
6167
6168 err = rmdir(path);
6169 if (err)
6170 return -errno;
6171
6172 return 0;
6173}
6174
6175int bpf_map__pin(struct bpf_map *map, const char *path)
6176{
6177 char *cp, errmsg[STRERR_BUFSIZE];
6178 int err;
6179
6180 if (map == NULL) {
6181 pr_warn("invalid map pointer\n");
6182 return -EINVAL;
6183 }
6184
6185 if (map->pin_path) {
6186 if (path && strcmp(path, map->pin_path)) {
6187 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
6188 bpf_map__name(map), map->pin_path, path);
6189 return -EINVAL;
6190 } else if (map->pinned) {
6191 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
6192 bpf_map__name(map), map->pin_path);
6193 return 0;
6194 }
6195 } else {
6196 if (!path) {
6197 pr_warn("missing a path to pin map '%s' at\n",
6198 bpf_map__name(map));
6199 return -EINVAL;
6200 } else if (map->pinned) {
6201 pr_warn("map '%s' already pinned\n", bpf_map__name(map));
6202 return -EEXIST;
6203 }
6204
6205 map->pin_path = strdup(path);
6206 if (!map->pin_path) {
6207 err = -errno;
6208 goto out_err;
6209 }
6210 }
6211
6212 err = make_parent_dir(map->pin_path);
6213 if (err)
6214 return err;
6215
6216 err = check_path(map->pin_path);
6217 if (err)
6218 return err;
6219
6220 if (bpf_obj_pin(map->fd, map->pin_path)) {
6221 err = -errno;
6222 goto out_err;
6223 }
6224
6225 map->pinned = true;
6226 pr_debug("pinned map '%s'\n", map->pin_path);
6227
6228 return 0;
6229
6230out_err:
6231 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
6232 pr_warn("failed to pin map: %s\n", cp);
6233 return err;
6234}
6235
6236int bpf_map__unpin(struct bpf_map *map, const char *path)
6237{
6238 int err;
6239
6240 if (map == NULL) {
6241 pr_warn("invalid map pointer\n");
6242 return -EINVAL;
6243 }
6244
6245 if (map->pin_path) {
6246 if (path && strcmp(path, map->pin_path)) {
6247 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
6248 bpf_map__name(map), map->pin_path, path);
6249 return -EINVAL;
6250 }
6251 path = map->pin_path;
6252 } else if (!path) {
6253 pr_warn("no path to unpin map '%s' from\n",
6254 bpf_map__name(map));
6255 return -EINVAL;
6256 }
6257
6258 err = check_path(path);
6259 if (err)
6260 return err;
6261
6262 err = unlink(path);
6263 if (err != 0)
6264 return -errno;
6265
6266 map->pinned = false;
6267 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
6268
6269 return 0;
6270}
6271
6272int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
6273{
6274 char *new = NULL;
6275
6276 if (path) {
6277 new = strdup(path);
6278 if (!new)
6279 return -errno;
6280 }
6281
6282 free(map->pin_path);
6283 map->pin_path = new;
6284 return 0;
6285}
6286
6287const char *bpf_map__get_pin_path(const struct bpf_map *map)
6288{
6289 return map->pin_path;
6290}
6291
6292bool bpf_map__is_pinned(const struct bpf_map *map)
6293{
6294 return map->pinned;
6295}
6296
6297int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
6298{
6299 struct bpf_map *map;
6300 int err;
6301
6302 if (!obj)
6303 return -ENOENT;
6304
6305 if (!obj->loaded) {
6306 pr_warn("object not yet loaded; load it first\n");
6307 return -ENOENT;
6308 }
6309
6310 bpf_object__for_each_map(map, obj) {
6311 char *pin_path = NULL;
6312 char buf[PATH_MAX];
6313
6314 if (path) {
6315 int len;
6316
6317 len = snprintf(buf, PATH_MAX, "%s/%s", path,
6318 bpf_map__name(map));
6319 if (len < 0) {
6320 err = -EINVAL;
6321 goto err_unpin_maps;
6322 } else if (len >= PATH_MAX) {
6323 err = -ENAMETOOLONG;
6324 goto err_unpin_maps;
6325 }
6326 pin_path = buf;
6327 } else if (!map->pin_path) {
6328 continue;
6329 }
6330
6331 err = bpf_map__pin(map, pin_path);
6332 if (err)
6333 goto err_unpin_maps;
6334 }
6335
6336 return 0;
6337
6338err_unpin_maps:
6339 while ((map = bpf_map__prev(map, obj))) {
6340 if (!map->pin_path)
6341 continue;
6342
6343 bpf_map__unpin(map, NULL);
6344 }
6345
6346 return err;
6347}
6348
6349int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
6350{
6351 struct bpf_map *map;
6352 int err;
6353
6354 if (!obj)
6355 return -ENOENT;
6356
6357 bpf_object__for_each_map(map, obj) {
6358 char *pin_path = NULL;
6359 char buf[PATH_MAX];
6360
6361 if (path) {
6362 int len;
6363
6364 len = snprintf(buf, PATH_MAX, "%s/%s", path,
6365 bpf_map__name(map));
6366 if (len < 0)
6367 return -EINVAL;
6368 else if (len >= PATH_MAX)
6369 return -ENAMETOOLONG;
6370 pin_path = buf;
6371 } else if (!map->pin_path) {
6372 continue;
6373 }
6374
6375 err = bpf_map__unpin(map, pin_path);
6376 if (err)
6377 return err;
6378 }
6379
6380 return 0;
6381}
6382
6383int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
6384{
6385 struct bpf_program *prog;
6386 int err;
6387
6388 if (!obj)
6389 return -ENOENT;
6390
6391 if (!obj->loaded) {
6392 pr_warn("object not yet loaded; load it first\n");
6393 return -ENOENT;
6394 }
6395
6396 bpf_object__for_each_program(prog, obj) {
6397 char buf[PATH_MAX];
6398 int len;
6399
6400 len = snprintf(buf, PATH_MAX, "%s/%s", path,
6401 prog->pin_name);
6402 if (len < 0) {
6403 err = -EINVAL;
6404 goto err_unpin_programs;
6405 } else if (len >= PATH_MAX) {
6406 err = -ENAMETOOLONG;
6407 goto err_unpin_programs;
6408 }
6409
6410 err = bpf_program__pin(prog, buf);
6411 if (err)
6412 goto err_unpin_programs;
6413 }
6414
6415 return 0;
6416
6417err_unpin_programs:
6418 while ((prog = bpf_program__prev(prog, obj))) {
6419 char buf[PATH_MAX];
6420 int len;
6421
6422 len = snprintf(buf, PATH_MAX, "%s/%s", path,
6423 prog->pin_name);
6424 if (len < 0)
6425 continue;
6426 else if (len >= PATH_MAX)
6427 continue;
6428
6429 bpf_program__unpin(prog, buf);
6430 }
6431
6432 return err;
6433}
6434
6435int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
6436{
6437 struct bpf_program *prog;
6438 int err;
6439
6440 if (!obj)
6441 return -ENOENT;
6442
6443 bpf_object__for_each_program(prog, obj) {
6444 char buf[PATH_MAX];
6445 int len;
6446
6447 len = snprintf(buf, PATH_MAX, "%s/%s", path,
6448 prog->pin_name);
6449 if (len < 0)
6450 return -EINVAL;
6451 else if (len >= PATH_MAX)
6452 return -ENAMETOOLONG;
6453
6454 err = bpf_program__unpin(prog, buf);
6455 if (err)
6456 return err;
6457 }
6458
6459 return 0;
6460}
6461
6462int bpf_object__pin(struct bpf_object *obj, const char *path)
6463{
6464 int err;
6465
6466 err = bpf_object__pin_maps(obj, path);
6467 if (err)
6468 return err;
6469
6470 err = bpf_object__pin_programs(obj, path);
6471 if (err) {
6472 bpf_object__unpin_maps(obj, path);
6473 return err;
6474 }
6475
6476 return 0;
6477}
6478
6479static void bpf_map__destroy(struct bpf_map *map)
6480{
6481 if (map->clear_priv)
6482 map->clear_priv(map, map->priv);
6483 map->priv = NULL;
6484 map->clear_priv = NULL;
6485
6486 if (map->inner_map) {
6487 bpf_map__destroy(map->inner_map);
6488 zfree(&map->inner_map);
6489 }
6490
6491 zfree(&map->init_slots);
6492 map->init_slots_sz = 0;
6493
6494 if (map->mmaped) {
6495 munmap(map->mmaped, bpf_map_mmap_sz(map));
6496 map->mmaped = NULL;
6497 }
6498
6499 if (map->st_ops) {
6500 zfree(&map->st_ops->data);
6501 zfree(&map->st_ops->progs);
6502 zfree(&map->st_ops->kern_func_off);
6503 zfree(&map->st_ops);
6504 }
6505
6506 zfree(&map->name);
6507 zfree(&map->pin_path);
6508
6509 if (map->fd >= 0)
6510 zclose(map->fd);
6511}
6512
6513void bpf_object__close(struct bpf_object *obj)
6514{
6515 size_t i;
6516
6517 if (IS_ERR_OR_NULL(obj))
6518 return;
6519
6520 if (obj->clear_priv)
6521 obj->clear_priv(obj, obj->priv);
6522
6523 bpf_object__elf_finish(obj);
6524 bpf_object__unload(obj);
6525 btf__free(obj->btf);
6526 btf_ext__free(obj->btf_ext);
6527
6528 for (i = 0; i < obj->nr_maps; i++)
6529 bpf_map__destroy(&obj->maps[i]);
6530
6531 zfree(&obj->kconfig);
6532 zfree(&obj->externs);
6533 obj->nr_extern = 0;
6534
6535 zfree(&obj->maps);
6536 obj->nr_maps = 0;
6537
6538 if (obj->programs && obj->nr_programs) {
6539 for (i = 0; i < obj->nr_programs; i++)
6540 bpf_program__exit(&obj->programs[i]);
6541 }
6542 zfree(&obj->programs);
6543
6544 list_del(&obj->list);
6545 free(obj);
6546}
6547
6548struct bpf_object *
6549bpf_object__next(struct bpf_object *prev)
6550{
6551 struct bpf_object *next;
6552
6553 if (!prev)
6554 next = list_first_entry(&bpf_objects_list,
6555 struct bpf_object,
6556 list);
6557 else
6558 next = list_next_entry(prev, list);
6559
6560
6561 if (&next->list == &bpf_objects_list)
6562 return NULL;
6563
6564 return next;
6565}
6566
6567const char *bpf_object__name(const struct bpf_object *obj)
6568{
6569 return obj ? obj->name : ERR_PTR(-EINVAL);
6570}
6571
6572unsigned int bpf_object__kversion(const struct bpf_object *obj)
6573{
6574 return obj ? obj->kern_version : 0;
6575}
6576
6577struct btf *bpf_object__btf(const struct bpf_object *obj)
6578{
6579 return obj ? obj->btf : NULL;
6580}
6581
6582int bpf_object__btf_fd(const struct bpf_object *obj)
6583{
6584 return obj->btf ? btf__fd(obj->btf) : -1;
6585}
6586
6587int bpf_object__set_priv(struct bpf_object *obj, void *priv,
6588 bpf_object_clear_priv_t clear_priv)
6589{
6590 if (obj->priv && obj->clear_priv)
6591 obj->clear_priv(obj, obj->priv);
6592
6593 obj->priv = priv;
6594 obj->clear_priv = clear_priv;
6595 return 0;
6596}
6597
6598void *bpf_object__priv(const struct bpf_object *obj)
6599{
6600 return obj ? obj->priv : ERR_PTR(-EINVAL);
6601}
6602
6603static struct bpf_program *
6604__bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
6605 bool forward)
6606{
6607 size_t nr_programs = obj->nr_programs;
6608 ssize_t idx;
6609
6610 if (!nr_programs)
6611 return NULL;
6612
6613 if (!p)
6614
6615 return forward ? &obj->programs[0] :
6616 &obj->programs[nr_programs - 1];
6617
6618 if (p->obj != obj) {
6619 pr_warn("error: program handler doesn't match object\n");
6620 return NULL;
6621 }
6622
6623 idx = (p - obj->programs) + (forward ? 1 : -1);
6624 if (idx >= obj->nr_programs || idx < 0)
6625 return NULL;
6626 return &obj->programs[idx];
6627}
6628
6629struct bpf_program *
6630bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
6631{
6632 struct bpf_program *prog = prev;
6633
6634 do {
6635 prog = __bpf_program__iter(prog, obj, true);
6636 } while (prog && bpf_program__is_function_storage(prog, obj));
6637
6638 return prog;
6639}
6640
6641struct bpf_program *
6642bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj)
6643{
6644 struct bpf_program *prog = next;
6645
6646 do {
6647 prog = __bpf_program__iter(prog, obj, false);
6648 } while (prog && bpf_program__is_function_storage(prog, obj));
6649
6650 return prog;
6651}
6652
6653int bpf_program__set_priv(struct bpf_program *prog, void *priv,
6654 bpf_program_clear_priv_t clear_priv)
6655{
6656 if (prog->priv && prog->clear_priv)
6657 prog->clear_priv(prog, prog->priv);
6658
6659 prog->priv = priv;
6660 prog->clear_priv = clear_priv;
6661 return 0;
6662}
6663
6664void *bpf_program__priv(const struct bpf_program *prog)
6665{
6666 return prog ? prog->priv : ERR_PTR(-EINVAL);
6667}
6668
6669void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
6670{
6671 prog->prog_ifindex = ifindex;
6672}
6673
6674const char *bpf_program__name(const struct bpf_program *prog)
6675{
6676 return prog->name;
6677}
6678
6679const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy)
6680{
6681 const char *title;
6682
6683 title = prog->section_name;
6684 if (needs_copy) {
6685 title = strdup(title);
6686 if (!title) {
6687 pr_warn("failed to strdup program title\n");
6688 return ERR_PTR(-ENOMEM);
6689 }
6690 }
6691
6692 return title;
6693}
6694
6695bool bpf_program__autoload(const struct bpf_program *prog)
6696{
6697 return prog->load;
6698}
6699
6700int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
6701{
6702 if (prog->obj->loaded)
6703 return -EINVAL;
6704
6705 prog->load = autoload;
6706 return 0;
6707}
6708
6709int bpf_program__fd(const struct bpf_program *prog)
6710{
6711 return bpf_program__nth_fd(prog, 0);
6712}
6713
6714size_t bpf_program__size(const struct bpf_program *prog)
6715{
6716 return prog->insns_cnt * sizeof(struct bpf_insn);
6717}
6718
6719int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
6720 bpf_program_prep_t prep)
6721{
6722 int *instances_fds;
6723
6724 if (nr_instances <= 0 || !prep)
6725 return -EINVAL;
6726
6727 if (prog->instances.nr > 0 || prog->instances.fds) {
6728 pr_warn("Can't set pre-processor after loading\n");
6729 return -EINVAL;
6730 }
6731
6732 instances_fds = malloc(sizeof(int) * nr_instances);
6733 if (!instances_fds) {
6734 pr_warn("alloc memory failed for fds\n");
6735 return -ENOMEM;
6736 }
6737
6738
6739 memset(instances_fds, -1, sizeof(int) * nr_instances);
6740
6741 prog->instances.nr = nr_instances;
6742 prog->instances.fds = instances_fds;
6743 prog->preprocessor = prep;
6744 return 0;
6745}
6746
6747int bpf_program__nth_fd(const struct bpf_program *prog, int n)
6748{
6749 int fd;
6750
6751 if (!prog)
6752 return -EINVAL;
6753
6754 if (n >= prog->instances.nr || n < 0) {
6755 pr_warn("Can't get the %dth fd from program %s: only %d instances\n",
6756 n, prog->section_name, prog->instances.nr);
6757 return -EINVAL;
6758 }
6759
6760 fd = prog->instances.fds[n];
6761 if (fd < 0) {
6762 pr_warn("%dth instance of program '%s' is invalid\n",
6763 n, prog->section_name);
6764 return -ENOENT;
6765 }
6766
6767 return fd;
6768}
6769
6770enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog)
6771{
6772 return prog->type;
6773}
6774
6775void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
6776{
6777 prog->type = type;
6778}
6779
6780static bool bpf_program__is_type(const struct bpf_program *prog,
6781 enum bpf_prog_type type)
6782{
6783 return prog ? (prog->type == type) : false;
6784}
6785
6786#define BPF_PROG_TYPE_FNS(NAME, TYPE) \
6787int bpf_program__set_##NAME(struct bpf_program *prog) \
6788{ \
6789 if (!prog) \
6790 return -EINVAL; \
6791 bpf_program__set_type(prog, TYPE); \
6792 return 0; \
6793} \
6794 \
6795bool bpf_program__is_##NAME(const struct bpf_program *prog) \
6796{ \
6797 return bpf_program__is_type(prog, TYPE); \
6798} \
6799
6800BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
6801BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM);
6802BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
6803BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
6804BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
6805BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
6806BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
6807BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
6808BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
6809BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
6810BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS);
6811BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT);
6812BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP);
6813
6814enum bpf_attach_type
6815bpf_program__get_expected_attach_type(struct bpf_program *prog)
6816{
6817 return prog->expected_attach_type;
6818}
6819
6820void bpf_program__set_expected_attach_type(struct bpf_program *prog,
6821 enum bpf_attach_type type)
6822{
6823 prog->expected_attach_type = type;
6824}
6825
6826#define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional, \
6827 attachable, attach_btf) \
6828 { \
6829 .sec = string, \
6830 .len = sizeof(string) - 1, \
6831 .prog_type = ptype, \
6832 .expected_attach_type = eatype, \
6833 .is_exp_attach_type_optional = eatype_optional, \
6834 .is_attachable = attachable, \
6835 .is_attach_btf = attach_btf, \
6836 }
6837
6838
6839#define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0)
6840
6841
6842#define BPF_APROG_SEC(string, ptype, atype) \
6843 BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0)
6844
6845
6846#define BPF_EAPROG_SEC(string, ptype, eatype) \
6847 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0)
6848
6849
6850#define BPF_PROG_BTF(string, ptype, eatype) \
6851 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1)
6852
6853
6854
6855
6856#define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
6857
6858#define SEC_DEF(sec_pfx, ptype, ...) { \
6859 .sec = sec_pfx, \
6860 .len = sizeof(sec_pfx) - 1, \
6861 .prog_type = BPF_PROG_TYPE_##ptype, \
6862 __VA_ARGS__ \
6863}
6864
6865static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
6866 struct bpf_program *prog);
6867static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
6868 struct bpf_program *prog);
6869static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
6870 struct bpf_program *prog);
6871static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
6872 struct bpf_program *prog);
6873static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
6874 struct bpf_program *prog);
6875static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
6876 struct bpf_program *prog);
6877
6878static const struct bpf_sec_def section_defs[] = {
6879 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),
6880 BPF_PROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT),
6881 SEC_DEF("kprobe/", KPROBE,
6882 .attach_fn = attach_kprobe),
6883 BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE),
6884 SEC_DEF("kretprobe/", KPROBE,
6885 .attach_fn = attach_kprobe),
6886 BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE),
6887 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS),
6888 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT),
6889 SEC_DEF("tracepoint/", TRACEPOINT,
6890 .attach_fn = attach_tp),
6891 SEC_DEF("tp/", TRACEPOINT,
6892 .attach_fn = attach_tp),
6893 SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT,
6894 .attach_fn = attach_raw_tp),
6895 SEC_DEF("raw_tp/", RAW_TRACEPOINT,
6896 .attach_fn = attach_raw_tp),
6897 SEC_DEF("tp_btf/", TRACING,
6898 .expected_attach_type = BPF_TRACE_RAW_TP,
6899 .is_attach_btf = true,
6900 .attach_fn = attach_trace),
6901 SEC_DEF("fentry/", TRACING,
6902 .expected_attach_type = BPF_TRACE_FENTRY,
6903 .is_attach_btf = true,
6904 .attach_fn = attach_trace),
6905 SEC_DEF("fmod_ret/", TRACING,
6906 .expected_attach_type = BPF_MODIFY_RETURN,
6907 .is_attach_btf = true,
6908 .attach_fn = attach_trace),
6909 SEC_DEF("fexit/", TRACING,
6910 .expected_attach_type = BPF_TRACE_FEXIT,
6911 .is_attach_btf = true,
6912 .attach_fn = attach_trace),
6913 SEC_DEF("freplace/", EXT,
6914 .is_attach_btf = true,
6915 .attach_fn = attach_trace),
6916 SEC_DEF("lsm/", LSM,
6917 .is_attach_btf = true,
6918 .expected_attach_type = BPF_LSM_MAC,
6919 .attach_fn = attach_lsm),
6920 SEC_DEF("iter/", TRACING,
6921 .expected_attach_type = BPF_TRACE_ITER,
6922 .is_attach_btf = true,
6923 .attach_fn = attach_iter),
6924 BPF_EAPROG_SEC("xdp_devmap/", BPF_PROG_TYPE_XDP,
6925 BPF_XDP_DEVMAP),
6926 BPF_EAPROG_SEC("xdp_cpumap/", BPF_PROG_TYPE_XDP,
6927 BPF_XDP_CPUMAP),
6928 BPF_APROG_SEC("xdp", BPF_PROG_TYPE_XDP,
6929 BPF_XDP),
6930 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),
6931 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN),
6932 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT),
6933 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT),
6934 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL),
6935 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB,
6936 BPF_CGROUP_INET_INGRESS),
6937 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB,
6938 BPF_CGROUP_INET_EGRESS),
6939 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB),
6940 BPF_EAPROG_SEC("cgroup/sock_create", BPF_PROG_TYPE_CGROUP_SOCK,
6941 BPF_CGROUP_INET_SOCK_CREATE),
6942 BPF_EAPROG_SEC("cgroup/sock_release", BPF_PROG_TYPE_CGROUP_SOCK,
6943 BPF_CGROUP_INET_SOCK_RELEASE),
6944 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK,
6945 BPF_CGROUP_INET_SOCK_CREATE),
6946 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK,
6947 BPF_CGROUP_INET4_POST_BIND),
6948 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK,
6949 BPF_CGROUP_INET6_POST_BIND),
6950 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE,
6951 BPF_CGROUP_DEVICE),
6952 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS,
6953 BPF_CGROUP_SOCK_OPS),
6954 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB,
6955 BPF_SK_SKB_STREAM_PARSER),
6956 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB,
6957 BPF_SK_SKB_STREAM_VERDICT),
6958 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB),
6959 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG,
6960 BPF_SK_MSG_VERDICT),
6961 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2,
6962 BPF_LIRC_MODE2),
6963 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR,
6964 BPF_FLOW_DISSECTOR),
6965 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6966 BPF_CGROUP_INET4_BIND),
6967 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6968 BPF_CGROUP_INET6_BIND),
6969 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6970 BPF_CGROUP_INET4_CONNECT),
6971 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6972 BPF_CGROUP_INET6_CONNECT),
6973 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6974 BPF_CGROUP_UDP4_SENDMSG),
6975 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6976 BPF_CGROUP_UDP6_SENDMSG),
6977 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6978 BPF_CGROUP_UDP4_RECVMSG),
6979 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6980 BPF_CGROUP_UDP6_RECVMSG),
6981 BPF_EAPROG_SEC("cgroup/getpeername4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6982 BPF_CGROUP_INET4_GETPEERNAME),
6983 BPF_EAPROG_SEC("cgroup/getpeername6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6984 BPF_CGROUP_INET6_GETPEERNAME),
6985 BPF_EAPROG_SEC("cgroup/getsockname4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6986 BPF_CGROUP_INET4_GETSOCKNAME),
6987 BPF_EAPROG_SEC("cgroup/getsockname6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
6988 BPF_CGROUP_INET6_GETSOCKNAME),
6989 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL,
6990 BPF_CGROUP_SYSCTL),
6991 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
6992 BPF_CGROUP_GETSOCKOPT),
6993 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
6994 BPF_CGROUP_SETSOCKOPT),
6995 BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS),
6996 BPF_EAPROG_SEC("sk_lookup/", BPF_PROG_TYPE_SK_LOOKUP,
6997 BPF_SK_LOOKUP),
6998};
6999
7000#undef BPF_PROG_SEC_IMPL
7001#undef BPF_PROG_SEC
7002#undef BPF_APROG_SEC
7003#undef BPF_EAPROG_SEC
7004#undef BPF_APROG_COMPAT
7005#undef SEC_DEF
7006
7007#define MAX_TYPE_NAME_SIZE 32
7008
7009static const struct bpf_sec_def *find_sec_def(const char *sec_name)
7010{
7011 int i, n = ARRAY_SIZE(section_defs);
7012
7013 for (i = 0; i < n; i++) {
7014 if (strncmp(sec_name,
7015 section_defs[i].sec, section_defs[i].len))
7016 continue;
7017 return §ion_defs[i];
7018 }
7019 return NULL;
7020}
7021
7022static char *libbpf_get_type_names(bool attach_type)
7023{
7024 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
7025 char *buf;
7026
7027 buf = malloc(len);
7028 if (!buf)
7029 return NULL;
7030
7031 buf[0] = '\0';
7032
7033 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
7034 if (attach_type && !section_defs[i].is_attachable)
7035 continue;
7036
7037 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
7038 free(buf);
7039 return NULL;
7040 }
7041 strcat(buf, " ");
7042 strcat(buf, section_defs[i].sec);
7043 }
7044
7045 return buf;
7046}
7047
7048int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
7049 enum bpf_attach_type *expected_attach_type)
7050{
7051 const struct bpf_sec_def *sec_def;
7052 char *type_names;
7053
7054 if (!name)
7055 return -EINVAL;
7056
7057 sec_def = find_sec_def(name);
7058 if (sec_def) {
7059 *prog_type = sec_def->prog_type;
7060 *expected_attach_type = sec_def->expected_attach_type;
7061 return 0;
7062 }
7063
7064 pr_debug("failed to guess program type from ELF section '%s'\n", name);
7065 type_names = libbpf_get_type_names(false);
7066 if (type_names != NULL) {
7067 pr_debug("supported section(type) names are:%s\n", type_names);
7068 free(type_names);
7069 }
7070
7071 return -ESRCH;
7072}
7073
7074static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
7075 size_t offset)
7076{
7077 struct bpf_map *map;
7078 size_t i;
7079
7080 for (i = 0; i < obj->nr_maps; i++) {
7081 map = &obj->maps[i];
7082 if (!bpf_map__is_struct_ops(map))
7083 continue;
7084 if (map->sec_offset <= offset &&
7085 offset - map->sec_offset < map->def.value_size)
7086 return map;
7087 }
7088
7089 return NULL;
7090}
7091
7092
7093static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
7094 GElf_Shdr *shdr, Elf_Data *data)
7095{
7096 const struct btf_member *member;
7097 struct bpf_struct_ops *st_ops;
7098 struct bpf_program *prog;
7099 unsigned int shdr_idx;
7100 const struct btf *btf;
7101 struct bpf_map *map;
7102 Elf_Data *symbols;
7103 unsigned int moff;
7104 const char *name;
7105 __u32 member_idx;
7106 GElf_Sym sym;
7107 GElf_Rel rel;
7108 int i, nrels;
7109
7110 symbols = obj->efile.symbols;
7111 btf = obj->btf;
7112 nrels = shdr->sh_size / shdr->sh_entsize;
7113 for (i = 0; i < nrels; i++) {
7114 if (!gelf_getrel(data, i, &rel)) {
7115 pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
7116 return -LIBBPF_ERRNO__FORMAT;
7117 }
7118
7119 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
7120 pr_warn("struct_ops reloc: symbol %zx not found\n",
7121 (size_t)GELF_R_SYM(rel.r_info));
7122 return -LIBBPF_ERRNO__FORMAT;
7123 }
7124
7125 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
7126 sym.st_name) ? : "<?>";
7127 map = find_struct_ops_map_by_offset(obj, rel.r_offset);
7128 if (!map) {
7129 pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n",
7130 (size_t)rel.r_offset);
7131 return -EINVAL;
7132 }
7133
7134 moff = rel.r_offset - map->sec_offset;
7135 shdr_idx = sym.st_shndx;
7136 st_ops = map->st_ops;
7137 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
7138 map->name,
7139 (long long)(rel.r_info >> 32),
7140 (long long)sym.st_value,
7141 shdr_idx, (size_t)rel.r_offset,
7142 map->sec_offset, sym.st_name, name);
7143
7144 if (shdr_idx >= SHN_LORESERVE) {
7145 pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n",
7146 map->name, (size_t)rel.r_offset, shdr_idx);
7147 return -LIBBPF_ERRNO__RELOC;
7148 }
7149
7150 member = find_member_by_offset(st_ops->type, moff * 8);
7151 if (!member) {
7152 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
7153 map->name, moff);
7154 return -EINVAL;
7155 }
7156 member_idx = member - btf_members(st_ops->type);
7157 name = btf__name_by_offset(btf, member->name_off);
7158
7159 if (!resolve_func_ptr(btf, member->type, NULL)) {
7160 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
7161 map->name, name);
7162 return -EINVAL;
7163 }
7164
7165 prog = bpf_object__find_prog_by_idx(obj, shdr_idx);
7166 if (!prog) {
7167 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
7168 map->name, shdr_idx, name);
7169 return -EINVAL;
7170 }
7171
7172 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
7173 const struct bpf_sec_def *sec_def;
7174
7175 sec_def = find_sec_def(prog->section_name);
7176 if (sec_def &&
7177 sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) {
7178
7179 prog->type = sec_def->prog_type;
7180 goto invalid_prog;
7181 }
7182
7183 prog->type = BPF_PROG_TYPE_STRUCT_OPS;
7184 prog->attach_btf_id = st_ops->type_id;
7185 prog->expected_attach_type = member_idx;
7186 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
7187 prog->attach_btf_id != st_ops->type_id ||
7188 prog->expected_attach_type != member_idx) {
7189 goto invalid_prog;
7190 }
7191 st_ops->progs[member_idx] = prog;
7192 }
7193
7194 return 0;
7195
7196invalid_prog:
7197 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
7198 map->name, prog->name, prog->section_name, prog->type,
7199 prog->attach_btf_id, prog->expected_attach_type, name);
7200 return -EINVAL;
7201}
7202
7203#define BTF_TRACE_PREFIX "btf_trace_"
7204#define BTF_LSM_PREFIX "bpf_lsm_"
7205#define BTF_ITER_PREFIX "bpf_iter_"
7206#define BTF_MAX_NAME_SIZE 128
7207
7208static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
7209 const char *name, __u32 kind)
7210{
7211 char btf_type_name[BTF_MAX_NAME_SIZE];
7212 int ret;
7213
7214 ret = snprintf(btf_type_name, sizeof(btf_type_name),
7215 "%s%s", prefix, name);
7216
7217
7218
7219
7220 if (ret < 0 || ret >= sizeof(btf_type_name))
7221 return -ENAMETOOLONG;
7222 return btf__find_by_name_kind(btf, btf_type_name, kind);
7223}
7224
7225static inline int __find_vmlinux_btf_id(struct btf *btf, const char *name,
7226 enum bpf_attach_type attach_type)
7227{
7228 int err;
7229
7230 if (attach_type == BPF_TRACE_RAW_TP)
7231 err = find_btf_by_prefix_kind(btf, BTF_TRACE_PREFIX, name,
7232 BTF_KIND_TYPEDEF);
7233 else if (attach_type == BPF_LSM_MAC)
7234 err = find_btf_by_prefix_kind(btf, BTF_LSM_PREFIX, name,
7235 BTF_KIND_FUNC);
7236 else if (attach_type == BPF_TRACE_ITER)
7237 err = find_btf_by_prefix_kind(btf, BTF_ITER_PREFIX, name,
7238 BTF_KIND_FUNC);
7239 else
7240 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
7241
7242 if (err <= 0)
7243 pr_warn("%s is not found in vmlinux BTF\n", name);
7244
7245 return err;
7246}
7247
7248int libbpf_find_vmlinux_btf_id(const char *name,
7249 enum bpf_attach_type attach_type)
7250{
7251 struct btf *btf;
7252 int err;
7253
7254 btf = libbpf_find_kernel_btf();
7255 if (IS_ERR(btf)) {
7256 pr_warn("vmlinux BTF is not found\n");
7257 return -EINVAL;
7258 }
7259
7260 err = __find_vmlinux_btf_id(btf, name, attach_type);
7261 btf__free(btf);
7262 return err;
7263}
7264
7265static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
7266{
7267 struct bpf_prog_info_linear *info_linear;
7268 struct bpf_prog_info *info;
7269 struct btf *btf = NULL;
7270 int err = -EINVAL;
7271
7272 info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0);
7273 if (IS_ERR_OR_NULL(info_linear)) {
7274 pr_warn("failed get_prog_info_linear for FD %d\n",
7275 attach_prog_fd);
7276 return -EINVAL;
7277 }
7278 info = &info_linear->info;
7279 if (!info->btf_id) {
7280 pr_warn("The target program doesn't have BTF\n");
7281 goto out;
7282 }
7283 if (btf__get_from_id(info->btf_id, &btf)) {
7284 pr_warn("Failed to get BTF of the program\n");
7285 goto out;
7286 }
7287 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
7288 btf__free(btf);
7289 if (err <= 0) {
7290 pr_warn("%s is not found in prog's BTF\n", name);
7291 goto out;
7292 }
7293out:
7294 free(info_linear);
7295 return err;
7296}
7297
7298static int libbpf_find_attach_btf_id(struct bpf_program *prog)
7299{
7300 enum bpf_attach_type attach_type = prog->expected_attach_type;
7301 __u32 attach_prog_fd = prog->attach_prog_fd;
7302 const char *name = prog->section_name;
7303 int i, err;
7304
7305 if (!name)
7306 return -EINVAL;
7307
7308 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
7309 if (!section_defs[i].is_attach_btf)
7310 continue;
7311 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
7312 continue;
7313 if (attach_prog_fd)
7314 err = libbpf_find_prog_btf_id(name + section_defs[i].len,
7315 attach_prog_fd);
7316 else
7317 err = __find_vmlinux_btf_id(prog->obj->btf_vmlinux,
7318 name + section_defs[i].len,
7319 attach_type);
7320 return err;
7321 }
7322 pr_warn("failed to identify btf_id based on ELF section name '%s'\n", name);
7323 return -ESRCH;
7324}
7325
7326int libbpf_attach_type_by_name(const char *name,
7327 enum bpf_attach_type *attach_type)
7328{
7329 char *type_names;
7330 int i;
7331
7332 if (!name)
7333 return -EINVAL;
7334
7335 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
7336 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
7337 continue;
7338 if (!section_defs[i].is_attachable)
7339 return -EINVAL;
7340 *attach_type = section_defs[i].expected_attach_type;
7341 return 0;
7342 }
7343 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
7344 type_names = libbpf_get_type_names(true);
7345 if (type_names != NULL) {
7346 pr_debug("attachable section(type) names are:%s\n", type_names);
7347 free(type_names);
7348 }
7349
7350 return -EINVAL;
7351}
7352
7353int bpf_map__fd(const struct bpf_map *map)
7354{
7355 return map ? map->fd : -EINVAL;
7356}
7357
7358const struct bpf_map_def *bpf_map__def(const struct bpf_map *map)
7359{
7360 return map ? &map->def : ERR_PTR(-EINVAL);
7361}
7362
7363const char *bpf_map__name(const struct bpf_map *map)
7364{
7365 return map ? map->name : NULL;
7366}
7367
7368enum bpf_map_type bpf_map__type(const struct bpf_map *map)
7369{
7370 return map->def.type;
7371}
7372
7373int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
7374{
7375 if (map->fd >= 0)
7376 return -EBUSY;
7377 map->def.type = type;
7378 return 0;
7379}
7380
7381__u32 bpf_map__map_flags(const struct bpf_map *map)
7382{
7383 return map->def.map_flags;
7384}
7385
7386int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
7387{
7388 if (map->fd >= 0)
7389 return -EBUSY;
7390 map->def.map_flags = flags;
7391 return 0;
7392}
7393
7394__u32 bpf_map__numa_node(const struct bpf_map *map)
7395{
7396 return map->numa_node;
7397}
7398
7399int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
7400{
7401 if (map->fd >= 0)
7402 return -EBUSY;
7403 map->numa_node = numa_node;
7404 return 0;
7405}
7406
7407__u32 bpf_map__key_size(const struct bpf_map *map)
7408{
7409 return map->def.key_size;
7410}
7411
7412int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
7413{
7414 if (map->fd >= 0)
7415 return -EBUSY;
7416 map->def.key_size = size;
7417 return 0;
7418}
7419
7420__u32 bpf_map__value_size(const struct bpf_map *map)
7421{
7422 return map->def.value_size;
7423}
7424
7425int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
7426{
7427 if (map->fd >= 0)
7428 return -EBUSY;
7429 map->def.value_size = size;
7430 return 0;
7431}
7432
7433__u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
7434{
7435 return map ? map->btf_key_type_id : 0;
7436}
7437
7438__u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
7439{
7440 return map ? map->btf_value_type_id : 0;
7441}
7442
7443int bpf_map__set_priv(struct bpf_map *map, void *priv,
7444 bpf_map_clear_priv_t clear_priv)
7445{
7446 if (!map)
7447 return -EINVAL;
7448
7449 if (map->priv) {
7450 if (map->clear_priv)
7451 map->clear_priv(map, map->priv);
7452 }
7453
7454 map->priv = priv;
7455 map->clear_priv = clear_priv;
7456 return 0;
7457}
7458
7459void *bpf_map__priv(const struct bpf_map *map)
7460{
7461 return map ? map->priv : ERR_PTR(-EINVAL);
7462}
7463
7464int bpf_map__set_initial_value(struct bpf_map *map,
7465 const void *data, size_t size)
7466{
7467 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
7468 size != map->def.value_size || map->fd >= 0)
7469 return -EINVAL;
7470
7471 memcpy(map->mmaped, data, size);
7472 return 0;
7473}
7474
7475bool bpf_map__is_offload_neutral(const struct bpf_map *map)
7476{
7477 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
7478}
7479
7480bool bpf_map__is_internal(const struct bpf_map *map)
7481{
7482 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
7483}
7484
7485__u32 bpf_map__ifindex(const struct bpf_map *map)
7486{
7487 return map->map_ifindex;
7488}
7489
7490int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
7491{
7492 if (map->fd >= 0)
7493 return -EBUSY;
7494 map->map_ifindex = ifindex;
7495 return 0;
7496}
7497
7498int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
7499{
7500 if (!bpf_map_type__is_map_in_map(map->def.type)) {
7501 pr_warn("error: unsupported map type\n");
7502 return -EINVAL;
7503 }
7504 if (map->inner_map_fd != -1) {
7505 pr_warn("error: inner_map_fd already specified\n");
7506 return -EINVAL;
7507 }
7508 map->inner_map_fd = fd;
7509 return 0;
7510}
7511
7512static struct bpf_map *
7513__bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
7514{
7515 ssize_t idx;
7516 struct bpf_map *s, *e;
7517
7518 if (!obj || !obj->maps)
7519 return NULL;
7520
7521 s = obj->maps;
7522 e = obj->maps + obj->nr_maps;
7523
7524 if ((m < s) || (m >= e)) {
7525 pr_warn("error in %s: map handler doesn't belong to object\n",
7526 __func__);
7527 return NULL;
7528 }
7529
7530 idx = (m - obj->maps) + i;
7531 if (idx >= obj->nr_maps || idx < 0)
7532 return NULL;
7533 return &obj->maps[idx];
7534}
7535
7536struct bpf_map *
7537bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
7538{
7539 if (prev == NULL)
7540 return obj->maps;
7541
7542 return __bpf_map__iter(prev, obj, 1);
7543}
7544
7545struct bpf_map *
7546bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj)
7547{
7548 if (next == NULL) {
7549 if (!obj->nr_maps)
7550 return NULL;
7551 return obj->maps + obj->nr_maps - 1;
7552 }
7553
7554 return __bpf_map__iter(next, obj, -1);
7555}
7556
7557struct bpf_map *
7558bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
7559{
7560 struct bpf_map *pos;
7561
7562 bpf_object__for_each_map(pos, obj) {
7563 if (pos->name && !strcmp(pos->name, name))
7564 return pos;
7565 }
7566 return NULL;
7567}
7568
7569int
7570bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
7571{
7572 return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
7573}
7574
7575struct bpf_map *
7576bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
7577{
7578 return ERR_PTR(-ENOTSUP);
7579}
7580
7581long libbpf_get_error(const void *ptr)
7582{
7583 return PTR_ERR_OR_ZERO(ptr);
7584}
7585
7586int bpf_prog_load(const char *file, enum bpf_prog_type type,
7587 struct bpf_object **pobj, int *prog_fd)
7588{
7589 struct bpf_prog_load_attr attr;
7590
7591 memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
7592 attr.file = file;
7593 attr.prog_type = type;
7594 attr.expected_attach_type = 0;
7595
7596 return bpf_prog_load_xattr(&attr, pobj, prog_fd);
7597}
7598
7599int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
7600 struct bpf_object **pobj, int *prog_fd)
7601{
7602 struct bpf_object_open_attr open_attr = {};
7603 struct bpf_program *prog, *first_prog = NULL;
7604 struct bpf_object *obj;
7605 struct bpf_map *map;
7606 int err;
7607
7608 if (!attr)
7609 return -EINVAL;
7610 if (!attr->file)
7611 return -EINVAL;
7612
7613 open_attr.file = attr->file;
7614 open_attr.prog_type = attr->prog_type;
7615
7616 obj = bpf_object__open_xattr(&open_attr);
7617 if (IS_ERR_OR_NULL(obj))
7618 return -ENOENT;
7619
7620 bpf_object__for_each_program(prog, obj) {
7621 enum bpf_attach_type attach_type = attr->expected_attach_type;
7622
7623
7624
7625
7626
7627 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) {
7628 bpf_program__set_type(prog, attr->prog_type);
7629 bpf_program__set_expected_attach_type(prog,
7630 attach_type);
7631 }
7632 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) {
7633
7634
7635
7636
7637 bpf_object__close(obj);
7638 return -EINVAL;
7639 }
7640
7641 prog->prog_ifindex = attr->ifindex;
7642 prog->log_level = attr->log_level;
7643 prog->prog_flags = attr->prog_flags;
7644 if (!first_prog)
7645 first_prog = prog;
7646 }
7647
7648 bpf_object__for_each_map(map, obj) {
7649 if (!bpf_map__is_offload_neutral(map))
7650 map->map_ifindex = attr->ifindex;
7651 }
7652
7653 if (!first_prog) {
7654 pr_warn("object file doesn't contain bpf program\n");
7655 bpf_object__close(obj);
7656 return -ENOENT;
7657 }
7658
7659 err = bpf_object__load(obj);
7660 if (err) {
7661 bpf_object__close(obj);
7662 return err;
7663 }
7664
7665 *pobj = obj;
7666 *prog_fd = bpf_program__fd(first_prog);
7667 return 0;
7668}
7669
7670struct bpf_link {
7671 int (*detach)(struct bpf_link *link);
7672 int (*destroy)(struct bpf_link *link);
7673 char *pin_path;
7674 int fd;
7675 bool disconnected;
7676};
7677
7678
7679int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
7680{
7681 return bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
7682}
7683
7684
7685
7686
7687
7688
7689
7690
7691
7692
7693
7694void bpf_link__disconnect(struct bpf_link *link)
7695{
7696 link->disconnected = true;
7697}
7698
7699int bpf_link__destroy(struct bpf_link *link)
7700{
7701 int err = 0;
7702
7703 if (IS_ERR_OR_NULL(link))
7704 return 0;
7705
7706 if (!link->disconnected && link->detach)
7707 err = link->detach(link);
7708 if (link->destroy)
7709 link->destroy(link);
7710 if (link->pin_path)
7711 free(link->pin_path);
7712 free(link);
7713
7714 return err;
7715}
7716
7717int bpf_link__fd(const struct bpf_link *link)
7718{
7719 return link->fd;
7720}
7721
7722const char *bpf_link__pin_path(const struct bpf_link *link)
7723{
7724 return link->pin_path;
7725}
7726
7727static int bpf_link__detach_fd(struct bpf_link *link)
7728{
7729 return close(link->fd);
7730}
7731
7732struct bpf_link *bpf_link__open(const char *path)
7733{
7734 struct bpf_link *link;
7735 int fd;
7736
7737 fd = bpf_obj_get(path);
7738 if (fd < 0) {
7739 fd = -errno;
7740 pr_warn("failed to open link at %s: %d\n", path, fd);
7741 return ERR_PTR(fd);
7742 }
7743
7744 link = calloc(1, sizeof(*link));
7745 if (!link) {
7746 close(fd);
7747 return ERR_PTR(-ENOMEM);
7748 }
7749 link->detach = &bpf_link__detach_fd;
7750 link->fd = fd;
7751
7752 link->pin_path = strdup(path);
7753 if (!link->pin_path) {
7754 bpf_link__destroy(link);
7755 return ERR_PTR(-ENOMEM);
7756 }
7757
7758 return link;
7759}
7760
7761int bpf_link__detach(struct bpf_link *link)
7762{
7763 return bpf_link_detach(link->fd) ? -errno : 0;
7764}
7765
7766int bpf_link__pin(struct bpf_link *link, const char *path)
7767{
7768 int err;
7769
7770 if (link->pin_path)
7771 return -EBUSY;
7772 err = make_parent_dir(path);
7773 if (err)
7774 return err;
7775 err = check_path(path);
7776 if (err)
7777 return err;
7778
7779 link->pin_path = strdup(path);
7780 if (!link->pin_path)
7781 return -ENOMEM;
7782
7783 if (bpf_obj_pin(link->fd, link->pin_path)) {
7784 err = -errno;
7785 zfree(&link->pin_path);
7786 return err;
7787 }
7788
7789 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
7790 return 0;
7791}
7792
7793int bpf_link__unpin(struct bpf_link *link)
7794{
7795 int err;
7796
7797 if (!link->pin_path)
7798 return -EINVAL;
7799
7800 err = unlink(link->pin_path);
7801 if (err != 0)
7802 return -errno;
7803
7804 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
7805 zfree(&link->pin_path);
7806 return 0;
7807}
7808
7809static int bpf_link__detach_perf_event(struct bpf_link *link)
7810{
7811 int err;
7812
7813 err = ioctl(link->fd, PERF_EVENT_IOC_DISABLE, 0);
7814 if (err)
7815 err = -errno;
7816
7817 close(link->fd);
7818 return err;
7819}
7820
7821struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog,
7822 int pfd)
7823{
7824 char errmsg[STRERR_BUFSIZE];
7825 struct bpf_link *link;
7826 int prog_fd, err;
7827
7828 if (pfd < 0) {
7829 pr_warn("program '%s': invalid perf event FD %d\n",
7830 bpf_program__title(prog, false), pfd);
7831 return ERR_PTR(-EINVAL);
7832 }
7833 prog_fd = bpf_program__fd(prog);
7834 if (prog_fd < 0) {
7835 pr_warn("program '%s': can't attach BPF program w/o FD (did you load it?)\n",
7836 bpf_program__title(prog, false));
7837 return ERR_PTR(-EINVAL);
7838 }
7839
7840 link = calloc(1, sizeof(*link));
7841 if (!link)
7842 return ERR_PTR(-ENOMEM);
7843 link->detach = &bpf_link__detach_perf_event;
7844 link->fd = pfd;
7845
7846 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
7847 err = -errno;
7848 free(link);
7849 pr_warn("program '%s': failed to attach to pfd %d: %s\n",
7850 bpf_program__title(prog, false), pfd,
7851 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7852 if (err == -EPROTO)
7853 pr_warn("program '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
7854 bpf_program__title(prog, false), pfd);
7855 return ERR_PTR(err);
7856 }
7857 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
7858 err = -errno;
7859 free(link);
7860 pr_warn("program '%s': failed to enable pfd %d: %s\n",
7861 bpf_program__title(prog, false), pfd,
7862 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7863 return ERR_PTR(err);
7864 }
7865 return link;
7866}
7867
7868
7869
7870
7871
7872
7873static int parse_uint_from_file(const char *file, const char *fmt)
7874{
7875 char buf[STRERR_BUFSIZE];
7876 int err, ret;
7877 FILE *f;
7878
7879 f = fopen(file, "r");
7880 if (!f) {
7881 err = -errno;
7882 pr_debug("failed to open '%s': %s\n", file,
7883 libbpf_strerror_r(err, buf, sizeof(buf)));
7884 return err;
7885 }
7886 err = fscanf(f, fmt, &ret);
7887 if (err != 1) {
7888 err = err == EOF ? -EIO : -errno;
7889 pr_debug("failed to parse '%s': %s\n", file,
7890 libbpf_strerror_r(err, buf, sizeof(buf)));
7891 fclose(f);
7892 return err;
7893 }
7894 fclose(f);
7895 return ret;
7896}
7897
7898static int determine_kprobe_perf_type(void)
7899{
7900 const char *file = "/sys/bus/event_source/devices/kprobe/type";
7901
7902 return parse_uint_from_file(file, "%d\n");
7903}
7904
7905static int determine_uprobe_perf_type(void)
7906{
7907 const char *file = "/sys/bus/event_source/devices/uprobe/type";
7908
7909 return parse_uint_from_file(file, "%d\n");
7910}
7911
7912static int determine_kprobe_retprobe_bit(void)
7913{
7914 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
7915
7916 return parse_uint_from_file(file, "config:%d\n");
7917}
7918
7919static int determine_uprobe_retprobe_bit(void)
7920{
7921 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
7922
7923 return parse_uint_from_file(file, "config:%d\n");
7924}
7925
7926static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
7927 uint64_t offset, int pid)
7928{
7929 struct perf_event_attr attr = {};
7930 char errmsg[STRERR_BUFSIZE];
7931 int type, pfd, err;
7932
7933 type = uprobe ? determine_uprobe_perf_type()
7934 : determine_kprobe_perf_type();
7935 if (type < 0) {
7936 pr_warn("failed to determine %s perf type: %s\n",
7937 uprobe ? "uprobe" : "kprobe",
7938 libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
7939 return type;
7940 }
7941 if (retprobe) {
7942 int bit = uprobe ? determine_uprobe_retprobe_bit()
7943 : determine_kprobe_retprobe_bit();
7944
7945 if (bit < 0) {
7946 pr_warn("failed to determine %s retprobe bit: %s\n",
7947 uprobe ? "uprobe" : "kprobe",
7948 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
7949 return bit;
7950 }
7951 attr.config |= 1 << bit;
7952 }
7953 attr.size = sizeof(attr);
7954 attr.type = type;
7955 attr.config1 = ptr_to_u64(name);
7956 attr.config2 = offset;
7957
7958
7959 pfd = syscall(__NR_perf_event_open, &attr,
7960 pid < 0 ? -1 : pid ,
7961 pid == -1 ? 0 : -1 ,
7962 -1 , PERF_FLAG_FD_CLOEXEC);
7963 if (pfd < 0) {
7964 err = -errno;
7965 pr_warn("%s perf_event_open() failed: %s\n",
7966 uprobe ? "uprobe" : "kprobe",
7967 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7968 return err;
7969 }
7970 return pfd;
7971}
7972
7973struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog,
7974 bool retprobe,
7975 const char *func_name)
7976{
7977 char errmsg[STRERR_BUFSIZE];
7978 struct bpf_link *link;
7979 int pfd, err;
7980
7981 pfd = perf_event_open_probe(false , retprobe, func_name,
7982 0 , -1 );
7983 if (pfd < 0) {
7984 pr_warn("program '%s': failed to create %s '%s' perf event: %s\n",
7985 bpf_program__title(prog, false),
7986 retprobe ? "kretprobe" : "kprobe", func_name,
7987 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
7988 return ERR_PTR(pfd);
7989 }
7990 link = bpf_program__attach_perf_event(prog, pfd);
7991 if (IS_ERR(link)) {
7992 close(pfd);
7993 err = PTR_ERR(link);
7994 pr_warn("program '%s': failed to attach to %s '%s': %s\n",
7995 bpf_program__title(prog, false),
7996 retprobe ? "kretprobe" : "kprobe", func_name,
7997 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
7998 return link;
7999 }
8000 return link;
8001}
8002
8003static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
8004 struct bpf_program *prog)
8005{
8006 const char *func_name;
8007 bool retprobe;
8008
8009 func_name = bpf_program__title(prog, false) + sec->len;
8010 retprobe = strcmp(sec->sec, "kretprobe/") == 0;
8011
8012 return bpf_program__attach_kprobe(prog, retprobe, func_name);
8013}
8014
8015struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog,
8016 bool retprobe, pid_t pid,
8017 const char *binary_path,
8018 size_t func_offset)
8019{
8020 char errmsg[STRERR_BUFSIZE];
8021 struct bpf_link *link;
8022 int pfd, err;
8023
8024 pfd = perf_event_open_probe(true , retprobe,
8025 binary_path, func_offset, pid);
8026 if (pfd < 0) {
8027 pr_warn("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
8028 bpf_program__title(prog, false),
8029 retprobe ? "uretprobe" : "uprobe",
8030 binary_path, func_offset,
8031 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
8032 return ERR_PTR(pfd);
8033 }
8034 link = bpf_program__attach_perf_event(prog, pfd);
8035 if (IS_ERR(link)) {
8036 close(pfd);
8037 err = PTR_ERR(link);
8038 pr_warn("program '%s': failed to attach to %s '%s:0x%zx': %s\n",
8039 bpf_program__title(prog, false),
8040 retprobe ? "uretprobe" : "uprobe",
8041 binary_path, func_offset,
8042 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
8043 return link;
8044 }
8045 return link;
8046}
8047
8048static int determine_tracepoint_id(const char *tp_category,
8049 const char *tp_name)
8050{
8051 char file[PATH_MAX];
8052 int ret;
8053
8054 ret = snprintf(file, sizeof(file),
8055 "/sys/kernel/debug/tracing/events/%s/%s/id",
8056 tp_category, tp_name);
8057 if (ret < 0)
8058 return -errno;
8059 if (ret >= sizeof(file)) {
8060 pr_debug("tracepoint %s/%s path is too long\n",
8061 tp_category, tp_name);
8062 return -E2BIG;
8063 }
8064 return parse_uint_from_file(file, "%d\n");
8065}
8066
8067static int perf_event_open_tracepoint(const char *tp_category,
8068 const char *tp_name)
8069{
8070 struct perf_event_attr attr = {};
8071 char errmsg[STRERR_BUFSIZE];
8072 int tp_id, pfd, err;
8073
8074 tp_id = determine_tracepoint_id(tp_category, tp_name);
8075 if (tp_id < 0) {
8076 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
8077 tp_category, tp_name,
8078 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
8079 return tp_id;
8080 }
8081
8082 attr.type = PERF_TYPE_TRACEPOINT;
8083 attr.size = sizeof(attr);
8084 attr.config = tp_id;
8085
8086 pfd = syscall(__NR_perf_event_open, &attr, -1 , 0 ,
8087 -1 , PERF_FLAG_FD_CLOEXEC);
8088 if (pfd < 0) {
8089 err = -errno;
8090 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
8091 tp_category, tp_name,
8092 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
8093 return err;
8094 }
8095 return pfd;
8096}
8097
8098struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,
8099 const char *tp_category,
8100 const char *tp_name)
8101{
8102 char errmsg[STRERR_BUFSIZE];
8103 struct bpf_link *link;
8104 int pfd, err;
8105
8106 pfd = perf_event_open_tracepoint(tp_category, tp_name);
8107 if (pfd < 0) {
8108 pr_warn("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
8109 bpf_program__title(prog, false),
8110 tp_category, tp_name,
8111 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
8112 return ERR_PTR(pfd);
8113 }
8114 link = bpf_program__attach_perf_event(prog, pfd);
8115 if (IS_ERR(link)) {
8116 close(pfd);
8117 err = PTR_ERR(link);
8118 pr_warn("program '%s': failed to attach to tracepoint '%s/%s': %s\n",
8119 bpf_program__title(prog, false),
8120 tp_category, tp_name,
8121 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
8122 return link;
8123 }
8124 return link;
8125}
8126
8127static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
8128 struct bpf_program *prog)
8129{
8130 char *sec_name, *tp_cat, *tp_name;
8131 struct bpf_link *link;
8132
8133 sec_name = strdup(bpf_program__title(prog, false));
8134 if (!sec_name)
8135 return ERR_PTR(-ENOMEM);
8136
8137
8138 tp_cat = sec_name + sec->len;
8139 tp_name = strchr(tp_cat, '/');
8140 if (!tp_name) {
8141 link = ERR_PTR(-EINVAL);
8142 goto out;
8143 }
8144 *tp_name = '\0';
8145 tp_name++;
8146
8147 link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
8148out:
8149 free(sec_name);
8150 return link;
8151}
8152
8153struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
8154 const char *tp_name)
8155{
8156 char errmsg[STRERR_BUFSIZE];
8157 struct bpf_link *link;
8158 int prog_fd, pfd;
8159
8160 prog_fd = bpf_program__fd(prog);
8161 if (prog_fd < 0) {
8162 pr_warn("program '%s': can't attach before loaded\n",
8163 bpf_program__title(prog, false));
8164 return ERR_PTR(-EINVAL);
8165 }
8166
8167 link = calloc(1, sizeof(*link));
8168 if (!link)
8169 return ERR_PTR(-ENOMEM);
8170 link->detach = &bpf_link__detach_fd;
8171
8172 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
8173 if (pfd < 0) {
8174 pfd = -errno;
8175 free(link);
8176 pr_warn("program '%s': failed to attach to raw tracepoint '%s': %s\n",
8177 bpf_program__title(prog, false), tp_name,
8178 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
8179 return ERR_PTR(pfd);
8180 }
8181 link->fd = pfd;
8182 return link;
8183}
8184
8185static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
8186 struct bpf_program *prog)
8187{
8188 const char *tp_name = bpf_program__title(prog, false) + sec->len;
8189
8190 return bpf_program__attach_raw_tracepoint(prog, tp_name);
8191}
8192
8193
8194static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog)
8195{
8196 char errmsg[STRERR_BUFSIZE];
8197 struct bpf_link *link;
8198 int prog_fd, pfd;
8199
8200 prog_fd = bpf_program__fd(prog);
8201 if (prog_fd < 0) {
8202 pr_warn("program '%s': can't attach before loaded\n",
8203 bpf_program__title(prog, false));
8204 return ERR_PTR(-EINVAL);
8205 }
8206
8207 link = calloc(1, sizeof(*link));
8208 if (!link)
8209 return ERR_PTR(-ENOMEM);
8210 link->detach = &bpf_link__detach_fd;
8211
8212 pfd = bpf_raw_tracepoint_open(NULL, prog_fd);
8213 if (pfd < 0) {
8214 pfd = -errno;
8215 free(link);
8216 pr_warn("program '%s': failed to attach: %s\n",
8217 bpf_program__title(prog, false),
8218 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
8219 return ERR_PTR(pfd);
8220 }
8221 link->fd = pfd;
8222 return (struct bpf_link *)link;
8223}
8224
8225struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog)
8226{
8227 return bpf_program__attach_btf_id(prog);
8228}
8229
8230struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog)
8231{
8232 return bpf_program__attach_btf_id(prog);
8233}
8234
8235static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
8236 struct bpf_program *prog)
8237{
8238 return bpf_program__attach_trace(prog);
8239}
8240
8241static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
8242 struct bpf_program *prog)
8243{
8244 return bpf_program__attach_lsm(prog);
8245}
8246
8247static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
8248 struct bpf_program *prog)
8249{
8250 return bpf_program__attach_iter(prog, NULL);
8251}
8252
8253static struct bpf_link *
8254bpf_program__attach_fd(struct bpf_program *prog, int target_fd,
8255 const char *target_name)
8256{
8257 enum bpf_attach_type attach_type;
8258 char errmsg[STRERR_BUFSIZE];
8259 struct bpf_link *link;
8260 int prog_fd, link_fd;
8261
8262 prog_fd = bpf_program__fd(prog);
8263 if (prog_fd < 0) {
8264 pr_warn("program '%s': can't attach before loaded\n",
8265 bpf_program__title(prog, false));
8266 return ERR_PTR(-EINVAL);
8267 }
8268
8269 link = calloc(1, sizeof(*link));
8270 if (!link)
8271 return ERR_PTR(-ENOMEM);
8272 link->detach = &bpf_link__detach_fd;
8273
8274 attach_type = bpf_program__get_expected_attach_type(prog);
8275 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, NULL);
8276 if (link_fd < 0) {
8277 link_fd = -errno;
8278 free(link);
8279 pr_warn("program '%s': failed to attach to %s: %s\n",
8280 bpf_program__title(prog, false), target_name,
8281 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
8282 return ERR_PTR(link_fd);
8283 }
8284 link->fd = link_fd;
8285 return link;
8286}
8287
8288struct bpf_link *
8289bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd)
8290{
8291 return bpf_program__attach_fd(prog, cgroup_fd, "cgroup");
8292}
8293
8294struct bpf_link *
8295bpf_program__attach_netns(struct bpf_program *prog, int netns_fd)
8296{
8297 return bpf_program__attach_fd(prog, netns_fd, "netns");
8298}
8299
8300struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex)
8301{
8302
8303 return bpf_program__attach_fd(prog, ifindex, "xdp");
8304}
8305
8306struct bpf_link *
8307bpf_program__attach_iter(struct bpf_program *prog,
8308 const struct bpf_iter_attach_opts *opts)
8309{
8310 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
8311 char errmsg[STRERR_BUFSIZE];
8312 struct bpf_link *link;
8313 int prog_fd, link_fd;
8314 __u32 target_fd = 0;
8315
8316 if (!OPTS_VALID(opts, bpf_iter_attach_opts))
8317 return ERR_PTR(-EINVAL);
8318
8319 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
8320 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
8321
8322 prog_fd = bpf_program__fd(prog);
8323 if (prog_fd < 0) {
8324 pr_warn("program '%s': can't attach before loaded\n",
8325 bpf_program__title(prog, false));
8326 return ERR_PTR(-EINVAL);
8327 }
8328
8329 link = calloc(1, sizeof(*link));
8330 if (!link)
8331 return ERR_PTR(-ENOMEM);
8332 link->detach = &bpf_link__detach_fd;
8333
8334 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
8335 &link_create_opts);
8336 if (link_fd < 0) {
8337 link_fd = -errno;
8338 free(link);
8339 pr_warn("program '%s': failed to attach to iterator: %s\n",
8340 bpf_program__title(prog, false),
8341 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
8342 return ERR_PTR(link_fd);
8343 }
8344 link->fd = link_fd;
8345 return link;
8346}
8347
8348struct bpf_link *bpf_program__attach(struct bpf_program *prog)
8349{
8350 const struct bpf_sec_def *sec_def;
8351
8352 sec_def = find_sec_def(bpf_program__title(prog, false));
8353 if (!sec_def || !sec_def->attach_fn)
8354 return ERR_PTR(-ESRCH);
8355
8356 return sec_def->attach_fn(sec_def, prog);
8357}
8358
8359static int bpf_link__detach_struct_ops(struct bpf_link *link)
8360{
8361 __u32 zero = 0;
8362
8363 if (bpf_map_delete_elem(link->fd, &zero))
8364 return -errno;
8365
8366 return 0;
8367}
8368
8369struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map)
8370{
8371 struct bpf_struct_ops *st_ops;
8372 struct bpf_link *link;
8373 __u32 i, zero = 0;
8374 int err;
8375
8376 if (!bpf_map__is_struct_ops(map) || map->fd == -1)
8377 return ERR_PTR(-EINVAL);
8378
8379 link = calloc(1, sizeof(*link));
8380 if (!link)
8381 return ERR_PTR(-EINVAL);
8382
8383 st_ops = map->st_ops;
8384 for (i = 0; i < btf_vlen(st_ops->type); i++) {
8385 struct bpf_program *prog = st_ops->progs[i];
8386 void *kern_data;
8387 int prog_fd;
8388
8389 if (!prog)
8390 continue;
8391
8392 prog_fd = bpf_program__fd(prog);
8393 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
8394 *(unsigned long *)kern_data = prog_fd;
8395 }
8396
8397 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
8398 if (err) {
8399 err = -errno;
8400 free(link);
8401 return ERR_PTR(err);
8402 }
8403
8404 link->detach = bpf_link__detach_struct_ops;
8405 link->fd = map->fd;
8406
8407 return link;
8408}
8409
8410enum bpf_perf_event_ret
8411bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
8412 void **copy_mem, size_t *copy_size,
8413 bpf_perf_event_print_t fn, void *private_data)
8414{
8415 struct perf_event_mmap_page *header = mmap_mem;
8416 __u64 data_head = ring_buffer_read_head(header);
8417 __u64 data_tail = header->data_tail;
8418 void *base = ((__u8 *)header) + page_size;
8419 int ret = LIBBPF_PERF_EVENT_CONT;
8420 struct perf_event_header *ehdr;
8421 size_t ehdr_size;
8422
8423 while (data_head != data_tail) {
8424 ehdr = base + (data_tail & (mmap_size - 1));
8425 ehdr_size = ehdr->size;
8426
8427 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
8428 void *copy_start = ehdr;
8429 size_t len_first = base + mmap_size - copy_start;
8430 size_t len_secnd = ehdr_size - len_first;
8431
8432 if (*copy_size < ehdr_size) {
8433 free(*copy_mem);
8434 *copy_mem = malloc(ehdr_size);
8435 if (!*copy_mem) {
8436 *copy_size = 0;
8437 ret = LIBBPF_PERF_EVENT_ERROR;
8438 break;
8439 }
8440 *copy_size = ehdr_size;
8441 }
8442
8443 memcpy(*copy_mem, copy_start, len_first);
8444 memcpy(*copy_mem + len_first, base, len_secnd);
8445 ehdr = *copy_mem;
8446 }
8447
8448 ret = fn(ehdr, private_data);
8449 data_tail += ehdr_size;
8450 if (ret != LIBBPF_PERF_EVENT_CONT)
8451 break;
8452 }
8453
8454 ring_buffer_write_tail(header, data_tail);
8455 return ret;
8456}
8457
8458struct perf_buffer;
8459
8460struct perf_buffer_params {
8461 struct perf_event_attr *attr;
8462
8463 perf_buffer_event_fn event_cb;
8464
8465 perf_buffer_sample_fn sample_cb;
8466 perf_buffer_lost_fn lost_cb;
8467 void *ctx;
8468 int cpu_cnt;
8469 int *cpus;
8470 int *map_keys;
8471};
8472
8473struct perf_cpu_buf {
8474 struct perf_buffer *pb;
8475 void *base;
8476 void *buf;
8477 size_t buf_size;
8478 int fd;
8479 int cpu;
8480 int map_key;
8481};
8482
8483struct perf_buffer {
8484 perf_buffer_event_fn event_cb;
8485 perf_buffer_sample_fn sample_cb;
8486 perf_buffer_lost_fn lost_cb;
8487 void *ctx;
8488
8489 size_t page_size;
8490 size_t mmap_size;
8491 struct perf_cpu_buf **cpu_bufs;
8492 struct epoll_event *events;
8493 int cpu_cnt;
8494 int epoll_fd;
8495 int map_fd;
8496};
8497
8498static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
8499 struct perf_cpu_buf *cpu_buf)
8500{
8501 if (!cpu_buf)
8502 return;
8503 if (cpu_buf->base &&
8504 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
8505 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
8506 if (cpu_buf->fd >= 0) {
8507 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
8508 close(cpu_buf->fd);
8509 }
8510 free(cpu_buf->buf);
8511 free(cpu_buf);
8512}
8513
8514void perf_buffer__free(struct perf_buffer *pb)
8515{
8516 int i;
8517
8518 if (IS_ERR_OR_NULL(pb))
8519 return;
8520 if (pb->cpu_bufs) {
8521 for (i = 0; i < pb->cpu_cnt; i++) {
8522 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
8523
8524 if (!cpu_buf)
8525 continue;
8526
8527 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
8528 perf_buffer__free_cpu_buf(pb, cpu_buf);
8529 }
8530 free(pb->cpu_bufs);
8531 }
8532 if (pb->epoll_fd >= 0)
8533 close(pb->epoll_fd);
8534 free(pb->events);
8535 free(pb);
8536}
8537
8538static struct perf_cpu_buf *
8539perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
8540 int cpu, int map_key)
8541{
8542 struct perf_cpu_buf *cpu_buf;
8543 char msg[STRERR_BUFSIZE];
8544 int err;
8545
8546 cpu_buf = calloc(1, sizeof(*cpu_buf));
8547 if (!cpu_buf)
8548 return ERR_PTR(-ENOMEM);
8549
8550 cpu_buf->pb = pb;
8551 cpu_buf->cpu = cpu;
8552 cpu_buf->map_key = map_key;
8553
8554 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 , cpu,
8555 -1, PERF_FLAG_FD_CLOEXEC);
8556 if (cpu_buf->fd < 0) {
8557 err = -errno;
8558 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
8559 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
8560 goto error;
8561 }
8562
8563 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
8564 PROT_READ | PROT_WRITE, MAP_SHARED,
8565 cpu_buf->fd, 0);
8566 if (cpu_buf->base == MAP_FAILED) {
8567 cpu_buf->base = NULL;
8568 err = -errno;
8569 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
8570 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
8571 goto error;
8572 }
8573
8574 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
8575 err = -errno;
8576 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
8577 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
8578 goto error;
8579 }
8580
8581 return cpu_buf;
8582
8583error:
8584 perf_buffer__free_cpu_buf(pb, cpu_buf);
8585 return (struct perf_cpu_buf *)ERR_PTR(err);
8586}
8587
8588static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
8589 struct perf_buffer_params *p);
8590
8591struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
8592 const struct perf_buffer_opts *opts)
8593{
8594 struct perf_buffer_params p = {};
8595 struct perf_event_attr attr = { 0, };
8596
8597 attr.config = PERF_COUNT_SW_BPF_OUTPUT,
8598 attr.type = PERF_TYPE_SOFTWARE;
8599 attr.sample_type = PERF_SAMPLE_RAW;
8600 attr.sample_period = 1;
8601 attr.wakeup_events = 1;
8602
8603 p.attr = &attr;
8604 p.sample_cb = opts ? opts->sample_cb : NULL;
8605 p.lost_cb = opts ? opts->lost_cb : NULL;
8606 p.ctx = opts ? opts->ctx : NULL;
8607
8608 return __perf_buffer__new(map_fd, page_cnt, &p);
8609}
8610
8611struct perf_buffer *
8612perf_buffer__new_raw(int map_fd, size_t page_cnt,
8613 const struct perf_buffer_raw_opts *opts)
8614{
8615 struct perf_buffer_params p = {};
8616
8617 p.attr = opts->attr;
8618 p.event_cb = opts->event_cb;
8619 p.ctx = opts->ctx;
8620 p.cpu_cnt = opts->cpu_cnt;
8621 p.cpus = opts->cpus;
8622 p.map_keys = opts->map_keys;
8623
8624 return __perf_buffer__new(map_fd, page_cnt, &p);
8625}
8626
8627static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
8628 struct perf_buffer_params *p)
8629{
8630 const char *online_cpus_file = "/sys/devices/system/cpu/online";
8631 struct bpf_map_info map;
8632 char msg[STRERR_BUFSIZE];
8633 struct perf_buffer *pb;
8634 bool *online = NULL;
8635 __u32 map_info_len;
8636 int err, i, j, n;
8637
8638 if (page_cnt & (page_cnt - 1)) {
8639 pr_warn("page count should be power of two, but is %zu\n",
8640 page_cnt);
8641 return ERR_PTR(-EINVAL);
8642 }
8643
8644
8645 memset(&map, 0, sizeof(map));
8646 map_info_len = sizeof(map);
8647 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
8648 if (err) {
8649 err = -errno;
8650
8651
8652
8653 if (err != -EINVAL) {
8654 pr_warn("failed to get map info for map FD %d: %s\n",
8655 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
8656 return ERR_PTR(err);
8657 }
8658 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
8659 map_fd);
8660 } else {
8661 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
8662 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
8663 map.name);
8664 return ERR_PTR(-EINVAL);
8665 }
8666 }
8667
8668 pb = calloc(1, sizeof(*pb));
8669 if (!pb)
8670 return ERR_PTR(-ENOMEM);
8671
8672 pb->event_cb = p->event_cb;
8673 pb->sample_cb = p->sample_cb;
8674 pb->lost_cb = p->lost_cb;
8675 pb->ctx = p->ctx;
8676
8677 pb->page_size = getpagesize();
8678 pb->mmap_size = pb->page_size * page_cnt;
8679 pb->map_fd = map_fd;
8680
8681 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
8682 if (pb->epoll_fd < 0) {
8683 err = -errno;
8684 pr_warn("failed to create epoll instance: %s\n",
8685 libbpf_strerror_r(err, msg, sizeof(msg)));
8686 goto error;
8687 }
8688
8689 if (p->cpu_cnt > 0) {
8690 pb->cpu_cnt = p->cpu_cnt;
8691 } else {
8692 pb->cpu_cnt = libbpf_num_possible_cpus();
8693 if (pb->cpu_cnt < 0) {
8694 err = pb->cpu_cnt;
8695 goto error;
8696 }
8697 if (map.max_entries && map.max_entries < pb->cpu_cnt)
8698 pb->cpu_cnt = map.max_entries;
8699 }
8700
8701 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
8702 if (!pb->events) {
8703 err = -ENOMEM;
8704 pr_warn("failed to allocate events: out of memory\n");
8705 goto error;
8706 }
8707 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
8708 if (!pb->cpu_bufs) {
8709 err = -ENOMEM;
8710 pr_warn("failed to allocate buffers: out of memory\n");
8711 goto error;
8712 }
8713
8714 err = parse_cpu_mask_file(online_cpus_file, &online, &n);
8715 if (err) {
8716 pr_warn("failed to get online CPU mask: %d\n", err);
8717 goto error;
8718 }
8719
8720 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
8721 struct perf_cpu_buf *cpu_buf;
8722 int cpu, map_key;
8723
8724 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
8725 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
8726
8727
8728
8729
8730 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
8731 continue;
8732
8733 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
8734 if (IS_ERR(cpu_buf)) {
8735 err = PTR_ERR(cpu_buf);
8736 goto error;
8737 }
8738
8739 pb->cpu_bufs[j] = cpu_buf;
8740
8741 err = bpf_map_update_elem(pb->map_fd, &map_key,
8742 &cpu_buf->fd, 0);
8743 if (err) {
8744 err = -errno;
8745 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
8746 cpu, map_key, cpu_buf->fd,
8747 libbpf_strerror_r(err, msg, sizeof(msg)));
8748 goto error;
8749 }
8750
8751 pb->events[j].events = EPOLLIN;
8752 pb->events[j].data.ptr = cpu_buf;
8753 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
8754 &pb->events[j]) < 0) {
8755 err = -errno;
8756 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
8757 cpu, cpu_buf->fd,
8758 libbpf_strerror_r(err, msg, sizeof(msg)));
8759 goto error;
8760 }
8761 j++;
8762 }
8763 pb->cpu_cnt = j;
8764 free(online);
8765
8766 return pb;
8767
8768error:
8769 free(online);
8770 if (pb)
8771 perf_buffer__free(pb);
8772 return ERR_PTR(err);
8773}
8774
8775struct perf_sample_raw {
8776 struct perf_event_header header;
8777 uint32_t size;
8778 char data[];
8779};
8780
8781struct perf_sample_lost {
8782 struct perf_event_header header;
8783 uint64_t id;
8784 uint64_t lost;
8785 uint64_t sample_id;
8786};
8787
8788static enum bpf_perf_event_ret
8789perf_buffer__process_record(struct perf_event_header *e, void *ctx)
8790{
8791 struct perf_cpu_buf *cpu_buf = ctx;
8792 struct perf_buffer *pb = cpu_buf->pb;
8793 void *data = e;
8794
8795
8796 if (pb->event_cb)
8797 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
8798
8799 switch (e->type) {
8800 case PERF_RECORD_SAMPLE: {
8801 struct perf_sample_raw *s = data;
8802
8803 if (pb->sample_cb)
8804 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
8805 break;
8806 }
8807 case PERF_RECORD_LOST: {
8808 struct perf_sample_lost *s = data;
8809
8810 if (pb->lost_cb)
8811 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
8812 break;
8813 }
8814 default:
8815 pr_warn("unknown perf sample type %d\n", e->type);
8816 return LIBBPF_PERF_EVENT_ERROR;
8817 }
8818 return LIBBPF_PERF_EVENT_CONT;
8819}
8820
8821static int perf_buffer__process_records(struct perf_buffer *pb,
8822 struct perf_cpu_buf *cpu_buf)
8823{
8824 enum bpf_perf_event_ret ret;
8825
8826 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,
8827 pb->page_size, &cpu_buf->buf,
8828 &cpu_buf->buf_size,
8829 perf_buffer__process_record, cpu_buf);
8830 if (ret != LIBBPF_PERF_EVENT_CONT)
8831 return ret;
8832 return 0;
8833}
8834
8835int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
8836{
8837 int i, cnt, err;
8838
8839 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
8840 for (i = 0; i < cnt; i++) {
8841 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
8842
8843 err = perf_buffer__process_records(pb, cpu_buf);
8844 if (err) {
8845 pr_warn("error while processing records: %d\n", err);
8846 return err;
8847 }
8848 }
8849 return cnt < 0 ? -errno : cnt;
8850}
8851
8852int perf_buffer__consume(struct perf_buffer *pb)
8853{
8854 int i, err;
8855
8856 for (i = 0; i < pb->cpu_cnt; i++) {
8857 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
8858
8859 if (!cpu_buf)
8860 continue;
8861
8862 err = perf_buffer__process_records(pb, cpu_buf);
8863 if (err) {
8864 pr_warn("error while processing records: %d\n", err);
8865 return err;
8866 }
8867 }
8868 return 0;
8869}
8870
8871struct bpf_prog_info_array_desc {
8872 int array_offset;
8873 int count_offset;
8874 int size_offset;
8875
8876
8877};
8878
8879static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = {
8880 [BPF_PROG_INFO_JITED_INSNS] = {
8881 offsetof(struct bpf_prog_info, jited_prog_insns),
8882 offsetof(struct bpf_prog_info, jited_prog_len),
8883 -1,
8884 },
8885 [BPF_PROG_INFO_XLATED_INSNS] = {
8886 offsetof(struct bpf_prog_info, xlated_prog_insns),
8887 offsetof(struct bpf_prog_info, xlated_prog_len),
8888 -1,
8889 },
8890 [BPF_PROG_INFO_MAP_IDS] = {
8891 offsetof(struct bpf_prog_info, map_ids),
8892 offsetof(struct bpf_prog_info, nr_map_ids),
8893 -(int)sizeof(__u32),
8894 },
8895 [BPF_PROG_INFO_JITED_KSYMS] = {
8896 offsetof(struct bpf_prog_info, jited_ksyms),
8897 offsetof(struct bpf_prog_info, nr_jited_ksyms),
8898 -(int)sizeof(__u64),
8899 },
8900 [BPF_PROG_INFO_JITED_FUNC_LENS] = {
8901 offsetof(struct bpf_prog_info, jited_func_lens),
8902 offsetof(struct bpf_prog_info, nr_jited_func_lens),
8903 -(int)sizeof(__u32),
8904 },
8905 [BPF_PROG_INFO_FUNC_INFO] = {
8906 offsetof(struct bpf_prog_info, func_info),
8907 offsetof(struct bpf_prog_info, nr_func_info),
8908 offsetof(struct bpf_prog_info, func_info_rec_size),
8909 },
8910 [BPF_PROG_INFO_LINE_INFO] = {
8911 offsetof(struct bpf_prog_info, line_info),
8912 offsetof(struct bpf_prog_info, nr_line_info),
8913 offsetof(struct bpf_prog_info, line_info_rec_size),
8914 },
8915 [BPF_PROG_INFO_JITED_LINE_INFO] = {
8916 offsetof(struct bpf_prog_info, jited_line_info),
8917 offsetof(struct bpf_prog_info, nr_jited_line_info),
8918 offsetof(struct bpf_prog_info, jited_line_info_rec_size),
8919 },
8920 [BPF_PROG_INFO_PROG_TAGS] = {
8921 offsetof(struct bpf_prog_info, prog_tags),
8922 offsetof(struct bpf_prog_info, nr_prog_tags),
8923 -(int)sizeof(__u8) * BPF_TAG_SIZE,
8924 },
8925
8926};
8927
8928static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,
8929 int offset)
8930{
8931 __u32 *array = (__u32 *)info;
8932
8933 if (offset >= 0)
8934 return array[offset / sizeof(__u32)];
8935 return -(int)offset;
8936}
8937
8938static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,
8939 int offset)
8940{
8941 __u64 *array = (__u64 *)info;
8942
8943 if (offset >= 0)
8944 return array[offset / sizeof(__u64)];
8945 return -(int)offset;
8946}
8947
8948static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
8949 __u32 val)
8950{
8951 __u32 *array = (__u32 *)info;
8952
8953 if (offset >= 0)
8954 array[offset / sizeof(__u32)] = val;
8955}
8956
8957static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
8958 __u64 val)
8959{
8960 __u64 *array = (__u64 *)info;
8961
8962 if (offset >= 0)
8963 array[offset / sizeof(__u64)] = val;
8964}
8965
8966struct bpf_prog_info_linear *
8967bpf_program__get_prog_info_linear(int fd, __u64 arrays)
8968{
8969 struct bpf_prog_info_linear *info_linear;
8970 struct bpf_prog_info info = {};
8971 __u32 info_len = sizeof(info);
8972 __u32 data_len = 0;
8973 int i, err;
8974 void *ptr;
8975
8976 if (arrays >> BPF_PROG_INFO_LAST_ARRAY)
8977 return ERR_PTR(-EINVAL);
8978
8979
8980 err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
8981 if (err) {
8982 pr_debug("can't get prog info: %s", strerror(errno));
8983 return ERR_PTR(-EFAULT);
8984 }
8985
8986
8987 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
8988 bool include_array = (arrays & (1UL << i)) > 0;
8989 struct bpf_prog_info_array_desc *desc;
8990 __u32 count, size;
8991
8992 desc = bpf_prog_info_array_desc + i;
8993
8994
8995 if (info_len < desc->array_offset + sizeof(__u32) ||
8996 info_len < desc->count_offset + sizeof(__u32) ||
8997 (desc->size_offset > 0 && info_len < desc->size_offset))
8998 include_array = false;
8999
9000 if (!include_array) {
9001 arrays &= ~(1UL << i);
9002 continue;
9003 }
9004
9005 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
9006 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
9007
9008 data_len += count * size;
9009 }
9010
9011
9012 data_len = roundup(data_len, sizeof(__u64));
9013 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len);
9014 if (!info_linear)
9015 return ERR_PTR(-ENOMEM);
9016
9017
9018 info_linear->arrays = arrays;
9019 memset(&info_linear->info, 0, sizeof(info));
9020 ptr = info_linear->data;
9021
9022 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
9023 struct bpf_prog_info_array_desc *desc;
9024 __u32 count, size;
9025
9026 if ((arrays & (1UL << i)) == 0)
9027 continue;
9028
9029 desc = bpf_prog_info_array_desc + i;
9030 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
9031 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
9032 bpf_prog_info_set_offset_u32(&info_linear->info,
9033 desc->count_offset, count);
9034 bpf_prog_info_set_offset_u32(&info_linear->info,
9035 desc->size_offset, size);
9036 bpf_prog_info_set_offset_u64(&info_linear->info,
9037 desc->array_offset,
9038 ptr_to_u64(ptr));
9039 ptr += count * size;
9040 }
9041
9042
9043 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
9044 if (err) {
9045 pr_debug("can't get prog info: %s", strerror(errno));
9046 free(info_linear);
9047 return ERR_PTR(-EFAULT);
9048 }
9049
9050
9051 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
9052 struct bpf_prog_info_array_desc *desc;
9053 __u32 v1, v2;
9054
9055 if ((arrays & (1UL << i)) == 0)
9056 continue;
9057
9058 desc = bpf_prog_info_array_desc + i;
9059 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
9060 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
9061 desc->count_offset);
9062 if (v1 != v2)
9063 pr_warn("%s: mismatch in element count\n", __func__);
9064
9065 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
9066 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
9067 desc->size_offset);
9068 if (v1 != v2)
9069 pr_warn("%s: mismatch in rec size\n", __func__);
9070 }
9071
9072
9073 info_linear->info_len = sizeof(struct bpf_prog_info);
9074 info_linear->data_len = data_len;
9075
9076 return info_linear;
9077}
9078
9079void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear)
9080{
9081 int i;
9082
9083 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
9084 struct bpf_prog_info_array_desc *desc;
9085 __u64 addr, offs;
9086
9087 if ((info_linear->arrays & (1UL << i)) == 0)
9088 continue;
9089
9090 desc = bpf_prog_info_array_desc + i;
9091 addr = bpf_prog_info_read_offset_u64(&info_linear->info,
9092 desc->array_offset);
9093 offs = addr - ptr_to_u64(info_linear->data);
9094 bpf_prog_info_set_offset_u64(&info_linear->info,
9095 desc->array_offset, offs);
9096 }
9097}
9098
9099void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear)
9100{
9101 int i;
9102
9103 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
9104 struct bpf_prog_info_array_desc *desc;
9105 __u64 addr, offs;
9106
9107 if ((info_linear->arrays & (1UL << i)) == 0)
9108 continue;
9109
9110 desc = bpf_prog_info_array_desc + i;
9111 offs = bpf_prog_info_read_offset_u64(&info_linear->info,
9112 desc->array_offset);
9113 addr = offs + ptr_to_u64(info_linear->data);
9114 bpf_prog_info_set_offset_u64(&info_linear->info,
9115 desc->array_offset, addr);
9116 }
9117}
9118
9119int bpf_program__set_attach_target(struct bpf_program *prog,
9120 int attach_prog_fd,
9121 const char *attach_func_name)
9122{
9123 int btf_id;
9124
9125 if (!prog || attach_prog_fd < 0 || !attach_func_name)
9126 return -EINVAL;
9127
9128 if (attach_prog_fd)
9129 btf_id = libbpf_find_prog_btf_id(attach_func_name,
9130 attach_prog_fd);
9131 else
9132 btf_id = __find_vmlinux_btf_id(prog->obj->btf_vmlinux,
9133 attach_func_name,
9134 prog->expected_attach_type);
9135
9136 if (btf_id < 0)
9137 return btf_id;
9138
9139 prog->attach_btf_id = btf_id;
9140 prog->attach_prog_fd = attach_prog_fd;
9141 return 0;
9142}
9143
9144int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
9145{
9146 int err = 0, n, len, start, end = -1;
9147 bool *tmp;
9148
9149 *mask = NULL;
9150 *mask_sz = 0;
9151
9152
9153 while (*s) {
9154 if (*s == ',' || *s == '\n') {
9155 s++;
9156 continue;
9157 }
9158 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
9159 if (n <= 0 || n > 2) {
9160 pr_warn("Failed to get CPU range %s: %d\n", s, n);
9161 err = -EINVAL;
9162 goto cleanup;
9163 } else if (n == 1) {
9164 end = start;
9165 }
9166 if (start < 0 || start > end) {
9167 pr_warn("Invalid CPU range [%d,%d] in %s\n",
9168 start, end, s);
9169 err = -EINVAL;
9170 goto cleanup;
9171 }
9172 tmp = realloc(*mask, end + 1);
9173 if (!tmp) {
9174 err = -ENOMEM;
9175 goto cleanup;
9176 }
9177 *mask = tmp;
9178 memset(tmp + *mask_sz, 0, start - *mask_sz);
9179 memset(tmp + start, 1, end - start + 1);
9180 *mask_sz = end + 1;
9181 s += len;
9182 }
9183 if (!*mask_sz) {
9184 pr_warn("Empty CPU range\n");
9185 return -EINVAL;
9186 }
9187 return 0;
9188cleanup:
9189 free(*mask);
9190 *mask = NULL;
9191 return err;
9192}
9193
9194int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
9195{
9196 int fd, err = 0, len;
9197 char buf[128];
9198
9199 fd = open(fcpu, O_RDONLY);
9200 if (fd < 0) {
9201 err = -errno;
9202 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
9203 return err;
9204 }
9205 len = read(fd, buf, sizeof(buf));
9206 close(fd);
9207 if (len <= 0) {
9208 err = len ? -errno : -EINVAL;
9209 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
9210 return err;
9211 }
9212 if (len >= sizeof(buf)) {
9213 pr_warn("CPU mask is too big in file %s\n", fcpu);
9214 return -E2BIG;
9215 }
9216 buf[len] = '\0';
9217
9218 return parse_cpu_mask_str(buf, mask, mask_sz);
9219}
9220
9221int libbpf_num_possible_cpus(void)
9222{
9223 static const char *fcpu = "/sys/devices/system/cpu/possible";
9224 static int cpus;
9225 int err, n, i, tmp_cpus;
9226 bool *mask;
9227
9228 tmp_cpus = READ_ONCE(cpus);
9229 if (tmp_cpus > 0)
9230 return tmp_cpus;
9231
9232 err = parse_cpu_mask_file(fcpu, &mask, &n);
9233 if (err)
9234 return err;
9235
9236 tmp_cpus = 0;
9237 for (i = 0; i < n; i++) {
9238 if (mask[i])
9239 tmp_cpus++;
9240 }
9241 free(mask);
9242
9243 WRITE_ONCE(cpus, tmp_cpus);
9244 return tmp_cpus;
9245}
9246
9247int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
9248 const struct bpf_object_open_opts *opts)
9249{
9250 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
9251 .object_name = s->name,
9252 );
9253 struct bpf_object *obj;
9254 int i;
9255
9256
9257
9258
9259
9260
9261
9262 if (opts) {
9263 memcpy(&skel_opts, opts, sizeof(*opts));
9264 if (!opts->object_name)
9265 skel_opts.object_name = s->name;
9266 }
9267
9268 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
9269 if (IS_ERR(obj)) {
9270 pr_warn("failed to initialize skeleton BPF object '%s': %ld\n",
9271 s->name, PTR_ERR(obj));
9272 return PTR_ERR(obj);
9273 }
9274
9275 *s->obj = obj;
9276
9277 for (i = 0; i < s->map_cnt; i++) {
9278 struct bpf_map **map = s->maps[i].map;
9279 const char *name = s->maps[i].name;
9280 void **mmaped = s->maps[i].mmaped;
9281
9282 *map = bpf_object__find_map_by_name(obj, name);
9283 if (!*map) {
9284 pr_warn("failed to find skeleton map '%s'\n", name);
9285 return -ESRCH;
9286 }
9287
9288
9289 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
9290 *mmaped = (*map)->mmaped;
9291 }
9292
9293 for (i = 0; i < s->prog_cnt; i++) {
9294 struct bpf_program **prog = s->progs[i].prog;
9295 const char *name = s->progs[i].name;
9296
9297 *prog = bpf_object__find_program_by_name(obj, name);
9298 if (!*prog) {
9299 pr_warn("failed to find skeleton program '%s'\n", name);
9300 return -ESRCH;
9301 }
9302 }
9303
9304 return 0;
9305}
9306
9307int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
9308{
9309 int i, err;
9310
9311 err = bpf_object__load(*s->obj);
9312 if (err) {
9313 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
9314 return err;
9315 }
9316
9317 for (i = 0; i < s->map_cnt; i++) {
9318 struct bpf_map *map = *s->maps[i].map;
9319 size_t mmap_sz = bpf_map_mmap_sz(map);
9320 int prot, map_fd = bpf_map__fd(map);
9321 void **mmaped = s->maps[i].mmaped;
9322
9323 if (!mmaped)
9324 continue;
9325
9326 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
9327 *mmaped = NULL;
9328 continue;
9329 }
9330
9331 if (map->def.map_flags & BPF_F_RDONLY_PROG)
9332 prot = PROT_READ;
9333 else
9334 prot = PROT_READ | PROT_WRITE;
9335
9336
9337
9338
9339
9340
9341
9342
9343
9344
9345
9346 *mmaped = mmap(map->mmaped, mmap_sz, prot,
9347 MAP_SHARED | MAP_FIXED, map_fd, 0);
9348 if (*mmaped == MAP_FAILED) {
9349 err = -errno;
9350 *mmaped = NULL;
9351 pr_warn("failed to re-mmap() map '%s': %d\n",
9352 bpf_map__name(map), err);
9353 return err;
9354 }
9355 }
9356
9357 return 0;
9358}
9359
9360int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
9361{
9362 int i;
9363
9364 for (i = 0; i < s->prog_cnt; i++) {
9365 struct bpf_program *prog = *s->progs[i].prog;
9366 struct bpf_link **link = s->progs[i].link;
9367 const struct bpf_sec_def *sec_def;
9368 const char *sec_name = bpf_program__title(prog, false);
9369
9370 if (!prog->load)
9371 continue;
9372
9373 sec_def = find_sec_def(sec_name);
9374 if (!sec_def || !sec_def->attach_fn)
9375 continue;
9376
9377 *link = sec_def->attach_fn(sec_def, prog);
9378 if (IS_ERR(*link)) {
9379 pr_warn("failed to auto-attach program '%s': %ld\n",
9380 bpf_program__name(prog), PTR_ERR(*link));
9381 return PTR_ERR(*link);
9382 }
9383 }
9384
9385 return 0;
9386}
9387
9388void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
9389{
9390 int i;
9391
9392 for (i = 0; i < s->prog_cnt; i++) {
9393 struct bpf_link **link = s->progs[i].link;
9394
9395 bpf_link__destroy(*link);
9396 *link = NULL;
9397 }
9398}
9399
9400void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
9401{
9402 if (s->progs)
9403 bpf_object__detach_skeleton(s);
9404 if (s->obj)
9405 bpf_object__close(*s->obj);
9406 free(s->maps);
9407 free(s->progs);
9408 free(s);
9409}
9410