1
2
3
4
5
6
7
8
9
10
11
12
13#ifndef _GNU_SOURCE
14#define _GNU_SOURCE
15#endif
16#include <stdlib.h>
17#include <stdio.h>
18#include <stdarg.h>
19#include <libgen.h>
20#include <inttypes.h>
21#include <limits.h>
22#include <string.h>
23#include <unistd.h>
24#include <endian.h>
25#include <fcntl.h>
26#include <errno.h>
27#include <ctype.h>
28#include <asm/unistd.h>
29#include <linux/err.h>
30#include <linux/kernel.h>
31#include <linux/bpf.h>
32#include <linux/btf.h>
33#include <linux/filter.h>
34#include <linux/list.h>
35#include <linux/limits.h>
36#include <linux/perf_event.h>
37#include <linux/ring_buffer.h>
38#include <linux/version.h>
39#include <sys/epoll.h>
40#include <sys/ioctl.h>
41#include <sys/mman.h>
42#include <sys/stat.h>
43#include <sys/types.h>
44#include <sys/vfs.h>
45#include <sys/utsname.h>
46#include <sys/resource.h>
47#include <libelf.h>
48#include <gelf.h>
49#include <zlib.h>
50
51#include "libbpf.h"
52#include "bpf.h"
53#include "btf.h"
54#include "str_error.h"
55#include "libbpf_internal.h"
56#include "hashmap.h"
57
58#ifndef EM_BPF
59#define EM_BPF 247
60#endif
61
62#ifndef BPF_FS_MAGIC
63#define BPF_FS_MAGIC 0xcafe4a11
64#endif
65
66#define BPF_INSN_SZ (sizeof(struct bpf_insn))
67
68
69
70
71#pragma GCC diagnostic ignored "-Wformat-nonliteral"
72
73#define __printf(a, b) __attribute__((format(printf, a, b)))
74
75static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
76static const struct btf_type *
77skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
78
79static int __base_pr(enum libbpf_print_level level, const char *format,
80 va_list args)
81{
82 if (level == LIBBPF_DEBUG)
83 return 0;
84
85 return vfprintf(stderr, format, args);
86}
87
88static libbpf_print_fn_t __libbpf_pr = __base_pr;
89
90libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
91{
92 libbpf_print_fn_t old_print_fn = __libbpf_pr;
93
94 __libbpf_pr = fn;
95 return old_print_fn;
96}
97
98__printf(2, 3)
99void libbpf_print(enum libbpf_print_level level, const char *format, ...)
100{
101 va_list args;
102
103 if (!__libbpf_pr)
104 return;
105
106 va_start(args, format);
107 __libbpf_pr(level, format, args);
108 va_end(args);
109}
110
111static void pr_perm_msg(int err)
112{
113 struct rlimit limit;
114 char buf[100];
115
116 if (err != -EPERM || geteuid() != 0)
117 return;
118
119 err = getrlimit(RLIMIT_MEMLOCK, &limit);
120 if (err)
121 return;
122
123 if (limit.rlim_cur == RLIM_INFINITY)
124 return;
125
126 if (limit.rlim_cur < 1024)
127 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
128 else if (limit.rlim_cur < 1024*1024)
129 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
130 else
131 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
132
133 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
134 buf);
135}
136
137#define STRERR_BUFSIZE 128
138
139
140#ifndef zfree
141# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
142#endif
143
144#ifndef zclose
145# define zclose(fd) ({ \
146 int ___err = 0; \
147 if ((fd) >= 0) \
148 ___err = close((fd)); \
149 fd = -1; \
150 ___err; })
151#endif
152
153static inline __u64 ptr_to_u64(const void *ptr)
154{
155 return (__u64) (unsigned long) ptr;
156}
157
158enum kern_feature_id {
159
160 FEAT_PROG_NAME,
161
162 FEAT_GLOBAL_DATA,
163
164 FEAT_BTF,
165
166 FEAT_BTF_FUNC,
167
168 FEAT_BTF_DATASEC,
169
170 FEAT_BTF_GLOBAL_FUNC,
171
172 FEAT_ARRAY_MMAP,
173
174 FEAT_EXP_ATTACH_TYPE,
175
176 FEAT_PROBE_READ_KERN,
177
178 FEAT_PROG_BIND_MAP,
179
180 FEAT_MODULE_BTF,
181 __FEAT_CNT,
182};
183
184static bool kernel_supports(enum kern_feature_id feat_id);
185
186enum reloc_type {
187 RELO_LD64,
188 RELO_CALL,
189 RELO_DATA,
190 RELO_EXTERN,
191};
192
193struct reloc_desc {
194 enum reloc_type type;
195 int insn_idx;
196 int map_idx;
197 int sym_off;
198 bool processed;
199};
200
201struct bpf_sec_def;
202
203typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec,
204 struct bpf_program *prog);
205
206struct bpf_sec_def {
207 const char *sec;
208 size_t len;
209 enum bpf_prog_type prog_type;
210 enum bpf_attach_type expected_attach_type;
211 bool is_exp_attach_type_optional;
212 bool is_attachable;
213 bool is_attach_btf;
214 bool is_sleepable;
215 attach_fn_t attach_fn;
216};
217
218
219
220
221
222struct bpf_program {
223 const struct bpf_sec_def *sec_def;
224 char *sec_name;
225 size_t sec_idx;
226
227
228
229 size_t sec_insn_off;
230
231
232
233
234 size_t sec_insn_cnt;
235
236
237
238
239
240
241
242
243 size_t sub_insn_off;
244
245 char *name;
246
247
248
249 char *pin_name;
250
251
252
253
254
255
256 struct bpf_insn *insns;
257
258
259
260
261 size_t insns_cnt;
262
263 struct reloc_desc *reloc_desc;
264 int nr_reloc;
265 int log_level;
266
267 struct {
268 int nr;
269 int *fds;
270 } instances;
271 bpf_program_prep_t preprocessor;
272
273 struct bpf_object *obj;
274 void *priv;
275 bpf_program_clear_priv_t clear_priv;
276
277 bool load;
278 enum bpf_prog_type type;
279 enum bpf_attach_type expected_attach_type;
280 int prog_ifindex;
281 __u32 attach_btf_obj_fd;
282 __u32 attach_btf_id;
283 __u32 attach_prog_fd;
284 void *func_info;
285 __u32 func_info_rec_size;
286 __u32 func_info_cnt;
287
288 void *line_info;
289 __u32 line_info_rec_size;
290 __u32 line_info_cnt;
291 __u32 prog_flags;
292};
293
294struct bpf_struct_ops {
295 const char *tname;
296 const struct btf_type *type;
297 struct bpf_program **progs;
298 __u32 *kern_func_off;
299
300 void *data;
301
302
303
304
305
306
307
308
309
310
311 void *kern_vdata;
312 __u32 type_id;
313};
314
315#define DATA_SEC ".data"
316#define BSS_SEC ".bss"
317#define RODATA_SEC ".rodata"
318#define KCONFIG_SEC ".kconfig"
319#define KSYMS_SEC ".ksyms"
320#define STRUCT_OPS_SEC ".struct_ops"
321
322enum libbpf_map_type {
323 LIBBPF_MAP_UNSPEC,
324 LIBBPF_MAP_DATA,
325 LIBBPF_MAP_BSS,
326 LIBBPF_MAP_RODATA,
327 LIBBPF_MAP_KCONFIG,
328};
329
330static const char * const libbpf_type_to_btf_name[] = {
331 [LIBBPF_MAP_DATA] = DATA_SEC,
332 [LIBBPF_MAP_BSS] = BSS_SEC,
333 [LIBBPF_MAP_RODATA] = RODATA_SEC,
334 [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC,
335};
336
337struct bpf_map {
338 char *name;
339 int fd;
340 int sec_idx;
341 size_t sec_offset;
342 int map_ifindex;
343 int inner_map_fd;
344 struct bpf_map_def def;
345 __u32 numa_node;
346 __u32 btf_var_idx;
347 __u32 btf_key_type_id;
348 __u32 btf_value_type_id;
349 __u32 btf_vmlinux_value_type_id;
350 void *priv;
351 bpf_map_clear_priv_t clear_priv;
352 enum libbpf_map_type libbpf_type;
353 void *mmaped;
354 struct bpf_struct_ops *st_ops;
355 struct bpf_map *inner_map;
356 void **init_slots;
357 int init_slots_sz;
358 char *pin_path;
359 bool pinned;
360 bool reused;
361};
362
363enum extern_type {
364 EXT_UNKNOWN,
365 EXT_KCFG,
366 EXT_KSYM,
367};
368
369enum kcfg_type {
370 KCFG_UNKNOWN,
371 KCFG_CHAR,
372 KCFG_BOOL,
373 KCFG_INT,
374 KCFG_TRISTATE,
375 KCFG_CHAR_ARR,
376};
377
378struct extern_desc {
379 enum extern_type type;
380 int sym_idx;
381 int btf_id;
382 int sec_btf_id;
383 const char *name;
384 bool is_set;
385 bool is_weak;
386 union {
387 struct {
388 enum kcfg_type type;
389 int sz;
390 int align;
391 int data_off;
392 bool is_signed;
393 } kcfg;
394 struct {
395 unsigned long long addr;
396
397
398 int vmlinux_btf_id;
399
400
401 __u32 type_id;
402 } ksym;
403 };
404};
405
406static LIST_HEAD(bpf_objects_list);
407
408struct module_btf {
409 struct btf *btf;
410 char *name;
411 __u32 id;
412 int fd;
413};
414
415struct bpf_object {
416 char name[BPF_OBJ_NAME_LEN];
417 char license[64];
418 __u32 kern_version;
419
420 struct bpf_program *programs;
421 size_t nr_programs;
422 struct bpf_map *maps;
423 size_t nr_maps;
424 size_t maps_cap;
425
426 char *kconfig;
427 struct extern_desc *externs;
428 int nr_extern;
429 int kconfig_map_idx;
430 int rodata_map_idx;
431
432 bool loaded;
433 bool has_subcalls;
434
435
436
437
438
439 struct {
440 int fd;
441 const void *obj_buf;
442 size_t obj_buf_sz;
443 Elf *elf;
444 GElf_Ehdr ehdr;
445 Elf_Data *symbols;
446 Elf_Data *data;
447 Elf_Data *rodata;
448 Elf_Data *bss;
449 Elf_Data *st_ops_data;
450 size_t shstrndx;
451 size_t strtabidx;
452 struct {
453 GElf_Shdr shdr;
454 Elf_Data *data;
455 } *reloc_sects;
456 int nr_reloc_sects;
457 int maps_shndx;
458 int btf_maps_shndx;
459 __u32 btf_maps_sec_btf_id;
460 int text_shndx;
461 int symbols_shndx;
462 int data_shndx;
463 int rodata_shndx;
464 int bss_shndx;
465 int st_ops_shndx;
466 } efile;
467
468
469
470
471
472 struct list_head list;
473
474 struct btf *btf;
475 struct btf_ext *btf_ext;
476
477
478
479
480 struct btf *btf_vmlinux;
481
482 struct btf *btf_vmlinux_override;
483
484 struct module_btf *btf_modules;
485 bool btf_modules_loaded;
486 size_t btf_module_cnt;
487 size_t btf_module_cap;
488
489 void *priv;
490 bpf_object_clear_priv_t clear_priv;
491
492 char path[];
493};
494#define obj_elf_valid(o) ((o)->efile.elf)
495
496static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
497static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
498static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
499static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
500static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr);
501static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
502static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
503static int elf_sym_by_sec_off(const struct bpf_object *obj, size_t sec_idx,
504 size_t off, __u32 sym_type, GElf_Sym *sym);
505
506void bpf_program__unload(struct bpf_program *prog)
507{
508 int i;
509
510 if (!prog)
511 return;
512
513
514
515
516
517 if (prog->instances.nr > 0) {
518 for (i = 0; i < prog->instances.nr; i++)
519 zclose(prog->instances.fds[i]);
520 } else if (prog->instances.nr != -1) {
521 pr_warn("Internal error: instances.nr is %d\n",
522 prog->instances.nr);
523 }
524
525 prog->instances.nr = -1;
526 zfree(&prog->instances.fds);
527
528 zfree(&prog->func_info);
529 zfree(&prog->line_info);
530}
531
532static void bpf_program__exit(struct bpf_program *prog)
533{
534 if (!prog)
535 return;
536
537 if (prog->clear_priv)
538 prog->clear_priv(prog, prog->priv);
539
540 prog->priv = NULL;
541 prog->clear_priv = NULL;
542
543 bpf_program__unload(prog);
544 zfree(&prog->name);
545 zfree(&prog->sec_name);
546 zfree(&prog->pin_name);
547 zfree(&prog->insns);
548 zfree(&prog->reloc_desc);
549
550 prog->nr_reloc = 0;
551 prog->insns_cnt = 0;
552 prog->sec_idx = -1;
553}
554
555static char *__bpf_program__pin_name(struct bpf_program *prog)
556{
557 char *name, *p;
558
559 name = p = strdup(prog->sec_name);
560 while ((p = strchr(p, '/')))
561 *p = '_';
562
563 return name;
564}
565
566static bool insn_is_subprog_call(const struct bpf_insn *insn)
567{
568 return BPF_CLASS(insn->code) == BPF_JMP &&
569 BPF_OP(insn->code) == BPF_CALL &&
570 BPF_SRC(insn->code) == BPF_K &&
571 insn->src_reg == BPF_PSEUDO_CALL &&
572 insn->dst_reg == 0 &&
573 insn->off == 0;
574}
575
576static int
577bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
578 const char *name, size_t sec_idx, const char *sec_name,
579 size_t sec_off, void *insn_data, size_t insn_data_sz)
580{
581 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
582 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
583 sec_name, name, sec_off, insn_data_sz);
584 return -EINVAL;
585 }
586
587 memset(prog, 0, sizeof(*prog));
588 prog->obj = obj;
589
590 prog->sec_idx = sec_idx;
591 prog->sec_insn_off = sec_off / BPF_INSN_SZ;
592 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
593
594 prog->insns_cnt = prog->sec_insn_cnt;
595
596 prog->type = BPF_PROG_TYPE_UNSPEC;
597 prog->load = true;
598
599 prog->instances.fds = NULL;
600 prog->instances.nr = -1;
601
602 prog->sec_name = strdup(sec_name);
603 if (!prog->sec_name)
604 goto errout;
605
606 prog->name = strdup(name);
607 if (!prog->name)
608 goto errout;
609
610 prog->pin_name = __bpf_program__pin_name(prog);
611 if (!prog->pin_name)
612 goto errout;
613
614 prog->insns = malloc(insn_data_sz);
615 if (!prog->insns)
616 goto errout;
617 memcpy(prog->insns, insn_data, insn_data_sz);
618
619 return 0;
620errout:
621 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
622 bpf_program__exit(prog);
623 return -ENOMEM;
624}
625
626static int
627bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
628 const char *sec_name, int sec_idx)
629{
630 struct bpf_program *prog, *progs;
631 void *data = sec_data->d_buf;
632 size_t sec_sz = sec_data->d_size, sec_off, prog_sz;
633 int nr_progs, err;
634 const char *name;
635 GElf_Sym sym;
636
637 progs = obj->programs;
638 nr_progs = obj->nr_programs;
639 sec_off = 0;
640
641 while (sec_off < sec_sz) {
642 if (elf_sym_by_sec_off(obj, sec_idx, sec_off, STT_FUNC, &sym)) {
643 pr_warn("sec '%s': failed to find program symbol at offset %zu\n",
644 sec_name, sec_off);
645 return -LIBBPF_ERRNO__FORMAT;
646 }
647
648 prog_sz = sym.st_size;
649
650 name = elf_sym_str(obj, sym.st_name);
651 if (!name) {
652 pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
653 sec_name, sec_off);
654 return -LIBBPF_ERRNO__FORMAT;
655 }
656
657 if (sec_off + prog_sz > sec_sz) {
658 pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
659 sec_name, sec_off);
660 return -LIBBPF_ERRNO__FORMAT;
661 }
662
663 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
664 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
665
666 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
667 if (!progs) {
668
669
670
671
672
673 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
674 sec_name, name);
675 return -ENOMEM;
676 }
677 obj->programs = progs;
678
679 prog = &progs[nr_progs];
680
681 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
682 sec_off, data + sec_off, prog_sz);
683 if (err)
684 return err;
685
686 nr_progs++;
687 obj->nr_programs = nr_progs;
688
689 sec_off += prog_sz;
690 }
691
692 return 0;
693}
694
695static __u32 get_kernel_version(void)
696{
697 __u32 major, minor, patch;
698 struct utsname info;
699
700 uname(&info);
701 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
702 return 0;
703 return KERNEL_VERSION(major, minor, patch);
704}
705
706static const struct btf_member *
707find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
708{
709 struct btf_member *m;
710 int i;
711
712 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
713 if (btf_member_bit_offset(t, i) == bit_offset)
714 return m;
715 }
716
717 return NULL;
718}
719
720static const struct btf_member *
721find_member_by_name(const struct btf *btf, const struct btf_type *t,
722 const char *name)
723{
724 struct btf_member *m;
725 int i;
726
727 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
728 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
729 return m;
730 }
731
732 return NULL;
733}
734
735#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
736static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
737 const char *name, __u32 kind);
738
739static int
740find_struct_ops_kern_types(const struct btf *btf, const char *tname,
741 const struct btf_type **type, __u32 *type_id,
742 const struct btf_type **vtype, __u32 *vtype_id,
743 const struct btf_member **data_member)
744{
745 const struct btf_type *kern_type, *kern_vtype;
746 const struct btf_member *kern_data_member;
747 __s32 kern_vtype_id, kern_type_id;
748 __u32 i;
749
750 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
751 if (kern_type_id < 0) {
752 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
753 tname);
754 return kern_type_id;
755 }
756 kern_type = btf__type_by_id(btf, kern_type_id);
757
758
759
760
761
762
763 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
764 tname, BTF_KIND_STRUCT);
765 if (kern_vtype_id < 0) {
766 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
767 STRUCT_OPS_VALUE_PREFIX, tname);
768 return kern_vtype_id;
769 }
770 kern_vtype = btf__type_by_id(btf, kern_vtype_id);
771
772
773
774
775
776
777
778 kern_data_member = btf_members(kern_vtype);
779 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
780 if (kern_data_member->type == kern_type_id)
781 break;
782 }
783 if (i == btf_vlen(kern_vtype)) {
784 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
785 tname, STRUCT_OPS_VALUE_PREFIX, tname);
786 return -EINVAL;
787 }
788
789 *type = kern_type;
790 *type_id = kern_type_id;
791 *vtype = kern_vtype;
792 *vtype_id = kern_vtype_id;
793 *data_member = kern_data_member;
794
795 return 0;
796}
797
798static bool bpf_map__is_struct_ops(const struct bpf_map *map)
799{
800 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
801}
802
803
804static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
805 const struct btf *btf,
806 const struct btf *kern_btf)
807{
808 const struct btf_member *member, *kern_member, *kern_data_member;
809 const struct btf_type *type, *kern_type, *kern_vtype;
810 __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
811 struct bpf_struct_ops *st_ops;
812 void *data, *kern_data;
813 const char *tname;
814 int err;
815
816 st_ops = map->st_ops;
817 type = st_ops->type;
818 tname = st_ops->tname;
819 err = find_struct_ops_kern_types(kern_btf, tname,
820 &kern_type, &kern_type_id,
821 &kern_vtype, &kern_vtype_id,
822 &kern_data_member);
823 if (err)
824 return err;
825
826 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
827 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
828
829 map->def.value_size = kern_vtype->size;
830 map->btf_vmlinux_value_type_id = kern_vtype_id;
831
832 st_ops->kern_vdata = calloc(1, kern_vtype->size);
833 if (!st_ops->kern_vdata)
834 return -ENOMEM;
835
836 data = st_ops->data;
837 kern_data_off = kern_data_member->offset / 8;
838 kern_data = st_ops->kern_vdata + kern_data_off;
839
840 member = btf_members(type);
841 for (i = 0; i < btf_vlen(type); i++, member++) {
842 const struct btf_type *mtype, *kern_mtype;
843 __u32 mtype_id, kern_mtype_id;
844 void *mdata, *kern_mdata;
845 __s64 msize, kern_msize;
846 __u32 moff, kern_moff;
847 __u32 kern_member_idx;
848 const char *mname;
849
850 mname = btf__name_by_offset(btf, member->name_off);
851 kern_member = find_member_by_name(kern_btf, kern_type, mname);
852 if (!kern_member) {
853 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
854 map->name, mname);
855 return -ENOTSUP;
856 }
857
858 kern_member_idx = kern_member - btf_members(kern_type);
859 if (btf_member_bitfield_size(type, i) ||
860 btf_member_bitfield_size(kern_type, kern_member_idx)) {
861 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
862 map->name, mname);
863 return -ENOTSUP;
864 }
865
866 moff = member->offset / 8;
867 kern_moff = kern_member->offset / 8;
868
869 mdata = data + moff;
870 kern_mdata = kern_data + kern_moff;
871
872 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
873 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
874 &kern_mtype_id);
875 if (BTF_INFO_KIND(mtype->info) !=
876 BTF_INFO_KIND(kern_mtype->info)) {
877 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
878 map->name, mname, BTF_INFO_KIND(mtype->info),
879 BTF_INFO_KIND(kern_mtype->info));
880 return -ENOTSUP;
881 }
882
883 if (btf_is_ptr(mtype)) {
884 struct bpf_program *prog;
885
886 mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id);
887 kern_mtype = skip_mods_and_typedefs(kern_btf,
888 kern_mtype->type,
889 &kern_mtype_id);
890 if (!btf_is_func_proto(mtype) ||
891 !btf_is_func_proto(kern_mtype)) {
892 pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n",
893 map->name, mname);
894 return -ENOTSUP;
895 }
896
897 prog = st_ops->progs[i];
898 if (!prog) {
899 pr_debug("struct_ops init_kern %s: func ptr %s is not set\n",
900 map->name, mname);
901 continue;
902 }
903
904 prog->attach_btf_id = kern_type_id;
905 prog->expected_attach_type = kern_member_idx;
906
907 st_ops->kern_func_off[i] = kern_data_off + kern_moff;
908
909 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
910 map->name, mname, prog->name, moff,
911 kern_moff);
912
913 continue;
914 }
915
916 msize = btf__resolve_size(btf, mtype_id);
917 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
918 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
919 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
920 map->name, mname, (ssize_t)msize,
921 (ssize_t)kern_msize);
922 return -ENOTSUP;
923 }
924
925 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
926 map->name, mname, (unsigned int)msize,
927 moff, kern_moff);
928 memcpy(kern_mdata, mdata, msize);
929 }
930
931 return 0;
932}
933
934static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
935{
936 struct bpf_map *map;
937 size_t i;
938 int err;
939
940 for (i = 0; i < obj->nr_maps; i++) {
941 map = &obj->maps[i];
942
943 if (!bpf_map__is_struct_ops(map))
944 continue;
945
946 err = bpf_map__init_kern_struct_ops(map, obj->btf,
947 obj->btf_vmlinux);
948 if (err)
949 return err;
950 }
951
952 return 0;
953}
954
955static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
956{
957 const struct btf_type *type, *datasec;
958 const struct btf_var_secinfo *vsi;
959 struct bpf_struct_ops *st_ops;
960 const char *tname, *var_name;
961 __s32 type_id, datasec_id;
962 const struct btf *btf;
963 struct bpf_map *map;
964 __u32 i;
965
966 if (obj->efile.st_ops_shndx == -1)
967 return 0;
968
969 btf = obj->btf;
970 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
971 BTF_KIND_DATASEC);
972 if (datasec_id < 0) {
973 pr_warn("struct_ops init: DATASEC %s not found\n",
974 STRUCT_OPS_SEC);
975 return -EINVAL;
976 }
977
978 datasec = btf__type_by_id(btf, datasec_id);
979 vsi = btf_var_secinfos(datasec);
980 for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
981 type = btf__type_by_id(obj->btf, vsi->type);
982 var_name = btf__name_by_offset(obj->btf, type->name_off);
983
984 type_id = btf__resolve_type(obj->btf, vsi->type);
985 if (type_id < 0) {
986 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
987 vsi->type, STRUCT_OPS_SEC);
988 return -EINVAL;
989 }
990
991 type = btf__type_by_id(obj->btf, type_id);
992 tname = btf__name_by_offset(obj->btf, type->name_off);
993 if (!tname[0]) {
994 pr_warn("struct_ops init: anonymous type is not supported\n");
995 return -ENOTSUP;
996 }
997 if (!btf_is_struct(type)) {
998 pr_warn("struct_ops init: %s is not a struct\n", tname);
999 return -EINVAL;
1000 }
1001
1002 map = bpf_object__add_map(obj);
1003 if (IS_ERR(map))
1004 return PTR_ERR(map);
1005
1006 map->sec_idx = obj->efile.st_ops_shndx;
1007 map->sec_offset = vsi->offset;
1008 map->name = strdup(var_name);
1009 if (!map->name)
1010 return -ENOMEM;
1011
1012 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1013 map->def.key_size = sizeof(int);
1014 map->def.value_size = type->size;
1015 map->def.max_entries = 1;
1016
1017 map->st_ops = calloc(1, sizeof(*map->st_ops));
1018 if (!map->st_ops)
1019 return -ENOMEM;
1020 st_ops = map->st_ops;
1021 st_ops->data = malloc(type->size);
1022 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1023 st_ops->kern_func_off = malloc(btf_vlen(type) *
1024 sizeof(*st_ops->kern_func_off));
1025 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1026 return -ENOMEM;
1027
1028 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
1029 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1030 var_name, STRUCT_OPS_SEC);
1031 return -EINVAL;
1032 }
1033
1034 memcpy(st_ops->data,
1035 obj->efile.st_ops_data->d_buf + vsi->offset,
1036 type->size);
1037 st_ops->tname = tname;
1038 st_ops->type = type;
1039 st_ops->type_id = type_id;
1040
1041 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1042 tname, type_id, var_name, vsi->offset);
1043 }
1044
1045 return 0;
1046}
1047
1048static struct bpf_object *bpf_object__new(const char *path,
1049 const void *obj_buf,
1050 size_t obj_buf_sz,
1051 const char *obj_name)
1052{
1053 struct bpf_object *obj;
1054 char *end;
1055
1056 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1057 if (!obj) {
1058 pr_warn("alloc memory failed for %s\n", path);
1059 return ERR_PTR(-ENOMEM);
1060 }
1061
1062 strcpy(obj->path, path);
1063 if (obj_name) {
1064 strncpy(obj->name, obj_name, sizeof(obj->name) - 1);
1065 obj->name[sizeof(obj->name) - 1] = 0;
1066 } else {
1067
1068 strncpy(obj->name, basename((void *)path),
1069 sizeof(obj->name) - 1);
1070 end = strchr(obj->name, '.');
1071 if (end)
1072 *end = 0;
1073 }
1074
1075 obj->efile.fd = -1;
1076
1077
1078
1079
1080
1081
1082 obj->efile.obj_buf = obj_buf;
1083 obj->efile.obj_buf_sz = obj_buf_sz;
1084 obj->efile.maps_shndx = -1;
1085 obj->efile.btf_maps_shndx = -1;
1086 obj->efile.data_shndx = -1;
1087 obj->efile.rodata_shndx = -1;
1088 obj->efile.bss_shndx = -1;
1089 obj->efile.st_ops_shndx = -1;
1090 obj->kconfig_map_idx = -1;
1091 obj->rodata_map_idx = -1;
1092
1093 obj->kern_version = get_kernel_version();
1094 obj->loaded = false;
1095
1096 INIT_LIST_HEAD(&obj->list);
1097 list_add(&obj->list, &bpf_objects_list);
1098 return obj;
1099}
1100
1101static void bpf_object__elf_finish(struct bpf_object *obj)
1102{
1103 if (!obj_elf_valid(obj))
1104 return;
1105
1106 if (obj->efile.elf) {
1107 elf_end(obj->efile.elf);
1108 obj->efile.elf = NULL;
1109 }
1110 obj->efile.symbols = NULL;
1111 obj->efile.data = NULL;
1112 obj->efile.rodata = NULL;
1113 obj->efile.bss = NULL;
1114 obj->efile.st_ops_data = NULL;
1115
1116 zfree(&obj->efile.reloc_sects);
1117 obj->efile.nr_reloc_sects = 0;
1118 zclose(obj->efile.fd);
1119 obj->efile.obj_buf = NULL;
1120 obj->efile.obj_buf_sz = 0;
1121}
1122
1123
1124#ifndef ELF_C_READ_MMAP
1125#define ELF_C_READ_MMAP ELF_C_READ
1126#endif
1127
1128static int bpf_object__elf_init(struct bpf_object *obj)
1129{
1130 int err = 0;
1131 GElf_Ehdr *ep;
1132
1133 if (obj_elf_valid(obj)) {
1134 pr_warn("elf: init internal error\n");
1135 return -LIBBPF_ERRNO__LIBELF;
1136 }
1137
1138 if (obj->efile.obj_buf_sz > 0) {
1139
1140
1141
1142
1143 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf,
1144 obj->efile.obj_buf_sz);
1145 } else {
1146 obj->efile.fd = open(obj->path, O_RDONLY);
1147 if (obj->efile.fd < 0) {
1148 char errmsg[STRERR_BUFSIZE], *cp;
1149
1150 err = -errno;
1151 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1152 pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1153 return err;
1154 }
1155
1156 obj->efile.elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1157 }
1158
1159 if (!obj->efile.elf) {
1160 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1161 err = -LIBBPF_ERRNO__LIBELF;
1162 goto errout;
1163 }
1164
1165 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
1166 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1167 err = -LIBBPF_ERRNO__FORMAT;
1168 goto errout;
1169 }
1170 ep = &obj->efile.ehdr;
1171
1172 if (elf_getshdrstrndx(obj->efile.elf, &obj->efile.shstrndx)) {
1173 pr_warn("elf: failed to get section names section index for %s: %s\n",
1174 obj->path, elf_errmsg(-1));
1175 err = -LIBBPF_ERRNO__FORMAT;
1176 goto errout;
1177 }
1178
1179
1180 if (!elf_rawdata(elf_getscn(obj->efile.elf, obj->efile.shstrndx), NULL)) {
1181 pr_warn("elf: failed to get section names strings from %s: %s\n",
1182 obj->path, elf_errmsg(-1));
1183 return -LIBBPF_ERRNO__FORMAT;
1184 }
1185
1186
1187 if (ep->e_type != ET_REL ||
1188 (ep->e_machine && ep->e_machine != EM_BPF)) {
1189 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1190 err = -LIBBPF_ERRNO__FORMAT;
1191 goto errout;
1192 }
1193
1194 return 0;
1195errout:
1196 bpf_object__elf_finish(obj);
1197 return err;
1198}
1199
1200static int bpf_object__check_endianness(struct bpf_object *obj)
1201{
1202#if __BYTE_ORDER == __LITTLE_ENDIAN
1203 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB)
1204 return 0;
1205#elif __BYTE_ORDER == __BIG_ENDIAN
1206 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB)
1207 return 0;
1208#else
1209# error "Unrecognized __BYTE_ORDER__"
1210#endif
1211 pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1212 return -LIBBPF_ERRNO__ENDIAN;
1213}
1214
1215static int
1216bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1217{
1218 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1));
1219 pr_debug("license of %s is %s\n", obj->path, obj->license);
1220 return 0;
1221}
1222
1223static int
1224bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1225{
1226 __u32 kver;
1227
1228 if (size != sizeof(kver)) {
1229 pr_warn("invalid kver section in %s\n", obj->path);
1230 return -LIBBPF_ERRNO__FORMAT;
1231 }
1232 memcpy(&kver, data, sizeof(kver));
1233 obj->kern_version = kver;
1234 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1235 return 0;
1236}
1237
1238static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1239{
1240 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1241 type == BPF_MAP_TYPE_HASH_OF_MAPS)
1242 return true;
1243 return false;
1244}
1245
1246int bpf_object__section_size(const struct bpf_object *obj, const char *name,
1247 __u32 *size)
1248{
1249 int ret = -ENOENT;
1250
1251 *size = 0;
1252 if (!name) {
1253 return -EINVAL;
1254 } else if (!strcmp(name, DATA_SEC)) {
1255 if (obj->efile.data)
1256 *size = obj->efile.data->d_size;
1257 } else if (!strcmp(name, BSS_SEC)) {
1258 if (obj->efile.bss)
1259 *size = obj->efile.bss->d_size;
1260 } else if (!strcmp(name, RODATA_SEC)) {
1261 if (obj->efile.rodata)
1262 *size = obj->efile.rodata->d_size;
1263 } else if (!strcmp(name, STRUCT_OPS_SEC)) {
1264 if (obj->efile.st_ops_data)
1265 *size = obj->efile.st_ops_data->d_size;
1266 } else {
1267 Elf_Scn *scn = elf_sec_by_name(obj, name);
1268 Elf_Data *data = elf_sec_data(obj, scn);
1269
1270 if (data) {
1271 ret = 0;
1272 *size = data->d_size;
1273 }
1274 }
1275
1276 return *size ? 0 : ret;
1277}
1278
1279int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
1280 __u32 *off)
1281{
1282 Elf_Data *symbols = obj->efile.symbols;
1283 const char *sname;
1284 size_t si;
1285
1286 if (!name || !off)
1287 return -EINVAL;
1288
1289 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) {
1290 GElf_Sym sym;
1291
1292 if (!gelf_getsym(symbols, si, &sym))
1293 continue;
1294 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
1295 GELF_ST_TYPE(sym.st_info) != STT_OBJECT)
1296 continue;
1297
1298 sname = elf_sym_str(obj, sym.st_name);
1299 if (!sname) {
1300 pr_warn("failed to get sym name string for var %s\n",
1301 name);
1302 return -EIO;
1303 }
1304 if (strcmp(name, sname) == 0) {
1305 *off = sym.st_value;
1306 return 0;
1307 }
1308 }
1309
1310 return -ENOENT;
1311}
1312
1313static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1314{
1315 struct bpf_map *new_maps;
1316 size_t new_cap;
1317 int i;
1318
1319 if (obj->nr_maps < obj->maps_cap)
1320 return &obj->maps[obj->nr_maps++];
1321
1322 new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
1323 new_maps = libbpf_reallocarray(obj->maps, new_cap, sizeof(*obj->maps));
1324 if (!new_maps) {
1325 pr_warn("alloc maps for object failed\n");
1326 return ERR_PTR(-ENOMEM);
1327 }
1328
1329 obj->maps_cap = new_cap;
1330 obj->maps = new_maps;
1331
1332
1333 memset(obj->maps + obj->nr_maps, 0,
1334 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps));
1335
1336
1337
1338
1339 for (i = obj->nr_maps; i < obj->maps_cap; i++) {
1340 obj->maps[i].fd = -1;
1341 obj->maps[i].inner_map_fd = -1;
1342 }
1343
1344 return &obj->maps[obj->nr_maps++];
1345}
1346
1347static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1348{
1349 long page_sz = sysconf(_SC_PAGE_SIZE);
1350 size_t map_sz;
1351
1352 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1353 map_sz = roundup(map_sz, page_sz);
1354 return map_sz;
1355}
1356
1357static char *internal_map_name(struct bpf_object *obj,
1358 enum libbpf_map_type type)
1359{
1360 char map_name[BPF_OBJ_NAME_LEN], *p;
1361 const char *sfx = libbpf_type_to_btf_name[type];
1362 int sfx_len = max((size_t)7, strlen(sfx));
1363 int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1,
1364 strlen(obj->name));
1365
1366 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1367 sfx_len, libbpf_type_to_btf_name[type]);
1368
1369
1370 for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1371 if (!isalnum(*p) && *p != '_' && *p != '.')
1372 *p = '_';
1373
1374 return strdup(map_name);
1375}
1376
1377static int
1378bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1379 int sec_idx, void *data, size_t data_sz)
1380{
1381 struct bpf_map_def *def;
1382 struct bpf_map *map;
1383 int err;
1384
1385 map = bpf_object__add_map(obj);
1386 if (IS_ERR(map))
1387 return PTR_ERR(map);
1388
1389 map->libbpf_type = type;
1390 map->sec_idx = sec_idx;
1391 map->sec_offset = 0;
1392 map->name = internal_map_name(obj, type);
1393 if (!map->name) {
1394 pr_warn("failed to alloc map name\n");
1395 return -ENOMEM;
1396 }
1397
1398 def = &map->def;
1399 def->type = BPF_MAP_TYPE_ARRAY;
1400 def->key_size = sizeof(int);
1401 def->value_size = data_sz;
1402 def->max_entries = 1;
1403 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1404 ? BPF_F_RDONLY_PROG : 0;
1405 def->map_flags |= BPF_F_MMAPABLE;
1406
1407 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1408 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1409
1410 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1411 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1412 if (map->mmaped == MAP_FAILED) {
1413 err = -errno;
1414 map->mmaped = NULL;
1415 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1416 map->name, err);
1417 zfree(&map->name);
1418 return err;
1419 }
1420
1421 if (data)
1422 memcpy(map->mmaped, data, data_sz);
1423
1424 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1425 return 0;
1426}
1427
1428static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1429{
1430 int err;
1431
1432
1433
1434
1435 if (obj->efile.data_shndx >= 0) {
1436 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1437 obj->efile.data_shndx,
1438 obj->efile.data->d_buf,
1439 obj->efile.data->d_size);
1440 if (err)
1441 return err;
1442 }
1443 if (obj->efile.rodata_shndx >= 0) {
1444 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1445 obj->efile.rodata_shndx,
1446 obj->efile.rodata->d_buf,
1447 obj->efile.rodata->d_size);
1448 if (err)
1449 return err;
1450
1451 obj->rodata_map_idx = obj->nr_maps - 1;
1452 }
1453 if (obj->efile.bss_shndx >= 0) {
1454 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1455 obj->efile.bss_shndx,
1456 NULL,
1457 obj->efile.bss->d_size);
1458 if (err)
1459 return err;
1460 }
1461 return 0;
1462}
1463
1464
1465static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1466 const void *name)
1467{
1468 int i;
1469
1470 for (i = 0; i < obj->nr_extern; i++) {
1471 if (strcmp(obj->externs[i].name, name) == 0)
1472 return &obj->externs[i];
1473 }
1474 return NULL;
1475}
1476
1477static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
1478 char value)
1479{
1480 switch (ext->kcfg.type) {
1481 case KCFG_BOOL:
1482 if (value == 'm') {
1483 pr_warn("extern (kcfg) %s=%c should be tristate or char\n",
1484 ext->name, value);
1485 return -EINVAL;
1486 }
1487 *(bool *)ext_val = value == 'y' ? true : false;
1488 break;
1489 case KCFG_TRISTATE:
1490 if (value == 'y')
1491 *(enum libbpf_tristate *)ext_val = TRI_YES;
1492 else if (value == 'm')
1493 *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1494 else
1495 *(enum libbpf_tristate *)ext_val = TRI_NO;
1496 break;
1497 case KCFG_CHAR:
1498 *(char *)ext_val = value;
1499 break;
1500 case KCFG_UNKNOWN:
1501 case KCFG_INT:
1502 case KCFG_CHAR_ARR:
1503 default:
1504 pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n",
1505 ext->name, value);
1506 return -EINVAL;
1507 }
1508 ext->is_set = true;
1509 return 0;
1510}
1511
1512static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
1513 const char *value)
1514{
1515 size_t len;
1516
1517 if (ext->kcfg.type != KCFG_CHAR_ARR) {
1518 pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value);
1519 return -EINVAL;
1520 }
1521
1522 len = strlen(value);
1523 if (value[len - 1] != '"') {
1524 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
1525 ext->name, value);
1526 return -EINVAL;
1527 }
1528
1529
1530 len -= 2;
1531 if (len >= ext->kcfg.sz) {
1532 pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n",
1533 ext->name, value, len, ext->kcfg.sz - 1);
1534 len = ext->kcfg.sz - 1;
1535 }
1536 memcpy(ext_val, value + 1, len);
1537 ext_val[len] = '\0';
1538 ext->is_set = true;
1539 return 0;
1540}
1541
1542static int parse_u64(const char *value, __u64 *res)
1543{
1544 char *value_end;
1545 int err;
1546
1547 errno = 0;
1548 *res = strtoull(value, &value_end, 0);
1549 if (errno) {
1550 err = -errno;
1551 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1552 return err;
1553 }
1554 if (*value_end) {
1555 pr_warn("failed to parse '%s' as integer completely\n", value);
1556 return -EINVAL;
1557 }
1558 return 0;
1559}
1560
1561static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
1562{
1563 int bit_sz = ext->kcfg.sz * 8;
1564
1565 if (ext->kcfg.sz == 8)
1566 return true;
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580 if (ext->kcfg.is_signed)
1581 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1582 else
1583 return (v >> bit_sz) == 0;
1584}
1585
1586static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
1587 __u64 value)
1588{
1589 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
1590 pr_warn("extern (kcfg) %s=%llu should be integer\n",
1591 ext->name, (unsigned long long)value);
1592 return -EINVAL;
1593 }
1594 if (!is_kcfg_value_in_range(ext, value)) {
1595 pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n",
1596 ext->name, (unsigned long long)value, ext->kcfg.sz);
1597 return -ERANGE;
1598 }
1599 switch (ext->kcfg.sz) {
1600 case 1: *(__u8 *)ext_val = value; break;
1601 case 2: *(__u16 *)ext_val = value; break;
1602 case 4: *(__u32 *)ext_val = value; break;
1603 case 8: *(__u64 *)ext_val = value; break;
1604 default:
1605 return -EINVAL;
1606 }
1607 ext->is_set = true;
1608 return 0;
1609}
1610
1611static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1612 char *buf, void *data)
1613{
1614 struct extern_desc *ext;
1615 char *sep, *value;
1616 int len, err = 0;
1617 void *ext_val;
1618 __u64 num;
1619
1620 if (strncmp(buf, "CONFIG_", 7))
1621 return 0;
1622
1623 sep = strchr(buf, '=');
1624 if (!sep) {
1625 pr_warn("failed to parse '%s': no separator\n", buf);
1626 return -EINVAL;
1627 }
1628
1629
1630 len = strlen(buf);
1631 if (buf[len - 1] == '\n')
1632 buf[len - 1] = '\0';
1633
1634 *sep = '\0';
1635 if (!sep[1]) {
1636 *sep = '=';
1637 pr_warn("failed to parse '%s': no value\n", buf);
1638 return -EINVAL;
1639 }
1640
1641 ext = find_extern_by_name(obj, buf);
1642 if (!ext || ext->is_set)
1643 return 0;
1644
1645 ext_val = data + ext->kcfg.data_off;
1646 value = sep + 1;
1647
1648 switch (*value) {
1649 case 'y': case 'n': case 'm':
1650 err = set_kcfg_value_tri(ext, ext_val, *value);
1651 break;
1652 case '"':
1653 err = set_kcfg_value_str(ext, ext_val, value);
1654 break;
1655 default:
1656
1657 err = parse_u64(value, &num);
1658 if (err) {
1659 pr_warn("extern (kcfg) %s=%s should be integer\n",
1660 ext->name, value);
1661 return err;
1662 }
1663 err = set_kcfg_value_num(ext, ext_val, num);
1664 break;
1665 }
1666 if (err)
1667 return err;
1668 pr_debug("extern (kcfg) %s=%s\n", ext->name, value);
1669 return 0;
1670}
1671
1672static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1673{
1674 char buf[PATH_MAX];
1675 struct utsname uts;
1676 int len, err = 0;
1677 gzFile file;
1678
1679 uname(&uts);
1680 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1681 if (len < 0)
1682 return -EINVAL;
1683 else if (len >= PATH_MAX)
1684 return -ENAMETOOLONG;
1685
1686
1687 file = gzopen(buf, "r");
1688 if (!file)
1689 file = gzopen("/proc/config.gz", "r");
1690
1691 if (!file) {
1692 pr_warn("failed to open system Kconfig\n");
1693 return -ENOENT;
1694 }
1695
1696 while (gzgets(file, buf, sizeof(buf))) {
1697 err = bpf_object__process_kconfig_line(obj, buf, data);
1698 if (err) {
1699 pr_warn("error parsing system Kconfig line '%s': %d\n",
1700 buf, err);
1701 goto out;
1702 }
1703 }
1704
1705out:
1706 gzclose(file);
1707 return err;
1708}
1709
1710static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1711 const char *config, void *data)
1712{
1713 char buf[PATH_MAX];
1714 int err = 0;
1715 FILE *file;
1716
1717 file = fmemopen((void *)config, strlen(config), "r");
1718 if (!file) {
1719 err = -errno;
1720 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1721 return err;
1722 }
1723
1724 while (fgets(buf, sizeof(buf), file)) {
1725 err = bpf_object__process_kconfig_line(obj, buf, data);
1726 if (err) {
1727 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
1728 buf, err);
1729 break;
1730 }
1731 }
1732
1733 fclose(file);
1734 return err;
1735}
1736
1737static int bpf_object__init_kconfig_map(struct bpf_object *obj)
1738{
1739 struct extern_desc *last_ext = NULL, *ext;
1740 size_t map_sz;
1741 int i, err;
1742
1743 for (i = 0; i < obj->nr_extern; i++) {
1744 ext = &obj->externs[i];
1745 if (ext->type == EXT_KCFG)
1746 last_ext = ext;
1747 }
1748
1749 if (!last_ext)
1750 return 0;
1751
1752 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
1753 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
1754 obj->efile.symbols_shndx,
1755 NULL, map_sz);
1756 if (err)
1757 return err;
1758
1759 obj->kconfig_map_idx = obj->nr_maps - 1;
1760
1761 return 0;
1762}
1763
1764static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
1765{
1766 Elf_Data *symbols = obj->efile.symbols;
1767 int i, map_def_sz = 0, nr_maps = 0, nr_syms;
1768 Elf_Data *data = NULL;
1769 Elf_Scn *scn;
1770
1771 if (obj->efile.maps_shndx < 0)
1772 return 0;
1773
1774 if (!symbols)
1775 return -EINVAL;
1776
1777
1778 scn = elf_sec_by_idx(obj, obj->efile.maps_shndx);
1779 data = elf_sec_data(obj, scn);
1780 if (!scn || !data) {
1781 pr_warn("elf: failed to get legacy map definitions for %s\n",
1782 obj->path);
1783 return -EINVAL;
1784 }
1785
1786
1787
1788
1789
1790
1791
1792
1793 nr_syms = symbols->d_size / sizeof(GElf_Sym);
1794 for (i = 0; i < nr_syms; i++) {
1795 GElf_Sym sym;
1796
1797 if (!gelf_getsym(symbols, i, &sym))
1798 continue;
1799 if (sym.st_shndx != obj->efile.maps_shndx)
1800 continue;
1801 nr_maps++;
1802 }
1803
1804 pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n",
1805 nr_maps, data->d_size, obj->path);
1806
1807 if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {
1808 pr_warn("elf: unable to determine legacy map definition size in %s\n",
1809 obj->path);
1810 return -EINVAL;
1811 }
1812 map_def_sz = data->d_size / nr_maps;
1813
1814
1815 for (i = 0; i < nr_syms; i++) {
1816 GElf_Sym sym;
1817 const char *map_name;
1818 struct bpf_map_def *def;
1819 struct bpf_map *map;
1820
1821 if (!gelf_getsym(symbols, i, &sym))
1822 continue;
1823 if (sym.st_shndx != obj->efile.maps_shndx)
1824 continue;
1825
1826 map = bpf_object__add_map(obj);
1827 if (IS_ERR(map))
1828 return PTR_ERR(map);
1829
1830 map_name = elf_sym_str(obj, sym.st_name);
1831 if (!map_name) {
1832 pr_warn("failed to get map #%d name sym string for obj %s\n",
1833 i, obj->path);
1834 return -LIBBPF_ERRNO__FORMAT;
1835 }
1836
1837 map->libbpf_type = LIBBPF_MAP_UNSPEC;
1838 map->sec_idx = sym.st_shndx;
1839 map->sec_offset = sym.st_value;
1840 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
1841 map_name, map->sec_idx, map->sec_offset);
1842 if (sym.st_value + map_def_sz > data->d_size) {
1843 pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
1844 obj->path, map_name);
1845 return -EINVAL;
1846 }
1847
1848 map->name = strdup(map_name);
1849 if (!map->name) {
1850 pr_warn("failed to alloc map name\n");
1851 return -ENOMEM;
1852 }
1853 pr_debug("map %d is \"%s\"\n", i, map->name);
1854 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
1855
1856
1857
1858
1859
1860
1861 if (map_def_sz <= sizeof(struct bpf_map_def)) {
1862 memcpy(&map->def, def, map_def_sz);
1863 } else {
1864
1865
1866
1867
1868
1869
1870 char *b;
1871
1872 for (b = ((char *)def) + sizeof(struct bpf_map_def);
1873 b < ((char *)def) + map_def_sz; b++) {
1874 if (*b != 0) {
1875 pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",
1876 obj->path, map_name);
1877 if (strict)
1878 return -EINVAL;
1879 }
1880 }
1881 memcpy(&map->def, def, sizeof(struct bpf_map_def));
1882 }
1883 }
1884 return 0;
1885}
1886
1887static const struct btf_type *
1888skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
1889{
1890 const struct btf_type *t = btf__type_by_id(btf, id);
1891
1892 if (res_id)
1893 *res_id = id;
1894
1895 while (btf_is_mod(t) || btf_is_typedef(t)) {
1896 if (res_id)
1897 *res_id = t->type;
1898 t = btf__type_by_id(btf, t->type);
1899 }
1900
1901 return t;
1902}
1903
1904static const struct btf_type *
1905resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
1906{
1907 const struct btf_type *t;
1908
1909 t = skip_mods_and_typedefs(btf, id, NULL);
1910 if (!btf_is_ptr(t))
1911 return NULL;
1912
1913 t = skip_mods_and_typedefs(btf, t->type, res_id);
1914
1915 return btf_is_func_proto(t) ? t : NULL;
1916}
1917
1918static const char *btf_kind_str(const struct btf_type *t)
1919{
1920 switch (btf_kind(t)) {
1921 case BTF_KIND_UNKN: return "void";
1922 case BTF_KIND_INT: return "int";
1923 case BTF_KIND_PTR: return "ptr";
1924 case BTF_KIND_ARRAY: return "array";
1925 case BTF_KIND_STRUCT: return "struct";
1926 case BTF_KIND_UNION: return "union";
1927 case BTF_KIND_ENUM: return "enum";
1928 case BTF_KIND_FWD: return "fwd";
1929 case BTF_KIND_TYPEDEF: return "typedef";
1930 case BTF_KIND_VOLATILE: return "volatile";
1931 case BTF_KIND_CONST: return "const";
1932 case BTF_KIND_RESTRICT: return "restrict";
1933 case BTF_KIND_FUNC: return "func";
1934 case BTF_KIND_FUNC_PROTO: return "func_proto";
1935 case BTF_KIND_VAR: return "var";
1936 case BTF_KIND_DATASEC: return "datasec";
1937 default: return "unknown";
1938 }
1939}
1940
1941
1942
1943
1944
1945
1946
1947
1948static bool get_map_field_int(const char *map_name, const struct btf *btf,
1949 const struct btf_member *m, __u32 *res)
1950{
1951 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
1952 const char *name = btf__name_by_offset(btf, m->name_off);
1953 const struct btf_array *arr_info;
1954 const struct btf_type *arr_t;
1955
1956 if (!btf_is_ptr(t)) {
1957 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
1958 map_name, name, btf_kind_str(t));
1959 return false;
1960 }
1961
1962 arr_t = btf__type_by_id(btf, t->type);
1963 if (!arr_t) {
1964 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
1965 map_name, name, t->type);
1966 return false;
1967 }
1968 if (!btf_is_array(arr_t)) {
1969 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
1970 map_name, name, btf_kind_str(arr_t));
1971 return false;
1972 }
1973 arr_info = btf_array(arr_t);
1974 *res = arr_info->nelems;
1975 return true;
1976}
1977
1978static int build_map_pin_path(struct bpf_map *map, const char *path)
1979{
1980 char buf[PATH_MAX];
1981 int len;
1982
1983 if (!path)
1984 path = "/sys/fs/bpf";
1985
1986 len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
1987 if (len < 0)
1988 return -EINVAL;
1989 else if (len >= PATH_MAX)
1990 return -ENAMETOOLONG;
1991
1992 return bpf_map__set_pin_path(map, buf);
1993}
1994
1995
1996static int parse_btf_map_def(struct bpf_object *obj,
1997 struct bpf_map *map,
1998 const struct btf_type *def,
1999 bool strict, bool is_inner,
2000 const char *pin_root_path)
2001{
2002 const struct btf_type *t;
2003 const struct btf_member *m;
2004 int vlen, i;
2005
2006 vlen = btf_vlen(def);
2007 m = btf_members(def);
2008 for (i = 0; i < vlen; i++, m++) {
2009 const char *name = btf__name_by_offset(obj->btf, m->name_off);
2010
2011 if (!name) {
2012 pr_warn("map '%s': invalid field #%d.\n", map->name, i);
2013 return -EINVAL;
2014 }
2015 if (strcmp(name, "type") == 0) {
2016 if (!get_map_field_int(map->name, obj->btf, m,
2017 &map->def.type))
2018 return -EINVAL;
2019 pr_debug("map '%s': found type = %u.\n",
2020 map->name, map->def.type);
2021 } else if (strcmp(name, "max_entries") == 0) {
2022 if (!get_map_field_int(map->name, obj->btf, m,
2023 &map->def.max_entries))
2024 return -EINVAL;
2025 pr_debug("map '%s': found max_entries = %u.\n",
2026 map->name, map->def.max_entries);
2027 } else if (strcmp(name, "map_flags") == 0) {
2028 if (!get_map_field_int(map->name, obj->btf, m,
2029 &map->def.map_flags))
2030 return -EINVAL;
2031 pr_debug("map '%s': found map_flags = %u.\n",
2032 map->name, map->def.map_flags);
2033 } else if (strcmp(name, "numa_node") == 0) {
2034 if (!get_map_field_int(map->name, obj->btf, m, &map->numa_node))
2035 return -EINVAL;
2036 pr_debug("map '%s': found numa_node = %u.\n", map->name, map->numa_node);
2037 } else if (strcmp(name, "key_size") == 0) {
2038 __u32 sz;
2039
2040 if (!get_map_field_int(map->name, obj->btf, m, &sz))
2041 return -EINVAL;
2042 pr_debug("map '%s': found key_size = %u.\n",
2043 map->name, sz);
2044 if (map->def.key_size && map->def.key_size != sz) {
2045 pr_warn("map '%s': conflicting key size %u != %u.\n",
2046 map->name, map->def.key_size, sz);
2047 return -EINVAL;
2048 }
2049 map->def.key_size = sz;
2050 } else if (strcmp(name, "key") == 0) {
2051 __s64 sz;
2052
2053 t = btf__type_by_id(obj->btf, m->type);
2054 if (!t) {
2055 pr_warn("map '%s': key type [%d] not found.\n",
2056 map->name, m->type);
2057 return -EINVAL;
2058 }
2059 if (!btf_is_ptr(t)) {
2060 pr_warn("map '%s': key spec is not PTR: %s.\n",
2061 map->name, btf_kind_str(t));
2062 return -EINVAL;
2063 }
2064 sz = btf__resolve_size(obj->btf, t->type);
2065 if (sz < 0) {
2066 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2067 map->name, t->type, (ssize_t)sz);
2068 return sz;
2069 }
2070 pr_debug("map '%s': found key [%u], sz = %zd.\n",
2071 map->name, t->type, (ssize_t)sz);
2072 if (map->def.key_size && map->def.key_size != sz) {
2073 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2074 map->name, map->def.key_size, (ssize_t)sz);
2075 return -EINVAL;
2076 }
2077 map->def.key_size = sz;
2078 map->btf_key_type_id = t->type;
2079 } else if (strcmp(name, "value_size") == 0) {
2080 __u32 sz;
2081
2082 if (!get_map_field_int(map->name, obj->btf, m, &sz))
2083 return -EINVAL;
2084 pr_debug("map '%s': found value_size = %u.\n",
2085 map->name, sz);
2086 if (map->def.value_size && map->def.value_size != sz) {
2087 pr_warn("map '%s': conflicting value size %u != %u.\n",
2088 map->name, map->def.value_size, sz);
2089 return -EINVAL;
2090 }
2091 map->def.value_size = sz;
2092 } else if (strcmp(name, "value") == 0) {
2093 __s64 sz;
2094
2095 t = btf__type_by_id(obj->btf, m->type);
2096 if (!t) {
2097 pr_warn("map '%s': value type [%d] not found.\n",
2098 map->name, m->type);
2099 return -EINVAL;
2100 }
2101 if (!btf_is_ptr(t)) {
2102 pr_warn("map '%s': value spec is not PTR: %s.\n",
2103 map->name, btf_kind_str(t));
2104 return -EINVAL;
2105 }
2106 sz = btf__resolve_size(obj->btf, t->type);
2107 if (sz < 0) {
2108 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2109 map->name, t->type, (ssize_t)sz);
2110 return sz;
2111 }
2112 pr_debug("map '%s': found value [%u], sz = %zd.\n",
2113 map->name, t->type, (ssize_t)sz);
2114 if (map->def.value_size && map->def.value_size != sz) {
2115 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2116 map->name, map->def.value_size, (ssize_t)sz);
2117 return -EINVAL;
2118 }
2119 map->def.value_size = sz;
2120 map->btf_value_type_id = t->type;
2121 }
2122 else if (strcmp(name, "values") == 0) {
2123 int err;
2124
2125 if (is_inner) {
2126 pr_warn("map '%s': multi-level inner maps not supported.\n",
2127 map->name);
2128 return -ENOTSUP;
2129 }
2130 if (i != vlen - 1) {
2131 pr_warn("map '%s': '%s' member should be last.\n",
2132 map->name, name);
2133 return -EINVAL;
2134 }
2135 if (!bpf_map_type__is_map_in_map(map->def.type)) {
2136 pr_warn("map '%s': should be map-in-map.\n",
2137 map->name);
2138 return -ENOTSUP;
2139 }
2140 if (map->def.value_size && map->def.value_size != 4) {
2141 pr_warn("map '%s': conflicting value size %u != 4.\n",
2142 map->name, map->def.value_size);
2143 return -EINVAL;
2144 }
2145 map->def.value_size = 4;
2146 t = btf__type_by_id(obj->btf, m->type);
2147 if (!t) {
2148 pr_warn("map '%s': map-in-map inner type [%d] not found.\n",
2149 map->name, m->type);
2150 return -EINVAL;
2151 }
2152 if (!btf_is_array(t) || btf_array(t)->nelems) {
2153 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n",
2154 map->name);
2155 return -EINVAL;
2156 }
2157 t = skip_mods_and_typedefs(obj->btf, btf_array(t)->type,
2158 NULL);
2159 if (!btf_is_ptr(t)) {
2160 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2161 map->name, btf_kind_str(t));
2162 return -EINVAL;
2163 }
2164 t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
2165 if (!btf_is_struct(t)) {
2166 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2167 map->name, btf_kind_str(t));
2168 return -EINVAL;
2169 }
2170
2171 map->inner_map = calloc(1, sizeof(*map->inner_map));
2172 if (!map->inner_map)
2173 return -ENOMEM;
2174 map->inner_map->sec_idx = obj->efile.btf_maps_shndx;
2175 map->inner_map->name = malloc(strlen(map->name) +
2176 sizeof(".inner") + 1);
2177 if (!map->inner_map->name)
2178 return -ENOMEM;
2179 sprintf(map->inner_map->name, "%s.inner", map->name);
2180
2181 err = parse_btf_map_def(obj, map->inner_map, t, strict,
2182 true , NULL);
2183 if (err)
2184 return err;
2185 } else if (strcmp(name, "pinning") == 0) {
2186 __u32 val;
2187 int err;
2188
2189 if (is_inner) {
2190 pr_debug("map '%s': inner def can't be pinned.\n",
2191 map->name);
2192 return -EINVAL;
2193 }
2194 if (!get_map_field_int(map->name, obj->btf, m, &val))
2195 return -EINVAL;
2196 pr_debug("map '%s': found pinning = %u.\n",
2197 map->name, val);
2198
2199 if (val != LIBBPF_PIN_NONE &&
2200 val != LIBBPF_PIN_BY_NAME) {
2201 pr_warn("map '%s': invalid pinning value %u.\n",
2202 map->name, val);
2203 return -EINVAL;
2204 }
2205 if (val == LIBBPF_PIN_BY_NAME) {
2206 err = build_map_pin_path(map, pin_root_path);
2207 if (err) {
2208 pr_warn("map '%s': couldn't build pin path.\n",
2209 map->name);
2210 return err;
2211 }
2212 }
2213 } else {
2214 if (strict) {
2215 pr_warn("map '%s': unknown field '%s'.\n",
2216 map->name, name);
2217 return -ENOTSUP;
2218 }
2219 pr_debug("map '%s': ignoring unknown field '%s'.\n",
2220 map->name, name);
2221 }
2222 }
2223
2224 if (map->def.type == BPF_MAP_TYPE_UNSPEC) {
2225 pr_warn("map '%s': map type isn't specified.\n", map->name);
2226 return -EINVAL;
2227 }
2228
2229 return 0;
2230}
2231
2232static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2233 const struct btf_type *sec,
2234 int var_idx, int sec_idx,
2235 const Elf_Data *data, bool strict,
2236 const char *pin_root_path)
2237{
2238 const struct btf_type *var, *def;
2239 const struct btf_var_secinfo *vi;
2240 const struct btf_var *var_extra;
2241 const char *map_name;
2242 struct bpf_map *map;
2243
2244 vi = btf_var_secinfos(sec) + var_idx;
2245 var = btf__type_by_id(obj->btf, vi->type);
2246 var_extra = btf_var(var);
2247 map_name = btf__name_by_offset(obj->btf, var->name_off);
2248
2249 if (map_name == NULL || map_name[0] == '\0') {
2250 pr_warn("map #%d: empty name.\n", var_idx);
2251 return -EINVAL;
2252 }
2253 if ((__u64)vi->offset + vi->size > data->d_size) {
2254 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2255 return -EINVAL;
2256 }
2257 if (!btf_is_var(var)) {
2258 pr_warn("map '%s': unexpected var kind %s.\n",
2259 map_name, btf_kind_str(var));
2260 return -EINVAL;
2261 }
2262 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&
2263 var_extra->linkage != BTF_VAR_STATIC) {
2264 pr_warn("map '%s': unsupported var linkage %u.\n",
2265 map_name, var_extra->linkage);
2266 return -EOPNOTSUPP;
2267 }
2268
2269 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2270 if (!btf_is_struct(def)) {
2271 pr_warn("map '%s': unexpected def kind %s.\n",
2272 map_name, btf_kind_str(var));
2273 return -EINVAL;
2274 }
2275 if (def->size > vi->size) {
2276 pr_warn("map '%s': invalid def size.\n", map_name);
2277 return -EINVAL;
2278 }
2279
2280 map = bpf_object__add_map(obj);
2281 if (IS_ERR(map))
2282 return PTR_ERR(map);
2283 map->name = strdup(map_name);
2284 if (!map->name) {
2285 pr_warn("map '%s': failed to alloc map name.\n", map_name);
2286 return -ENOMEM;
2287 }
2288 map->libbpf_type = LIBBPF_MAP_UNSPEC;
2289 map->def.type = BPF_MAP_TYPE_UNSPEC;
2290 map->sec_idx = sec_idx;
2291 map->sec_offset = vi->offset;
2292 map->btf_var_idx = var_idx;
2293 pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2294 map_name, map->sec_idx, map->sec_offset);
2295
2296 return parse_btf_map_def(obj, map, def, strict, false, pin_root_path);
2297}
2298
2299static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2300 const char *pin_root_path)
2301{
2302 const struct btf_type *sec = NULL;
2303 int nr_types, i, vlen, err;
2304 const struct btf_type *t;
2305 const char *name;
2306 Elf_Data *data;
2307 Elf_Scn *scn;
2308
2309 if (obj->efile.btf_maps_shndx < 0)
2310 return 0;
2311
2312 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2313 data = elf_sec_data(obj, scn);
2314 if (!scn || !data) {
2315 pr_warn("elf: failed to get %s map definitions for %s\n",
2316 MAPS_ELF_SEC, obj->path);
2317 return -EINVAL;
2318 }
2319
2320 nr_types = btf__get_nr_types(obj->btf);
2321 for (i = 1; i <= nr_types; i++) {
2322 t = btf__type_by_id(obj->btf, i);
2323 if (!btf_is_datasec(t))
2324 continue;
2325 name = btf__name_by_offset(obj->btf, t->name_off);
2326 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2327 sec = t;
2328 obj->efile.btf_maps_sec_btf_id = i;
2329 break;
2330 }
2331 }
2332
2333 if (!sec) {
2334 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2335 return -ENOENT;
2336 }
2337
2338 vlen = btf_vlen(sec);
2339 for (i = 0; i < vlen; i++) {
2340 err = bpf_object__init_user_btf_map(obj, sec, i,
2341 obj->efile.btf_maps_shndx,
2342 data, strict,
2343 pin_root_path);
2344 if (err)
2345 return err;
2346 }
2347
2348 return 0;
2349}
2350
2351static int bpf_object__init_maps(struct bpf_object *obj,
2352 const struct bpf_object_open_opts *opts)
2353{
2354 const char *pin_root_path;
2355 bool strict;
2356 int err;
2357
2358 strict = !OPTS_GET(opts, relaxed_maps, false);
2359 pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2360
2361 err = bpf_object__init_user_maps(obj, strict);
2362 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2363 err = err ?: bpf_object__init_global_data_maps(obj);
2364 err = err ?: bpf_object__init_kconfig_map(obj);
2365 err = err ?: bpf_object__init_struct_ops_maps(obj);
2366 if (err)
2367 return err;
2368
2369 return 0;
2370}
2371
2372static bool section_have_execinstr(struct bpf_object *obj, int idx)
2373{
2374 GElf_Shdr sh;
2375
2376 if (elf_sec_hdr(obj, elf_sec_by_idx(obj, idx), &sh))
2377 return false;
2378
2379 return sh.sh_flags & SHF_EXECINSTR;
2380}
2381
2382static bool btf_needs_sanitization(struct bpf_object *obj)
2383{
2384 bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC);
2385 bool has_datasec = kernel_supports(FEAT_BTF_DATASEC);
2386 bool has_func = kernel_supports(FEAT_BTF_FUNC);
2387
2388 return !has_func || !has_datasec || !has_func_global;
2389}
2390
2391static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2392{
2393 bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC);
2394 bool has_datasec = kernel_supports(FEAT_BTF_DATASEC);
2395 bool has_func = kernel_supports(FEAT_BTF_FUNC);
2396 struct btf_type *t;
2397 int i, j, vlen;
2398
2399 for (i = 1; i <= btf__get_nr_types(btf); i++) {
2400 t = (struct btf_type *)btf__type_by_id(btf, i);
2401
2402 if (!has_datasec && btf_is_var(t)) {
2403
2404 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2405
2406
2407
2408
2409
2410 t->size = 1;
2411 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2412 } else if (!has_datasec && btf_is_datasec(t)) {
2413
2414 const struct btf_var_secinfo *v = btf_var_secinfos(t);
2415 struct btf_member *m = btf_members(t);
2416 struct btf_type *vt;
2417 char *name;
2418
2419 name = (char *)btf__name_by_offset(btf, t->name_off);
2420 while (*name) {
2421 if (*name == '.')
2422 *name = '_';
2423 name++;
2424 }
2425
2426 vlen = btf_vlen(t);
2427 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2428 for (j = 0; j < vlen; j++, v++, m++) {
2429
2430 m->offset = v->offset * 8;
2431 m->type = v->type;
2432
2433 vt = (void *)btf__type_by_id(btf, v->type);
2434 m->name_off = vt->name_off;
2435 }
2436 } else if (!has_func && btf_is_func_proto(t)) {
2437
2438 vlen = btf_vlen(t);
2439 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2440 t->size = sizeof(__u32);
2441 } else if (!has_func && btf_is_func(t)) {
2442
2443 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2444 } else if (!has_func_global && btf_is_func(t)) {
2445
2446 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2447 }
2448 }
2449}
2450
2451static bool libbpf_needs_btf(const struct bpf_object *obj)
2452{
2453 return obj->efile.btf_maps_shndx >= 0 ||
2454 obj->efile.st_ops_shndx >= 0 ||
2455 obj->nr_extern > 0;
2456}
2457
2458static bool kernel_needs_btf(const struct bpf_object *obj)
2459{
2460 return obj->efile.st_ops_shndx >= 0;
2461}
2462
2463static int bpf_object__init_btf(struct bpf_object *obj,
2464 Elf_Data *btf_data,
2465 Elf_Data *btf_ext_data)
2466{
2467 int err = -ENOENT;
2468
2469 if (btf_data) {
2470 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2471 if (IS_ERR(obj->btf)) {
2472 err = PTR_ERR(obj->btf);
2473 obj->btf = NULL;
2474 pr_warn("Error loading ELF section %s: %d.\n",
2475 BTF_ELF_SEC, err);
2476 goto out;
2477 }
2478
2479 btf__set_pointer_size(obj->btf, 8);
2480 err = 0;
2481 }
2482 if (btf_ext_data) {
2483 if (!obj->btf) {
2484 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2485 BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2486 goto out;
2487 }
2488 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
2489 btf_ext_data->d_size);
2490 if (IS_ERR(obj->btf_ext)) {
2491 pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n",
2492 BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));
2493 obj->btf_ext = NULL;
2494 goto out;
2495 }
2496 }
2497out:
2498 if (err && libbpf_needs_btf(obj)) {
2499 pr_warn("BTF is required, but is missing or corrupted.\n");
2500 return err;
2501 }
2502 return 0;
2503}
2504
2505static int bpf_object__finalize_btf(struct bpf_object *obj)
2506{
2507 int err;
2508
2509 if (!obj->btf)
2510 return 0;
2511
2512 err = btf__finalize_data(obj, obj->btf);
2513 if (err) {
2514 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
2515 return err;
2516 }
2517
2518 return 0;
2519}
2520
2521static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
2522{
2523 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
2524 prog->type == BPF_PROG_TYPE_LSM)
2525 return true;
2526
2527
2528
2529
2530 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
2531 return true;
2532
2533 return false;
2534}
2535
2536static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
2537{
2538 struct bpf_program *prog;
2539 int i;
2540
2541
2542 if (obj->btf_ext && obj->btf_ext->core_relo_info.len)
2543 return true;
2544
2545
2546 for (i = 0; i < obj->nr_extern; i++) {
2547 const struct extern_desc *ext;
2548
2549 ext = &obj->externs[i];
2550 if (ext->type == EXT_KSYM && ext->ksym.type_id)
2551 return true;
2552 }
2553
2554 bpf_object__for_each_program(prog, obj) {
2555 if (!prog->load)
2556 continue;
2557 if (prog_needs_vmlinux_btf(prog))
2558 return true;
2559 }
2560
2561 return false;
2562}
2563
2564static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
2565{
2566 int err;
2567
2568
2569 if (obj->btf_vmlinux)
2570 return 0;
2571
2572 if (!force && !obj_needs_vmlinux_btf(obj))
2573 return 0;
2574
2575 obj->btf_vmlinux = libbpf_find_kernel_btf();
2576 if (IS_ERR(obj->btf_vmlinux)) {
2577 err = PTR_ERR(obj->btf_vmlinux);
2578 pr_warn("Error loading vmlinux BTF: %d\n", err);
2579 obj->btf_vmlinux = NULL;
2580 return err;
2581 }
2582 return 0;
2583}
2584
2585static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
2586{
2587 struct btf *kern_btf = obj->btf;
2588 bool btf_mandatory, sanitize;
2589 int err = 0;
2590
2591 if (!obj->btf)
2592 return 0;
2593
2594 if (!kernel_supports(FEAT_BTF)) {
2595 if (kernel_needs_btf(obj)) {
2596 err = -EOPNOTSUPP;
2597 goto report;
2598 }
2599 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
2600 return 0;
2601 }
2602
2603 sanitize = btf_needs_sanitization(obj);
2604 if (sanitize) {
2605 const void *raw_data;
2606 __u32 sz;
2607
2608
2609 raw_data = btf__get_raw_data(obj->btf, &sz);
2610 kern_btf = btf__new(raw_data, sz);
2611 if (IS_ERR(kern_btf))
2612 return PTR_ERR(kern_btf);
2613
2614
2615 btf__set_pointer_size(obj->btf, 8);
2616 bpf_object__sanitize_btf(obj, kern_btf);
2617 }
2618
2619 err = btf__load(kern_btf);
2620 if (sanitize) {
2621 if (!err) {
2622
2623 btf__set_fd(obj->btf, btf__fd(kern_btf));
2624 btf__set_fd(kern_btf, -1);
2625 }
2626 btf__free(kern_btf);
2627 }
2628report:
2629 if (err) {
2630 btf_mandatory = kernel_needs_btf(obj);
2631 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
2632 btf_mandatory ? "BTF is mandatory, can't proceed."
2633 : "BTF is optional, ignoring.");
2634 if (!btf_mandatory)
2635 err = 0;
2636 }
2637 return err;
2638}
2639
2640static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
2641{
2642 const char *name;
2643
2644 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
2645 if (!name) {
2646 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
2647 off, obj->path, elf_errmsg(-1));
2648 return NULL;
2649 }
2650
2651 return name;
2652}
2653
2654static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
2655{
2656 const char *name;
2657
2658 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
2659 if (!name) {
2660 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
2661 off, obj->path, elf_errmsg(-1));
2662 return NULL;
2663 }
2664
2665 return name;
2666}
2667
2668static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
2669{
2670 Elf_Scn *scn;
2671
2672 scn = elf_getscn(obj->efile.elf, idx);
2673 if (!scn) {
2674 pr_warn("elf: failed to get section(%zu) from %s: %s\n",
2675 idx, obj->path, elf_errmsg(-1));
2676 return NULL;
2677 }
2678 return scn;
2679}
2680
2681static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
2682{
2683 Elf_Scn *scn = NULL;
2684 Elf *elf = obj->efile.elf;
2685 const char *sec_name;
2686
2687 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2688 sec_name = elf_sec_name(obj, scn);
2689 if (!sec_name)
2690 return NULL;
2691
2692 if (strcmp(sec_name, name) != 0)
2693 continue;
2694
2695 return scn;
2696 }
2697 return NULL;
2698}
2699
2700static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr)
2701{
2702 if (!scn)
2703 return -EINVAL;
2704
2705 if (gelf_getshdr(scn, hdr) != hdr) {
2706 pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
2707 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
2708 return -EINVAL;
2709 }
2710
2711 return 0;
2712}
2713
2714static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
2715{
2716 const char *name;
2717 GElf_Shdr sh;
2718
2719 if (!scn)
2720 return NULL;
2721
2722 if (elf_sec_hdr(obj, scn, &sh))
2723 return NULL;
2724
2725 name = elf_sec_str(obj, sh.sh_name);
2726 if (!name) {
2727 pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
2728 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
2729 return NULL;
2730 }
2731
2732 return name;
2733}
2734
2735static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
2736{
2737 Elf_Data *data;
2738
2739 if (!scn)
2740 return NULL;
2741
2742 data = elf_getdata(scn, 0);
2743 if (!data) {
2744 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
2745 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
2746 obj->path, elf_errmsg(-1));
2747 return NULL;
2748 }
2749
2750 return data;
2751}
2752
2753static int elf_sym_by_sec_off(const struct bpf_object *obj, size_t sec_idx,
2754 size_t off, __u32 sym_type, GElf_Sym *sym)
2755{
2756 Elf_Data *symbols = obj->efile.symbols;
2757 size_t n = symbols->d_size / sizeof(GElf_Sym);
2758 int i;
2759
2760 for (i = 0; i < n; i++) {
2761 if (!gelf_getsym(symbols, i, sym))
2762 continue;
2763 if (sym->st_shndx != sec_idx || sym->st_value != off)
2764 continue;
2765 if (GELF_ST_TYPE(sym->st_info) != sym_type)
2766 continue;
2767 return 0;
2768 }
2769
2770 return -ENOENT;
2771}
2772
2773static bool is_sec_name_dwarf(const char *name)
2774{
2775
2776 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
2777}
2778
2779static bool ignore_elf_section(GElf_Shdr *hdr, const char *name)
2780{
2781
2782 if (hdr->sh_type == SHT_STRTAB)
2783 return true;
2784
2785
2786 if (hdr->sh_type == 0x6FFF4C03 )
2787 return true;
2788
2789
2790 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
2791 strcmp(name, ".text") == 0)
2792 return true;
2793
2794
2795 if (is_sec_name_dwarf(name))
2796 return true;
2797
2798 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
2799 name += sizeof(".rel") - 1;
2800
2801 if (is_sec_name_dwarf(name))
2802 return true;
2803
2804
2805 if (strcmp(name, BTF_ELF_SEC) == 0 ||
2806 strcmp(name, BTF_EXT_ELF_SEC) == 0)
2807 return true;
2808 }
2809
2810 return false;
2811}
2812
2813static int cmp_progs(const void *_a, const void *_b)
2814{
2815 const struct bpf_program *a = _a;
2816 const struct bpf_program *b = _b;
2817
2818 if (a->sec_idx != b->sec_idx)
2819 return a->sec_idx < b->sec_idx ? -1 : 1;
2820
2821
2822 return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
2823}
2824
2825static int bpf_object__elf_collect(struct bpf_object *obj)
2826{
2827 Elf *elf = obj->efile.elf;
2828 Elf_Data *btf_ext_data = NULL;
2829 Elf_Data *btf_data = NULL;
2830 int idx = 0, err = 0;
2831 const char *name;
2832 Elf_Data *data;
2833 Elf_Scn *scn;
2834 GElf_Shdr sh;
2835
2836
2837
2838
2839 scn = NULL;
2840 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2841 if (elf_sec_hdr(obj, scn, &sh))
2842 return -LIBBPF_ERRNO__FORMAT;
2843
2844 if (sh.sh_type == SHT_SYMTAB) {
2845 if (obj->efile.symbols) {
2846 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
2847 return -LIBBPF_ERRNO__FORMAT;
2848 }
2849
2850 data = elf_sec_data(obj, scn);
2851 if (!data)
2852 return -LIBBPF_ERRNO__FORMAT;
2853
2854 obj->efile.symbols = data;
2855 obj->efile.symbols_shndx = elf_ndxscn(scn);
2856 obj->efile.strtabidx = sh.sh_link;
2857 }
2858 }
2859
2860 scn = NULL;
2861 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2862 idx++;
2863
2864 if (elf_sec_hdr(obj, scn, &sh))
2865 return -LIBBPF_ERRNO__FORMAT;
2866
2867 name = elf_sec_str(obj, sh.sh_name);
2868 if (!name)
2869 return -LIBBPF_ERRNO__FORMAT;
2870
2871 if (ignore_elf_section(&sh, name))
2872 continue;
2873
2874 data = elf_sec_data(obj, scn);
2875 if (!data)
2876 return -LIBBPF_ERRNO__FORMAT;
2877
2878 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
2879 idx, name, (unsigned long)data->d_size,
2880 (int)sh.sh_link, (unsigned long)sh.sh_flags,
2881 (int)sh.sh_type);
2882
2883 if (strcmp(name, "license") == 0) {
2884 err = bpf_object__init_license(obj, data->d_buf, data->d_size);
2885 if (err)
2886 return err;
2887 } else if (strcmp(name, "version") == 0) {
2888 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
2889 if (err)
2890 return err;
2891 } else if (strcmp(name, "maps") == 0) {
2892 obj->efile.maps_shndx = idx;
2893 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
2894 obj->efile.btf_maps_shndx = idx;
2895 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
2896 btf_data = data;
2897 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
2898 btf_ext_data = data;
2899 } else if (sh.sh_type == SHT_SYMTAB) {
2900
2901 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
2902 if (sh.sh_flags & SHF_EXECINSTR) {
2903 if (strcmp(name, ".text") == 0)
2904 obj->efile.text_shndx = idx;
2905 err = bpf_object__add_programs(obj, data, name, idx);
2906 if (err)
2907 return err;
2908 } else if (strcmp(name, DATA_SEC) == 0) {
2909 obj->efile.data = data;
2910 obj->efile.data_shndx = idx;
2911 } else if (strcmp(name, RODATA_SEC) == 0) {
2912 obj->efile.rodata = data;
2913 obj->efile.rodata_shndx = idx;
2914 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
2915 obj->efile.st_ops_data = data;
2916 obj->efile.st_ops_shndx = idx;
2917 } else {
2918 pr_info("elf: skipping unrecognized data section(%d) %s\n",
2919 idx, name);
2920 }
2921 } else if (sh.sh_type == SHT_REL) {
2922 int nr_sects = obj->efile.nr_reloc_sects;
2923 void *sects = obj->efile.reloc_sects;
2924 int sec = sh.sh_info;
2925
2926
2927 if (!section_have_execinstr(obj, sec) &&
2928 strcmp(name, ".rel" STRUCT_OPS_SEC) &&
2929 strcmp(name, ".rel" MAPS_ELF_SEC)) {
2930 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
2931 idx, name, sec,
2932 elf_sec_name(obj, elf_sec_by_idx(obj, sec)) ?: "<?>");
2933 continue;
2934 }
2935
2936 sects = libbpf_reallocarray(sects, nr_sects + 1,
2937 sizeof(*obj->efile.reloc_sects));
2938 if (!sects)
2939 return -ENOMEM;
2940
2941 obj->efile.reloc_sects = sects;
2942 obj->efile.nr_reloc_sects++;
2943
2944 obj->efile.reloc_sects[nr_sects].shdr = sh;
2945 obj->efile.reloc_sects[nr_sects].data = data;
2946 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) {
2947 obj->efile.bss = data;
2948 obj->efile.bss_shndx = idx;
2949 } else {
2950 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
2951 (size_t)sh.sh_size);
2952 }
2953 }
2954
2955 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
2956 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
2957 return -LIBBPF_ERRNO__FORMAT;
2958 }
2959
2960
2961
2962 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
2963
2964 return bpf_object__init_btf(obj, btf_data, btf_ext_data);
2965}
2966
2967static bool sym_is_extern(const GElf_Sym *sym)
2968{
2969 int bind = GELF_ST_BIND(sym->st_info);
2970
2971 return sym->st_shndx == SHN_UNDEF &&
2972 (bind == STB_GLOBAL || bind == STB_WEAK) &&
2973 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE;
2974}
2975
2976static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
2977{
2978 const struct btf_type *t;
2979 const char *var_name;
2980 int i, n;
2981
2982 if (!btf)
2983 return -ESRCH;
2984
2985 n = btf__get_nr_types(btf);
2986 for (i = 1; i <= n; i++) {
2987 t = btf__type_by_id(btf, i);
2988
2989 if (!btf_is_var(t))
2990 continue;
2991
2992 var_name = btf__name_by_offset(btf, t->name_off);
2993 if (strcmp(var_name, ext_name))
2994 continue;
2995
2996 if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
2997 return -EINVAL;
2998
2999 return i;
3000 }
3001
3002 return -ENOENT;
3003}
3004
3005static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
3006 const struct btf_var_secinfo *vs;
3007 const struct btf_type *t;
3008 int i, j, n;
3009
3010 if (!btf)
3011 return -ESRCH;
3012
3013 n = btf__get_nr_types(btf);
3014 for (i = 1; i <= n; i++) {
3015 t = btf__type_by_id(btf, i);
3016
3017 if (!btf_is_datasec(t))
3018 continue;
3019
3020 vs = btf_var_secinfos(t);
3021 for (j = 0; j < btf_vlen(t); j++, vs++) {
3022 if (vs->type == ext_btf_id)
3023 return i;
3024 }
3025 }
3026
3027 return -ENOENT;
3028}
3029
3030static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
3031 bool *is_signed)
3032{
3033 const struct btf_type *t;
3034 const char *name;
3035
3036 t = skip_mods_and_typedefs(btf, id, NULL);
3037 name = btf__name_by_offset(btf, t->name_off);
3038
3039 if (is_signed)
3040 *is_signed = false;
3041 switch (btf_kind(t)) {
3042 case BTF_KIND_INT: {
3043 int enc = btf_int_encoding(t);
3044
3045 if (enc & BTF_INT_BOOL)
3046 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
3047 if (is_signed)
3048 *is_signed = enc & BTF_INT_SIGNED;
3049 if (t->size == 1)
3050 return KCFG_CHAR;
3051 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
3052 return KCFG_UNKNOWN;
3053 return KCFG_INT;
3054 }
3055 case BTF_KIND_ENUM:
3056 if (t->size != 4)
3057 return KCFG_UNKNOWN;
3058 if (strcmp(name, "libbpf_tristate"))
3059 return KCFG_UNKNOWN;
3060 return KCFG_TRISTATE;
3061 case BTF_KIND_ARRAY:
3062 if (btf_array(t)->nelems == 0)
3063 return KCFG_UNKNOWN;
3064 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
3065 return KCFG_UNKNOWN;
3066 return KCFG_CHAR_ARR;
3067 default:
3068 return KCFG_UNKNOWN;
3069 }
3070}
3071
3072static int cmp_externs(const void *_a, const void *_b)
3073{
3074 const struct extern_desc *a = _a;
3075 const struct extern_desc *b = _b;
3076
3077 if (a->type != b->type)
3078 return a->type < b->type ? -1 : 1;
3079
3080 if (a->type == EXT_KCFG) {
3081
3082 if (a->kcfg.align != b->kcfg.align)
3083 return a->kcfg.align > b->kcfg.align ? -1 : 1;
3084
3085 if (a->kcfg.sz != b->kcfg.sz)
3086 return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
3087 }
3088
3089
3090 return strcmp(a->name, b->name);
3091}
3092
3093static int find_int_btf_id(const struct btf *btf)
3094{
3095 const struct btf_type *t;
3096 int i, n;
3097
3098 n = btf__get_nr_types(btf);
3099 for (i = 1; i <= n; i++) {
3100 t = btf__type_by_id(btf, i);
3101
3102 if (btf_is_int(t) && btf_int_bits(t) == 32)
3103 return i;
3104 }
3105
3106 return 0;
3107}
3108
3109static int bpf_object__collect_externs(struct bpf_object *obj)
3110{
3111 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
3112 const struct btf_type *t;
3113 struct extern_desc *ext;
3114 int i, n, off;
3115 const char *ext_name, *sec_name;
3116 Elf_Scn *scn;
3117 GElf_Shdr sh;
3118
3119 if (!obj->efile.symbols)
3120 return 0;
3121
3122 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
3123 if (elf_sec_hdr(obj, scn, &sh))
3124 return -LIBBPF_ERRNO__FORMAT;
3125
3126 n = sh.sh_size / sh.sh_entsize;
3127 pr_debug("looking for externs among %d symbols...\n", n);
3128
3129 for (i = 0; i < n; i++) {
3130 GElf_Sym sym;
3131
3132 if (!gelf_getsym(obj->efile.symbols, i, &sym))
3133 return -LIBBPF_ERRNO__FORMAT;
3134 if (!sym_is_extern(&sym))
3135 continue;
3136 ext_name = elf_sym_str(obj, sym.st_name);
3137 if (!ext_name || !ext_name[0])
3138 continue;
3139
3140 ext = obj->externs;
3141 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
3142 if (!ext)
3143 return -ENOMEM;
3144 obj->externs = ext;
3145 ext = &ext[obj->nr_extern];
3146 memset(ext, 0, sizeof(*ext));
3147 obj->nr_extern++;
3148
3149 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
3150 if (ext->btf_id <= 0) {
3151 pr_warn("failed to find BTF for extern '%s': %d\n",
3152 ext_name, ext->btf_id);
3153 return ext->btf_id;
3154 }
3155 t = btf__type_by_id(obj->btf, ext->btf_id);
3156 ext->name = btf__name_by_offset(obj->btf, t->name_off);
3157 ext->sym_idx = i;
3158 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK;
3159
3160 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
3161 if (ext->sec_btf_id <= 0) {
3162 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
3163 ext_name, ext->btf_id, ext->sec_btf_id);
3164 return ext->sec_btf_id;
3165 }
3166 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
3167 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3168
3169 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
3170 kcfg_sec = sec;
3171 ext->type = EXT_KCFG;
3172 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
3173 if (ext->kcfg.sz <= 0) {
3174 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
3175 ext_name, ext->kcfg.sz);
3176 return ext->kcfg.sz;
3177 }
3178 ext->kcfg.align = btf__align_of(obj->btf, t->type);
3179 if (ext->kcfg.align <= 0) {
3180 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
3181 ext_name, ext->kcfg.align);
3182 return -EINVAL;
3183 }
3184 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
3185 &ext->kcfg.is_signed);
3186 if (ext->kcfg.type == KCFG_UNKNOWN) {
3187 pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name);
3188 return -ENOTSUP;
3189 }
3190 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3191 ksym_sec = sec;
3192 ext->type = EXT_KSYM;
3193 skip_mods_and_typedefs(obj->btf, t->type,
3194 &ext->ksym.type_id);
3195 } else {
3196 pr_warn("unrecognized extern section '%s'\n", sec_name);
3197 return -ENOTSUP;
3198 }
3199 }
3200 pr_debug("collected %d externs total\n", obj->nr_extern);
3201
3202 if (!obj->nr_extern)
3203 return 0;
3204
3205
3206 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
3207
3208
3209
3210
3211
3212 if (ksym_sec) {
3213
3214
3215
3216 int int_btf_id = find_int_btf_id(obj->btf);
3217
3218 for (i = 0; i < obj->nr_extern; i++) {
3219 ext = &obj->externs[i];
3220 if (ext->type != EXT_KSYM)
3221 continue;
3222 pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
3223 i, ext->sym_idx, ext->name);
3224 }
3225
3226 sec = ksym_sec;
3227 n = btf_vlen(sec);
3228 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
3229 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3230 struct btf_type *vt;
3231
3232 vt = (void *)btf__type_by_id(obj->btf, vs->type);
3233 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
3234 ext = find_extern_by_name(obj, ext_name);
3235 if (!ext) {
3236 pr_warn("failed to find extern definition for BTF var '%s'\n",
3237 ext_name);
3238 return -ESRCH;
3239 }
3240 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3241 vt->type = int_btf_id;
3242 vs->offset = off;
3243 vs->size = sizeof(int);
3244 }
3245 sec->size = off;
3246 }
3247
3248 if (kcfg_sec) {
3249 sec = kcfg_sec;
3250
3251 off = 0;
3252 for (i = 0; i < obj->nr_extern; i++) {
3253 ext = &obj->externs[i];
3254 if (ext->type != EXT_KCFG)
3255 continue;
3256
3257 ext->kcfg.data_off = roundup(off, ext->kcfg.align);
3258 off = ext->kcfg.data_off + ext->kcfg.sz;
3259 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
3260 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
3261 }
3262 sec->size = off;
3263 n = btf_vlen(sec);
3264 for (i = 0; i < n; i++) {
3265 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3266
3267 t = btf__type_by_id(obj->btf, vs->type);
3268 ext_name = btf__name_by_offset(obj->btf, t->name_off);
3269 ext = find_extern_by_name(obj, ext_name);
3270 if (!ext) {
3271 pr_warn("failed to find extern definition for BTF var '%s'\n",
3272 ext_name);
3273 return -ESRCH;
3274 }
3275 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3276 vs->offset = ext->kcfg.data_off;
3277 }
3278 }
3279 return 0;
3280}
3281
3282struct bpf_program *
3283bpf_object__find_program_by_title(const struct bpf_object *obj,
3284 const char *title)
3285{
3286 struct bpf_program *pos;
3287
3288 bpf_object__for_each_program(pos, obj) {
3289 if (pos->sec_name && !strcmp(pos->sec_name, title))
3290 return pos;
3291 }
3292 return NULL;
3293}
3294
3295static bool prog_is_subprog(const struct bpf_object *obj,
3296 const struct bpf_program *prog)
3297{
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
3311}
3312
3313struct bpf_program *
3314bpf_object__find_program_by_name(const struct bpf_object *obj,
3315 const char *name)
3316{
3317 struct bpf_program *prog;
3318
3319 bpf_object__for_each_program(prog, obj) {
3320 if (prog_is_subprog(obj, prog))
3321 continue;
3322 if (!strcmp(prog->name, name))
3323 return prog;
3324 }
3325 return NULL;
3326}
3327
3328static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
3329 int shndx)
3330{
3331 return shndx == obj->efile.data_shndx ||
3332 shndx == obj->efile.bss_shndx ||
3333 shndx == obj->efile.rodata_shndx;
3334}
3335
3336static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
3337 int shndx)
3338{
3339 return shndx == obj->efile.maps_shndx ||
3340 shndx == obj->efile.btf_maps_shndx;
3341}
3342
3343static enum libbpf_map_type
3344bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
3345{
3346 if (shndx == obj->efile.data_shndx)
3347 return LIBBPF_MAP_DATA;
3348 else if (shndx == obj->efile.bss_shndx)
3349 return LIBBPF_MAP_BSS;
3350 else if (shndx == obj->efile.rodata_shndx)
3351 return LIBBPF_MAP_RODATA;
3352 else if (shndx == obj->efile.symbols_shndx)
3353 return LIBBPF_MAP_KCONFIG;
3354 else
3355 return LIBBPF_MAP_UNSPEC;
3356}
3357
3358static int bpf_program__record_reloc(struct bpf_program *prog,
3359 struct reloc_desc *reloc_desc,
3360 __u32 insn_idx, const char *sym_name,
3361 const GElf_Sym *sym, const GElf_Rel *rel)
3362{
3363 struct bpf_insn *insn = &prog->insns[insn_idx];
3364 size_t map_idx, nr_maps = prog->obj->nr_maps;
3365 struct bpf_object *obj = prog->obj;
3366 __u32 shdr_idx = sym->st_shndx;
3367 enum libbpf_map_type type;
3368 const char *sym_sec_name;
3369 struct bpf_map *map;
3370
3371 reloc_desc->processed = false;
3372
3373
3374 if (insn->code == (BPF_JMP | BPF_CALL)) {
3375 if (insn->src_reg != BPF_PSEUDO_CALL) {
3376 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
3377 return -LIBBPF_ERRNO__RELOC;
3378 }
3379
3380 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
3381 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3382 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
3383 prog->name, sym_name, sym_sec_name);
3384 return -LIBBPF_ERRNO__RELOC;
3385 }
3386 if (sym->st_value % BPF_INSN_SZ) {
3387 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
3388 prog->name, sym_name, (size_t)sym->st_value);
3389 return -LIBBPF_ERRNO__RELOC;
3390 }
3391 reloc_desc->type = RELO_CALL;
3392 reloc_desc->insn_idx = insn_idx;
3393 reloc_desc->sym_off = sym->st_value;
3394 return 0;
3395 }
3396
3397 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) {
3398 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
3399 prog->name, sym_name, insn_idx, insn->code);
3400 return -LIBBPF_ERRNO__RELOC;
3401 }
3402
3403 if (sym_is_extern(sym)) {
3404 int sym_idx = GELF_R_SYM(rel->r_info);
3405 int i, n = obj->nr_extern;
3406 struct extern_desc *ext;
3407
3408 for (i = 0; i < n; i++) {
3409 ext = &obj->externs[i];
3410 if (ext->sym_idx == sym_idx)
3411 break;
3412 }
3413 if (i >= n) {
3414 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
3415 prog->name, sym_name, sym_idx);
3416 return -LIBBPF_ERRNO__RELOC;
3417 }
3418 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
3419 prog->name, i, ext->name, ext->sym_idx, insn_idx);
3420 reloc_desc->type = RELO_EXTERN;
3421 reloc_desc->insn_idx = insn_idx;
3422 reloc_desc->sym_off = i;
3423 return 0;
3424 }
3425
3426 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
3427 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
3428 prog->name, sym_name, shdr_idx);
3429 return -LIBBPF_ERRNO__RELOC;
3430 }
3431
3432 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
3433 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3434
3435
3436 if (type == LIBBPF_MAP_UNSPEC) {
3437 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
3438 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
3439 prog->name, sym_name, sym_sec_name);
3440 return -LIBBPF_ERRNO__RELOC;
3441 }
3442 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3443 map = &obj->maps[map_idx];
3444 if (map->libbpf_type != type ||
3445 map->sec_idx != sym->st_shndx ||
3446 map->sec_offset != sym->st_value)
3447 continue;
3448 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
3449 prog->name, map_idx, map->name, map->sec_idx,
3450 map->sec_offset, insn_idx);
3451 break;
3452 }
3453 if (map_idx >= nr_maps) {
3454 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
3455 prog->name, sym_sec_name, (size_t)sym->st_value);
3456 return -LIBBPF_ERRNO__RELOC;
3457 }
3458 reloc_desc->type = RELO_LD64;
3459 reloc_desc->insn_idx = insn_idx;
3460 reloc_desc->map_idx = map_idx;
3461 reloc_desc->sym_off = 0;
3462 return 0;
3463 }
3464
3465
3466 if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
3467 pr_warn("prog '%s': bad data relo against section '%s'\n",
3468 prog->name, sym_sec_name);
3469 return -LIBBPF_ERRNO__RELOC;
3470 }
3471 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3472 map = &obj->maps[map_idx];
3473 if (map->libbpf_type != type)
3474 continue;
3475 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
3476 prog->name, map_idx, map->name, map->sec_idx,
3477 map->sec_offset, insn_idx);
3478 break;
3479 }
3480 if (map_idx >= nr_maps) {
3481 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
3482 prog->name, sym_sec_name);
3483 return -LIBBPF_ERRNO__RELOC;
3484 }
3485
3486 reloc_desc->type = RELO_DATA;
3487 reloc_desc->insn_idx = insn_idx;
3488 reloc_desc->map_idx = map_idx;
3489 reloc_desc->sym_off = sym->st_value;
3490 return 0;
3491}
3492
3493static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
3494{
3495 return insn_idx >= prog->sec_insn_off &&
3496 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
3497}
3498
3499static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
3500 size_t sec_idx, size_t insn_idx)
3501{
3502 int l = 0, r = obj->nr_programs - 1, m;
3503 struct bpf_program *prog;
3504
3505 while (l < r) {
3506 m = l + (r - l + 1) / 2;
3507 prog = &obj->programs[m];
3508
3509 if (prog->sec_idx < sec_idx ||
3510 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
3511 l = m;
3512 else
3513 r = m - 1;
3514 }
3515
3516
3517
3518 prog = &obj->programs[l];
3519 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
3520 return prog;
3521 return NULL;
3522}
3523
3524static int
3525bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data *data)
3526{
3527 Elf_Data *symbols = obj->efile.symbols;
3528 const char *relo_sec_name, *sec_name;
3529 size_t sec_idx = shdr->sh_info;
3530 struct bpf_program *prog;
3531 struct reloc_desc *relos;
3532 int err, i, nrels;
3533 const char *sym_name;
3534 __u32 insn_idx;
3535 GElf_Sym sym;
3536 GElf_Rel rel;
3537
3538 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
3539 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
3540 if (!relo_sec_name || !sec_name)
3541 return -EINVAL;
3542
3543 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
3544 relo_sec_name, sec_idx, sec_name);
3545 nrels = shdr->sh_size / shdr->sh_entsize;
3546
3547 for (i = 0; i < nrels; i++) {
3548 if (!gelf_getrel(data, i, &rel)) {
3549 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
3550 return -LIBBPF_ERRNO__FORMAT;
3551 }
3552 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
3553 pr_warn("sec '%s': symbol 0x%zx not found for relo #%d\n",
3554 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);
3555 return -LIBBPF_ERRNO__FORMAT;
3556 }
3557 if (rel.r_offset % BPF_INSN_SZ) {
3558 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
3559 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);
3560 return -LIBBPF_ERRNO__FORMAT;
3561 }
3562
3563 insn_idx = rel.r_offset / BPF_INSN_SZ;
3564
3565
3566
3567
3568
3569
3570 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION && sym.st_name == 0)
3571 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym.st_shndx));
3572 else
3573 sym_name = elf_sym_str(obj, sym.st_name);
3574 sym_name = sym_name ?: "<?";
3575
3576 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
3577 relo_sec_name, i, insn_idx, sym_name);
3578
3579 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
3580 if (!prog) {
3581 pr_warn("sec '%s': relo #%d: program not found in section '%s' for insn #%u\n",
3582 relo_sec_name, i, sec_name, insn_idx);
3583 return -LIBBPF_ERRNO__RELOC;
3584 }
3585
3586 relos = libbpf_reallocarray(prog->reloc_desc,
3587 prog->nr_reloc + 1, sizeof(*relos));
3588 if (!relos)
3589 return -ENOMEM;
3590 prog->reloc_desc = relos;
3591
3592
3593 insn_idx -= prog->sec_insn_off;
3594 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
3595 insn_idx, sym_name, &sym, &rel);
3596 if (err)
3597 return err;
3598
3599 prog->nr_reloc++;
3600 }
3601 return 0;
3602}
3603
3604static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
3605{
3606 struct bpf_map_def *def = &map->def;
3607 __u32 key_type_id = 0, value_type_id = 0;
3608 int ret;
3609
3610
3611
3612
3613
3614 if (map->sec_idx == obj->efile.btf_maps_shndx ||
3615 bpf_map__is_struct_ops(map))
3616 return 0;
3617
3618 if (!bpf_map__is_internal(map)) {
3619 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size,
3620 def->value_size, &key_type_id,
3621 &value_type_id);
3622 } else {
3623
3624
3625
3626
3627 ret = btf__find_by_name(obj->btf,
3628 libbpf_type_to_btf_name[map->libbpf_type]);
3629 }
3630 if (ret < 0)
3631 return ret;
3632
3633 map->btf_key_type_id = key_type_id;
3634 map->btf_value_type_id = bpf_map__is_internal(map) ?
3635 ret : value_type_id;
3636 return 0;
3637}
3638
3639int bpf_map__reuse_fd(struct bpf_map *map, int fd)
3640{
3641 struct bpf_map_info info = {};
3642 __u32 len = sizeof(info);
3643 int new_fd, err;
3644 char *new_name;
3645
3646 err = bpf_obj_get_info_by_fd(fd, &info, &len);
3647 if (err)
3648 return err;
3649
3650 new_name = strdup(info.name);
3651 if (!new_name)
3652 return -errno;
3653
3654 new_fd = open("/", O_RDONLY | O_CLOEXEC);
3655 if (new_fd < 0) {
3656 err = -errno;
3657 goto err_free_new_name;
3658 }
3659
3660 new_fd = dup3(fd, new_fd, O_CLOEXEC);
3661 if (new_fd < 0) {
3662 err = -errno;
3663 goto err_close_new_fd;
3664 }
3665
3666 err = zclose(map->fd);
3667 if (err) {
3668 err = -errno;
3669 goto err_close_new_fd;
3670 }
3671 free(map->name);
3672
3673 map->fd = new_fd;
3674 map->name = new_name;
3675 map->def.type = info.type;
3676 map->def.key_size = info.key_size;
3677 map->def.value_size = info.value_size;
3678 map->def.max_entries = info.max_entries;
3679 map->def.map_flags = info.map_flags;
3680 map->btf_key_type_id = info.btf_key_type_id;
3681 map->btf_value_type_id = info.btf_value_type_id;
3682 map->reused = true;
3683
3684 return 0;
3685
3686err_close_new_fd:
3687 close(new_fd);
3688err_free_new_name:
3689 free(new_name);
3690 return err;
3691}
3692
3693__u32 bpf_map__max_entries(const struct bpf_map *map)
3694{
3695 return map->def.max_entries;
3696}
3697
3698int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
3699{
3700 if (map->fd >= 0)
3701 return -EBUSY;
3702 map->def.max_entries = max_entries;
3703 return 0;
3704}
3705
3706int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
3707{
3708 if (!map || !max_entries)
3709 return -EINVAL;
3710
3711 return bpf_map__set_max_entries(map, max_entries);
3712}
3713
3714static int
3715bpf_object__probe_loading(struct bpf_object *obj)
3716{
3717 struct bpf_load_program_attr attr;
3718 char *cp, errmsg[STRERR_BUFSIZE];
3719 struct bpf_insn insns[] = {
3720 BPF_MOV64_IMM(BPF_REG_0, 0),
3721 BPF_EXIT_INSN(),
3722 };
3723 int ret;
3724
3725
3726
3727 memset(&attr, 0, sizeof(attr));
3728 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3729 attr.insns = insns;
3730 attr.insns_cnt = ARRAY_SIZE(insns);
3731 attr.license = "GPL";
3732
3733 ret = bpf_load_program_xattr(&attr, NULL, 0);
3734 if (ret < 0) {
3735 ret = errno;
3736 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3737 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
3738 "program. Make sure your kernel supports BPF "
3739 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
3740 "set to big enough value.\n", __func__, cp, ret);
3741 return -ret;
3742 }
3743 close(ret);
3744
3745 return 0;
3746}
3747
3748static int probe_fd(int fd)
3749{
3750 if (fd >= 0)
3751 close(fd);
3752 return fd >= 0;
3753}
3754
3755static int probe_kern_prog_name(void)
3756{
3757 struct bpf_load_program_attr attr;
3758 struct bpf_insn insns[] = {
3759 BPF_MOV64_IMM(BPF_REG_0, 0),
3760 BPF_EXIT_INSN(),
3761 };
3762 int ret;
3763
3764
3765
3766 memset(&attr, 0, sizeof(attr));
3767 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3768 attr.insns = insns;
3769 attr.insns_cnt = ARRAY_SIZE(insns);
3770 attr.license = "GPL";
3771 attr.name = "test";
3772 ret = bpf_load_program_xattr(&attr, NULL, 0);
3773 return probe_fd(ret);
3774}
3775
3776static int probe_kern_global_data(void)
3777{
3778 struct bpf_load_program_attr prg_attr;
3779 struct bpf_create_map_attr map_attr;
3780 char *cp, errmsg[STRERR_BUFSIZE];
3781 struct bpf_insn insns[] = {
3782 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
3783 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
3784 BPF_MOV64_IMM(BPF_REG_0, 0),
3785 BPF_EXIT_INSN(),
3786 };
3787 int ret, map;
3788
3789 memset(&map_attr, 0, sizeof(map_attr));
3790 map_attr.map_type = BPF_MAP_TYPE_ARRAY;
3791 map_attr.key_size = sizeof(int);
3792 map_attr.value_size = 32;
3793 map_attr.max_entries = 1;
3794
3795 map = bpf_create_map_xattr(&map_attr);
3796 if (map < 0) {
3797 ret = -errno;
3798 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3799 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
3800 __func__, cp, -ret);
3801 return ret;
3802 }
3803
3804 insns[0].imm = map;
3805
3806 memset(&prg_attr, 0, sizeof(prg_attr));
3807 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3808 prg_attr.insns = insns;
3809 prg_attr.insns_cnt = ARRAY_SIZE(insns);
3810 prg_attr.license = "GPL";
3811
3812 ret = bpf_load_program_xattr(&prg_attr, NULL, 0);
3813 close(map);
3814 return probe_fd(ret);
3815}
3816
3817static int probe_kern_btf(void)
3818{
3819 static const char strs[] = "\0int";
3820 __u32 types[] = {
3821
3822 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
3823 };
3824
3825 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3826 strs, sizeof(strs)));
3827}
3828
3829static int probe_kern_btf_func(void)
3830{
3831 static const char strs[] = "\0int\0x\0a";
3832
3833 __u32 types[] = {
3834
3835 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
3836
3837 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3838 BTF_PARAM_ENC(7, 1),
3839
3840 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
3841 };
3842
3843 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3844 strs, sizeof(strs)));
3845}
3846
3847static int probe_kern_btf_func_global(void)
3848{
3849 static const char strs[] = "\0int\0x\0a";
3850
3851 __u32 types[] = {
3852
3853 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
3854
3855 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
3856 BTF_PARAM_ENC(7, 1),
3857
3858 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
3859 };
3860
3861 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3862 strs, sizeof(strs)));
3863}
3864
3865static int probe_kern_btf_datasec(void)
3866{
3867 static const char strs[] = "\0x\0.data";
3868
3869 __u32 types[] = {
3870
3871 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
3872
3873 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
3874 BTF_VAR_STATIC,
3875
3876 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
3877 BTF_VAR_SECINFO_ENC(2, 0, 4),
3878 };
3879
3880 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
3881 strs, sizeof(strs)));
3882}
3883
3884static int probe_kern_array_mmap(void)
3885{
3886 struct bpf_create_map_attr attr = {
3887 .map_type = BPF_MAP_TYPE_ARRAY,
3888 .map_flags = BPF_F_MMAPABLE,
3889 .key_size = sizeof(int),
3890 .value_size = sizeof(int),
3891 .max_entries = 1,
3892 };
3893
3894 return probe_fd(bpf_create_map_xattr(&attr));
3895}
3896
3897static int probe_kern_exp_attach_type(void)
3898{
3899 struct bpf_load_program_attr attr;
3900 struct bpf_insn insns[] = {
3901 BPF_MOV64_IMM(BPF_REG_0, 0),
3902 BPF_EXIT_INSN(),
3903 };
3904
3905 memset(&attr, 0, sizeof(attr));
3906
3907
3908
3909
3910
3911 attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK;
3912 attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE;
3913 attr.insns = insns;
3914 attr.insns_cnt = ARRAY_SIZE(insns);
3915 attr.license = "GPL";
3916
3917 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0));
3918}
3919
3920static int probe_kern_probe_read_kernel(void)
3921{
3922 struct bpf_load_program_attr attr;
3923 struct bpf_insn insns[] = {
3924 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
3925 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
3926 BPF_MOV64_IMM(BPF_REG_2, 8),
3927 BPF_MOV64_IMM(BPF_REG_3, 0),
3928 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel),
3929 BPF_EXIT_INSN(),
3930 };
3931
3932 memset(&attr, 0, sizeof(attr));
3933 attr.prog_type = BPF_PROG_TYPE_KPROBE;
3934 attr.insns = insns;
3935 attr.insns_cnt = ARRAY_SIZE(insns);
3936 attr.license = "GPL";
3937
3938 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0));
3939}
3940
3941static int probe_prog_bind_map(void)
3942{
3943 struct bpf_load_program_attr prg_attr;
3944 struct bpf_create_map_attr map_attr;
3945 char *cp, errmsg[STRERR_BUFSIZE];
3946 struct bpf_insn insns[] = {
3947 BPF_MOV64_IMM(BPF_REG_0, 0),
3948 BPF_EXIT_INSN(),
3949 };
3950 int ret, map, prog;
3951
3952 memset(&map_attr, 0, sizeof(map_attr));
3953 map_attr.map_type = BPF_MAP_TYPE_ARRAY;
3954 map_attr.key_size = sizeof(int);
3955 map_attr.value_size = 32;
3956 map_attr.max_entries = 1;
3957
3958 map = bpf_create_map_xattr(&map_attr);
3959 if (map < 0) {
3960 ret = -errno;
3961 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
3962 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
3963 __func__, cp, -ret);
3964 return ret;
3965 }
3966
3967 memset(&prg_attr, 0, sizeof(prg_attr));
3968 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
3969 prg_attr.insns = insns;
3970 prg_attr.insns_cnt = ARRAY_SIZE(insns);
3971 prg_attr.license = "GPL";
3972
3973 prog = bpf_load_program_xattr(&prg_attr, NULL, 0);
3974 if (prog < 0) {
3975 close(map);
3976 return 0;
3977 }
3978
3979 ret = bpf_prog_bind_map(prog, map, NULL);
3980
3981 close(map);
3982 close(prog);
3983
3984 return ret >= 0;
3985}
3986
3987static int probe_module_btf(void)
3988{
3989 static const char strs[] = "\0int";
3990 __u32 types[] = {
3991
3992 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
3993 };
3994 struct bpf_btf_info info;
3995 __u32 len = sizeof(info);
3996 char name[16];
3997 int fd, err;
3998
3999 fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
4000 if (fd < 0)
4001 return 0;
4002
4003 memset(&info, 0, sizeof(info));
4004 info.name = ptr_to_u64(name);
4005 info.name_len = sizeof(name);
4006
4007
4008
4009
4010
4011 err = bpf_obj_get_info_by_fd(fd, &info, &len);
4012 close(fd);
4013 return !err;
4014}
4015
4016enum kern_feature_result {
4017 FEAT_UNKNOWN = 0,
4018 FEAT_SUPPORTED = 1,
4019 FEAT_MISSING = 2,
4020};
4021
4022typedef int (*feature_probe_fn)(void);
4023
4024static struct kern_feature_desc {
4025 const char *desc;
4026 feature_probe_fn probe;
4027 enum kern_feature_result res;
4028} feature_probes[__FEAT_CNT] = {
4029 [FEAT_PROG_NAME] = {
4030 "BPF program name", probe_kern_prog_name,
4031 },
4032 [FEAT_GLOBAL_DATA] = {
4033 "global variables", probe_kern_global_data,
4034 },
4035 [FEAT_BTF] = {
4036 "minimal BTF", probe_kern_btf,
4037 },
4038 [FEAT_BTF_FUNC] = {
4039 "BTF functions", probe_kern_btf_func,
4040 },
4041 [FEAT_BTF_GLOBAL_FUNC] = {
4042 "BTF global function", probe_kern_btf_func_global,
4043 },
4044 [FEAT_BTF_DATASEC] = {
4045 "BTF data section and variable", probe_kern_btf_datasec,
4046 },
4047 [FEAT_ARRAY_MMAP] = {
4048 "ARRAY map mmap()", probe_kern_array_mmap,
4049 },
4050 [FEAT_EXP_ATTACH_TYPE] = {
4051 "BPF_PROG_LOAD expected_attach_type attribute",
4052 probe_kern_exp_attach_type,
4053 },
4054 [FEAT_PROBE_READ_KERN] = {
4055 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel,
4056 },
4057 [FEAT_PROG_BIND_MAP] = {
4058 "BPF_PROG_BIND_MAP support", probe_prog_bind_map,
4059 },
4060 [FEAT_MODULE_BTF] = {
4061 "module BTF support", probe_module_btf,
4062 },
4063};
4064
4065static bool kernel_supports(enum kern_feature_id feat_id)
4066{
4067 struct kern_feature_desc *feat = &feature_probes[feat_id];
4068 int ret;
4069
4070 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
4071 ret = feat->probe();
4072 if (ret > 0) {
4073 WRITE_ONCE(feat->res, FEAT_SUPPORTED);
4074 } else if (ret == 0) {
4075 WRITE_ONCE(feat->res, FEAT_MISSING);
4076 } else {
4077 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
4078 WRITE_ONCE(feat->res, FEAT_MISSING);
4079 }
4080 }
4081
4082 return READ_ONCE(feat->res) == FEAT_SUPPORTED;
4083}
4084
4085static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
4086{
4087 struct bpf_map_info map_info = {};
4088 char msg[STRERR_BUFSIZE];
4089 __u32 map_info_len;
4090
4091 map_info_len = sizeof(map_info);
4092
4093 if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) {
4094 pr_warn("failed to get map info for map FD %d: %s\n",
4095 map_fd, libbpf_strerror_r(errno, msg, sizeof(msg)));
4096 return false;
4097 }
4098
4099 return (map_info.type == map->def.type &&
4100 map_info.key_size == map->def.key_size &&
4101 map_info.value_size == map->def.value_size &&
4102 map_info.max_entries == map->def.max_entries &&
4103 map_info.map_flags == map->def.map_flags);
4104}
4105
4106static int
4107bpf_object__reuse_map(struct bpf_map *map)
4108{
4109 char *cp, errmsg[STRERR_BUFSIZE];
4110 int err, pin_fd;
4111
4112 pin_fd = bpf_obj_get(map->pin_path);
4113 if (pin_fd < 0) {
4114 err = -errno;
4115 if (err == -ENOENT) {
4116 pr_debug("found no pinned map to reuse at '%s'\n",
4117 map->pin_path);
4118 return 0;
4119 }
4120
4121 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
4122 pr_warn("couldn't retrieve pinned map '%s': %s\n",
4123 map->pin_path, cp);
4124 return err;
4125 }
4126
4127 if (!map_is_reuse_compat(map, pin_fd)) {
4128 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
4129 map->pin_path);
4130 close(pin_fd);
4131 return -EINVAL;
4132 }
4133
4134 err = bpf_map__reuse_fd(map, pin_fd);
4135 if (err) {
4136 close(pin_fd);
4137 return err;
4138 }
4139 map->pinned = true;
4140 pr_debug("reused pinned map at '%s'\n", map->pin_path);
4141
4142 return 0;
4143}
4144
4145static int
4146bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
4147{
4148 enum libbpf_map_type map_type = map->libbpf_type;
4149 char *cp, errmsg[STRERR_BUFSIZE];
4150 int err, zero = 0;
4151
4152 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
4153 if (err) {
4154 err = -errno;
4155 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4156 pr_warn("Error setting initial map(%s) contents: %s\n",
4157 map->name, cp);
4158 return err;
4159 }
4160
4161
4162 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
4163 err = bpf_map_freeze(map->fd);
4164 if (err) {
4165 err = -errno;
4166 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4167 pr_warn("Error freezing map(%s) as read-only: %s\n",
4168 map->name, cp);
4169 return err;
4170 }
4171 }
4172 return 0;
4173}
4174
4175static void bpf_map__destroy(struct bpf_map *map);
4176
4177static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map)
4178{
4179 struct bpf_create_map_attr create_attr;
4180 struct bpf_map_def *def = &map->def;
4181
4182 memset(&create_attr, 0, sizeof(create_attr));
4183
4184 if (kernel_supports(FEAT_PROG_NAME))
4185 create_attr.name = map->name;
4186 create_attr.map_ifindex = map->map_ifindex;
4187 create_attr.map_type = def->type;
4188 create_attr.map_flags = def->map_flags;
4189 create_attr.key_size = def->key_size;
4190 create_attr.value_size = def->value_size;
4191 create_attr.numa_node = map->numa_node;
4192
4193 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) {
4194 int nr_cpus;
4195
4196 nr_cpus = libbpf_num_possible_cpus();
4197 if (nr_cpus < 0) {
4198 pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
4199 map->name, nr_cpus);
4200 return nr_cpus;
4201 }
4202 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
4203 create_attr.max_entries = nr_cpus;
4204 } else {
4205 create_attr.max_entries = def->max_entries;
4206 }
4207
4208 if (bpf_map__is_struct_ops(map))
4209 create_attr.btf_vmlinux_value_type_id =
4210 map->btf_vmlinux_value_type_id;
4211
4212 create_attr.btf_fd = 0;
4213 create_attr.btf_key_type_id = 0;
4214 create_attr.btf_value_type_id = 0;
4215 if (obj->btf && btf__fd(obj->btf) >= 0 && !bpf_map_find_btf_info(obj, map)) {
4216 create_attr.btf_fd = btf__fd(obj->btf);
4217 create_attr.btf_key_type_id = map->btf_key_type_id;
4218 create_attr.btf_value_type_id = map->btf_value_type_id;
4219 }
4220
4221 if (bpf_map_type__is_map_in_map(def->type)) {
4222 if (map->inner_map) {
4223 int err;
4224
4225 err = bpf_object__create_map(obj, map->inner_map);
4226 if (err) {
4227 pr_warn("map '%s': failed to create inner map: %d\n",
4228 map->name, err);
4229 return err;
4230 }
4231 map->inner_map_fd = bpf_map__fd(map->inner_map);
4232 }
4233 if (map->inner_map_fd >= 0)
4234 create_attr.inner_map_fd = map->inner_map_fd;
4235 }
4236
4237 map->fd = bpf_create_map_xattr(&create_attr);
4238 if (map->fd < 0 && (create_attr.btf_key_type_id ||
4239 create_attr.btf_value_type_id)) {
4240 char *cp, errmsg[STRERR_BUFSIZE];
4241 int err = -errno;
4242
4243 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4244 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
4245 map->name, cp, err);
4246 create_attr.btf_fd = 0;
4247 create_attr.btf_key_type_id = 0;
4248 create_attr.btf_value_type_id = 0;
4249 map->btf_key_type_id = 0;
4250 map->btf_value_type_id = 0;
4251 map->fd = bpf_create_map_xattr(&create_attr);
4252 }
4253
4254 if (map->fd < 0)
4255 return -errno;
4256
4257 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
4258 bpf_map__destroy(map->inner_map);
4259 zfree(&map->inner_map);
4260 }
4261
4262 return 0;
4263}
4264
4265static int init_map_slots(struct bpf_map *map)
4266{
4267 const struct bpf_map *targ_map;
4268 unsigned int i;
4269 int fd, err;
4270
4271 for (i = 0; i < map->init_slots_sz; i++) {
4272 if (!map->init_slots[i])
4273 continue;
4274
4275 targ_map = map->init_slots[i];
4276 fd = bpf_map__fd(targ_map);
4277 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
4278 if (err) {
4279 err = -errno;
4280 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
4281 map->name, i, targ_map->name,
4282 fd, err);
4283 return err;
4284 }
4285 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
4286 map->name, i, targ_map->name, fd);
4287 }
4288
4289 zfree(&map->init_slots);
4290 map->init_slots_sz = 0;
4291
4292 return 0;
4293}
4294
4295static int
4296bpf_object__create_maps(struct bpf_object *obj)
4297{
4298 struct bpf_map *map;
4299 char *cp, errmsg[STRERR_BUFSIZE];
4300 unsigned int i, j;
4301 int err;
4302
4303 for (i = 0; i < obj->nr_maps; i++) {
4304 map = &obj->maps[i];
4305
4306 if (map->pin_path) {
4307 err = bpf_object__reuse_map(map);
4308 if (err) {
4309 pr_warn("map '%s': error reusing pinned map\n",
4310 map->name);
4311 goto err_out;
4312 }
4313 }
4314
4315 if (map->fd >= 0) {
4316 pr_debug("map '%s': skipping creation (preset fd=%d)\n",
4317 map->name, map->fd);
4318 } else {
4319 err = bpf_object__create_map(obj, map);
4320 if (err)
4321 goto err_out;
4322
4323 pr_debug("map '%s': created successfully, fd=%d\n",
4324 map->name, map->fd);
4325
4326 if (bpf_map__is_internal(map)) {
4327 err = bpf_object__populate_internal_map(obj, map);
4328 if (err < 0) {
4329 zclose(map->fd);
4330 goto err_out;
4331 }
4332 }
4333
4334 if (map->init_slots_sz) {
4335 err = init_map_slots(map);
4336 if (err < 0) {
4337 zclose(map->fd);
4338 goto err_out;
4339 }
4340 }
4341 }
4342
4343 if (map->pin_path && !map->pinned) {
4344 err = bpf_map__pin(map, NULL);
4345 if (err) {
4346 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
4347 map->name, map->pin_path, err);
4348 zclose(map->fd);
4349 goto err_out;
4350 }
4351 }
4352 }
4353
4354 return 0;
4355
4356err_out:
4357 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4358 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
4359 pr_perm_msg(err);
4360 for (j = 0; j < i; j++)
4361 zclose(obj->maps[j].fd);
4362 return err;
4363}
4364
4365#define BPF_CORE_SPEC_MAX_LEN 64
4366
4367
4368struct bpf_core_accessor {
4369 __u32 type_id;
4370 __u32 idx;
4371 const char *name;
4372};
4373
4374struct bpf_core_spec {
4375 const struct btf *btf;
4376
4377 struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN];
4378
4379 __u32 root_type_id;
4380
4381 enum bpf_core_relo_kind relo_kind;
4382
4383 int len;
4384
4385 int raw_spec[BPF_CORE_SPEC_MAX_LEN];
4386
4387 int raw_len;
4388
4389 __u32 bit_offset;
4390};
4391
4392static bool str_is_empty(const char *s)
4393{
4394 return !s || !s[0];
4395}
4396
4397static bool is_flex_arr(const struct btf *btf,
4398 const struct bpf_core_accessor *acc,
4399 const struct btf_array *arr)
4400{
4401 const struct btf_type *t;
4402
4403
4404 if (!acc->name || arr->nelems > 0)
4405 return false;
4406
4407
4408 t = btf__type_by_id(btf, acc->type_id);
4409 return acc->idx == btf_vlen(t) - 1;
4410}
4411
4412static const char *core_relo_kind_str(enum bpf_core_relo_kind kind)
4413{
4414 switch (kind) {
4415 case BPF_FIELD_BYTE_OFFSET: return "byte_off";
4416 case BPF_FIELD_BYTE_SIZE: return "byte_sz";
4417 case BPF_FIELD_EXISTS: return "field_exists";
4418 case BPF_FIELD_SIGNED: return "signed";
4419 case BPF_FIELD_LSHIFT_U64: return "lshift_u64";
4420 case BPF_FIELD_RSHIFT_U64: return "rshift_u64";
4421 case BPF_TYPE_ID_LOCAL: return "local_type_id";
4422 case BPF_TYPE_ID_TARGET: return "target_type_id";
4423 case BPF_TYPE_EXISTS: return "type_exists";
4424 case BPF_TYPE_SIZE: return "type_size";
4425 case BPF_ENUMVAL_EXISTS: return "enumval_exists";
4426 case BPF_ENUMVAL_VALUE: return "enumval_value";
4427 default: return "unknown";
4428 }
4429}
4430
4431static bool core_relo_is_field_based(enum bpf_core_relo_kind kind)
4432{
4433 switch (kind) {
4434 case BPF_FIELD_BYTE_OFFSET:
4435 case BPF_FIELD_BYTE_SIZE:
4436 case BPF_FIELD_EXISTS:
4437 case BPF_FIELD_SIGNED:
4438 case BPF_FIELD_LSHIFT_U64:
4439 case BPF_FIELD_RSHIFT_U64:
4440 return true;
4441 default:
4442 return false;
4443 }
4444}
4445
4446static bool core_relo_is_type_based(enum bpf_core_relo_kind kind)
4447{
4448 switch (kind) {
4449 case BPF_TYPE_ID_LOCAL:
4450 case BPF_TYPE_ID_TARGET:
4451 case BPF_TYPE_EXISTS:
4452 case BPF_TYPE_SIZE:
4453 return true;
4454 default:
4455 return false;
4456 }
4457}
4458
4459static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind)
4460{
4461 switch (kind) {
4462 case BPF_ENUMVAL_EXISTS:
4463 case BPF_ENUMVAL_VALUE:
4464 return true;
4465 default:
4466 return false;
4467 }
4468}
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507static int bpf_core_parse_spec(const struct btf *btf,
4508 __u32 type_id,
4509 const char *spec_str,
4510 enum bpf_core_relo_kind relo_kind,
4511 struct bpf_core_spec *spec)
4512{
4513 int access_idx, parsed_len, i;
4514 struct bpf_core_accessor *acc;
4515 const struct btf_type *t;
4516 const char *name;
4517 __u32 id;
4518 __s64 sz;
4519
4520 if (str_is_empty(spec_str) || *spec_str == ':')
4521 return -EINVAL;
4522
4523 memset(spec, 0, sizeof(*spec));
4524 spec->btf = btf;
4525 spec->root_type_id = type_id;
4526 spec->relo_kind = relo_kind;
4527
4528
4529 if (core_relo_is_type_based(relo_kind)) {
4530 if (strcmp(spec_str, "0"))
4531 return -EINVAL;
4532 return 0;
4533 }
4534
4535
4536 while (*spec_str) {
4537 if (*spec_str == ':')
4538 ++spec_str;
4539 if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)
4540 return -EINVAL;
4541 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4542 return -E2BIG;
4543 spec_str += parsed_len;
4544 spec->raw_spec[spec->raw_len++] = access_idx;
4545 }
4546
4547 if (spec->raw_len == 0)
4548 return -EINVAL;
4549
4550 t = skip_mods_and_typedefs(btf, type_id, &id);
4551 if (!t)
4552 return -EINVAL;
4553
4554 access_idx = spec->raw_spec[0];
4555 acc = &spec->spec[0];
4556 acc->type_id = id;
4557 acc->idx = access_idx;
4558 spec->len++;
4559
4560 if (core_relo_is_enumval_based(relo_kind)) {
4561 if (!btf_is_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
4562 return -EINVAL;
4563
4564
4565 acc->name = btf__name_by_offset(btf, btf_enum(t)[access_idx].name_off);
4566 return 0;
4567 }
4568
4569 if (!core_relo_is_field_based(relo_kind))
4570 return -EINVAL;
4571
4572 sz = btf__resolve_size(btf, id);
4573 if (sz < 0)
4574 return sz;
4575 spec->bit_offset = access_idx * sz * 8;
4576
4577 for (i = 1; i < spec->raw_len; i++) {
4578 t = skip_mods_and_typedefs(btf, id, &id);
4579 if (!t)
4580 return -EINVAL;
4581
4582 access_idx = spec->raw_spec[i];
4583 acc = &spec->spec[spec->len];
4584
4585 if (btf_is_composite(t)) {
4586 const struct btf_member *m;
4587 __u32 bit_offset;
4588
4589 if (access_idx >= btf_vlen(t))
4590 return -EINVAL;
4591
4592 bit_offset = btf_member_bit_offset(t, access_idx);
4593 spec->bit_offset += bit_offset;
4594
4595 m = btf_members(t) + access_idx;
4596 if (m->name_off) {
4597 name = btf__name_by_offset(btf, m->name_off);
4598 if (str_is_empty(name))
4599 return -EINVAL;
4600
4601 acc->type_id = id;
4602 acc->idx = access_idx;
4603 acc->name = name;
4604 spec->len++;
4605 }
4606
4607 id = m->type;
4608 } else if (btf_is_array(t)) {
4609 const struct btf_array *a = btf_array(t);
4610 bool flex;
4611
4612 t = skip_mods_and_typedefs(btf, a->type, &id);
4613 if (!t)
4614 return -EINVAL;
4615
4616 flex = is_flex_arr(btf, acc - 1, a);
4617 if (!flex && access_idx >= a->nelems)
4618 return -EINVAL;
4619
4620 spec->spec[spec->len].type_id = id;
4621 spec->spec[spec->len].idx = access_idx;
4622 spec->len++;
4623
4624 sz = btf__resolve_size(btf, id);
4625 if (sz < 0)
4626 return sz;
4627 spec->bit_offset += access_idx * sz * 8;
4628 } else {
4629 pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n",
4630 type_id, spec_str, i, id, btf_kind_str(t));
4631 return -EINVAL;
4632 }
4633 }
4634
4635 return 0;
4636}
4637
4638static bool bpf_core_is_flavor_sep(const char *s)
4639{
4640
4641 return s[0] != '_' &&
4642 s[1] == '_' && s[2] == '_' && s[3] == '_' &&
4643 s[4] != '_';
4644}
4645
4646
4647
4648
4649
4650static size_t bpf_core_essential_name_len(const char *name)
4651{
4652 size_t n = strlen(name);
4653 int i;
4654
4655 for (i = n - 5; i >= 0; i--) {
4656 if (bpf_core_is_flavor_sep(name + i))
4657 return i + 1;
4658 }
4659 return n;
4660}
4661
4662struct core_cand
4663{
4664 const struct btf *btf;
4665 const struct btf_type *t;
4666 const char *name;
4667 __u32 id;
4668};
4669
4670
4671struct core_cand_list {
4672 struct core_cand *cands;
4673 int len;
4674};
4675
4676static void bpf_core_free_cands(struct core_cand_list *cands)
4677{
4678 free(cands->cands);
4679 free(cands);
4680}
4681
4682static int bpf_core_add_cands(struct core_cand *local_cand,
4683 size_t local_essent_len,
4684 const struct btf *targ_btf,
4685 const char *targ_btf_name,
4686 int targ_start_id,
4687 struct core_cand_list *cands)
4688{
4689 struct core_cand *new_cands, *cand;
4690 const struct btf_type *t;
4691 const char *targ_name;
4692 size_t targ_essent_len;
4693 int n, i;
4694
4695 n = btf__get_nr_types(targ_btf);
4696 for (i = targ_start_id; i <= n; i++) {
4697 t = btf__type_by_id(targ_btf, i);
4698 if (btf_kind(t) != btf_kind(local_cand->t))
4699 continue;
4700
4701 targ_name = btf__name_by_offset(targ_btf, t->name_off);
4702 if (str_is_empty(targ_name))
4703 continue;
4704
4705 targ_essent_len = bpf_core_essential_name_len(targ_name);
4706 if (targ_essent_len != local_essent_len)
4707 continue;
4708
4709 if (strncmp(local_cand->name, targ_name, local_essent_len) != 0)
4710 continue;
4711
4712 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
4713 local_cand->id, btf_kind_str(local_cand->t),
4714 local_cand->name, i, btf_kind_str(t), targ_name,
4715 targ_btf_name);
4716 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
4717 sizeof(*cands->cands));
4718 if (!new_cands)
4719 return -ENOMEM;
4720
4721 cand = &new_cands[cands->len];
4722 cand->btf = targ_btf;
4723 cand->t = t;
4724 cand->name = targ_name;
4725 cand->id = i;
4726
4727 cands->cands = new_cands;
4728 cands->len++;
4729 }
4730 return 0;
4731}
4732
4733static int load_module_btfs(struct bpf_object *obj)
4734{
4735 struct bpf_btf_info info;
4736 struct module_btf *mod_btf;
4737 struct btf *btf;
4738 char name[64];
4739 __u32 id = 0, len;
4740 int err, fd;
4741
4742 if (obj->btf_modules_loaded)
4743 return 0;
4744
4745
4746 obj->btf_modules_loaded = true;
4747
4748
4749 if (!kernel_supports(FEAT_MODULE_BTF))
4750 return 0;
4751
4752 while (true) {
4753 err = bpf_btf_get_next_id(id, &id);
4754 if (err && errno == ENOENT)
4755 return 0;
4756 if (err) {
4757 err = -errno;
4758 pr_warn("failed to iterate BTF objects: %d\n", err);
4759 return err;
4760 }
4761
4762 fd = bpf_btf_get_fd_by_id(id);
4763 if (fd < 0) {
4764 if (errno == ENOENT)
4765 continue;
4766 err = -errno;
4767 pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
4768 return err;
4769 }
4770
4771 len = sizeof(info);
4772 memset(&info, 0, sizeof(info));
4773 info.name = ptr_to_u64(name);
4774 info.name_len = sizeof(name);
4775
4776 err = bpf_obj_get_info_by_fd(fd, &info, &len);
4777 if (err) {
4778 err = -errno;
4779 pr_warn("failed to get BTF object #%d info: %d\n", id, err);
4780 goto err_out;
4781 }
4782
4783
4784 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
4785 close(fd);
4786 continue;
4787 }
4788
4789 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
4790 if (IS_ERR(btf)) {
4791 pr_warn("failed to load module [%s]'s BTF object #%d: %ld\n",
4792 name, id, PTR_ERR(btf));
4793 err = PTR_ERR(btf);
4794 goto err_out;
4795 }
4796
4797 err = btf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
4798 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
4799 if (err)
4800 goto err_out;
4801
4802 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
4803
4804 mod_btf->btf = btf;
4805 mod_btf->id = id;
4806 mod_btf->fd = fd;
4807 mod_btf->name = strdup(name);
4808 if (!mod_btf->name) {
4809 err = -ENOMEM;
4810 goto err_out;
4811 }
4812 continue;
4813
4814err_out:
4815 close(fd);
4816 return err;
4817 }
4818
4819 return 0;
4820}
4821
4822static struct core_cand_list *
4823bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
4824{
4825 struct core_cand local_cand = {};
4826 struct core_cand_list *cands;
4827 const struct btf *main_btf;
4828 size_t local_essent_len;
4829 int err, i;
4830
4831 local_cand.btf = local_btf;
4832 local_cand.t = btf__type_by_id(local_btf, local_type_id);
4833 if (!local_cand.t)
4834 return ERR_PTR(-EINVAL);
4835
4836 local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off);
4837 if (str_is_empty(local_cand.name))
4838 return ERR_PTR(-EINVAL);
4839 local_essent_len = bpf_core_essential_name_len(local_cand.name);
4840
4841 cands = calloc(1, sizeof(*cands));
4842 if (!cands)
4843 return ERR_PTR(-ENOMEM);
4844
4845
4846 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
4847 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
4848 if (err)
4849 goto err_out;
4850
4851
4852 if (cands->len)
4853 return cands;
4854
4855
4856 if (obj->btf_vmlinux_override)
4857 return cands;
4858
4859
4860 err = load_module_btfs(obj);
4861 if (err)
4862 goto err_out;
4863
4864 for (i = 0; i < obj->btf_module_cnt; i++) {
4865 err = bpf_core_add_cands(&local_cand, local_essent_len,
4866 obj->btf_modules[i].btf,
4867 obj->btf_modules[i].name,
4868 btf__get_nr_types(obj->btf_vmlinux) + 1,
4869 cands);
4870 if (err)
4871 goto err_out;
4872 }
4873
4874 return cands;
4875err_out:
4876 bpf_core_free_cands(cands);
4877 return ERR_PTR(err);
4878}
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896static int bpf_core_fields_are_compat(const struct btf *local_btf,
4897 __u32 local_id,
4898 const struct btf *targ_btf,
4899 __u32 targ_id)
4900{
4901 const struct btf_type *local_type, *targ_type;
4902
4903recur:
4904 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
4905 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
4906 if (!local_type || !targ_type)
4907 return -EINVAL;
4908
4909 if (btf_is_composite(local_type) && btf_is_composite(targ_type))
4910 return 1;
4911 if (btf_kind(local_type) != btf_kind(targ_type))
4912 return 0;
4913
4914 switch (btf_kind(local_type)) {
4915 case BTF_KIND_PTR:
4916 return 1;
4917 case BTF_KIND_FWD:
4918 case BTF_KIND_ENUM: {
4919 const char *local_name, *targ_name;
4920 size_t local_len, targ_len;
4921
4922 local_name = btf__name_by_offset(local_btf,
4923 local_type->name_off);
4924 targ_name = btf__name_by_offset(targ_btf, targ_type->name_off);
4925 local_len = bpf_core_essential_name_len(local_name);
4926 targ_len = bpf_core_essential_name_len(targ_name);
4927
4928 return local_len == 0 || targ_len == 0 ||
4929 (local_len == targ_len &&
4930 strncmp(local_name, targ_name, local_len) == 0);
4931 }
4932 case BTF_KIND_INT:
4933
4934
4935
4936 return btf_int_offset(local_type) == 0 &&
4937 btf_int_offset(targ_type) == 0;
4938 case BTF_KIND_ARRAY:
4939 local_id = btf_array(local_type)->type;
4940 targ_id = btf_array(targ_type)->type;
4941 goto recur;
4942 default:
4943 pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n",
4944 btf_kind(local_type), local_id, targ_id);
4945 return 0;
4946 }
4947}
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965static int bpf_core_match_member(const struct btf *local_btf,
4966 const struct bpf_core_accessor *local_acc,
4967 const struct btf *targ_btf,
4968 __u32 targ_id,
4969 struct bpf_core_spec *spec,
4970 __u32 *next_targ_id)
4971{
4972 const struct btf_type *local_type, *targ_type;
4973 const struct btf_member *local_member, *m;
4974 const char *local_name, *targ_name;
4975 __u32 local_id;
4976 int i, n, found;
4977
4978 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
4979 if (!targ_type)
4980 return -EINVAL;
4981 if (!btf_is_composite(targ_type))
4982 return 0;
4983
4984 local_id = local_acc->type_id;
4985 local_type = btf__type_by_id(local_btf, local_id);
4986 local_member = btf_members(local_type) + local_acc->idx;
4987 local_name = btf__name_by_offset(local_btf, local_member->name_off);
4988
4989 n = btf_vlen(targ_type);
4990 m = btf_members(targ_type);
4991 for (i = 0; i < n; i++, m++) {
4992 __u32 bit_offset;
4993
4994 bit_offset = btf_member_bit_offset(targ_type, i);
4995
4996
4997 if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
4998 return -E2BIG;
4999
5000
5001 spec->bit_offset += bit_offset;
5002 spec->raw_spec[spec->raw_len++] = i;
5003
5004 targ_name = btf__name_by_offset(targ_btf, m->name_off);
5005 if (str_is_empty(targ_name)) {
5006
5007 found = bpf_core_match_member(local_btf, local_acc,
5008 targ_btf, m->type,
5009 spec, next_targ_id);
5010 if (found)
5011 return found;
5012 } else if (strcmp(local_name, targ_name) == 0) {
5013
5014 struct bpf_core_accessor *targ_acc;
5015
5016 targ_acc = &spec->spec[spec->len++];
5017 targ_acc->type_id = targ_id;
5018 targ_acc->idx = i;
5019 targ_acc->name = targ_name;
5020
5021 *next_targ_id = m->type;
5022 found = bpf_core_fields_are_compat(local_btf,
5023 local_member->type,
5024 targ_btf, m->type);
5025 if (!found)
5026 spec->len--;
5027 return found;
5028 }
5029
5030 spec->bit_offset -= bit_offset;
5031 spec->raw_len--;
5032 }
5033
5034 return 0;
5035}
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056static int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
5057 const struct btf *targ_btf, __u32 targ_id)
5058{
5059 const struct btf_type *local_type, *targ_type;
5060 int depth = 32;
5061
5062
5063 local_type = btf__type_by_id(local_btf, local_id);
5064 targ_type = btf__type_by_id(targ_btf, targ_id);
5065 if (btf_kind(local_type) != btf_kind(targ_type))
5066 return 0;
5067
5068recur:
5069 depth--;
5070 if (depth < 0)
5071 return -EINVAL;
5072
5073 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
5074 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
5075 if (!local_type || !targ_type)
5076 return -EINVAL;
5077
5078 if (btf_kind(local_type) != btf_kind(targ_type))
5079 return 0;
5080
5081 switch (btf_kind(local_type)) {
5082 case BTF_KIND_UNKN:
5083 case BTF_KIND_STRUCT:
5084 case BTF_KIND_UNION:
5085 case BTF_KIND_ENUM:
5086 case BTF_KIND_FWD:
5087 return 1;
5088 case BTF_KIND_INT:
5089
5090
5091
5092 return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0;
5093 case BTF_KIND_PTR:
5094 local_id = local_type->type;
5095 targ_id = targ_type->type;
5096 goto recur;
5097 case BTF_KIND_ARRAY:
5098 local_id = btf_array(local_type)->type;
5099 targ_id = btf_array(targ_type)->type;
5100 goto recur;
5101 case BTF_KIND_FUNC_PROTO: {
5102 struct btf_param *local_p = btf_params(local_type);
5103 struct btf_param *targ_p = btf_params(targ_type);
5104 __u16 local_vlen = btf_vlen(local_type);
5105 __u16 targ_vlen = btf_vlen(targ_type);
5106 int i, err;
5107
5108 if (local_vlen != targ_vlen)
5109 return 0;
5110
5111 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
5112 skip_mods_and_typedefs(local_btf, local_p->type, &local_id);
5113 skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id);
5114 err = bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id);
5115 if (err <= 0)
5116 return err;
5117 }
5118
5119
5120 skip_mods_and_typedefs(local_btf, local_type->type, &local_id);
5121 skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id);
5122 goto recur;
5123 }
5124 default:
5125 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
5126 btf_kind_str(local_type), local_id, targ_id);
5127 return 0;
5128 }
5129}
5130
5131
5132
5133
5134
5135static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
5136 const struct btf *targ_btf, __u32 targ_id,
5137 struct bpf_core_spec *targ_spec)
5138{
5139 const struct btf_type *targ_type;
5140 const struct bpf_core_accessor *local_acc;
5141 struct bpf_core_accessor *targ_acc;
5142 int i, sz, matched;
5143
5144 memset(targ_spec, 0, sizeof(*targ_spec));
5145 targ_spec->btf = targ_btf;
5146 targ_spec->root_type_id = targ_id;
5147 targ_spec->relo_kind = local_spec->relo_kind;
5148
5149 if (core_relo_is_type_based(local_spec->relo_kind)) {
5150 return bpf_core_types_are_compat(local_spec->btf,
5151 local_spec->root_type_id,
5152 targ_btf, targ_id);
5153 }
5154
5155 local_acc = &local_spec->spec[0];
5156 targ_acc = &targ_spec->spec[0];
5157
5158 if (core_relo_is_enumval_based(local_spec->relo_kind)) {
5159 size_t local_essent_len, targ_essent_len;
5160 const struct btf_enum *e;
5161 const char *targ_name;
5162
5163
5164 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id);
5165 if (!btf_is_enum(targ_type))
5166 return 0;
5167
5168 local_essent_len = bpf_core_essential_name_len(local_acc->name);
5169
5170 for (i = 0, e = btf_enum(targ_type); i < btf_vlen(targ_type); i++, e++) {
5171 targ_name = btf__name_by_offset(targ_spec->btf, e->name_off);
5172 targ_essent_len = bpf_core_essential_name_len(targ_name);
5173 if (targ_essent_len != local_essent_len)
5174 continue;
5175 if (strncmp(local_acc->name, targ_name, local_essent_len) == 0) {
5176 targ_acc->type_id = targ_id;
5177 targ_acc->idx = i;
5178 targ_acc->name = targ_name;
5179 targ_spec->len++;
5180 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
5181 targ_spec->raw_len++;
5182 return 1;
5183 }
5184 }
5185 return 0;
5186 }
5187
5188 if (!core_relo_is_field_based(local_spec->relo_kind))
5189 return -EINVAL;
5190
5191 for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) {
5192 targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id,
5193 &targ_id);
5194 if (!targ_type)
5195 return -EINVAL;
5196
5197 if (local_acc->name) {
5198 matched = bpf_core_match_member(local_spec->btf,
5199 local_acc,
5200 targ_btf, targ_id,
5201 targ_spec, &targ_id);
5202 if (matched <= 0)
5203 return matched;
5204 } else {
5205
5206
5207
5208
5209 if (i > 0) {
5210 const struct btf_array *a;
5211 bool flex;
5212
5213 if (!btf_is_array(targ_type))
5214 return 0;
5215
5216 a = btf_array(targ_type);
5217 flex = is_flex_arr(targ_btf, targ_acc - 1, a);
5218 if (!flex && local_acc->idx >= a->nelems)
5219 return 0;
5220 if (!skip_mods_and_typedefs(targ_btf, a->type,
5221 &targ_id))
5222 return -EINVAL;
5223 }
5224
5225
5226 if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
5227 return -E2BIG;
5228
5229 targ_acc->type_id = targ_id;
5230 targ_acc->idx = local_acc->idx;
5231 targ_acc->name = NULL;
5232 targ_spec->len++;
5233 targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx;
5234 targ_spec->raw_len++;
5235
5236 sz = btf__resolve_size(targ_btf, targ_id);
5237 if (sz < 0)
5238 return sz;
5239 targ_spec->bit_offset += local_acc->idx * sz * 8;
5240 }
5241 }
5242
5243 return 1;
5244}
5245
5246static int bpf_core_calc_field_relo(const struct bpf_program *prog,
5247 const struct bpf_core_relo *relo,
5248 const struct bpf_core_spec *spec,
5249 __u32 *val, __u32 *field_sz, __u32 *type_id,
5250 bool *validate)
5251{
5252 const struct bpf_core_accessor *acc;
5253 const struct btf_type *t;
5254 __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id;
5255 const struct btf_member *m;
5256 const struct btf_type *mt;
5257 bool bitfield;
5258 __s64 sz;
5259
5260 *field_sz = 0;
5261
5262 if (relo->kind == BPF_FIELD_EXISTS) {
5263 *val = spec ? 1 : 0;
5264 return 0;
5265 }
5266
5267 if (!spec)
5268 return -EUCLEAN;
5269
5270 acc = &spec->spec[spec->len - 1];
5271 t = btf__type_by_id(spec->btf, acc->type_id);
5272
5273
5274 if (!acc->name) {
5275 if (relo->kind == BPF_FIELD_BYTE_OFFSET) {
5276 *val = spec->bit_offset / 8;
5277
5278 sz = btf__resolve_size(spec->btf, acc->type_id);
5279 if (sz < 0)
5280 return -EINVAL;
5281 *field_sz = sz;
5282 *type_id = acc->type_id;
5283 } else if (relo->kind == BPF_FIELD_BYTE_SIZE) {
5284 sz = btf__resolve_size(spec->btf, acc->type_id);
5285 if (sz < 0)
5286 return -EINVAL;
5287 *val = sz;
5288 } else {
5289 pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n",
5290 prog->name, relo->kind, relo->insn_off / 8);
5291 return -EINVAL;
5292 }
5293 if (validate)
5294 *validate = true;
5295 return 0;
5296 }
5297
5298 m = btf_members(t) + acc->idx;
5299 mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id);
5300 bit_off = spec->bit_offset;
5301 bit_sz = btf_member_bitfield_size(t, acc->idx);
5302
5303 bitfield = bit_sz > 0;
5304 if (bitfield) {
5305 byte_sz = mt->size;
5306 byte_off = bit_off / 8 / byte_sz * byte_sz;
5307
5308 while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) {
5309 if (byte_sz >= 8) {
5310
5311 pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n",
5312 prog->name, relo->kind, relo->insn_off / 8);
5313 return -E2BIG;
5314 }
5315 byte_sz *= 2;
5316 byte_off = bit_off / 8 / byte_sz * byte_sz;
5317 }
5318 } else {
5319 sz = btf__resolve_size(spec->btf, field_type_id);
5320 if (sz < 0)
5321 return -EINVAL;
5322 byte_sz = sz;
5323 byte_off = spec->bit_offset / 8;
5324 bit_sz = byte_sz * 8;
5325 }
5326
5327
5328
5329
5330
5331 if (validate)
5332 *validate = !bitfield;
5333
5334 switch (relo->kind) {
5335 case BPF_FIELD_BYTE_OFFSET:
5336 *val = byte_off;
5337 if (!bitfield) {
5338 *field_sz = byte_sz;
5339 *type_id = field_type_id;
5340 }
5341 break;
5342 case BPF_FIELD_BYTE_SIZE:
5343 *val = byte_sz;
5344 break;
5345 case BPF_FIELD_SIGNED:
5346
5347 *val = btf_is_enum(mt) ||
5348 (btf_int_encoding(mt) & BTF_INT_SIGNED);
5349 if (validate)
5350 *validate = true;
5351 break;
5352 case BPF_FIELD_LSHIFT_U64:
5353#if __BYTE_ORDER == __LITTLE_ENDIAN
5354 *val = 64 - (bit_off + bit_sz - byte_off * 8);
5355#else
5356 *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8);
5357#endif
5358 break;
5359 case BPF_FIELD_RSHIFT_U64:
5360 *val = 64 - bit_sz;
5361 if (validate)
5362 *validate = true;
5363 break;
5364 case BPF_FIELD_EXISTS:
5365 default:
5366 return -EOPNOTSUPP;
5367 }
5368
5369 return 0;
5370}
5371
5372static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
5373 const struct bpf_core_spec *spec,
5374 __u32 *val)
5375{
5376 __s64 sz;
5377
5378
5379 if (!spec) {
5380 *val = 0;
5381 return 0;
5382 }
5383
5384 switch (relo->kind) {
5385 case BPF_TYPE_ID_TARGET:
5386 *val = spec->root_type_id;
5387 break;
5388 case BPF_TYPE_EXISTS:
5389 *val = 1;
5390 break;
5391 case BPF_TYPE_SIZE:
5392 sz = btf__resolve_size(spec->btf, spec->root_type_id);
5393 if (sz < 0)
5394 return -EINVAL;
5395 *val = sz;
5396 break;
5397 case BPF_TYPE_ID_LOCAL:
5398
5399 default:
5400 return -EOPNOTSUPP;
5401 }
5402
5403 return 0;
5404}
5405
5406static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,
5407 const struct bpf_core_spec *spec,
5408 __u32 *val)
5409{
5410 const struct btf_type *t;
5411 const struct btf_enum *e;
5412
5413 switch (relo->kind) {
5414 case BPF_ENUMVAL_EXISTS:
5415 *val = spec ? 1 : 0;
5416 break;
5417 case BPF_ENUMVAL_VALUE:
5418 if (!spec)
5419 return -EUCLEAN;
5420 t = btf__type_by_id(spec->btf, spec->spec[0].type_id);
5421 e = btf_enum(t) + spec->spec[0].idx;
5422 *val = e->val;
5423 break;
5424 default:
5425 return -EOPNOTSUPP;
5426 }
5427
5428 return 0;
5429}
5430
5431struct bpf_core_relo_res
5432{
5433
5434 __u32 orig_val;
5435
5436 __u32 new_val;
5437
5438 bool poison;
5439
5440 bool validate;
5441
5442
5443
5444
5445
5446
5447
5448
5449 bool fail_memsz_adjust;
5450 __u32 orig_sz;
5451 __u32 orig_type_id;
5452 __u32 new_sz;
5453 __u32 new_type_id;
5454};
5455
5456
5457
5458
5459
5460
5461
5462static int bpf_core_calc_relo(const struct bpf_program *prog,
5463 const struct bpf_core_relo *relo,
5464 int relo_idx,
5465 const struct bpf_core_spec *local_spec,
5466 const struct bpf_core_spec *targ_spec,
5467 struct bpf_core_relo_res *res)
5468{
5469 int err = -EOPNOTSUPP;
5470
5471 res->orig_val = 0;
5472 res->new_val = 0;
5473 res->poison = false;
5474 res->validate = true;
5475 res->fail_memsz_adjust = false;
5476 res->orig_sz = res->new_sz = 0;
5477 res->orig_type_id = res->new_type_id = 0;
5478
5479 if (core_relo_is_field_based(relo->kind)) {
5480 err = bpf_core_calc_field_relo(prog, relo, local_spec,
5481 &res->orig_val, &res->orig_sz,
5482 &res->orig_type_id, &res->validate);
5483 err = err ?: bpf_core_calc_field_relo(prog, relo, targ_spec,
5484 &res->new_val, &res->new_sz,
5485 &res->new_type_id, NULL);
5486 if (err)
5487 goto done;
5488
5489
5490
5491
5492 res->fail_memsz_adjust = false;
5493 if (res->orig_sz != res->new_sz) {
5494 const struct btf_type *orig_t, *new_t;
5495
5496 orig_t = btf__type_by_id(local_spec->btf, res->orig_type_id);
5497 new_t = btf__type_by_id(targ_spec->btf, res->new_type_id);
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513 if (btf_is_ptr(orig_t) && btf_is_ptr(new_t))
5514 goto done;
5515 if (btf_is_int(orig_t) && btf_is_int(new_t) &&
5516 btf_int_encoding(orig_t) != BTF_INT_SIGNED &&
5517 btf_int_encoding(new_t) != BTF_INT_SIGNED)
5518 goto done;
5519
5520
5521
5522
5523 res->fail_memsz_adjust = true;
5524 }
5525 } else if (core_relo_is_type_based(relo->kind)) {
5526 err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val);
5527 err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val);
5528 } else if (core_relo_is_enumval_based(relo->kind)) {
5529 err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val);
5530 err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);
5531 }
5532
5533done:
5534 if (err == -EUCLEAN) {
5535
5536 res->poison = true;
5537 err = 0;
5538 } else if (err == -EOPNOTSUPP) {
5539
5540 pr_warn("prog '%s': relo #%d: unrecognized CO-RE relocation %s (%d) at insn #%d\n",
5541 prog->name, relo_idx, core_relo_kind_str(relo->kind),
5542 relo->kind, relo->insn_off / 8);
5543 }
5544
5545 return err;
5546}
5547
5548
5549
5550
5551
5552static void bpf_core_poison_insn(struct bpf_program *prog, int relo_idx,
5553 int insn_idx, struct bpf_insn *insn)
5554{
5555 pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n",
5556 prog->name, relo_idx, insn_idx);
5557 insn->code = BPF_JMP | BPF_CALL;
5558 insn->dst_reg = 0;
5559 insn->src_reg = 0;
5560 insn->off = 0;
5561
5562
5563
5564
5565 insn->imm = 195896080;
5566}
5567
5568static bool is_ldimm64(struct bpf_insn *insn)
5569{
5570 return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
5571}
5572
5573static int insn_bpf_size_to_bytes(struct bpf_insn *insn)
5574{
5575 switch (BPF_SIZE(insn->code)) {
5576 case BPF_DW: return 8;
5577 case BPF_W: return 4;
5578 case BPF_H: return 2;
5579 case BPF_B: return 1;
5580 default: return -1;
5581 }
5582}
5583
5584static int insn_bytes_to_bpf_size(__u32 sz)
5585{
5586 switch (sz) {
5587 case 8: return BPF_DW;
5588 case 4: return BPF_W;
5589 case 2: return BPF_H;
5590 case 1: return BPF_B;
5591 default: return -1;
5592 }
5593}
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612static int bpf_core_patch_insn(struct bpf_program *prog,
5613 const struct bpf_core_relo *relo,
5614 int relo_idx,
5615 const struct bpf_core_relo_res *res)
5616{
5617 __u32 orig_val, new_val;
5618 struct bpf_insn *insn;
5619 int insn_idx;
5620 __u8 class;
5621
5622 if (relo->insn_off % BPF_INSN_SZ)
5623 return -EINVAL;
5624 insn_idx = relo->insn_off / BPF_INSN_SZ;
5625
5626
5627
5628
5629 insn_idx = insn_idx - prog->sec_insn_off;
5630 insn = &prog->insns[insn_idx];
5631 class = BPF_CLASS(insn->code);
5632
5633 if (res->poison) {
5634poison:
5635
5636
5637
5638 if (is_ldimm64(insn))
5639 bpf_core_poison_insn(prog, relo_idx, insn_idx + 1, insn + 1);
5640 bpf_core_poison_insn(prog, relo_idx, insn_idx, insn);
5641 return 0;
5642 }
5643
5644 orig_val = res->orig_val;
5645 new_val = res->new_val;
5646
5647 switch (class) {
5648 case BPF_ALU:
5649 case BPF_ALU64:
5650 if (BPF_SRC(insn->code) != BPF_K)
5651 return -EINVAL;
5652 if (res->validate && insn->imm != orig_val) {
5653 pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n",
5654 prog->name, relo_idx,
5655 insn_idx, insn->imm, orig_val, new_val);
5656 return -EINVAL;
5657 }
5658 orig_val = insn->imm;
5659 insn->imm = new_val;
5660 pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n",
5661 prog->name, relo_idx, insn_idx,
5662 orig_val, new_val);
5663 break;
5664 case BPF_LDX:
5665 case BPF_ST:
5666 case BPF_STX:
5667 if (res->validate && insn->off != orig_val) {
5668 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %u -> %u\n",
5669 prog->name, relo_idx, insn_idx, insn->off, orig_val, new_val);
5670 return -EINVAL;
5671 }
5672 if (new_val > SHRT_MAX) {
5673 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n",
5674 prog->name, relo_idx, insn_idx, new_val);
5675 return -ERANGE;
5676 }
5677 if (res->fail_memsz_adjust) {
5678 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. "
5679 "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n",
5680 prog->name, relo_idx, insn_idx);
5681 goto poison;
5682 }
5683
5684 orig_val = insn->off;
5685 insn->off = new_val;
5686 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",
5687 prog->name, relo_idx, insn_idx, orig_val, new_val);
5688
5689 if (res->new_sz != res->orig_sz) {
5690 int insn_bytes_sz, insn_bpf_sz;
5691
5692 insn_bytes_sz = insn_bpf_size_to_bytes(insn);
5693 if (insn_bytes_sz != res->orig_sz) {
5694 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n",
5695 prog->name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz);
5696 return -EINVAL;
5697 }
5698
5699 insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz);
5700 if (insn_bpf_sz < 0) {
5701 pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n",
5702 prog->name, relo_idx, insn_idx, res->new_sz);
5703 return -EINVAL;
5704 }
5705
5706 insn->code = BPF_MODE(insn->code) | insn_bpf_sz | BPF_CLASS(insn->code);
5707 pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n",
5708 prog->name, relo_idx, insn_idx, res->orig_sz, res->new_sz);
5709 }
5710 break;
5711 case BPF_LD: {
5712 __u64 imm;
5713
5714 if (!is_ldimm64(insn) ||
5715 insn[0].src_reg != 0 || insn[0].off != 0 ||
5716 insn_idx + 1 >= prog->insns_cnt ||
5717 insn[1].code != 0 || insn[1].dst_reg != 0 ||
5718 insn[1].src_reg != 0 || insn[1].off != 0) {
5719 pr_warn("prog '%s': relo #%d: insn #%d (LDIMM64) has unexpected form\n",
5720 prog->name, relo_idx, insn_idx);
5721 return -EINVAL;
5722 }
5723
5724 imm = insn[0].imm + ((__u64)insn[1].imm << 32);
5725 if (res->validate && imm != orig_val) {
5726 pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %u -> %u\n",
5727 prog->name, relo_idx,
5728 insn_idx, (unsigned long long)imm,
5729 orig_val, new_val);
5730 return -EINVAL;
5731 }
5732
5733 insn[0].imm = new_val;
5734 insn[1].imm = 0;
5735 pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %u\n",
5736 prog->name, relo_idx, insn_idx,
5737 (unsigned long long)imm, new_val);
5738 break;
5739 }
5740 default:
5741 pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:0x%x, src:0x%x, dst:0x%x, off:0x%x, imm:0x%x\n",
5742 prog->name, relo_idx, insn_idx, insn->code,
5743 insn->src_reg, insn->dst_reg, insn->off, insn->imm);
5744 return -EINVAL;
5745 }
5746
5747 return 0;
5748}
5749
5750
5751
5752
5753
5754static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec)
5755{
5756 const struct btf_type *t;
5757 const struct btf_enum *e;
5758 const char *s;
5759 __u32 type_id;
5760 int i;
5761
5762 type_id = spec->root_type_id;
5763 t = btf__type_by_id(spec->btf, type_id);
5764 s = btf__name_by_offset(spec->btf, t->name_off);
5765
5766 libbpf_print(level, "[%u] %s %s", type_id, btf_kind_str(t), str_is_empty(s) ? "<anon>" : s);
5767
5768 if (core_relo_is_type_based(spec->relo_kind))
5769 return;
5770
5771 if (core_relo_is_enumval_based(spec->relo_kind)) {
5772 t = skip_mods_and_typedefs(spec->btf, type_id, NULL);
5773 e = btf_enum(t) + spec->raw_spec[0];
5774 s = btf__name_by_offset(spec->btf, e->name_off);
5775
5776 libbpf_print(level, "::%s = %u", s, e->val);
5777 return;
5778 }
5779
5780 if (core_relo_is_field_based(spec->relo_kind)) {
5781 for (i = 0; i < spec->len; i++) {
5782 if (spec->spec[i].name)
5783 libbpf_print(level, ".%s", spec->spec[i].name);
5784 else if (i > 0 || spec->spec[i].idx > 0)
5785 libbpf_print(level, "[%u]", spec->spec[i].idx);
5786 }
5787
5788 libbpf_print(level, " (");
5789 for (i = 0; i < spec->raw_len; i++)
5790 libbpf_print(level, "%s%d", i == 0 ? "" : ":", spec->raw_spec[i]);
5791
5792 if (spec->bit_offset % 8)
5793 libbpf_print(level, " @ offset %u.%u)",
5794 spec->bit_offset / 8, spec->bit_offset % 8);
5795 else
5796 libbpf_print(level, " @ offset %u)", spec->bit_offset / 8);
5797 return;
5798 }
5799}
5800
5801static size_t bpf_core_hash_fn(const void *key, void *ctx)
5802{
5803 return (size_t)key;
5804}
5805
5806static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
5807{
5808 return k1 == k2;
5809}
5810
5811static void *u32_as_hash_key(__u32 x)
5812{
5813 return (void *)(uintptr_t)x;
5814}
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866static int bpf_core_apply_relo(struct bpf_program *prog,
5867 const struct bpf_core_relo *relo,
5868 int relo_idx,
5869 const struct btf *local_btf,
5870 struct hashmap *cand_cache)
5871{
5872 struct bpf_core_spec local_spec, cand_spec, targ_spec = {};
5873 const void *type_key = u32_as_hash_key(relo->type_id);
5874 struct bpf_core_relo_res cand_res, targ_res;
5875 const struct btf_type *local_type;
5876 const char *local_name;
5877 struct core_cand_list *cands = NULL;
5878 __u32 local_id;
5879 const char *spec_str;
5880 int i, j, err;
5881
5882 local_id = relo->type_id;
5883 local_type = btf__type_by_id(local_btf, local_id);
5884 if (!local_type)
5885 return -EINVAL;
5886
5887 local_name = btf__name_by_offset(local_btf, local_type->name_off);
5888 if (!local_name)
5889 return -EINVAL;
5890
5891 spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
5892 if (str_is_empty(spec_str))
5893 return -EINVAL;
5894
5895 err = bpf_core_parse_spec(local_btf, local_id, spec_str, relo->kind, &local_spec);
5896 if (err) {
5897 pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",
5898 prog->name, relo_idx, local_id, btf_kind_str(local_type),
5899 str_is_empty(local_name) ? "<anon>" : local_name,
5900 spec_str, err);
5901 return -EINVAL;
5902 }
5903
5904 pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog->name,
5905 relo_idx, core_relo_kind_str(relo->kind), relo->kind);
5906 bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec);
5907 libbpf_print(LIBBPF_DEBUG, "\n");
5908
5909
5910 if (relo->kind == BPF_TYPE_ID_LOCAL) {
5911 targ_res.validate = true;
5912 targ_res.poison = false;
5913 targ_res.orig_val = local_spec.root_type_id;
5914 targ_res.new_val = local_spec.root_type_id;
5915 goto patch_insn;
5916 }
5917
5918
5919 if (str_is_empty(spec_str)) {
5920 pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n",
5921 prog->name, relo_idx, core_relo_kind_str(relo->kind), relo->kind);
5922 return -EOPNOTSUPP;
5923 }
5924
5925 if (!hashmap__find(cand_cache, type_key, (void **)&cands)) {
5926 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
5927 if (IS_ERR(cands)) {
5928 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
5929 prog->name, relo_idx, local_id, btf_kind_str(local_type),
5930 local_name, PTR_ERR(cands));
5931 return PTR_ERR(cands);
5932 }
5933 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL);
5934 if (err) {
5935 bpf_core_free_cands(cands);
5936 return err;
5937 }
5938 }
5939
5940 for (i = 0, j = 0; i < cands->len; i++) {
5941 err = bpf_core_spec_match(&local_spec, cands->cands[i].btf,
5942 cands->cands[i].id, &cand_spec);
5943 if (err < 0) {
5944 pr_warn("prog '%s': relo #%d: error matching candidate #%d ",
5945 prog->name, relo_idx, i);
5946 bpf_core_dump_spec(LIBBPF_WARN, &cand_spec);
5947 libbpf_print(LIBBPF_WARN, ": %d\n", err);
5948 return err;
5949 }
5950
5951 pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog->name,
5952 relo_idx, err == 0 ? "non-matching" : "matching", i);
5953 bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec);
5954 libbpf_print(LIBBPF_DEBUG, "\n");
5955
5956 if (err == 0)
5957 continue;
5958
5959 err = bpf_core_calc_relo(prog, relo, relo_idx, &local_spec, &cand_spec, &cand_res);
5960 if (err)
5961 return err;
5962
5963 if (j == 0) {
5964 targ_res = cand_res;
5965 targ_spec = cand_spec;
5966 } else if (cand_spec.bit_offset != targ_spec.bit_offset) {
5967
5968
5969
5970 pr_warn("prog '%s': relo #%d: field offset ambiguity: %u != %u\n",
5971 prog->name, relo_idx, cand_spec.bit_offset,
5972 targ_spec.bit_offset);
5973 return -EINVAL;
5974 } else if (cand_res.poison != targ_res.poison || cand_res.new_val != targ_res.new_val) {
5975
5976
5977
5978
5979 pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %u != %s %u\n",
5980 prog->name, relo_idx,
5981 cand_res.poison ? "failure" : "success", cand_res.new_val,
5982 targ_res.poison ? "failure" : "success", targ_res.new_val);
5983 return -EINVAL;
5984 }
5985
5986 cands->cands[j++] = cands->cands[i];
5987 }
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997 if (j > 0)
5998 cands->len = j;
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011 if (j == 0) {
6012 pr_debug("prog '%s': relo #%d: no matching targets found\n",
6013 prog->name, relo_idx);
6014
6015
6016 err = bpf_core_calc_relo(prog, relo, relo_idx, &local_spec, NULL, &targ_res);
6017 if (err)
6018 return err;
6019 }
6020
6021patch_insn:
6022
6023 err = bpf_core_patch_insn(prog, relo, relo_idx, &targ_res);
6024 if (err) {
6025 pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",
6026 prog->name, relo_idx, relo->insn_off, err);
6027 return -EINVAL;
6028 }
6029
6030 return 0;
6031}
6032
6033static int
6034bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
6035{
6036 const struct btf_ext_info_sec *sec;
6037 const struct bpf_core_relo *rec;
6038 const struct btf_ext_info *seg;
6039 struct hashmap_entry *entry;
6040 struct hashmap *cand_cache = NULL;
6041 struct bpf_program *prog;
6042 const char *sec_name;
6043 int i, err = 0, insn_idx, sec_idx;
6044
6045 if (obj->btf_ext->core_relo_info.len == 0)
6046 return 0;
6047
6048 if (targ_btf_path) {
6049 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
6050 if (IS_ERR_OR_NULL(obj->btf_vmlinux_override)) {
6051 err = PTR_ERR(obj->btf_vmlinux_override);
6052 pr_warn("failed to parse target BTF: %d\n", err);
6053 return err;
6054 }
6055 }
6056
6057 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
6058 if (IS_ERR(cand_cache)) {
6059 err = PTR_ERR(cand_cache);
6060 goto out;
6061 }
6062
6063 seg = &obj->btf_ext->core_relo_info;
6064 for_each_btf_ext_sec(seg, sec) {
6065 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6066 if (str_is_empty(sec_name)) {
6067 err = -EINVAL;
6068 goto out;
6069 }
6070
6071
6072
6073
6074
6075
6076 prog = NULL;
6077 for (i = 0; i < obj->nr_programs; i++) {
6078 prog = &obj->programs[i];
6079 if (strcmp(prog->sec_name, sec_name) == 0)
6080 break;
6081 }
6082 if (!prog) {
6083 pr_warn("sec '%s': failed to find a BPF program\n", sec_name);
6084 return -ENOENT;
6085 }
6086 sec_idx = prog->sec_idx;
6087
6088 pr_debug("sec '%s': found %d CO-RE relocations\n",
6089 sec_name, sec->num_info);
6090
6091 for_each_btf_ext_rec(seg, sec, i, rec) {
6092 insn_idx = rec->insn_off / BPF_INSN_SZ;
6093 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
6094 if (!prog) {
6095 pr_warn("sec '%s': failed to find program at insn #%d for CO-RE offset relocation #%d\n",
6096 sec_name, insn_idx, i);
6097 err = -EINVAL;
6098 goto out;
6099 }
6100
6101
6102
6103 if (!prog->load)
6104 continue;
6105
6106 err = bpf_core_apply_relo(prog, rec, i, obj->btf, cand_cache);
6107 if (err) {
6108 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
6109 prog->name, i, err);
6110 goto out;
6111 }
6112 }
6113 }
6114
6115out:
6116
6117 btf__free(obj->btf_vmlinux_override);
6118 obj->btf_vmlinux_override = NULL;
6119
6120 if (!IS_ERR_OR_NULL(cand_cache)) {
6121 hashmap__for_each_entry(cand_cache, entry, i) {
6122 bpf_core_free_cands(entry->value);
6123 }
6124 hashmap__free(cand_cache);
6125 }
6126 return err;
6127}
6128
6129
6130
6131
6132
6133
6134static int
6135bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
6136{
6137 int i;
6138
6139 for (i = 0; i < prog->nr_reloc; i++) {
6140 struct reloc_desc *relo = &prog->reloc_desc[i];
6141 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6142 struct extern_desc *ext;
6143
6144 switch (relo->type) {
6145 case RELO_LD64:
6146 insn[0].src_reg = BPF_PSEUDO_MAP_FD;
6147 insn[0].imm = obj->maps[relo->map_idx].fd;
6148 relo->processed = true;
6149 break;
6150 case RELO_DATA:
6151 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6152 insn[1].imm = insn[0].imm + relo->sym_off;
6153 insn[0].imm = obj->maps[relo->map_idx].fd;
6154 relo->processed = true;
6155 break;
6156 case RELO_EXTERN:
6157 ext = &obj->externs[relo->sym_off];
6158 if (ext->type == EXT_KCFG) {
6159 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
6160 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6161 insn[1].imm = ext->kcfg.data_off;
6162 } else {
6163 if (ext->ksym.type_id) {
6164 insn[0].src_reg = BPF_PSEUDO_BTF_ID;
6165 insn[0].imm = ext->ksym.vmlinux_btf_id;
6166 } else {
6167 insn[0].imm = (__u32)ext->ksym.addr;
6168 insn[1].imm = ext->ksym.addr >> 32;
6169 }
6170 }
6171 relo->processed = true;
6172 break;
6173 case RELO_CALL:
6174
6175 break;
6176 default:
6177 pr_warn("prog '%s': relo #%d: bad relo type %d\n",
6178 prog->name, i, relo->type);
6179 return -EINVAL;
6180 }
6181 }
6182
6183 return 0;
6184}
6185
6186static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
6187 const struct bpf_program *prog,
6188 const struct btf_ext_info *ext_info,
6189 void **prog_info, __u32 *prog_rec_cnt,
6190 __u32 *prog_rec_sz)
6191{
6192 void *copy_start = NULL, *copy_end = NULL;
6193 void *rec, *rec_end, *new_prog_info;
6194 const struct btf_ext_info_sec *sec;
6195 size_t old_sz, new_sz;
6196 const char *sec_name;
6197 int i, off_adj;
6198
6199 for_each_btf_ext_sec(ext_info, sec) {
6200 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
6201 if (!sec_name)
6202 return -EINVAL;
6203 if (strcmp(sec_name, prog->sec_name) != 0)
6204 continue;
6205
6206 for_each_btf_ext_rec(ext_info, sec, i, rec) {
6207 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;
6208
6209 if (insn_off < prog->sec_insn_off)
6210 continue;
6211 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6212 break;
6213
6214 if (!copy_start)
6215 copy_start = rec;
6216 copy_end = rec + ext_info->rec_size;
6217 }
6218
6219 if (!copy_start)
6220 return -ENOENT;
6221
6222
6223
6224
6225 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
6226 new_sz = old_sz + (copy_end - copy_start);
6227 new_prog_info = realloc(*prog_info, new_sz);
6228 if (!new_prog_info)
6229 return -ENOMEM;
6230 *prog_info = new_prog_info;
6231 *prog_rec_cnt = new_sz / ext_info->rec_size;
6232 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
6233
6234
6235
6236
6237
6238
6239
6240 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6241 rec = new_prog_info + old_sz;
6242 rec_end = new_prog_info + new_sz;
6243 for (; rec < rec_end; rec += ext_info->rec_size) {
6244 __u32 *insn_off = rec;
6245
6246 *insn_off = *insn_off / BPF_INSN_SZ + off_adj;
6247 }
6248 *prog_rec_sz = ext_info->rec_size;
6249 return 0;
6250 }
6251
6252 return -ENOENT;
6253}
6254
6255static int
6256reloc_prog_func_and_line_info(const struct bpf_object *obj,
6257 struct bpf_program *main_prog,
6258 const struct bpf_program *prog)
6259{
6260 int err;
6261
6262
6263
6264
6265 if (!obj->btf_ext || !kernel_supports(FEAT_BTF_FUNC))
6266 return 0;
6267
6268
6269
6270
6271 if (main_prog != prog && !main_prog->func_info)
6272 goto line_info;
6273
6274 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6275 &main_prog->func_info,
6276 &main_prog->func_info_cnt,
6277 &main_prog->func_info_rec_size);
6278 if (err) {
6279 if (err != -ENOENT) {
6280 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
6281 prog->name, err);
6282 return err;
6283 }
6284 if (main_prog->func_info) {
6285
6286
6287
6288
6289 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6290 return err;
6291 }
6292
6293 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
6294 prog->name);
6295 }
6296
6297line_info:
6298
6299 if (main_prog != prog && !main_prog->line_info)
6300 return 0;
6301
6302 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6303 &main_prog->line_info,
6304 &main_prog->line_info_cnt,
6305 &main_prog->line_info_rec_size);
6306 if (err) {
6307 if (err != -ENOENT) {
6308 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
6309 prog->name, err);
6310 return err;
6311 }
6312 if (main_prog->line_info) {
6313
6314
6315
6316
6317 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6318 return err;
6319 }
6320
6321 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6322 prog->name);
6323 }
6324 return 0;
6325}
6326
6327static int cmp_relo_by_insn_idx(const void *key, const void *elem)
6328{
6329 size_t insn_idx = *(const size_t *)key;
6330 const struct reloc_desc *relo = elem;
6331
6332 if (insn_idx == relo->insn_idx)
6333 return 0;
6334 return insn_idx < relo->insn_idx ? -1 : 1;
6335}
6336
6337static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6338{
6339 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6340 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6341}
6342
6343static int
6344bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
6345 struct bpf_program *prog)
6346{
6347 size_t sub_insn_idx, insn_idx, new_cnt;
6348 struct bpf_program *subprog;
6349 struct bpf_insn *insns, *insn;
6350 struct reloc_desc *relo;
6351 int err;
6352
6353 err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6354 if (err)
6355 return err;
6356
6357 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6358 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6359 if (!insn_is_subprog_call(insn))
6360 continue;
6361
6362 relo = find_prog_insn_relo(prog, insn_idx);
6363 if (relo && relo->type != RELO_CALL) {
6364 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6365 prog->name, insn_idx, relo->type);
6366 return -LIBBPF_ERRNO__RELOC;
6367 }
6368 if (relo) {
6369
6370
6371
6372
6373
6374
6375
6376 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
6377 } else {
6378
6379
6380
6381
6382
6383
6384 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6385 }
6386
6387
6388 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6389 if (!subprog) {
6390 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6391 prog->name);
6392 return -LIBBPF_ERRNO__RELOC;
6393 }
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405 if (subprog->sub_insn_off == 0) {
6406 subprog->sub_insn_off = main_prog->insns_cnt;
6407
6408 new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
6409 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
6410 if (!insns) {
6411 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6412 return -ENOMEM;
6413 }
6414 main_prog->insns = insns;
6415 main_prog->insns_cnt = new_cnt;
6416
6417 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
6418 subprog->insns_cnt * sizeof(*insns));
6419
6420 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6421 main_prog->name, subprog->insns_cnt, subprog->name);
6422
6423 err = bpf_object__reloc_code(obj, main_prog, subprog);
6424 if (err)
6425 return err;
6426 }
6427
6428
6429
6430
6431 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6432
6433
6434
6435
6436
6437 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6438
6439 if (relo)
6440 relo->processed = true;
6441
6442 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6443 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6444 }
6445
6446 return 0;
6447}
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530static int
6531bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6532{
6533 struct bpf_program *subprog;
6534 int i, j, err;
6535
6536
6537
6538
6539 for (i = 0; i < obj->nr_programs; i++) {
6540 subprog = &obj->programs[i];
6541 if (!prog_is_subprog(obj, subprog))
6542 continue;
6543
6544 subprog->sub_insn_off = 0;
6545 for (j = 0; j < subprog->nr_reloc; j++)
6546 if (subprog->reloc_desc[j].type == RELO_CALL)
6547 subprog->reloc_desc[j].processed = false;
6548 }
6549
6550 err = bpf_object__reloc_code(obj, prog, prog);
6551 if (err)
6552 return err;
6553
6554
6555 return 0;
6556}
6557
6558static int
6559bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
6560{
6561 struct bpf_program *prog;
6562 size_t i;
6563 int err;
6564
6565 if (obj->btf_ext) {
6566 err = bpf_object__relocate_core(obj, targ_btf_path);
6567 if (err) {
6568 pr_warn("failed to perform CO-RE relocations: %d\n",
6569 err);
6570 return err;
6571 }
6572 }
6573
6574
6575
6576
6577 for (i = 0; i < obj->nr_programs; i++) {
6578 prog = &obj->programs[i];
6579 err = bpf_object__relocate_data(obj, prog);
6580 if (err) {
6581 pr_warn("prog '%s': failed to relocate data references: %d\n",
6582 prog->name, err);
6583 return err;
6584 }
6585 }
6586
6587
6588
6589
6590
6591 for (i = 0; i < obj->nr_programs; i++) {
6592 prog = &obj->programs[i];
6593
6594
6595
6596 if (prog_is_subprog(obj, prog))
6597 continue;
6598
6599 err = bpf_object__relocate_calls(obj, prog);
6600 if (err) {
6601 pr_warn("prog '%s': failed to relocate calls: %d\n",
6602 prog->name, err);
6603 return err;
6604 }
6605 }
6606
6607 for (i = 0; i < obj->nr_programs; i++) {
6608 prog = &obj->programs[i];
6609 zfree(&prog->reloc_desc);
6610 prog->nr_reloc = 0;
6611 }
6612 return 0;
6613}
6614
6615static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
6616 GElf_Shdr *shdr, Elf_Data *data);
6617
6618static int bpf_object__collect_map_relos(struct bpf_object *obj,
6619 GElf_Shdr *shdr, Elf_Data *data)
6620{
6621 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
6622 int i, j, nrels, new_sz;
6623 const struct btf_var_secinfo *vi = NULL;
6624 const struct btf_type *sec, *var, *def;
6625 struct bpf_map *map = NULL, *targ_map;
6626 const struct btf_member *member;
6627 const char *name, *mname;
6628 Elf_Data *symbols;
6629 unsigned int moff;
6630 GElf_Sym sym;
6631 GElf_Rel rel;
6632 void *tmp;
6633
6634 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
6635 return -EINVAL;
6636 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
6637 if (!sec)
6638 return -EINVAL;
6639
6640 symbols = obj->efile.symbols;
6641 nrels = shdr->sh_size / shdr->sh_entsize;
6642 for (i = 0; i < nrels; i++) {
6643 if (!gelf_getrel(data, i, &rel)) {
6644 pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
6645 return -LIBBPF_ERRNO__FORMAT;
6646 }
6647 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
6648 pr_warn(".maps relo #%d: symbol %zx not found\n",
6649 i, (size_t)GELF_R_SYM(rel.r_info));
6650 return -LIBBPF_ERRNO__FORMAT;
6651 }
6652 name = elf_sym_str(obj, sym.st_name) ?: "<?>";
6653 if (sym.st_shndx != obj->efile.btf_maps_shndx) {
6654 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
6655 i, name);
6656 return -LIBBPF_ERRNO__RELOC;
6657 }
6658
6659 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n",
6660 i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value,
6661 (size_t)rel.r_offset, sym.st_name, name);
6662
6663 for (j = 0; j < obj->nr_maps; j++) {
6664 map = &obj->maps[j];
6665 if (map->sec_idx != obj->efile.btf_maps_shndx)
6666 continue;
6667
6668 vi = btf_var_secinfos(sec) + map->btf_var_idx;
6669 if (vi->offset <= rel.r_offset &&
6670 rel.r_offset + bpf_ptr_sz <= vi->offset + vi->size)
6671 break;
6672 }
6673 if (j == obj->nr_maps) {
6674 pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n",
6675 i, name, (size_t)rel.r_offset);
6676 return -EINVAL;
6677 }
6678
6679 if (!bpf_map_type__is_map_in_map(map->def.type))
6680 return -EINVAL;
6681 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
6682 map->def.key_size != sizeof(int)) {
6683 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
6684 i, map->name, sizeof(int));
6685 return -EINVAL;
6686 }
6687
6688 targ_map = bpf_object__find_map_by_name(obj, name);
6689 if (!targ_map)
6690 return -ESRCH;
6691
6692 var = btf__type_by_id(obj->btf, vi->type);
6693 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
6694 if (btf_vlen(def) == 0)
6695 return -EINVAL;
6696 member = btf_members(def) + btf_vlen(def) - 1;
6697 mname = btf__name_by_offset(obj->btf, member->name_off);
6698 if (strcmp(mname, "values"))
6699 return -EINVAL;
6700
6701 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
6702 if (rel.r_offset - vi->offset < moff)
6703 return -EINVAL;
6704
6705 moff = rel.r_offset - vi->offset - moff;
6706
6707
6708
6709 if (moff % bpf_ptr_sz)
6710 return -EINVAL;
6711 moff /= bpf_ptr_sz;
6712 if (moff >= map->init_slots_sz) {
6713 new_sz = moff + 1;
6714 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
6715 if (!tmp)
6716 return -ENOMEM;
6717 map->init_slots = tmp;
6718 memset(map->init_slots + map->init_slots_sz, 0,
6719 (new_sz - map->init_slots_sz) * host_ptr_sz);
6720 map->init_slots_sz = new_sz;
6721 }
6722 map->init_slots[moff] = targ_map;
6723
6724 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n",
6725 i, map->name, moff, name);
6726 }
6727
6728 return 0;
6729}
6730
6731static int cmp_relocs(const void *_a, const void *_b)
6732{
6733 const struct reloc_desc *a = _a;
6734 const struct reloc_desc *b = _b;
6735
6736 if (a->insn_idx != b->insn_idx)
6737 return a->insn_idx < b->insn_idx ? -1 : 1;
6738
6739
6740 if (a->type != b->type)
6741 return a->type < b->type ? -1 : 1;
6742
6743 return 0;
6744}
6745
6746static int bpf_object__collect_relos(struct bpf_object *obj)
6747{
6748 int i, err;
6749
6750 for (i = 0; i < obj->efile.nr_reloc_sects; i++) {
6751 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr;
6752 Elf_Data *data = obj->efile.reloc_sects[i].data;
6753 int idx = shdr->sh_info;
6754
6755 if (shdr->sh_type != SHT_REL) {
6756 pr_warn("internal error at %d\n", __LINE__);
6757 return -LIBBPF_ERRNO__INTERNAL;
6758 }
6759
6760 if (idx == obj->efile.st_ops_shndx)
6761 err = bpf_object__collect_st_ops_relos(obj, shdr, data);
6762 else if (idx == obj->efile.btf_maps_shndx)
6763 err = bpf_object__collect_map_relos(obj, shdr, data);
6764 else
6765 err = bpf_object__collect_prog_relos(obj, shdr, data);
6766 if (err)
6767 return err;
6768 }
6769
6770 for (i = 0; i < obj->nr_programs; i++) {
6771 struct bpf_program *p = &obj->programs[i];
6772
6773 if (!p->nr_reloc)
6774 continue;
6775
6776 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6777 }
6778 return 0;
6779}
6780
6781static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)
6782{
6783 if (BPF_CLASS(insn->code) == BPF_JMP &&
6784 BPF_OP(insn->code) == BPF_CALL &&
6785 BPF_SRC(insn->code) == BPF_K &&
6786 insn->src_reg == 0 &&
6787 insn->dst_reg == 0) {
6788 *func_id = insn->imm;
6789 return true;
6790 }
6791 return false;
6792}
6793
6794static int bpf_object__sanitize_prog(struct bpf_object* obj, struct bpf_program *prog)
6795{
6796 struct bpf_insn *insn = prog->insns;
6797 enum bpf_func_id func_id;
6798 int i;
6799
6800 for (i = 0; i < prog->insns_cnt; i++, insn++) {
6801 if (!insn_is_helper_call(insn, &func_id))
6802 continue;
6803
6804
6805
6806
6807
6808 switch (func_id) {
6809 case BPF_FUNC_probe_read_kernel:
6810 case BPF_FUNC_probe_read_user:
6811 if (!kernel_supports(FEAT_PROBE_READ_KERN))
6812 insn->imm = BPF_FUNC_probe_read;
6813 break;
6814 case BPF_FUNC_probe_read_kernel_str:
6815 case BPF_FUNC_probe_read_user_str:
6816 if (!kernel_supports(FEAT_PROBE_READ_KERN))
6817 insn->imm = BPF_FUNC_probe_read_str;
6818 break;
6819 default:
6820 break;
6821 }
6822 }
6823 return 0;
6824}
6825
6826static int
6827load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
6828 char *license, __u32 kern_version, int *pfd)
6829{
6830 struct bpf_prog_load_params load_attr = {};
6831 char *cp, errmsg[STRERR_BUFSIZE];
6832 size_t log_buf_size = 0;
6833 char *log_buf = NULL;
6834 int btf_fd, ret;
6835
6836 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
6837
6838
6839
6840
6841 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
6842 prog->name, prog->sec_name);
6843 return -EINVAL;
6844 }
6845
6846 if (!insns || !insns_cnt)
6847 return -EINVAL;
6848
6849 load_attr.prog_type = prog->type;
6850
6851 if (!kernel_supports(FEAT_EXP_ATTACH_TYPE) && prog->sec_def &&
6852 prog->sec_def->is_exp_attach_type_optional)
6853 load_attr.expected_attach_type = 0;
6854 else
6855 load_attr.expected_attach_type = prog->expected_attach_type;
6856 if (kernel_supports(FEAT_PROG_NAME))
6857 load_attr.name = prog->name;
6858 load_attr.insns = insns;
6859 load_attr.insn_cnt = insns_cnt;
6860 load_attr.license = license;
6861 load_attr.attach_btf_id = prog->attach_btf_id;
6862 if (prog->attach_prog_fd)
6863 load_attr.attach_prog_fd = prog->attach_prog_fd;
6864 else
6865 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
6866 load_attr.attach_btf_id = prog->attach_btf_id;
6867 load_attr.kern_version = kern_version;
6868 load_attr.prog_ifindex = prog->prog_ifindex;
6869
6870
6871 btf_fd = bpf_object__btf_fd(prog->obj);
6872 if (btf_fd >= 0 && kernel_supports(FEAT_BTF_FUNC)) {
6873 load_attr.prog_btf_fd = btf_fd;
6874 load_attr.func_info = prog->func_info;
6875 load_attr.func_info_rec_size = prog->func_info_rec_size;
6876 load_attr.func_info_cnt = prog->func_info_cnt;
6877 load_attr.line_info = prog->line_info;
6878 load_attr.line_info_rec_size = prog->line_info_rec_size;
6879 load_attr.line_info_cnt = prog->line_info_cnt;
6880 }
6881 load_attr.log_level = prog->log_level;
6882 load_attr.prog_flags = prog->prog_flags;
6883
6884retry_load:
6885 if (log_buf_size) {
6886 log_buf = malloc(log_buf_size);
6887 if (!log_buf)
6888 return -ENOMEM;
6889
6890 *log_buf = 0;
6891 }
6892
6893 load_attr.log_buf = log_buf;
6894 load_attr.log_buf_sz = log_buf_size;
6895 ret = libbpf__bpf_prog_load(&load_attr);
6896
6897 if (ret >= 0) {
6898 if (log_buf && load_attr.log_level)
6899 pr_debug("verifier log:\n%s", log_buf);
6900
6901 if (prog->obj->rodata_map_idx >= 0 &&
6902 kernel_supports(FEAT_PROG_BIND_MAP)) {
6903 struct bpf_map *rodata_map =
6904 &prog->obj->maps[prog->obj->rodata_map_idx];
6905
6906 if (bpf_prog_bind_map(ret, bpf_map__fd(rodata_map), NULL)) {
6907 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6908 pr_warn("prog '%s': failed to bind .rodata map: %s\n",
6909 prog->name, cp);
6910
6911 }
6912 }
6913
6914 *pfd = ret;
6915 ret = 0;
6916 goto out;
6917 }
6918
6919 if (!log_buf || errno == ENOSPC) {
6920 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE,
6921 log_buf_size << 1);
6922
6923 free(log_buf);
6924 goto retry_load;
6925 }
6926 ret = errno ? -errno : -LIBBPF_ERRNO__LOAD;
6927 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6928 pr_warn("load bpf program failed: %s\n", cp);
6929 pr_perm_msg(ret);
6930
6931 if (log_buf && log_buf[0] != '\0') {
6932 ret = -LIBBPF_ERRNO__VERIFY;
6933 pr_warn("-- BEGIN DUMP LOG ---\n");
6934 pr_warn("\n%s\n", log_buf);
6935 pr_warn("-- END LOG --\n");
6936 } else if (load_attr.insn_cnt >= BPF_MAXINSNS) {
6937 pr_warn("Program too large (%zu insns), at most %d insns\n",
6938 load_attr.insn_cnt, BPF_MAXINSNS);
6939 ret = -LIBBPF_ERRNO__PROG2BIG;
6940 } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
6941
6942 int fd;
6943
6944 load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
6945 load_attr.expected_attach_type = 0;
6946 load_attr.log_buf = NULL;
6947 load_attr.log_buf_sz = 0;
6948 fd = libbpf__bpf_prog_load(&load_attr);
6949 if (fd >= 0) {
6950 close(fd);
6951 ret = -LIBBPF_ERRNO__PROGTYPE;
6952 goto out;
6953 }
6954 }
6955
6956out:
6957 free(log_buf);
6958 return ret;
6959}
6960
6961static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id);
6962
6963int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
6964{
6965 int err = 0, fd, i;
6966
6967 if (prog->obj->loaded) {
6968 pr_warn("prog '%s': can't load after object was loaded\n", prog->name);
6969 return -EINVAL;
6970 }
6971
6972 if ((prog->type == BPF_PROG_TYPE_TRACING ||
6973 prog->type == BPF_PROG_TYPE_LSM ||
6974 prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) {
6975 int btf_obj_fd = 0, btf_type_id = 0;
6976
6977 err = libbpf_find_attach_btf_id(prog, &btf_obj_fd, &btf_type_id);
6978 if (err)
6979 return err;
6980
6981 prog->attach_btf_obj_fd = btf_obj_fd;
6982 prog->attach_btf_id = btf_type_id;
6983 }
6984
6985 if (prog->instances.nr < 0 || !prog->instances.fds) {
6986 if (prog->preprocessor) {
6987 pr_warn("Internal error: can't load program '%s'\n",
6988 prog->name);
6989 return -LIBBPF_ERRNO__INTERNAL;
6990 }
6991
6992 prog->instances.fds = malloc(sizeof(int));
6993 if (!prog->instances.fds) {
6994 pr_warn("Not enough memory for BPF fds\n");
6995 return -ENOMEM;
6996 }
6997 prog->instances.nr = 1;
6998 prog->instances.fds[0] = -1;
6999 }
7000
7001 if (!prog->preprocessor) {
7002 if (prog->instances.nr != 1) {
7003 pr_warn("prog '%s': inconsistent nr(%d) != 1\n",
7004 prog->name, prog->instances.nr);
7005 }
7006 err = load_program(prog, prog->insns, prog->insns_cnt,
7007 license, kern_ver, &fd);
7008 if (!err)
7009 prog->instances.fds[0] = fd;
7010 goto out;
7011 }
7012
7013 for (i = 0; i < prog->instances.nr; i++) {
7014 struct bpf_prog_prep_result result;
7015 bpf_program_prep_t preprocessor = prog->preprocessor;
7016
7017 memset(&result, 0, sizeof(result));
7018 err = preprocessor(prog, i, prog->insns,
7019 prog->insns_cnt, &result);
7020 if (err) {
7021 pr_warn("Preprocessing the %dth instance of program '%s' failed\n",
7022 i, prog->name);
7023 goto out;
7024 }
7025
7026 if (!result.new_insn_ptr || !result.new_insn_cnt) {
7027 pr_debug("Skip loading the %dth instance of program '%s'\n",
7028 i, prog->name);
7029 prog->instances.fds[i] = -1;
7030 if (result.pfd)
7031 *result.pfd = -1;
7032 continue;
7033 }
7034
7035 err = load_program(prog, result.new_insn_ptr,
7036 result.new_insn_cnt, license, kern_ver, &fd);
7037 if (err) {
7038 pr_warn("Loading the %dth instance of program '%s' failed\n",
7039 i, prog->name);
7040 goto out;
7041 }
7042
7043 if (result.pfd)
7044 *result.pfd = fd;
7045 prog->instances.fds[i] = fd;
7046 }
7047out:
7048 if (err)
7049 pr_warn("failed to load program '%s'\n", prog->name);
7050 zfree(&prog->insns);
7051 prog->insns_cnt = 0;
7052 return err;
7053}
7054
7055static int
7056bpf_object__load_progs(struct bpf_object *obj, int log_level)
7057{
7058 struct bpf_program *prog;
7059 size_t i;
7060 int err;
7061
7062 for (i = 0; i < obj->nr_programs; i++) {
7063 prog = &obj->programs[i];
7064 err = bpf_object__sanitize_prog(obj, prog);
7065 if (err)
7066 return err;
7067 }
7068
7069 for (i = 0; i < obj->nr_programs; i++) {
7070 prog = &obj->programs[i];
7071 if (prog_is_subprog(obj, prog))
7072 continue;
7073 if (!prog->load) {
7074 pr_debug("prog '%s': skipped loading\n", prog->name);
7075 continue;
7076 }
7077 prog->log_level |= log_level;
7078 err = bpf_program__load(prog, obj->license, obj->kern_version);
7079 if (err)
7080 return err;
7081 }
7082 return 0;
7083}
7084
7085static const struct bpf_sec_def *find_sec_def(const char *sec_name);
7086
7087static struct bpf_object *
7088__bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,
7089 const struct bpf_object_open_opts *opts)
7090{
7091 const char *obj_name, *kconfig;
7092 struct bpf_program *prog;
7093 struct bpf_object *obj;
7094 char tmp_name[64];
7095 int err;
7096
7097 if (elf_version(EV_CURRENT) == EV_NONE) {
7098 pr_warn("failed to init libelf for %s\n",
7099 path ? : "(mem buf)");
7100 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
7101 }
7102
7103 if (!OPTS_VALID(opts, bpf_object_open_opts))
7104 return ERR_PTR(-EINVAL);
7105
7106 obj_name = OPTS_GET(opts, object_name, NULL);
7107 if (obj_buf) {
7108 if (!obj_name) {
7109 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
7110 (unsigned long)obj_buf,
7111 (unsigned long)obj_buf_sz);
7112 obj_name = tmp_name;
7113 }
7114 path = obj_name;
7115 pr_debug("loading object '%s' from buffer\n", obj_name);
7116 }
7117
7118 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
7119 if (IS_ERR(obj))
7120 return obj;
7121
7122 kconfig = OPTS_GET(opts, kconfig, NULL);
7123 if (kconfig) {
7124 obj->kconfig = strdup(kconfig);
7125 if (!obj->kconfig)
7126 return ERR_PTR(-ENOMEM);
7127 }
7128
7129 err = bpf_object__elf_init(obj);
7130 err = err ? : bpf_object__check_endianness(obj);
7131 err = err ? : bpf_object__elf_collect(obj);
7132 err = err ? : bpf_object__collect_externs(obj);
7133 err = err ? : bpf_object__finalize_btf(obj);
7134 err = err ? : bpf_object__init_maps(obj, opts);
7135 err = err ? : bpf_object__collect_relos(obj);
7136 if (err)
7137 goto out;
7138 bpf_object__elf_finish(obj);
7139
7140 bpf_object__for_each_program(prog, obj) {
7141 prog->sec_def = find_sec_def(prog->sec_name);
7142 if (!prog->sec_def) {
7143
7144 pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
7145 prog->name, prog->sec_name);
7146 continue;
7147 }
7148
7149 if (prog->sec_def->is_sleepable)
7150 prog->prog_flags |= BPF_F_SLEEPABLE;
7151 bpf_program__set_type(prog, prog->sec_def->prog_type);
7152 bpf_program__set_expected_attach_type(prog,
7153 prog->sec_def->expected_attach_type);
7154
7155 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING ||
7156 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT)
7157 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
7158 }
7159
7160 return obj;
7161out:
7162 bpf_object__close(obj);
7163 return ERR_PTR(err);
7164}
7165
7166static struct bpf_object *
7167__bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)
7168{
7169 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
7170 .relaxed_maps = flags & MAPS_RELAX_COMPAT,
7171 );
7172
7173
7174 if (!attr->file)
7175 return NULL;
7176
7177 pr_debug("loading %s\n", attr->file);
7178 return __bpf_object__open(attr->file, NULL, 0, &opts);
7179}
7180
7181struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
7182{
7183 return __bpf_object__open_xattr(attr, 0);
7184}
7185
7186struct bpf_object *bpf_object__open(const char *path)
7187{
7188 struct bpf_object_open_attr attr = {
7189 .file = path,
7190 .prog_type = BPF_PROG_TYPE_UNSPEC,
7191 };
7192
7193 return bpf_object__open_xattr(&attr);
7194}
7195
7196struct bpf_object *
7197bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
7198{
7199 if (!path)
7200 return ERR_PTR(-EINVAL);
7201
7202 pr_debug("loading %s\n", path);
7203
7204 return __bpf_object__open(path, NULL, 0, opts);
7205}
7206
7207struct bpf_object *
7208bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
7209 const struct bpf_object_open_opts *opts)
7210{
7211 if (!obj_buf || obj_buf_sz == 0)
7212 return ERR_PTR(-EINVAL);
7213
7214 return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts);
7215}
7216
7217struct bpf_object *
7218bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
7219 const char *name)
7220{
7221 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
7222 .object_name = name,
7223
7224 .relaxed_maps = true,
7225 );
7226
7227
7228 if (!obj_buf || obj_buf_sz == 0)
7229 return NULL;
7230
7231 return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts);
7232}
7233
7234int bpf_object__unload(struct bpf_object *obj)
7235{
7236 size_t i;
7237
7238 if (!obj)
7239 return -EINVAL;
7240
7241 for (i = 0; i < obj->nr_maps; i++) {
7242 zclose(obj->maps[i].fd);
7243 if (obj->maps[i].st_ops)
7244 zfree(&obj->maps[i].st_ops->kern_vdata);
7245 }
7246
7247 for (i = 0; i < obj->nr_programs; i++)
7248 bpf_program__unload(&obj->programs[i]);
7249
7250 return 0;
7251}
7252
7253static int bpf_object__sanitize_maps(struct bpf_object *obj)
7254{
7255 struct bpf_map *m;
7256
7257 bpf_object__for_each_map(m, obj) {
7258 if (!bpf_map__is_internal(m))
7259 continue;
7260 if (!kernel_supports(FEAT_GLOBAL_DATA)) {
7261 pr_warn("kernel doesn't support global data\n");
7262 return -ENOTSUP;
7263 }
7264 if (!kernel_supports(FEAT_ARRAY_MMAP))
7265 m->def.map_flags ^= BPF_F_MMAPABLE;
7266 }
7267
7268 return 0;
7269}
7270
7271static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
7272{
7273 char sym_type, sym_name[500];
7274 unsigned long long sym_addr;
7275 struct extern_desc *ext;
7276 int ret, err = 0;
7277 FILE *f;
7278
7279 f = fopen("/proc/kallsyms", "r");
7280 if (!f) {
7281 err = -errno;
7282 pr_warn("failed to open /proc/kallsyms: %d\n", err);
7283 return err;
7284 }
7285
7286 while (true) {
7287 ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
7288 &sym_addr, &sym_type, sym_name);
7289 if (ret == EOF && feof(f))
7290 break;
7291 if (ret != 3) {
7292 pr_warn("failed to read kallsyms entry: %d\n", ret);
7293 err = -EINVAL;
7294 goto out;
7295 }
7296
7297 ext = find_extern_by_name(obj, sym_name);
7298 if (!ext || ext->type != EXT_KSYM)
7299 continue;
7300
7301 if (ext->is_set && ext->ksym.addr != sym_addr) {
7302 pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n",
7303 sym_name, ext->ksym.addr, sym_addr);
7304 err = -EINVAL;
7305 goto out;
7306 }
7307 if (!ext->is_set) {
7308 ext->is_set = true;
7309 ext->ksym.addr = sym_addr;
7310 pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr);
7311 }
7312 }
7313
7314out:
7315 fclose(f);
7316 return err;
7317}
7318
7319static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
7320{
7321 struct extern_desc *ext;
7322 int i, id;
7323
7324 for (i = 0; i < obj->nr_extern; i++) {
7325 const struct btf_type *targ_var, *targ_type;
7326 __u32 targ_type_id, local_type_id;
7327 const char *targ_var_name;
7328 int ret;
7329
7330 ext = &obj->externs[i];
7331 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
7332 continue;
7333
7334 id = btf__find_by_name_kind(obj->btf_vmlinux, ext->name,
7335 BTF_KIND_VAR);
7336 if (id <= 0) {
7337 pr_warn("extern (ksym) '%s': failed to find BTF ID in vmlinux BTF.\n",
7338 ext->name);
7339 return -ESRCH;
7340 }
7341
7342
7343 local_type_id = ext->ksym.type_id;
7344
7345
7346 targ_var = btf__type_by_id(obj->btf_vmlinux, id);
7347 targ_var_name = btf__name_by_offset(obj->btf_vmlinux,
7348 targ_var->name_off);
7349 targ_type = skip_mods_and_typedefs(obj->btf_vmlinux,
7350 targ_var->type,
7351 &targ_type_id);
7352
7353 ret = bpf_core_types_are_compat(obj->btf, local_type_id,
7354 obj->btf_vmlinux, targ_type_id);
7355 if (ret <= 0) {
7356 const struct btf_type *local_type;
7357 const char *targ_name, *local_name;
7358
7359 local_type = btf__type_by_id(obj->btf, local_type_id);
7360 local_name = btf__name_by_offset(obj->btf,
7361 local_type->name_off);
7362 targ_name = btf__name_by_offset(obj->btf_vmlinux,
7363 targ_type->name_off);
7364
7365 pr_warn("extern (ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
7366 ext->name, local_type_id,
7367 btf_kind_str(local_type), local_name, targ_type_id,
7368 btf_kind_str(targ_type), targ_name);
7369 return -EINVAL;
7370 }
7371
7372 ext->is_set = true;
7373 ext->ksym.vmlinux_btf_id = id;
7374 pr_debug("extern (ksym) '%s': resolved to [%d] %s %s\n",
7375 ext->name, id, btf_kind_str(targ_var), targ_var_name);
7376 }
7377 return 0;
7378}
7379
7380static int bpf_object__resolve_externs(struct bpf_object *obj,
7381 const char *extra_kconfig)
7382{
7383 bool need_config = false, need_kallsyms = false;
7384 bool need_vmlinux_btf = false;
7385 struct extern_desc *ext;
7386 void *kcfg_data = NULL;
7387 int err, i;
7388
7389 if (obj->nr_extern == 0)
7390 return 0;
7391
7392 if (obj->kconfig_map_idx >= 0)
7393 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
7394
7395 for (i = 0; i < obj->nr_extern; i++) {
7396 ext = &obj->externs[i];
7397
7398 if (ext->type == EXT_KCFG &&
7399 strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
7400 void *ext_val = kcfg_data + ext->kcfg.data_off;
7401 __u32 kver = get_kernel_version();
7402
7403 if (!kver) {
7404 pr_warn("failed to get kernel version\n");
7405 return -EINVAL;
7406 }
7407 err = set_kcfg_value_num(ext, ext_val, kver);
7408 if (err)
7409 return err;
7410 pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver);
7411 } else if (ext->type == EXT_KCFG &&
7412 strncmp(ext->name, "CONFIG_", 7) == 0) {
7413 need_config = true;
7414 } else if (ext->type == EXT_KSYM) {
7415 if (ext->ksym.type_id)
7416 need_vmlinux_btf = true;
7417 else
7418 need_kallsyms = true;
7419 } else {
7420 pr_warn("unrecognized extern '%s'\n", ext->name);
7421 return -EINVAL;
7422 }
7423 }
7424 if (need_config && extra_kconfig) {
7425 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
7426 if (err)
7427 return -EINVAL;
7428 need_config = false;
7429 for (i = 0; i < obj->nr_extern; i++) {
7430 ext = &obj->externs[i];
7431 if (ext->type == EXT_KCFG && !ext->is_set) {
7432 need_config = true;
7433 break;
7434 }
7435 }
7436 }
7437 if (need_config) {
7438 err = bpf_object__read_kconfig_file(obj, kcfg_data);
7439 if (err)
7440 return -EINVAL;
7441 }
7442 if (need_kallsyms) {
7443 err = bpf_object__read_kallsyms_file(obj);
7444 if (err)
7445 return -EINVAL;
7446 }
7447 if (need_vmlinux_btf) {
7448 err = bpf_object__resolve_ksyms_btf_id(obj);
7449 if (err)
7450 return -EINVAL;
7451 }
7452 for (i = 0; i < obj->nr_extern; i++) {
7453 ext = &obj->externs[i];
7454
7455 if (!ext->is_set && !ext->is_weak) {
7456 pr_warn("extern %s (strong) not resolved\n", ext->name);
7457 return -ESRCH;
7458 } else if (!ext->is_set) {
7459 pr_debug("extern %s (weak) not resolved, defaulting to zero\n",
7460 ext->name);
7461 }
7462 }
7463
7464 return 0;
7465}
7466
7467int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
7468{
7469 struct bpf_object *obj;
7470 int err, i;
7471
7472 if (!attr)
7473 return -EINVAL;
7474 obj = attr->obj;
7475 if (!obj)
7476 return -EINVAL;
7477
7478 if (obj->loaded) {
7479 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
7480 return -EINVAL;
7481 }
7482
7483 err = bpf_object__probe_loading(obj);
7484 err = err ? : bpf_object__load_vmlinux_btf(obj, false);
7485 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
7486 err = err ? : bpf_object__sanitize_and_load_btf(obj);
7487 err = err ? : bpf_object__sanitize_maps(obj);
7488 err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
7489 err = err ? : bpf_object__create_maps(obj);
7490 err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
7491 err = err ? : bpf_object__load_progs(obj, attr->log_level);
7492
7493
7494 for (i = 0; i < obj->btf_module_cnt; i++) {
7495 close(obj->btf_modules[i].fd);
7496 btf__free(obj->btf_modules[i].btf);
7497 free(obj->btf_modules[i].name);
7498 }
7499 free(obj->btf_modules);
7500
7501
7502 btf__free(obj->btf_vmlinux);
7503 obj->btf_vmlinux = NULL;
7504
7505 obj->loaded = true;
7506
7507 if (err)
7508 goto out;
7509
7510 return 0;
7511out:
7512
7513 for (i = 0; i < obj->nr_maps; i++)
7514 if (obj->maps[i].pinned && !obj->maps[i].reused)
7515 bpf_map__unpin(&obj->maps[i], NULL);
7516
7517 bpf_object__unload(obj);
7518 pr_warn("failed to load object '%s'\n", obj->path);
7519 return err;
7520}
7521
7522int bpf_object__load(struct bpf_object *obj)
7523{
7524 struct bpf_object_load_attr attr = {
7525 .obj = obj,
7526 };
7527
7528 return bpf_object__load_xattr(&attr);
7529}
7530
7531static int make_parent_dir(const char *path)
7532{
7533 char *cp, errmsg[STRERR_BUFSIZE];
7534 char *dname, *dir;
7535 int err = 0;
7536
7537 dname = strdup(path);
7538 if (dname == NULL)
7539 return -ENOMEM;
7540
7541 dir = dirname(dname);
7542 if (mkdir(dir, 0700) && errno != EEXIST)
7543 err = -errno;
7544
7545 free(dname);
7546 if (err) {
7547 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7548 pr_warn("failed to mkdir %s: %s\n", path, cp);
7549 }
7550 return err;
7551}
7552
7553static int check_path(const char *path)
7554{
7555 char *cp, errmsg[STRERR_BUFSIZE];
7556 struct statfs st_fs;
7557 char *dname, *dir;
7558 int err = 0;
7559
7560 if (path == NULL)
7561 return -EINVAL;
7562
7563 dname = strdup(path);
7564 if (dname == NULL)
7565 return -ENOMEM;
7566
7567 dir = dirname(dname);
7568 if (statfs(dir, &st_fs)) {
7569 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7570 pr_warn("failed to statfs %s: %s\n", dir, cp);
7571 err = -errno;
7572 }
7573 free(dname);
7574
7575 if (!err && st_fs.f_type != BPF_FS_MAGIC) {
7576 pr_warn("specified path %s is not on BPF FS\n", path);
7577 err = -EINVAL;
7578 }
7579
7580 return err;
7581}
7582
7583int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
7584 int instance)
7585{
7586 char *cp, errmsg[STRERR_BUFSIZE];
7587 int err;
7588
7589 err = make_parent_dir(path);
7590 if (err)
7591 return err;
7592
7593 err = check_path(path);
7594 if (err)
7595 return err;
7596
7597 if (prog == NULL) {
7598 pr_warn("invalid program pointer\n");
7599 return -EINVAL;
7600 }
7601
7602 if (instance < 0 || instance >= prog->instances.nr) {
7603 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
7604 instance, prog->name, prog->instances.nr);
7605 return -EINVAL;
7606 }
7607
7608 if (bpf_obj_pin(prog->instances.fds[instance], path)) {
7609 err = -errno;
7610 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
7611 pr_warn("failed to pin program: %s\n", cp);
7612 return err;
7613 }
7614 pr_debug("pinned program '%s'\n", path);
7615
7616 return 0;
7617}
7618
7619int bpf_program__unpin_instance(struct bpf_program *prog, const char *path,
7620 int instance)
7621{
7622 int err;
7623
7624 err = check_path(path);
7625 if (err)
7626 return err;
7627
7628 if (prog == NULL) {
7629 pr_warn("invalid program pointer\n");
7630 return -EINVAL;
7631 }
7632
7633 if (instance < 0 || instance >= prog->instances.nr) {
7634 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
7635 instance, prog->name, prog->instances.nr);
7636 return -EINVAL;
7637 }
7638
7639 err = unlink(path);
7640 if (err != 0)
7641 return -errno;
7642 pr_debug("unpinned program '%s'\n", path);
7643
7644 return 0;
7645}
7646
7647int bpf_program__pin(struct bpf_program *prog, const char *path)
7648{
7649 int i, err;
7650
7651 err = make_parent_dir(path);
7652 if (err)
7653 return err;
7654
7655 err = check_path(path);
7656 if (err)
7657 return err;
7658
7659 if (prog == NULL) {
7660 pr_warn("invalid program pointer\n");
7661 return -EINVAL;
7662 }
7663
7664 if (prog->instances.nr <= 0) {
7665 pr_warn("no instances of prog %s to pin\n", prog->name);
7666 return -EINVAL;
7667 }
7668
7669 if (prog->instances.nr == 1) {
7670
7671 return bpf_program__pin_instance(prog, path, 0);
7672 }
7673
7674 for (i = 0; i < prog->instances.nr; i++) {
7675 char buf[PATH_MAX];
7676 int len;
7677
7678 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7679 if (len < 0) {
7680 err = -EINVAL;
7681 goto err_unpin;
7682 } else if (len >= PATH_MAX) {
7683 err = -ENAMETOOLONG;
7684 goto err_unpin;
7685 }
7686
7687 err = bpf_program__pin_instance(prog, buf, i);
7688 if (err)
7689 goto err_unpin;
7690 }
7691
7692 return 0;
7693
7694err_unpin:
7695 for (i = i - 1; i >= 0; i--) {
7696 char buf[PATH_MAX];
7697 int len;
7698
7699 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7700 if (len < 0)
7701 continue;
7702 else if (len >= PATH_MAX)
7703 continue;
7704
7705 bpf_program__unpin_instance(prog, buf, i);
7706 }
7707
7708 rmdir(path);
7709
7710 return err;
7711}
7712
7713int bpf_program__unpin(struct bpf_program *prog, const char *path)
7714{
7715 int i, err;
7716
7717 err = check_path(path);
7718 if (err)
7719 return err;
7720
7721 if (prog == NULL) {
7722 pr_warn("invalid program pointer\n");
7723 return -EINVAL;
7724 }
7725
7726 if (prog->instances.nr <= 0) {
7727 pr_warn("no instances of prog %s to pin\n", prog->name);
7728 return -EINVAL;
7729 }
7730
7731 if (prog->instances.nr == 1) {
7732
7733 return bpf_program__unpin_instance(prog, path, 0);
7734 }
7735
7736 for (i = 0; i < prog->instances.nr; i++) {
7737 char buf[PATH_MAX];
7738 int len;
7739
7740 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7741 if (len < 0)
7742 return -EINVAL;
7743 else if (len >= PATH_MAX)
7744 return -ENAMETOOLONG;
7745
7746 err = bpf_program__unpin_instance(prog, buf, i);
7747 if (err)
7748 return err;
7749 }
7750
7751 err = rmdir(path);
7752 if (err)
7753 return -errno;
7754
7755 return 0;
7756}
7757
7758int bpf_map__pin(struct bpf_map *map, const char *path)
7759{
7760 char *cp, errmsg[STRERR_BUFSIZE];
7761 int err;
7762
7763 if (map == NULL) {
7764 pr_warn("invalid map pointer\n");
7765 return -EINVAL;
7766 }
7767
7768 if (map->pin_path) {
7769 if (path && strcmp(path, map->pin_path)) {
7770 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7771 bpf_map__name(map), map->pin_path, path);
7772 return -EINVAL;
7773 } else if (map->pinned) {
7774 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
7775 bpf_map__name(map), map->pin_path);
7776 return 0;
7777 }
7778 } else {
7779 if (!path) {
7780 pr_warn("missing a path to pin map '%s' at\n",
7781 bpf_map__name(map));
7782 return -EINVAL;
7783 } else if (map->pinned) {
7784 pr_warn("map '%s' already pinned\n", bpf_map__name(map));
7785 return -EEXIST;
7786 }
7787
7788 map->pin_path = strdup(path);
7789 if (!map->pin_path) {
7790 err = -errno;
7791 goto out_err;
7792 }
7793 }
7794
7795 err = make_parent_dir(map->pin_path);
7796 if (err)
7797 return err;
7798
7799 err = check_path(map->pin_path);
7800 if (err)
7801 return err;
7802
7803 if (bpf_obj_pin(map->fd, map->pin_path)) {
7804 err = -errno;
7805 goto out_err;
7806 }
7807
7808 map->pinned = true;
7809 pr_debug("pinned map '%s'\n", map->pin_path);
7810
7811 return 0;
7812
7813out_err:
7814 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7815 pr_warn("failed to pin map: %s\n", cp);
7816 return err;
7817}
7818
7819int bpf_map__unpin(struct bpf_map *map, const char *path)
7820{
7821 int err;
7822
7823 if (map == NULL) {
7824 pr_warn("invalid map pointer\n");
7825 return -EINVAL;
7826 }
7827
7828 if (map->pin_path) {
7829 if (path && strcmp(path, map->pin_path)) {
7830 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7831 bpf_map__name(map), map->pin_path, path);
7832 return -EINVAL;
7833 }
7834 path = map->pin_path;
7835 } else if (!path) {
7836 pr_warn("no path to unpin map '%s' from\n",
7837 bpf_map__name(map));
7838 return -EINVAL;
7839 }
7840
7841 err = check_path(path);
7842 if (err)
7843 return err;
7844
7845 err = unlink(path);
7846 if (err != 0)
7847 return -errno;
7848
7849 map->pinned = false;
7850 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
7851
7852 return 0;
7853}
7854
7855int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
7856{
7857 char *new = NULL;
7858
7859 if (path) {
7860 new = strdup(path);
7861 if (!new)
7862 return -errno;
7863 }
7864
7865 free(map->pin_path);
7866 map->pin_path = new;
7867 return 0;
7868}
7869
7870const char *bpf_map__get_pin_path(const struct bpf_map *map)
7871{
7872 return map->pin_path;
7873}
7874
7875bool bpf_map__is_pinned(const struct bpf_map *map)
7876{
7877 return map->pinned;
7878}
7879
7880static void sanitize_pin_path(char *s)
7881{
7882
7883 while (*s) {
7884 if (*s == '.')
7885 *s = '_';
7886 s++;
7887 }
7888}
7889
7890int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
7891{
7892 struct bpf_map *map;
7893 int err;
7894
7895 if (!obj)
7896 return -ENOENT;
7897
7898 if (!obj->loaded) {
7899 pr_warn("object not yet loaded; load it first\n");
7900 return -ENOENT;
7901 }
7902
7903 bpf_object__for_each_map(map, obj) {
7904 char *pin_path = NULL;
7905 char buf[PATH_MAX];
7906
7907 if (path) {
7908 int len;
7909
7910 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7911 bpf_map__name(map));
7912 if (len < 0) {
7913 err = -EINVAL;
7914 goto err_unpin_maps;
7915 } else if (len >= PATH_MAX) {
7916 err = -ENAMETOOLONG;
7917 goto err_unpin_maps;
7918 }
7919 sanitize_pin_path(buf);
7920 pin_path = buf;
7921 } else if (!map->pin_path) {
7922 continue;
7923 }
7924
7925 err = bpf_map__pin(map, pin_path);
7926 if (err)
7927 goto err_unpin_maps;
7928 }
7929
7930 return 0;
7931
7932err_unpin_maps:
7933 while ((map = bpf_map__prev(map, obj))) {
7934 if (!map->pin_path)
7935 continue;
7936
7937 bpf_map__unpin(map, NULL);
7938 }
7939
7940 return err;
7941}
7942
7943int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
7944{
7945 struct bpf_map *map;
7946 int err;
7947
7948 if (!obj)
7949 return -ENOENT;
7950
7951 bpf_object__for_each_map(map, obj) {
7952 char *pin_path = NULL;
7953 char buf[PATH_MAX];
7954
7955 if (path) {
7956 int len;
7957
7958 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7959 bpf_map__name(map));
7960 if (len < 0)
7961 return -EINVAL;
7962 else if (len >= PATH_MAX)
7963 return -ENAMETOOLONG;
7964 sanitize_pin_path(buf);
7965 pin_path = buf;
7966 } else if (!map->pin_path) {
7967 continue;
7968 }
7969
7970 err = bpf_map__unpin(map, pin_path);
7971 if (err)
7972 return err;
7973 }
7974
7975 return 0;
7976}
7977
7978int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
7979{
7980 struct bpf_program *prog;
7981 int err;
7982
7983 if (!obj)
7984 return -ENOENT;
7985
7986 if (!obj->loaded) {
7987 pr_warn("object not yet loaded; load it first\n");
7988 return -ENOENT;
7989 }
7990
7991 bpf_object__for_each_program(prog, obj) {
7992 char buf[PATH_MAX];
7993 int len;
7994
7995 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7996 prog->pin_name);
7997 if (len < 0) {
7998 err = -EINVAL;
7999 goto err_unpin_programs;
8000 } else if (len >= PATH_MAX) {
8001 err = -ENAMETOOLONG;
8002 goto err_unpin_programs;
8003 }
8004
8005 err = bpf_program__pin(prog, buf);
8006 if (err)
8007 goto err_unpin_programs;
8008 }
8009
8010 return 0;
8011
8012err_unpin_programs:
8013 while ((prog = bpf_program__prev(prog, obj))) {
8014 char buf[PATH_MAX];
8015 int len;
8016
8017 len = snprintf(buf, PATH_MAX, "%s/%s", path,
8018 prog->pin_name);
8019 if (len < 0)
8020 continue;
8021 else if (len >= PATH_MAX)
8022 continue;
8023
8024 bpf_program__unpin(prog, buf);
8025 }
8026
8027 return err;
8028}
8029
8030int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
8031{
8032 struct bpf_program *prog;
8033 int err;
8034
8035 if (!obj)
8036 return -ENOENT;
8037
8038 bpf_object__for_each_program(prog, obj) {
8039 char buf[PATH_MAX];
8040 int len;
8041
8042 len = snprintf(buf, PATH_MAX, "%s/%s", path,
8043 prog->pin_name);
8044 if (len < 0)
8045 return -EINVAL;
8046 else if (len >= PATH_MAX)
8047 return -ENAMETOOLONG;
8048
8049 err = bpf_program__unpin(prog, buf);
8050 if (err)
8051 return err;
8052 }
8053
8054 return 0;
8055}
8056
8057int bpf_object__pin(struct bpf_object *obj, const char *path)
8058{
8059 int err;
8060
8061 err = bpf_object__pin_maps(obj, path);
8062 if (err)
8063 return err;
8064
8065 err = bpf_object__pin_programs(obj, path);
8066 if (err) {
8067 bpf_object__unpin_maps(obj, path);
8068 return err;
8069 }
8070
8071 return 0;
8072}
8073
8074static void bpf_map__destroy(struct bpf_map *map)
8075{
8076 if (map->clear_priv)
8077 map->clear_priv(map, map->priv);
8078 map->priv = NULL;
8079 map->clear_priv = NULL;
8080
8081 if (map->inner_map) {
8082 bpf_map__destroy(map->inner_map);
8083 zfree(&map->inner_map);
8084 }
8085
8086 zfree(&map->init_slots);
8087 map->init_slots_sz = 0;
8088
8089 if (map->mmaped) {
8090 munmap(map->mmaped, bpf_map_mmap_sz(map));
8091 map->mmaped = NULL;
8092 }
8093
8094 if (map->st_ops) {
8095 zfree(&map->st_ops->data);
8096 zfree(&map->st_ops->progs);
8097 zfree(&map->st_ops->kern_func_off);
8098 zfree(&map->st_ops);
8099 }
8100
8101 zfree(&map->name);
8102 zfree(&map->pin_path);
8103
8104 if (map->fd >= 0)
8105 zclose(map->fd);
8106}
8107
8108void bpf_object__close(struct bpf_object *obj)
8109{
8110 size_t i;
8111
8112 if (IS_ERR_OR_NULL(obj))
8113 return;
8114
8115 if (obj->clear_priv)
8116 obj->clear_priv(obj, obj->priv);
8117
8118 bpf_object__elf_finish(obj);
8119 bpf_object__unload(obj);
8120 btf__free(obj->btf);
8121 btf_ext__free(obj->btf_ext);
8122
8123 for (i = 0; i < obj->nr_maps; i++)
8124 bpf_map__destroy(&obj->maps[i]);
8125
8126 zfree(&obj->kconfig);
8127 zfree(&obj->externs);
8128 obj->nr_extern = 0;
8129
8130 zfree(&obj->maps);
8131 obj->nr_maps = 0;
8132
8133 if (obj->programs && obj->nr_programs) {
8134 for (i = 0; i < obj->nr_programs; i++)
8135 bpf_program__exit(&obj->programs[i]);
8136 }
8137 zfree(&obj->programs);
8138
8139 list_del(&obj->list);
8140 free(obj);
8141}
8142
8143struct bpf_object *
8144bpf_object__next(struct bpf_object *prev)
8145{
8146 struct bpf_object *next;
8147
8148 if (!prev)
8149 next = list_first_entry(&bpf_objects_list,
8150 struct bpf_object,
8151 list);
8152 else
8153 next = list_next_entry(prev, list);
8154
8155
8156 if (&next->list == &bpf_objects_list)
8157 return NULL;
8158
8159 return next;
8160}
8161
8162const char *bpf_object__name(const struct bpf_object *obj)
8163{
8164 return obj ? obj->name : ERR_PTR(-EINVAL);
8165}
8166
8167unsigned int bpf_object__kversion(const struct bpf_object *obj)
8168{
8169 return obj ? obj->kern_version : 0;
8170}
8171
8172struct btf *bpf_object__btf(const struct bpf_object *obj)
8173{
8174 return obj ? obj->btf : NULL;
8175}
8176
8177int bpf_object__btf_fd(const struct bpf_object *obj)
8178{
8179 return obj->btf ? btf__fd(obj->btf) : -1;
8180}
8181
8182int bpf_object__set_priv(struct bpf_object *obj, void *priv,
8183 bpf_object_clear_priv_t clear_priv)
8184{
8185 if (obj->priv && obj->clear_priv)
8186 obj->clear_priv(obj, obj->priv);
8187
8188 obj->priv = priv;
8189 obj->clear_priv = clear_priv;
8190 return 0;
8191}
8192
8193void *bpf_object__priv(const struct bpf_object *obj)
8194{
8195 return obj ? obj->priv : ERR_PTR(-EINVAL);
8196}
8197
8198static struct bpf_program *
8199__bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
8200 bool forward)
8201{
8202 size_t nr_programs = obj->nr_programs;
8203 ssize_t idx;
8204
8205 if (!nr_programs)
8206 return NULL;
8207
8208 if (!p)
8209
8210 return forward ? &obj->programs[0] :
8211 &obj->programs[nr_programs - 1];
8212
8213 if (p->obj != obj) {
8214 pr_warn("error: program handler doesn't match object\n");
8215 return NULL;
8216 }
8217
8218 idx = (p - obj->programs) + (forward ? 1 : -1);
8219 if (idx >= obj->nr_programs || idx < 0)
8220 return NULL;
8221 return &obj->programs[idx];
8222}
8223
8224struct bpf_program *
8225bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
8226{
8227 struct bpf_program *prog = prev;
8228
8229 do {
8230 prog = __bpf_program__iter(prog, obj, true);
8231 } while (prog && prog_is_subprog(obj, prog));
8232
8233 return prog;
8234}
8235
8236struct bpf_program *
8237bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj)
8238{
8239 struct bpf_program *prog = next;
8240
8241 do {
8242 prog = __bpf_program__iter(prog, obj, false);
8243 } while (prog && prog_is_subprog(obj, prog));
8244
8245 return prog;
8246}
8247
8248int bpf_program__set_priv(struct bpf_program *prog, void *priv,
8249 bpf_program_clear_priv_t clear_priv)
8250{
8251 if (prog->priv && prog->clear_priv)
8252 prog->clear_priv(prog, prog->priv);
8253
8254 prog->priv = priv;
8255 prog->clear_priv = clear_priv;
8256 return 0;
8257}
8258
8259void *bpf_program__priv(const struct bpf_program *prog)
8260{
8261 return prog ? prog->priv : ERR_PTR(-EINVAL);
8262}
8263
8264void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
8265{
8266 prog->prog_ifindex = ifindex;
8267}
8268
8269const char *bpf_program__name(const struct bpf_program *prog)
8270{
8271 return prog->name;
8272}
8273
8274const char *bpf_program__section_name(const struct bpf_program *prog)
8275{
8276 return prog->sec_name;
8277}
8278
8279const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy)
8280{
8281 const char *title;
8282
8283 title = prog->sec_name;
8284 if (needs_copy) {
8285 title = strdup(title);
8286 if (!title) {
8287 pr_warn("failed to strdup program title\n");
8288 return ERR_PTR(-ENOMEM);
8289 }
8290 }
8291
8292 return title;
8293}
8294
8295bool bpf_program__autoload(const struct bpf_program *prog)
8296{
8297 return prog->load;
8298}
8299
8300int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
8301{
8302 if (prog->obj->loaded)
8303 return -EINVAL;
8304
8305 prog->load = autoload;
8306 return 0;
8307}
8308
8309int bpf_program__fd(const struct bpf_program *prog)
8310{
8311 return bpf_program__nth_fd(prog, 0);
8312}
8313
8314size_t bpf_program__size(const struct bpf_program *prog)
8315{
8316 return prog->insns_cnt * BPF_INSN_SZ;
8317}
8318
8319int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
8320 bpf_program_prep_t prep)
8321{
8322 int *instances_fds;
8323
8324 if (nr_instances <= 0 || !prep)
8325 return -EINVAL;
8326
8327 if (prog->instances.nr > 0 || prog->instances.fds) {
8328 pr_warn("Can't set pre-processor after loading\n");
8329 return -EINVAL;
8330 }
8331
8332 instances_fds = malloc(sizeof(int) * nr_instances);
8333 if (!instances_fds) {
8334 pr_warn("alloc memory failed for fds\n");
8335 return -ENOMEM;
8336 }
8337
8338
8339 memset(instances_fds, -1, sizeof(int) * nr_instances);
8340
8341 prog->instances.nr = nr_instances;
8342 prog->instances.fds = instances_fds;
8343 prog->preprocessor = prep;
8344 return 0;
8345}
8346
8347int bpf_program__nth_fd(const struct bpf_program *prog, int n)
8348{
8349 int fd;
8350
8351 if (!prog)
8352 return -EINVAL;
8353
8354 if (n >= prog->instances.nr || n < 0) {
8355 pr_warn("Can't get the %dth fd from program %s: only %d instances\n",
8356 n, prog->name, prog->instances.nr);
8357 return -EINVAL;
8358 }
8359
8360 fd = prog->instances.fds[n];
8361 if (fd < 0) {
8362 pr_warn("%dth instance of program '%s' is invalid\n",
8363 n, prog->name);
8364 return -ENOENT;
8365 }
8366
8367 return fd;
8368}
8369
8370enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog)
8371{
8372 return prog->type;
8373}
8374
8375void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
8376{
8377 prog->type = type;
8378}
8379
8380static bool bpf_program__is_type(const struct bpf_program *prog,
8381 enum bpf_prog_type type)
8382{
8383 return prog ? (prog->type == type) : false;
8384}
8385
8386#define BPF_PROG_TYPE_FNS(NAME, TYPE) \
8387int bpf_program__set_##NAME(struct bpf_program *prog) \
8388{ \
8389 if (!prog) \
8390 return -EINVAL; \
8391 bpf_program__set_type(prog, TYPE); \
8392 return 0; \
8393} \
8394 \
8395bool bpf_program__is_##NAME(const struct bpf_program *prog) \
8396{ \
8397 return bpf_program__is_type(prog, TYPE); \
8398} \
8399
8400BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
8401BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM);
8402BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
8403BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
8404BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
8405BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
8406BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
8407BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
8408BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
8409BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
8410BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS);
8411BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT);
8412BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP);
8413
8414enum bpf_attach_type
8415bpf_program__get_expected_attach_type(struct bpf_program *prog)
8416{
8417 return prog->expected_attach_type;
8418}
8419
8420void bpf_program__set_expected_attach_type(struct bpf_program *prog,
8421 enum bpf_attach_type type)
8422{
8423 prog->expected_attach_type = type;
8424}
8425
8426#define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional, \
8427 attachable, attach_btf) \
8428 { \
8429 .sec = string, \
8430 .len = sizeof(string) - 1, \
8431 .prog_type = ptype, \
8432 .expected_attach_type = eatype, \
8433 .is_exp_attach_type_optional = eatype_optional, \
8434 .is_attachable = attachable, \
8435 .is_attach_btf = attach_btf, \
8436 }
8437
8438
8439#define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0)
8440
8441
8442#define BPF_APROG_SEC(string, ptype, atype) \
8443 BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0)
8444
8445
8446#define BPF_EAPROG_SEC(string, ptype, eatype) \
8447 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0)
8448
8449
8450#define BPF_PROG_BTF(string, ptype, eatype) \
8451 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1)
8452
8453
8454
8455
8456#define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
8457
8458#define SEC_DEF(sec_pfx, ptype, ...) { \
8459 .sec = sec_pfx, \
8460 .len = sizeof(sec_pfx) - 1, \
8461 .prog_type = BPF_PROG_TYPE_##ptype, \
8462 __VA_ARGS__ \
8463}
8464
8465static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
8466 struct bpf_program *prog);
8467static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
8468 struct bpf_program *prog);
8469static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
8470 struct bpf_program *prog);
8471static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
8472 struct bpf_program *prog);
8473static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
8474 struct bpf_program *prog);
8475static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
8476 struct bpf_program *prog);
8477
8478static const struct bpf_sec_def section_defs[] = {
8479 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),
8480 BPF_PROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT),
8481 SEC_DEF("kprobe/", KPROBE,
8482 .attach_fn = attach_kprobe),
8483 BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE),
8484 SEC_DEF("kretprobe/", KPROBE,
8485 .attach_fn = attach_kprobe),
8486 BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE),
8487 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS),
8488 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT),
8489 SEC_DEF("tracepoint/", TRACEPOINT,
8490 .attach_fn = attach_tp),
8491 SEC_DEF("tp/", TRACEPOINT,
8492 .attach_fn = attach_tp),
8493 SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT,
8494 .attach_fn = attach_raw_tp),
8495 SEC_DEF("raw_tp/", RAW_TRACEPOINT,
8496 .attach_fn = attach_raw_tp),
8497 SEC_DEF("tp_btf/", TRACING,
8498 .expected_attach_type = BPF_TRACE_RAW_TP,
8499 .is_attach_btf = true,
8500 .attach_fn = attach_trace),
8501 SEC_DEF("fentry/", TRACING,
8502 .expected_attach_type = BPF_TRACE_FENTRY,
8503 .is_attach_btf = true,
8504 .attach_fn = attach_trace),
8505 SEC_DEF("fmod_ret/", TRACING,
8506 .expected_attach_type = BPF_MODIFY_RETURN,
8507 .is_attach_btf = true,
8508 .attach_fn = attach_trace),
8509 SEC_DEF("fexit/", TRACING,
8510 .expected_attach_type = BPF_TRACE_FEXIT,
8511 .is_attach_btf = true,
8512 .attach_fn = attach_trace),
8513 SEC_DEF("fentry.s/", TRACING,
8514 .expected_attach_type = BPF_TRACE_FENTRY,
8515 .is_attach_btf = true,
8516 .is_sleepable = true,
8517 .attach_fn = attach_trace),
8518 SEC_DEF("fmod_ret.s/", TRACING,
8519 .expected_attach_type = BPF_MODIFY_RETURN,
8520 .is_attach_btf = true,
8521 .is_sleepable = true,
8522 .attach_fn = attach_trace),
8523 SEC_DEF("fexit.s/", TRACING,
8524 .expected_attach_type = BPF_TRACE_FEXIT,
8525 .is_attach_btf = true,
8526 .is_sleepable = true,
8527 .attach_fn = attach_trace),
8528 SEC_DEF("freplace/", EXT,
8529 .is_attach_btf = true,
8530 .attach_fn = attach_trace),
8531 SEC_DEF("lsm/", LSM,
8532 .is_attach_btf = true,
8533 .expected_attach_type = BPF_LSM_MAC,
8534 .attach_fn = attach_lsm),
8535 SEC_DEF("lsm.s/", LSM,
8536 .is_attach_btf = true,
8537 .is_sleepable = true,
8538 .expected_attach_type = BPF_LSM_MAC,
8539 .attach_fn = attach_lsm),
8540 SEC_DEF("iter/", TRACING,
8541 .expected_attach_type = BPF_TRACE_ITER,
8542 .is_attach_btf = true,
8543 .attach_fn = attach_iter),
8544 BPF_EAPROG_SEC("xdp_devmap/", BPF_PROG_TYPE_XDP,
8545 BPF_XDP_DEVMAP),
8546 BPF_EAPROG_SEC("xdp_cpumap/", BPF_PROG_TYPE_XDP,
8547 BPF_XDP_CPUMAP),
8548 BPF_APROG_SEC("xdp", BPF_PROG_TYPE_XDP,
8549 BPF_XDP),
8550 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),
8551 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN),
8552 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT),
8553 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT),
8554 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL),
8555 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB,
8556 BPF_CGROUP_INET_INGRESS),
8557 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB,
8558 BPF_CGROUP_INET_EGRESS),
8559 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB),
8560 BPF_EAPROG_SEC("cgroup/sock_create", BPF_PROG_TYPE_CGROUP_SOCK,
8561 BPF_CGROUP_INET_SOCK_CREATE),
8562 BPF_EAPROG_SEC("cgroup/sock_release", BPF_PROG_TYPE_CGROUP_SOCK,
8563 BPF_CGROUP_INET_SOCK_RELEASE),
8564 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK,
8565 BPF_CGROUP_INET_SOCK_CREATE),
8566 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK,
8567 BPF_CGROUP_INET4_POST_BIND),
8568 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK,
8569 BPF_CGROUP_INET6_POST_BIND),
8570 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE,
8571 BPF_CGROUP_DEVICE),
8572 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS,
8573 BPF_CGROUP_SOCK_OPS),
8574 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB,
8575 BPF_SK_SKB_STREAM_PARSER),
8576 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB,
8577 BPF_SK_SKB_STREAM_VERDICT),
8578 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB),
8579 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG,
8580 BPF_SK_MSG_VERDICT),
8581 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2,
8582 BPF_LIRC_MODE2),
8583 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR,
8584 BPF_FLOW_DISSECTOR),
8585 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8586 BPF_CGROUP_INET4_BIND),
8587 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8588 BPF_CGROUP_INET6_BIND),
8589 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8590 BPF_CGROUP_INET4_CONNECT),
8591 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8592 BPF_CGROUP_INET6_CONNECT),
8593 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8594 BPF_CGROUP_UDP4_SENDMSG),
8595 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8596 BPF_CGROUP_UDP6_SENDMSG),
8597 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8598 BPF_CGROUP_UDP4_RECVMSG),
8599 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8600 BPF_CGROUP_UDP6_RECVMSG),
8601 BPF_EAPROG_SEC("cgroup/getpeername4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8602 BPF_CGROUP_INET4_GETPEERNAME),
8603 BPF_EAPROG_SEC("cgroup/getpeername6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8604 BPF_CGROUP_INET6_GETPEERNAME),
8605 BPF_EAPROG_SEC("cgroup/getsockname4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8606 BPF_CGROUP_INET4_GETSOCKNAME),
8607 BPF_EAPROG_SEC("cgroup/getsockname6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8608 BPF_CGROUP_INET6_GETSOCKNAME),
8609 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL,
8610 BPF_CGROUP_SYSCTL),
8611 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
8612 BPF_CGROUP_GETSOCKOPT),
8613 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
8614 BPF_CGROUP_SETSOCKOPT),
8615 BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS),
8616 BPF_EAPROG_SEC("sk_lookup/", BPF_PROG_TYPE_SK_LOOKUP,
8617 BPF_SK_LOOKUP),
8618};
8619
8620#undef BPF_PROG_SEC_IMPL
8621#undef BPF_PROG_SEC
8622#undef BPF_APROG_SEC
8623#undef BPF_EAPROG_SEC
8624#undef BPF_APROG_COMPAT
8625#undef SEC_DEF
8626
8627#define MAX_TYPE_NAME_SIZE 32
8628
8629static const struct bpf_sec_def *find_sec_def(const char *sec_name)
8630{
8631 int i, n = ARRAY_SIZE(section_defs);
8632
8633 for (i = 0; i < n; i++) {
8634 if (strncmp(sec_name,
8635 section_defs[i].sec, section_defs[i].len))
8636 continue;
8637 return §ion_defs[i];
8638 }
8639 return NULL;
8640}
8641
8642static char *libbpf_get_type_names(bool attach_type)
8643{
8644 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
8645 char *buf;
8646
8647 buf = malloc(len);
8648 if (!buf)
8649 return NULL;
8650
8651 buf[0] = '\0';
8652
8653 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8654 if (attach_type && !section_defs[i].is_attachable)
8655 continue;
8656
8657 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
8658 free(buf);
8659 return NULL;
8660 }
8661 strcat(buf, " ");
8662 strcat(buf, section_defs[i].sec);
8663 }
8664
8665 return buf;
8666}
8667
8668int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
8669 enum bpf_attach_type *expected_attach_type)
8670{
8671 const struct bpf_sec_def *sec_def;
8672 char *type_names;
8673
8674 if (!name)
8675 return -EINVAL;
8676
8677 sec_def = find_sec_def(name);
8678 if (sec_def) {
8679 *prog_type = sec_def->prog_type;
8680 *expected_attach_type = sec_def->expected_attach_type;
8681 return 0;
8682 }
8683
8684 pr_debug("failed to guess program type from ELF section '%s'\n", name);
8685 type_names = libbpf_get_type_names(false);
8686 if (type_names != NULL) {
8687 pr_debug("supported section(type) names are:%s\n", type_names);
8688 free(type_names);
8689 }
8690
8691 return -ESRCH;
8692}
8693
8694static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
8695 size_t offset)
8696{
8697 struct bpf_map *map;
8698 size_t i;
8699
8700 for (i = 0; i < obj->nr_maps; i++) {
8701 map = &obj->maps[i];
8702 if (!bpf_map__is_struct_ops(map))
8703 continue;
8704 if (map->sec_offset <= offset &&
8705 offset - map->sec_offset < map->def.value_size)
8706 return map;
8707 }
8708
8709 return NULL;
8710}
8711
8712
8713static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
8714 GElf_Shdr *shdr, Elf_Data *data)
8715{
8716 const struct btf_member *member;
8717 struct bpf_struct_ops *st_ops;
8718 struct bpf_program *prog;
8719 unsigned int shdr_idx;
8720 const struct btf *btf;
8721 struct bpf_map *map;
8722 Elf_Data *symbols;
8723 unsigned int moff, insn_idx;
8724 const char *name;
8725 __u32 member_idx;
8726 GElf_Sym sym;
8727 GElf_Rel rel;
8728 int i, nrels;
8729
8730 symbols = obj->efile.symbols;
8731 btf = obj->btf;
8732 nrels = shdr->sh_size / shdr->sh_entsize;
8733 for (i = 0; i < nrels; i++) {
8734 if (!gelf_getrel(data, i, &rel)) {
8735 pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
8736 return -LIBBPF_ERRNO__FORMAT;
8737 }
8738
8739 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
8740 pr_warn("struct_ops reloc: symbol %zx not found\n",
8741 (size_t)GELF_R_SYM(rel.r_info));
8742 return -LIBBPF_ERRNO__FORMAT;
8743 }
8744
8745 name = elf_sym_str(obj, sym.st_name) ?: "<?>";
8746 map = find_struct_ops_map_by_offset(obj, rel.r_offset);
8747 if (!map) {
8748 pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n",
8749 (size_t)rel.r_offset);
8750 return -EINVAL;
8751 }
8752
8753 moff = rel.r_offset - map->sec_offset;
8754 shdr_idx = sym.st_shndx;
8755 st_ops = map->st_ops;
8756 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
8757 map->name,
8758 (long long)(rel.r_info >> 32),
8759 (long long)sym.st_value,
8760 shdr_idx, (size_t)rel.r_offset,
8761 map->sec_offset, sym.st_name, name);
8762
8763 if (shdr_idx >= SHN_LORESERVE) {
8764 pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n",
8765 map->name, (size_t)rel.r_offset, shdr_idx);
8766 return -LIBBPF_ERRNO__RELOC;
8767 }
8768 if (sym.st_value % BPF_INSN_SZ) {
8769 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
8770 map->name, (unsigned long long)sym.st_value);
8771 return -LIBBPF_ERRNO__FORMAT;
8772 }
8773 insn_idx = sym.st_value / BPF_INSN_SZ;
8774
8775 member = find_member_by_offset(st_ops->type, moff * 8);
8776 if (!member) {
8777 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
8778 map->name, moff);
8779 return -EINVAL;
8780 }
8781 member_idx = member - btf_members(st_ops->type);
8782 name = btf__name_by_offset(btf, member->name_off);
8783
8784 if (!resolve_func_ptr(btf, member->type, NULL)) {
8785 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
8786 map->name, name);
8787 return -EINVAL;
8788 }
8789
8790 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
8791 if (!prog) {
8792 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
8793 map->name, shdr_idx, name);
8794 return -EINVAL;
8795 }
8796
8797 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
8798 const struct bpf_sec_def *sec_def;
8799
8800 sec_def = find_sec_def(prog->sec_name);
8801 if (sec_def &&
8802 sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) {
8803
8804 prog->type = sec_def->prog_type;
8805 goto invalid_prog;
8806 }
8807
8808 prog->type = BPF_PROG_TYPE_STRUCT_OPS;
8809 prog->attach_btf_id = st_ops->type_id;
8810 prog->expected_attach_type = member_idx;
8811 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
8812 prog->attach_btf_id != st_ops->type_id ||
8813 prog->expected_attach_type != member_idx) {
8814 goto invalid_prog;
8815 }
8816 st_ops->progs[member_idx] = prog;
8817 }
8818
8819 return 0;
8820
8821invalid_prog:
8822 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
8823 map->name, prog->name, prog->sec_name, prog->type,
8824 prog->attach_btf_id, prog->expected_attach_type, name);
8825 return -EINVAL;
8826}
8827
8828#define BTF_TRACE_PREFIX "btf_trace_"
8829#define BTF_LSM_PREFIX "bpf_lsm_"
8830#define BTF_ITER_PREFIX "bpf_iter_"
8831#define BTF_MAX_NAME_SIZE 128
8832
8833static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
8834 const char *name, __u32 kind)
8835{
8836 char btf_type_name[BTF_MAX_NAME_SIZE];
8837 int ret;
8838
8839 ret = snprintf(btf_type_name, sizeof(btf_type_name),
8840 "%s%s", prefix, name);
8841
8842
8843
8844
8845 if (ret < 0 || ret >= sizeof(btf_type_name))
8846 return -ENAMETOOLONG;
8847 return btf__find_by_name_kind(btf, btf_type_name, kind);
8848}
8849
8850static inline int find_attach_btf_id(struct btf *btf, const char *name,
8851 enum bpf_attach_type attach_type)
8852{
8853 int err;
8854
8855 if (attach_type == BPF_TRACE_RAW_TP)
8856 err = find_btf_by_prefix_kind(btf, BTF_TRACE_PREFIX, name,
8857 BTF_KIND_TYPEDEF);
8858 else if (attach_type == BPF_LSM_MAC)
8859 err = find_btf_by_prefix_kind(btf, BTF_LSM_PREFIX, name,
8860 BTF_KIND_FUNC);
8861 else if (attach_type == BPF_TRACE_ITER)
8862 err = find_btf_by_prefix_kind(btf, BTF_ITER_PREFIX, name,
8863 BTF_KIND_FUNC);
8864 else
8865 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
8866
8867 return err;
8868}
8869
8870int libbpf_find_vmlinux_btf_id(const char *name,
8871 enum bpf_attach_type attach_type)
8872{
8873 struct btf *btf;
8874 int err;
8875
8876 btf = libbpf_find_kernel_btf();
8877 if (IS_ERR(btf)) {
8878 pr_warn("vmlinux BTF is not found\n");
8879 return -EINVAL;
8880 }
8881
8882 err = find_attach_btf_id(btf, name, attach_type);
8883 if (err <= 0)
8884 pr_warn("%s is not found in vmlinux BTF\n", name);
8885
8886 btf__free(btf);
8887 return err;
8888}
8889
8890static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
8891{
8892 struct bpf_prog_info_linear *info_linear;
8893 struct bpf_prog_info *info;
8894 struct btf *btf = NULL;
8895 int err = -EINVAL;
8896
8897 info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0);
8898 if (IS_ERR_OR_NULL(info_linear)) {
8899 pr_warn("failed get_prog_info_linear for FD %d\n",
8900 attach_prog_fd);
8901 return -EINVAL;
8902 }
8903 info = &info_linear->info;
8904 if (!info->btf_id) {
8905 pr_warn("The target program doesn't have BTF\n");
8906 goto out;
8907 }
8908 if (btf__get_from_id(info->btf_id, &btf)) {
8909 pr_warn("Failed to get BTF of the program\n");
8910 goto out;
8911 }
8912 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
8913 btf__free(btf);
8914 if (err <= 0) {
8915 pr_warn("%s is not found in prog's BTF\n", name);
8916 goto out;
8917 }
8918out:
8919 free(info_linear);
8920 return err;
8921}
8922
8923static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
8924 enum bpf_attach_type attach_type,
8925 int *btf_obj_fd, int *btf_type_id)
8926{
8927 int ret, i;
8928
8929 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
8930 if (ret > 0) {
8931 *btf_obj_fd = 0;
8932 *btf_type_id = ret;
8933 return 0;
8934 }
8935 if (ret != -ENOENT)
8936 return ret;
8937
8938 ret = load_module_btfs(obj);
8939 if (ret)
8940 return ret;
8941
8942 for (i = 0; i < obj->btf_module_cnt; i++) {
8943 const struct module_btf *mod = &obj->btf_modules[i];
8944
8945 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
8946 if (ret > 0) {
8947 *btf_obj_fd = mod->fd;
8948 *btf_type_id = ret;
8949 return 0;
8950 }
8951 if (ret == -ENOENT)
8952 continue;
8953
8954 return ret;
8955 }
8956
8957 return -ESRCH;
8958}
8959
8960static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id)
8961{
8962 enum bpf_attach_type attach_type = prog->expected_attach_type;
8963 __u32 attach_prog_fd = prog->attach_prog_fd;
8964 const char *name = prog->sec_name, *attach_name;
8965 const struct bpf_sec_def *sec = NULL;
8966 int i, err;
8967
8968 if (!name)
8969 return -EINVAL;
8970
8971 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8972 if (!section_defs[i].is_attach_btf)
8973 continue;
8974 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
8975 continue;
8976
8977 sec = §ion_defs[i];
8978 break;
8979 }
8980
8981 if (!sec) {
8982 pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", name);
8983 return -ESRCH;
8984 }
8985 attach_name = name + sec->len;
8986
8987
8988 if (attach_prog_fd) {
8989 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
8990 if (err < 0) {
8991 pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
8992 attach_prog_fd, attach_name, err);
8993 return err;
8994 }
8995 *btf_obj_fd = 0;
8996 *btf_type_id = err;
8997 return 0;
8998 }
8999
9000
9001 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9002 if (err) {
9003 pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err);
9004 return err;
9005 }
9006 return 0;
9007}
9008
9009int libbpf_attach_type_by_name(const char *name,
9010 enum bpf_attach_type *attach_type)
9011{
9012 char *type_names;
9013 int i;
9014
9015 if (!name)
9016 return -EINVAL;
9017
9018 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
9019 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
9020 continue;
9021 if (!section_defs[i].is_attachable)
9022 return -EINVAL;
9023 *attach_type = section_defs[i].expected_attach_type;
9024 return 0;
9025 }
9026 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
9027 type_names = libbpf_get_type_names(true);
9028 if (type_names != NULL) {
9029 pr_debug("attachable section(type) names are:%s\n", type_names);
9030 free(type_names);
9031 }
9032
9033 return -EINVAL;
9034}
9035
9036int bpf_map__fd(const struct bpf_map *map)
9037{
9038 return map ? map->fd : -EINVAL;
9039}
9040
9041const struct bpf_map_def *bpf_map__def(const struct bpf_map *map)
9042{
9043 return map ? &map->def : ERR_PTR(-EINVAL);
9044}
9045
9046const char *bpf_map__name(const struct bpf_map *map)
9047{
9048 return map ? map->name : NULL;
9049}
9050
9051enum bpf_map_type bpf_map__type(const struct bpf_map *map)
9052{
9053 return map->def.type;
9054}
9055
9056int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
9057{
9058 if (map->fd >= 0)
9059 return -EBUSY;
9060 map->def.type = type;
9061 return 0;
9062}
9063
9064__u32 bpf_map__map_flags(const struct bpf_map *map)
9065{
9066 return map->def.map_flags;
9067}
9068
9069int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
9070{
9071 if (map->fd >= 0)
9072 return -EBUSY;
9073 map->def.map_flags = flags;
9074 return 0;
9075}
9076
9077__u32 bpf_map__numa_node(const struct bpf_map *map)
9078{
9079 return map->numa_node;
9080}
9081
9082int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
9083{
9084 if (map->fd >= 0)
9085 return -EBUSY;
9086 map->numa_node = numa_node;
9087 return 0;
9088}
9089
9090__u32 bpf_map__key_size(const struct bpf_map *map)
9091{
9092 return map->def.key_size;
9093}
9094
9095int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
9096{
9097 if (map->fd >= 0)
9098 return -EBUSY;
9099 map->def.key_size = size;
9100 return 0;
9101}
9102
9103__u32 bpf_map__value_size(const struct bpf_map *map)
9104{
9105 return map->def.value_size;
9106}
9107
9108int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
9109{
9110 if (map->fd >= 0)
9111 return -EBUSY;
9112 map->def.value_size = size;
9113 return 0;
9114}
9115
9116__u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
9117{
9118 return map ? map->btf_key_type_id : 0;
9119}
9120
9121__u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
9122{
9123 return map ? map->btf_value_type_id : 0;
9124}
9125
9126int bpf_map__set_priv(struct bpf_map *map, void *priv,
9127 bpf_map_clear_priv_t clear_priv)
9128{
9129 if (!map)
9130 return -EINVAL;
9131
9132 if (map->priv) {
9133 if (map->clear_priv)
9134 map->clear_priv(map, map->priv);
9135 }
9136
9137 map->priv = priv;
9138 map->clear_priv = clear_priv;
9139 return 0;
9140}
9141
9142void *bpf_map__priv(const struct bpf_map *map)
9143{
9144 return map ? map->priv : ERR_PTR(-EINVAL);
9145}
9146
9147int bpf_map__set_initial_value(struct bpf_map *map,
9148 const void *data, size_t size)
9149{
9150 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
9151 size != map->def.value_size || map->fd >= 0)
9152 return -EINVAL;
9153
9154 memcpy(map->mmaped, data, size);
9155 return 0;
9156}
9157
9158bool bpf_map__is_offload_neutral(const struct bpf_map *map)
9159{
9160 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
9161}
9162
9163bool bpf_map__is_internal(const struct bpf_map *map)
9164{
9165 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
9166}
9167
9168__u32 bpf_map__ifindex(const struct bpf_map *map)
9169{
9170 return map->map_ifindex;
9171}
9172
9173int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
9174{
9175 if (map->fd >= 0)
9176 return -EBUSY;
9177 map->map_ifindex = ifindex;
9178 return 0;
9179}
9180
9181int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
9182{
9183 if (!bpf_map_type__is_map_in_map(map->def.type)) {
9184 pr_warn("error: unsupported map type\n");
9185 return -EINVAL;
9186 }
9187 if (map->inner_map_fd != -1) {
9188 pr_warn("error: inner_map_fd already specified\n");
9189 return -EINVAL;
9190 }
9191 map->inner_map_fd = fd;
9192 return 0;
9193}
9194
9195static struct bpf_map *
9196__bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
9197{
9198 ssize_t idx;
9199 struct bpf_map *s, *e;
9200
9201 if (!obj || !obj->maps)
9202 return NULL;
9203
9204 s = obj->maps;
9205 e = obj->maps + obj->nr_maps;
9206
9207 if ((m < s) || (m >= e)) {
9208 pr_warn("error in %s: map handler doesn't belong to object\n",
9209 __func__);
9210 return NULL;
9211 }
9212
9213 idx = (m - obj->maps) + i;
9214 if (idx >= obj->nr_maps || idx < 0)
9215 return NULL;
9216 return &obj->maps[idx];
9217}
9218
9219struct bpf_map *
9220bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
9221{
9222 if (prev == NULL)
9223 return obj->maps;
9224
9225 return __bpf_map__iter(prev, obj, 1);
9226}
9227
9228struct bpf_map *
9229bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj)
9230{
9231 if (next == NULL) {
9232 if (!obj->nr_maps)
9233 return NULL;
9234 return obj->maps + obj->nr_maps - 1;
9235 }
9236
9237 return __bpf_map__iter(next, obj, -1);
9238}
9239
9240struct bpf_map *
9241bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
9242{
9243 struct bpf_map *pos;
9244
9245 bpf_object__for_each_map(pos, obj) {
9246 if (pos->name && !strcmp(pos->name, name))
9247 return pos;
9248 }
9249 return NULL;
9250}
9251
9252int
9253bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
9254{
9255 return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
9256}
9257
9258struct bpf_map *
9259bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
9260{
9261 return ERR_PTR(-ENOTSUP);
9262}
9263
9264long libbpf_get_error(const void *ptr)
9265{
9266 return PTR_ERR_OR_ZERO(ptr);
9267}
9268
9269int bpf_prog_load(const char *file, enum bpf_prog_type type,
9270 struct bpf_object **pobj, int *prog_fd)
9271{
9272 struct bpf_prog_load_attr attr;
9273
9274 memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
9275 attr.file = file;
9276 attr.prog_type = type;
9277 attr.expected_attach_type = 0;
9278
9279 return bpf_prog_load_xattr(&attr, pobj, prog_fd);
9280}
9281
9282int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
9283 struct bpf_object **pobj, int *prog_fd)
9284{
9285 struct bpf_object_open_attr open_attr = {};
9286 struct bpf_program *prog, *first_prog = NULL;
9287 struct bpf_object *obj;
9288 struct bpf_map *map;
9289 int err;
9290
9291 if (!attr)
9292 return -EINVAL;
9293 if (!attr->file)
9294 return -EINVAL;
9295
9296 open_attr.file = attr->file;
9297 open_attr.prog_type = attr->prog_type;
9298
9299 obj = bpf_object__open_xattr(&open_attr);
9300 if (IS_ERR_OR_NULL(obj))
9301 return -ENOENT;
9302
9303 bpf_object__for_each_program(prog, obj) {
9304 enum bpf_attach_type attach_type = attr->expected_attach_type;
9305
9306
9307
9308
9309
9310 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) {
9311 bpf_program__set_type(prog, attr->prog_type);
9312 bpf_program__set_expected_attach_type(prog,
9313 attach_type);
9314 }
9315 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) {
9316
9317
9318
9319
9320 bpf_object__close(obj);
9321 return -EINVAL;
9322 }
9323
9324 prog->prog_ifindex = attr->ifindex;
9325 prog->log_level = attr->log_level;
9326 prog->prog_flags |= attr->prog_flags;
9327 if (!first_prog)
9328 first_prog = prog;
9329 }
9330
9331 bpf_object__for_each_map(map, obj) {
9332 if (!bpf_map__is_offload_neutral(map))
9333 map->map_ifindex = attr->ifindex;
9334 }
9335
9336 if (!first_prog) {
9337 pr_warn("object file doesn't contain bpf program\n");
9338 bpf_object__close(obj);
9339 return -ENOENT;
9340 }
9341
9342 err = bpf_object__load(obj);
9343 if (err) {
9344 bpf_object__close(obj);
9345 return err;
9346 }
9347
9348 *pobj = obj;
9349 *prog_fd = bpf_program__fd(first_prog);
9350 return 0;
9351}
9352
9353struct bpf_link {
9354 int (*detach)(struct bpf_link *link);
9355 int (*destroy)(struct bpf_link *link);
9356 char *pin_path;
9357 int fd;
9358 bool disconnected;
9359};
9360
9361
9362int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
9363{
9364 return bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
9365}
9366
9367
9368
9369
9370
9371
9372
9373
9374
9375
9376
9377void bpf_link__disconnect(struct bpf_link *link)
9378{
9379 link->disconnected = true;
9380}
9381
9382int bpf_link__destroy(struct bpf_link *link)
9383{
9384 int err = 0;
9385
9386 if (IS_ERR_OR_NULL(link))
9387 return 0;
9388
9389 if (!link->disconnected && link->detach)
9390 err = link->detach(link);
9391 if (link->destroy)
9392 link->destroy(link);
9393 if (link->pin_path)
9394 free(link->pin_path);
9395 free(link);
9396
9397 return err;
9398}
9399
9400int bpf_link__fd(const struct bpf_link *link)
9401{
9402 return link->fd;
9403}
9404
9405const char *bpf_link__pin_path(const struct bpf_link *link)
9406{
9407 return link->pin_path;
9408}
9409
9410static int bpf_link__detach_fd(struct bpf_link *link)
9411{
9412 return close(link->fd);
9413}
9414
9415struct bpf_link *bpf_link__open(const char *path)
9416{
9417 struct bpf_link *link;
9418 int fd;
9419
9420 fd = bpf_obj_get(path);
9421 if (fd < 0) {
9422 fd = -errno;
9423 pr_warn("failed to open link at %s: %d\n", path, fd);
9424 return ERR_PTR(fd);
9425 }
9426
9427 link = calloc(1, sizeof(*link));
9428 if (!link) {
9429 close(fd);
9430 return ERR_PTR(-ENOMEM);
9431 }
9432 link->detach = &bpf_link__detach_fd;
9433 link->fd = fd;
9434
9435 link->pin_path = strdup(path);
9436 if (!link->pin_path) {
9437 bpf_link__destroy(link);
9438 return ERR_PTR(-ENOMEM);
9439 }
9440
9441 return link;
9442}
9443
9444int bpf_link__detach(struct bpf_link *link)
9445{
9446 return bpf_link_detach(link->fd) ? -errno : 0;
9447}
9448
9449int bpf_link__pin(struct bpf_link *link, const char *path)
9450{
9451 int err;
9452
9453 if (link->pin_path)
9454 return -EBUSY;
9455 err = make_parent_dir(path);
9456 if (err)
9457 return err;
9458 err = check_path(path);
9459 if (err)
9460 return err;
9461
9462 link->pin_path = strdup(path);
9463 if (!link->pin_path)
9464 return -ENOMEM;
9465
9466 if (bpf_obj_pin(link->fd, link->pin_path)) {
9467 err = -errno;
9468 zfree(&link->pin_path);
9469 return err;
9470 }
9471
9472 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
9473 return 0;
9474}
9475
9476int bpf_link__unpin(struct bpf_link *link)
9477{
9478 int err;
9479
9480 if (!link->pin_path)
9481 return -EINVAL;
9482
9483 err = unlink(link->pin_path);
9484 if (err != 0)
9485 return -errno;
9486
9487 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
9488 zfree(&link->pin_path);
9489 return 0;
9490}
9491
9492static int bpf_link__detach_perf_event(struct bpf_link *link)
9493{
9494 int err;
9495
9496 err = ioctl(link->fd, PERF_EVENT_IOC_DISABLE, 0);
9497 if (err)
9498 err = -errno;
9499
9500 close(link->fd);
9501 return err;
9502}
9503
9504struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog,
9505 int pfd)
9506{
9507 char errmsg[STRERR_BUFSIZE];
9508 struct bpf_link *link;
9509 int prog_fd, err;
9510
9511 if (pfd < 0) {
9512 pr_warn("prog '%s': invalid perf event FD %d\n",
9513 prog->name, pfd);
9514 return ERR_PTR(-EINVAL);
9515 }
9516 prog_fd = bpf_program__fd(prog);
9517 if (prog_fd < 0) {
9518 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
9519 prog->name);
9520 return ERR_PTR(-EINVAL);
9521 }
9522
9523 link = calloc(1, sizeof(*link));
9524 if (!link)
9525 return ERR_PTR(-ENOMEM);
9526 link->detach = &bpf_link__detach_perf_event;
9527 link->fd = pfd;
9528
9529 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
9530 err = -errno;
9531 free(link);
9532 pr_warn("prog '%s': failed to attach to pfd %d: %s\n",
9533 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9534 if (err == -EPROTO)
9535 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
9536 prog->name, pfd);
9537 return ERR_PTR(err);
9538 }
9539 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
9540 err = -errno;
9541 free(link);
9542 pr_warn("prog '%s': failed to enable pfd %d: %s\n",
9543 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9544 return ERR_PTR(err);
9545 }
9546 return link;
9547}
9548
9549
9550
9551
9552
9553
9554static int parse_uint_from_file(const char *file, const char *fmt)
9555{
9556 char buf[STRERR_BUFSIZE];
9557 int err, ret;
9558 FILE *f;
9559
9560 f = fopen(file, "r");
9561 if (!f) {
9562 err = -errno;
9563 pr_debug("failed to open '%s': %s\n", file,
9564 libbpf_strerror_r(err, buf, sizeof(buf)));
9565 return err;
9566 }
9567 err = fscanf(f, fmt, &ret);
9568 if (err != 1) {
9569 err = err == EOF ? -EIO : -errno;
9570 pr_debug("failed to parse '%s': %s\n", file,
9571 libbpf_strerror_r(err, buf, sizeof(buf)));
9572 fclose(f);
9573 return err;
9574 }
9575 fclose(f);
9576 return ret;
9577}
9578
9579static int determine_kprobe_perf_type(void)
9580{
9581 const char *file = "/sys/bus/event_source/devices/kprobe/type";
9582
9583 return parse_uint_from_file(file, "%d\n");
9584}
9585
9586static int determine_uprobe_perf_type(void)
9587{
9588 const char *file = "/sys/bus/event_source/devices/uprobe/type";
9589
9590 return parse_uint_from_file(file, "%d\n");
9591}
9592
9593static int determine_kprobe_retprobe_bit(void)
9594{
9595 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
9596
9597 return parse_uint_from_file(file, "config:%d\n");
9598}
9599
9600static int determine_uprobe_retprobe_bit(void)
9601{
9602 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
9603
9604 return parse_uint_from_file(file, "config:%d\n");
9605}
9606
9607static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
9608 uint64_t offset, int pid)
9609{
9610 struct perf_event_attr attr = {};
9611 char errmsg[STRERR_BUFSIZE];
9612 int type, pfd, err;
9613
9614 type = uprobe ? determine_uprobe_perf_type()
9615 : determine_kprobe_perf_type();
9616 if (type < 0) {
9617 pr_warn("failed to determine %s perf type: %s\n",
9618 uprobe ? "uprobe" : "kprobe",
9619 libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
9620 return type;
9621 }
9622 if (retprobe) {
9623 int bit = uprobe ? determine_uprobe_retprobe_bit()
9624 : determine_kprobe_retprobe_bit();
9625
9626 if (bit < 0) {
9627 pr_warn("failed to determine %s retprobe bit: %s\n",
9628 uprobe ? "uprobe" : "kprobe",
9629 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
9630 return bit;
9631 }
9632 attr.config |= 1 << bit;
9633 }
9634 attr.size = sizeof(attr);
9635 attr.type = type;
9636 attr.config1 = ptr_to_u64(name);
9637 attr.config2 = offset;
9638
9639
9640 pfd = syscall(__NR_perf_event_open, &attr,
9641 pid < 0 ? -1 : pid ,
9642 pid == -1 ? 0 : -1 ,
9643 -1 , PERF_FLAG_FD_CLOEXEC);
9644 if (pfd < 0) {
9645 err = -errno;
9646 pr_warn("%s perf_event_open() failed: %s\n",
9647 uprobe ? "uprobe" : "kprobe",
9648 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9649 return err;
9650 }
9651 return pfd;
9652}
9653
9654struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog,
9655 bool retprobe,
9656 const char *func_name)
9657{
9658 char errmsg[STRERR_BUFSIZE];
9659 struct bpf_link *link;
9660 int pfd, err;
9661
9662 pfd = perf_event_open_probe(false , retprobe, func_name,
9663 0 , -1 );
9664 if (pfd < 0) {
9665 pr_warn("prog '%s': failed to create %s '%s' perf event: %s\n",
9666 prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
9667 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9668 return ERR_PTR(pfd);
9669 }
9670 link = bpf_program__attach_perf_event(prog, pfd);
9671 if (IS_ERR(link)) {
9672 close(pfd);
9673 err = PTR_ERR(link);
9674 pr_warn("prog '%s': failed to attach to %s '%s': %s\n",
9675 prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
9676 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9677 return link;
9678 }
9679 return link;
9680}
9681
9682static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
9683 struct bpf_program *prog)
9684{
9685 const char *func_name;
9686 bool retprobe;
9687
9688 func_name = prog->sec_name + sec->len;
9689 retprobe = strcmp(sec->sec, "kretprobe/") == 0;
9690
9691 return bpf_program__attach_kprobe(prog, retprobe, func_name);
9692}
9693
9694struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog,
9695 bool retprobe, pid_t pid,
9696 const char *binary_path,
9697 size_t func_offset)
9698{
9699 char errmsg[STRERR_BUFSIZE];
9700 struct bpf_link *link;
9701 int pfd, err;
9702
9703 pfd = perf_event_open_probe(true , retprobe,
9704 binary_path, func_offset, pid);
9705 if (pfd < 0) {
9706 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
9707 prog->name, retprobe ? "uretprobe" : "uprobe",
9708 binary_path, func_offset,
9709 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9710 return ERR_PTR(pfd);
9711 }
9712 link = bpf_program__attach_perf_event(prog, pfd);
9713 if (IS_ERR(link)) {
9714 close(pfd);
9715 err = PTR_ERR(link);
9716 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
9717 prog->name, retprobe ? "uretprobe" : "uprobe",
9718 binary_path, func_offset,
9719 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9720 return link;
9721 }
9722 return link;
9723}
9724
9725static int determine_tracepoint_id(const char *tp_category,
9726 const char *tp_name)
9727{
9728 char file[PATH_MAX];
9729 int ret;
9730
9731 ret = snprintf(file, sizeof(file),
9732 "/sys/kernel/debug/tracing/events/%s/%s/id",
9733 tp_category, tp_name);
9734 if (ret < 0)
9735 return -errno;
9736 if (ret >= sizeof(file)) {
9737 pr_debug("tracepoint %s/%s path is too long\n",
9738 tp_category, tp_name);
9739 return -E2BIG;
9740 }
9741 return parse_uint_from_file(file, "%d\n");
9742}
9743
9744static int perf_event_open_tracepoint(const char *tp_category,
9745 const char *tp_name)
9746{
9747 struct perf_event_attr attr = {};
9748 char errmsg[STRERR_BUFSIZE];
9749 int tp_id, pfd, err;
9750
9751 tp_id = determine_tracepoint_id(tp_category, tp_name);
9752 if (tp_id < 0) {
9753 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
9754 tp_category, tp_name,
9755 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
9756 return tp_id;
9757 }
9758
9759 attr.type = PERF_TYPE_TRACEPOINT;
9760 attr.size = sizeof(attr);
9761 attr.config = tp_id;
9762
9763 pfd = syscall(__NR_perf_event_open, &attr, -1 , 0 ,
9764 -1 , PERF_FLAG_FD_CLOEXEC);
9765 if (pfd < 0) {
9766 err = -errno;
9767 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
9768 tp_category, tp_name,
9769 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9770 return err;
9771 }
9772 return pfd;
9773}
9774
9775struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,
9776 const char *tp_category,
9777 const char *tp_name)
9778{
9779 char errmsg[STRERR_BUFSIZE];
9780 struct bpf_link *link;
9781 int pfd, err;
9782
9783 pfd = perf_event_open_tracepoint(tp_category, tp_name);
9784 if (pfd < 0) {
9785 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
9786 prog->name, tp_category, tp_name,
9787 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9788 return ERR_PTR(pfd);
9789 }
9790 link = bpf_program__attach_perf_event(prog, pfd);
9791 if (IS_ERR(link)) {
9792 close(pfd);
9793 err = PTR_ERR(link);
9794 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
9795 prog->name, tp_category, tp_name,
9796 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9797 return link;
9798 }
9799 return link;
9800}
9801
9802static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
9803 struct bpf_program *prog)
9804{
9805 char *sec_name, *tp_cat, *tp_name;
9806 struct bpf_link *link;
9807
9808 sec_name = strdup(prog->sec_name);
9809 if (!sec_name)
9810 return ERR_PTR(-ENOMEM);
9811
9812
9813 tp_cat = sec_name + sec->len;
9814 tp_name = strchr(tp_cat, '/');
9815 if (!tp_name) {
9816 link = ERR_PTR(-EINVAL);
9817 goto out;
9818 }
9819 *tp_name = '\0';
9820 tp_name++;
9821
9822 link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
9823out:
9824 free(sec_name);
9825 return link;
9826}
9827
9828struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
9829 const char *tp_name)
9830{
9831 char errmsg[STRERR_BUFSIZE];
9832 struct bpf_link *link;
9833 int prog_fd, pfd;
9834
9835 prog_fd = bpf_program__fd(prog);
9836 if (prog_fd < 0) {
9837 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9838 return ERR_PTR(-EINVAL);
9839 }
9840
9841 link = calloc(1, sizeof(*link));
9842 if (!link)
9843 return ERR_PTR(-ENOMEM);
9844 link->detach = &bpf_link__detach_fd;
9845
9846 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
9847 if (pfd < 0) {
9848 pfd = -errno;
9849 free(link);
9850 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
9851 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9852 return ERR_PTR(pfd);
9853 }
9854 link->fd = pfd;
9855 return link;
9856}
9857
9858static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
9859 struct bpf_program *prog)
9860{
9861 const char *tp_name = prog->sec_name + sec->len;
9862
9863 return bpf_program__attach_raw_tracepoint(prog, tp_name);
9864}
9865
9866
9867static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog)
9868{
9869 char errmsg[STRERR_BUFSIZE];
9870 struct bpf_link *link;
9871 int prog_fd, pfd;
9872
9873 prog_fd = bpf_program__fd(prog);
9874 if (prog_fd < 0) {
9875 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9876 return ERR_PTR(-EINVAL);
9877 }
9878
9879 link = calloc(1, sizeof(*link));
9880 if (!link)
9881 return ERR_PTR(-ENOMEM);
9882 link->detach = &bpf_link__detach_fd;
9883
9884 pfd = bpf_raw_tracepoint_open(NULL, prog_fd);
9885 if (pfd < 0) {
9886 pfd = -errno;
9887 free(link);
9888 pr_warn("prog '%s': failed to attach: %s\n",
9889 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9890 return ERR_PTR(pfd);
9891 }
9892 link->fd = pfd;
9893 return (struct bpf_link *)link;
9894}
9895
9896struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog)
9897{
9898 return bpf_program__attach_btf_id(prog);
9899}
9900
9901struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog)
9902{
9903 return bpf_program__attach_btf_id(prog);
9904}
9905
9906static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
9907 struct bpf_program *prog)
9908{
9909 return bpf_program__attach_trace(prog);
9910}
9911
9912static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
9913 struct bpf_program *prog)
9914{
9915 return bpf_program__attach_lsm(prog);
9916}
9917
9918static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
9919 struct bpf_program *prog)
9920{
9921 return bpf_program__attach_iter(prog, NULL);
9922}
9923
9924static struct bpf_link *
9925bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id,
9926 const char *target_name)
9927{
9928 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
9929 .target_btf_id = btf_id);
9930 enum bpf_attach_type attach_type;
9931 char errmsg[STRERR_BUFSIZE];
9932 struct bpf_link *link;
9933 int prog_fd, link_fd;
9934
9935 prog_fd = bpf_program__fd(prog);
9936 if (prog_fd < 0) {
9937 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9938 return ERR_PTR(-EINVAL);
9939 }
9940
9941 link = calloc(1, sizeof(*link));
9942 if (!link)
9943 return ERR_PTR(-ENOMEM);
9944 link->detach = &bpf_link__detach_fd;
9945
9946 attach_type = bpf_program__get_expected_attach_type(prog);
9947 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
9948 if (link_fd < 0) {
9949 link_fd = -errno;
9950 free(link);
9951 pr_warn("prog '%s': failed to attach to %s: %s\n",
9952 prog->name, target_name,
9953 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
9954 return ERR_PTR(link_fd);
9955 }
9956 link->fd = link_fd;
9957 return link;
9958}
9959
9960struct bpf_link *
9961bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd)
9962{
9963 return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
9964}
9965
9966struct bpf_link *
9967bpf_program__attach_netns(struct bpf_program *prog, int netns_fd)
9968{
9969 return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
9970}
9971
9972struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex)
9973{
9974
9975 return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
9976}
9977
9978struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog,
9979 int target_fd,
9980 const char *attach_func_name)
9981{
9982 int btf_id;
9983
9984 if (!!target_fd != !!attach_func_name) {
9985 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
9986 prog->name);
9987 return ERR_PTR(-EINVAL);
9988 }
9989
9990 if (prog->type != BPF_PROG_TYPE_EXT) {
9991 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
9992 prog->name);
9993 return ERR_PTR(-EINVAL);
9994 }
9995
9996 if (target_fd) {
9997 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
9998 if (btf_id < 0)
9999 return ERR_PTR(btf_id);
10000
10001 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
10002 } else {
10003
10004
10005
10006 return bpf_program__attach_trace(prog);
10007 }
10008}
10009
10010struct bpf_link *
10011bpf_program__attach_iter(struct bpf_program *prog,
10012 const struct bpf_iter_attach_opts *opts)
10013{
10014 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
10015 char errmsg[STRERR_BUFSIZE];
10016 struct bpf_link *link;
10017 int prog_fd, link_fd;
10018 __u32 target_fd = 0;
10019
10020 if (!OPTS_VALID(opts, bpf_iter_attach_opts))
10021 return ERR_PTR(-EINVAL);
10022
10023 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
10024 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
10025
10026 prog_fd = bpf_program__fd(prog);
10027 if (prog_fd < 0) {
10028 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
10029 return ERR_PTR(-EINVAL);
10030 }
10031
10032 link = calloc(1, sizeof(*link));
10033 if (!link)
10034 return ERR_PTR(-ENOMEM);
10035 link->detach = &bpf_link__detach_fd;
10036
10037 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
10038 &link_create_opts);
10039 if (link_fd < 0) {
10040 link_fd = -errno;
10041 free(link);
10042 pr_warn("prog '%s': failed to attach to iterator: %s\n",
10043 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
10044 return ERR_PTR(link_fd);
10045 }
10046 link->fd = link_fd;
10047 return link;
10048}
10049
10050struct bpf_link *bpf_program__attach(struct bpf_program *prog)
10051{
10052 const struct bpf_sec_def *sec_def;
10053
10054 sec_def = find_sec_def(prog->sec_name);
10055 if (!sec_def || !sec_def->attach_fn)
10056 return ERR_PTR(-ESRCH);
10057
10058 return sec_def->attach_fn(sec_def, prog);
10059}
10060
10061static int bpf_link__detach_struct_ops(struct bpf_link *link)
10062{
10063 __u32 zero = 0;
10064
10065 if (bpf_map_delete_elem(link->fd, &zero))
10066 return -errno;
10067
10068 return 0;
10069}
10070
10071struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map)
10072{
10073 struct bpf_struct_ops *st_ops;
10074 struct bpf_link *link;
10075 __u32 i, zero = 0;
10076 int err;
10077
10078 if (!bpf_map__is_struct_ops(map) || map->fd == -1)
10079 return ERR_PTR(-EINVAL);
10080
10081 link = calloc(1, sizeof(*link));
10082 if (!link)
10083 return ERR_PTR(-EINVAL);
10084
10085 st_ops = map->st_ops;
10086 for (i = 0; i < btf_vlen(st_ops->type); i++) {
10087 struct bpf_program *prog = st_ops->progs[i];
10088 void *kern_data;
10089 int prog_fd;
10090
10091 if (!prog)
10092 continue;
10093
10094 prog_fd = bpf_program__fd(prog);
10095 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
10096 *(unsigned long *)kern_data = prog_fd;
10097 }
10098
10099 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
10100 if (err) {
10101 err = -errno;
10102 free(link);
10103 return ERR_PTR(err);
10104 }
10105
10106 link->detach = bpf_link__detach_struct_ops;
10107 link->fd = map->fd;
10108
10109 return link;
10110}
10111
10112enum bpf_perf_event_ret
10113bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
10114 void **copy_mem, size_t *copy_size,
10115 bpf_perf_event_print_t fn, void *private_data)
10116{
10117 struct perf_event_mmap_page *header = mmap_mem;
10118 __u64 data_head = ring_buffer_read_head(header);
10119 __u64 data_tail = header->data_tail;
10120 void *base = ((__u8 *)header) + page_size;
10121 int ret = LIBBPF_PERF_EVENT_CONT;
10122 struct perf_event_header *ehdr;
10123 size_t ehdr_size;
10124
10125 while (data_head != data_tail) {
10126 ehdr = base + (data_tail & (mmap_size - 1));
10127 ehdr_size = ehdr->size;
10128
10129 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
10130 void *copy_start = ehdr;
10131 size_t len_first = base + mmap_size - copy_start;
10132 size_t len_secnd = ehdr_size - len_first;
10133
10134 if (*copy_size < ehdr_size) {
10135 free(*copy_mem);
10136 *copy_mem = malloc(ehdr_size);
10137 if (!*copy_mem) {
10138 *copy_size = 0;
10139 ret = LIBBPF_PERF_EVENT_ERROR;
10140 break;
10141 }
10142 *copy_size = ehdr_size;
10143 }
10144
10145 memcpy(*copy_mem, copy_start, len_first);
10146 memcpy(*copy_mem + len_first, base, len_secnd);
10147 ehdr = *copy_mem;
10148 }
10149
10150 ret = fn(ehdr, private_data);
10151 data_tail += ehdr_size;
10152 if (ret != LIBBPF_PERF_EVENT_CONT)
10153 break;
10154 }
10155
10156 ring_buffer_write_tail(header, data_tail);
10157 return ret;
10158}
10159
10160struct perf_buffer;
10161
10162struct perf_buffer_params {
10163 struct perf_event_attr *attr;
10164
10165 perf_buffer_event_fn event_cb;
10166
10167 perf_buffer_sample_fn sample_cb;
10168 perf_buffer_lost_fn lost_cb;
10169 void *ctx;
10170 int cpu_cnt;
10171 int *cpus;
10172 int *map_keys;
10173};
10174
10175struct perf_cpu_buf {
10176 struct perf_buffer *pb;
10177 void *base;
10178 void *buf;
10179 size_t buf_size;
10180 int fd;
10181 int cpu;
10182 int map_key;
10183};
10184
10185struct perf_buffer {
10186 perf_buffer_event_fn event_cb;
10187 perf_buffer_sample_fn sample_cb;
10188 perf_buffer_lost_fn lost_cb;
10189 void *ctx;
10190
10191 size_t page_size;
10192 size_t mmap_size;
10193 struct perf_cpu_buf **cpu_bufs;
10194 struct epoll_event *events;
10195 int cpu_cnt;
10196 int epoll_fd;
10197 int map_fd;
10198};
10199
10200static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
10201 struct perf_cpu_buf *cpu_buf)
10202{
10203 if (!cpu_buf)
10204 return;
10205 if (cpu_buf->base &&
10206 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
10207 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
10208 if (cpu_buf->fd >= 0) {
10209 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
10210 close(cpu_buf->fd);
10211 }
10212 free(cpu_buf->buf);
10213 free(cpu_buf);
10214}
10215
10216void perf_buffer__free(struct perf_buffer *pb)
10217{
10218 int i;
10219
10220 if (IS_ERR_OR_NULL(pb))
10221 return;
10222 if (pb->cpu_bufs) {
10223 for (i = 0; i < pb->cpu_cnt; i++) {
10224 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
10225
10226 if (!cpu_buf)
10227 continue;
10228
10229 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
10230 perf_buffer__free_cpu_buf(pb, cpu_buf);
10231 }
10232 free(pb->cpu_bufs);
10233 }
10234 if (pb->epoll_fd >= 0)
10235 close(pb->epoll_fd);
10236 free(pb->events);
10237 free(pb);
10238}
10239
10240static struct perf_cpu_buf *
10241perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
10242 int cpu, int map_key)
10243{
10244 struct perf_cpu_buf *cpu_buf;
10245 char msg[STRERR_BUFSIZE];
10246 int err;
10247
10248 cpu_buf = calloc(1, sizeof(*cpu_buf));
10249 if (!cpu_buf)
10250 return ERR_PTR(-ENOMEM);
10251
10252 cpu_buf->pb = pb;
10253 cpu_buf->cpu = cpu;
10254 cpu_buf->map_key = map_key;
10255
10256 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 , cpu,
10257 -1, PERF_FLAG_FD_CLOEXEC);
10258 if (cpu_buf->fd < 0) {
10259 err = -errno;
10260 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
10261 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10262 goto error;
10263 }
10264
10265 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
10266 PROT_READ | PROT_WRITE, MAP_SHARED,
10267 cpu_buf->fd, 0);
10268 if (cpu_buf->base == MAP_FAILED) {
10269 cpu_buf->base = NULL;
10270 err = -errno;
10271 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
10272 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10273 goto error;
10274 }
10275
10276 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
10277 err = -errno;
10278 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
10279 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
10280 goto error;
10281 }
10282
10283 return cpu_buf;
10284
10285error:
10286 perf_buffer__free_cpu_buf(pb, cpu_buf);
10287 return (struct perf_cpu_buf *)ERR_PTR(err);
10288}
10289
10290static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
10291 struct perf_buffer_params *p);
10292
10293struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
10294 const struct perf_buffer_opts *opts)
10295{
10296 struct perf_buffer_params p = {};
10297 struct perf_event_attr attr = { 0, };
10298
10299 attr.config = PERF_COUNT_SW_BPF_OUTPUT;
10300 attr.type = PERF_TYPE_SOFTWARE;
10301 attr.sample_type = PERF_SAMPLE_RAW;
10302 attr.sample_period = 1;
10303 attr.wakeup_events = 1;
10304
10305 p.attr = &attr;
10306 p.sample_cb = opts ? opts->sample_cb : NULL;
10307 p.lost_cb = opts ? opts->lost_cb : NULL;
10308 p.ctx = opts ? opts->ctx : NULL;
10309
10310 return __perf_buffer__new(map_fd, page_cnt, &p);
10311}
10312
10313struct perf_buffer *
10314perf_buffer__new_raw(int map_fd, size_t page_cnt,
10315 const struct perf_buffer_raw_opts *opts)
10316{
10317 struct perf_buffer_params p = {};
10318
10319 p.attr = opts->attr;
10320 p.event_cb = opts->event_cb;
10321 p.ctx = opts->ctx;
10322 p.cpu_cnt = opts->cpu_cnt;
10323 p.cpus = opts->cpus;
10324 p.map_keys = opts->map_keys;
10325
10326 return __perf_buffer__new(map_fd, page_cnt, &p);
10327}
10328
10329static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
10330 struct perf_buffer_params *p)
10331{
10332 const char *online_cpus_file = "/sys/devices/system/cpu/online";
10333 struct bpf_map_info map;
10334 char msg[STRERR_BUFSIZE];
10335 struct perf_buffer *pb;
10336 bool *online = NULL;
10337 __u32 map_info_len;
10338 int err, i, j, n;
10339
10340 if (page_cnt & (page_cnt - 1)) {
10341 pr_warn("page count should be power of two, but is %zu\n",
10342 page_cnt);
10343 return ERR_PTR(-EINVAL);
10344 }
10345
10346
10347 memset(&map, 0, sizeof(map));
10348 map_info_len = sizeof(map);
10349 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
10350 if (err) {
10351 err = -errno;
10352
10353
10354
10355 if (err != -EINVAL) {
10356 pr_warn("failed to get map info for map FD %d: %s\n",
10357 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
10358 return ERR_PTR(err);
10359 }
10360 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
10361 map_fd);
10362 } else {
10363 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
10364 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
10365 map.name);
10366 return ERR_PTR(-EINVAL);
10367 }
10368 }
10369
10370 pb = calloc(1, sizeof(*pb));
10371 if (!pb)
10372 return ERR_PTR(-ENOMEM);
10373
10374 pb->event_cb = p->event_cb;
10375 pb->sample_cb = p->sample_cb;
10376 pb->lost_cb = p->lost_cb;
10377 pb->ctx = p->ctx;
10378
10379 pb->page_size = getpagesize();
10380 pb->mmap_size = pb->page_size * page_cnt;
10381 pb->map_fd = map_fd;
10382
10383 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
10384 if (pb->epoll_fd < 0) {
10385 err = -errno;
10386 pr_warn("failed to create epoll instance: %s\n",
10387 libbpf_strerror_r(err, msg, sizeof(msg)));
10388 goto error;
10389 }
10390
10391 if (p->cpu_cnt > 0) {
10392 pb->cpu_cnt = p->cpu_cnt;
10393 } else {
10394 pb->cpu_cnt = libbpf_num_possible_cpus();
10395 if (pb->cpu_cnt < 0) {
10396 err = pb->cpu_cnt;
10397 goto error;
10398 }
10399 if (map.max_entries && map.max_entries < pb->cpu_cnt)
10400 pb->cpu_cnt = map.max_entries;
10401 }
10402
10403 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
10404 if (!pb->events) {
10405 err = -ENOMEM;
10406 pr_warn("failed to allocate events: out of memory\n");
10407 goto error;
10408 }
10409 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
10410 if (!pb->cpu_bufs) {
10411 err = -ENOMEM;
10412 pr_warn("failed to allocate buffers: out of memory\n");
10413 goto error;
10414 }
10415
10416 err = parse_cpu_mask_file(online_cpus_file, &online, &n);
10417 if (err) {
10418 pr_warn("failed to get online CPU mask: %d\n", err);
10419 goto error;
10420 }
10421
10422 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
10423 struct perf_cpu_buf *cpu_buf;
10424 int cpu, map_key;
10425
10426 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
10427 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
10428
10429
10430
10431
10432 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
10433 continue;
10434
10435 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
10436 if (IS_ERR(cpu_buf)) {
10437 err = PTR_ERR(cpu_buf);
10438 goto error;
10439 }
10440
10441 pb->cpu_bufs[j] = cpu_buf;
10442
10443 err = bpf_map_update_elem(pb->map_fd, &map_key,
10444 &cpu_buf->fd, 0);
10445 if (err) {
10446 err = -errno;
10447 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
10448 cpu, map_key, cpu_buf->fd,
10449 libbpf_strerror_r(err, msg, sizeof(msg)));
10450 goto error;
10451 }
10452
10453 pb->events[j].events = EPOLLIN;
10454 pb->events[j].data.ptr = cpu_buf;
10455 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
10456 &pb->events[j]) < 0) {
10457 err = -errno;
10458 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
10459 cpu, cpu_buf->fd,
10460 libbpf_strerror_r(err, msg, sizeof(msg)));
10461 goto error;
10462 }
10463 j++;
10464 }
10465 pb->cpu_cnt = j;
10466 free(online);
10467
10468 return pb;
10469
10470error:
10471 free(online);
10472 if (pb)
10473 perf_buffer__free(pb);
10474 return ERR_PTR(err);
10475}
10476
10477struct perf_sample_raw {
10478 struct perf_event_header header;
10479 uint32_t size;
10480 char data[];
10481};
10482
10483struct perf_sample_lost {
10484 struct perf_event_header header;
10485 uint64_t id;
10486 uint64_t lost;
10487 uint64_t sample_id;
10488};
10489
10490static enum bpf_perf_event_ret
10491perf_buffer__process_record(struct perf_event_header *e, void *ctx)
10492{
10493 struct perf_cpu_buf *cpu_buf = ctx;
10494 struct perf_buffer *pb = cpu_buf->pb;
10495 void *data = e;
10496
10497
10498 if (pb->event_cb)
10499 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
10500
10501 switch (e->type) {
10502 case PERF_RECORD_SAMPLE: {
10503 struct perf_sample_raw *s = data;
10504
10505 if (pb->sample_cb)
10506 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
10507 break;
10508 }
10509 case PERF_RECORD_LOST: {
10510 struct perf_sample_lost *s = data;
10511
10512 if (pb->lost_cb)
10513 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
10514 break;
10515 }
10516 default:
10517 pr_warn("unknown perf sample type %d\n", e->type);
10518 return LIBBPF_PERF_EVENT_ERROR;
10519 }
10520 return LIBBPF_PERF_EVENT_CONT;
10521}
10522
10523static int perf_buffer__process_records(struct perf_buffer *pb,
10524 struct perf_cpu_buf *cpu_buf)
10525{
10526 enum bpf_perf_event_ret ret;
10527
10528 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,
10529 pb->page_size, &cpu_buf->buf,
10530 &cpu_buf->buf_size,
10531 perf_buffer__process_record, cpu_buf);
10532 if (ret != LIBBPF_PERF_EVENT_CONT)
10533 return ret;
10534 return 0;
10535}
10536
10537int perf_buffer__epoll_fd(const struct perf_buffer *pb)
10538{
10539 return pb->epoll_fd;
10540}
10541
10542int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
10543{
10544 int i, cnt, err;
10545
10546 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
10547 for (i = 0; i < cnt; i++) {
10548 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
10549
10550 err = perf_buffer__process_records(pb, cpu_buf);
10551 if (err) {
10552 pr_warn("error while processing records: %d\n", err);
10553 return err;
10554 }
10555 }
10556 return cnt < 0 ? -errno : cnt;
10557}
10558
10559
10560
10561
10562size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
10563{
10564 return pb->cpu_cnt;
10565}
10566
10567
10568
10569
10570
10571
10572int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
10573{
10574 struct perf_cpu_buf *cpu_buf;
10575
10576 if (buf_idx >= pb->cpu_cnt)
10577 return -EINVAL;
10578
10579 cpu_buf = pb->cpu_bufs[buf_idx];
10580 if (!cpu_buf)
10581 return -ENOENT;
10582
10583 return cpu_buf->fd;
10584}
10585
10586
10587
10588
10589
10590
10591
10592
10593
10594int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
10595{
10596 struct perf_cpu_buf *cpu_buf;
10597
10598 if (buf_idx >= pb->cpu_cnt)
10599 return -EINVAL;
10600
10601 cpu_buf = pb->cpu_bufs[buf_idx];
10602 if (!cpu_buf)
10603 return -ENOENT;
10604
10605 return perf_buffer__process_records(pb, cpu_buf);
10606}
10607
10608int perf_buffer__consume(struct perf_buffer *pb)
10609{
10610 int i, err;
10611
10612 for (i = 0; i < pb->cpu_cnt; i++) {
10613 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
10614
10615 if (!cpu_buf)
10616 continue;
10617
10618 err = perf_buffer__process_records(pb, cpu_buf);
10619 if (err) {
10620 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
10621 return err;
10622 }
10623 }
10624 return 0;
10625}
10626
10627struct bpf_prog_info_array_desc {
10628 int array_offset;
10629 int count_offset;
10630 int size_offset;
10631
10632
10633};
10634
10635static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = {
10636 [BPF_PROG_INFO_JITED_INSNS] = {
10637 offsetof(struct bpf_prog_info, jited_prog_insns),
10638 offsetof(struct bpf_prog_info, jited_prog_len),
10639 -1,
10640 },
10641 [BPF_PROG_INFO_XLATED_INSNS] = {
10642 offsetof(struct bpf_prog_info, xlated_prog_insns),
10643 offsetof(struct bpf_prog_info, xlated_prog_len),
10644 -1,
10645 },
10646 [BPF_PROG_INFO_MAP_IDS] = {
10647 offsetof(struct bpf_prog_info, map_ids),
10648 offsetof(struct bpf_prog_info, nr_map_ids),
10649 -(int)sizeof(__u32),
10650 },
10651 [BPF_PROG_INFO_JITED_KSYMS] = {
10652 offsetof(struct bpf_prog_info, jited_ksyms),
10653 offsetof(struct bpf_prog_info, nr_jited_ksyms),
10654 -(int)sizeof(__u64),
10655 },
10656 [BPF_PROG_INFO_JITED_FUNC_LENS] = {
10657 offsetof(struct bpf_prog_info, jited_func_lens),
10658 offsetof(struct bpf_prog_info, nr_jited_func_lens),
10659 -(int)sizeof(__u32),
10660 },
10661 [BPF_PROG_INFO_FUNC_INFO] = {
10662 offsetof(struct bpf_prog_info, func_info),
10663 offsetof(struct bpf_prog_info, nr_func_info),
10664 offsetof(struct bpf_prog_info, func_info_rec_size),
10665 },
10666 [BPF_PROG_INFO_LINE_INFO] = {
10667 offsetof(struct bpf_prog_info, line_info),
10668 offsetof(struct bpf_prog_info, nr_line_info),
10669 offsetof(struct bpf_prog_info, line_info_rec_size),
10670 },
10671 [BPF_PROG_INFO_JITED_LINE_INFO] = {
10672 offsetof(struct bpf_prog_info, jited_line_info),
10673 offsetof(struct bpf_prog_info, nr_jited_line_info),
10674 offsetof(struct bpf_prog_info, jited_line_info_rec_size),
10675 },
10676 [BPF_PROG_INFO_PROG_TAGS] = {
10677 offsetof(struct bpf_prog_info, prog_tags),
10678 offsetof(struct bpf_prog_info, nr_prog_tags),
10679 -(int)sizeof(__u8) * BPF_TAG_SIZE,
10680 },
10681
10682};
10683
10684static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,
10685 int offset)
10686{
10687 __u32 *array = (__u32 *)info;
10688
10689 if (offset >= 0)
10690 return array[offset / sizeof(__u32)];
10691 return -(int)offset;
10692}
10693
10694static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,
10695 int offset)
10696{
10697 __u64 *array = (__u64 *)info;
10698
10699 if (offset >= 0)
10700 return array[offset / sizeof(__u64)];
10701 return -(int)offset;
10702}
10703
10704static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
10705 __u32 val)
10706{
10707 __u32 *array = (__u32 *)info;
10708
10709 if (offset >= 0)
10710 array[offset / sizeof(__u32)] = val;
10711}
10712
10713static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
10714 __u64 val)
10715{
10716 __u64 *array = (__u64 *)info;
10717
10718 if (offset >= 0)
10719 array[offset / sizeof(__u64)] = val;
10720}
10721
10722struct bpf_prog_info_linear *
10723bpf_program__get_prog_info_linear(int fd, __u64 arrays)
10724{
10725 struct bpf_prog_info_linear *info_linear;
10726 struct bpf_prog_info info = {};
10727 __u32 info_len = sizeof(info);
10728 __u32 data_len = 0;
10729 int i, err;
10730 void *ptr;
10731
10732 if (arrays >> BPF_PROG_INFO_LAST_ARRAY)
10733 return ERR_PTR(-EINVAL);
10734
10735
10736 err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
10737 if (err) {
10738 pr_debug("can't get prog info: %s", strerror(errno));
10739 return ERR_PTR(-EFAULT);
10740 }
10741
10742
10743 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10744 bool include_array = (arrays & (1UL << i)) > 0;
10745 struct bpf_prog_info_array_desc *desc;
10746 __u32 count, size;
10747
10748 desc = bpf_prog_info_array_desc + i;
10749
10750
10751 if (info_len < desc->array_offset + sizeof(__u32) ||
10752 info_len < desc->count_offset + sizeof(__u32) ||
10753 (desc->size_offset > 0 && info_len < desc->size_offset))
10754 include_array = false;
10755
10756 if (!include_array) {
10757 arrays &= ~(1UL << i);
10758 continue;
10759 }
10760
10761 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10762 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10763
10764 data_len += count * size;
10765 }
10766
10767
10768 data_len = roundup(data_len, sizeof(__u64));
10769 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len);
10770 if (!info_linear)
10771 return ERR_PTR(-ENOMEM);
10772
10773
10774 info_linear->arrays = arrays;
10775 memset(&info_linear->info, 0, sizeof(info));
10776 ptr = info_linear->data;
10777
10778 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10779 struct bpf_prog_info_array_desc *desc;
10780 __u32 count, size;
10781
10782 if ((arrays & (1UL << i)) == 0)
10783 continue;
10784
10785 desc = bpf_prog_info_array_desc + i;
10786 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10787 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10788 bpf_prog_info_set_offset_u32(&info_linear->info,
10789 desc->count_offset, count);
10790 bpf_prog_info_set_offset_u32(&info_linear->info,
10791 desc->size_offset, size);
10792 bpf_prog_info_set_offset_u64(&info_linear->info,
10793 desc->array_offset,
10794 ptr_to_u64(ptr));
10795 ptr += count * size;
10796 }
10797
10798
10799 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
10800 if (err) {
10801 pr_debug("can't get prog info: %s", strerror(errno));
10802 free(info_linear);
10803 return ERR_PTR(-EFAULT);
10804 }
10805
10806
10807 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10808 struct bpf_prog_info_array_desc *desc;
10809 __u32 v1, v2;
10810
10811 if ((arrays & (1UL << i)) == 0)
10812 continue;
10813
10814 desc = bpf_prog_info_array_desc + i;
10815 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10816 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
10817 desc->count_offset);
10818 if (v1 != v2)
10819 pr_warn("%s: mismatch in element count\n", __func__);
10820
10821 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10822 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
10823 desc->size_offset);
10824 if (v1 != v2)
10825 pr_warn("%s: mismatch in rec size\n", __func__);
10826 }
10827
10828
10829 info_linear->info_len = sizeof(struct bpf_prog_info);
10830 info_linear->data_len = data_len;
10831
10832 return info_linear;
10833}
10834
10835void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear)
10836{
10837 int i;
10838
10839 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10840 struct bpf_prog_info_array_desc *desc;
10841 __u64 addr, offs;
10842
10843 if ((info_linear->arrays & (1UL << i)) == 0)
10844 continue;
10845
10846 desc = bpf_prog_info_array_desc + i;
10847 addr = bpf_prog_info_read_offset_u64(&info_linear->info,
10848 desc->array_offset);
10849 offs = addr - ptr_to_u64(info_linear->data);
10850 bpf_prog_info_set_offset_u64(&info_linear->info,
10851 desc->array_offset, offs);
10852 }
10853}
10854
10855void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear)
10856{
10857 int i;
10858
10859 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10860 struct bpf_prog_info_array_desc *desc;
10861 __u64 addr, offs;
10862
10863 if ((info_linear->arrays & (1UL << i)) == 0)
10864 continue;
10865
10866 desc = bpf_prog_info_array_desc + i;
10867 offs = bpf_prog_info_read_offset_u64(&info_linear->info,
10868 desc->array_offset);
10869 addr = offs + ptr_to_u64(info_linear->data);
10870 bpf_prog_info_set_offset_u64(&info_linear->info,
10871 desc->array_offset, addr);
10872 }
10873}
10874
10875int bpf_program__set_attach_target(struct bpf_program *prog,
10876 int attach_prog_fd,
10877 const char *attach_func_name)
10878{
10879 int btf_obj_fd = 0, btf_id = 0, err;
10880
10881 if (!prog || attach_prog_fd < 0 || !attach_func_name)
10882 return -EINVAL;
10883
10884 if (prog->obj->loaded)
10885 return -EINVAL;
10886
10887 if (attach_prog_fd) {
10888 btf_id = libbpf_find_prog_btf_id(attach_func_name,
10889 attach_prog_fd);
10890 if (btf_id < 0)
10891 return btf_id;
10892 } else {
10893
10894 err = bpf_object__load_vmlinux_btf(prog->obj, true);
10895 if (err)
10896 return err;
10897 err = find_kernel_btf_id(prog->obj, attach_func_name,
10898 prog->expected_attach_type,
10899 &btf_obj_fd, &btf_id);
10900 if (err)
10901 return err;
10902 }
10903
10904 prog->attach_btf_id = btf_id;
10905 prog->attach_btf_obj_fd = btf_obj_fd;
10906 prog->attach_prog_fd = attach_prog_fd;
10907 return 0;
10908}
10909
10910int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
10911{
10912 int err = 0, n, len, start, end = -1;
10913 bool *tmp;
10914
10915 *mask = NULL;
10916 *mask_sz = 0;
10917
10918
10919 while (*s) {
10920 if (*s == ',' || *s == '\n') {
10921 s++;
10922 continue;
10923 }
10924 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
10925 if (n <= 0 || n > 2) {
10926 pr_warn("Failed to get CPU range %s: %d\n", s, n);
10927 err = -EINVAL;
10928 goto cleanup;
10929 } else if (n == 1) {
10930 end = start;
10931 }
10932 if (start < 0 || start > end) {
10933 pr_warn("Invalid CPU range [%d,%d] in %s\n",
10934 start, end, s);
10935 err = -EINVAL;
10936 goto cleanup;
10937 }
10938 tmp = realloc(*mask, end + 1);
10939 if (!tmp) {
10940 err = -ENOMEM;
10941 goto cleanup;
10942 }
10943 *mask = tmp;
10944 memset(tmp + *mask_sz, 0, start - *mask_sz);
10945 memset(tmp + start, 1, end - start + 1);
10946 *mask_sz = end + 1;
10947 s += len;
10948 }
10949 if (!*mask_sz) {
10950 pr_warn("Empty CPU range\n");
10951 return -EINVAL;
10952 }
10953 return 0;
10954cleanup:
10955 free(*mask);
10956 *mask = NULL;
10957 return err;
10958}
10959
10960int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
10961{
10962 int fd, err = 0, len;
10963 char buf[128];
10964
10965 fd = open(fcpu, O_RDONLY);
10966 if (fd < 0) {
10967 err = -errno;
10968 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
10969 return err;
10970 }
10971 len = read(fd, buf, sizeof(buf));
10972 close(fd);
10973 if (len <= 0) {
10974 err = len ? -errno : -EINVAL;
10975 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
10976 return err;
10977 }
10978 if (len >= sizeof(buf)) {
10979 pr_warn("CPU mask is too big in file %s\n", fcpu);
10980 return -E2BIG;
10981 }
10982 buf[len] = '\0';
10983
10984 return parse_cpu_mask_str(buf, mask, mask_sz);
10985}
10986
10987int libbpf_num_possible_cpus(void)
10988{
10989 static const char *fcpu = "/sys/devices/system/cpu/possible";
10990 static int cpus;
10991 int err, n, i, tmp_cpus;
10992 bool *mask;
10993
10994 tmp_cpus = READ_ONCE(cpus);
10995 if (tmp_cpus > 0)
10996 return tmp_cpus;
10997
10998 err = parse_cpu_mask_file(fcpu, &mask, &n);
10999 if (err)
11000 return err;
11001
11002 tmp_cpus = 0;
11003 for (i = 0; i < n; i++) {
11004 if (mask[i])
11005 tmp_cpus++;
11006 }
11007 free(mask);
11008
11009 WRITE_ONCE(cpus, tmp_cpus);
11010 return tmp_cpus;
11011}
11012
11013int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
11014 const struct bpf_object_open_opts *opts)
11015{
11016 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
11017 .object_name = s->name,
11018 );
11019 struct bpf_object *obj;
11020 int i;
11021
11022
11023
11024
11025
11026
11027
11028 if (opts) {
11029 memcpy(&skel_opts, opts, sizeof(*opts));
11030 if (!opts->object_name)
11031 skel_opts.object_name = s->name;
11032 }
11033
11034 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
11035 if (IS_ERR(obj)) {
11036 pr_warn("failed to initialize skeleton BPF object '%s': %ld\n",
11037 s->name, PTR_ERR(obj));
11038 return PTR_ERR(obj);
11039 }
11040
11041 *s->obj = obj;
11042
11043 for (i = 0; i < s->map_cnt; i++) {
11044 struct bpf_map **map = s->maps[i].map;
11045 const char *name = s->maps[i].name;
11046 void **mmaped = s->maps[i].mmaped;
11047
11048 *map = bpf_object__find_map_by_name(obj, name);
11049 if (!*map) {
11050 pr_warn("failed to find skeleton map '%s'\n", name);
11051 return -ESRCH;
11052 }
11053
11054
11055 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
11056 *mmaped = (*map)->mmaped;
11057 }
11058
11059 for (i = 0; i < s->prog_cnt; i++) {
11060 struct bpf_program **prog = s->progs[i].prog;
11061 const char *name = s->progs[i].name;
11062
11063 *prog = bpf_object__find_program_by_name(obj, name);
11064 if (!*prog) {
11065 pr_warn("failed to find skeleton program '%s'\n", name);
11066 return -ESRCH;
11067 }
11068 }
11069
11070 return 0;
11071}
11072
11073int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
11074{
11075 int i, err;
11076
11077 err = bpf_object__load(*s->obj);
11078 if (err) {
11079 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
11080 return err;
11081 }
11082
11083 for (i = 0; i < s->map_cnt; i++) {
11084 struct bpf_map *map = *s->maps[i].map;
11085 size_t mmap_sz = bpf_map_mmap_sz(map);
11086 int prot, map_fd = bpf_map__fd(map);
11087 void **mmaped = s->maps[i].mmaped;
11088
11089 if (!mmaped)
11090 continue;
11091
11092 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
11093 *mmaped = NULL;
11094 continue;
11095 }
11096
11097 if (map->def.map_flags & BPF_F_RDONLY_PROG)
11098 prot = PROT_READ;
11099 else
11100 prot = PROT_READ | PROT_WRITE;
11101
11102
11103
11104
11105
11106
11107
11108
11109
11110
11111
11112 *mmaped = mmap(map->mmaped, mmap_sz, prot,
11113 MAP_SHARED | MAP_FIXED, map_fd, 0);
11114 if (*mmaped == MAP_FAILED) {
11115 err = -errno;
11116 *mmaped = NULL;
11117 pr_warn("failed to re-mmap() map '%s': %d\n",
11118 bpf_map__name(map), err);
11119 return err;
11120 }
11121 }
11122
11123 return 0;
11124}
11125
11126int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
11127{
11128 int i;
11129
11130 for (i = 0; i < s->prog_cnt; i++) {
11131 struct bpf_program *prog = *s->progs[i].prog;
11132 struct bpf_link **link = s->progs[i].link;
11133 const struct bpf_sec_def *sec_def;
11134
11135 if (!prog->load)
11136 continue;
11137
11138 sec_def = find_sec_def(prog->sec_name);
11139 if (!sec_def || !sec_def->attach_fn)
11140 continue;
11141
11142 *link = sec_def->attach_fn(sec_def, prog);
11143 if (IS_ERR(*link)) {
11144 pr_warn("failed to auto-attach program '%s': %ld\n",
11145 bpf_program__name(prog), PTR_ERR(*link));
11146 return PTR_ERR(*link);
11147 }
11148 }
11149
11150 return 0;
11151}
11152
11153void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
11154{
11155 int i;
11156
11157 for (i = 0; i < s->prog_cnt; i++) {
11158 struct bpf_link **link = s->progs[i].link;
11159
11160 bpf_link__destroy(*link);
11161 *link = NULL;
11162 }
11163}
11164
11165void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
11166{
11167 if (s->progs)
11168 bpf_object__detach_skeleton(s);
11169 if (s->obj)
11170 bpf_object__close(*s->obj);
11171 free(s->maps);
11172 free(s->progs);
11173 free(s);
11174}
11175