1
2
3
4
5
6
7
8
9
10
11
12
13#ifndef _GNU_SOURCE
14#define _GNU_SOURCE
15#endif
16#include <stdlib.h>
17#include <stdio.h>
18#include <stdarg.h>
19#include <libgen.h>
20#include <inttypes.h>
21#include <limits.h>
22#include <string.h>
23#include <unistd.h>
24#include <endian.h>
25#include <fcntl.h>
26#include <errno.h>
27#include <ctype.h>
28#include <asm/unistd.h>
29#include <linux/err.h>
30#include <linux/kernel.h>
31#include <linux/bpf.h>
32#include <linux/btf.h>
33#include <linux/filter.h>
34#include <linux/list.h>
35#include <linux/limits.h>
36#include <linux/perf_event.h>
37#include <linux/ring_buffer.h>
38#include <linux/version.h>
39#include <sys/epoll.h>
40#include <sys/ioctl.h>
41#include <sys/mman.h>
42#include <sys/stat.h>
43#include <sys/types.h>
44#include <sys/vfs.h>
45#include <sys/utsname.h>
46#include <sys/resource.h>
47#include <libelf.h>
48#include <gelf.h>
49#include <zlib.h>
50
51#include "libbpf.h"
52#include "bpf.h"
53#include "btf.h"
54#include "str_error.h"
55#include "libbpf_internal.h"
56#include "hashmap.h"
57#include "bpf_gen_internal.h"
58
59#ifndef BPF_FS_MAGIC
60#define BPF_FS_MAGIC 0xcafe4a11
61#endif
62
63#define BPF_INSN_SZ (sizeof(struct bpf_insn))
64
65
66
67
68#pragma GCC diagnostic ignored "-Wformat-nonliteral"
69
70#define __printf(a, b) __attribute__((format(printf, a, b)))
71
72static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
73static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
74
75static int __base_pr(enum libbpf_print_level level, const char *format,
76 va_list args)
77{
78 if (level == LIBBPF_DEBUG)
79 return 0;
80
81 return vfprintf(stderr, format, args);
82}
83
84static libbpf_print_fn_t __libbpf_pr = __base_pr;
85
86libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
87{
88 libbpf_print_fn_t old_print_fn = __libbpf_pr;
89
90 __libbpf_pr = fn;
91 return old_print_fn;
92}
93
94__printf(2, 3)
95void libbpf_print(enum libbpf_print_level level, const char *format, ...)
96{
97 va_list args;
98
99 if (!__libbpf_pr)
100 return;
101
102 va_start(args, format);
103 __libbpf_pr(level, format, args);
104 va_end(args);
105}
106
107static void pr_perm_msg(int err)
108{
109 struct rlimit limit;
110 char buf[100];
111
112 if (err != -EPERM || geteuid() != 0)
113 return;
114
115 err = getrlimit(RLIMIT_MEMLOCK, &limit);
116 if (err)
117 return;
118
119 if (limit.rlim_cur == RLIM_INFINITY)
120 return;
121
122 if (limit.rlim_cur < 1024)
123 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
124 else if (limit.rlim_cur < 1024*1024)
125 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
126 else
127 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
128
129 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
130 buf);
131}
132
133#define STRERR_BUFSIZE 128
134
135
136#ifndef zfree
137# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
138#endif
139
140#ifndef zclose
141# define zclose(fd) ({ \
142 int ___err = 0; \
143 if ((fd) >= 0) \
144 ___err = close((fd)); \
145 fd = -1; \
146 ___err; })
147#endif
148
149static inline __u64 ptr_to_u64(const void *ptr)
150{
151 return (__u64) (unsigned long) ptr;
152}
153
154
155enum libbpf_strict_mode libbpf_mode = LIBBPF_STRICT_NONE;
156
157int libbpf_set_strict_mode(enum libbpf_strict_mode mode)
158{
159
160
161
162
163 if (mode != LIBBPF_STRICT_ALL
164 && (mode & ~((__LIBBPF_STRICT_LAST - 1) * 2 - 1)))
165 return errno = EINVAL, -EINVAL;
166
167 libbpf_mode = mode;
168 return 0;
169}
170
171enum kern_feature_id {
172
173 FEAT_PROG_NAME,
174
175 FEAT_GLOBAL_DATA,
176
177 FEAT_BTF,
178
179 FEAT_BTF_FUNC,
180
181 FEAT_BTF_DATASEC,
182
183 FEAT_BTF_GLOBAL_FUNC,
184
185 FEAT_ARRAY_MMAP,
186
187 FEAT_EXP_ATTACH_TYPE,
188
189 FEAT_PROBE_READ_KERN,
190
191 FEAT_PROG_BIND_MAP,
192
193 FEAT_MODULE_BTF,
194
195 FEAT_BTF_FLOAT,
196
197 FEAT_PERF_LINK,
198 __FEAT_CNT,
199};
200
201static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id);
202
203enum reloc_type {
204 RELO_LD64,
205 RELO_CALL,
206 RELO_DATA,
207 RELO_EXTERN_VAR,
208 RELO_EXTERN_FUNC,
209 RELO_SUBPROG_ADDR,
210};
211
212struct reloc_desc {
213 enum reloc_type type;
214 int insn_idx;
215 int map_idx;
216 int sym_off;
217};
218
219struct bpf_sec_def;
220
221typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec,
222 struct bpf_program *prog);
223
224struct bpf_sec_def {
225 const char *sec;
226 size_t len;
227 enum bpf_prog_type prog_type;
228 enum bpf_attach_type expected_attach_type;
229 bool is_exp_attach_type_optional;
230 bool is_attachable;
231 bool is_attach_btf;
232 bool is_sleepable;
233 attach_fn_t attach_fn;
234};
235
236
237
238
239
240struct bpf_program {
241 const struct bpf_sec_def *sec_def;
242 char *sec_name;
243 size_t sec_idx;
244
245
246
247 size_t sec_insn_off;
248
249
250
251
252 size_t sec_insn_cnt;
253
254
255
256
257
258
259
260
261 size_t sub_insn_off;
262
263 char *name;
264
265
266
267 char *pin_name;
268
269
270
271
272
273
274 struct bpf_insn *insns;
275
276
277
278
279 size_t insns_cnt;
280
281 struct reloc_desc *reloc_desc;
282 int nr_reloc;
283 int log_level;
284
285 struct {
286 int nr;
287 int *fds;
288 } instances;
289 bpf_program_prep_t preprocessor;
290
291 struct bpf_object *obj;
292 void *priv;
293 bpf_program_clear_priv_t clear_priv;
294
295 bool load;
296 bool mark_btf_static;
297 enum bpf_prog_type type;
298 enum bpf_attach_type expected_attach_type;
299 int prog_ifindex;
300 __u32 attach_btf_obj_fd;
301 __u32 attach_btf_id;
302 __u32 attach_prog_fd;
303 void *func_info;
304 __u32 func_info_rec_size;
305 __u32 func_info_cnt;
306
307 void *line_info;
308 __u32 line_info_rec_size;
309 __u32 line_info_cnt;
310 __u32 prog_flags;
311};
312
313struct bpf_struct_ops {
314 const char *tname;
315 const struct btf_type *type;
316 struct bpf_program **progs;
317 __u32 *kern_func_off;
318
319 void *data;
320
321
322
323
324
325
326
327
328
329
330 void *kern_vdata;
331 __u32 type_id;
332};
333
334#define DATA_SEC ".data"
335#define BSS_SEC ".bss"
336#define RODATA_SEC ".rodata"
337#define KCONFIG_SEC ".kconfig"
338#define KSYMS_SEC ".ksyms"
339#define STRUCT_OPS_SEC ".struct_ops"
340
341enum libbpf_map_type {
342 LIBBPF_MAP_UNSPEC,
343 LIBBPF_MAP_DATA,
344 LIBBPF_MAP_BSS,
345 LIBBPF_MAP_RODATA,
346 LIBBPF_MAP_KCONFIG,
347};
348
349static const char * const libbpf_type_to_btf_name[] = {
350 [LIBBPF_MAP_DATA] = DATA_SEC,
351 [LIBBPF_MAP_BSS] = BSS_SEC,
352 [LIBBPF_MAP_RODATA] = RODATA_SEC,
353 [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC,
354};
355
356struct bpf_map {
357 char *name;
358 int fd;
359 int sec_idx;
360 size_t sec_offset;
361 int map_ifindex;
362 int inner_map_fd;
363 struct bpf_map_def def;
364 __u32 numa_node;
365 __u32 btf_var_idx;
366 __u32 btf_key_type_id;
367 __u32 btf_value_type_id;
368 __u32 btf_vmlinux_value_type_id;
369 void *priv;
370 bpf_map_clear_priv_t clear_priv;
371 enum libbpf_map_type libbpf_type;
372 void *mmaped;
373 struct bpf_struct_ops *st_ops;
374 struct bpf_map *inner_map;
375 void **init_slots;
376 int init_slots_sz;
377 char *pin_path;
378 bool pinned;
379 bool reused;
380};
381
382enum extern_type {
383 EXT_UNKNOWN,
384 EXT_KCFG,
385 EXT_KSYM,
386};
387
388enum kcfg_type {
389 KCFG_UNKNOWN,
390 KCFG_CHAR,
391 KCFG_BOOL,
392 KCFG_INT,
393 KCFG_TRISTATE,
394 KCFG_CHAR_ARR,
395};
396
397struct extern_desc {
398 enum extern_type type;
399 int sym_idx;
400 int btf_id;
401 int sec_btf_id;
402 const char *name;
403 bool is_set;
404 bool is_weak;
405 union {
406 struct {
407 enum kcfg_type type;
408 int sz;
409 int align;
410 int data_off;
411 bool is_signed;
412 } kcfg;
413 struct {
414 unsigned long long addr;
415
416
417 int kernel_btf_obj_fd;
418 int kernel_btf_id;
419
420
421 __u32 type_id;
422 } ksym;
423 };
424};
425
426static LIST_HEAD(bpf_objects_list);
427
428struct module_btf {
429 struct btf *btf;
430 char *name;
431 __u32 id;
432 int fd;
433};
434
435struct bpf_object {
436 char name[BPF_OBJ_NAME_LEN];
437 char license[64];
438 __u32 kern_version;
439
440 struct bpf_program *programs;
441 size_t nr_programs;
442 struct bpf_map *maps;
443 size_t nr_maps;
444 size_t maps_cap;
445
446 char *kconfig;
447 struct extern_desc *externs;
448 int nr_extern;
449 int kconfig_map_idx;
450 int rodata_map_idx;
451
452 bool loaded;
453 bool has_subcalls;
454
455 struct bpf_gen *gen_loader;
456
457
458
459
460
461 struct {
462 int fd;
463 const void *obj_buf;
464 size_t obj_buf_sz;
465 Elf *elf;
466 GElf_Ehdr ehdr;
467 Elf_Data *symbols;
468 Elf_Data *data;
469 Elf_Data *rodata;
470 Elf_Data *bss;
471 Elf_Data *st_ops_data;
472 size_t shstrndx;
473 size_t strtabidx;
474 struct {
475 GElf_Shdr shdr;
476 Elf_Data *data;
477 } *reloc_sects;
478 int nr_reloc_sects;
479 int maps_shndx;
480 int btf_maps_shndx;
481 __u32 btf_maps_sec_btf_id;
482 int text_shndx;
483 int symbols_shndx;
484 int data_shndx;
485 int rodata_shndx;
486 int bss_shndx;
487 int st_ops_shndx;
488 } efile;
489
490
491
492
493
494 struct list_head list;
495
496 struct btf *btf;
497 struct btf_ext *btf_ext;
498
499
500
501
502 struct btf *btf_vmlinux;
503
504
505
506 char *btf_custom_path;
507
508 struct btf *btf_vmlinux_override;
509
510 struct module_btf *btf_modules;
511 bool btf_modules_loaded;
512 size_t btf_module_cnt;
513 size_t btf_module_cap;
514
515 void *priv;
516 bpf_object_clear_priv_t clear_priv;
517
518 char path[];
519};
520#define obj_elf_valid(o) ((o)->efile.elf)
521
522static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
523static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
524static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
525static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
526static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr);
527static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
528static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
529
530void bpf_program__unload(struct bpf_program *prog)
531{
532 int i;
533
534 if (!prog)
535 return;
536
537
538
539
540
541 if (prog->instances.nr > 0) {
542 for (i = 0; i < prog->instances.nr; i++)
543 zclose(prog->instances.fds[i]);
544 } else if (prog->instances.nr != -1) {
545 pr_warn("Internal error: instances.nr is %d\n",
546 prog->instances.nr);
547 }
548
549 prog->instances.nr = -1;
550 zfree(&prog->instances.fds);
551
552 zfree(&prog->func_info);
553 zfree(&prog->line_info);
554}
555
556static void bpf_program__exit(struct bpf_program *prog)
557{
558 if (!prog)
559 return;
560
561 if (prog->clear_priv)
562 prog->clear_priv(prog, prog->priv);
563
564 prog->priv = NULL;
565 prog->clear_priv = NULL;
566
567 bpf_program__unload(prog);
568 zfree(&prog->name);
569 zfree(&prog->sec_name);
570 zfree(&prog->pin_name);
571 zfree(&prog->insns);
572 zfree(&prog->reloc_desc);
573
574 prog->nr_reloc = 0;
575 prog->insns_cnt = 0;
576 prog->sec_idx = -1;
577}
578
579static char *__bpf_program__pin_name(struct bpf_program *prog)
580{
581 char *name, *p;
582
583 name = p = strdup(prog->sec_name);
584 while ((p = strchr(p, '/')))
585 *p = '_';
586
587 return name;
588}
589
590static bool insn_is_subprog_call(const struct bpf_insn *insn)
591{
592 return BPF_CLASS(insn->code) == BPF_JMP &&
593 BPF_OP(insn->code) == BPF_CALL &&
594 BPF_SRC(insn->code) == BPF_K &&
595 insn->src_reg == BPF_PSEUDO_CALL &&
596 insn->dst_reg == 0 &&
597 insn->off == 0;
598}
599
600static bool is_call_insn(const struct bpf_insn *insn)
601{
602 return insn->code == (BPF_JMP | BPF_CALL);
603}
604
605static bool insn_is_pseudo_func(struct bpf_insn *insn)
606{
607 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
608}
609
610static int
611bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
612 const char *name, size_t sec_idx, const char *sec_name,
613 size_t sec_off, void *insn_data, size_t insn_data_sz)
614{
615 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
616 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
617 sec_name, name, sec_off, insn_data_sz);
618 return -EINVAL;
619 }
620
621 memset(prog, 0, sizeof(*prog));
622 prog->obj = obj;
623
624 prog->sec_idx = sec_idx;
625 prog->sec_insn_off = sec_off / BPF_INSN_SZ;
626 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
627
628 prog->insns_cnt = prog->sec_insn_cnt;
629
630 prog->type = BPF_PROG_TYPE_UNSPEC;
631 prog->load = true;
632
633 prog->instances.fds = NULL;
634 prog->instances.nr = -1;
635
636 prog->sec_name = strdup(sec_name);
637 if (!prog->sec_name)
638 goto errout;
639
640 prog->name = strdup(name);
641 if (!prog->name)
642 goto errout;
643
644 prog->pin_name = __bpf_program__pin_name(prog);
645 if (!prog->pin_name)
646 goto errout;
647
648 prog->insns = malloc(insn_data_sz);
649 if (!prog->insns)
650 goto errout;
651 memcpy(prog->insns, insn_data, insn_data_sz);
652
653 return 0;
654errout:
655 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
656 bpf_program__exit(prog);
657 return -ENOMEM;
658}
659
660static int
661bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
662 const char *sec_name, int sec_idx)
663{
664 Elf_Data *symbols = obj->efile.symbols;
665 struct bpf_program *prog, *progs;
666 void *data = sec_data->d_buf;
667 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
668 int nr_progs, err, i;
669 const char *name;
670 GElf_Sym sym;
671
672 progs = obj->programs;
673 nr_progs = obj->nr_programs;
674 nr_syms = symbols->d_size / sizeof(GElf_Sym);
675 sec_off = 0;
676
677 for (i = 0; i < nr_syms; i++) {
678 if (!gelf_getsym(symbols, i, &sym))
679 continue;
680 if (sym.st_shndx != sec_idx)
681 continue;
682 if (GELF_ST_TYPE(sym.st_info) != STT_FUNC)
683 continue;
684
685 prog_sz = sym.st_size;
686 sec_off = sym.st_value;
687
688 name = elf_sym_str(obj, sym.st_name);
689 if (!name) {
690 pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
691 sec_name, sec_off);
692 return -LIBBPF_ERRNO__FORMAT;
693 }
694
695 if (sec_off + prog_sz > sec_sz) {
696 pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
697 sec_name, sec_off);
698 return -LIBBPF_ERRNO__FORMAT;
699 }
700
701 if (sec_idx != obj->efile.text_shndx && GELF_ST_BIND(sym.st_info) == STB_LOCAL) {
702 pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name);
703 return -ENOTSUP;
704 }
705
706 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
707 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
708
709 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
710 if (!progs) {
711
712
713
714
715
716 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
717 sec_name, name);
718 return -ENOMEM;
719 }
720 obj->programs = progs;
721
722 prog = &progs[nr_progs];
723
724 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
725 sec_off, data + sec_off, prog_sz);
726 if (err)
727 return err;
728
729
730
731
732
733
734 if (GELF_ST_BIND(sym.st_info) != STB_LOCAL
735 && (GELF_ST_VISIBILITY(sym.st_other) == STV_HIDDEN
736 || GELF_ST_VISIBILITY(sym.st_other) == STV_INTERNAL))
737 prog->mark_btf_static = true;
738
739 nr_progs++;
740 obj->nr_programs = nr_progs;
741 }
742
743 return 0;
744}
745
746static __u32 get_kernel_version(void)
747{
748 __u32 major, minor, patch;
749 struct utsname info;
750
751 uname(&info);
752 if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
753 return 0;
754 return KERNEL_VERSION(major, minor, patch);
755}
756
757static const struct btf_member *
758find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
759{
760 struct btf_member *m;
761 int i;
762
763 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
764 if (btf_member_bit_offset(t, i) == bit_offset)
765 return m;
766 }
767
768 return NULL;
769}
770
771static const struct btf_member *
772find_member_by_name(const struct btf *btf, const struct btf_type *t,
773 const char *name)
774{
775 struct btf_member *m;
776 int i;
777
778 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
779 if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
780 return m;
781 }
782
783 return NULL;
784}
785
786#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
787static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
788 const char *name, __u32 kind);
789
790static int
791find_struct_ops_kern_types(const struct btf *btf, const char *tname,
792 const struct btf_type **type, __u32 *type_id,
793 const struct btf_type **vtype, __u32 *vtype_id,
794 const struct btf_member **data_member)
795{
796 const struct btf_type *kern_type, *kern_vtype;
797 const struct btf_member *kern_data_member;
798 __s32 kern_vtype_id, kern_type_id;
799 __u32 i;
800
801 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
802 if (kern_type_id < 0) {
803 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
804 tname);
805 return kern_type_id;
806 }
807 kern_type = btf__type_by_id(btf, kern_type_id);
808
809
810
811
812
813
814 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
815 tname, BTF_KIND_STRUCT);
816 if (kern_vtype_id < 0) {
817 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
818 STRUCT_OPS_VALUE_PREFIX, tname);
819 return kern_vtype_id;
820 }
821 kern_vtype = btf__type_by_id(btf, kern_vtype_id);
822
823
824
825
826
827
828
829 kern_data_member = btf_members(kern_vtype);
830 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
831 if (kern_data_member->type == kern_type_id)
832 break;
833 }
834 if (i == btf_vlen(kern_vtype)) {
835 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
836 tname, STRUCT_OPS_VALUE_PREFIX, tname);
837 return -EINVAL;
838 }
839
840 *type = kern_type;
841 *type_id = kern_type_id;
842 *vtype = kern_vtype;
843 *vtype_id = kern_vtype_id;
844 *data_member = kern_data_member;
845
846 return 0;
847}
848
849static bool bpf_map__is_struct_ops(const struct bpf_map *map)
850{
851 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
852}
853
854
855static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
856 const struct btf *btf,
857 const struct btf *kern_btf)
858{
859 const struct btf_member *member, *kern_member, *kern_data_member;
860 const struct btf_type *type, *kern_type, *kern_vtype;
861 __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
862 struct bpf_struct_ops *st_ops;
863 void *data, *kern_data;
864 const char *tname;
865 int err;
866
867 st_ops = map->st_ops;
868 type = st_ops->type;
869 tname = st_ops->tname;
870 err = find_struct_ops_kern_types(kern_btf, tname,
871 &kern_type, &kern_type_id,
872 &kern_vtype, &kern_vtype_id,
873 &kern_data_member);
874 if (err)
875 return err;
876
877 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
878 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
879
880 map->def.value_size = kern_vtype->size;
881 map->btf_vmlinux_value_type_id = kern_vtype_id;
882
883 st_ops->kern_vdata = calloc(1, kern_vtype->size);
884 if (!st_ops->kern_vdata)
885 return -ENOMEM;
886
887 data = st_ops->data;
888 kern_data_off = kern_data_member->offset / 8;
889 kern_data = st_ops->kern_vdata + kern_data_off;
890
891 member = btf_members(type);
892 for (i = 0; i < btf_vlen(type); i++, member++) {
893 const struct btf_type *mtype, *kern_mtype;
894 __u32 mtype_id, kern_mtype_id;
895 void *mdata, *kern_mdata;
896 __s64 msize, kern_msize;
897 __u32 moff, kern_moff;
898 __u32 kern_member_idx;
899 const char *mname;
900
901 mname = btf__name_by_offset(btf, member->name_off);
902 kern_member = find_member_by_name(kern_btf, kern_type, mname);
903 if (!kern_member) {
904 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
905 map->name, mname);
906 return -ENOTSUP;
907 }
908
909 kern_member_idx = kern_member - btf_members(kern_type);
910 if (btf_member_bitfield_size(type, i) ||
911 btf_member_bitfield_size(kern_type, kern_member_idx)) {
912 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
913 map->name, mname);
914 return -ENOTSUP;
915 }
916
917 moff = member->offset / 8;
918 kern_moff = kern_member->offset / 8;
919
920 mdata = data + moff;
921 kern_mdata = kern_data + kern_moff;
922
923 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
924 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
925 &kern_mtype_id);
926 if (BTF_INFO_KIND(mtype->info) !=
927 BTF_INFO_KIND(kern_mtype->info)) {
928 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
929 map->name, mname, BTF_INFO_KIND(mtype->info),
930 BTF_INFO_KIND(kern_mtype->info));
931 return -ENOTSUP;
932 }
933
934 if (btf_is_ptr(mtype)) {
935 struct bpf_program *prog;
936
937 prog = st_ops->progs[i];
938 if (!prog)
939 continue;
940
941 kern_mtype = skip_mods_and_typedefs(kern_btf,
942 kern_mtype->type,
943 &kern_mtype_id);
944
945
946
947
948
949 if (!btf_is_func_proto(kern_mtype)) {
950 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
951 map->name, mname);
952 return -ENOTSUP;
953 }
954
955 prog->attach_btf_id = kern_type_id;
956 prog->expected_attach_type = kern_member_idx;
957
958 st_ops->kern_func_off[i] = kern_data_off + kern_moff;
959
960 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
961 map->name, mname, prog->name, moff,
962 kern_moff);
963
964 continue;
965 }
966
967 msize = btf__resolve_size(btf, mtype_id);
968 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
969 if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
970 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
971 map->name, mname, (ssize_t)msize,
972 (ssize_t)kern_msize);
973 return -ENOTSUP;
974 }
975
976 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
977 map->name, mname, (unsigned int)msize,
978 moff, kern_moff);
979 memcpy(kern_mdata, mdata, msize);
980 }
981
982 return 0;
983}
984
985static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
986{
987 struct bpf_map *map;
988 size_t i;
989 int err;
990
991 for (i = 0; i < obj->nr_maps; i++) {
992 map = &obj->maps[i];
993
994 if (!bpf_map__is_struct_ops(map))
995 continue;
996
997 err = bpf_map__init_kern_struct_ops(map, obj->btf,
998 obj->btf_vmlinux);
999 if (err)
1000 return err;
1001 }
1002
1003 return 0;
1004}
1005
1006static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
1007{
1008 const struct btf_type *type, *datasec;
1009 const struct btf_var_secinfo *vsi;
1010 struct bpf_struct_ops *st_ops;
1011 const char *tname, *var_name;
1012 __s32 type_id, datasec_id;
1013 const struct btf *btf;
1014 struct bpf_map *map;
1015 __u32 i;
1016
1017 if (obj->efile.st_ops_shndx == -1)
1018 return 0;
1019
1020 btf = obj->btf;
1021 datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
1022 BTF_KIND_DATASEC);
1023 if (datasec_id < 0) {
1024 pr_warn("struct_ops init: DATASEC %s not found\n",
1025 STRUCT_OPS_SEC);
1026 return -EINVAL;
1027 }
1028
1029 datasec = btf__type_by_id(btf, datasec_id);
1030 vsi = btf_var_secinfos(datasec);
1031 for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
1032 type = btf__type_by_id(obj->btf, vsi->type);
1033 var_name = btf__name_by_offset(obj->btf, type->name_off);
1034
1035 type_id = btf__resolve_type(obj->btf, vsi->type);
1036 if (type_id < 0) {
1037 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
1038 vsi->type, STRUCT_OPS_SEC);
1039 return -EINVAL;
1040 }
1041
1042 type = btf__type_by_id(obj->btf, type_id);
1043 tname = btf__name_by_offset(obj->btf, type->name_off);
1044 if (!tname[0]) {
1045 pr_warn("struct_ops init: anonymous type is not supported\n");
1046 return -ENOTSUP;
1047 }
1048 if (!btf_is_struct(type)) {
1049 pr_warn("struct_ops init: %s is not a struct\n", tname);
1050 return -EINVAL;
1051 }
1052
1053 map = bpf_object__add_map(obj);
1054 if (IS_ERR(map))
1055 return PTR_ERR(map);
1056
1057 map->sec_idx = obj->efile.st_ops_shndx;
1058 map->sec_offset = vsi->offset;
1059 map->name = strdup(var_name);
1060 if (!map->name)
1061 return -ENOMEM;
1062
1063 map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1064 map->def.key_size = sizeof(int);
1065 map->def.value_size = type->size;
1066 map->def.max_entries = 1;
1067
1068 map->st_ops = calloc(1, sizeof(*map->st_ops));
1069 if (!map->st_ops)
1070 return -ENOMEM;
1071 st_ops = map->st_ops;
1072 st_ops->data = malloc(type->size);
1073 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1074 st_ops->kern_func_off = malloc(btf_vlen(type) *
1075 sizeof(*st_ops->kern_func_off));
1076 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1077 return -ENOMEM;
1078
1079 if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
1080 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1081 var_name, STRUCT_OPS_SEC);
1082 return -EINVAL;
1083 }
1084
1085 memcpy(st_ops->data,
1086 obj->efile.st_ops_data->d_buf + vsi->offset,
1087 type->size);
1088 st_ops->tname = tname;
1089 st_ops->type = type;
1090 st_ops->type_id = type_id;
1091
1092 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1093 tname, type_id, var_name, vsi->offset);
1094 }
1095
1096 return 0;
1097}
1098
1099static struct bpf_object *bpf_object__new(const char *path,
1100 const void *obj_buf,
1101 size_t obj_buf_sz,
1102 const char *obj_name)
1103{
1104 struct bpf_object *obj;
1105 char *end;
1106
1107 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1108 if (!obj) {
1109 pr_warn("alloc memory failed for %s\n", path);
1110 return ERR_PTR(-ENOMEM);
1111 }
1112
1113 strcpy(obj->path, path);
1114 if (obj_name) {
1115 strncpy(obj->name, obj_name, sizeof(obj->name) - 1);
1116 obj->name[sizeof(obj->name) - 1] = 0;
1117 } else {
1118
1119 strncpy(obj->name, basename((void *)path),
1120 sizeof(obj->name) - 1);
1121 end = strchr(obj->name, '.');
1122 if (end)
1123 *end = 0;
1124 }
1125
1126 obj->efile.fd = -1;
1127
1128
1129
1130
1131
1132
1133 obj->efile.obj_buf = obj_buf;
1134 obj->efile.obj_buf_sz = obj_buf_sz;
1135 obj->efile.maps_shndx = -1;
1136 obj->efile.btf_maps_shndx = -1;
1137 obj->efile.data_shndx = -1;
1138 obj->efile.rodata_shndx = -1;
1139 obj->efile.bss_shndx = -1;
1140 obj->efile.st_ops_shndx = -1;
1141 obj->kconfig_map_idx = -1;
1142 obj->rodata_map_idx = -1;
1143
1144 obj->kern_version = get_kernel_version();
1145 obj->loaded = false;
1146
1147 INIT_LIST_HEAD(&obj->list);
1148 list_add(&obj->list, &bpf_objects_list);
1149 return obj;
1150}
1151
1152static void bpf_object__elf_finish(struct bpf_object *obj)
1153{
1154 if (!obj_elf_valid(obj))
1155 return;
1156
1157 if (obj->efile.elf) {
1158 elf_end(obj->efile.elf);
1159 obj->efile.elf = NULL;
1160 }
1161 obj->efile.symbols = NULL;
1162 obj->efile.data = NULL;
1163 obj->efile.rodata = NULL;
1164 obj->efile.bss = NULL;
1165 obj->efile.st_ops_data = NULL;
1166
1167 zfree(&obj->efile.reloc_sects);
1168 obj->efile.nr_reloc_sects = 0;
1169 zclose(obj->efile.fd);
1170 obj->efile.obj_buf = NULL;
1171 obj->efile.obj_buf_sz = 0;
1172}
1173
1174static int bpf_object__elf_init(struct bpf_object *obj)
1175{
1176 int err = 0;
1177 GElf_Ehdr *ep;
1178
1179 if (obj_elf_valid(obj)) {
1180 pr_warn("elf: init internal error\n");
1181 return -LIBBPF_ERRNO__LIBELF;
1182 }
1183
1184 if (obj->efile.obj_buf_sz > 0) {
1185
1186
1187
1188
1189 obj->efile.elf = elf_memory((char *)obj->efile.obj_buf,
1190 obj->efile.obj_buf_sz);
1191 } else {
1192 obj->efile.fd = open(obj->path, O_RDONLY);
1193 if (obj->efile.fd < 0) {
1194 char errmsg[STRERR_BUFSIZE], *cp;
1195
1196 err = -errno;
1197 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1198 pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1199 return err;
1200 }
1201
1202 obj->efile.elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1203 }
1204
1205 if (!obj->efile.elf) {
1206 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1207 err = -LIBBPF_ERRNO__LIBELF;
1208 goto errout;
1209 }
1210
1211 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
1212 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1213 err = -LIBBPF_ERRNO__FORMAT;
1214 goto errout;
1215 }
1216 ep = &obj->efile.ehdr;
1217
1218 if (elf_getshdrstrndx(obj->efile.elf, &obj->efile.shstrndx)) {
1219 pr_warn("elf: failed to get section names section index for %s: %s\n",
1220 obj->path, elf_errmsg(-1));
1221 err = -LIBBPF_ERRNO__FORMAT;
1222 goto errout;
1223 }
1224
1225
1226 if (!elf_rawdata(elf_getscn(obj->efile.elf, obj->efile.shstrndx), NULL)) {
1227 pr_warn("elf: failed to get section names strings from %s: %s\n",
1228 obj->path, elf_errmsg(-1));
1229 err = -LIBBPF_ERRNO__FORMAT;
1230 goto errout;
1231 }
1232
1233
1234 if (ep->e_type != ET_REL ||
1235 (ep->e_machine && ep->e_machine != EM_BPF)) {
1236 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1237 err = -LIBBPF_ERRNO__FORMAT;
1238 goto errout;
1239 }
1240
1241 return 0;
1242errout:
1243 bpf_object__elf_finish(obj);
1244 return err;
1245}
1246
1247static int bpf_object__check_endianness(struct bpf_object *obj)
1248{
1249#if __BYTE_ORDER == __LITTLE_ENDIAN
1250 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB)
1251 return 0;
1252#elif __BYTE_ORDER == __BIG_ENDIAN
1253 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB)
1254 return 0;
1255#else
1256# error "Unrecognized __BYTE_ORDER__"
1257#endif
1258 pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1259 return -LIBBPF_ERRNO__ENDIAN;
1260}
1261
1262static int
1263bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1264{
1265 memcpy(obj->license, data, min(size, sizeof(obj->license) - 1));
1266 pr_debug("license of %s is %s\n", obj->path, obj->license);
1267 return 0;
1268}
1269
1270static int
1271bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1272{
1273 __u32 kver;
1274
1275 if (size != sizeof(kver)) {
1276 pr_warn("invalid kver section in %s\n", obj->path);
1277 return -LIBBPF_ERRNO__FORMAT;
1278 }
1279 memcpy(&kver, data, sizeof(kver));
1280 obj->kern_version = kver;
1281 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1282 return 0;
1283}
1284
1285static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1286{
1287 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1288 type == BPF_MAP_TYPE_HASH_OF_MAPS)
1289 return true;
1290 return false;
1291}
1292
1293int bpf_object__section_size(const struct bpf_object *obj, const char *name,
1294 __u32 *size)
1295{
1296 int ret = -ENOENT;
1297
1298 *size = 0;
1299 if (!name) {
1300 return -EINVAL;
1301 } else if (!strcmp(name, DATA_SEC)) {
1302 if (obj->efile.data)
1303 *size = obj->efile.data->d_size;
1304 } else if (!strcmp(name, BSS_SEC)) {
1305 if (obj->efile.bss)
1306 *size = obj->efile.bss->d_size;
1307 } else if (!strcmp(name, RODATA_SEC)) {
1308 if (obj->efile.rodata)
1309 *size = obj->efile.rodata->d_size;
1310 } else if (!strcmp(name, STRUCT_OPS_SEC)) {
1311 if (obj->efile.st_ops_data)
1312 *size = obj->efile.st_ops_data->d_size;
1313 } else {
1314 Elf_Scn *scn = elf_sec_by_name(obj, name);
1315 Elf_Data *data = elf_sec_data(obj, scn);
1316
1317 if (data) {
1318 ret = 0;
1319 *size = data->d_size;
1320 }
1321 }
1322
1323 return *size ? 0 : ret;
1324}
1325
1326int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
1327 __u32 *off)
1328{
1329 Elf_Data *symbols = obj->efile.symbols;
1330 const char *sname;
1331 size_t si;
1332
1333 if (!name || !off)
1334 return -EINVAL;
1335
1336 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) {
1337 GElf_Sym sym;
1338
1339 if (!gelf_getsym(symbols, si, &sym))
1340 continue;
1341 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
1342 GELF_ST_TYPE(sym.st_info) != STT_OBJECT)
1343 continue;
1344
1345 sname = elf_sym_str(obj, sym.st_name);
1346 if (!sname) {
1347 pr_warn("failed to get sym name string for var %s\n",
1348 name);
1349 return -EIO;
1350 }
1351 if (strcmp(name, sname) == 0) {
1352 *off = sym.st_value;
1353 return 0;
1354 }
1355 }
1356
1357 return -ENOENT;
1358}
1359
1360static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1361{
1362 struct bpf_map *new_maps;
1363 size_t new_cap;
1364 int i;
1365
1366 if (obj->nr_maps < obj->maps_cap)
1367 return &obj->maps[obj->nr_maps++];
1368
1369 new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
1370 new_maps = libbpf_reallocarray(obj->maps, new_cap, sizeof(*obj->maps));
1371 if (!new_maps) {
1372 pr_warn("alloc maps for object failed\n");
1373 return ERR_PTR(-ENOMEM);
1374 }
1375
1376 obj->maps_cap = new_cap;
1377 obj->maps = new_maps;
1378
1379
1380 memset(obj->maps + obj->nr_maps, 0,
1381 (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps));
1382
1383
1384
1385
1386 for (i = obj->nr_maps; i < obj->maps_cap; i++) {
1387 obj->maps[i].fd = -1;
1388 obj->maps[i].inner_map_fd = -1;
1389 }
1390
1391 return &obj->maps[obj->nr_maps++];
1392}
1393
1394static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1395{
1396 long page_sz = sysconf(_SC_PAGE_SIZE);
1397 size_t map_sz;
1398
1399 map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
1400 map_sz = roundup(map_sz, page_sz);
1401 return map_sz;
1402}
1403
1404static char *internal_map_name(struct bpf_object *obj,
1405 enum libbpf_map_type type)
1406{
1407 char map_name[BPF_OBJ_NAME_LEN], *p;
1408 const char *sfx = libbpf_type_to_btf_name[type];
1409 int sfx_len = max((size_t)7, strlen(sfx));
1410 int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1,
1411 strlen(obj->name));
1412
1413 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1414 sfx_len, libbpf_type_to_btf_name[type]);
1415
1416
1417 for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1418 if (!isalnum(*p) && *p != '_' && *p != '.')
1419 *p = '_';
1420
1421 return strdup(map_name);
1422}
1423
1424static int
1425bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1426 int sec_idx, void *data, size_t data_sz)
1427{
1428 struct bpf_map_def *def;
1429 struct bpf_map *map;
1430 int err;
1431
1432 map = bpf_object__add_map(obj);
1433 if (IS_ERR(map))
1434 return PTR_ERR(map);
1435
1436 map->libbpf_type = type;
1437 map->sec_idx = sec_idx;
1438 map->sec_offset = 0;
1439 map->name = internal_map_name(obj, type);
1440 if (!map->name) {
1441 pr_warn("failed to alloc map name\n");
1442 return -ENOMEM;
1443 }
1444
1445 def = &map->def;
1446 def->type = BPF_MAP_TYPE_ARRAY;
1447 def->key_size = sizeof(int);
1448 def->value_size = data_sz;
1449 def->max_entries = 1;
1450 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1451 ? BPF_F_RDONLY_PROG : 0;
1452 def->map_flags |= BPF_F_MMAPABLE;
1453
1454 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1455 map->name, map->sec_idx, map->sec_offset, def->map_flags);
1456
1457 map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
1458 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1459 if (map->mmaped == MAP_FAILED) {
1460 err = -errno;
1461 map->mmaped = NULL;
1462 pr_warn("failed to alloc map '%s' content buffer: %d\n",
1463 map->name, err);
1464 zfree(&map->name);
1465 return err;
1466 }
1467
1468 if (data)
1469 memcpy(map->mmaped, data, data_sz);
1470
1471 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1472 return 0;
1473}
1474
1475static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1476{
1477 int err;
1478
1479
1480
1481
1482 if (obj->efile.data_shndx >= 0) {
1483 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1484 obj->efile.data_shndx,
1485 obj->efile.data->d_buf,
1486 obj->efile.data->d_size);
1487 if (err)
1488 return err;
1489 }
1490 if (obj->efile.rodata_shndx >= 0) {
1491 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1492 obj->efile.rodata_shndx,
1493 obj->efile.rodata->d_buf,
1494 obj->efile.rodata->d_size);
1495 if (err)
1496 return err;
1497
1498 obj->rodata_map_idx = obj->nr_maps - 1;
1499 }
1500 if (obj->efile.bss_shndx >= 0) {
1501 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1502 obj->efile.bss_shndx,
1503 NULL,
1504 obj->efile.bss->d_size);
1505 if (err)
1506 return err;
1507 }
1508 return 0;
1509}
1510
1511
1512static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1513 const void *name)
1514{
1515 int i;
1516
1517 for (i = 0; i < obj->nr_extern; i++) {
1518 if (strcmp(obj->externs[i].name, name) == 0)
1519 return &obj->externs[i];
1520 }
1521 return NULL;
1522}
1523
1524static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
1525 char value)
1526{
1527 switch (ext->kcfg.type) {
1528 case KCFG_BOOL:
1529 if (value == 'm') {
1530 pr_warn("extern (kcfg) %s=%c should be tristate or char\n",
1531 ext->name, value);
1532 return -EINVAL;
1533 }
1534 *(bool *)ext_val = value == 'y' ? true : false;
1535 break;
1536 case KCFG_TRISTATE:
1537 if (value == 'y')
1538 *(enum libbpf_tristate *)ext_val = TRI_YES;
1539 else if (value == 'm')
1540 *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1541 else
1542 *(enum libbpf_tristate *)ext_val = TRI_NO;
1543 break;
1544 case KCFG_CHAR:
1545 *(char *)ext_val = value;
1546 break;
1547 case KCFG_UNKNOWN:
1548 case KCFG_INT:
1549 case KCFG_CHAR_ARR:
1550 default:
1551 pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n",
1552 ext->name, value);
1553 return -EINVAL;
1554 }
1555 ext->is_set = true;
1556 return 0;
1557}
1558
1559static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
1560 const char *value)
1561{
1562 size_t len;
1563
1564 if (ext->kcfg.type != KCFG_CHAR_ARR) {
1565 pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value);
1566 return -EINVAL;
1567 }
1568
1569 len = strlen(value);
1570 if (value[len - 1] != '"') {
1571 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
1572 ext->name, value);
1573 return -EINVAL;
1574 }
1575
1576
1577 len -= 2;
1578 if (len >= ext->kcfg.sz) {
1579 pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n",
1580 ext->name, value, len, ext->kcfg.sz - 1);
1581 len = ext->kcfg.sz - 1;
1582 }
1583 memcpy(ext_val, value + 1, len);
1584 ext_val[len] = '\0';
1585 ext->is_set = true;
1586 return 0;
1587}
1588
1589static int parse_u64(const char *value, __u64 *res)
1590{
1591 char *value_end;
1592 int err;
1593
1594 errno = 0;
1595 *res = strtoull(value, &value_end, 0);
1596 if (errno) {
1597 err = -errno;
1598 pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1599 return err;
1600 }
1601 if (*value_end) {
1602 pr_warn("failed to parse '%s' as integer completely\n", value);
1603 return -EINVAL;
1604 }
1605 return 0;
1606}
1607
1608static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
1609{
1610 int bit_sz = ext->kcfg.sz * 8;
1611
1612 if (ext->kcfg.sz == 8)
1613 return true;
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627 if (ext->kcfg.is_signed)
1628 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1629 else
1630 return (v >> bit_sz) == 0;
1631}
1632
1633static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
1634 __u64 value)
1635{
1636 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
1637 pr_warn("extern (kcfg) %s=%llu should be integer\n",
1638 ext->name, (unsigned long long)value);
1639 return -EINVAL;
1640 }
1641 if (!is_kcfg_value_in_range(ext, value)) {
1642 pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n",
1643 ext->name, (unsigned long long)value, ext->kcfg.sz);
1644 return -ERANGE;
1645 }
1646 switch (ext->kcfg.sz) {
1647 case 1: *(__u8 *)ext_val = value; break;
1648 case 2: *(__u16 *)ext_val = value; break;
1649 case 4: *(__u32 *)ext_val = value; break;
1650 case 8: *(__u64 *)ext_val = value; break;
1651 default:
1652 return -EINVAL;
1653 }
1654 ext->is_set = true;
1655 return 0;
1656}
1657
1658static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1659 char *buf, void *data)
1660{
1661 struct extern_desc *ext;
1662 char *sep, *value;
1663 int len, err = 0;
1664 void *ext_val;
1665 __u64 num;
1666
1667 if (strncmp(buf, "CONFIG_", 7))
1668 return 0;
1669
1670 sep = strchr(buf, '=');
1671 if (!sep) {
1672 pr_warn("failed to parse '%s': no separator\n", buf);
1673 return -EINVAL;
1674 }
1675
1676
1677 len = strlen(buf);
1678 if (buf[len - 1] == '\n')
1679 buf[len - 1] = '\0';
1680
1681 *sep = '\0';
1682 if (!sep[1]) {
1683 *sep = '=';
1684 pr_warn("failed to parse '%s': no value\n", buf);
1685 return -EINVAL;
1686 }
1687
1688 ext = find_extern_by_name(obj, buf);
1689 if (!ext || ext->is_set)
1690 return 0;
1691
1692 ext_val = data + ext->kcfg.data_off;
1693 value = sep + 1;
1694
1695 switch (*value) {
1696 case 'y': case 'n': case 'm':
1697 err = set_kcfg_value_tri(ext, ext_val, *value);
1698 break;
1699 case '"':
1700 err = set_kcfg_value_str(ext, ext_val, value);
1701 break;
1702 default:
1703
1704 err = parse_u64(value, &num);
1705 if (err) {
1706 pr_warn("extern (kcfg) %s=%s should be integer\n",
1707 ext->name, value);
1708 return err;
1709 }
1710 err = set_kcfg_value_num(ext, ext_val, num);
1711 break;
1712 }
1713 if (err)
1714 return err;
1715 pr_debug("extern (kcfg) %s=%s\n", ext->name, value);
1716 return 0;
1717}
1718
1719static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1720{
1721 char buf[PATH_MAX];
1722 struct utsname uts;
1723 int len, err = 0;
1724 gzFile file;
1725
1726 uname(&uts);
1727 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1728 if (len < 0)
1729 return -EINVAL;
1730 else if (len >= PATH_MAX)
1731 return -ENAMETOOLONG;
1732
1733
1734 file = gzopen(buf, "r");
1735 if (!file)
1736 file = gzopen("/proc/config.gz", "r");
1737
1738 if (!file) {
1739 pr_warn("failed to open system Kconfig\n");
1740 return -ENOENT;
1741 }
1742
1743 while (gzgets(file, buf, sizeof(buf))) {
1744 err = bpf_object__process_kconfig_line(obj, buf, data);
1745 if (err) {
1746 pr_warn("error parsing system Kconfig line '%s': %d\n",
1747 buf, err);
1748 goto out;
1749 }
1750 }
1751
1752out:
1753 gzclose(file);
1754 return err;
1755}
1756
1757static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
1758 const char *config, void *data)
1759{
1760 char buf[PATH_MAX];
1761 int err = 0;
1762 FILE *file;
1763
1764 file = fmemopen((void *)config, strlen(config), "r");
1765 if (!file) {
1766 err = -errno;
1767 pr_warn("failed to open in-memory Kconfig: %d\n", err);
1768 return err;
1769 }
1770
1771 while (fgets(buf, sizeof(buf), file)) {
1772 err = bpf_object__process_kconfig_line(obj, buf, data);
1773 if (err) {
1774 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
1775 buf, err);
1776 break;
1777 }
1778 }
1779
1780 fclose(file);
1781 return err;
1782}
1783
1784static int bpf_object__init_kconfig_map(struct bpf_object *obj)
1785{
1786 struct extern_desc *last_ext = NULL, *ext;
1787 size_t map_sz;
1788 int i, err;
1789
1790 for (i = 0; i < obj->nr_extern; i++) {
1791 ext = &obj->externs[i];
1792 if (ext->type == EXT_KCFG)
1793 last_ext = ext;
1794 }
1795
1796 if (!last_ext)
1797 return 0;
1798
1799 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
1800 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
1801 obj->efile.symbols_shndx,
1802 NULL, map_sz);
1803 if (err)
1804 return err;
1805
1806 obj->kconfig_map_idx = obj->nr_maps - 1;
1807
1808 return 0;
1809}
1810
1811static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
1812{
1813 Elf_Data *symbols = obj->efile.symbols;
1814 int i, map_def_sz = 0, nr_maps = 0, nr_syms;
1815 Elf_Data *data = NULL;
1816 Elf_Scn *scn;
1817
1818 if (obj->efile.maps_shndx < 0)
1819 return 0;
1820
1821 if (!symbols)
1822 return -EINVAL;
1823
1824 scn = elf_sec_by_idx(obj, obj->efile.maps_shndx);
1825 data = elf_sec_data(obj, scn);
1826 if (!scn || !data) {
1827 pr_warn("elf: failed to get legacy map definitions for %s\n",
1828 obj->path);
1829 return -EINVAL;
1830 }
1831
1832
1833
1834
1835
1836
1837
1838
1839 nr_syms = symbols->d_size / sizeof(GElf_Sym);
1840 for (i = 0; i < nr_syms; i++) {
1841 GElf_Sym sym;
1842
1843 if (!gelf_getsym(symbols, i, &sym))
1844 continue;
1845 if (sym.st_shndx != obj->efile.maps_shndx)
1846 continue;
1847 nr_maps++;
1848 }
1849
1850 pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n",
1851 nr_maps, data->d_size, obj->path);
1852
1853 if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {
1854 pr_warn("elf: unable to determine legacy map definition size in %s\n",
1855 obj->path);
1856 return -EINVAL;
1857 }
1858 map_def_sz = data->d_size / nr_maps;
1859
1860
1861 for (i = 0; i < nr_syms; i++) {
1862 GElf_Sym sym;
1863 const char *map_name;
1864 struct bpf_map_def *def;
1865 struct bpf_map *map;
1866
1867 if (!gelf_getsym(symbols, i, &sym))
1868 continue;
1869 if (sym.st_shndx != obj->efile.maps_shndx)
1870 continue;
1871
1872 map = bpf_object__add_map(obj);
1873 if (IS_ERR(map))
1874 return PTR_ERR(map);
1875
1876 map_name = elf_sym_str(obj, sym.st_name);
1877 if (!map_name) {
1878 pr_warn("failed to get map #%d name sym string for obj %s\n",
1879 i, obj->path);
1880 return -LIBBPF_ERRNO__FORMAT;
1881 }
1882
1883 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION
1884 || GELF_ST_BIND(sym.st_info) == STB_LOCAL) {
1885 pr_warn("map '%s' (legacy): static maps are not supported\n", map_name);
1886 return -ENOTSUP;
1887 }
1888
1889 map->libbpf_type = LIBBPF_MAP_UNSPEC;
1890 map->sec_idx = sym.st_shndx;
1891 map->sec_offset = sym.st_value;
1892 pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
1893 map_name, map->sec_idx, map->sec_offset);
1894 if (sym.st_value + map_def_sz > data->d_size) {
1895 pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
1896 obj->path, map_name);
1897 return -EINVAL;
1898 }
1899
1900 map->name = strdup(map_name);
1901 if (!map->name) {
1902 pr_warn("failed to alloc map name\n");
1903 return -ENOMEM;
1904 }
1905 pr_debug("map %d is \"%s\"\n", i, map->name);
1906 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
1907
1908
1909
1910
1911
1912
1913 if (map_def_sz <= sizeof(struct bpf_map_def)) {
1914 memcpy(&map->def, def, map_def_sz);
1915 } else {
1916
1917
1918
1919
1920
1921
1922 char *b;
1923
1924 for (b = ((char *)def) + sizeof(struct bpf_map_def);
1925 b < ((char *)def) + map_def_sz; b++) {
1926 if (*b != 0) {
1927 pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",
1928 obj->path, map_name);
1929 if (strict)
1930 return -EINVAL;
1931 }
1932 }
1933 memcpy(&map->def, def, sizeof(struct bpf_map_def));
1934 }
1935 }
1936 return 0;
1937}
1938
1939const struct btf_type *
1940skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
1941{
1942 const struct btf_type *t = btf__type_by_id(btf, id);
1943
1944 if (res_id)
1945 *res_id = id;
1946
1947 while (btf_is_mod(t) || btf_is_typedef(t)) {
1948 if (res_id)
1949 *res_id = t->type;
1950 t = btf__type_by_id(btf, t->type);
1951 }
1952
1953 return t;
1954}
1955
1956static const struct btf_type *
1957resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
1958{
1959 const struct btf_type *t;
1960
1961 t = skip_mods_and_typedefs(btf, id, NULL);
1962 if (!btf_is_ptr(t))
1963 return NULL;
1964
1965 t = skip_mods_and_typedefs(btf, t->type, res_id);
1966
1967 return btf_is_func_proto(t) ? t : NULL;
1968}
1969
1970static const char *__btf_kind_str(__u16 kind)
1971{
1972 switch (kind) {
1973 case BTF_KIND_UNKN: return "void";
1974 case BTF_KIND_INT: return "int";
1975 case BTF_KIND_PTR: return "ptr";
1976 case BTF_KIND_ARRAY: return "array";
1977 case BTF_KIND_STRUCT: return "struct";
1978 case BTF_KIND_UNION: return "union";
1979 case BTF_KIND_ENUM: return "enum";
1980 case BTF_KIND_FWD: return "fwd";
1981 case BTF_KIND_TYPEDEF: return "typedef";
1982 case BTF_KIND_VOLATILE: return "volatile";
1983 case BTF_KIND_CONST: return "const";
1984 case BTF_KIND_RESTRICT: return "restrict";
1985 case BTF_KIND_FUNC: return "func";
1986 case BTF_KIND_FUNC_PROTO: return "func_proto";
1987 case BTF_KIND_VAR: return "var";
1988 case BTF_KIND_DATASEC: return "datasec";
1989 case BTF_KIND_FLOAT: return "float";
1990 default: return "unknown";
1991 }
1992}
1993
1994const char *btf_kind_str(const struct btf_type *t)
1995{
1996 return __btf_kind_str(btf_kind(t));
1997}
1998
1999
2000
2001
2002
2003
2004
2005
2006static bool get_map_field_int(const char *map_name, const struct btf *btf,
2007 const struct btf_member *m, __u32 *res)
2008{
2009 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2010 const char *name = btf__name_by_offset(btf, m->name_off);
2011 const struct btf_array *arr_info;
2012 const struct btf_type *arr_t;
2013
2014 if (!btf_is_ptr(t)) {
2015 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
2016 map_name, name, btf_kind_str(t));
2017 return false;
2018 }
2019
2020 arr_t = btf__type_by_id(btf, t->type);
2021 if (!arr_t) {
2022 pr_warn("map '%s': attr '%s': type [%u] not found.\n",
2023 map_name, name, t->type);
2024 return false;
2025 }
2026 if (!btf_is_array(arr_t)) {
2027 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
2028 map_name, name, btf_kind_str(arr_t));
2029 return false;
2030 }
2031 arr_info = btf_array(arr_t);
2032 *res = arr_info->nelems;
2033 return true;
2034}
2035
2036static int build_map_pin_path(struct bpf_map *map, const char *path)
2037{
2038 char buf[PATH_MAX];
2039 int len;
2040
2041 if (!path)
2042 path = "/sys/fs/bpf";
2043
2044 len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
2045 if (len < 0)
2046 return -EINVAL;
2047 else if (len >= PATH_MAX)
2048 return -ENAMETOOLONG;
2049
2050 return bpf_map__set_pin_path(map, buf);
2051}
2052
2053int parse_btf_map_def(const char *map_name, struct btf *btf,
2054 const struct btf_type *def_t, bool strict,
2055 struct btf_map_def *map_def, struct btf_map_def *inner_def)
2056{
2057 const struct btf_type *t;
2058 const struct btf_member *m;
2059 bool is_inner = inner_def == NULL;
2060 int vlen, i;
2061
2062 vlen = btf_vlen(def_t);
2063 m = btf_members(def_t);
2064 for (i = 0; i < vlen; i++, m++) {
2065 const char *name = btf__name_by_offset(btf, m->name_off);
2066
2067 if (!name) {
2068 pr_warn("map '%s': invalid field #%d.\n", map_name, i);
2069 return -EINVAL;
2070 }
2071 if (strcmp(name, "type") == 0) {
2072 if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2073 return -EINVAL;
2074 map_def->parts |= MAP_DEF_MAP_TYPE;
2075 } else if (strcmp(name, "max_entries") == 0) {
2076 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2077 return -EINVAL;
2078 map_def->parts |= MAP_DEF_MAX_ENTRIES;
2079 } else if (strcmp(name, "map_flags") == 0) {
2080 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2081 return -EINVAL;
2082 map_def->parts |= MAP_DEF_MAP_FLAGS;
2083 } else if (strcmp(name, "numa_node") == 0) {
2084 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2085 return -EINVAL;
2086 map_def->parts |= MAP_DEF_NUMA_NODE;
2087 } else if (strcmp(name, "key_size") == 0) {
2088 __u32 sz;
2089
2090 if (!get_map_field_int(map_name, btf, m, &sz))
2091 return -EINVAL;
2092 if (map_def->key_size && map_def->key_size != sz) {
2093 pr_warn("map '%s': conflicting key size %u != %u.\n",
2094 map_name, map_def->key_size, sz);
2095 return -EINVAL;
2096 }
2097 map_def->key_size = sz;
2098 map_def->parts |= MAP_DEF_KEY_SIZE;
2099 } else if (strcmp(name, "key") == 0) {
2100 __s64 sz;
2101
2102 t = btf__type_by_id(btf, m->type);
2103 if (!t) {
2104 pr_warn("map '%s': key type [%d] not found.\n",
2105 map_name, m->type);
2106 return -EINVAL;
2107 }
2108 if (!btf_is_ptr(t)) {
2109 pr_warn("map '%s': key spec is not PTR: %s.\n",
2110 map_name, btf_kind_str(t));
2111 return -EINVAL;
2112 }
2113 sz = btf__resolve_size(btf, t->type);
2114 if (sz < 0) {
2115 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2116 map_name, t->type, (ssize_t)sz);
2117 return sz;
2118 }
2119 if (map_def->key_size && map_def->key_size != sz) {
2120 pr_warn("map '%s': conflicting key size %u != %zd.\n",
2121 map_name, map_def->key_size, (ssize_t)sz);
2122 return -EINVAL;
2123 }
2124 map_def->key_size = sz;
2125 map_def->key_type_id = t->type;
2126 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2127 } else if (strcmp(name, "value_size") == 0) {
2128 __u32 sz;
2129
2130 if (!get_map_field_int(map_name, btf, m, &sz))
2131 return -EINVAL;
2132 if (map_def->value_size && map_def->value_size != sz) {
2133 pr_warn("map '%s': conflicting value size %u != %u.\n",
2134 map_name, map_def->value_size, sz);
2135 return -EINVAL;
2136 }
2137 map_def->value_size = sz;
2138 map_def->parts |= MAP_DEF_VALUE_SIZE;
2139 } else if (strcmp(name, "value") == 0) {
2140 __s64 sz;
2141
2142 t = btf__type_by_id(btf, m->type);
2143 if (!t) {
2144 pr_warn("map '%s': value type [%d] not found.\n",
2145 map_name, m->type);
2146 return -EINVAL;
2147 }
2148 if (!btf_is_ptr(t)) {
2149 pr_warn("map '%s': value spec is not PTR: %s.\n",
2150 map_name, btf_kind_str(t));
2151 return -EINVAL;
2152 }
2153 sz = btf__resolve_size(btf, t->type);
2154 if (sz < 0) {
2155 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2156 map_name, t->type, (ssize_t)sz);
2157 return sz;
2158 }
2159 if (map_def->value_size && map_def->value_size != sz) {
2160 pr_warn("map '%s': conflicting value size %u != %zd.\n",
2161 map_name, map_def->value_size, (ssize_t)sz);
2162 return -EINVAL;
2163 }
2164 map_def->value_size = sz;
2165 map_def->value_type_id = t->type;
2166 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2167 }
2168 else if (strcmp(name, "values") == 0) {
2169 char inner_map_name[128];
2170 int err;
2171
2172 if (is_inner) {
2173 pr_warn("map '%s': multi-level inner maps not supported.\n",
2174 map_name);
2175 return -ENOTSUP;
2176 }
2177 if (i != vlen - 1) {
2178 pr_warn("map '%s': '%s' member should be last.\n",
2179 map_name, name);
2180 return -EINVAL;
2181 }
2182 if (!bpf_map_type__is_map_in_map(map_def->map_type)) {
2183 pr_warn("map '%s': should be map-in-map.\n",
2184 map_name);
2185 return -ENOTSUP;
2186 }
2187 if (map_def->value_size && map_def->value_size != 4) {
2188 pr_warn("map '%s': conflicting value size %u != 4.\n",
2189 map_name, map_def->value_size);
2190 return -EINVAL;
2191 }
2192 map_def->value_size = 4;
2193 t = btf__type_by_id(btf, m->type);
2194 if (!t) {
2195 pr_warn("map '%s': map-in-map inner type [%d] not found.\n",
2196 map_name, m->type);
2197 return -EINVAL;
2198 }
2199 if (!btf_is_array(t) || btf_array(t)->nelems) {
2200 pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n",
2201 map_name);
2202 return -EINVAL;
2203 }
2204 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2205 if (!btf_is_ptr(t)) {
2206 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2207 map_name, btf_kind_str(t));
2208 return -EINVAL;
2209 }
2210 t = skip_mods_and_typedefs(btf, t->type, NULL);
2211 if (!btf_is_struct(t)) {
2212 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2213 map_name, btf_kind_str(t));
2214 return -EINVAL;
2215 }
2216
2217 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);
2218 err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);
2219 if (err)
2220 return err;
2221
2222 map_def->parts |= MAP_DEF_INNER_MAP;
2223 } else if (strcmp(name, "pinning") == 0) {
2224 __u32 val;
2225
2226 if (is_inner) {
2227 pr_warn("map '%s': inner def can't be pinned.\n", map_name);
2228 return -EINVAL;
2229 }
2230 if (!get_map_field_int(map_name, btf, m, &val))
2231 return -EINVAL;
2232 if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {
2233 pr_warn("map '%s': invalid pinning value %u.\n",
2234 map_name, val);
2235 return -EINVAL;
2236 }
2237 map_def->pinning = val;
2238 map_def->parts |= MAP_DEF_PINNING;
2239 } else {
2240 if (strict) {
2241 pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
2242 return -ENOTSUP;
2243 }
2244 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
2245 }
2246 }
2247
2248 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2249 pr_warn("map '%s': map type isn't specified.\n", map_name);
2250 return -EINVAL;
2251 }
2252
2253 return 0;
2254}
2255
2256static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)
2257{
2258 map->def.type = def->map_type;
2259 map->def.key_size = def->key_size;
2260 map->def.value_size = def->value_size;
2261 map->def.max_entries = def->max_entries;
2262 map->def.map_flags = def->map_flags;
2263
2264 map->numa_node = def->numa_node;
2265 map->btf_key_type_id = def->key_type_id;
2266 map->btf_value_type_id = def->value_type_id;
2267
2268 if (def->parts & MAP_DEF_MAP_TYPE)
2269 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2270
2271 if (def->parts & MAP_DEF_KEY_TYPE)
2272 pr_debug("map '%s': found key [%u], sz = %u.\n",
2273 map->name, def->key_type_id, def->key_size);
2274 else if (def->parts & MAP_DEF_KEY_SIZE)
2275 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2276
2277 if (def->parts & MAP_DEF_VALUE_TYPE)
2278 pr_debug("map '%s': found value [%u], sz = %u.\n",
2279 map->name, def->value_type_id, def->value_size);
2280 else if (def->parts & MAP_DEF_VALUE_SIZE)
2281 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2282
2283 if (def->parts & MAP_DEF_MAX_ENTRIES)
2284 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2285 if (def->parts & MAP_DEF_MAP_FLAGS)
2286 pr_debug("map '%s': found map_flags = %u.\n", map->name, def->map_flags);
2287 if (def->parts & MAP_DEF_PINNING)
2288 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2289 if (def->parts & MAP_DEF_NUMA_NODE)
2290 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2291
2292 if (def->parts & MAP_DEF_INNER_MAP)
2293 pr_debug("map '%s': found inner map definition.\n", map->name);
2294}
2295
2296static const char *btf_var_linkage_str(__u32 linkage)
2297{
2298 switch (linkage) {
2299 case BTF_VAR_STATIC: return "static";
2300 case BTF_VAR_GLOBAL_ALLOCATED: return "global";
2301 case BTF_VAR_GLOBAL_EXTERN: return "extern";
2302 default: return "unknown";
2303 }
2304}
2305
2306static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2307 const struct btf_type *sec,
2308 int var_idx, int sec_idx,
2309 const Elf_Data *data, bool strict,
2310 const char *pin_root_path)
2311{
2312 struct btf_map_def map_def = {}, inner_def = {};
2313 const struct btf_type *var, *def;
2314 const struct btf_var_secinfo *vi;
2315 const struct btf_var *var_extra;
2316 const char *map_name;
2317 struct bpf_map *map;
2318 int err;
2319
2320 vi = btf_var_secinfos(sec) + var_idx;
2321 var = btf__type_by_id(obj->btf, vi->type);
2322 var_extra = btf_var(var);
2323 map_name = btf__name_by_offset(obj->btf, var->name_off);
2324
2325 if (map_name == NULL || map_name[0] == '\0') {
2326 pr_warn("map #%d: empty name.\n", var_idx);
2327 return -EINVAL;
2328 }
2329 if ((__u64)vi->offset + vi->size > data->d_size) {
2330 pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2331 return -EINVAL;
2332 }
2333 if (!btf_is_var(var)) {
2334 pr_warn("map '%s': unexpected var kind %s.\n",
2335 map_name, btf_kind_str(var));
2336 return -EINVAL;
2337 }
2338 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2339 pr_warn("map '%s': unsupported map linkage %s.\n",
2340 map_name, btf_var_linkage_str(var_extra->linkage));
2341 return -EOPNOTSUPP;
2342 }
2343
2344 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2345 if (!btf_is_struct(def)) {
2346 pr_warn("map '%s': unexpected def kind %s.\n",
2347 map_name, btf_kind_str(var));
2348 return -EINVAL;
2349 }
2350 if (def->size > vi->size) {
2351 pr_warn("map '%s': invalid def size.\n", map_name);
2352 return -EINVAL;
2353 }
2354
2355 map = bpf_object__add_map(obj);
2356 if (IS_ERR(map))
2357 return PTR_ERR(map);
2358 map->name = strdup(map_name);
2359 if (!map->name) {
2360 pr_warn("map '%s': failed to alloc map name.\n", map_name);
2361 return -ENOMEM;
2362 }
2363 map->libbpf_type = LIBBPF_MAP_UNSPEC;
2364 map->def.type = BPF_MAP_TYPE_UNSPEC;
2365 map->sec_idx = sec_idx;
2366 map->sec_offset = vi->offset;
2367 map->btf_var_idx = var_idx;
2368 pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2369 map_name, map->sec_idx, map->sec_offset);
2370
2371 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2372 if (err)
2373 return err;
2374
2375 fill_map_from_def(map, &map_def);
2376
2377 if (map_def.pinning == LIBBPF_PIN_BY_NAME) {
2378 err = build_map_pin_path(map, pin_root_path);
2379 if (err) {
2380 pr_warn("map '%s': couldn't build pin path.\n", map->name);
2381 return err;
2382 }
2383 }
2384
2385 if (map_def.parts & MAP_DEF_INNER_MAP) {
2386 map->inner_map = calloc(1, sizeof(*map->inner_map));
2387 if (!map->inner_map)
2388 return -ENOMEM;
2389 map->inner_map->fd = -1;
2390 map->inner_map->sec_idx = sec_idx;
2391 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2392 if (!map->inner_map->name)
2393 return -ENOMEM;
2394 sprintf(map->inner_map->name, "%s.inner", map_name);
2395
2396 fill_map_from_def(map->inner_map, &inner_def);
2397 }
2398
2399 return 0;
2400}
2401
2402static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2403 const char *pin_root_path)
2404{
2405 const struct btf_type *sec = NULL;
2406 int nr_types, i, vlen, err;
2407 const struct btf_type *t;
2408 const char *name;
2409 Elf_Data *data;
2410 Elf_Scn *scn;
2411
2412 if (obj->efile.btf_maps_shndx < 0)
2413 return 0;
2414
2415 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2416 data = elf_sec_data(obj, scn);
2417 if (!scn || !data) {
2418 pr_warn("elf: failed to get %s map definitions for %s\n",
2419 MAPS_ELF_SEC, obj->path);
2420 return -EINVAL;
2421 }
2422
2423 nr_types = btf__get_nr_types(obj->btf);
2424 for (i = 1; i <= nr_types; i++) {
2425 t = btf__type_by_id(obj->btf, i);
2426 if (!btf_is_datasec(t))
2427 continue;
2428 name = btf__name_by_offset(obj->btf, t->name_off);
2429 if (strcmp(name, MAPS_ELF_SEC) == 0) {
2430 sec = t;
2431 obj->efile.btf_maps_sec_btf_id = i;
2432 break;
2433 }
2434 }
2435
2436 if (!sec) {
2437 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2438 return -ENOENT;
2439 }
2440
2441 vlen = btf_vlen(sec);
2442 for (i = 0; i < vlen; i++) {
2443 err = bpf_object__init_user_btf_map(obj, sec, i,
2444 obj->efile.btf_maps_shndx,
2445 data, strict,
2446 pin_root_path);
2447 if (err)
2448 return err;
2449 }
2450
2451 return 0;
2452}
2453
2454static int bpf_object__init_maps(struct bpf_object *obj,
2455 const struct bpf_object_open_opts *opts)
2456{
2457 const char *pin_root_path;
2458 bool strict;
2459 int err;
2460
2461 strict = !OPTS_GET(opts, relaxed_maps, false);
2462 pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2463
2464 err = bpf_object__init_user_maps(obj, strict);
2465 err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2466 err = err ?: bpf_object__init_global_data_maps(obj);
2467 err = err ?: bpf_object__init_kconfig_map(obj);
2468 err = err ?: bpf_object__init_struct_ops_maps(obj);
2469
2470 return err;
2471}
2472
2473static bool section_have_execinstr(struct bpf_object *obj, int idx)
2474{
2475 GElf_Shdr sh;
2476
2477 if (elf_sec_hdr(obj, elf_sec_by_idx(obj, idx), &sh))
2478 return false;
2479
2480 return sh.sh_flags & SHF_EXECINSTR;
2481}
2482
2483static bool btf_needs_sanitization(struct bpf_object *obj)
2484{
2485 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2486 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2487 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2488 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2489
2490 return !has_func || !has_datasec || !has_func_global || !has_float;
2491}
2492
2493static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2494{
2495 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2496 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2497 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2498 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2499 struct btf_type *t;
2500 int i, j, vlen;
2501
2502 for (i = 1; i <= btf__get_nr_types(btf); i++) {
2503 t = (struct btf_type *)btf__type_by_id(btf, i);
2504
2505 if (!has_datasec && btf_is_var(t)) {
2506
2507 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2508
2509
2510
2511
2512
2513 t->size = 1;
2514 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2515 } else if (!has_datasec && btf_is_datasec(t)) {
2516
2517 const struct btf_var_secinfo *v = btf_var_secinfos(t);
2518 struct btf_member *m = btf_members(t);
2519 struct btf_type *vt;
2520 char *name;
2521
2522 name = (char *)btf__name_by_offset(btf, t->name_off);
2523 while (*name) {
2524 if (*name == '.')
2525 *name = '_';
2526 name++;
2527 }
2528
2529 vlen = btf_vlen(t);
2530 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2531 for (j = 0; j < vlen; j++, v++, m++) {
2532
2533 m->offset = v->offset * 8;
2534 m->type = v->type;
2535
2536 vt = (void *)btf__type_by_id(btf, v->type);
2537 m->name_off = vt->name_off;
2538 }
2539 } else if (!has_func && btf_is_func_proto(t)) {
2540
2541 vlen = btf_vlen(t);
2542 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2543 t->size = sizeof(__u32);
2544 } else if (!has_func && btf_is_func(t)) {
2545
2546 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2547 } else if (!has_func_global && btf_is_func(t)) {
2548
2549 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2550 } else if (!has_float && btf_is_float(t)) {
2551
2552
2553
2554
2555 t->name_off = 0;
2556 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
2557 }
2558 }
2559}
2560
2561static bool libbpf_needs_btf(const struct bpf_object *obj)
2562{
2563 return obj->efile.btf_maps_shndx >= 0 ||
2564 obj->efile.st_ops_shndx >= 0 ||
2565 obj->nr_extern > 0;
2566}
2567
2568static bool kernel_needs_btf(const struct bpf_object *obj)
2569{
2570 return obj->efile.st_ops_shndx >= 0;
2571}
2572
2573static int bpf_object__init_btf(struct bpf_object *obj,
2574 Elf_Data *btf_data,
2575 Elf_Data *btf_ext_data)
2576{
2577 int err = -ENOENT;
2578
2579 if (btf_data) {
2580 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2581 err = libbpf_get_error(obj->btf);
2582 if (err) {
2583 obj->btf = NULL;
2584 pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err);
2585 goto out;
2586 }
2587
2588 btf__set_pointer_size(obj->btf, 8);
2589 }
2590 if (btf_ext_data) {
2591 if (!obj->btf) {
2592 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2593 BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2594 goto out;
2595 }
2596 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
2597 err = libbpf_get_error(obj->btf_ext);
2598 if (err) {
2599 pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n",
2600 BTF_EXT_ELF_SEC, err);
2601 obj->btf_ext = NULL;
2602 goto out;
2603 }
2604 }
2605out:
2606 if (err && libbpf_needs_btf(obj)) {
2607 pr_warn("BTF is required, but is missing or corrupted.\n");
2608 return err;
2609 }
2610 return 0;
2611}
2612
2613static int bpf_object__finalize_btf(struct bpf_object *obj)
2614{
2615 int err;
2616
2617 if (!obj->btf)
2618 return 0;
2619
2620 err = btf__finalize_data(obj, obj->btf);
2621 if (err) {
2622 pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
2623 return err;
2624 }
2625
2626 return 0;
2627}
2628
2629static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
2630{
2631 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
2632 prog->type == BPF_PROG_TYPE_LSM)
2633 return true;
2634
2635
2636
2637
2638 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
2639 return true;
2640
2641 return false;
2642}
2643
2644static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
2645{
2646 struct bpf_program *prog;
2647 int i;
2648
2649
2650
2651
2652 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
2653 return true;
2654
2655
2656 for (i = 0; i < obj->nr_extern; i++) {
2657 const struct extern_desc *ext;
2658
2659 ext = &obj->externs[i];
2660 if (ext->type == EXT_KSYM && ext->ksym.type_id)
2661 return true;
2662 }
2663
2664 bpf_object__for_each_program(prog, obj) {
2665 if (!prog->load)
2666 continue;
2667 if (prog_needs_vmlinux_btf(prog))
2668 return true;
2669 }
2670
2671 return false;
2672}
2673
2674static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
2675{
2676 int err;
2677
2678
2679 if (obj->btf_vmlinux || obj->gen_loader)
2680 return 0;
2681
2682 if (!force && !obj_needs_vmlinux_btf(obj))
2683 return 0;
2684
2685 obj->btf_vmlinux = btf__load_vmlinux_btf();
2686 err = libbpf_get_error(obj->btf_vmlinux);
2687 if (err) {
2688 pr_warn("Error loading vmlinux BTF: %d\n", err);
2689 obj->btf_vmlinux = NULL;
2690 return err;
2691 }
2692 return 0;
2693}
2694
2695static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
2696{
2697 struct btf *kern_btf = obj->btf;
2698 bool btf_mandatory, sanitize;
2699 int i, err = 0;
2700
2701 if (!obj->btf)
2702 return 0;
2703
2704 if (!kernel_supports(obj, FEAT_BTF)) {
2705 if (kernel_needs_btf(obj)) {
2706 err = -EOPNOTSUPP;
2707 goto report;
2708 }
2709 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
2710 return 0;
2711 }
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721 for (i = 0; i < obj->nr_programs; i++) {
2722 struct bpf_program *prog = &obj->programs[i];
2723 struct btf_type *t;
2724 const char *name;
2725 int j, n;
2726
2727 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
2728 continue;
2729
2730 n = btf__get_nr_types(obj->btf);
2731 for (j = 1; j <= n; j++) {
2732 t = btf_type_by_id(obj->btf, j);
2733 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL)
2734 continue;
2735
2736 name = btf__str_by_offset(obj->btf, t->name_off);
2737 if (strcmp(name, prog->name) != 0)
2738 continue;
2739
2740 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
2741 break;
2742 }
2743 }
2744
2745 sanitize = btf_needs_sanitization(obj);
2746 if (sanitize) {
2747 const void *raw_data;
2748 __u32 sz;
2749
2750
2751 raw_data = btf__get_raw_data(obj->btf, &sz);
2752 kern_btf = btf__new(raw_data, sz);
2753 err = libbpf_get_error(kern_btf);
2754 if (err)
2755 return err;
2756
2757
2758 btf__set_pointer_size(obj->btf, 8);
2759 bpf_object__sanitize_btf(obj, kern_btf);
2760 }
2761
2762 if (obj->gen_loader) {
2763 __u32 raw_size = 0;
2764 const void *raw_data = btf__get_raw_data(kern_btf, &raw_size);
2765
2766 if (!raw_data)
2767 return -ENOMEM;
2768 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
2769
2770
2771
2772 btf__set_fd(kern_btf, 0);
2773 } else {
2774 err = btf__load_into_kernel(kern_btf);
2775 }
2776 if (sanitize) {
2777 if (!err) {
2778
2779 btf__set_fd(obj->btf, btf__fd(kern_btf));
2780 btf__set_fd(kern_btf, -1);
2781 }
2782 btf__free(kern_btf);
2783 }
2784report:
2785 if (err) {
2786 btf_mandatory = kernel_needs_btf(obj);
2787 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
2788 btf_mandatory ? "BTF is mandatory, can't proceed."
2789 : "BTF is optional, ignoring.");
2790 if (!btf_mandatory)
2791 err = 0;
2792 }
2793 return err;
2794}
2795
2796static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
2797{
2798 const char *name;
2799
2800 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
2801 if (!name) {
2802 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
2803 off, obj->path, elf_errmsg(-1));
2804 return NULL;
2805 }
2806
2807 return name;
2808}
2809
2810static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
2811{
2812 const char *name;
2813
2814 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
2815 if (!name) {
2816 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
2817 off, obj->path, elf_errmsg(-1));
2818 return NULL;
2819 }
2820
2821 return name;
2822}
2823
2824static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
2825{
2826 Elf_Scn *scn;
2827
2828 scn = elf_getscn(obj->efile.elf, idx);
2829 if (!scn) {
2830 pr_warn("elf: failed to get section(%zu) from %s: %s\n",
2831 idx, obj->path, elf_errmsg(-1));
2832 return NULL;
2833 }
2834 return scn;
2835}
2836
2837static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
2838{
2839 Elf_Scn *scn = NULL;
2840 Elf *elf = obj->efile.elf;
2841 const char *sec_name;
2842
2843 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2844 sec_name = elf_sec_name(obj, scn);
2845 if (!sec_name)
2846 return NULL;
2847
2848 if (strcmp(sec_name, name) != 0)
2849 continue;
2850
2851 return scn;
2852 }
2853 return NULL;
2854}
2855
2856static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr)
2857{
2858 if (!scn)
2859 return -EINVAL;
2860
2861 if (gelf_getshdr(scn, hdr) != hdr) {
2862 pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
2863 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
2864 return -EINVAL;
2865 }
2866
2867 return 0;
2868}
2869
2870static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
2871{
2872 const char *name;
2873 GElf_Shdr sh;
2874
2875 if (!scn)
2876 return NULL;
2877
2878 if (elf_sec_hdr(obj, scn, &sh))
2879 return NULL;
2880
2881 name = elf_sec_str(obj, sh.sh_name);
2882 if (!name) {
2883 pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
2884 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
2885 return NULL;
2886 }
2887
2888 return name;
2889}
2890
2891static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
2892{
2893 Elf_Data *data;
2894
2895 if (!scn)
2896 return NULL;
2897
2898 data = elf_getdata(scn, 0);
2899 if (!data) {
2900 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
2901 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
2902 obj->path, elf_errmsg(-1));
2903 return NULL;
2904 }
2905
2906 return data;
2907}
2908
2909static bool is_sec_name_dwarf(const char *name)
2910{
2911
2912 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
2913}
2914
2915static bool ignore_elf_section(GElf_Shdr *hdr, const char *name)
2916{
2917
2918 if (hdr->sh_type == SHT_STRTAB)
2919 return true;
2920
2921
2922 if (hdr->sh_type == SHT_LLVM_ADDRSIG)
2923 return true;
2924
2925
2926 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
2927 strcmp(name, ".text") == 0)
2928 return true;
2929
2930
2931 if (is_sec_name_dwarf(name))
2932 return true;
2933
2934 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
2935 name += sizeof(".rel") - 1;
2936
2937 if (is_sec_name_dwarf(name))
2938 return true;
2939
2940
2941 if (strcmp(name, BTF_ELF_SEC) == 0 ||
2942 strcmp(name, BTF_EXT_ELF_SEC) == 0)
2943 return true;
2944 }
2945
2946 return false;
2947}
2948
2949static int cmp_progs(const void *_a, const void *_b)
2950{
2951 const struct bpf_program *a = _a;
2952 const struct bpf_program *b = _b;
2953
2954 if (a->sec_idx != b->sec_idx)
2955 return a->sec_idx < b->sec_idx ? -1 : 1;
2956
2957
2958 return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
2959}
2960
2961static int bpf_object__elf_collect(struct bpf_object *obj)
2962{
2963 Elf *elf = obj->efile.elf;
2964 Elf_Data *btf_ext_data = NULL;
2965 Elf_Data *btf_data = NULL;
2966 int idx = 0, err = 0;
2967 const char *name;
2968 Elf_Data *data;
2969 Elf_Scn *scn;
2970 GElf_Shdr sh;
2971
2972
2973
2974
2975 scn = NULL;
2976 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2977 if (elf_sec_hdr(obj, scn, &sh))
2978 return -LIBBPF_ERRNO__FORMAT;
2979
2980 if (sh.sh_type == SHT_SYMTAB) {
2981 if (obj->efile.symbols) {
2982 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
2983 return -LIBBPF_ERRNO__FORMAT;
2984 }
2985
2986 data = elf_sec_data(obj, scn);
2987 if (!data)
2988 return -LIBBPF_ERRNO__FORMAT;
2989
2990 obj->efile.symbols = data;
2991 obj->efile.symbols_shndx = elf_ndxscn(scn);
2992 obj->efile.strtabidx = sh.sh_link;
2993 }
2994 }
2995
2996 scn = NULL;
2997 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2998 idx++;
2999
3000 if (elf_sec_hdr(obj, scn, &sh))
3001 return -LIBBPF_ERRNO__FORMAT;
3002
3003 name = elf_sec_str(obj, sh.sh_name);
3004 if (!name)
3005 return -LIBBPF_ERRNO__FORMAT;
3006
3007 if (ignore_elf_section(&sh, name))
3008 continue;
3009
3010 data = elf_sec_data(obj, scn);
3011 if (!data)
3012 return -LIBBPF_ERRNO__FORMAT;
3013
3014 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
3015 idx, name, (unsigned long)data->d_size,
3016 (int)sh.sh_link, (unsigned long)sh.sh_flags,
3017 (int)sh.sh_type);
3018
3019 if (strcmp(name, "license") == 0) {
3020 err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3021 if (err)
3022 return err;
3023 } else if (strcmp(name, "version") == 0) {
3024 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3025 if (err)
3026 return err;
3027 } else if (strcmp(name, "maps") == 0) {
3028 obj->efile.maps_shndx = idx;
3029 } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
3030 obj->efile.btf_maps_shndx = idx;
3031 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
3032 btf_data = data;
3033 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
3034 btf_ext_data = data;
3035 } else if (sh.sh_type == SHT_SYMTAB) {
3036
3037 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
3038 if (sh.sh_flags & SHF_EXECINSTR) {
3039 if (strcmp(name, ".text") == 0)
3040 obj->efile.text_shndx = idx;
3041 err = bpf_object__add_programs(obj, data, name, idx);
3042 if (err)
3043 return err;
3044 } else if (strcmp(name, DATA_SEC) == 0) {
3045 obj->efile.data = data;
3046 obj->efile.data_shndx = idx;
3047 } else if (strcmp(name, RODATA_SEC) == 0) {
3048 obj->efile.rodata = data;
3049 obj->efile.rodata_shndx = idx;
3050 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
3051 obj->efile.st_ops_data = data;
3052 obj->efile.st_ops_shndx = idx;
3053 } else {
3054 pr_info("elf: skipping unrecognized data section(%d) %s\n",
3055 idx, name);
3056 }
3057 } else if (sh.sh_type == SHT_REL) {
3058 int nr_sects = obj->efile.nr_reloc_sects;
3059 void *sects = obj->efile.reloc_sects;
3060 int sec = sh.sh_info;
3061
3062
3063 if (!section_have_execinstr(obj, sec) &&
3064 strcmp(name, ".rel" STRUCT_OPS_SEC) &&
3065 strcmp(name, ".rel" MAPS_ELF_SEC)) {
3066 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
3067 idx, name, sec,
3068 elf_sec_name(obj, elf_sec_by_idx(obj, sec)) ?: "<?>");
3069 continue;
3070 }
3071
3072 sects = libbpf_reallocarray(sects, nr_sects + 1,
3073 sizeof(*obj->efile.reloc_sects));
3074 if (!sects)
3075 return -ENOMEM;
3076
3077 obj->efile.reloc_sects = sects;
3078 obj->efile.nr_reloc_sects++;
3079
3080 obj->efile.reloc_sects[nr_sects].shdr = sh;
3081 obj->efile.reloc_sects[nr_sects].data = data;
3082 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) {
3083 obj->efile.bss = data;
3084 obj->efile.bss_shndx = idx;
3085 } else {
3086 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
3087 (size_t)sh.sh_size);
3088 }
3089 }
3090
3091 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3092 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3093 return -LIBBPF_ERRNO__FORMAT;
3094 }
3095
3096
3097
3098 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3099
3100 return bpf_object__init_btf(obj, btf_data, btf_ext_data);
3101}
3102
3103static bool sym_is_extern(const GElf_Sym *sym)
3104{
3105 int bind = GELF_ST_BIND(sym->st_info);
3106
3107 return sym->st_shndx == SHN_UNDEF &&
3108 (bind == STB_GLOBAL || bind == STB_WEAK) &&
3109 GELF_ST_TYPE(sym->st_info) == STT_NOTYPE;
3110}
3111
3112static bool sym_is_subprog(const GElf_Sym *sym, int text_shndx)
3113{
3114 int bind = GELF_ST_BIND(sym->st_info);
3115 int type = GELF_ST_TYPE(sym->st_info);
3116
3117
3118 if (sym->st_shndx != text_shndx)
3119 return false;
3120
3121
3122 if (bind == STB_LOCAL && type == STT_SECTION)
3123 return true;
3124
3125
3126 return bind == STB_GLOBAL && type == STT_FUNC;
3127}
3128
3129static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
3130{
3131 const struct btf_type *t;
3132 const char *tname;
3133 int i, n;
3134
3135 if (!btf)
3136 return -ESRCH;
3137
3138 n = btf__get_nr_types(btf);
3139 for (i = 1; i <= n; i++) {
3140 t = btf__type_by_id(btf, i);
3141
3142 if (!btf_is_var(t) && !btf_is_func(t))
3143 continue;
3144
3145 tname = btf__name_by_offset(btf, t->name_off);
3146 if (strcmp(tname, ext_name))
3147 continue;
3148
3149 if (btf_is_var(t) &&
3150 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
3151 return -EINVAL;
3152
3153 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
3154 return -EINVAL;
3155
3156 return i;
3157 }
3158
3159 return -ENOENT;
3160}
3161
3162static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
3163 const struct btf_var_secinfo *vs;
3164 const struct btf_type *t;
3165 int i, j, n;
3166
3167 if (!btf)
3168 return -ESRCH;
3169
3170 n = btf__get_nr_types(btf);
3171 for (i = 1; i <= n; i++) {
3172 t = btf__type_by_id(btf, i);
3173
3174 if (!btf_is_datasec(t))
3175 continue;
3176
3177 vs = btf_var_secinfos(t);
3178 for (j = 0; j < btf_vlen(t); j++, vs++) {
3179 if (vs->type == ext_btf_id)
3180 return i;
3181 }
3182 }
3183
3184 return -ENOENT;
3185}
3186
3187static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
3188 bool *is_signed)
3189{
3190 const struct btf_type *t;
3191 const char *name;
3192
3193 t = skip_mods_and_typedefs(btf, id, NULL);
3194 name = btf__name_by_offset(btf, t->name_off);
3195
3196 if (is_signed)
3197 *is_signed = false;
3198 switch (btf_kind(t)) {
3199 case BTF_KIND_INT: {
3200 int enc = btf_int_encoding(t);
3201
3202 if (enc & BTF_INT_BOOL)
3203 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
3204 if (is_signed)
3205 *is_signed = enc & BTF_INT_SIGNED;
3206 if (t->size == 1)
3207 return KCFG_CHAR;
3208 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
3209 return KCFG_UNKNOWN;
3210 return KCFG_INT;
3211 }
3212 case BTF_KIND_ENUM:
3213 if (t->size != 4)
3214 return KCFG_UNKNOWN;
3215 if (strcmp(name, "libbpf_tristate"))
3216 return KCFG_UNKNOWN;
3217 return KCFG_TRISTATE;
3218 case BTF_KIND_ARRAY:
3219 if (btf_array(t)->nelems == 0)
3220 return KCFG_UNKNOWN;
3221 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
3222 return KCFG_UNKNOWN;
3223 return KCFG_CHAR_ARR;
3224 default:
3225 return KCFG_UNKNOWN;
3226 }
3227}
3228
3229static int cmp_externs(const void *_a, const void *_b)
3230{
3231 const struct extern_desc *a = _a;
3232 const struct extern_desc *b = _b;
3233
3234 if (a->type != b->type)
3235 return a->type < b->type ? -1 : 1;
3236
3237 if (a->type == EXT_KCFG) {
3238
3239 if (a->kcfg.align != b->kcfg.align)
3240 return a->kcfg.align > b->kcfg.align ? -1 : 1;
3241
3242 if (a->kcfg.sz != b->kcfg.sz)
3243 return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
3244 }
3245
3246
3247 return strcmp(a->name, b->name);
3248}
3249
3250static int find_int_btf_id(const struct btf *btf)
3251{
3252 const struct btf_type *t;
3253 int i, n;
3254
3255 n = btf__get_nr_types(btf);
3256 for (i = 1; i <= n; i++) {
3257 t = btf__type_by_id(btf, i);
3258
3259 if (btf_is_int(t) && btf_int_bits(t) == 32)
3260 return i;
3261 }
3262
3263 return 0;
3264}
3265
3266static int add_dummy_ksym_var(struct btf *btf)
3267{
3268 int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
3269 const struct btf_var_secinfo *vs;
3270 const struct btf_type *sec;
3271
3272 if (!btf)
3273 return 0;
3274
3275 sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
3276 BTF_KIND_DATASEC);
3277 if (sec_btf_id < 0)
3278 return 0;
3279
3280 sec = btf__type_by_id(btf, sec_btf_id);
3281 vs = btf_var_secinfos(sec);
3282 for (i = 0; i < btf_vlen(sec); i++, vs++) {
3283 const struct btf_type *vt;
3284
3285 vt = btf__type_by_id(btf, vs->type);
3286 if (btf_is_func(vt))
3287 break;
3288 }
3289
3290
3291 if (i == btf_vlen(sec))
3292 return 0;
3293
3294 int_btf_id = find_int_btf_id(btf);
3295 dummy_var_btf_id = btf__add_var(btf,
3296 "dummy_ksym",
3297 BTF_VAR_GLOBAL_ALLOCATED,
3298 int_btf_id);
3299 if (dummy_var_btf_id < 0)
3300 pr_warn("cannot create a dummy_ksym var\n");
3301
3302 return dummy_var_btf_id;
3303}
3304
3305static int bpf_object__collect_externs(struct bpf_object *obj)
3306{
3307 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
3308 const struct btf_type *t;
3309 struct extern_desc *ext;
3310 int i, n, off, dummy_var_btf_id;
3311 const char *ext_name, *sec_name;
3312 Elf_Scn *scn;
3313 GElf_Shdr sh;
3314
3315 if (!obj->efile.symbols)
3316 return 0;
3317
3318 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
3319 if (elf_sec_hdr(obj, scn, &sh))
3320 return -LIBBPF_ERRNO__FORMAT;
3321
3322 dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
3323 if (dummy_var_btf_id < 0)
3324 return dummy_var_btf_id;
3325
3326 n = sh.sh_size / sh.sh_entsize;
3327 pr_debug("looking for externs among %d symbols...\n", n);
3328
3329 for (i = 0; i < n; i++) {
3330 GElf_Sym sym;
3331
3332 if (!gelf_getsym(obj->efile.symbols, i, &sym))
3333 return -LIBBPF_ERRNO__FORMAT;
3334 if (!sym_is_extern(&sym))
3335 continue;
3336 ext_name = elf_sym_str(obj, sym.st_name);
3337 if (!ext_name || !ext_name[0])
3338 continue;
3339
3340 ext = obj->externs;
3341 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
3342 if (!ext)
3343 return -ENOMEM;
3344 obj->externs = ext;
3345 ext = &ext[obj->nr_extern];
3346 memset(ext, 0, sizeof(*ext));
3347 obj->nr_extern++;
3348
3349 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
3350 if (ext->btf_id <= 0) {
3351 pr_warn("failed to find BTF for extern '%s': %d\n",
3352 ext_name, ext->btf_id);
3353 return ext->btf_id;
3354 }
3355 t = btf__type_by_id(obj->btf, ext->btf_id);
3356 ext->name = btf__name_by_offset(obj->btf, t->name_off);
3357 ext->sym_idx = i;
3358 ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK;
3359
3360 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
3361 if (ext->sec_btf_id <= 0) {
3362 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
3363 ext_name, ext->btf_id, ext->sec_btf_id);
3364 return ext->sec_btf_id;
3365 }
3366 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
3367 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3368
3369 if (strcmp(sec_name, KCONFIG_SEC) == 0) {
3370 if (btf_is_func(t)) {
3371 pr_warn("extern function %s is unsupported under %s section\n",
3372 ext->name, KCONFIG_SEC);
3373 return -ENOTSUP;
3374 }
3375 kcfg_sec = sec;
3376 ext->type = EXT_KCFG;
3377 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
3378 if (ext->kcfg.sz <= 0) {
3379 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
3380 ext_name, ext->kcfg.sz);
3381 return ext->kcfg.sz;
3382 }
3383 ext->kcfg.align = btf__align_of(obj->btf, t->type);
3384 if (ext->kcfg.align <= 0) {
3385 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
3386 ext_name, ext->kcfg.align);
3387 return -EINVAL;
3388 }
3389 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
3390 &ext->kcfg.is_signed);
3391 if (ext->kcfg.type == KCFG_UNKNOWN) {
3392 pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name);
3393 return -ENOTSUP;
3394 }
3395 } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3396 if (btf_is_func(t) && ext->is_weak) {
3397 pr_warn("extern weak function %s is unsupported\n",
3398 ext->name);
3399 return -ENOTSUP;
3400 }
3401 ksym_sec = sec;
3402 ext->type = EXT_KSYM;
3403 skip_mods_and_typedefs(obj->btf, t->type,
3404 &ext->ksym.type_id);
3405 } else {
3406 pr_warn("unrecognized extern section '%s'\n", sec_name);
3407 return -ENOTSUP;
3408 }
3409 }
3410 pr_debug("collected %d externs total\n", obj->nr_extern);
3411
3412 if (!obj->nr_extern)
3413 return 0;
3414
3415
3416 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
3417
3418
3419
3420
3421
3422 if (ksym_sec) {
3423
3424
3425
3426 int int_btf_id = find_int_btf_id(obj->btf);
3427
3428
3429
3430
3431
3432 const struct btf_type *dummy_var;
3433
3434 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
3435 for (i = 0; i < obj->nr_extern; i++) {
3436 ext = &obj->externs[i];
3437 if (ext->type != EXT_KSYM)
3438 continue;
3439 pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
3440 i, ext->sym_idx, ext->name);
3441 }
3442
3443 sec = ksym_sec;
3444 n = btf_vlen(sec);
3445 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
3446 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3447 struct btf_type *vt;
3448
3449 vt = (void *)btf__type_by_id(obj->btf, vs->type);
3450 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
3451 ext = find_extern_by_name(obj, ext_name);
3452 if (!ext) {
3453 pr_warn("failed to find extern definition for BTF %s '%s'\n",
3454 btf_kind_str(vt), ext_name);
3455 return -ESRCH;
3456 }
3457 if (btf_is_func(vt)) {
3458 const struct btf_type *func_proto;
3459 struct btf_param *param;
3460 int j;
3461
3462 func_proto = btf__type_by_id(obj->btf,
3463 vt->type);
3464 param = btf_params(func_proto);
3465
3466
3467
3468 for (j = 0; j < btf_vlen(func_proto); j++)
3469 if (param[j].type && !param[j].name_off)
3470 param[j].name_off =
3471 dummy_var->name_off;
3472 vs->type = dummy_var_btf_id;
3473 vt->info &= ~0xffff;
3474 vt->info |= BTF_FUNC_GLOBAL;
3475 } else {
3476 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3477 vt->type = int_btf_id;
3478 }
3479 vs->offset = off;
3480 vs->size = sizeof(int);
3481 }
3482 sec->size = off;
3483 }
3484
3485 if (kcfg_sec) {
3486 sec = kcfg_sec;
3487
3488 off = 0;
3489 for (i = 0; i < obj->nr_extern; i++) {
3490 ext = &obj->externs[i];
3491 if (ext->type != EXT_KCFG)
3492 continue;
3493
3494 ext->kcfg.data_off = roundup(off, ext->kcfg.align);
3495 off = ext->kcfg.data_off + ext->kcfg.sz;
3496 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
3497 i, ext->sym_idx, ext->kcfg.data_off, ext->name);
3498 }
3499 sec->size = off;
3500 n = btf_vlen(sec);
3501 for (i = 0; i < n; i++) {
3502 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
3503
3504 t = btf__type_by_id(obj->btf, vs->type);
3505 ext_name = btf__name_by_offset(obj->btf, t->name_off);
3506 ext = find_extern_by_name(obj, ext_name);
3507 if (!ext) {
3508 pr_warn("failed to find extern definition for BTF var '%s'\n",
3509 ext_name);
3510 return -ESRCH;
3511 }
3512 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
3513 vs->offset = ext->kcfg.data_off;
3514 }
3515 }
3516 return 0;
3517}
3518
3519struct bpf_program *
3520bpf_object__find_program_by_title(const struct bpf_object *obj,
3521 const char *title)
3522{
3523 struct bpf_program *pos;
3524
3525 bpf_object__for_each_program(pos, obj) {
3526 if (pos->sec_name && !strcmp(pos->sec_name, title))
3527 return pos;
3528 }
3529 return errno = ENOENT, NULL;
3530}
3531
3532static bool prog_is_subprog(const struct bpf_object *obj,
3533 const struct bpf_program *prog)
3534{
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
3548}
3549
3550struct bpf_program *
3551bpf_object__find_program_by_name(const struct bpf_object *obj,
3552 const char *name)
3553{
3554 struct bpf_program *prog;
3555
3556 bpf_object__for_each_program(prog, obj) {
3557 if (prog_is_subprog(obj, prog))
3558 continue;
3559 if (!strcmp(prog->name, name))
3560 return prog;
3561 }
3562 return errno = ENOENT, NULL;
3563}
3564
3565static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
3566 int shndx)
3567{
3568 return shndx == obj->efile.data_shndx ||
3569 shndx == obj->efile.bss_shndx ||
3570 shndx == obj->efile.rodata_shndx;
3571}
3572
3573static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
3574 int shndx)
3575{
3576 return shndx == obj->efile.maps_shndx ||
3577 shndx == obj->efile.btf_maps_shndx;
3578}
3579
3580static enum libbpf_map_type
3581bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
3582{
3583 if (shndx == obj->efile.data_shndx)
3584 return LIBBPF_MAP_DATA;
3585 else if (shndx == obj->efile.bss_shndx)
3586 return LIBBPF_MAP_BSS;
3587 else if (shndx == obj->efile.rodata_shndx)
3588 return LIBBPF_MAP_RODATA;
3589 else if (shndx == obj->efile.symbols_shndx)
3590 return LIBBPF_MAP_KCONFIG;
3591 else
3592 return LIBBPF_MAP_UNSPEC;
3593}
3594
3595static int bpf_program__record_reloc(struct bpf_program *prog,
3596 struct reloc_desc *reloc_desc,
3597 __u32 insn_idx, const char *sym_name,
3598 const GElf_Sym *sym, const GElf_Rel *rel)
3599{
3600 struct bpf_insn *insn = &prog->insns[insn_idx];
3601 size_t map_idx, nr_maps = prog->obj->nr_maps;
3602 struct bpf_object *obj = prog->obj;
3603 __u32 shdr_idx = sym->st_shndx;
3604 enum libbpf_map_type type;
3605 const char *sym_sec_name;
3606 struct bpf_map *map;
3607
3608 if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
3609 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
3610 prog->name, sym_name, insn_idx, insn->code);
3611 return -LIBBPF_ERRNO__RELOC;
3612 }
3613
3614 if (sym_is_extern(sym)) {
3615 int sym_idx = GELF_R_SYM(rel->r_info);
3616 int i, n = obj->nr_extern;
3617 struct extern_desc *ext;
3618
3619 for (i = 0; i < n; i++) {
3620 ext = &obj->externs[i];
3621 if (ext->sym_idx == sym_idx)
3622 break;
3623 }
3624 if (i >= n) {
3625 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
3626 prog->name, sym_name, sym_idx);
3627 return -LIBBPF_ERRNO__RELOC;
3628 }
3629 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
3630 prog->name, i, ext->name, ext->sym_idx, insn_idx);
3631 if (insn->code == (BPF_JMP | BPF_CALL))
3632 reloc_desc->type = RELO_EXTERN_FUNC;
3633 else
3634 reloc_desc->type = RELO_EXTERN_VAR;
3635 reloc_desc->insn_idx = insn_idx;
3636 reloc_desc->sym_off = i;
3637 return 0;
3638 }
3639
3640
3641 if (is_call_insn(insn)) {
3642 if (insn->src_reg != BPF_PSEUDO_CALL) {
3643 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
3644 return -LIBBPF_ERRNO__RELOC;
3645 }
3646
3647 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
3648 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3649 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
3650 prog->name, sym_name, sym_sec_name);
3651 return -LIBBPF_ERRNO__RELOC;
3652 }
3653 if (sym->st_value % BPF_INSN_SZ) {
3654 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
3655 prog->name, sym_name, (size_t)sym->st_value);
3656 return -LIBBPF_ERRNO__RELOC;
3657 }
3658 reloc_desc->type = RELO_CALL;
3659 reloc_desc->insn_idx = insn_idx;
3660 reloc_desc->sym_off = sym->st_value;
3661 return 0;
3662 }
3663
3664 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
3665 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
3666 prog->name, sym_name, shdr_idx);
3667 return -LIBBPF_ERRNO__RELOC;
3668 }
3669
3670
3671 if (sym_is_subprog(sym, obj->efile.text_shndx)) {
3672
3673
3674
3675 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
3676 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
3677 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
3678 return -LIBBPF_ERRNO__RELOC;
3679 }
3680
3681 reloc_desc->type = RELO_SUBPROG_ADDR;
3682 reloc_desc->insn_idx = insn_idx;
3683 reloc_desc->sym_off = sym->st_value;
3684 return 0;
3685 }
3686
3687 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
3688 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
3689
3690
3691 if (type == LIBBPF_MAP_UNSPEC) {
3692 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
3693 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
3694 prog->name, sym_name, sym_sec_name);
3695 return -LIBBPF_ERRNO__RELOC;
3696 }
3697 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3698 map = &obj->maps[map_idx];
3699 if (map->libbpf_type != type ||
3700 map->sec_idx != sym->st_shndx ||
3701 map->sec_offset != sym->st_value)
3702 continue;
3703 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
3704 prog->name, map_idx, map->name, map->sec_idx,
3705 map->sec_offset, insn_idx);
3706 break;
3707 }
3708 if (map_idx >= nr_maps) {
3709 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
3710 prog->name, sym_sec_name, (size_t)sym->st_value);
3711 return -LIBBPF_ERRNO__RELOC;
3712 }
3713 reloc_desc->type = RELO_LD64;
3714 reloc_desc->insn_idx = insn_idx;
3715 reloc_desc->map_idx = map_idx;
3716 reloc_desc->sym_off = 0;
3717 return 0;
3718 }
3719
3720
3721 if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
3722 pr_warn("prog '%s': bad data relo against section '%s'\n",
3723 prog->name, sym_sec_name);
3724 return -LIBBPF_ERRNO__RELOC;
3725 }
3726 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
3727 map = &obj->maps[map_idx];
3728 if (map->libbpf_type != type)
3729 continue;
3730 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
3731 prog->name, map_idx, map->name, map->sec_idx,
3732 map->sec_offset, insn_idx);
3733 break;
3734 }
3735 if (map_idx >= nr_maps) {
3736 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
3737 prog->name, sym_sec_name);
3738 return -LIBBPF_ERRNO__RELOC;
3739 }
3740
3741 reloc_desc->type = RELO_DATA;
3742 reloc_desc->insn_idx = insn_idx;
3743 reloc_desc->map_idx = map_idx;
3744 reloc_desc->sym_off = sym->st_value;
3745 return 0;
3746}
3747
3748static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
3749{
3750 return insn_idx >= prog->sec_insn_off &&
3751 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
3752}
3753
3754static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
3755 size_t sec_idx, size_t insn_idx)
3756{
3757 int l = 0, r = obj->nr_programs - 1, m;
3758 struct bpf_program *prog;
3759
3760 while (l < r) {
3761 m = l + (r - l + 1) / 2;
3762 prog = &obj->programs[m];
3763
3764 if (prog->sec_idx < sec_idx ||
3765 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
3766 l = m;
3767 else
3768 r = m - 1;
3769 }
3770
3771
3772
3773 prog = &obj->programs[l];
3774 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
3775 return prog;
3776 return NULL;
3777}
3778
3779static int
3780bpf_object__collect_prog_relos(struct bpf_object *obj, GElf_Shdr *shdr, Elf_Data *data)
3781{
3782 Elf_Data *symbols = obj->efile.symbols;
3783 const char *relo_sec_name, *sec_name;
3784 size_t sec_idx = shdr->sh_info;
3785 struct bpf_program *prog;
3786 struct reloc_desc *relos;
3787 int err, i, nrels;
3788 const char *sym_name;
3789 __u32 insn_idx;
3790 Elf_Scn *scn;
3791 Elf_Data *scn_data;
3792 GElf_Sym sym;
3793 GElf_Rel rel;
3794
3795 scn = elf_sec_by_idx(obj, sec_idx);
3796 scn_data = elf_sec_data(obj, scn);
3797
3798 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
3799 sec_name = elf_sec_name(obj, scn);
3800 if (!relo_sec_name || !sec_name)
3801 return -EINVAL;
3802
3803 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
3804 relo_sec_name, sec_idx, sec_name);
3805 nrels = shdr->sh_size / shdr->sh_entsize;
3806
3807 for (i = 0; i < nrels; i++) {
3808 if (!gelf_getrel(data, i, &rel)) {
3809 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
3810 return -LIBBPF_ERRNO__FORMAT;
3811 }
3812 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
3813 pr_warn("sec '%s': symbol 0x%zx not found for relo #%d\n",
3814 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);
3815 return -LIBBPF_ERRNO__FORMAT;
3816 }
3817
3818 if (rel.r_offset % BPF_INSN_SZ || rel.r_offset >= scn_data->d_size) {
3819 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
3820 relo_sec_name, (size_t)GELF_R_SYM(rel.r_info), i);
3821 return -LIBBPF_ERRNO__FORMAT;
3822 }
3823
3824 insn_idx = rel.r_offset / BPF_INSN_SZ;
3825
3826
3827
3828
3829
3830
3831 if (GELF_ST_TYPE(sym.st_info) == STT_SECTION && sym.st_name == 0)
3832 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym.st_shndx));
3833 else
3834 sym_name = elf_sym_str(obj, sym.st_name);
3835 sym_name = sym_name ?: "<?";
3836
3837 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
3838 relo_sec_name, i, insn_idx, sym_name);
3839
3840 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
3841 if (!prog) {
3842 pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",
3843 relo_sec_name, i, sec_name, insn_idx);
3844 continue;
3845 }
3846
3847 relos = libbpf_reallocarray(prog->reloc_desc,
3848 prog->nr_reloc + 1, sizeof(*relos));
3849 if (!relos)
3850 return -ENOMEM;
3851 prog->reloc_desc = relos;
3852
3853
3854 insn_idx -= prog->sec_insn_off;
3855 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
3856 insn_idx, sym_name, &sym, &rel);
3857 if (err)
3858 return err;
3859
3860 prog->nr_reloc++;
3861 }
3862 return 0;
3863}
3864
3865static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
3866{
3867 struct bpf_map_def *def = &map->def;
3868 __u32 key_type_id = 0, value_type_id = 0;
3869 int ret;
3870
3871
3872
3873
3874
3875 if (map->sec_idx == obj->efile.btf_maps_shndx ||
3876 bpf_map__is_struct_ops(map))
3877 return 0;
3878
3879 if (!bpf_map__is_internal(map)) {
3880 ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size,
3881 def->value_size, &key_type_id,
3882 &value_type_id);
3883 } else {
3884
3885
3886
3887
3888 ret = btf__find_by_name(obj->btf,
3889 libbpf_type_to_btf_name[map->libbpf_type]);
3890 }
3891 if (ret < 0)
3892 return ret;
3893
3894 map->btf_key_type_id = key_type_id;
3895 map->btf_value_type_id = bpf_map__is_internal(map) ?
3896 ret : value_type_id;
3897 return 0;
3898}
3899
3900static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)
3901{
3902 char file[PATH_MAX], buff[4096];
3903 FILE *fp;
3904 __u32 val;
3905 int err;
3906
3907 snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);
3908 memset(info, 0, sizeof(*info));
3909
3910 fp = fopen(file, "r");
3911 if (!fp) {
3912 err = -errno;
3913 pr_warn("failed to open %s: %d. No procfs support?\n", file,
3914 err);
3915 return err;
3916 }
3917
3918 while (fgets(buff, sizeof(buff), fp)) {
3919 if (sscanf(buff, "map_type:\t%u", &val) == 1)
3920 info->type = val;
3921 else if (sscanf(buff, "key_size:\t%u", &val) == 1)
3922 info->key_size = val;
3923 else if (sscanf(buff, "value_size:\t%u", &val) == 1)
3924 info->value_size = val;
3925 else if (sscanf(buff, "max_entries:\t%u", &val) == 1)
3926 info->max_entries = val;
3927 else if (sscanf(buff, "map_flags:\t%i", &val) == 1)
3928 info->map_flags = val;
3929 }
3930
3931 fclose(fp);
3932
3933 return 0;
3934}
3935
3936int bpf_map__reuse_fd(struct bpf_map *map, int fd)
3937{
3938 struct bpf_map_info info = {};
3939 __u32 len = sizeof(info);
3940 int new_fd, err;
3941 char *new_name;
3942
3943 err = bpf_obj_get_info_by_fd(fd, &info, &len);
3944 if (err && errno == EINVAL)
3945 err = bpf_get_map_info_from_fdinfo(fd, &info);
3946 if (err)
3947 return libbpf_err(err);
3948
3949 new_name = strdup(info.name);
3950 if (!new_name)
3951 return libbpf_err(-errno);
3952
3953 new_fd = open("/", O_RDONLY | O_CLOEXEC);
3954 if (new_fd < 0) {
3955 err = -errno;
3956 goto err_free_new_name;
3957 }
3958
3959 new_fd = dup3(fd, new_fd, O_CLOEXEC);
3960 if (new_fd < 0) {
3961 err = -errno;
3962 goto err_close_new_fd;
3963 }
3964
3965 err = zclose(map->fd);
3966 if (err) {
3967 err = -errno;
3968 goto err_close_new_fd;
3969 }
3970 free(map->name);
3971
3972 map->fd = new_fd;
3973 map->name = new_name;
3974 map->def.type = info.type;
3975 map->def.key_size = info.key_size;
3976 map->def.value_size = info.value_size;
3977 map->def.max_entries = info.max_entries;
3978 map->def.map_flags = info.map_flags;
3979 map->btf_key_type_id = info.btf_key_type_id;
3980 map->btf_value_type_id = info.btf_value_type_id;
3981 map->reused = true;
3982
3983 return 0;
3984
3985err_close_new_fd:
3986 close(new_fd);
3987err_free_new_name:
3988 free(new_name);
3989 return libbpf_err(err);
3990}
3991
3992__u32 bpf_map__max_entries(const struct bpf_map *map)
3993{
3994 return map->def.max_entries;
3995}
3996
3997struct bpf_map *bpf_map__inner_map(struct bpf_map *map)
3998{
3999 if (!bpf_map_type__is_map_in_map(map->def.type))
4000 return errno = EINVAL, NULL;
4001
4002 return map->inner_map;
4003}
4004
4005int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
4006{
4007 if (map->fd >= 0)
4008 return libbpf_err(-EBUSY);
4009 map->def.max_entries = max_entries;
4010 return 0;
4011}
4012
4013int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
4014{
4015 if (!map || !max_entries)
4016 return libbpf_err(-EINVAL);
4017
4018 return bpf_map__set_max_entries(map, max_entries);
4019}
4020
4021static int
4022bpf_object__probe_loading(struct bpf_object *obj)
4023{
4024 struct bpf_load_program_attr attr;
4025 char *cp, errmsg[STRERR_BUFSIZE];
4026 struct bpf_insn insns[] = {
4027 BPF_MOV64_IMM(BPF_REG_0, 0),
4028 BPF_EXIT_INSN(),
4029 };
4030 int ret;
4031
4032 if (obj->gen_loader)
4033 return 0;
4034
4035
4036
4037 memset(&attr, 0, sizeof(attr));
4038 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
4039 attr.insns = insns;
4040 attr.insns_cnt = ARRAY_SIZE(insns);
4041 attr.license = "GPL";
4042
4043 ret = bpf_load_program_xattr(&attr, NULL, 0);
4044 if (ret < 0) {
4045 attr.prog_type = BPF_PROG_TYPE_TRACEPOINT;
4046 ret = bpf_load_program_xattr(&attr, NULL, 0);
4047 }
4048 if (ret < 0) {
4049 ret = errno;
4050 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4051 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
4052 "program. Make sure your kernel supports BPF "
4053 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
4054 "set to big enough value.\n", __func__, cp, ret);
4055 return -ret;
4056 }
4057 close(ret);
4058
4059 return 0;
4060}
4061
4062static int probe_fd(int fd)
4063{
4064 if (fd >= 0)
4065 close(fd);
4066 return fd >= 0;
4067}
4068
4069static int probe_kern_prog_name(void)
4070{
4071 struct bpf_load_program_attr attr;
4072 struct bpf_insn insns[] = {
4073 BPF_MOV64_IMM(BPF_REG_0, 0),
4074 BPF_EXIT_INSN(),
4075 };
4076 int ret;
4077
4078
4079
4080 memset(&attr, 0, sizeof(attr));
4081 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
4082 attr.insns = insns;
4083 attr.insns_cnt = ARRAY_SIZE(insns);
4084 attr.license = "GPL";
4085 attr.name = "test";
4086 ret = bpf_load_program_xattr(&attr, NULL, 0);
4087 return probe_fd(ret);
4088}
4089
4090static int probe_kern_global_data(void)
4091{
4092 struct bpf_load_program_attr prg_attr;
4093 struct bpf_create_map_attr map_attr;
4094 char *cp, errmsg[STRERR_BUFSIZE];
4095 struct bpf_insn insns[] = {
4096 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
4097 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
4098 BPF_MOV64_IMM(BPF_REG_0, 0),
4099 BPF_EXIT_INSN(),
4100 };
4101 int ret, map;
4102
4103 memset(&map_attr, 0, sizeof(map_attr));
4104 map_attr.map_type = BPF_MAP_TYPE_ARRAY;
4105 map_attr.key_size = sizeof(int);
4106 map_attr.value_size = 32;
4107 map_attr.max_entries = 1;
4108
4109 map = bpf_create_map_xattr(&map_attr);
4110 if (map < 0) {
4111 ret = -errno;
4112 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4113 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4114 __func__, cp, -ret);
4115 return ret;
4116 }
4117
4118 insns[0].imm = map;
4119
4120 memset(&prg_attr, 0, sizeof(prg_attr));
4121 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
4122 prg_attr.insns = insns;
4123 prg_attr.insns_cnt = ARRAY_SIZE(insns);
4124 prg_attr.license = "GPL";
4125
4126 ret = bpf_load_program_xattr(&prg_attr, NULL, 0);
4127 close(map);
4128 return probe_fd(ret);
4129}
4130
4131static int probe_kern_btf(void)
4132{
4133 static const char strs[] = "\0int";
4134 __u32 types[] = {
4135
4136 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4137 };
4138
4139 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4140 strs, sizeof(strs)));
4141}
4142
4143static int probe_kern_btf_func(void)
4144{
4145 static const char strs[] = "\0int\0x\0a";
4146
4147 __u32 types[] = {
4148
4149 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4150
4151 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4152 BTF_PARAM_ENC(7, 1),
4153
4154 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
4155 };
4156
4157 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4158 strs, sizeof(strs)));
4159}
4160
4161static int probe_kern_btf_func_global(void)
4162{
4163 static const char strs[] = "\0int\0x\0a";
4164
4165 __u32 types[] = {
4166
4167 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4168
4169 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4170 BTF_PARAM_ENC(7, 1),
4171
4172 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
4173 };
4174
4175 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4176 strs, sizeof(strs)));
4177}
4178
4179static int probe_kern_btf_datasec(void)
4180{
4181 static const char strs[] = "\0x\0.data";
4182
4183 __u32 types[] = {
4184
4185 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
4186
4187 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4188 BTF_VAR_STATIC,
4189
4190 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
4191 BTF_VAR_SECINFO_ENC(2, 0, 4),
4192 };
4193
4194 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4195 strs, sizeof(strs)));
4196}
4197
4198static int probe_kern_btf_float(void)
4199{
4200 static const char strs[] = "\0float";
4201 __u32 types[] = {
4202
4203 BTF_TYPE_FLOAT_ENC(1, 4),
4204 };
4205
4206 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4207 strs, sizeof(strs)));
4208}
4209
4210static int probe_kern_array_mmap(void)
4211{
4212 struct bpf_create_map_attr attr = {
4213 .map_type = BPF_MAP_TYPE_ARRAY,
4214 .map_flags = BPF_F_MMAPABLE,
4215 .key_size = sizeof(int),
4216 .value_size = sizeof(int),
4217 .max_entries = 1,
4218 };
4219
4220 return probe_fd(bpf_create_map_xattr(&attr));
4221}
4222
4223static int probe_kern_exp_attach_type(void)
4224{
4225 struct bpf_load_program_attr attr;
4226 struct bpf_insn insns[] = {
4227 BPF_MOV64_IMM(BPF_REG_0, 0),
4228 BPF_EXIT_INSN(),
4229 };
4230
4231 memset(&attr, 0, sizeof(attr));
4232
4233
4234
4235
4236
4237 attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK;
4238 attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE;
4239 attr.insns = insns;
4240 attr.insns_cnt = ARRAY_SIZE(insns);
4241 attr.license = "GPL";
4242
4243 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0));
4244}
4245
4246static int probe_kern_probe_read_kernel(void)
4247{
4248 struct bpf_load_program_attr attr;
4249 struct bpf_insn insns[] = {
4250 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
4251 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8),
4252 BPF_MOV64_IMM(BPF_REG_2, 8),
4253 BPF_MOV64_IMM(BPF_REG_3, 0),
4254 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel),
4255 BPF_EXIT_INSN(),
4256 };
4257
4258 memset(&attr, 0, sizeof(attr));
4259 attr.prog_type = BPF_PROG_TYPE_KPROBE;
4260 attr.insns = insns;
4261 attr.insns_cnt = ARRAY_SIZE(insns);
4262 attr.license = "GPL";
4263
4264 return probe_fd(bpf_load_program_xattr(&attr, NULL, 0));
4265}
4266
4267static int probe_prog_bind_map(void)
4268{
4269 struct bpf_load_program_attr prg_attr;
4270 struct bpf_create_map_attr map_attr;
4271 char *cp, errmsg[STRERR_BUFSIZE];
4272 struct bpf_insn insns[] = {
4273 BPF_MOV64_IMM(BPF_REG_0, 0),
4274 BPF_EXIT_INSN(),
4275 };
4276 int ret, map, prog;
4277
4278 memset(&map_attr, 0, sizeof(map_attr));
4279 map_attr.map_type = BPF_MAP_TYPE_ARRAY;
4280 map_attr.key_size = sizeof(int);
4281 map_attr.value_size = 32;
4282 map_attr.max_entries = 1;
4283
4284 map = bpf_create_map_xattr(&map_attr);
4285 if (map < 0) {
4286 ret = -errno;
4287 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4288 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4289 __func__, cp, -ret);
4290 return ret;
4291 }
4292
4293 memset(&prg_attr, 0, sizeof(prg_attr));
4294 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
4295 prg_attr.insns = insns;
4296 prg_attr.insns_cnt = ARRAY_SIZE(insns);
4297 prg_attr.license = "GPL";
4298
4299 prog = bpf_load_program_xattr(&prg_attr, NULL, 0);
4300 if (prog < 0) {
4301 close(map);
4302 return 0;
4303 }
4304
4305 ret = bpf_prog_bind_map(prog, map, NULL);
4306
4307 close(map);
4308 close(prog);
4309
4310 return ret >= 0;
4311}
4312
4313static int probe_module_btf(void)
4314{
4315 static const char strs[] = "\0int";
4316 __u32 types[] = {
4317
4318 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4319 };
4320 struct bpf_btf_info info;
4321 __u32 len = sizeof(info);
4322 char name[16];
4323 int fd, err;
4324
4325 fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
4326 if (fd < 0)
4327 return 0;
4328
4329 memset(&info, 0, sizeof(info));
4330 info.name = ptr_to_u64(name);
4331 info.name_len = sizeof(name);
4332
4333
4334
4335
4336
4337 err = bpf_obj_get_info_by_fd(fd, &info, &len);
4338 close(fd);
4339 return !err;
4340}
4341
4342static int probe_perf_link(void)
4343{
4344 struct bpf_load_program_attr attr;
4345 struct bpf_insn insns[] = {
4346 BPF_MOV64_IMM(BPF_REG_0, 0),
4347 BPF_EXIT_INSN(),
4348 };
4349 int prog_fd, link_fd, err;
4350
4351 memset(&attr, 0, sizeof(attr));
4352 attr.prog_type = BPF_PROG_TYPE_TRACEPOINT;
4353 attr.insns = insns;
4354 attr.insns_cnt = ARRAY_SIZE(insns);
4355 attr.license = "GPL";
4356 prog_fd = bpf_load_program_xattr(&attr, NULL, 0);
4357 if (prog_fd < 0)
4358 return -errno;
4359
4360
4361
4362
4363 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL);
4364 err = -errno;
4365
4366 if (link_fd >= 0)
4367 close(link_fd);
4368 close(prog_fd);
4369
4370 return link_fd < 0 && err == -EBADF;
4371}
4372
4373enum kern_feature_result {
4374 FEAT_UNKNOWN = 0,
4375 FEAT_SUPPORTED = 1,
4376 FEAT_MISSING = 2,
4377};
4378
4379typedef int (*feature_probe_fn)(void);
4380
4381static struct kern_feature_desc {
4382 const char *desc;
4383 feature_probe_fn probe;
4384 enum kern_feature_result res;
4385} feature_probes[__FEAT_CNT] = {
4386 [FEAT_PROG_NAME] = {
4387 "BPF program name", probe_kern_prog_name,
4388 },
4389 [FEAT_GLOBAL_DATA] = {
4390 "global variables", probe_kern_global_data,
4391 },
4392 [FEAT_BTF] = {
4393 "minimal BTF", probe_kern_btf,
4394 },
4395 [FEAT_BTF_FUNC] = {
4396 "BTF functions", probe_kern_btf_func,
4397 },
4398 [FEAT_BTF_GLOBAL_FUNC] = {
4399 "BTF global function", probe_kern_btf_func_global,
4400 },
4401 [FEAT_BTF_DATASEC] = {
4402 "BTF data section and variable", probe_kern_btf_datasec,
4403 },
4404 [FEAT_ARRAY_MMAP] = {
4405 "ARRAY map mmap()", probe_kern_array_mmap,
4406 },
4407 [FEAT_EXP_ATTACH_TYPE] = {
4408 "BPF_PROG_LOAD expected_attach_type attribute",
4409 probe_kern_exp_attach_type,
4410 },
4411 [FEAT_PROBE_READ_KERN] = {
4412 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel,
4413 },
4414 [FEAT_PROG_BIND_MAP] = {
4415 "BPF_PROG_BIND_MAP support", probe_prog_bind_map,
4416 },
4417 [FEAT_MODULE_BTF] = {
4418 "module BTF support", probe_module_btf,
4419 },
4420 [FEAT_BTF_FLOAT] = {
4421 "BTF_KIND_FLOAT support", probe_kern_btf_float,
4422 },
4423 [FEAT_PERF_LINK] = {
4424 "BPF perf link support", probe_perf_link,
4425 },
4426};
4427
4428static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
4429{
4430 struct kern_feature_desc *feat = &feature_probes[feat_id];
4431 int ret;
4432
4433 if (obj->gen_loader)
4434
4435
4436
4437 return true;
4438
4439 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
4440 ret = feat->probe();
4441 if (ret > 0) {
4442 WRITE_ONCE(feat->res, FEAT_SUPPORTED);
4443 } else if (ret == 0) {
4444 WRITE_ONCE(feat->res, FEAT_MISSING);
4445 } else {
4446 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
4447 WRITE_ONCE(feat->res, FEAT_MISSING);
4448 }
4449 }
4450
4451 return READ_ONCE(feat->res) == FEAT_SUPPORTED;
4452}
4453
4454static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
4455{
4456 struct bpf_map_info map_info = {};
4457 char msg[STRERR_BUFSIZE];
4458 __u32 map_info_len;
4459 int err;
4460
4461 map_info_len = sizeof(map_info);
4462
4463 err = bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len);
4464 if (err && errno == EINVAL)
4465 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
4466 if (err) {
4467 pr_warn("failed to get map info for map FD %d: %s\n", map_fd,
4468 libbpf_strerror_r(errno, msg, sizeof(msg)));
4469 return false;
4470 }
4471
4472 return (map_info.type == map->def.type &&
4473 map_info.key_size == map->def.key_size &&
4474 map_info.value_size == map->def.value_size &&
4475 map_info.max_entries == map->def.max_entries &&
4476 map_info.map_flags == map->def.map_flags);
4477}
4478
4479static int
4480bpf_object__reuse_map(struct bpf_map *map)
4481{
4482 char *cp, errmsg[STRERR_BUFSIZE];
4483 int err, pin_fd;
4484
4485 pin_fd = bpf_obj_get(map->pin_path);
4486 if (pin_fd < 0) {
4487 err = -errno;
4488 if (err == -ENOENT) {
4489 pr_debug("found no pinned map to reuse at '%s'\n",
4490 map->pin_path);
4491 return 0;
4492 }
4493
4494 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
4495 pr_warn("couldn't retrieve pinned map '%s': %s\n",
4496 map->pin_path, cp);
4497 return err;
4498 }
4499
4500 if (!map_is_reuse_compat(map, pin_fd)) {
4501 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
4502 map->pin_path);
4503 close(pin_fd);
4504 return -EINVAL;
4505 }
4506
4507 err = bpf_map__reuse_fd(map, pin_fd);
4508 if (err) {
4509 close(pin_fd);
4510 return err;
4511 }
4512 map->pinned = true;
4513 pr_debug("reused pinned map at '%s'\n", map->pin_path);
4514
4515 return 0;
4516}
4517
4518static int
4519bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
4520{
4521 enum libbpf_map_type map_type = map->libbpf_type;
4522 char *cp, errmsg[STRERR_BUFSIZE];
4523 int err, zero = 0;
4524
4525 if (obj->gen_loader) {
4526 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
4527 map->mmaped, map->def.value_size);
4528 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG)
4529 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
4530 return 0;
4531 }
4532 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
4533 if (err) {
4534 err = -errno;
4535 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4536 pr_warn("Error setting initial map(%s) contents: %s\n",
4537 map->name, cp);
4538 return err;
4539 }
4540
4541
4542 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
4543 err = bpf_map_freeze(map->fd);
4544 if (err) {
4545 err = -errno;
4546 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4547 pr_warn("Error freezing map(%s) as read-only: %s\n",
4548 map->name, cp);
4549 return err;
4550 }
4551 }
4552 return 0;
4553}
4554
4555static void bpf_map__destroy(struct bpf_map *map);
4556
4557static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)
4558{
4559 struct bpf_create_map_attr create_attr;
4560 struct bpf_map_def *def = &map->def;
4561 int err = 0;
4562
4563 memset(&create_attr, 0, sizeof(create_attr));
4564
4565 if (kernel_supports(obj, FEAT_PROG_NAME))
4566 create_attr.name = map->name;
4567 create_attr.map_ifindex = map->map_ifindex;
4568 create_attr.map_type = def->type;
4569 create_attr.map_flags = def->map_flags;
4570 create_attr.key_size = def->key_size;
4571 create_attr.value_size = def->value_size;
4572 create_attr.numa_node = map->numa_node;
4573
4574 if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !def->max_entries) {
4575 int nr_cpus;
4576
4577 nr_cpus = libbpf_num_possible_cpus();
4578 if (nr_cpus < 0) {
4579 pr_warn("map '%s': failed to determine number of system CPUs: %d\n",
4580 map->name, nr_cpus);
4581 return nr_cpus;
4582 }
4583 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);
4584 create_attr.max_entries = nr_cpus;
4585 } else {
4586 create_attr.max_entries = def->max_entries;
4587 }
4588
4589 if (bpf_map__is_struct_ops(map))
4590 create_attr.btf_vmlinux_value_type_id =
4591 map->btf_vmlinux_value_type_id;
4592
4593 create_attr.btf_fd = 0;
4594 create_attr.btf_key_type_id = 0;
4595 create_attr.btf_value_type_id = 0;
4596 if (obj->btf && btf__fd(obj->btf) >= 0 && !bpf_map_find_btf_info(obj, map)) {
4597 create_attr.btf_fd = btf__fd(obj->btf);
4598 create_attr.btf_key_type_id = map->btf_key_type_id;
4599 create_attr.btf_value_type_id = map->btf_value_type_id;
4600 }
4601
4602 if (bpf_map_type__is_map_in_map(def->type)) {
4603 if (map->inner_map) {
4604 err = bpf_object__create_map(obj, map->inner_map, true);
4605 if (err) {
4606 pr_warn("map '%s': failed to create inner map: %d\n",
4607 map->name, err);
4608 return err;
4609 }
4610 map->inner_map_fd = bpf_map__fd(map->inner_map);
4611 }
4612 if (map->inner_map_fd >= 0)
4613 create_attr.inner_map_fd = map->inner_map_fd;
4614 }
4615
4616 if (obj->gen_loader) {
4617 bpf_gen__map_create(obj->gen_loader, &create_attr, is_inner ? -1 : map - obj->maps);
4618
4619
4620
4621 map->fd = 0;
4622 } else {
4623 map->fd = bpf_create_map_xattr(&create_attr);
4624 }
4625 if (map->fd < 0 && (create_attr.btf_key_type_id ||
4626 create_attr.btf_value_type_id)) {
4627 char *cp, errmsg[STRERR_BUFSIZE];
4628
4629 err = -errno;
4630 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4631 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
4632 map->name, cp, err);
4633 create_attr.btf_fd = 0;
4634 create_attr.btf_key_type_id = 0;
4635 create_attr.btf_value_type_id = 0;
4636 map->btf_key_type_id = 0;
4637 map->btf_value_type_id = 0;
4638 map->fd = bpf_create_map_xattr(&create_attr);
4639 }
4640
4641 err = map->fd < 0 ? -errno : 0;
4642
4643 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {
4644 if (obj->gen_loader)
4645 map->inner_map->fd = -1;
4646 bpf_map__destroy(map->inner_map);
4647 zfree(&map->inner_map);
4648 }
4649
4650 return err;
4651}
4652
4653static int init_map_slots(struct bpf_object *obj, struct bpf_map *map)
4654{
4655 const struct bpf_map *targ_map;
4656 unsigned int i;
4657 int fd, err = 0;
4658
4659 for (i = 0; i < map->init_slots_sz; i++) {
4660 if (!map->init_slots[i])
4661 continue;
4662
4663 targ_map = map->init_slots[i];
4664 fd = bpf_map__fd(targ_map);
4665 if (obj->gen_loader) {
4666 pr_warn("// TODO map_update_elem: idx %td key %d value==map_idx %td\n",
4667 map - obj->maps, i, targ_map - obj->maps);
4668 return -ENOTSUP;
4669 } else {
4670 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
4671 }
4672 if (err) {
4673 err = -errno;
4674 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
4675 map->name, i, targ_map->name,
4676 fd, err);
4677 return err;
4678 }
4679 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
4680 map->name, i, targ_map->name, fd);
4681 }
4682
4683 zfree(&map->init_slots);
4684 map->init_slots_sz = 0;
4685
4686 return 0;
4687}
4688
4689static int
4690bpf_object__create_maps(struct bpf_object *obj)
4691{
4692 struct bpf_map *map;
4693 char *cp, errmsg[STRERR_BUFSIZE];
4694 unsigned int i, j;
4695 int err;
4696 bool retried;
4697
4698 for (i = 0; i < obj->nr_maps; i++) {
4699 map = &obj->maps[i];
4700
4701 retried = false;
4702retry:
4703 if (map->pin_path) {
4704 err = bpf_object__reuse_map(map);
4705 if (err) {
4706 pr_warn("map '%s': error reusing pinned map\n",
4707 map->name);
4708 goto err_out;
4709 }
4710 if (retried && map->fd < 0) {
4711 pr_warn("map '%s': cannot find pinned map\n",
4712 map->name);
4713 err = -ENOENT;
4714 goto err_out;
4715 }
4716 }
4717
4718 if (map->fd >= 0) {
4719 pr_debug("map '%s': skipping creation (preset fd=%d)\n",
4720 map->name, map->fd);
4721 } else {
4722 err = bpf_object__create_map(obj, map, false);
4723 if (err)
4724 goto err_out;
4725
4726 pr_debug("map '%s': created successfully, fd=%d\n",
4727 map->name, map->fd);
4728
4729 if (bpf_map__is_internal(map)) {
4730 err = bpf_object__populate_internal_map(obj, map);
4731 if (err < 0) {
4732 zclose(map->fd);
4733 goto err_out;
4734 }
4735 }
4736
4737 if (map->init_slots_sz) {
4738 err = init_map_slots(obj, map);
4739 if (err < 0) {
4740 zclose(map->fd);
4741 goto err_out;
4742 }
4743 }
4744 }
4745
4746 if (map->pin_path && !map->pinned) {
4747 err = bpf_map__pin(map, NULL);
4748 if (err) {
4749 zclose(map->fd);
4750 if (!retried && err == -EEXIST) {
4751 retried = true;
4752 goto retry;
4753 }
4754 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",
4755 map->name, map->pin_path, err);
4756 goto err_out;
4757 }
4758 }
4759 }
4760
4761 return 0;
4762
4763err_out:
4764 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
4765 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
4766 pr_perm_msg(err);
4767 for (j = 0; j < i; j++)
4768 zclose(obj->maps[j].fd);
4769 return err;
4770}
4771
4772static bool bpf_core_is_flavor_sep(const char *s)
4773{
4774
4775 return s[0] != '_' &&
4776 s[1] == '_' && s[2] == '_' && s[3] == '_' &&
4777 s[4] != '_';
4778}
4779
4780
4781
4782
4783
4784size_t bpf_core_essential_name_len(const char *name)
4785{
4786 size_t n = strlen(name);
4787 int i;
4788
4789 for (i = n - 5; i >= 0; i--) {
4790 if (bpf_core_is_flavor_sep(name + i))
4791 return i + 1;
4792 }
4793 return n;
4794}
4795
4796static void bpf_core_free_cands(struct bpf_core_cand_list *cands)
4797{
4798 free(cands->cands);
4799 free(cands);
4800}
4801
4802static int bpf_core_add_cands(struct bpf_core_cand *local_cand,
4803 size_t local_essent_len,
4804 const struct btf *targ_btf,
4805 const char *targ_btf_name,
4806 int targ_start_id,
4807 struct bpf_core_cand_list *cands)
4808{
4809 struct bpf_core_cand *new_cands, *cand;
4810 const struct btf_type *t;
4811 const char *targ_name;
4812 size_t targ_essent_len;
4813 int n, i;
4814
4815 n = btf__get_nr_types(targ_btf);
4816 for (i = targ_start_id; i <= n; i++) {
4817 t = btf__type_by_id(targ_btf, i);
4818 if (btf_kind(t) != btf_kind(local_cand->t))
4819 continue;
4820
4821 targ_name = btf__name_by_offset(targ_btf, t->name_off);
4822 if (str_is_empty(targ_name))
4823 continue;
4824
4825 targ_essent_len = bpf_core_essential_name_len(targ_name);
4826 if (targ_essent_len != local_essent_len)
4827 continue;
4828
4829 if (strncmp(local_cand->name, targ_name, local_essent_len) != 0)
4830 continue;
4831
4832 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",
4833 local_cand->id, btf_kind_str(local_cand->t),
4834 local_cand->name, i, btf_kind_str(t), targ_name,
4835 targ_btf_name);
4836 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,
4837 sizeof(*cands->cands));
4838 if (!new_cands)
4839 return -ENOMEM;
4840
4841 cand = &new_cands[cands->len];
4842 cand->btf = targ_btf;
4843 cand->t = t;
4844 cand->name = targ_name;
4845 cand->id = i;
4846
4847 cands->cands = new_cands;
4848 cands->len++;
4849 }
4850 return 0;
4851}
4852
4853static int load_module_btfs(struct bpf_object *obj)
4854{
4855 struct bpf_btf_info info;
4856 struct module_btf *mod_btf;
4857 struct btf *btf;
4858 char name[64];
4859 __u32 id = 0, len;
4860 int err, fd;
4861
4862 if (obj->btf_modules_loaded)
4863 return 0;
4864
4865 if (obj->gen_loader)
4866 return 0;
4867
4868
4869 obj->btf_modules_loaded = true;
4870
4871
4872 if (!kernel_supports(obj, FEAT_MODULE_BTF))
4873 return 0;
4874
4875 while (true) {
4876 err = bpf_btf_get_next_id(id, &id);
4877 if (err && errno == ENOENT)
4878 return 0;
4879 if (err) {
4880 err = -errno;
4881 pr_warn("failed to iterate BTF objects: %d\n", err);
4882 return err;
4883 }
4884
4885 fd = bpf_btf_get_fd_by_id(id);
4886 if (fd < 0) {
4887 if (errno == ENOENT)
4888 continue;
4889 err = -errno;
4890 pr_warn("failed to get BTF object #%d FD: %d\n", id, err);
4891 return err;
4892 }
4893
4894 len = sizeof(info);
4895 memset(&info, 0, sizeof(info));
4896 info.name = ptr_to_u64(name);
4897 info.name_len = sizeof(name);
4898
4899 err = bpf_obj_get_info_by_fd(fd, &info, &len);
4900 if (err) {
4901 err = -errno;
4902 pr_warn("failed to get BTF object #%d info: %d\n", id, err);
4903 goto err_out;
4904 }
4905
4906
4907 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) {
4908 close(fd);
4909 continue;
4910 }
4911
4912 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
4913 err = libbpf_get_error(btf);
4914 if (err) {
4915 pr_warn("failed to load module [%s]'s BTF object #%d: %d\n",
4916 name, id, err);
4917 goto err_out;
4918 }
4919
4920 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
4921 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
4922 if (err)
4923 goto err_out;
4924
4925 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
4926
4927 mod_btf->btf = btf;
4928 mod_btf->id = id;
4929 mod_btf->fd = fd;
4930 mod_btf->name = strdup(name);
4931 if (!mod_btf->name) {
4932 err = -ENOMEM;
4933 goto err_out;
4934 }
4935 continue;
4936
4937err_out:
4938 close(fd);
4939 return err;
4940 }
4941
4942 return 0;
4943}
4944
4945static struct bpf_core_cand_list *
4946bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
4947{
4948 struct bpf_core_cand local_cand = {};
4949 struct bpf_core_cand_list *cands;
4950 const struct btf *main_btf;
4951 size_t local_essent_len;
4952 int err, i;
4953
4954 local_cand.btf = local_btf;
4955 local_cand.t = btf__type_by_id(local_btf, local_type_id);
4956 if (!local_cand.t)
4957 return ERR_PTR(-EINVAL);
4958
4959 local_cand.name = btf__name_by_offset(local_btf, local_cand.t->name_off);
4960 if (str_is_empty(local_cand.name))
4961 return ERR_PTR(-EINVAL);
4962 local_essent_len = bpf_core_essential_name_len(local_cand.name);
4963
4964 cands = calloc(1, sizeof(*cands));
4965 if (!cands)
4966 return ERR_PTR(-ENOMEM);
4967
4968
4969 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
4970 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands);
4971 if (err)
4972 goto err_out;
4973
4974
4975 if (cands->len)
4976 return cands;
4977
4978
4979 if (obj->btf_vmlinux_override)
4980 return cands;
4981
4982
4983 err = load_module_btfs(obj);
4984 if (err)
4985 goto err_out;
4986
4987 for (i = 0; i < obj->btf_module_cnt; i++) {
4988 err = bpf_core_add_cands(&local_cand, local_essent_len,
4989 obj->btf_modules[i].btf,
4990 obj->btf_modules[i].name,
4991 btf__get_nr_types(obj->btf_vmlinux) + 1,
4992 cands);
4993 if (err)
4994 goto err_out;
4995 }
4996
4997 return cands;
4998err_out:
4999 bpf_core_free_cands(cands);
5000 return ERR_PTR(err);
5001}
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
5023 const struct btf *targ_btf, __u32 targ_id)
5024{
5025 const struct btf_type *local_type, *targ_type;
5026 int depth = 32;
5027
5028
5029 local_type = btf__type_by_id(local_btf, local_id);
5030 targ_type = btf__type_by_id(targ_btf, targ_id);
5031 if (btf_kind(local_type) != btf_kind(targ_type))
5032 return 0;
5033
5034recur:
5035 depth--;
5036 if (depth < 0)
5037 return -EINVAL;
5038
5039 local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
5040 targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
5041 if (!local_type || !targ_type)
5042 return -EINVAL;
5043
5044 if (btf_kind(local_type) != btf_kind(targ_type))
5045 return 0;
5046
5047 switch (btf_kind(local_type)) {
5048 case BTF_KIND_UNKN:
5049 case BTF_KIND_STRUCT:
5050 case BTF_KIND_UNION:
5051 case BTF_KIND_ENUM:
5052 case BTF_KIND_FWD:
5053 return 1;
5054 case BTF_KIND_INT:
5055
5056
5057
5058 return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0;
5059 case BTF_KIND_PTR:
5060 local_id = local_type->type;
5061 targ_id = targ_type->type;
5062 goto recur;
5063 case BTF_KIND_ARRAY:
5064 local_id = btf_array(local_type)->type;
5065 targ_id = btf_array(targ_type)->type;
5066 goto recur;
5067 case BTF_KIND_FUNC_PROTO: {
5068 struct btf_param *local_p = btf_params(local_type);
5069 struct btf_param *targ_p = btf_params(targ_type);
5070 __u16 local_vlen = btf_vlen(local_type);
5071 __u16 targ_vlen = btf_vlen(targ_type);
5072 int i, err;
5073
5074 if (local_vlen != targ_vlen)
5075 return 0;
5076
5077 for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
5078 skip_mods_and_typedefs(local_btf, local_p->type, &local_id);
5079 skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id);
5080 err = bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id);
5081 if (err <= 0)
5082 return err;
5083 }
5084
5085
5086 skip_mods_and_typedefs(local_btf, local_type->type, &local_id);
5087 skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id);
5088 goto recur;
5089 }
5090 default:
5091 pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
5092 btf_kind_str(local_type), local_id, targ_id);
5093 return 0;
5094 }
5095}
5096
5097static size_t bpf_core_hash_fn(const void *key, void *ctx)
5098{
5099 return (size_t)key;
5100}
5101
5102static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx)
5103{
5104 return k1 == k2;
5105}
5106
5107static void *u32_as_hash_key(__u32 x)
5108{
5109 return (void *)(uintptr_t)x;
5110}
5111
5112static int bpf_core_apply_relo(struct bpf_program *prog,
5113 const struct bpf_core_relo *relo,
5114 int relo_idx,
5115 const struct btf *local_btf,
5116 struct hashmap *cand_cache)
5117{
5118 const void *type_key = u32_as_hash_key(relo->type_id);
5119 struct bpf_core_cand_list *cands = NULL;
5120 const char *prog_name = prog->name;
5121 const struct btf_type *local_type;
5122 const char *local_name;
5123 __u32 local_id = relo->type_id;
5124 struct bpf_insn *insn;
5125 int insn_idx, err;
5126
5127 if (relo->insn_off % BPF_INSN_SZ)
5128 return -EINVAL;
5129 insn_idx = relo->insn_off / BPF_INSN_SZ;
5130
5131
5132
5133
5134 insn_idx = insn_idx - prog->sec_insn_off;
5135 if (insn_idx > prog->insns_cnt)
5136 return -EINVAL;
5137 insn = &prog->insns[insn_idx];
5138
5139 local_type = btf__type_by_id(local_btf, local_id);
5140 if (!local_type)
5141 return -EINVAL;
5142
5143 local_name = btf__name_by_offset(local_btf, local_type->name_off);
5144 if (!local_name)
5145 return -EINVAL;
5146
5147 if (prog->obj->gen_loader) {
5148 pr_warn("// TODO core_relo: prog %td insn[%d] %s kind %d\n",
5149 prog - prog->obj->programs, relo->insn_off / 8,
5150 local_name, relo->kind);
5151 return -ENOTSUP;
5152 }
5153
5154 if (relo->kind != BPF_TYPE_ID_LOCAL &&
5155 !hashmap__find(cand_cache, type_key, (void **)&cands)) {
5156 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
5157 if (IS_ERR(cands)) {
5158 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
5159 prog_name, relo_idx, local_id, btf_kind_str(local_type),
5160 local_name, PTR_ERR(cands));
5161 return PTR_ERR(cands);
5162 }
5163 err = hashmap__set(cand_cache, type_key, cands, NULL, NULL);
5164 if (err) {
5165 bpf_core_free_cands(cands);
5166 return err;
5167 }
5168 }
5169
5170 return bpf_core_apply_relo_insn(prog_name, insn, insn_idx, relo, relo_idx, local_btf, cands);
5171}
5172
5173static int
5174bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
5175{
5176 const struct btf_ext_info_sec *sec;
5177 const struct bpf_core_relo *rec;
5178 const struct btf_ext_info *seg;
5179 struct hashmap_entry *entry;
5180 struct hashmap *cand_cache = NULL;
5181 struct bpf_program *prog;
5182 const char *sec_name;
5183 int i, err = 0, insn_idx, sec_idx;
5184
5185 if (obj->btf_ext->core_relo_info.len == 0)
5186 return 0;
5187
5188 if (targ_btf_path) {
5189 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
5190 err = libbpf_get_error(obj->btf_vmlinux_override);
5191 if (err) {
5192 pr_warn("failed to parse target BTF: %d\n", err);
5193 return err;
5194 }
5195 }
5196
5197 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL);
5198 if (IS_ERR(cand_cache)) {
5199 err = PTR_ERR(cand_cache);
5200 goto out;
5201 }
5202
5203 seg = &obj->btf_ext->core_relo_info;
5204 for_each_btf_ext_sec(seg, sec) {
5205 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
5206 if (str_is_empty(sec_name)) {
5207 err = -EINVAL;
5208 goto out;
5209 }
5210
5211
5212
5213
5214
5215
5216 prog = NULL;
5217 for (i = 0; i < obj->nr_programs; i++) {
5218 prog = &obj->programs[i];
5219 if (strcmp(prog->sec_name, sec_name) == 0)
5220 break;
5221 }
5222 if (!prog) {
5223 pr_warn("sec '%s': failed to find a BPF program\n", sec_name);
5224 return -ENOENT;
5225 }
5226 sec_idx = prog->sec_idx;
5227
5228 pr_debug("sec '%s': found %d CO-RE relocations\n",
5229 sec_name, sec->num_info);
5230
5231 for_each_btf_ext_rec(seg, sec, i, rec) {
5232 insn_idx = rec->insn_off / BPF_INSN_SZ;
5233 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
5234 if (!prog) {
5235 pr_warn("sec '%s': failed to find program at insn #%d for CO-RE offset relocation #%d\n",
5236 sec_name, insn_idx, i);
5237 err = -EINVAL;
5238 goto out;
5239 }
5240
5241
5242
5243 if (!prog->load)
5244 continue;
5245
5246 err = bpf_core_apply_relo(prog, rec, i, obj->btf, cand_cache);
5247 if (err) {
5248 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
5249 prog->name, i, err);
5250 goto out;
5251 }
5252 }
5253 }
5254
5255out:
5256
5257 btf__free(obj->btf_vmlinux_override);
5258 obj->btf_vmlinux_override = NULL;
5259
5260 if (!IS_ERR_OR_NULL(cand_cache)) {
5261 hashmap__for_each_entry(cand_cache, entry, i) {
5262 bpf_core_free_cands(entry->value);
5263 }
5264 hashmap__free(cand_cache);
5265 }
5266 return err;
5267}
5268
5269
5270
5271
5272
5273
5274static int
5275bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
5276{
5277 int i;
5278
5279 for (i = 0; i < prog->nr_reloc; i++) {
5280 struct reloc_desc *relo = &prog->reloc_desc[i];
5281 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
5282 struct extern_desc *ext;
5283
5284 switch (relo->type) {
5285 case RELO_LD64:
5286 if (obj->gen_loader) {
5287 insn[0].src_reg = BPF_PSEUDO_MAP_IDX;
5288 insn[0].imm = relo->map_idx;
5289 } else {
5290 insn[0].src_reg = BPF_PSEUDO_MAP_FD;
5291 insn[0].imm = obj->maps[relo->map_idx].fd;
5292 }
5293 break;
5294 case RELO_DATA:
5295 insn[1].imm = insn[0].imm + relo->sym_off;
5296 if (obj->gen_loader) {
5297 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
5298 insn[0].imm = relo->map_idx;
5299 } else {
5300 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
5301 insn[0].imm = obj->maps[relo->map_idx].fd;
5302 }
5303 break;
5304 case RELO_EXTERN_VAR:
5305 ext = &obj->externs[relo->sym_off];
5306 if (ext->type == EXT_KCFG) {
5307 if (obj->gen_loader) {
5308 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE;
5309 insn[0].imm = obj->kconfig_map_idx;
5310 } else {
5311 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE;
5312 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
5313 }
5314 insn[1].imm = ext->kcfg.data_off;
5315 } else {
5316 if (ext->ksym.type_id && ext->is_set) {
5317 insn[0].src_reg = BPF_PSEUDO_BTF_ID;
5318 insn[0].imm = ext->ksym.kernel_btf_id;
5319 insn[1].imm = ext->ksym.kernel_btf_obj_fd;
5320 } else {
5321 insn[0].imm = (__u32)ext->ksym.addr;
5322 insn[1].imm = ext->ksym.addr >> 32;
5323 }
5324 }
5325 break;
5326 case RELO_EXTERN_FUNC:
5327 ext = &obj->externs[relo->sym_off];
5328 insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL;
5329 insn[0].imm = ext->ksym.kernel_btf_id;
5330 break;
5331 case RELO_SUBPROG_ADDR:
5332 if (insn[0].src_reg != BPF_PSEUDO_FUNC) {
5333 pr_warn("prog '%s': relo #%d: bad insn\n",
5334 prog->name, i);
5335 return -EINVAL;
5336 }
5337
5338 break;
5339 case RELO_CALL:
5340
5341 break;
5342 default:
5343 pr_warn("prog '%s': relo #%d: bad relo type %d\n",
5344 prog->name, i, relo->type);
5345 return -EINVAL;
5346 }
5347 }
5348
5349 return 0;
5350}
5351
5352static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
5353 const struct bpf_program *prog,
5354 const struct btf_ext_info *ext_info,
5355 void **prog_info, __u32 *prog_rec_cnt,
5356 __u32 *prog_rec_sz)
5357{
5358 void *copy_start = NULL, *copy_end = NULL;
5359 void *rec, *rec_end, *new_prog_info;
5360 const struct btf_ext_info_sec *sec;
5361 size_t old_sz, new_sz;
5362 const char *sec_name;
5363 int i, off_adj;
5364
5365 for_each_btf_ext_sec(ext_info, sec) {
5366 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
5367 if (!sec_name)
5368 return -EINVAL;
5369 if (strcmp(sec_name, prog->sec_name) != 0)
5370 continue;
5371
5372 for_each_btf_ext_rec(ext_info, sec, i, rec) {
5373 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ;
5374
5375 if (insn_off < prog->sec_insn_off)
5376 continue;
5377 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
5378 break;
5379
5380 if (!copy_start)
5381 copy_start = rec;
5382 copy_end = rec + ext_info->rec_size;
5383 }
5384
5385 if (!copy_start)
5386 return -ENOENT;
5387
5388
5389
5390
5391 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;
5392 new_sz = old_sz + (copy_end - copy_start);
5393 new_prog_info = realloc(*prog_info, new_sz);
5394 if (!new_prog_info)
5395 return -ENOMEM;
5396 *prog_info = new_prog_info;
5397 *prog_rec_cnt = new_sz / ext_info->rec_size;
5398 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);
5399
5400
5401
5402
5403
5404
5405
5406 off_adj = prog->sub_insn_off - prog->sec_insn_off;
5407 rec = new_prog_info + old_sz;
5408 rec_end = new_prog_info + new_sz;
5409 for (; rec < rec_end; rec += ext_info->rec_size) {
5410 __u32 *insn_off = rec;
5411
5412 *insn_off = *insn_off / BPF_INSN_SZ + off_adj;
5413 }
5414 *prog_rec_sz = ext_info->rec_size;
5415 return 0;
5416 }
5417
5418 return -ENOENT;
5419}
5420
5421static int
5422reloc_prog_func_and_line_info(const struct bpf_object *obj,
5423 struct bpf_program *main_prog,
5424 const struct bpf_program *prog)
5425{
5426 int err;
5427
5428
5429
5430
5431 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
5432 return 0;
5433
5434
5435
5436
5437 if (main_prog != prog && !main_prog->func_info)
5438 goto line_info;
5439
5440 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
5441 &main_prog->func_info,
5442 &main_prog->func_info_cnt,
5443 &main_prog->func_info_rec_size);
5444 if (err) {
5445 if (err != -ENOENT) {
5446 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
5447 prog->name, err);
5448 return err;
5449 }
5450 if (main_prog->func_info) {
5451
5452
5453
5454
5455 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
5456 return err;
5457 }
5458
5459 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
5460 prog->name);
5461 }
5462
5463line_info:
5464
5465 if (main_prog != prog && !main_prog->line_info)
5466 return 0;
5467
5468 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
5469 &main_prog->line_info,
5470 &main_prog->line_info_cnt,
5471 &main_prog->line_info_rec_size);
5472 if (err) {
5473 if (err != -ENOENT) {
5474 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
5475 prog->name, err);
5476 return err;
5477 }
5478 if (main_prog->line_info) {
5479
5480
5481
5482
5483 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
5484 return err;
5485 }
5486
5487 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
5488 prog->name);
5489 }
5490 return 0;
5491}
5492
5493static int cmp_relo_by_insn_idx(const void *key, const void *elem)
5494{
5495 size_t insn_idx = *(const size_t *)key;
5496 const struct reloc_desc *relo = elem;
5497
5498 if (insn_idx == relo->insn_idx)
5499 return 0;
5500 return insn_idx < relo->insn_idx ? -1 : 1;
5501}
5502
5503static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
5504{
5505 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
5506 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
5507}
5508
5509static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog)
5510{
5511 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;
5512 struct reloc_desc *relos;
5513 int i;
5514
5515 if (main_prog == subprog)
5516 return 0;
5517 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));
5518 if (!relos)
5519 return -ENOMEM;
5520 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,
5521 sizeof(*relos) * subprog->nr_reloc);
5522
5523 for (i = main_prog->nr_reloc; i < new_cnt; i++)
5524 relos[i].insn_idx += subprog->sub_insn_off;
5525
5526
5527
5528 main_prog->reloc_desc = relos;
5529 main_prog->nr_reloc = new_cnt;
5530 return 0;
5531}
5532
5533static int
5534bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
5535 struct bpf_program *prog)
5536{
5537 size_t sub_insn_idx, insn_idx, new_cnt;
5538 struct bpf_program *subprog;
5539 struct bpf_insn *insns, *insn;
5540 struct reloc_desc *relo;
5541 int err;
5542
5543 err = reloc_prog_func_and_line_info(obj, main_prog, prog);
5544 if (err)
5545 return err;
5546
5547 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
5548 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
5549 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn))
5550 continue;
5551
5552 relo = find_prog_insn_relo(prog, insn_idx);
5553 if (relo && relo->type == RELO_EXTERN_FUNC)
5554
5555
5556
5557 continue;
5558 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {
5559 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
5560 prog->name, insn_idx, relo->type);
5561 return -LIBBPF_ERRNO__RELOC;
5562 }
5563 if (relo) {
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574 if (relo->type == RELO_CALL)
5575 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;
5576 else
5577 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;
5578 } else if (insn_is_pseudo_func(insn)) {
5579
5580
5581
5582
5583 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
5584 prog->name, insn_idx);
5585 return -LIBBPF_ERRNO__RELOC;
5586 } else {
5587
5588
5589
5590
5591
5592
5593 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
5594 }
5595
5596
5597 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
5598 if (!subprog) {
5599 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
5600 prog->name);
5601 return -LIBBPF_ERRNO__RELOC;
5602 }
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614 if (subprog->sub_insn_off == 0) {
5615 subprog->sub_insn_off = main_prog->insns_cnt;
5616
5617 new_cnt = main_prog->insns_cnt + subprog->insns_cnt;
5618 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));
5619 if (!insns) {
5620 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
5621 return -ENOMEM;
5622 }
5623 main_prog->insns = insns;
5624 main_prog->insns_cnt = new_cnt;
5625
5626 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,
5627 subprog->insns_cnt * sizeof(*insns));
5628
5629 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
5630 main_prog->name, subprog->insns_cnt, subprog->name);
5631
5632
5633 err = append_subprog_relos(main_prog, subprog);
5634 if (err)
5635 return err;
5636 err = bpf_object__reloc_code(obj, main_prog, subprog);
5637 if (err)
5638 return err;
5639 }
5640
5641
5642
5643
5644 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
5645
5646
5647
5648
5649
5650 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
5651
5652 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
5653 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
5654 }
5655
5656 return 0;
5657}
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740static int
5741bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
5742{
5743 struct bpf_program *subprog;
5744 int i, err;
5745
5746
5747
5748
5749 for (i = 0; i < obj->nr_programs; i++) {
5750 subprog = &obj->programs[i];
5751 if (!prog_is_subprog(obj, subprog))
5752 continue;
5753
5754 subprog->sub_insn_off = 0;
5755 }
5756
5757 err = bpf_object__reloc_code(obj, prog, prog);
5758 if (err)
5759 return err;
5760
5761
5762 return 0;
5763}
5764
5765static void
5766bpf_object__free_relocs(struct bpf_object *obj)
5767{
5768 struct bpf_program *prog;
5769 int i;
5770
5771
5772 for (i = 0; i < obj->nr_programs; i++) {
5773 prog = &obj->programs[i];
5774 zfree(&prog->reloc_desc);
5775 prog->nr_reloc = 0;
5776 }
5777}
5778
5779static int
5780bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
5781{
5782 struct bpf_program *prog;
5783 size_t i, j;
5784 int err;
5785
5786 if (obj->btf_ext) {
5787 err = bpf_object__relocate_core(obj, targ_btf_path);
5788 if (err) {
5789 pr_warn("failed to perform CO-RE relocations: %d\n",
5790 err);
5791 return err;
5792 }
5793 }
5794
5795
5796
5797
5798
5799
5800
5801
5802 for (i = 0; i < obj->nr_programs; i++) {
5803 prog = &obj->programs[i];
5804 for (j = 0; j < prog->nr_reloc; j++) {
5805 struct reloc_desc *relo = &prog->reloc_desc[j];
5806 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
5807
5808
5809 if (relo->type == RELO_SUBPROG_ADDR)
5810 insn[0].src_reg = BPF_PSEUDO_FUNC;
5811 }
5812 }
5813
5814
5815
5816
5817
5818
5819
5820
5821 for (i = 0; i < obj->nr_programs; i++) {
5822 prog = &obj->programs[i];
5823
5824
5825
5826 if (prog_is_subprog(obj, prog))
5827 continue;
5828
5829 err = bpf_object__relocate_calls(obj, prog);
5830 if (err) {
5831 pr_warn("prog '%s': failed to relocate calls: %d\n",
5832 prog->name, err);
5833 return err;
5834 }
5835 }
5836
5837 for (i = 0; i < obj->nr_programs; i++) {
5838 prog = &obj->programs[i];
5839 if (prog_is_subprog(obj, prog))
5840 continue;
5841 err = bpf_object__relocate_data(obj, prog);
5842 if (err) {
5843 pr_warn("prog '%s': failed to relocate data references: %d\n",
5844 prog->name, err);
5845 return err;
5846 }
5847 }
5848 if (!obj->gen_loader)
5849 bpf_object__free_relocs(obj);
5850 return 0;
5851}
5852
5853static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
5854 GElf_Shdr *shdr, Elf_Data *data);
5855
5856static int bpf_object__collect_map_relos(struct bpf_object *obj,
5857 GElf_Shdr *shdr, Elf_Data *data)
5858{
5859 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *);
5860 int i, j, nrels, new_sz;
5861 const struct btf_var_secinfo *vi = NULL;
5862 const struct btf_type *sec, *var, *def;
5863 struct bpf_map *map = NULL, *targ_map;
5864 const struct btf_member *member;
5865 const char *name, *mname;
5866 Elf_Data *symbols;
5867 unsigned int moff;
5868 GElf_Sym sym;
5869 GElf_Rel rel;
5870 void *tmp;
5871
5872 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
5873 return -EINVAL;
5874 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
5875 if (!sec)
5876 return -EINVAL;
5877
5878 symbols = obj->efile.symbols;
5879 nrels = shdr->sh_size / shdr->sh_entsize;
5880 for (i = 0; i < nrels; i++) {
5881 if (!gelf_getrel(data, i, &rel)) {
5882 pr_warn(".maps relo #%d: failed to get ELF relo\n", i);
5883 return -LIBBPF_ERRNO__FORMAT;
5884 }
5885 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
5886 pr_warn(".maps relo #%d: symbol %zx not found\n",
5887 i, (size_t)GELF_R_SYM(rel.r_info));
5888 return -LIBBPF_ERRNO__FORMAT;
5889 }
5890 name = elf_sym_str(obj, sym.st_name) ?: "<?>";
5891 if (sym.st_shndx != obj->efile.btf_maps_shndx) {
5892 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",
5893 i, name);
5894 return -LIBBPF_ERRNO__RELOC;
5895 }
5896
5897 pr_debug(".maps relo #%d: for %zd value %zd rel.r_offset %zu name %d ('%s')\n",
5898 i, (ssize_t)(rel.r_info >> 32), (size_t)sym.st_value,
5899 (size_t)rel.r_offset, sym.st_name, name);
5900
5901 for (j = 0; j < obj->nr_maps; j++) {
5902 map = &obj->maps[j];
5903 if (map->sec_idx != obj->efile.btf_maps_shndx)
5904 continue;
5905
5906 vi = btf_var_secinfos(sec) + map->btf_var_idx;
5907 if (vi->offset <= rel.r_offset &&
5908 rel.r_offset + bpf_ptr_sz <= vi->offset + vi->size)
5909 break;
5910 }
5911 if (j == obj->nr_maps) {
5912 pr_warn(".maps relo #%d: cannot find map '%s' at rel.r_offset %zu\n",
5913 i, name, (size_t)rel.r_offset);
5914 return -EINVAL;
5915 }
5916
5917 if (!bpf_map_type__is_map_in_map(map->def.type))
5918 return -EINVAL;
5919 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&
5920 map->def.key_size != sizeof(int)) {
5921 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",
5922 i, map->name, sizeof(int));
5923 return -EINVAL;
5924 }
5925
5926 targ_map = bpf_object__find_map_by_name(obj, name);
5927 if (!targ_map)
5928 return -ESRCH;
5929
5930 var = btf__type_by_id(obj->btf, vi->type);
5931 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
5932 if (btf_vlen(def) == 0)
5933 return -EINVAL;
5934 member = btf_members(def) + btf_vlen(def) - 1;
5935 mname = btf__name_by_offset(obj->btf, member->name_off);
5936 if (strcmp(mname, "values"))
5937 return -EINVAL;
5938
5939 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;
5940 if (rel.r_offset - vi->offset < moff)
5941 return -EINVAL;
5942
5943 moff = rel.r_offset - vi->offset - moff;
5944
5945
5946
5947 if (moff % bpf_ptr_sz)
5948 return -EINVAL;
5949 moff /= bpf_ptr_sz;
5950 if (moff >= map->init_slots_sz) {
5951 new_sz = moff + 1;
5952 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);
5953 if (!tmp)
5954 return -ENOMEM;
5955 map->init_slots = tmp;
5956 memset(map->init_slots + map->init_slots_sz, 0,
5957 (new_sz - map->init_slots_sz) * host_ptr_sz);
5958 map->init_slots_sz = new_sz;
5959 }
5960 map->init_slots[moff] = targ_map;
5961
5962 pr_debug(".maps relo #%d: map '%s' slot [%d] points to map '%s'\n",
5963 i, map->name, moff, name);
5964 }
5965
5966 return 0;
5967}
5968
5969static int cmp_relocs(const void *_a, const void *_b)
5970{
5971 const struct reloc_desc *a = _a;
5972 const struct reloc_desc *b = _b;
5973
5974 if (a->insn_idx != b->insn_idx)
5975 return a->insn_idx < b->insn_idx ? -1 : 1;
5976
5977
5978 if (a->type != b->type)
5979 return a->type < b->type ? -1 : 1;
5980
5981 return 0;
5982}
5983
5984static int bpf_object__collect_relos(struct bpf_object *obj)
5985{
5986 int i, err;
5987
5988 for (i = 0; i < obj->efile.nr_reloc_sects; i++) {
5989 GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr;
5990 Elf_Data *data = obj->efile.reloc_sects[i].data;
5991 int idx = shdr->sh_info;
5992
5993 if (shdr->sh_type != SHT_REL) {
5994 pr_warn("internal error at %d\n", __LINE__);
5995 return -LIBBPF_ERRNO__INTERNAL;
5996 }
5997
5998 if (idx == obj->efile.st_ops_shndx)
5999 err = bpf_object__collect_st_ops_relos(obj, shdr, data);
6000 else if (idx == obj->efile.btf_maps_shndx)
6001 err = bpf_object__collect_map_relos(obj, shdr, data);
6002 else
6003 err = bpf_object__collect_prog_relos(obj, shdr, data);
6004 if (err)
6005 return err;
6006 }
6007
6008 for (i = 0; i < obj->nr_programs; i++) {
6009 struct bpf_program *p = &obj->programs[i];
6010
6011 if (!p->nr_reloc)
6012 continue;
6013
6014 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);
6015 }
6016 return 0;
6017}
6018
6019static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id)
6020{
6021 if (BPF_CLASS(insn->code) == BPF_JMP &&
6022 BPF_OP(insn->code) == BPF_CALL &&
6023 BPF_SRC(insn->code) == BPF_K &&
6024 insn->src_reg == 0 &&
6025 insn->dst_reg == 0) {
6026 *func_id = insn->imm;
6027 return true;
6028 }
6029 return false;
6030}
6031
6032static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)
6033{
6034 struct bpf_insn *insn = prog->insns;
6035 enum bpf_func_id func_id;
6036 int i;
6037
6038 if (obj->gen_loader)
6039 return 0;
6040
6041 for (i = 0; i < prog->insns_cnt; i++, insn++) {
6042 if (!insn_is_helper_call(insn, &func_id))
6043 continue;
6044
6045
6046
6047
6048
6049 switch (func_id) {
6050 case BPF_FUNC_probe_read_kernel:
6051 case BPF_FUNC_probe_read_user:
6052 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
6053 insn->imm = BPF_FUNC_probe_read;
6054 break;
6055 case BPF_FUNC_probe_read_kernel_str:
6056 case BPF_FUNC_probe_read_user_str:
6057 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
6058 insn->imm = BPF_FUNC_probe_read_str;
6059 break;
6060 default:
6061 break;
6062 }
6063 }
6064 return 0;
6065}
6066
6067static int
6068load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
6069 char *license, __u32 kern_version, int *pfd)
6070{
6071 struct bpf_prog_load_params load_attr = {};
6072 char *cp, errmsg[STRERR_BUFSIZE];
6073 size_t log_buf_size = 0;
6074 char *log_buf = NULL;
6075 int btf_fd, ret;
6076
6077 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
6078
6079
6080
6081
6082 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
6083 prog->name, prog->sec_name);
6084 return -EINVAL;
6085 }
6086
6087 if (!insns || !insns_cnt)
6088 return -EINVAL;
6089
6090 load_attr.prog_type = prog->type;
6091
6092 if (!kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE) && prog->sec_def &&
6093 prog->sec_def->is_exp_attach_type_optional)
6094 load_attr.expected_attach_type = 0;
6095 else
6096 load_attr.expected_attach_type = prog->expected_attach_type;
6097 if (kernel_supports(prog->obj, FEAT_PROG_NAME))
6098 load_attr.name = prog->name;
6099 load_attr.insns = insns;
6100 load_attr.insn_cnt = insns_cnt;
6101 load_attr.license = license;
6102 load_attr.attach_btf_id = prog->attach_btf_id;
6103 if (prog->attach_prog_fd)
6104 load_attr.attach_prog_fd = prog->attach_prog_fd;
6105 else
6106 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
6107 load_attr.attach_btf_id = prog->attach_btf_id;
6108 load_attr.kern_version = kern_version;
6109 load_attr.prog_ifindex = prog->prog_ifindex;
6110
6111
6112 btf_fd = bpf_object__btf_fd(prog->obj);
6113 if (btf_fd >= 0 && kernel_supports(prog->obj, FEAT_BTF_FUNC)) {
6114 load_attr.prog_btf_fd = btf_fd;
6115 load_attr.func_info = prog->func_info;
6116 load_attr.func_info_rec_size = prog->func_info_rec_size;
6117 load_attr.func_info_cnt = prog->func_info_cnt;
6118 load_attr.line_info = prog->line_info;
6119 load_attr.line_info_rec_size = prog->line_info_rec_size;
6120 load_attr.line_info_cnt = prog->line_info_cnt;
6121 }
6122 load_attr.log_level = prog->log_level;
6123 load_attr.prog_flags = prog->prog_flags;
6124
6125 if (prog->obj->gen_loader) {
6126 bpf_gen__prog_load(prog->obj->gen_loader, &load_attr,
6127 prog - prog->obj->programs);
6128 *pfd = -1;
6129 return 0;
6130 }
6131retry_load:
6132 if (log_buf_size) {
6133 log_buf = malloc(log_buf_size);
6134 if (!log_buf)
6135 return -ENOMEM;
6136
6137 *log_buf = 0;
6138 }
6139
6140 load_attr.log_buf = log_buf;
6141 load_attr.log_buf_sz = log_buf_size;
6142 ret = libbpf__bpf_prog_load(&load_attr);
6143
6144 if (ret >= 0) {
6145 if (log_buf && load_attr.log_level)
6146 pr_debug("verifier log:\n%s", log_buf);
6147
6148 if (prog->obj->rodata_map_idx >= 0 &&
6149 kernel_supports(prog->obj, FEAT_PROG_BIND_MAP)) {
6150 struct bpf_map *rodata_map =
6151 &prog->obj->maps[prog->obj->rodata_map_idx];
6152
6153 if (bpf_prog_bind_map(ret, bpf_map__fd(rodata_map), NULL)) {
6154 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6155 pr_warn("prog '%s': failed to bind .rodata map: %s\n",
6156 prog->name, cp);
6157
6158 }
6159 }
6160
6161 *pfd = ret;
6162 ret = 0;
6163 goto out;
6164 }
6165
6166 if (!log_buf || errno == ENOSPC) {
6167 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE,
6168 log_buf_size << 1);
6169
6170 free(log_buf);
6171 goto retry_load;
6172 }
6173 ret = errno ? -errno : -LIBBPF_ERRNO__LOAD;
6174 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6175 pr_warn("load bpf program failed: %s\n", cp);
6176 pr_perm_msg(ret);
6177
6178 if (log_buf && log_buf[0] != '\0') {
6179 ret = -LIBBPF_ERRNO__VERIFY;
6180 pr_warn("-- BEGIN DUMP LOG ---\n");
6181 pr_warn("\n%s\n", log_buf);
6182 pr_warn("-- END LOG --\n");
6183 } else if (load_attr.insn_cnt >= BPF_MAXINSNS) {
6184 pr_warn("Program too large (%zu insns), at most %d insns\n",
6185 load_attr.insn_cnt, BPF_MAXINSNS);
6186 ret = -LIBBPF_ERRNO__PROG2BIG;
6187 } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
6188
6189 int fd;
6190
6191 load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
6192 load_attr.expected_attach_type = 0;
6193 load_attr.log_buf = NULL;
6194 load_attr.log_buf_sz = 0;
6195 fd = libbpf__bpf_prog_load(&load_attr);
6196 if (fd >= 0) {
6197 close(fd);
6198 ret = -LIBBPF_ERRNO__PROGTYPE;
6199 goto out;
6200 }
6201 }
6202
6203out:
6204 free(log_buf);
6205 return ret;
6206}
6207
6208static int bpf_program__record_externs(struct bpf_program *prog)
6209{
6210 struct bpf_object *obj = prog->obj;
6211 int i;
6212
6213 for (i = 0; i < prog->nr_reloc; i++) {
6214 struct reloc_desc *relo = &prog->reloc_desc[i];
6215 struct extern_desc *ext = &obj->externs[relo->sym_off];
6216
6217 switch (relo->type) {
6218 case RELO_EXTERN_VAR:
6219 if (ext->type != EXT_KSYM)
6220 continue;
6221 if (!ext->ksym.type_id) {
6222 pr_warn("typeless ksym %s is not supported yet\n",
6223 ext->name);
6224 return -ENOTSUP;
6225 }
6226 bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_VAR,
6227 relo->insn_idx);
6228 break;
6229 case RELO_EXTERN_FUNC:
6230 bpf_gen__record_extern(obj->gen_loader, ext->name, BTF_KIND_FUNC,
6231 relo->insn_idx);
6232 break;
6233 default:
6234 continue;
6235 }
6236 }
6237 return 0;
6238}
6239
6240static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id);
6241
6242int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver)
6243{
6244 int err = 0, fd, i;
6245
6246 if (prog->obj->loaded) {
6247 pr_warn("prog '%s': can't load after object was loaded\n", prog->name);
6248 return libbpf_err(-EINVAL);
6249 }
6250
6251 if ((prog->type == BPF_PROG_TYPE_TRACING ||
6252 prog->type == BPF_PROG_TYPE_LSM ||
6253 prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) {
6254 int btf_obj_fd = 0, btf_type_id = 0;
6255
6256 err = libbpf_find_attach_btf_id(prog, &btf_obj_fd, &btf_type_id);
6257 if (err)
6258 return libbpf_err(err);
6259
6260 prog->attach_btf_obj_fd = btf_obj_fd;
6261 prog->attach_btf_id = btf_type_id;
6262 }
6263
6264 if (prog->instances.nr < 0 || !prog->instances.fds) {
6265 if (prog->preprocessor) {
6266 pr_warn("Internal error: can't load program '%s'\n",
6267 prog->name);
6268 return libbpf_err(-LIBBPF_ERRNO__INTERNAL);
6269 }
6270
6271 prog->instances.fds = malloc(sizeof(int));
6272 if (!prog->instances.fds) {
6273 pr_warn("Not enough memory for BPF fds\n");
6274 return libbpf_err(-ENOMEM);
6275 }
6276 prog->instances.nr = 1;
6277 prog->instances.fds[0] = -1;
6278 }
6279
6280 if (!prog->preprocessor) {
6281 if (prog->instances.nr != 1) {
6282 pr_warn("prog '%s': inconsistent nr(%d) != 1\n",
6283 prog->name, prog->instances.nr);
6284 }
6285 if (prog->obj->gen_loader)
6286 bpf_program__record_externs(prog);
6287 err = load_program(prog, prog->insns, prog->insns_cnt,
6288 license, kern_ver, &fd);
6289 if (!err)
6290 prog->instances.fds[0] = fd;
6291 goto out;
6292 }
6293
6294 for (i = 0; i < prog->instances.nr; i++) {
6295 struct bpf_prog_prep_result result;
6296 bpf_program_prep_t preprocessor = prog->preprocessor;
6297
6298 memset(&result, 0, sizeof(result));
6299 err = preprocessor(prog, i, prog->insns,
6300 prog->insns_cnt, &result);
6301 if (err) {
6302 pr_warn("Preprocessing the %dth instance of program '%s' failed\n",
6303 i, prog->name);
6304 goto out;
6305 }
6306
6307 if (!result.new_insn_ptr || !result.new_insn_cnt) {
6308 pr_debug("Skip loading the %dth instance of program '%s'\n",
6309 i, prog->name);
6310 prog->instances.fds[i] = -1;
6311 if (result.pfd)
6312 *result.pfd = -1;
6313 continue;
6314 }
6315
6316 err = load_program(prog, result.new_insn_ptr,
6317 result.new_insn_cnt, license, kern_ver, &fd);
6318 if (err) {
6319 pr_warn("Loading the %dth instance of program '%s' failed\n",
6320 i, prog->name);
6321 goto out;
6322 }
6323
6324 if (result.pfd)
6325 *result.pfd = fd;
6326 prog->instances.fds[i] = fd;
6327 }
6328out:
6329 if (err)
6330 pr_warn("failed to load program '%s'\n", prog->name);
6331 zfree(&prog->insns);
6332 prog->insns_cnt = 0;
6333 return libbpf_err(err);
6334}
6335
6336static int
6337bpf_object__load_progs(struct bpf_object *obj, int log_level)
6338{
6339 struct bpf_program *prog;
6340 size_t i;
6341 int err;
6342
6343 for (i = 0; i < obj->nr_programs; i++) {
6344 prog = &obj->programs[i];
6345 err = bpf_object__sanitize_prog(obj, prog);
6346 if (err)
6347 return err;
6348 }
6349
6350 for (i = 0; i < obj->nr_programs; i++) {
6351 prog = &obj->programs[i];
6352 if (prog_is_subprog(obj, prog))
6353 continue;
6354 if (!prog->load) {
6355 pr_debug("prog '%s': skipped loading\n", prog->name);
6356 continue;
6357 }
6358 prog->log_level |= log_level;
6359 err = bpf_program__load(prog, obj->license, obj->kern_version);
6360 if (err)
6361 return err;
6362 }
6363 if (obj->gen_loader)
6364 bpf_object__free_relocs(obj);
6365 return 0;
6366}
6367
6368static const struct bpf_sec_def *find_sec_def(const char *sec_name);
6369
6370static struct bpf_object *
6371__bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,
6372 const struct bpf_object_open_opts *opts)
6373{
6374 const char *obj_name, *kconfig, *btf_tmp_path;
6375 struct bpf_program *prog;
6376 struct bpf_object *obj;
6377 char tmp_name[64];
6378 int err;
6379
6380 if (elf_version(EV_CURRENT) == EV_NONE) {
6381 pr_warn("failed to init libelf for %s\n",
6382 path ? : "(mem buf)");
6383 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
6384 }
6385
6386 if (!OPTS_VALID(opts, bpf_object_open_opts))
6387 return ERR_PTR(-EINVAL);
6388
6389 obj_name = OPTS_GET(opts, object_name, NULL);
6390 if (obj_buf) {
6391 if (!obj_name) {
6392 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
6393 (unsigned long)obj_buf,
6394 (unsigned long)obj_buf_sz);
6395 obj_name = tmp_name;
6396 }
6397 path = obj_name;
6398 pr_debug("loading object '%s' from buffer\n", obj_name);
6399 }
6400
6401 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
6402 if (IS_ERR(obj))
6403 return obj;
6404
6405 btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL);
6406 if (btf_tmp_path) {
6407 if (strlen(btf_tmp_path) >= PATH_MAX) {
6408 err = -ENAMETOOLONG;
6409 goto out;
6410 }
6411 obj->btf_custom_path = strdup(btf_tmp_path);
6412 if (!obj->btf_custom_path) {
6413 err = -ENOMEM;
6414 goto out;
6415 }
6416 }
6417
6418 kconfig = OPTS_GET(opts, kconfig, NULL);
6419 if (kconfig) {
6420 obj->kconfig = strdup(kconfig);
6421 if (!obj->kconfig) {
6422 err = -ENOMEM;
6423 goto out;
6424 }
6425 }
6426
6427 err = bpf_object__elf_init(obj);
6428 err = err ? : bpf_object__check_endianness(obj);
6429 err = err ? : bpf_object__elf_collect(obj);
6430 err = err ? : bpf_object__collect_externs(obj);
6431 err = err ? : bpf_object__finalize_btf(obj);
6432 err = err ? : bpf_object__init_maps(obj, opts);
6433 err = err ? : bpf_object__collect_relos(obj);
6434 if (err)
6435 goto out;
6436 bpf_object__elf_finish(obj);
6437
6438 bpf_object__for_each_program(prog, obj) {
6439 prog->sec_def = find_sec_def(prog->sec_name);
6440 if (!prog->sec_def) {
6441
6442 pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
6443 prog->name, prog->sec_name);
6444 continue;
6445 }
6446
6447 if (prog->sec_def->is_sleepable)
6448 prog->prog_flags |= BPF_F_SLEEPABLE;
6449 bpf_program__set_type(prog, prog->sec_def->prog_type);
6450 bpf_program__set_expected_attach_type(prog,
6451 prog->sec_def->expected_attach_type);
6452
6453 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING ||
6454 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT)
6455 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
6456 }
6457
6458 return obj;
6459out:
6460 bpf_object__close(obj);
6461 return ERR_PTR(err);
6462}
6463
6464static struct bpf_object *
6465__bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)
6466{
6467 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
6468 .relaxed_maps = flags & MAPS_RELAX_COMPAT,
6469 );
6470
6471
6472 if (!attr->file)
6473 return NULL;
6474
6475 pr_debug("loading %s\n", attr->file);
6476 return __bpf_object__open(attr->file, NULL, 0, &opts);
6477}
6478
6479struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
6480{
6481 return libbpf_ptr(__bpf_object__open_xattr(attr, 0));
6482}
6483
6484struct bpf_object *bpf_object__open(const char *path)
6485{
6486 struct bpf_object_open_attr attr = {
6487 .file = path,
6488 .prog_type = BPF_PROG_TYPE_UNSPEC,
6489 };
6490
6491 return libbpf_ptr(__bpf_object__open_xattr(&attr, 0));
6492}
6493
6494struct bpf_object *
6495bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
6496{
6497 if (!path)
6498 return libbpf_err_ptr(-EINVAL);
6499
6500 pr_debug("loading %s\n", path);
6501
6502 return libbpf_ptr(__bpf_object__open(path, NULL, 0, opts));
6503}
6504
6505struct bpf_object *
6506bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
6507 const struct bpf_object_open_opts *opts)
6508{
6509 if (!obj_buf || obj_buf_sz == 0)
6510 return libbpf_err_ptr(-EINVAL);
6511
6512 return libbpf_ptr(__bpf_object__open(NULL, obj_buf, obj_buf_sz, opts));
6513}
6514
6515struct bpf_object *
6516bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
6517 const char *name)
6518{
6519 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
6520 .object_name = name,
6521
6522 .relaxed_maps = true,
6523 );
6524
6525
6526 if (!obj_buf || obj_buf_sz == 0)
6527 return errno = EINVAL, NULL;
6528
6529 return libbpf_ptr(__bpf_object__open(NULL, obj_buf, obj_buf_sz, &opts));
6530}
6531
6532int bpf_object__unload(struct bpf_object *obj)
6533{
6534 size_t i;
6535
6536 if (!obj)
6537 return libbpf_err(-EINVAL);
6538
6539 for (i = 0; i < obj->nr_maps; i++) {
6540 zclose(obj->maps[i].fd);
6541 if (obj->maps[i].st_ops)
6542 zfree(&obj->maps[i].st_ops->kern_vdata);
6543 }
6544
6545 for (i = 0; i < obj->nr_programs; i++)
6546 bpf_program__unload(&obj->programs[i]);
6547
6548 return 0;
6549}
6550
6551static int bpf_object__sanitize_maps(struct bpf_object *obj)
6552{
6553 struct bpf_map *m;
6554
6555 bpf_object__for_each_map(m, obj) {
6556 if (!bpf_map__is_internal(m))
6557 continue;
6558 if (!kernel_supports(obj, FEAT_GLOBAL_DATA)) {
6559 pr_warn("kernel doesn't support global data\n");
6560 return -ENOTSUP;
6561 }
6562 if (!kernel_supports(obj, FEAT_ARRAY_MMAP))
6563 m->def.map_flags ^= BPF_F_MMAPABLE;
6564 }
6565
6566 return 0;
6567}
6568
6569static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
6570{
6571 char sym_type, sym_name[500];
6572 unsigned long long sym_addr;
6573 const struct btf_type *t;
6574 struct extern_desc *ext;
6575 int ret, err = 0;
6576 FILE *f;
6577
6578 f = fopen("/proc/kallsyms", "r");
6579 if (!f) {
6580 err = -errno;
6581 pr_warn("failed to open /proc/kallsyms: %d\n", err);
6582 return err;
6583 }
6584
6585 while (true) {
6586 ret = fscanf(f, "%llx %c %499s%*[^\n]\n",
6587 &sym_addr, &sym_type, sym_name);
6588 if (ret == EOF && feof(f))
6589 break;
6590 if (ret != 3) {
6591 pr_warn("failed to read kallsyms entry: %d\n", ret);
6592 err = -EINVAL;
6593 goto out;
6594 }
6595
6596 ext = find_extern_by_name(obj, sym_name);
6597 if (!ext || ext->type != EXT_KSYM)
6598 continue;
6599
6600 t = btf__type_by_id(obj->btf, ext->btf_id);
6601 if (!btf_is_var(t))
6602 continue;
6603
6604 if (ext->is_set && ext->ksym.addr != sym_addr) {
6605 pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n",
6606 sym_name, ext->ksym.addr, sym_addr);
6607 err = -EINVAL;
6608 goto out;
6609 }
6610 if (!ext->is_set) {
6611 ext->is_set = true;
6612 ext->ksym.addr = sym_addr;
6613 pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr);
6614 }
6615 }
6616
6617out:
6618 fclose(f);
6619 return err;
6620}
6621
6622static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
6623 __u16 kind, struct btf **res_btf,
6624 int *res_btf_fd)
6625{
6626 int i, id, btf_fd, err;
6627 struct btf *btf;
6628
6629 btf = obj->btf_vmlinux;
6630 btf_fd = 0;
6631 id = btf__find_by_name_kind(btf, ksym_name, kind);
6632
6633 if (id == -ENOENT) {
6634 err = load_module_btfs(obj);
6635 if (err)
6636 return err;
6637
6638 for (i = 0; i < obj->btf_module_cnt; i++) {
6639 btf = obj->btf_modules[i].btf;
6640
6641 btf_fd = obj->btf_modules[i].fd;
6642 id = btf__find_by_name_kind(btf, ksym_name, kind);
6643 if (id != -ENOENT)
6644 break;
6645 }
6646 }
6647 if (id <= 0)
6648 return -ESRCH;
6649
6650 *res_btf = btf;
6651 *res_btf_fd = btf_fd;
6652 return id;
6653}
6654
6655static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
6656 struct extern_desc *ext)
6657{
6658 const struct btf_type *targ_var, *targ_type;
6659 __u32 targ_type_id, local_type_id;
6660 const char *targ_var_name;
6661 int id, btf_fd = 0, err;
6662 struct btf *btf = NULL;
6663
6664 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &btf_fd);
6665 if (id == -ESRCH && ext->is_weak) {
6666 return 0;
6667 } else if (id < 0) {
6668 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n",
6669 ext->name);
6670 return id;
6671 }
6672
6673
6674 local_type_id = ext->ksym.type_id;
6675
6676
6677 targ_var = btf__type_by_id(btf, id);
6678 targ_var_name = btf__name_by_offset(btf, targ_var->name_off);
6679 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);
6680
6681 err = bpf_core_types_are_compat(obj->btf, local_type_id,
6682 btf, targ_type_id);
6683 if (err <= 0) {
6684 const struct btf_type *local_type;
6685 const char *targ_name, *local_name;
6686
6687 local_type = btf__type_by_id(obj->btf, local_type_id);
6688 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
6689 targ_name = btf__name_by_offset(btf, targ_type->name_off);
6690
6691 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
6692 ext->name, local_type_id,
6693 btf_kind_str(local_type), local_name, targ_type_id,
6694 btf_kind_str(targ_type), targ_name);
6695 return -EINVAL;
6696 }
6697
6698 ext->is_set = true;
6699 ext->ksym.kernel_btf_obj_fd = btf_fd;
6700 ext->ksym.kernel_btf_id = id;
6701 pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n",
6702 ext->name, id, btf_kind_str(targ_var), targ_var_name);
6703
6704 return 0;
6705}
6706
6707static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
6708 struct extern_desc *ext)
6709{
6710 int local_func_proto_id, kfunc_proto_id, kfunc_id;
6711 const struct btf_type *kern_func;
6712 struct btf *kern_btf = NULL;
6713 int ret, kern_btf_fd = 0;
6714
6715 local_func_proto_id = ext->ksym.type_id;
6716
6717 kfunc_id = find_ksym_btf_id(obj, ext->name, BTF_KIND_FUNC,
6718 &kern_btf, &kern_btf_fd);
6719 if (kfunc_id < 0) {
6720 pr_warn("extern (func ksym) '%s': not found in kernel BTF\n",
6721 ext->name);
6722 return kfunc_id;
6723 }
6724
6725 if (kern_btf != obj->btf_vmlinux) {
6726 pr_warn("extern (func ksym) '%s': function in kernel module is not supported\n",
6727 ext->name);
6728 return -ENOTSUP;
6729 }
6730
6731 kern_func = btf__type_by_id(kern_btf, kfunc_id);
6732 kfunc_proto_id = kern_func->type;
6733
6734 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
6735 kern_btf, kfunc_proto_id);
6736 if (ret <= 0) {
6737 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with kernel [%d]\n",
6738 ext->name, local_func_proto_id, kfunc_proto_id);
6739 return -EINVAL;
6740 }
6741
6742 ext->is_set = true;
6743 ext->ksym.kernel_btf_obj_fd = kern_btf_fd;
6744 ext->ksym.kernel_btf_id = kfunc_id;
6745 pr_debug("extern (func ksym) '%s': resolved to kernel [%d]\n",
6746 ext->name, kfunc_id);
6747
6748 return 0;
6749}
6750
6751static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
6752{
6753 const struct btf_type *t;
6754 struct extern_desc *ext;
6755 int i, err;
6756
6757 for (i = 0; i < obj->nr_extern; i++) {
6758 ext = &obj->externs[i];
6759 if (ext->type != EXT_KSYM || !ext->ksym.type_id)
6760 continue;
6761
6762 if (obj->gen_loader) {
6763 ext->is_set = true;
6764 ext->ksym.kernel_btf_obj_fd = 0;
6765 ext->ksym.kernel_btf_id = 0;
6766 continue;
6767 }
6768 t = btf__type_by_id(obj->btf, ext->btf_id);
6769 if (btf_is_var(t))
6770 err = bpf_object__resolve_ksym_var_btf_id(obj, ext);
6771 else
6772 err = bpf_object__resolve_ksym_func_btf_id(obj, ext);
6773 if (err)
6774 return err;
6775 }
6776 return 0;
6777}
6778
6779static int bpf_object__resolve_externs(struct bpf_object *obj,
6780 const char *extra_kconfig)
6781{
6782 bool need_config = false, need_kallsyms = false;
6783 bool need_vmlinux_btf = false;
6784 struct extern_desc *ext;
6785 void *kcfg_data = NULL;
6786 int err, i;
6787
6788 if (obj->nr_extern == 0)
6789 return 0;
6790
6791 if (obj->kconfig_map_idx >= 0)
6792 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
6793
6794 for (i = 0; i < obj->nr_extern; i++) {
6795 ext = &obj->externs[i];
6796
6797 if (ext->type == EXT_KCFG &&
6798 strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
6799 void *ext_val = kcfg_data + ext->kcfg.data_off;
6800 __u32 kver = get_kernel_version();
6801
6802 if (!kver) {
6803 pr_warn("failed to get kernel version\n");
6804 return -EINVAL;
6805 }
6806 err = set_kcfg_value_num(ext, ext_val, kver);
6807 if (err)
6808 return err;
6809 pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver);
6810 } else if (ext->type == EXT_KCFG &&
6811 strncmp(ext->name, "CONFIG_", 7) == 0) {
6812 need_config = true;
6813 } else if (ext->type == EXT_KSYM) {
6814 if (ext->ksym.type_id)
6815 need_vmlinux_btf = true;
6816 else
6817 need_kallsyms = true;
6818 } else {
6819 pr_warn("unrecognized extern '%s'\n", ext->name);
6820 return -EINVAL;
6821 }
6822 }
6823 if (need_config && extra_kconfig) {
6824 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
6825 if (err)
6826 return -EINVAL;
6827 need_config = false;
6828 for (i = 0; i < obj->nr_extern; i++) {
6829 ext = &obj->externs[i];
6830 if (ext->type == EXT_KCFG && !ext->is_set) {
6831 need_config = true;
6832 break;
6833 }
6834 }
6835 }
6836 if (need_config) {
6837 err = bpf_object__read_kconfig_file(obj, kcfg_data);
6838 if (err)
6839 return -EINVAL;
6840 }
6841 if (need_kallsyms) {
6842 err = bpf_object__read_kallsyms_file(obj);
6843 if (err)
6844 return -EINVAL;
6845 }
6846 if (need_vmlinux_btf) {
6847 err = bpf_object__resolve_ksyms_btf_id(obj);
6848 if (err)
6849 return -EINVAL;
6850 }
6851 for (i = 0; i < obj->nr_extern; i++) {
6852 ext = &obj->externs[i];
6853
6854 if (!ext->is_set && !ext->is_weak) {
6855 pr_warn("extern %s (strong) not resolved\n", ext->name);
6856 return -ESRCH;
6857 } else if (!ext->is_set) {
6858 pr_debug("extern %s (weak) not resolved, defaulting to zero\n",
6859 ext->name);
6860 }
6861 }
6862
6863 return 0;
6864}
6865
6866int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
6867{
6868 struct bpf_object *obj;
6869 int err, i;
6870
6871 if (!attr)
6872 return libbpf_err(-EINVAL);
6873 obj = attr->obj;
6874 if (!obj)
6875 return libbpf_err(-EINVAL);
6876
6877 if (obj->loaded) {
6878 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
6879 return libbpf_err(-EINVAL);
6880 }
6881
6882 if (obj->gen_loader)
6883 bpf_gen__init(obj->gen_loader, attr->log_level);
6884
6885 err = bpf_object__probe_loading(obj);
6886 err = err ? : bpf_object__load_vmlinux_btf(obj, false);
6887 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
6888 err = err ? : bpf_object__sanitize_and_load_btf(obj);
6889 err = err ? : bpf_object__sanitize_maps(obj);
6890 err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
6891 err = err ? : bpf_object__create_maps(obj);
6892 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : attr->target_btf_path);
6893 err = err ? : bpf_object__load_progs(obj, attr->log_level);
6894
6895 if (obj->gen_loader) {
6896
6897 if (obj->btf)
6898 btf__set_fd(obj->btf, -1);
6899 for (i = 0; i < obj->nr_maps; i++)
6900 obj->maps[i].fd = -1;
6901 if (!err)
6902 err = bpf_gen__finish(obj->gen_loader);
6903 }
6904
6905
6906 for (i = 0; i < obj->btf_module_cnt; i++) {
6907 close(obj->btf_modules[i].fd);
6908 btf__free(obj->btf_modules[i].btf);
6909 free(obj->btf_modules[i].name);
6910 }
6911 free(obj->btf_modules);
6912
6913
6914 btf__free(obj->btf_vmlinux);
6915 obj->btf_vmlinux = NULL;
6916
6917 obj->loaded = true;
6918
6919 if (err)
6920 goto out;
6921
6922 return 0;
6923out:
6924
6925 for (i = 0; i < obj->nr_maps; i++)
6926 if (obj->maps[i].pinned && !obj->maps[i].reused)
6927 bpf_map__unpin(&obj->maps[i], NULL);
6928
6929 bpf_object__unload(obj);
6930 pr_warn("failed to load object '%s'\n", obj->path);
6931 return libbpf_err(err);
6932}
6933
6934int bpf_object__load(struct bpf_object *obj)
6935{
6936 struct bpf_object_load_attr attr = {
6937 .obj = obj,
6938 };
6939
6940 return bpf_object__load_xattr(&attr);
6941}
6942
6943static int make_parent_dir(const char *path)
6944{
6945 char *cp, errmsg[STRERR_BUFSIZE];
6946 char *dname, *dir;
6947 int err = 0;
6948
6949 dname = strdup(path);
6950 if (dname == NULL)
6951 return -ENOMEM;
6952
6953 dir = dirname(dname);
6954 if (mkdir(dir, 0700) && errno != EEXIST)
6955 err = -errno;
6956
6957 free(dname);
6958 if (err) {
6959 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
6960 pr_warn("failed to mkdir %s: %s\n", path, cp);
6961 }
6962 return err;
6963}
6964
6965static int check_path(const char *path)
6966{
6967 char *cp, errmsg[STRERR_BUFSIZE];
6968 struct statfs st_fs;
6969 char *dname, *dir;
6970 int err = 0;
6971
6972 if (path == NULL)
6973 return -EINVAL;
6974
6975 dname = strdup(path);
6976 if (dname == NULL)
6977 return -ENOMEM;
6978
6979 dir = dirname(dname);
6980 if (statfs(dir, &st_fs)) {
6981 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
6982 pr_warn("failed to statfs %s: %s\n", dir, cp);
6983 err = -errno;
6984 }
6985 free(dname);
6986
6987 if (!err && st_fs.f_type != BPF_FS_MAGIC) {
6988 pr_warn("specified path %s is not on BPF FS\n", path);
6989 err = -EINVAL;
6990 }
6991
6992 return err;
6993}
6994
6995int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
6996 int instance)
6997{
6998 char *cp, errmsg[STRERR_BUFSIZE];
6999 int err;
7000
7001 err = make_parent_dir(path);
7002 if (err)
7003 return libbpf_err(err);
7004
7005 err = check_path(path);
7006 if (err)
7007 return libbpf_err(err);
7008
7009 if (prog == NULL) {
7010 pr_warn("invalid program pointer\n");
7011 return libbpf_err(-EINVAL);
7012 }
7013
7014 if (instance < 0 || instance >= prog->instances.nr) {
7015 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
7016 instance, prog->name, prog->instances.nr);
7017 return libbpf_err(-EINVAL);
7018 }
7019
7020 if (bpf_obj_pin(prog->instances.fds[instance], path)) {
7021 err = -errno;
7022 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
7023 pr_warn("failed to pin program: %s\n", cp);
7024 return libbpf_err(err);
7025 }
7026 pr_debug("pinned program '%s'\n", path);
7027
7028 return 0;
7029}
7030
7031int bpf_program__unpin_instance(struct bpf_program *prog, const char *path,
7032 int instance)
7033{
7034 int err;
7035
7036 err = check_path(path);
7037 if (err)
7038 return libbpf_err(err);
7039
7040 if (prog == NULL) {
7041 pr_warn("invalid program pointer\n");
7042 return libbpf_err(-EINVAL);
7043 }
7044
7045 if (instance < 0 || instance >= prog->instances.nr) {
7046 pr_warn("invalid prog instance %d of prog %s (max %d)\n",
7047 instance, prog->name, prog->instances.nr);
7048 return libbpf_err(-EINVAL);
7049 }
7050
7051 err = unlink(path);
7052 if (err != 0)
7053 return libbpf_err(-errno);
7054
7055 pr_debug("unpinned program '%s'\n", path);
7056
7057 return 0;
7058}
7059
7060int bpf_program__pin(struct bpf_program *prog, const char *path)
7061{
7062 int i, err;
7063
7064 err = make_parent_dir(path);
7065 if (err)
7066 return libbpf_err(err);
7067
7068 err = check_path(path);
7069 if (err)
7070 return libbpf_err(err);
7071
7072 if (prog == NULL) {
7073 pr_warn("invalid program pointer\n");
7074 return libbpf_err(-EINVAL);
7075 }
7076
7077 if (prog->instances.nr <= 0) {
7078 pr_warn("no instances of prog %s to pin\n", prog->name);
7079 return libbpf_err(-EINVAL);
7080 }
7081
7082 if (prog->instances.nr == 1) {
7083
7084 return bpf_program__pin_instance(prog, path, 0);
7085 }
7086
7087 for (i = 0; i < prog->instances.nr; i++) {
7088 char buf[PATH_MAX];
7089 int len;
7090
7091 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7092 if (len < 0) {
7093 err = -EINVAL;
7094 goto err_unpin;
7095 } else if (len >= PATH_MAX) {
7096 err = -ENAMETOOLONG;
7097 goto err_unpin;
7098 }
7099
7100 err = bpf_program__pin_instance(prog, buf, i);
7101 if (err)
7102 goto err_unpin;
7103 }
7104
7105 return 0;
7106
7107err_unpin:
7108 for (i = i - 1; i >= 0; i--) {
7109 char buf[PATH_MAX];
7110 int len;
7111
7112 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7113 if (len < 0)
7114 continue;
7115 else if (len >= PATH_MAX)
7116 continue;
7117
7118 bpf_program__unpin_instance(prog, buf, i);
7119 }
7120
7121 rmdir(path);
7122
7123 return libbpf_err(err);
7124}
7125
7126int bpf_program__unpin(struct bpf_program *prog, const char *path)
7127{
7128 int i, err;
7129
7130 err = check_path(path);
7131 if (err)
7132 return libbpf_err(err);
7133
7134 if (prog == NULL) {
7135 pr_warn("invalid program pointer\n");
7136 return libbpf_err(-EINVAL);
7137 }
7138
7139 if (prog->instances.nr <= 0) {
7140 pr_warn("no instances of prog %s to pin\n", prog->name);
7141 return libbpf_err(-EINVAL);
7142 }
7143
7144 if (prog->instances.nr == 1) {
7145
7146 return bpf_program__unpin_instance(prog, path, 0);
7147 }
7148
7149 for (i = 0; i < prog->instances.nr; i++) {
7150 char buf[PATH_MAX];
7151 int len;
7152
7153 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
7154 if (len < 0)
7155 return libbpf_err(-EINVAL);
7156 else if (len >= PATH_MAX)
7157 return libbpf_err(-ENAMETOOLONG);
7158
7159 err = bpf_program__unpin_instance(prog, buf, i);
7160 if (err)
7161 return err;
7162 }
7163
7164 err = rmdir(path);
7165 if (err)
7166 return libbpf_err(-errno);
7167
7168 return 0;
7169}
7170
7171int bpf_map__pin(struct bpf_map *map, const char *path)
7172{
7173 char *cp, errmsg[STRERR_BUFSIZE];
7174 int err;
7175
7176 if (map == NULL) {
7177 pr_warn("invalid map pointer\n");
7178 return libbpf_err(-EINVAL);
7179 }
7180
7181 if (map->pin_path) {
7182 if (path && strcmp(path, map->pin_path)) {
7183 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7184 bpf_map__name(map), map->pin_path, path);
7185 return libbpf_err(-EINVAL);
7186 } else if (map->pinned) {
7187 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",
7188 bpf_map__name(map), map->pin_path);
7189 return 0;
7190 }
7191 } else {
7192 if (!path) {
7193 pr_warn("missing a path to pin map '%s' at\n",
7194 bpf_map__name(map));
7195 return libbpf_err(-EINVAL);
7196 } else if (map->pinned) {
7197 pr_warn("map '%s' already pinned\n", bpf_map__name(map));
7198 return libbpf_err(-EEXIST);
7199 }
7200
7201 map->pin_path = strdup(path);
7202 if (!map->pin_path) {
7203 err = -errno;
7204 goto out_err;
7205 }
7206 }
7207
7208 err = make_parent_dir(map->pin_path);
7209 if (err)
7210 return libbpf_err(err);
7211
7212 err = check_path(map->pin_path);
7213 if (err)
7214 return libbpf_err(err);
7215
7216 if (bpf_obj_pin(map->fd, map->pin_path)) {
7217 err = -errno;
7218 goto out_err;
7219 }
7220
7221 map->pinned = true;
7222 pr_debug("pinned map '%s'\n", map->pin_path);
7223
7224 return 0;
7225
7226out_err:
7227 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
7228 pr_warn("failed to pin map: %s\n", cp);
7229 return libbpf_err(err);
7230}
7231
7232int bpf_map__unpin(struct bpf_map *map, const char *path)
7233{
7234 int err;
7235
7236 if (map == NULL) {
7237 pr_warn("invalid map pointer\n");
7238 return libbpf_err(-EINVAL);
7239 }
7240
7241 if (map->pin_path) {
7242 if (path && strcmp(path, map->pin_path)) {
7243 pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
7244 bpf_map__name(map), map->pin_path, path);
7245 return libbpf_err(-EINVAL);
7246 }
7247 path = map->pin_path;
7248 } else if (!path) {
7249 pr_warn("no path to unpin map '%s' from\n",
7250 bpf_map__name(map));
7251 return libbpf_err(-EINVAL);
7252 }
7253
7254 err = check_path(path);
7255 if (err)
7256 return libbpf_err(err);
7257
7258 err = unlink(path);
7259 if (err != 0)
7260 return libbpf_err(-errno);
7261
7262 map->pinned = false;
7263 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path);
7264
7265 return 0;
7266}
7267
7268int bpf_map__set_pin_path(struct bpf_map *map, const char *path)
7269{
7270 char *new = NULL;
7271
7272 if (path) {
7273 new = strdup(path);
7274 if (!new)
7275 return libbpf_err(-errno);
7276 }
7277
7278 free(map->pin_path);
7279 map->pin_path = new;
7280 return 0;
7281}
7282
7283const char *bpf_map__get_pin_path(const struct bpf_map *map)
7284{
7285 return map->pin_path;
7286}
7287
7288const char *bpf_map__pin_path(const struct bpf_map *map)
7289{
7290 return map->pin_path;
7291}
7292
7293bool bpf_map__is_pinned(const struct bpf_map *map)
7294{
7295 return map->pinned;
7296}
7297
7298static void sanitize_pin_path(char *s)
7299{
7300
7301 while (*s) {
7302 if (*s == '.')
7303 *s = '_';
7304 s++;
7305 }
7306}
7307
7308int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
7309{
7310 struct bpf_map *map;
7311 int err;
7312
7313 if (!obj)
7314 return libbpf_err(-ENOENT);
7315
7316 if (!obj->loaded) {
7317 pr_warn("object not yet loaded; load it first\n");
7318 return libbpf_err(-ENOENT);
7319 }
7320
7321 bpf_object__for_each_map(map, obj) {
7322 char *pin_path = NULL;
7323 char buf[PATH_MAX];
7324
7325 if (path) {
7326 int len;
7327
7328 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7329 bpf_map__name(map));
7330 if (len < 0) {
7331 err = -EINVAL;
7332 goto err_unpin_maps;
7333 } else if (len >= PATH_MAX) {
7334 err = -ENAMETOOLONG;
7335 goto err_unpin_maps;
7336 }
7337 sanitize_pin_path(buf);
7338 pin_path = buf;
7339 } else if (!map->pin_path) {
7340 continue;
7341 }
7342
7343 err = bpf_map__pin(map, pin_path);
7344 if (err)
7345 goto err_unpin_maps;
7346 }
7347
7348 return 0;
7349
7350err_unpin_maps:
7351 while ((map = bpf_map__prev(map, obj))) {
7352 if (!map->pin_path)
7353 continue;
7354
7355 bpf_map__unpin(map, NULL);
7356 }
7357
7358 return libbpf_err(err);
7359}
7360
7361int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
7362{
7363 struct bpf_map *map;
7364 int err;
7365
7366 if (!obj)
7367 return libbpf_err(-ENOENT);
7368
7369 bpf_object__for_each_map(map, obj) {
7370 char *pin_path = NULL;
7371 char buf[PATH_MAX];
7372
7373 if (path) {
7374 int len;
7375
7376 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7377 bpf_map__name(map));
7378 if (len < 0)
7379 return libbpf_err(-EINVAL);
7380 else if (len >= PATH_MAX)
7381 return libbpf_err(-ENAMETOOLONG);
7382 sanitize_pin_path(buf);
7383 pin_path = buf;
7384 } else if (!map->pin_path) {
7385 continue;
7386 }
7387
7388 err = bpf_map__unpin(map, pin_path);
7389 if (err)
7390 return libbpf_err(err);
7391 }
7392
7393 return 0;
7394}
7395
7396int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
7397{
7398 struct bpf_program *prog;
7399 int err;
7400
7401 if (!obj)
7402 return libbpf_err(-ENOENT);
7403
7404 if (!obj->loaded) {
7405 pr_warn("object not yet loaded; load it first\n");
7406 return libbpf_err(-ENOENT);
7407 }
7408
7409 bpf_object__for_each_program(prog, obj) {
7410 char buf[PATH_MAX];
7411 int len;
7412
7413 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7414 prog->pin_name);
7415 if (len < 0) {
7416 err = -EINVAL;
7417 goto err_unpin_programs;
7418 } else if (len >= PATH_MAX) {
7419 err = -ENAMETOOLONG;
7420 goto err_unpin_programs;
7421 }
7422
7423 err = bpf_program__pin(prog, buf);
7424 if (err)
7425 goto err_unpin_programs;
7426 }
7427
7428 return 0;
7429
7430err_unpin_programs:
7431 while ((prog = bpf_program__prev(prog, obj))) {
7432 char buf[PATH_MAX];
7433 int len;
7434
7435 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7436 prog->pin_name);
7437 if (len < 0)
7438 continue;
7439 else if (len >= PATH_MAX)
7440 continue;
7441
7442 bpf_program__unpin(prog, buf);
7443 }
7444
7445 return libbpf_err(err);
7446}
7447
7448int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
7449{
7450 struct bpf_program *prog;
7451 int err;
7452
7453 if (!obj)
7454 return libbpf_err(-ENOENT);
7455
7456 bpf_object__for_each_program(prog, obj) {
7457 char buf[PATH_MAX];
7458 int len;
7459
7460 len = snprintf(buf, PATH_MAX, "%s/%s", path,
7461 prog->pin_name);
7462 if (len < 0)
7463 return libbpf_err(-EINVAL);
7464 else if (len >= PATH_MAX)
7465 return libbpf_err(-ENAMETOOLONG);
7466
7467 err = bpf_program__unpin(prog, buf);
7468 if (err)
7469 return libbpf_err(err);
7470 }
7471
7472 return 0;
7473}
7474
7475int bpf_object__pin(struct bpf_object *obj, const char *path)
7476{
7477 int err;
7478
7479 err = bpf_object__pin_maps(obj, path);
7480 if (err)
7481 return libbpf_err(err);
7482
7483 err = bpf_object__pin_programs(obj, path);
7484 if (err) {
7485 bpf_object__unpin_maps(obj, path);
7486 return libbpf_err(err);
7487 }
7488
7489 return 0;
7490}
7491
7492static void bpf_map__destroy(struct bpf_map *map)
7493{
7494 if (map->clear_priv)
7495 map->clear_priv(map, map->priv);
7496 map->priv = NULL;
7497 map->clear_priv = NULL;
7498
7499 if (map->inner_map) {
7500 bpf_map__destroy(map->inner_map);
7501 zfree(&map->inner_map);
7502 }
7503
7504 zfree(&map->init_slots);
7505 map->init_slots_sz = 0;
7506
7507 if (map->mmaped) {
7508 munmap(map->mmaped, bpf_map_mmap_sz(map));
7509 map->mmaped = NULL;
7510 }
7511
7512 if (map->st_ops) {
7513 zfree(&map->st_ops->data);
7514 zfree(&map->st_ops->progs);
7515 zfree(&map->st_ops->kern_func_off);
7516 zfree(&map->st_ops);
7517 }
7518
7519 zfree(&map->name);
7520 zfree(&map->pin_path);
7521
7522 if (map->fd >= 0)
7523 zclose(map->fd);
7524}
7525
7526void bpf_object__close(struct bpf_object *obj)
7527{
7528 size_t i;
7529
7530 if (IS_ERR_OR_NULL(obj))
7531 return;
7532
7533 if (obj->clear_priv)
7534 obj->clear_priv(obj, obj->priv);
7535
7536 bpf_gen__free(obj->gen_loader);
7537 bpf_object__elf_finish(obj);
7538 bpf_object__unload(obj);
7539 btf__free(obj->btf);
7540 btf_ext__free(obj->btf_ext);
7541
7542 for (i = 0; i < obj->nr_maps; i++)
7543 bpf_map__destroy(&obj->maps[i]);
7544
7545 zfree(&obj->btf_custom_path);
7546 zfree(&obj->kconfig);
7547 zfree(&obj->externs);
7548 obj->nr_extern = 0;
7549
7550 zfree(&obj->maps);
7551 obj->nr_maps = 0;
7552
7553 if (obj->programs && obj->nr_programs) {
7554 for (i = 0; i < obj->nr_programs; i++)
7555 bpf_program__exit(&obj->programs[i]);
7556 }
7557 zfree(&obj->programs);
7558
7559 list_del(&obj->list);
7560 free(obj);
7561}
7562
7563struct bpf_object *
7564bpf_object__next(struct bpf_object *prev)
7565{
7566 struct bpf_object *next;
7567
7568 if (!prev)
7569 next = list_first_entry(&bpf_objects_list,
7570 struct bpf_object,
7571 list);
7572 else
7573 next = list_next_entry(prev, list);
7574
7575
7576 if (&next->list == &bpf_objects_list)
7577 return NULL;
7578
7579 return next;
7580}
7581
7582const char *bpf_object__name(const struct bpf_object *obj)
7583{
7584 return obj ? obj->name : libbpf_err_ptr(-EINVAL);
7585}
7586
7587unsigned int bpf_object__kversion(const struct bpf_object *obj)
7588{
7589 return obj ? obj->kern_version : 0;
7590}
7591
7592struct btf *bpf_object__btf(const struct bpf_object *obj)
7593{
7594 return obj ? obj->btf : NULL;
7595}
7596
7597int bpf_object__btf_fd(const struct bpf_object *obj)
7598{
7599 return obj->btf ? btf__fd(obj->btf) : -1;
7600}
7601
7602int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
7603{
7604 if (obj->loaded)
7605 return libbpf_err(-EINVAL);
7606
7607 obj->kern_version = kern_version;
7608
7609 return 0;
7610}
7611
7612int bpf_object__set_priv(struct bpf_object *obj, void *priv,
7613 bpf_object_clear_priv_t clear_priv)
7614{
7615 if (obj->priv && obj->clear_priv)
7616 obj->clear_priv(obj, obj->priv);
7617
7618 obj->priv = priv;
7619 obj->clear_priv = clear_priv;
7620 return 0;
7621}
7622
7623void *bpf_object__priv(const struct bpf_object *obj)
7624{
7625 return obj ? obj->priv : libbpf_err_ptr(-EINVAL);
7626}
7627
7628int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts)
7629{
7630 struct bpf_gen *gen;
7631
7632 if (!opts)
7633 return -EFAULT;
7634 if (!OPTS_VALID(opts, gen_loader_opts))
7635 return -EINVAL;
7636 gen = calloc(sizeof(*gen), 1);
7637 if (!gen)
7638 return -ENOMEM;
7639 gen->opts = opts;
7640 obj->gen_loader = gen;
7641 return 0;
7642}
7643
7644static struct bpf_program *
7645__bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
7646 bool forward)
7647{
7648 size_t nr_programs = obj->nr_programs;
7649 ssize_t idx;
7650
7651 if (!nr_programs)
7652 return NULL;
7653
7654 if (!p)
7655
7656 return forward ? &obj->programs[0] :
7657 &obj->programs[nr_programs - 1];
7658
7659 if (p->obj != obj) {
7660 pr_warn("error: program handler doesn't match object\n");
7661 return errno = EINVAL, NULL;
7662 }
7663
7664 idx = (p - obj->programs) + (forward ? 1 : -1);
7665 if (idx >= obj->nr_programs || idx < 0)
7666 return NULL;
7667 return &obj->programs[idx];
7668}
7669
7670struct bpf_program *
7671bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
7672{
7673 struct bpf_program *prog = prev;
7674
7675 do {
7676 prog = __bpf_program__iter(prog, obj, true);
7677 } while (prog && prog_is_subprog(obj, prog));
7678
7679 return prog;
7680}
7681
7682struct bpf_program *
7683bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj)
7684{
7685 struct bpf_program *prog = next;
7686
7687 do {
7688 prog = __bpf_program__iter(prog, obj, false);
7689 } while (prog && prog_is_subprog(obj, prog));
7690
7691 return prog;
7692}
7693
7694int bpf_program__set_priv(struct bpf_program *prog, void *priv,
7695 bpf_program_clear_priv_t clear_priv)
7696{
7697 if (prog->priv && prog->clear_priv)
7698 prog->clear_priv(prog, prog->priv);
7699
7700 prog->priv = priv;
7701 prog->clear_priv = clear_priv;
7702 return 0;
7703}
7704
7705void *bpf_program__priv(const struct bpf_program *prog)
7706{
7707 return prog ? prog->priv : libbpf_err_ptr(-EINVAL);
7708}
7709
7710void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
7711{
7712 prog->prog_ifindex = ifindex;
7713}
7714
7715const char *bpf_program__name(const struct bpf_program *prog)
7716{
7717 return prog->name;
7718}
7719
7720const char *bpf_program__section_name(const struct bpf_program *prog)
7721{
7722 return prog->sec_name;
7723}
7724
7725const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy)
7726{
7727 const char *title;
7728
7729 title = prog->sec_name;
7730 if (needs_copy) {
7731 title = strdup(title);
7732 if (!title) {
7733 pr_warn("failed to strdup program title\n");
7734 return libbpf_err_ptr(-ENOMEM);
7735 }
7736 }
7737
7738 return title;
7739}
7740
7741bool bpf_program__autoload(const struct bpf_program *prog)
7742{
7743 return prog->load;
7744}
7745
7746int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
7747{
7748 if (prog->obj->loaded)
7749 return libbpf_err(-EINVAL);
7750
7751 prog->load = autoload;
7752 return 0;
7753}
7754
7755int bpf_program__fd(const struct bpf_program *prog)
7756{
7757 return bpf_program__nth_fd(prog, 0);
7758}
7759
7760size_t bpf_program__size(const struct bpf_program *prog)
7761{
7762 return prog->insns_cnt * BPF_INSN_SZ;
7763}
7764
7765int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
7766 bpf_program_prep_t prep)
7767{
7768 int *instances_fds;
7769
7770 if (nr_instances <= 0 || !prep)
7771 return libbpf_err(-EINVAL);
7772
7773 if (prog->instances.nr > 0 || prog->instances.fds) {
7774 pr_warn("Can't set pre-processor after loading\n");
7775 return libbpf_err(-EINVAL);
7776 }
7777
7778 instances_fds = malloc(sizeof(int) * nr_instances);
7779 if (!instances_fds) {
7780 pr_warn("alloc memory failed for fds\n");
7781 return libbpf_err(-ENOMEM);
7782 }
7783
7784
7785 memset(instances_fds, -1, sizeof(int) * nr_instances);
7786
7787 prog->instances.nr = nr_instances;
7788 prog->instances.fds = instances_fds;
7789 prog->preprocessor = prep;
7790 return 0;
7791}
7792
7793int bpf_program__nth_fd(const struct bpf_program *prog, int n)
7794{
7795 int fd;
7796
7797 if (!prog)
7798 return libbpf_err(-EINVAL);
7799
7800 if (n >= prog->instances.nr || n < 0) {
7801 pr_warn("Can't get the %dth fd from program %s: only %d instances\n",
7802 n, prog->name, prog->instances.nr);
7803 return libbpf_err(-EINVAL);
7804 }
7805
7806 fd = prog->instances.fds[n];
7807 if (fd < 0) {
7808 pr_warn("%dth instance of program '%s' is invalid\n",
7809 n, prog->name);
7810 return libbpf_err(-ENOENT);
7811 }
7812
7813 return fd;
7814}
7815
7816enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog)
7817{
7818 return prog->type;
7819}
7820
7821void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
7822{
7823 prog->type = type;
7824}
7825
7826static bool bpf_program__is_type(const struct bpf_program *prog,
7827 enum bpf_prog_type type)
7828{
7829 return prog ? (prog->type == type) : false;
7830}
7831
7832#define BPF_PROG_TYPE_FNS(NAME, TYPE) \
7833int bpf_program__set_##NAME(struct bpf_program *prog) \
7834{ \
7835 if (!prog) \
7836 return libbpf_err(-EINVAL); \
7837 bpf_program__set_type(prog, TYPE); \
7838 return 0; \
7839} \
7840 \
7841bool bpf_program__is_##NAME(const struct bpf_program *prog) \
7842{ \
7843 return bpf_program__is_type(prog, TYPE); \
7844} \
7845
7846BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
7847BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM);
7848BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
7849BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
7850BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
7851BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
7852BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
7853BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
7854BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
7855BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
7856BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS);
7857BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT);
7858BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP);
7859
7860enum bpf_attach_type
7861bpf_program__get_expected_attach_type(const struct bpf_program *prog)
7862{
7863 return prog->expected_attach_type;
7864}
7865
7866void bpf_program__set_expected_attach_type(struct bpf_program *prog,
7867 enum bpf_attach_type type)
7868{
7869 prog->expected_attach_type = type;
7870}
7871
7872#define BPF_PROG_SEC_IMPL(string, ptype, eatype, eatype_optional, \
7873 attachable, attach_btf) \
7874 { \
7875 .sec = string, \
7876 .len = sizeof(string) - 1, \
7877 .prog_type = ptype, \
7878 .expected_attach_type = eatype, \
7879 .is_exp_attach_type_optional = eatype_optional, \
7880 .is_attachable = attachable, \
7881 .is_attach_btf = attach_btf, \
7882 }
7883
7884
7885#define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0)
7886
7887
7888#define BPF_APROG_SEC(string, ptype, atype) \
7889 BPF_PROG_SEC_IMPL(string, ptype, atype, true, 1, 0)
7890
7891
7892#define BPF_EAPROG_SEC(string, ptype, eatype) \
7893 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 1, 0)
7894
7895
7896#define BPF_PROG_BTF(string, ptype, eatype) \
7897 BPF_PROG_SEC_IMPL(string, ptype, eatype, false, 0, 1)
7898
7899
7900
7901
7902#define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
7903
7904#define SEC_DEF(sec_pfx, ptype, ...) { \
7905 .sec = sec_pfx, \
7906 .len = sizeof(sec_pfx) - 1, \
7907 .prog_type = BPF_PROG_TYPE_##ptype, \
7908 __VA_ARGS__ \
7909}
7910
7911static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
7912 struct bpf_program *prog);
7913static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
7914 struct bpf_program *prog);
7915static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
7916 struct bpf_program *prog);
7917static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
7918 struct bpf_program *prog);
7919static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
7920 struct bpf_program *prog);
7921static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
7922 struct bpf_program *prog);
7923
7924static const struct bpf_sec_def section_defs[] = {
7925 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),
7926 BPF_EAPROG_SEC("sk_reuseport/migrate", BPF_PROG_TYPE_SK_REUSEPORT,
7927 BPF_SK_REUSEPORT_SELECT_OR_MIGRATE),
7928 BPF_EAPROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT,
7929 BPF_SK_REUSEPORT_SELECT),
7930 SEC_DEF("kprobe/", KPROBE,
7931 .attach_fn = attach_kprobe),
7932 BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE),
7933 SEC_DEF("kretprobe/", KPROBE,
7934 .attach_fn = attach_kprobe),
7935 BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE),
7936 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS),
7937 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT),
7938 SEC_DEF("tracepoint/", TRACEPOINT,
7939 .attach_fn = attach_tp),
7940 SEC_DEF("tp/", TRACEPOINT,
7941 .attach_fn = attach_tp),
7942 SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT,
7943 .attach_fn = attach_raw_tp),
7944 SEC_DEF("raw_tp/", RAW_TRACEPOINT,
7945 .attach_fn = attach_raw_tp),
7946 SEC_DEF("tp_btf/", TRACING,
7947 .expected_attach_type = BPF_TRACE_RAW_TP,
7948 .is_attach_btf = true,
7949 .attach_fn = attach_trace),
7950 SEC_DEF("fentry/", TRACING,
7951 .expected_attach_type = BPF_TRACE_FENTRY,
7952 .is_attach_btf = true,
7953 .attach_fn = attach_trace),
7954 SEC_DEF("fmod_ret/", TRACING,
7955 .expected_attach_type = BPF_MODIFY_RETURN,
7956 .is_attach_btf = true,
7957 .attach_fn = attach_trace),
7958 SEC_DEF("fexit/", TRACING,
7959 .expected_attach_type = BPF_TRACE_FEXIT,
7960 .is_attach_btf = true,
7961 .attach_fn = attach_trace),
7962 SEC_DEF("fentry.s/", TRACING,
7963 .expected_attach_type = BPF_TRACE_FENTRY,
7964 .is_attach_btf = true,
7965 .is_sleepable = true,
7966 .attach_fn = attach_trace),
7967 SEC_DEF("fmod_ret.s/", TRACING,
7968 .expected_attach_type = BPF_MODIFY_RETURN,
7969 .is_attach_btf = true,
7970 .is_sleepable = true,
7971 .attach_fn = attach_trace),
7972 SEC_DEF("fexit.s/", TRACING,
7973 .expected_attach_type = BPF_TRACE_FEXIT,
7974 .is_attach_btf = true,
7975 .is_sleepable = true,
7976 .attach_fn = attach_trace),
7977 SEC_DEF("freplace/", EXT,
7978 .is_attach_btf = true,
7979 .attach_fn = attach_trace),
7980 SEC_DEF("lsm/", LSM,
7981 .is_attach_btf = true,
7982 .expected_attach_type = BPF_LSM_MAC,
7983 .attach_fn = attach_lsm),
7984 SEC_DEF("lsm.s/", LSM,
7985 .is_attach_btf = true,
7986 .is_sleepable = true,
7987 .expected_attach_type = BPF_LSM_MAC,
7988 .attach_fn = attach_lsm),
7989 SEC_DEF("iter/", TRACING,
7990 .expected_attach_type = BPF_TRACE_ITER,
7991 .is_attach_btf = true,
7992 .attach_fn = attach_iter),
7993 SEC_DEF("syscall", SYSCALL,
7994 .is_sleepable = true),
7995 BPF_EAPROG_SEC("xdp_devmap/", BPF_PROG_TYPE_XDP,
7996 BPF_XDP_DEVMAP),
7997 BPF_EAPROG_SEC("xdp_cpumap/", BPF_PROG_TYPE_XDP,
7998 BPF_XDP_CPUMAP),
7999 BPF_APROG_SEC("xdp", BPF_PROG_TYPE_XDP,
8000 BPF_XDP),
8001 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),
8002 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN),
8003 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT),
8004 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT),
8005 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL),
8006 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB,
8007 BPF_CGROUP_INET_INGRESS),
8008 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB,
8009 BPF_CGROUP_INET_EGRESS),
8010 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB),
8011 BPF_EAPROG_SEC("cgroup/sock_create", BPF_PROG_TYPE_CGROUP_SOCK,
8012 BPF_CGROUP_INET_SOCK_CREATE),
8013 BPF_EAPROG_SEC("cgroup/sock_release", BPF_PROG_TYPE_CGROUP_SOCK,
8014 BPF_CGROUP_INET_SOCK_RELEASE),
8015 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK,
8016 BPF_CGROUP_INET_SOCK_CREATE),
8017 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK,
8018 BPF_CGROUP_INET4_POST_BIND),
8019 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK,
8020 BPF_CGROUP_INET6_POST_BIND),
8021 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE,
8022 BPF_CGROUP_DEVICE),
8023 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS,
8024 BPF_CGROUP_SOCK_OPS),
8025 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB,
8026 BPF_SK_SKB_STREAM_PARSER),
8027 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB,
8028 BPF_SK_SKB_STREAM_VERDICT),
8029 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB),
8030 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG,
8031 BPF_SK_MSG_VERDICT),
8032 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2,
8033 BPF_LIRC_MODE2),
8034 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR,
8035 BPF_FLOW_DISSECTOR),
8036 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8037 BPF_CGROUP_INET4_BIND),
8038 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8039 BPF_CGROUP_INET6_BIND),
8040 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8041 BPF_CGROUP_INET4_CONNECT),
8042 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8043 BPF_CGROUP_INET6_CONNECT),
8044 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8045 BPF_CGROUP_UDP4_SENDMSG),
8046 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8047 BPF_CGROUP_UDP6_SENDMSG),
8048 BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8049 BPF_CGROUP_UDP4_RECVMSG),
8050 BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8051 BPF_CGROUP_UDP6_RECVMSG),
8052 BPF_EAPROG_SEC("cgroup/getpeername4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8053 BPF_CGROUP_INET4_GETPEERNAME),
8054 BPF_EAPROG_SEC("cgroup/getpeername6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8055 BPF_CGROUP_INET6_GETPEERNAME),
8056 BPF_EAPROG_SEC("cgroup/getsockname4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8057 BPF_CGROUP_INET4_GETSOCKNAME),
8058 BPF_EAPROG_SEC("cgroup/getsockname6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
8059 BPF_CGROUP_INET6_GETSOCKNAME),
8060 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL,
8061 BPF_CGROUP_SYSCTL),
8062 BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
8063 BPF_CGROUP_GETSOCKOPT),
8064 BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
8065 BPF_CGROUP_SETSOCKOPT),
8066 BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS),
8067 BPF_EAPROG_SEC("sk_lookup/", BPF_PROG_TYPE_SK_LOOKUP,
8068 BPF_SK_LOOKUP),
8069};
8070
8071#undef BPF_PROG_SEC_IMPL
8072#undef BPF_PROG_SEC
8073#undef BPF_APROG_SEC
8074#undef BPF_EAPROG_SEC
8075#undef BPF_APROG_COMPAT
8076#undef SEC_DEF
8077
8078#define MAX_TYPE_NAME_SIZE 32
8079
8080static const struct bpf_sec_def *find_sec_def(const char *sec_name)
8081{
8082 int i, n = ARRAY_SIZE(section_defs);
8083
8084 for (i = 0; i < n; i++) {
8085 if (strncmp(sec_name,
8086 section_defs[i].sec, section_defs[i].len))
8087 continue;
8088 return §ion_defs[i];
8089 }
8090 return NULL;
8091}
8092
8093static char *libbpf_get_type_names(bool attach_type)
8094{
8095 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE;
8096 char *buf;
8097
8098 buf = malloc(len);
8099 if (!buf)
8100 return NULL;
8101
8102 buf[0] = '\0';
8103
8104 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8105 if (attach_type && !section_defs[i].is_attachable)
8106 continue;
8107
8108 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) {
8109 free(buf);
8110 return NULL;
8111 }
8112 strcat(buf, " ");
8113 strcat(buf, section_defs[i].sec);
8114 }
8115
8116 return buf;
8117}
8118
8119int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
8120 enum bpf_attach_type *expected_attach_type)
8121{
8122 const struct bpf_sec_def *sec_def;
8123 char *type_names;
8124
8125 if (!name)
8126 return libbpf_err(-EINVAL);
8127
8128 sec_def = find_sec_def(name);
8129 if (sec_def) {
8130 *prog_type = sec_def->prog_type;
8131 *expected_attach_type = sec_def->expected_attach_type;
8132 return 0;
8133 }
8134
8135 pr_debug("failed to guess program type from ELF section '%s'\n", name);
8136 type_names = libbpf_get_type_names(false);
8137 if (type_names != NULL) {
8138 pr_debug("supported section(type) names are:%s\n", type_names);
8139 free(type_names);
8140 }
8141
8142 return libbpf_err(-ESRCH);
8143}
8144
8145static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
8146 size_t offset)
8147{
8148 struct bpf_map *map;
8149 size_t i;
8150
8151 for (i = 0; i < obj->nr_maps; i++) {
8152 map = &obj->maps[i];
8153 if (!bpf_map__is_struct_ops(map))
8154 continue;
8155 if (map->sec_offset <= offset &&
8156 offset - map->sec_offset < map->def.value_size)
8157 return map;
8158 }
8159
8160 return NULL;
8161}
8162
8163
8164static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
8165 GElf_Shdr *shdr, Elf_Data *data)
8166{
8167 const struct btf_member *member;
8168 struct bpf_struct_ops *st_ops;
8169 struct bpf_program *prog;
8170 unsigned int shdr_idx;
8171 const struct btf *btf;
8172 struct bpf_map *map;
8173 Elf_Data *symbols;
8174 unsigned int moff, insn_idx;
8175 const char *name;
8176 __u32 member_idx;
8177 GElf_Sym sym;
8178 GElf_Rel rel;
8179 int i, nrels;
8180
8181 symbols = obj->efile.symbols;
8182 btf = obj->btf;
8183 nrels = shdr->sh_size / shdr->sh_entsize;
8184 for (i = 0; i < nrels; i++) {
8185 if (!gelf_getrel(data, i, &rel)) {
8186 pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
8187 return -LIBBPF_ERRNO__FORMAT;
8188 }
8189
8190 if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
8191 pr_warn("struct_ops reloc: symbol %zx not found\n",
8192 (size_t)GELF_R_SYM(rel.r_info));
8193 return -LIBBPF_ERRNO__FORMAT;
8194 }
8195
8196 name = elf_sym_str(obj, sym.st_name) ?: "<?>";
8197 map = find_struct_ops_map_by_offset(obj, rel.r_offset);
8198 if (!map) {
8199 pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n",
8200 (size_t)rel.r_offset);
8201 return -EINVAL;
8202 }
8203
8204 moff = rel.r_offset - map->sec_offset;
8205 shdr_idx = sym.st_shndx;
8206 st_ops = map->st_ops;
8207 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
8208 map->name,
8209 (long long)(rel.r_info >> 32),
8210 (long long)sym.st_value,
8211 shdr_idx, (size_t)rel.r_offset,
8212 map->sec_offset, sym.st_name, name);
8213
8214 if (shdr_idx >= SHN_LORESERVE) {
8215 pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n",
8216 map->name, (size_t)rel.r_offset, shdr_idx);
8217 return -LIBBPF_ERRNO__RELOC;
8218 }
8219 if (sym.st_value % BPF_INSN_SZ) {
8220 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n",
8221 map->name, (unsigned long long)sym.st_value);
8222 return -LIBBPF_ERRNO__FORMAT;
8223 }
8224 insn_idx = sym.st_value / BPF_INSN_SZ;
8225
8226 member = find_member_by_offset(st_ops->type, moff * 8);
8227 if (!member) {
8228 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
8229 map->name, moff);
8230 return -EINVAL;
8231 }
8232 member_idx = member - btf_members(st_ops->type);
8233 name = btf__name_by_offset(btf, member->name_off);
8234
8235 if (!resolve_func_ptr(btf, member->type, NULL)) {
8236 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
8237 map->name, name);
8238 return -EINVAL;
8239 }
8240
8241 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
8242 if (!prog) {
8243 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
8244 map->name, shdr_idx, name);
8245 return -EINVAL;
8246 }
8247
8248 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
8249 const struct bpf_sec_def *sec_def;
8250
8251 sec_def = find_sec_def(prog->sec_name);
8252 if (sec_def &&
8253 sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) {
8254
8255 prog->type = sec_def->prog_type;
8256 goto invalid_prog;
8257 }
8258
8259 prog->type = BPF_PROG_TYPE_STRUCT_OPS;
8260 prog->attach_btf_id = st_ops->type_id;
8261 prog->expected_attach_type = member_idx;
8262 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
8263 prog->attach_btf_id != st_ops->type_id ||
8264 prog->expected_attach_type != member_idx) {
8265 goto invalid_prog;
8266 }
8267 st_ops->progs[member_idx] = prog;
8268 }
8269
8270 return 0;
8271
8272invalid_prog:
8273 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
8274 map->name, prog->name, prog->sec_name, prog->type,
8275 prog->attach_btf_id, prog->expected_attach_type, name);
8276 return -EINVAL;
8277}
8278
8279#define BTF_TRACE_PREFIX "btf_trace_"
8280#define BTF_LSM_PREFIX "bpf_lsm_"
8281#define BTF_ITER_PREFIX "bpf_iter_"
8282#define BTF_MAX_NAME_SIZE 128
8283
8284void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,
8285 const char **prefix, int *kind)
8286{
8287 switch (attach_type) {
8288 case BPF_TRACE_RAW_TP:
8289 *prefix = BTF_TRACE_PREFIX;
8290 *kind = BTF_KIND_TYPEDEF;
8291 break;
8292 case BPF_LSM_MAC:
8293 *prefix = BTF_LSM_PREFIX;
8294 *kind = BTF_KIND_FUNC;
8295 break;
8296 case BPF_TRACE_ITER:
8297 *prefix = BTF_ITER_PREFIX;
8298 *kind = BTF_KIND_FUNC;
8299 break;
8300 default:
8301 *prefix = "";
8302 *kind = BTF_KIND_FUNC;
8303 }
8304}
8305
8306static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
8307 const char *name, __u32 kind)
8308{
8309 char btf_type_name[BTF_MAX_NAME_SIZE];
8310 int ret;
8311
8312 ret = snprintf(btf_type_name, sizeof(btf_type_name),
8313 "%s%s", prefix, name);
8314
8315
8316
8317
8318 if (ret < 0 || ret >= sizeof(btf_type_name))
8319 return -ENAMETOOLONG;
8320 return btf__find_by_name_kind(btf, btf_type_name, kind);
8321}
8322
8323static inline int find_attach_btf_id(struct btf *btf, const char *name,
8324 enum bpf_attach_type attach_type)
8325{
8326 const char *prefix;
8327 int kind;
8328
8329 btf_get_kernel_prefix_kind(attach_type, &prefix, &kind);
8330 return find_btf_by_prefix_kind(btf, prefix, name, kind);
8331}
8332
8333int libbpf_find_vmlinux_btf_id(const char *name,
8334 enum bpf_attach_type attach_type)
8335{
8336 struct btf *btf;
8337 int err;
8338
8339 btf = btf__load_vmlinux_btf();
8340 err = libbpf_get_error(btf);
8341 if (err) {
8342 pr_warn("vmlinux BTF is not found\n");
8343 return libbpf_err(err);
8344 }
8345
8346 err = find_attach_btf_id(btf, name, attach_type);
8347 if (err <= 0)
8348 pr_warn("%s is not found in vmlinux BTF\n", name);
8349
8350 btf__free(btf);
8351 return libbpf_err(err);
8352}
8353
8354static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
8355{
8356 struct bpf_prog_info_linear *info_linear;
8357 struct bpf_prog_info *info;
8358 struct btf *btf;
8359 int err;
8360
8361 info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0);
8362 err = libbpf_get_error(info_linear);
8363 if (err) {
8364 pr_warn("failed get_prog_info_linear for FD %d\n",
8365 attach_prog_fd);
8366 return err;
8367 }
8368
8369 err = -EINVAL;
8370 info = &info_linear->info;
8371 if (!info->btf_id) {
8372 pr_warn("The target program doesn't have BTF\n");
8373 goto out;
8374 }
8375 btf = btf__load_from_kernel_by_id(info->btf_id);
8376 if (libbpf_get_error(btf)) {
8377 pr_warn("Failed to get BTF of the program\n");
8378 goto out;
8379 }
8380 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
8381 btf__free(btf);
8382 if (err <= 0) {
8383 pr_warn("%s is not found in prog's BTF\n", name);
8384 goto out;
8385 }
8386out:
8387 free(info_linear);
8388 return err;
8389}
8390
8391static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
8392 enum bpf_attach_type attach_type,
8393 int *btf_obj_fd, int *btf_type_id)
8394{
8395 int ret, i;
8396
8397 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
8398 if (ret > 0) {
8399 *btf_obj_fd = 0;
8400 *btf_type_id = ret;
8401 return 0;
8402 }
8403 if (ret != -ENOENT)
8404 return ret;
8405
8406 ret = load_module_btfs(obj);
8407 if (ret)
8408 return ret;
8409
8410 for (i = 0; i < obj->btf_module_cnt; i++) {
8411 const struct module_btf *mod = &obj->btf_modules[i];
8412
8413 ret = find_attach_btf_id(mod->btf, attach_name, attach_type);
8414 if (ret > 0) {
8415 *btf_obj_fd = mod->fd;
8416 *btf_type_id = ret;
8417 return 0;
8418 }
8419 if (ret == -ENOENT)
8420 continue;
8421
8422 return ret;
8423 }
8424
8425 return -ESRCH;
8426}
8427
8428static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, int *btf_type_id)
8429{
8430 enum bpf_attach_type attach_type = prog->expected_attach_type;
8431 __u32 attach_prog_fd = prog->attach_prog_fd;
8432 const char *name = prog->sec_name, *attach_name;
8433 const struct bpf_sec_def *sec = NULL;
8434 int i, err = 0;
8435
8436 if (!name)
8437 return -EINVAL;
8438
8439 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8440 if (!section_defs[i].is_attach_btf)
8441 continue;
8442 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
8443 continue;
8444
8445 sec = §ion_defs[i];
8446 break;
8447 }
8448
8449 if (!sec) {
8450 pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", name);
8451 return -ESRCH;
8452 }
8453 attach_name = name + sec->len;
8454
8455
8456 if (attach_prog_fd) {
8457 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
8458 if (err < 0) {
8459 pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
8460 attach_prog_fd, attach_name, err);
8461 return err;
8462 }
8463 *btf_obj_fd = 0;
8464 *btf_type_id = err;
8465 return 0;
8466 }
8467
8468
8469 if (prog->obj->gen_loader) {
8470 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
8471 *btf_obj_fd = 0;
8472 *btf_type_id = 1;
8473 } else {
8474 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
8475 }
8476 if (err) {
8477 pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err);
8478 return err;
8479 }
8480 return 0;
8481}
8482
8483int libbpf_attach_type_by_name(const char *name,
8484 enum bpf_attach_type *attach_type)
8485{
8486 char *type_names;
8487 int i;
8488
8489 if (!name)
8490 return libbpf_err(-EINVAL);
8491
8492 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8493 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
8494 continue;
8495 if (!section_defs[i].is_attachable)
8496 return libbpf_err(-EINVAL);
8497 *attach_type = section_defs[i].expected_attach_type;
8498 return 0;
8499 }
8500 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
8501 type_names = libbpf_get_type_names(true);
8502 if (type_names != NULL) {
8503 pr_debug("attachable section(type) names are:%s\n", type_names);
8504 free(type_names);
8505 }
8506
8507 return libbpf_err(-EINVAL);
8508}
8509
8510int bpf_map__fd(const struct bpf_map *map)
8511{
8512 return map ? map->fd : libbpf_err(-EINVAL);
8513}
8514
8515const struct bpf_map_def *bpf_map__def(const struct bpf_map *map)
8516{
8517 return map ? &map->def : libbpf_err_ptr(-EINVAL);
8518}
8519
8520const char *bpf_map__name(const struct bpf_map *map)
8521{
8522 return map ? map->name : NULL;
8523}
8524
8525enum bpf_map_type bpf_map__type(const struct bpf_map *map)
8526{
8527 return map->def.type;
8528}
8529
8530int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type)
8531{
8532 if (map->fd >= 0)
8533 return libbpf_err(-EBUSY);
8534 map->def.type = type;
8535 return 0;
8536}
8537
8538__u32 bpf_map__map_flags(const struct bpf_map *map)
8539{
8540 return map->def.map_flags;
8541}
8542
8543int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags)
8544{
8545 if (map->fd >= 0)
8546 return libbpf_err(-EBUSY);
8547 map->def.map_flags = flags;
8548 return 0;
8549}
8550
8551__u32 bpf_map__numa_node(const struct bpf_map *map)
8552{
8553 return map->numa_node;
8554}
8555
8556int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node)
8557{
8558 if (map->fd >= 0)
8559 return libbpf_err(-EBUSY);
8560 map->numa_node = numa_node;
8561 return 0;
8562}
8563
8564__u32 bpf_map__key_size(const struct bpf_map *map)
8565{
8566 return map->def.key_size;
8567}
8568
8569int bpf_map__set_key_size(struct bpf_map *map, __u32 size)
8570{
8571 if (map->fd >= 0)
8572 return libbpf_err(-EBUSY);
8573 map->def.key_size = size;
8574 return 0;
8575}
8576
8577__u32 bpf_map__value_size(const struct bpf_map *map)
8578{
8579 return map->def.value_size;
8580}
8581
8582int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
8583{
8584 if (map->fd >= 0)
8585 return libbpf_err(-EBUSY);
8586 map->def.value_size = size;
8587 return 0;
8588}
8589
8590__u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
8591{
8592 return map ? map->btf_key_type_id : 0;
8593}
8594
8595__u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
8596{
8597 return map ? map->btf_value_type_id : 0;
8598}
8599
8600int bpf_map__set_priv(struct bpf_map *map, void *priv,
8601 bpf_map_clear_priv_t clear_priv)
8602{
8603 if (!map)
8604 return libbpf_err(-EINVAL);
8605
8606 if (map->priv) {
8607 if (map->clear_priv)
8608 map->clear_priv(map, map->priv);
8609 }
8610
8611 map->priv = priv;
8612 map->clear_priv = clear_priv;
8613 return 0;
8614}
8615
8616void *bpf_map__priv(const struct bpf_map *map)
8617{
8618 return map ? map->priv : libbpf_err_ptr(-EINVAL);
8619}
8620
8621int bpf_map__set_initial_value(struct bpf_map *map,
8622 const void *data, size_t size)
8623{
8624 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG ||
8625 size != map->def.value_size || map->fd >= 0)
8626 return libbpf_err(-EINVAL);
8627
8628 memcpy(map->mmaped, data, size);
8629 return 0;
8630}
8631
8632const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize)
8633{
8634 if (!map->mmaped)
8635 return NULL;
8636 *psize = map->def.value_size;
8637 return map->mmaped;
8638}
8639
8640bool bpf_map__is_offload_neutral(const struct bpf_map *map)
8641{
8642 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
8643}
8644
8645bool bpf_map__is_internal(const struct bpf_map *map)
8646{
8647 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
8648}
8649
8650__u32 bpf_map__ifindex(const struct bpf_map *map)
8651{
8652 return map->map_ifindex;
8653}
8654
8655int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
8656{
8657 if (map->fd >= 0)
8658 return libbpf_err(-EBUSY);
8659 map->map_ifindex = ifindex;
8660 return 0;
8661}
8662
8663int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
8664{
8665 if (!bpf_map_type__is_map_in_map(map->def.type)) {
8666 pr_warn("error: unsupported map type\n");
8667 return libbpf_err(-EINVAL);
8668 }
8669 if (map->inner_map_fd != -1) {
8670 pr_warn("error: inner_map_fd already specified\n");
8671 return libbpf_err(-EINVAL);
8672 }
8673 zfree(&map->inner_map);
8674 map->inner_map_fd = fd;
8675 return 0;
8676}
8677
8678static struct bpf_map *
8679__bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
8680{
8681 ssize_t idx;
8682 struct bpf_map *s, *e;
8683
8684 if (!obj || !obj->maps)
8685 return errno = EINVAL, NULL;
8686
8687 s = obj->maps;
8688 e = obj->maps + obj->nr_maps;
8689
8690 if ((m < s) || (m >= e)) {
8691 pr_warn("error in %s: map handler doesn't belong to object\n",
8692 __func__);
8693 return errno = EINVAL, NULL;
8694 }
8695
8696 idx = (m - obj->maps) + i;
8697 if (idx >= obj->nr_maps || idx < 0)
8698 return NULL;
8699 return &obj->maps[idx];
8700}
8701
8702struct bpf_map *
8703bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
8704{
8705 if (prev == NULL)
8706 return obj->maps;
8707
8708 return __bpf_map__iter(prev, obj, 1);
8709}
8710
8711struct bpf_map *
8712bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj)
8713{
8714 if (next == NULL) {
8715 if (!obj->nr_maps)
8716 return NULL;
8717 return obj->maps + obj->nr_maps - 1;
8718 }
8719
8720 return __bpf_map__iter(next, obj, -1);
8721}
8722
8723struct bpf_map *
8724bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
8725{
8726 struct bpf_map *pos;
8727
8728 bpf_object__for_each_map(pos, obj) {
8729 if (pos->name && !strcmp(pos->name, name))
8730 return pos;
8731 }
8732 return errno = ENOENT, NULL;
8733}
8734
8735int
8736bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
8737{
8738 return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
8739}
8740
8741struct bpf_map *
8742bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
8743{
8744 return libbpf_err_ptr(-ENOTSUP);
8745}
8746
8747long libbpf_get_error(const void *ptr)
8748{
8749 if (!IS_ERR_OR_NULL(ptr))
8750 return 0;
8751
8752 if (IS_ERR(ptr))
8753 errno = -PTR_ERR(ptr);
8754
8755
8756
8757
8758
8759
8760 return -errno;
8761}
8762
8763int bpf_prog_load(const char *file, enum bpf_prog_type type,
8764 struct bpf_object **pobj, int *prog_fd)
8765{
8766 struct bpf_prog_load_attr attr;
8767
8768 memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
8769 attr.file = file;
8770 attr.prog_type = type;
8771 attr.expected_attach_type = 0;
8772
8773 return bpf_prog_load_xattr(&attr, pobj, prog_fd);
8774}
8775
8776int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
8777 struct bpf_object **pobj, int *prog_fd)
8778{
8779 struct bpf_object_open_attr open_attr = {};
8780 struct bpf_program *prog, *first_prog = NULL;
8781 struct bpf_object *obj;
8782 struct bpf_map *map;
8783 int err;
8784
8785 if (!attr)
8786 return libbpf_err(-EINVAL);
8787 if (!attr->file)
8788 return libbpf_err(-EINVAL);
8789
8790 open_attr.file = attr->file;
8791 open_attr.prog_type = attr->prog_type;
8792
8793 obj = bpf_object__open_xattr(&open_attr);
8794 err = libbpf_get_error(obj);
8795 if (err)
8796 return libbpf_err(-ENOENT);
8797
8798 bpf_object__for_each_program(prog, obj) {
8799 enum bpf_attach_type attach_type = attr->expected_attach_type;
8800
8801
8802
8803
8804
8805 if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) {
8806 bpf_program__set_type(prog, attr->prog_type);
8807 bpf_program__set_expected_attach_type(prog,
8808 attach_type);
8809 }
8810 if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) {
8811
8812
8813
8814
8815 bpf_object__close(obj);
8816 return libbpf_err(-EINVAL);
8817 }
8818
8819 prog->prog_ifindex = attr->ifindex;
8820 prog->log_level = attr->log_level;
8821 prog->prog_flags |= attr->prog_flags;
8822 if (!first_prog)
8823 first_prog = prog;
8824 }
8825
8826 bpf_object__for_each_map(map, obj) {
8827 if (!bpf_map__is_offload_neutral(map))
8828 map->map_ifindex = attr->ifindex;
8829 }
8830
8831 if (!first_prog) {
8832 pr_warn("object file doesn't contain bpf program\n");
8833 bpf_object__close(obj);
8834 return libbpf_err(-ENOENT);
8835 }
8836
8837 err = bpf_object__load(obj);
8838 if (err) {
8839 bpf_object__close(obj);
8840 return libbpf_err(err);
8841 }
8842
8843 *pobj = obj;
8844 *prog_fd = bpf_program__fd(first_prog);
8845 return 0;
8846}
8847
8848struct bpf_link {
8849 int (*detach)(struct bpf_link *link);
8850 void (*dealloc)(struct bpf_link *link);
8851 char *pin_path;
8852 int fd;
8853 bool disconnected;
8854};
8855
8856
8857int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
8858{
8859 int ret;
8860
8861 ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
8862 return libbpf_err_errno(ret);
8863}
8864
8865
8866
8867
8868
8869
8870
8871
8872
8873
8874
8875void bpf_link__disconnect(struct bpf_link *link)
8876{
8877 link->disconnected = true;
8878}
8879
8880int bpf_link__destroy(struct bpf_link *link)
8881{
8882 int err = 0;
8883
8884 if (IS_ERR_OR_NULL(link))
8885 return 0;
8886
8887 if (!link->disconnected && link->detach)
8888 err = link->detach(link);
8889 if (link->pin_path)
8890 free(link->pin_path);
8891 if (link->dealloc)
8892 link->dealloc(link);
8893 else
8894 free(link);
8895
8896 return libbpf_err(err);
8897}
8898
8899int bpf_link__fd(const struct bpf_link *link)
8900{
8901 return link->fd;
8902}
8903
8904const char *bpf_link__pin_path(const struct bpf_link *link)
8905{
8906 return link->pin_path;
8907}
8908
8909static int bpf_link__detach_fd(struct bpf_link *link)
8910{
8911 return libbpf_err_errno(close(link->fd));
8912}
8913
8914struct bpf_link *bpf_link__open(const char *path)
8915{
8916 struct bpf_link *link;
8917 int fd;
8918
8919 fd = bpf_obj_get(path);
8920 if (fd < 0) {
8921 fd = -errno;
8922 pr_warn("failed to open link at %s: %d\n", path, fd);
8923 return libbpf_err_ptr(fd);
8924 }
8925
8926 link = calloc(1, sizeof(*link));
8927 if (!link) {
8928 close(fd);
8929 return libbpf_err_ptr(-ENOMEM);
8930 }
8931 link->detach = &bpf_link__detach_fd;
8932 link->fd = fd;
8933
8934 link->pin_path = strdup(path);
8935 if (!link->pin_path) {
8936 bpf_link__destroy(link);
8937 return libbpf_err_ptr(-ENOMEM);
8938 }
8939
8940 return link;
8941}
8942
8943int bpf_link__detach(struct bpf_link *link)
8944{
8945 return bpf_link_detach(link->fd) ? -errno : 0;
8946}
8947
8948int bpf_link__pin(struct bpf_link *link, const char *path)
8949{
8950 int err;
8951
8952 if (link->pin_path)
8953 return libbpf_err(-EBUSY);
8954 err = make_parent_dir(path);
8955 if (err)
8956 return libbpf_err(err);
8957 err = check_path(path);
8958 if (err)
8959 return libbpf_err(err);
8960
8961 link->pin_path = strdup(path);
8962 if (!link->pin_path)
8963 return libbpf_err(-ENOMEM);
8964
8965 if (bpf_obj_pin(link->fd, link->pin_path)) {
8966 err = -errno;
8967 zfree(&link->pin_path);
8968 return libbpf_err(err);
8969 }
8970
8971 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);
8972 return 0;
8973}
8974
8975int bpf_link__unpin(struct bpf_link *link)
8976{
8977 int err;
8978
8979 if (!link->pin_path)
8980 return libbpf_err(-EINVAL);
8981
8982 err = unlink(link->pin_path);
8983 if (err != 0)
8984 return -errno;
8985
8986 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);
8987 zfree(&link->pin_path);
8988 return 0;
8989}
8990
8991struct bpf_link_perf {
8992 struct bpf_link link;
8993 int perf_event_fd;
8994};
8995
8996static int bpf_link_perf_detach(struct bpf_link *link)
8997{
8998 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
8999 int err = 0;
9000
9001 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
9002 err = -errno;
9003
9004 if (perf_link->perf_event_fd != link->fd)
9005 close(perf_link->perf_event_fd);
9006 close(link->fd);
9007
9008 return libbpf_err(err);
9009}
9010
9011static void bpf_link_perf_dealloc(struct bpf_link *link)
9012{
9013 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
9014
9015 free(perf_link);
9016}
9017
9018struct bpf_link *bpf_program__attach_perf_event_opts(struct bpf_program *prog, int pfd,
9019 const struct bpf_perf_event_opts *opts)
9020{
9021 char errmsg[STRERR_BUFSIZE];
9022 struct bpf_link_perf *link;
9023 int prog_fd, link_fd = -1, err;
9024
9025 if (!OPTS_VALID(opts, bpf_perf_event_opts))
9026 return libbpf_err_ptr(-EINVAL);
9027
9028 if (pfd < 0) {
9029 pr_warn("prog '%s': invalid perf event FD %d\n",
9030 prog->name, pfd);
9031 return libbpf_err_ptr(-EINVAL);
9032 }
9033 prog_fd = bpf_program__fd(prog);
9034 if (prog_fd < 0) {
9035 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
9036 prog->name);
9037 return libbpf_err_ptr(-EINVAL);
9038 }
9039
9040 link = calloc(1, sizeof(*link));
9041 if (!link)
9042 return libbpf_err_ptr(-ENOMEM);
9043 link->link.detach = &bpf_link_perf_detach;
9044 link->link.dealloc = &bpf_link_perf_dealloc;
9045 link->perf_event_fd = pfd;
9046
9047 if (kernel_supports(prog->obj, FEAT_PERF_LINK)) {
9048 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts,
9049 .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0));
9050
9051 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts);
9052 if (link_fd < 0) {
9053 err = -errno;
9054 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n",
9055 prog->name, pfd,
9056 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9057 goto err_out;
9058 }
9059 link->link.fd = link_fd;
9060 } else {
9061 if (OPTS_GET(opts, bpf_cookie, 0)) {
9062 pr_warn("prog '%s': user context value is not supported\n", prog->name);
9063 err = -EOPNOTSUPP;
9064 goto err_out;
9065 }
9066
9067 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
9068 err = -errno;
9069 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
9070 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9071 if (err == -EPROTO)
9072 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
9073 prog->name, pfd);
9074 goto err_out;
9075 }
9076 link->link.fd = pfd;
9077 }
9078 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
9079 err = -errno;
9080 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
9081 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9082 goto err_out;
9083 }
9084
9085 return &link->link;
9086err_out:
9087 if (link_fd >= 0)
9088 close(link_fd);
9089 free(link);
9090 return libbpf_err_ptr(err);
9091}
9092
9093struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, int pfd)
9094{
9095 return bpf_program__attach_perf_event_opts(prog, pfd, NULL);
9096}
9097
9098
9099
9100
9101
9102
9103static int parse_uint_from_file(const char *file, const char *fmt)
9104{
9105 char buf[STRERR_BUFSIZE];
9106 int err, ret;
9107 FILE *f;
9108
9109 f = fopen(file, "r");
9110 if (!f) {
9111 err = -errno;
9112 pr_debug("failed to open '%s': %s\n", file,
9113 libbpf_strerror_r(err, buf, sizeof(buf)));
9114 return err;
9115 }
9116 err = fscanf(f, fmt, &ret);
9117 if (err != 1) {
9118 err = err == EOF ? -EIO : -errno;
9119 pr_debug("failed to parse '%s': %s\n", file,
9120 libbpf_strerror_r(err, buf, sizeof(buf)));
9121 fclose(f);
9122 return err;
9123 }
9124 fclose(f);
9125 return ret;
9126}
9127
9128static int determine_kprobe_perf_type(void)
9129{
9130 const char *file = "/sys/bus/event_source/devices/kprobe/type";
9131
9132 return parse_uint_from_file(file, "%d\n");
9133}
9134
9135static int determine_uprobe_perf_type(void)
9136{
9137 const char *file = "/sys/bus/event_source/devices/uprobe/type";
9138
9139 return parse_uint_from_file(file, "%d\n");
9140}
9141
9142static int determine_kprobe_retprobe_bit(void)
9143{
9144 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe";
9145
9146 return parse_uint_from_file(file, "config:%d\n");
9147}
9148
9149static int determine_uprobe_retprobe_bit(void)
9150{
9151 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe";
9152
9153 return parse_uint_from_file(file, "config:%d\n");
9154}
9155
9156#define PERF_UPROBE_REF_CTR_OFFSET_BITS 32
9157#define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32
9158
9159static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
9160 uint64_t offset, int pid, size_t ref_ctr_off)
9161{
9162 struct perf_event_attr attr = {};
9163 char errmsg[STRERR_BUFSIZE];
9164 int type, pfd, err;
9165
9166 if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))
9167 return -EINVAL;
9168
9169 type = uprobe ? determine_uprobe_perf_type()
9170 : determine_kprobe_perf_type();
9171 if (type < 0) {
9172 pr_warn("failed to determine %s perf type: %s\n",
9173 uprobe ? "uprobe" : "kprobe",
9174 libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
9175 return type;
9176 }
9177 if (retprobe) {
9178 int bit = uprobe ? determine_uprobe_retprobe_bit()
9179 : determine_kprobe_retprobe_bit();
9180
9181 if (bit < 0) {
9182 pr_warn("failed to determine %s retprobe bit: %s\n",
9183 uprobe ? "uprobe" : "kprobe",
9184 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
9185 return bit;
9186 }
9187 attr.config |= 1 << bit;
9188 }
9189 attr.size = sizeof(attr);
9190 attr.type = type;
9191 attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
9192 attr.config1 = ptr_to_u64(name);
9193 attr.config2 = offset;
9194
9195
9196 pfd = syscall(__NR_perf_event_open, &attr,
9197 pid < 0 ? -1 : pid ,
9198 pid == -1 ? 0 : -1 ,
9199 -1 , PERF_FLAG_FD_CLOEXEC);
9200 if (pfd < 0) {
9201 err = -errno;
9202 pr_warn("%s perf_event_open() failed: %s\n",
9203 uprobe ? "uprobe" : "kprobe",
9204 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9205 return err;
9206 }
9207 return pfd;
9208}
9209
9210struct bpf_link *
9211bpf_program__attach_kprobe_opts(struct bpf_program *prog,
9212 const char *func_name,
9213 const struct bpf_kprobe_opts *opts)
9214{
9215 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
9216 char errmsg[STRERR_BUFSIZE];
9217 struct bpf_link *link;
9218 unsigned long offset;
9219 bool retprobe;
9220 int pfd, err;
9221
9222 if (!OPTS_VALID(opts, bpf_kprobe_opts))
9223 return libbpf_err_ptr(-EINVAL);
9224
9225 retprobe = OPTS_GET(opts, retprobe, false);
9226 offset = OPTS_GET(opts, offset, 0);
9227 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
9228
9229 pfd = perf_event_open_probe(false , retprobe, func_name,
9230 offset, -1 , 0 );
9231 if (pfd < 0) {
9232 pr_warn("prog '%s': failed to create %s '%s' perf event: %s\n",
9233 prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
9234 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9235 return libbpf_err_ptr(pfd);
9236 }
9237 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
9238 err = libbpf_get_error(link);
9239 if (err) {
9240 close(pfd);
9241 pr_warn("prog '%s': failed to attach to %s '%s': %s\n",
9242 prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
9243 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9244 return libbpf_err_ptr(err);
9245 }
9246 return link;
9247}
9248
9249struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog,
9250 bool retprobe,
9251 const char *func_name)
9252{
9253 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts,
9254 .retprobe = retprobe,
9255 );
9256
9257 return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
9258}
9259
9260static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,
9261 struct bpf_program *prog)
9262{
9263 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts);
9264 unsigned long offset = 0;
9265 struct bpf_link *link;
9266 const char *func_name;
9267 char *func;
9268 int n, err;
9269
9270 func_name = prog->sec_name + sec->len;
9271 opts.retprobe = strcmp(sec->sec, "kretprobe/") == 0;
9272
9273 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
9274 if (n < 1) {
9275 err = -EINVAL;
9276 pr_warn("kprobe name is invalid: %s\n", func_name);
9277 return libbpf_err_ptr(err);
9278 }
9279 if (opts.retprobe && offset != 0) {
9280 free(func);
9281 err = -EINVAL;
9282 pr_warn("kretprobes do not support offset specification\n");
9283 return libbpf_err_ptr(err);
9284 }
9285
9286 opts.offset = offset;
9287 link = bpf_program__attach_kprobe_opts(prog, func, &opts);
9288 free(func);
9289 return link;
9290}
9291
9292LIBBPF_API struct bpf_link *
9293bpf_program__attach_uprobe_opts(struct bpf_program *prog, pid_t pid,
9294 const char *binary_path, size_t func_offset,
9295 const struct bpf_uprobe_opts *opts)
9296{
9297 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
9298 char errmsg[STRERR_BUFSIZE];
9299 struct bpf_link *link;
9300 size_t ref_ctr_off;
9301 int pfd, err;
9302 bool retprobe;
9303
9304 if (!OPTS_VALID(opts, bpf_uprobe_opts))
9305 return libbpf_err_ptr(-EINVAL);
9306
9307 retprobe = OPTS_GET(opts, retprobe, false);
9308 ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0);
9309 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
9310
9311 pfd = perf_event_open_probe(true , retprobe, binary_path,
9312 func_offset, pid, ref_ctr_off);
9313 if (pfd < 0) {
9314 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
9315 prog->name, retprobe ? "uretprobe" : "uprobe",
9316 binary_path, func_offset,
9317 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9318 return libbpf_err_ptr(pfd);
9319 }
9320 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
9321 err = libbpf_get_error(link);
9322 if (err) {
9323 close(pfd);
9324 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
9325 prog->name, retprobe ? "uretprobe" : "uprobe",
9326 binary_path, func_offset,
9327 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9328 return libbpf_err_ptr(err);
9329 }
9330 return link;
9331}
9332
9333struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog,
9334 bool retprobe, pid_t pid,
9335 const char *binary_path,
9336 size_t func_offset)
9337{
9338 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe);
9339
9340 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts);
9341}
9342
9343static int determine_tracepoint_id(const char *tp_category,
9344 const char *tp_name)
9345{
9346 char file[PATH_MAX];
9347 int ret;
9348
9349 ret = snprintf(file, sizeof(file),
9350 "/sys/kernel/debug/tracing/events/%s/%s/id",
9351 tp_category, tp_name);
9352 if (ret < 0)
9353 return -errno;
9354 if (ret >= sizeof(file)) {
9355 pr_debug("tracepoint %s/%s path is too long\n",
9356 tp_category, tp_name);
9357 return -E2BIG;
9358 }
9359 return parse_uint_from_file(file, "%d\n");
9360}
9361
9362static int perf_event_open_tracepoint(const char *tp_category,
9363 const char *tp_name)
9364{
9365 struct perf_event_attr attr = {};
9366 char errmsg[STRERR_BUFSIZE];
9367 int tp_id, pfd, err;
9368
9369 tp_id = determine_tracepoint_id(tp_category, tp_name);
9370 if (tp_id < 0) {
9371 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
9372 tp_category, tp_name,
9373 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
9374 return tp_id;
9375 }
9376
9377 attr.type = PERF_TYPE_TRACEPOINT;
9378 attr.size = sizeof(attr);
9379 attr.config = tp_id;
9380
9381 pfd = syscall(__NR_perf_event_open, &attr, -1 , 0 ,
9382 -1 , PERF_FLAG_FD_CLOEXEC);
9383 if (pfd < 0) {
9384 err = -errno;
9385 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
9386 tp_category, tp_name,
9387 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9388 return err;
9389 }
9390 return pfd;
9391}
9392
9393struct bpf_link *bpf_program__attach_tracepoint_opts(struct bpf_program *prog,
9394 const char *tp_category,
9395 const char *tp_name,
9396 const struct bpf_tracepoint_opts *opts)
9397{
9398 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
9399 char errmsg[STRERR_BUFSIZE];
9400 struct bpf_link *link;
9401 int pfd, err;
9402
9403 if (!OPTS_VALID(opts, bpf_tracepoint_opts))
9404 return libbpf_err_ptr(-EINVAL);
9405
9406 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
9407
9408 pfd = perf_event_open_tracepoint(tp_category, tp_name);
9409 if (pfd < 0) {
9410 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
9411 prog->name, tp_category, tp_name,
9412 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9413 return libbpf_err_ptr(pfd);
9414 }
9415 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
9416 err = libbpf_get_error(link);
9417 if (err) {
9418 close(pfd);
9419 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
9420 prog->name, tp_category, tp_name,
9421 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9422 return libbpf_err_ptr(err);
9423 }
9424 return link;
9425}
9426
9427struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,
9428 const char *tp_category,
9429 const char *tp_name)
9430{
9431 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);
9432}
9433
9434static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,
9435 struct bpf_program *prog)
9436{
9437 char *sec_name, *tp_cat, *tp_name;
9438 struct bpf_link *link;
9439
9440 sec_name = strdup(prog->sec_name);
9441 if (!sec_name)
9442 return libbpf_err_ptr(-ENOMEM);
9443
9444
9445 tp_cat = sec_name + sec->len;
9446 tp_name = strchr(tp_cat, '/');
9447 if (!tp_name) {
9448 free(sec_name);
9449 return libbpf_err_ptr(-EINVAL);
9450 }
9451 *tp_name = '\0';
9452 tp_name++;
9453
9454 link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
9455 free(sec_name);
9456 return link;
9457}
9458
9459struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,
9460 const char *tp_name)
9461{
9462 char errmsg[STRERR_BUFSIZE];
9463 struct bpf_link *link;
9464 int prog_fd, pfd;
9465
9466 prog_fd = bpf_program__fd(prog);
9467 if (prog_fd < 0) {
9468 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9469 return libbpf_err_ptr(-EINVAL);
9470 }
9471
9472 link = calloc(1, sizeof(*link));
9473 if (!link)
9474 return libbpf_err_ptr(-ENOMEM);
9475 link->detach = &bpf_link__detach_fd;
9476
9477 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
9478 if (pfd < 0) {
9479 pfd = -errno;
9480 free(link);
9481 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
9482 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9483 return libbpf_err_ptr(pfd);
9484 }
9485 link->fd = pfd;
9486 return link;
9487}
9488
9489static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,
9490 struct bpf_program *prog)
9491{
9492 const char *tp_name = prog->sec_name + sec->len;
9493
9494 return bpf_program__attach_raw_tracepoint(prog, tp_name);
9495}
9496
9497
9498static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog)
9499{
9500 char errmsg[STRERR_BUFSIZE];
9501 struct bpf_link *link;
9502 int prog_fd, pfd;
9503
9504 prog_fd = bpf_program__fd(prog);
9505 if (prog_fd < 0) {
9506 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9507 return libbpf_err_ptr(-EINVAL);
9508 }
9509
9510 link = calloc(1, sizeof(*link));
9511 if (!link)
9512 return libbpf_err_ptr(-ENOMEM);
9513 link->detach = &bpf_link__detach_fd;
9514
9515 pfd = bpf_raw_tracepoint_open(NULL, prog_fd);
9516 if (pfd < 0) {
9517 pfd = -errno;
9518 free(link);
9519 pr_warn("prog '%s': failed to attach: %s\n",
9520 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
9521 return libbpf_err_ptr(pfd);
9522 }
9523 link->fd = pfd;
9524 return (struct bpf_link *)link;
9525}
9526
9527struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog)
9528{
9529 return bpf_program__attach_btf_id(prog);
9530}
9531
9532struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog)
9533{
9534 return bpf_program__attach_btf_id(prog);
9535}
9536
9537static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
9538 struct bpf_program *prog)
9539{
9540 return bpf_program__attach_trace(prog);
9541}
9542
9543static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
9544 struct bpf_program *prog)
9545{
9546 return bpf_program__attach_lsm(prog);
9547}
9548
9549static struct bpf_link *
9550bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id,
9551 const char *target_name)
9552{
9553 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
9554 .target_btf_id = btf_id);
9555 enum bpf_attach_type attach_type;
9556 char errmsg[STRERR_BUFSIZE];
9557 struct bpf_link *link;
9558 int prog_fd, link_fd;
9559
9560 prog_fd = bpf_program__fd(prog);
9561 if (prog_fd < 0) {
9562 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9563 return libbpf_err_ptr(-EINVAL);
9564 }
9565
9566 link = calloc(1, sizeof(*link));
9567 if (!link)
9568 return libbpf_err_ptr(-ENOMEM);
9569 link->detach = &bpf_link__detach_fd;
9570
9571 attach_type = bpf_program__get_expected_attach_type(prog);
9572 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
9573 if (link_fd < 0) {
9574 link_fd = -errno;
9575 free(link);
9576 pr_warn("prog '%s': failed to attach to %s: %s\n",
9577 prog->name, target_name,
9578 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
9579 return libbpf_err_ptr(link_fd);
9580 }
9581 link->fd = link_fd;
9582 return link;
9583}
9584
9585struct bpf_link *
9586bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd)
9587{
9588 return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
9589}
9590
9591struct bpf_link *
9592bpf_program__attach_netns(struct bpf_program *prog, int netns_fd)
9593{
9594 return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
9595}
9596
9597struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex)
9598{
9599
9600 return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
9601}
9602
9603struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog,
9604 int target_fd,
9605 const char *attach_func_name)
9606{
9607 int btf_id;
9608
9609 if (!!target_fd != !!attach_func_name) {
9610 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
9611 prog->name);
9612 return libbpf_err_ptr(-EINVAL);
9613 }
9614
9615 if (prog->type != BPF_PROG_TYPE_EXT) {
9616 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
9617 prog->name);
9618 return libbpf_err_ptr(-EINVAL);
9619 }
9620
9621 if (target_fd) {
9622 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
9623 if (btf_id < 0)
9624 return libbpf_err_ptr(btf_id);
9625
9626 return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
9627 } else {
9628
9629
9630
9631 return bpf_program__attach_trace(prog);
9632 }
9633}
9634
9635struct bpf_link *
9636bpf_program__attach_iter(struct bpf_program *prog,
9637 const struct bpf_iter_attach_opts *opts)
9638{
9639 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
9640 char errmsg[STRERR_BUFSIZE];
9641 struct bpf_link *link;
9642 int prog_fd, link_fd;
9643 __u32 target_fd = 0;
9644
9645 if (!OPTS_VALID(opts, bpf_iter_attach_opts))
9646 return libbpf_err_ptr(-EINVAL);
9647
9648 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0);
9649 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0);
9650
9651 prog_fd = bpf_program__fd(prog);
9652 if (prog_fd < 0) {
9653 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
9654 return libbpf_err_ptr(-EINVAL);
9655 }
9656
9657 link = calloc(1, sizeof(*link));
9658 if (!link)
9659 return libbpf_err_ptr(-ENOMEM);
9660 link->detach = &bpf_link__detach_fd;
9661
9662 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER,
9663 &link_create_opts);
9664 if (link_fd < 0) {
9665 link_fd = -errno;
9666 free(link);
9667 pr_warn("prog '%s': failed to attach to iterator: %s\n",
9668 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
9669 return libbpf_err_ptr(link_fd);
9670 }
9671 link->fd = link_fd;
9672 return link;
9673}
9674
9675static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
9676 struct bpf_program *prog)
9677{
9678 return bpf_program__attach_iter(prog, NULL);
9679}
9680
9681struct bpf_link *bpf_program__attach(struct bpf_program *prog)
9682{
9683 const struct bpf_sec_def *sec_def;
9684
9685 sec_def = find_sec_def(prog->sec_name);
9686 if (!sec_def || !sec_def->attach_fn)
9687 return libbpf_err_ptr(-ESRCH);
9688
9689 return sec_def->attach_fn(sec_def, prog);
9690}
9691
9692static int bpf_link__detach_struct_ops(struct bpf_link *link)
9693{
9694 __u32 zero = 0;
9695
9696 if (bpf_map_delete_elem(link->fd, &zero))
9697 return -errno;
9698
9699 return 0;
9700}
9701
9702struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map)
9703{
9704 struct bpf_struct_ops *st_ops;
9705 struct bpf_link *link;
9706 __u32 i, zero = 0;
9707 int err;
9708
9709 if (!bpf_map__is_struct_ops(map) || map->fd == -1)
9710 return libbpf_err_ptr(-EINVAL);
9711
9712 link = calloc(1, sizeof(*link));
9713 if (!link)
9714 return libbpf_err_ptr(-EINVAL);
9715
9716 st_ops = map->st_ops;
9717 for (i = 0; i < btf_vlen(st_ops->type); i++) {
9718 struct bpf_program *prog = st_ops->progs[i];
9719 void *kern_data;
9720 int prog_fd;
9721
9722 if (!prog)
9723 continue;
9724
9725 prog_fd = bpf_program__fd(prog);
9726 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
9727 *(unsigned long *)kern_data = prog_fd;
9728 }
9729
9730 err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
9731 if (err) {
9732 err = -errno;
9733 free(link);
9734 return libbpf_err_ptr(err);
9735 }
9736
9737 link->detach = bpf_link__detach_struct_ops;
9738 link->fd = map->fd;
9739
9740 return link;
9741}
9742
9743enum bpf_perf_event_ret
9744bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
9745 void **copy_mem, size_t *copy_size,
9746 bpf_perf_event_print_t fn, void *private_data)
9747{
9748 struct perf_event_mmap_page *header = mmap_mem;
9749 __u64 data_head = ring_buffer_read_head(header);
9750 __u64 data_tail = header->data_tail;
9751 void *base = ((__u8 *)header) + page_size;
9752 int ret = LIBBPF_PERF_EVENT_CONT;
9753 struct perf_event_header *ehdr;
9754 size_t ehdr_size;
9755
9756 while (data_head != data_tail) {
9757 ehdr = base + (data_tail & (mmap_size - 1));
9758 ehdr_size = ehdr->size;
9759
9760 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
9761 void *copy_start = ehdr;
9762 size_t len_first = base + mmap_size - copy_start;
9763 size_t len_secnd = ehdr_size - len_first;
9764
9765 if (*copy_size < ehdr_size) {
9766 free(*copy_mem);
9767 *copy_mem = malloc(ehdr_size);
9768 if (!*copy_mem) {
9769 *copy_size = 0;
9770 ret = LIBBPF_PERF_EVENT_ERROR;
9771 break;
9772 }
9773 *copy_size = ehdr_size;
9774 }
9775
9776 memcpy(*copy_mem, copy_start, len_first);
9777 memcpy(*copy_mem + len_first, base, len_secnd);
9778 ehdr = *copy_mem;
9779 }
9780
9781 ret = fn(ehdr, private_data);
9782 data_tail += ehdr_size;
9783 if (ret != LIBBPF_PERF_EVENT_CONT)
9784 break;
9785 }
9786
9787 ring_buffer_write_tail(header, data_tail);
9788 return libbpf_err(ret);
9789}
9790
9791struct perf_buffer;
9792
9793struct perf_buffer_params {
9794 struct perf_event_attr *attr;
9795
9796 perf_buffer_event_fn event_cb;
9797
9798 perf_buffer_sample_fn sample_cb;
9799 perf_buffer_lost_fn lost_cb;
9800 void *ctx;
9801 int cpu_cnt;
9802 int *cpus;
9803 int *map_keys;
9804};
9805
9806struct perf_cpu_buf {
9807 struct perf_buffer *pb;
9808 void *base;
9809 void *buf;
9810 size_t buf_size;
9811 int fd;
9812 int cpu;
9813 int map_key;
9814};
9815
9816struct perf_buffer {
9817 perf_buffer_event_fn event_cb;
9818 perf_buffer_sample_fn sample_cb;
9819 perf_buffer_lost_fn lost_cb;
9820 void *ctx;
9821
9822 size_t page_size;
9823 size_t mmap_size;
9824 struct perf_cpu_buf **cpu_bufs;
9825 struct epoll_event *events;
9826 int cpu_cnt;
9827 int epoll_fd;
9828 int map_fd;
9829};
9830
9831static void perf_buffer__free_cpu_buf(struct perf_buffer *pb,
9832 struct perf_cpu_buf *cpu_buf)
9833{
9834 if (!cpu_buf)
9835 return;
9836 if (cpu_buf->base &&
9837 munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
9838 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
9839 if (cpu_buf->fd >= 0) {
9840 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
9841 close(cpu_buf->fd);
9842 }
9843 free(cpu_buf->buf);
9844 free(cpu_buf);
9845}
9846
9847void perf_buffer__free(struct perf_buffer *pb)
9848{
9849 int i;
9850
9851 if (IS_ERR_OR_NULL(pb))
9852 return;
9853 if (pb->cpu_bufs) {
9854 for (i = 0; i < pb->cpu_cnt; i++) {
9855 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
9856
9857 if (!cpu_buf)
9858 continue;
9859
9860 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);
9861 perf_buffer__free_cpu_buf(pb, cpu_buf);
9862 }
9863 free(pb->cpu_bufs);
9864 }
9865 if (pb->epoll_fd >= 0)
9866 close(pb->epoll_fd);
9867 free(pb->events);
9868 free(pb);
9869}
9870
9871static struct perf_cpu_buf *
9872perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr,
9873 int cpu, int map_key)
9874{
9875 struct perf_cpu_buf *cpu_buf;
9876 char msg[STRERR_BUFSIZE];
9877 int err;
9878
9879 cpu_buf = calloc(1, sizeof(*cpu_buf));
9880 if (!cpu_buf)
9881 return ERR_PTR(-ENOMEM);
9882
9883 cpu_buf->pb = pb;
9884 cpu_buf->cpu = cpu;
9885 cpu_buf->map_key = map_key;
9886
9887 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 , cpu,
9888 -1, PERF_FLAG_FD_CLOEXEC);
9889 if (cpu_buf->fd < 0) {
9890 err = -errno;
9891 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
9892 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
9893 goto error;
9894 }
9895
9896 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,
9897 PROT_READ | PROT_WRITE, MAP_SHARED,
9898 cpu_buf->fd, 0);
9899 if (cpu_buf->base == MAP_FAILED) {
9900 cpu_buf->base = NULL;
9901 err = -errno;
9902 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
9903 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
9904 goto error;
9905 }
9906
9907 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
9908 err = -errno;
9909 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
9910 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
9911 goto error;
9912 }
9913
9914 return cpu_buf;
9915
9916error:
9917 perf_buffer__free_cpu_buf(pb, cpu_buf);
9918 return (struct perf_cpu_buf *)ERR_PTR(err);
9919}
9920
9921static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
9922 struct perf_buffer_params *p);
9923
9924struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
9925 const struct perf_buffer_opts *opts)
9926{
9927 struct perf_buffer_params p = {};
9928 struct perf_event_attr attr = { 0, };
9929
9930 attr.config = PERF_COUNT_SW_BPF_OUTPUT;
9931 attr.type = PERF_TYPE_SOFTWARE;
9932 attr.sample_type = PERF_SAMPLE_RAW;
9933 attr.sample_period = 1;
9934 attr.wakeup_events = 1;
9935
9936 p.attr = &attr;
9937 p.sample_cb = opts ? opts->sample_cb : NULL;
9938 p.lost_cb = opts ? opts->lost_cb : NULL;
9939 p.ctx = opts ? opts->ctx : NULL;
9940
9941 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
9942}
9943
9944struct perf_buffer *
9945perf_buffer__new_raw(int map_fd, size_t page_cnt,
9946 const struct perf_buffer_raw_opts *opts)
9947{
9948 struct perf_buffer_params p = {};
9949
9950 p.attr = opts->attr;
9951 p.event_cb = opts->event_cb;
9952 p.ctx = opts->ctx;
9953 p.cpu_cnt = opts->cpu_cnt;
9954 p.cpus = opts->cpus;
9955 p.map_keys = opts->map_keys;
9956
9957 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
9958}
9959
9960static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
9961 struct perf_buffer_params *p)
9962{
9963 const char *online_cpus_file = "/sys/devices/system/cpu/online";
9964 struct bpf_map_info map;
9965 char msg[STRERR_BUFSIZE];
9966 struct perf_buffer *pb;
9967 bool *online = NULL;
9968 __u32 map_info_len;
9969 int err, i, j, n;
9970
9971 if (page_cnt & (page_cnt - 1)) {
9972 pr_warn("page count should be power of two, but is %zu\n",
9973 page_cnt);
9974 return ERR_PTR(-EINVAL);
9975 }
9976
9977
9978 memset(&map, 0, sizeof(map));
9979 map_info_len = sizeof(map);
9980 err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
9981 if (err) {
9982 err = -errno;
9983
9984
9985
9986 if (err != -EINVAL) {
9987 pr_warn("failed to get map info for map FD %d: %s\n",
9988 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
9989 return ERR_PTR(err);
9990 }
9991 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
9992 map_fd);
9993 } else {
9994 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
9995 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
9996 map.name);
9997 return ERR_PTR(-EINVAL);
9998 }
9999 }
10000
10001 pb = calloc(1, sizeof(*pb));
10002 if (!pb)
10003 return ERR_PTR(-ENOMEM);
10004
10005 pb->event_cb = p->event_cb;
10006 pb->sample_cb = p->sample_cb;
10007 pb->lost_cb = p->lost_cb;
10008 pb->ctx = p->ctx;
10009
10010 pb->page_size = getpagesize();
10011 pb->mmap_size = pb->page_size * page_cnt;
10012 pb->map_fd = map_fd;
10013
10014 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
10015 if (pb->epoll_fd < 0) {
10016 err = -errno;
10017 pr_warn("failed to create epoll instance: %s\n",
10018 libbpf_strerror_r(err, msg, sizeof(msg)));
10019 goto error;
10020 }
10021
10022 if (p->cpu_cnt > 0) {
10023 pb->cpu_cnt = p->cpu_cnt;
10024 } else {
10025 pb->cpu_cnt = libbpf_num_possible_cpus();
10026 if (pb->cpu_cnt < 0) {
10027 err = pb->cpu_cnt;
10028 goto error;
10029 }
10030 if (map.max_entries && map.max_entries < pb->cpu_cnt)
10031 pb->cpu_cnt = map.max_entries;
10032 }
10033
10034 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
10035 if (!pb->events) {
10036 err = -ENOMEM;
10037 pr_warn("failed to allocate events: out of memory\n");
10038 goto error;
10039 }
10040 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
10041 if (!pb->cpu_bufs) {
10042 err = -ENOMEM;
10043 pr_warn("failed to allocate buffers: out of memory\n");
10044 goto error;
10045 }
10046
10047 err = parse_cpu_mask_file(online_cpus_file, &online, &n);
10048 if (err) {
10049 pr_warn("failed to get online CPU mask: %d\n", err);
10050 goto error;
10051 }
10052
10053 for (i = 0, j = 0; i < pb->cpu_cnt; i++) {
10054 struct perf_cpu_buf *cpu_buf;
10055 int cpu, map_key;
10056
10057 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;
10058 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;
10059
10060
10061
10062
10063 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))
10064 continue;
10065
10066 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);
10067 if (IS_ERR(cpu_buf)) {
10068 err = PTR_ERR(cpu_buf);
10069 goto error;
10070 }
10071
10072 pb->cpu_bufs[j] = cpu_buf;
10073
10074 err = bpf_map_update_elem(pb->map_fd, &map_key,
10075 &cpu_buf->fd, 0);
10076 if (err) {
10077 err = -errno;
10078 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
10079 cpu, map_key, cpu_buf->fd,
10080 libbpf_strerror_r(err, msg, sizeof(msg)));
10081 goto error;
10082 }
10083
10084 pb->events[j].events = EPOLLIN;
10085 pb->events[j].data.ptr = cpu_buf;
10086 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
10087 &pb->events[j]) < 0) {
10088 err = -errno;
10089 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
10090 cpu, cpu_buf->fd,
10091 libbpf_strerror_r(err, msg, sizeof(msg)));
10092 goto error;
10093 }
10094 j++;
10095 }
10096 pb->cpu_cnt = j;
10097 free(online);
10098
10099 return pb;
10100
10101error:
10102 free(online);
10103 if (pb)
10104 perf_buffer__free(pb);
10105 return ERR_PTR(err);
10106}
10107
10108struct perf_sample_raw {
10109 struct perf_event_header header;
10110 uint32_t size;
10111 char data[];
10112};
10113
10114struct perf_sample_lost {
10115 struct perf_event_header header;
10116 uint64_t id;
10117 uint64_t lost;
10118 uint64_t sample_id;
10119};
10120
10121static enum bpf_perf_event_ret
10122perf_buffer__process_record(struct perf_event_header *e, void *ctx)
10123{
10124 struct perf_cpu_buf *cpu_buf = ctx;
10125 struct perf_buffer *pb = cpu_buf->pb;
10126 void *data = e;
10127
10128
10129 if (pb->event_cb)
10130 return pb->event_cb(pb->ctx, cpu_buf->cpu, e);
10131
10132 switch (e->type) {
10133 case PERF_RECORD_SAMPLE: {
10134 struct perf_sample_raw *s = data;
10135
10136 if (pb->sample_cb)
10137 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);
10138 break;
10139 }
10140 case PERF_RECORD_LOST: {
10141 struct perf_sample_lost *s = data;
10142
10143 if (pb->lost_cb)
10144 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);
10145 break;
10146 }
10147 default:
10148 pr_warn("unknown perf sample type %d\n", e->type);
10149 return LIBBPF_PERF_EVENT_ERROR;
10150 }
10151 return LIBBPF_PERF_EVENT_CONT;
10152}
10153
10154static int perf_buffer__process_records(struct perf_buffer *pb,
10155 struct perf_cpu_buf *cpu_buf)
10156{
10157 enum bpf_perf_event_ret ret;
10158
10159 ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size,
10160 pb->page_size, &cpu_buf->buf,
10161 &cpu_buf->buf_size,
10162 perf_buffer__process_record, cpu_buf);
10163 if (ret != LIBBPF_PERF_EVENT_CONT)
10164 return ret;
10165 return 0;
10166}
10167
10168int perf_buffer__epoll_fd(const struct perf_buffer *pb)
10169{
10170 return pb->epoll_fd;
10171}
10172
10173int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms)
10174{
10175 int i, cnt, err;
10176
10177 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);
10178 if (cnt < 0)
10179 return -errno;
10180
10181 for (i = 0; i < cnt; i++) {
10182 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;
10183
10184 err = perf_buffer__process_records(pb, cpu_buf);
10185 if (err) {
10186 pr_warn("error while processing records: %d\n", err);
10187 return libbpf_err(err);
10188 }
10189 }
10190 return cnt;
10191}
10192
10193
10194
10195
10196size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb)
10197{
10198 return pb->cpu_cnt;
10199}
10200
10201
10202
10203
10204
10205
10206int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
10207{
10208 struct perf_cpu_buf *cpu_buf;
10209
10210 if (buf_idx >= pb->cpu_cnt)
10211 return libbpf_err(-EINVAL);
10212
10213 cpu_buf = pb->cpu_bufs[buf_idx];
10214 if (!cpu_buf)
10215 return libbpf_err(-ENOENT);
10216
10217 return cpu_buf->fd;
10218}
10219
10220
10221
10222
10223
10224
10225
10226
10227
10228int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx)
10229{
10230 struct perf_cpu_buf *cpu_buf;
10231
10232 if (buf_idx >= pb->cpu_cnt)
10233 return libbpf_err(-EINVAL);
10234
10235 cpu_buf = pb->cpu_bufs[buf_idx];
10236 if (!cpu_buf)
10237 return libbpf_err(-ENOENT);
10238
10239 return perf_buffer__process_records(pb, cpu_buf);
10240}
10241
10242int perf_buffer__consume(struct perf_buffer *pb)
10243{
10244 int i, err;
10245
10246 for (i = 0; i < pb->cpu_cnt; i++) {
10247 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];
10248
10249 if (!cpu_buf)
10250 continue;
10251
10252 err = perf_buffer__process_records(pb, cpu_buf);
10253 if (err) {
10254 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);
10255 return libbpf_err(err);
10256 }
10257 }
10258 return 0;
10259}
10260
10261struct bpf_prog_info_array_desc {
10262 int array_offset;
10263 int count_offset;
10264 int size_offset;
10265
10266
10267};
10268
10269static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = {
10270 [BPF_PROG_INFO_JITED_INSNS] = {
10271 offsetof(struct bpf_prog_info, jited_prog_insns),
10272 offsetof(struct bpf_prog_info, jited_prog_len),
10273 -1,
10274 },
10275 [BPF_PROG_INFO_XLATED_INSNS] = {
10276 offsetof(struct bpf_prog_info, xlated_prog_insns),
10277 offsetof(struct bpf_prog_info, xlated_prog_len),
10278 -1,
10279 },
10280 [BPF_PROG_INFO_MAP_IDS] = {
10281 offsetof(struct bpf_prog_info, map_ids),
10282 offsetof(struct bpf_prog_info, nr_map_ids),
10283 -(int)sizeof(__u32),
10284 },
10285 [BPF_PROG_INFO_JITED_KSYMS] = {
10286 offsetof(struct bpf_prog_info, jited_ksyms),
10287 offsetof(struct bpf_prog_info, nr_jited_ksyms),
10288 -(int)sizeof(__u64),
10289 },
10290 [BPF_PROG_INFO_JITED_FUNC_LENS] = {
10291 offsetof(struct bpf_prog_info, jited_func_lens),
10292 offsetof(struct bpf_prog_info, nr_jited_func_lens),
10293 -(int)sizeof(__u32),
10294 },
10295 [BPF_PROG_INFO_FUNC_INFO] = {
10296 offsetof(struct bpf_prog_info, func_info),
10297 offsetof(struct bpf_prog_info, nr_func_info),
10298 offsetof(struct bpf_prog_info, func_info_rec_size),
10299 },
10300 [BPF_PROG_INFO_LINE_INFO] = {
10301 offsetof(struct bpf_prog_info, line_info),
10302 offsetof(struct bpf_prog_info, nr_line_info),
10303 offsetof(struct bpf_prog_info, line_info_rec_size),
10304 },
10305 [BPF_PROG_INFO_JITED_LINE_INFO] = {
10306 offsetof(struct bpf_prog_info, jited_line_info),
10307 offsetof(struct bpf_prog_info, nr_jited_line_info),
10308 offsetof(struct bpf_prog_info, jited_line_info_rec_size),
10309 },
10310 [BPF_PROG_INFO_PROG_TAGS] = {
10311 offsetof(struct bpf_prog_info, prog_tags),
10312 offsetof(struct bpf_prog_info, nr_prog_tags),
10313 -(int)sizeof(__u8) * BPF_TAG_SIZE,
10314 },
10315
10316};
10317
10318static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,
10319 int offset)
10320{
10321 __u32 *array = (__u32 *)info;
10322
10323 if (offset >= 0)
10324 return array[offset / sizeof(__u32)];
10325 return -(int)offset;
10326}
10327
10328static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,
10329 int offset)
10330{
10331 __u64 *array = (__u64 *)info;
10332
10333 if (offset >= 0)
10334 return array[offset / sizeof(__u64)];
10335 return -(int)offset;
10336}
10337
10338static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
10339 __u32 val)
10340{
10341 __u32 *array = (__u32 *)info;
10342
10343 if (offset >= 0)
10344 array[offset / sizeof(__u32)] = val;
10345}
10346
10347static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
10348 __u64 val)
10349{
10350 __u64 *array = (__u64 *)info;
10351
10352 if (offset >= 0)
10353 array[offset / sizeof(__u64)] = val;
10354}
10355
10356struct bpf_prog_info_linear *
10357bpf_program__get_prog_info_linear(int fd, __u64 arrays)
10358{
10359 struct bpf_prog_info_linear *info_linear;
10360 struct bpf_prog_info info = {};
10361 __u32 info_len = sizeof(info);
10362 __u32 data_len = 0;
10363 int i, err;
10364 void *ptr;
10365
10366 if (arrays >> BPF_PROG_INFO_LAST_ARRAY)
10367 return libbpf_err_ptr(-EINVAL);
10368
10369
10370 err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
10371 if (err) {
10372 pr_debug("can't get prog info: %s", strerror(errno));
10373 return libbpf_err_ptr(-EFAULT);
10374 }
10375
10376
10377 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10378 bool include_array = (arrays & (1UL << i)) > 0;
10379 struct bpf_prog_info_array_desc *desc;
10380 __u32 count, size;
10381
10382 desc = bpf_prog_info_array_desc + i;
10383
10384
10385 if (info_len < desc->array_offset + sizeof(__u32) ||
10386 info_len < desc->count_offset + sizeof(__u32) ||
10387 (desc->size_offset > 0 && info_len < desc->size_offset))
10388 include_array = false;
10389
10390 if (!include_array) {
10391 arrays &= ~(1UL << i);
10392 continue;
10393 }
10394
10395 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10396 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10397
10398 data_len += count * size;
10399 }
10400
10401
10402 data_len = roundup(data_len, sizeof(__u64));
10403 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len);
10404 if (!info_linear)
10405 return libbpf_err_ptr(-ENOMEM);
10406
10407
10408 info_linear->arrays = arrays;
10409 memset(&info_linear->info, 0, sizeof(info));
10410 ptr = info_linear->data;
10411
10412 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10413 struct bpf_prog_info_array_desc *desc;
10414 __u32 count, size;
10415
10416 if ((arrays & (1UL << i)) == 0)
10417 continue;
10418
10419 desc = bpf_prog_info_array_desc + i;
10420 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10421 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10422 bpf_prog_info_set_offset_u32(&info_linear->info,
10423 desc->count_offset, count);
10424 bpf_prog_info_set_offset_u32(&info_linear->info,
10425 desc->size_offset, size);
10426 bpf_prog_info_set_offset_u64(&info_linear->info,
10427 desc->array_offset,
10428 ptr_to_u64(ptr));
10429 ptr += count * size;
10430 }
10431
10432
10433 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
10434 if (err) {
10435 pr_debug("can't get prog info: %s", strerror(errno));
10436 free(info_linear);
10437 return libbpf_err_ptr(-EFAULT);
10438 }
10439
10440
10441 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10442 struct bpf_prog_info_array_desc *desc;
10443 __u32 v1, v2;
10444
10445 if ((arrays & (1UL << i)) == 0)
10446 continue;
10447
10448 desc = bpf_prog_info_array_desc + i;
10449 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
10450 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
10451 desc->count_offset);
10452 if (v1 != v2)
10453 pr_warn("%s: mismatch in element count\n", __func__);
10454
10455 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
10456 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
10457 desc->size_offset);
10458 if (v1 != v2)
10459 pr_warn("%s: mismatch in rec size\n", __func__);
10460 }
10461
10462
10463 info_linear->info_len = sizeof(struct bpf_prog_info);
10464 info_linear->data_len = data_len;
10465
10466 return info_linear;
10467}
10468
10469void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear)
10470{
10471 int i;
10472
10473 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10474 struct bpf_prog_info_array_desc *desc;
10475 __u64 addr, offs;
10476
10477 if ((info_linear->arrays & (1UL << i)) == 0)
10478 continue;
10479
10480 desc = bpf_prog_info_array_desc + i;
10481 addr = bpf_prog_info_read_offset_u64(&info_linear->info,
10482 desc->array_offset);
10483 offs = addr - ptr_to_u64(info_linear->data);
10484 bpf_prog_info_set_offset_u64(&info_linear->info,
10485 desc->array_offset, offs);
10486 }
10487}
10488
10489void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear)
10490{
10491 int i;
10492
10493 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
10494 struct bpf_prog_info_array_desc *desc;
10495 __u64 addr, offs;
10496
10497 if ((info_linear->arrays & (1UL << i)) == 0)
10498 continue;
10499
10500 desc = bpf_prog_info_array_desc + i;
10501 offs = bpf_prog_info_read_offset_u64(&info_linear->info,
10502 desc->array_offset);
10503 addr = offs + ptr_to_u64(info_linear->data);
10504 bpf_prog_info_set_offset_u64(&info_linear->info,
10505 desc->array_offset, addr);
10506 }
10507}
10508
10509int bpf_program__set_attach_target(struct bpf_program *prog,
10510 int attach_prog_fd,
10511 const char *attach_func_name)
10512{
10513 int btf_obj_fd = 0, btf_id = 0, err;
10514
10515 if (!prog || attach_prog_fd < 0 || !attach_func_name)
10516 return libbpf_err(-EINVAL);
10517
10518 if (prog->obj->loaded)
10519 return libbpf_err(-EINVAL);
10520
10521 if (attach_prog_fd) {
10522 btf_id = libbpf_find_prog_btf_id(attach_func_name,
10523 attach_prog_fd);
10524 if (btf_id < 0)
10525 return libbpf_err(btf_id);
10526 } else {
10527
10528 err = bpf_object__load_vmlinux_btf(prog->obj, true);
10529 if (err)
10530 return libbpf_err(err);
10531 err = find_kernel_btf_id(prog->obj, attach_func_name,
10532 prog->expected_attach_type,
10533 &btf_obj_fd, &btf_id);
10534 if (err)
10535 return libbpf_err(err);
10536 }
10537
10538 prog->attach_btf_id = btf_id;
10539 prog->attach_btf_obj_fd = btf_obj_fd;
10540 prog->attach_prog_fd = attach_prog_fd;
10541 return 0;
10542}
10543
10544int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz)
10545{
10546 int err = 0, n, len, start, end = -1;
10547 bool *tmp;
10548
10549 *mask = NULL;
10550 *mask_sz = 0;
10551
10552
10553 while (*s) {
10554 if (*s == ',' || *s == '\n') {
10555 s++;
10556 continue;
10557 }
10558 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);
10559 if (n <= 0 || n > 2) {
10560 pr_warn("Failed to get CPU range %s: %d\n", s, n);
10561 err = -EINVAL;
10562 goto cleanup;
10563 } else if (n == 1) {
10564 end = start;
10565 }
10566 if (start < 0 || start > end) {
10567 pr_warn("Invalid CPU range [%d,%d] in %s\n",
10568 start, end, s);
10569 err = -EINVAL;
10570 goto cleanup;
10571 }
10572 tmp = realloc(*mask, end + 1);
10573 if (!tmp) {
10574 err = -ENOMEM;
10575 goto cleanup;
10576 }
10577 *mask = tmp;
10578 memset(tmp + *mask_sz, 0, start - *mask_sz);
10579 memset(tmp + start, 1, end - start + 1);
10580 *mask_sz = end + 1;
10581 s += len;
10582 }
10583 if (!*mask_sz) {
10584 pr_warn("Empty CPU range\n");
10585 return -EINVAL;
10586 }
10587 return 0;
10588cleanup:
10589 free(*mask);
10590 *mask = NULL;
10591 return err;
10592}
10593
10594int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz)
10595{
10596 int fd, err = 0, len;
10597 char buf[128];
10598
10599 fd = open(fcpu, O_RDONLY);
10600 if (fd < 0) {
10601 err = -errno;
10602 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);
10603 return err;
10604 }
10605 len = read(fd, buf, sizeof(buf));
10606 close(fd);
10607 if (len <= 0) {
10608 err = len ? -errno : -EINVAL;
10609 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);
10610 return err;
10611 }
10612 if (len >= sizeof(buf)) {
10613 pr_warn("CPU mask is too big in file %s\n", fcpu);
10614 return -E2BIG;
10615 }
10616 buf[len] = '\0';
10617
10618 return parse_cpu_mask_str(buf, mask, mask_sz);
10619}
10620
10621int libbpf_num_possible_cpus(void)
10622{
10623 static const char *fcpu = "/sys/devices/system/cpu/possible";
10624 static int cpus;
10625 int err, n, i, tmp_cpus;
10626 bool *mask;
10627
10628 tmp_cpus = READ_ONCE(cpus);
10629 if (tmp_cpus > 0)
10630 return tmp_cpus;
10631
10632 err = parse_cpu_mask_file(fcpu, &mask, &n);
10633 if (err)
10634 return libbpf_err(err);
10635
10636 tmp_cpus = 0;
10637 for (i = 0; i < n; i++) {
10638 if (mask[i])
10639 tmp_cpus++;
10640 }
10641 free(mask);
10642
10643 WRITE_ONCE(cpus, tmp_cpus);
10644 return tmp_cpus;
10645}
10646
10647int bpf_object__open_skeleton(struct bpf_object_skeleton *s,
10648 const struct bpf_object_open_opts *opts)
10649{
10650 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts,
10651 .object_name = s->name,
10652 );
10653 struct bpf_object *obj;
10654 int i, err;
10655
10656
10657
10658
10659
10660
10661
10662 if (opts) {
10663 memcpy(&skel_opts, opts, sizeof(*opts));
10664 if (!opts->object_name)
10665 skel_opts.object_name = s->name;
10666 }
10667
10668 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
10669 err = libbpf_get_error(obj);
10670 if (err) {
10671 pr_warn("failed to initialize skeleton BPF object '%s': %d\n",
10672 s->name, err);
10673 return libbpf_err(err);
10674 }
10675
10676 *s->obj = obj;
10677
10678 for (i = 0; i < s->map_cnt; i++) {
10679 struct bpf_map **map = s->maps[i].map;
10680 const char *name = s->maps[i].name;
10681 void **mmaped = s->maps[i].mmaped;
10682
10683 *map = bpf_object__find_map_by_name(obj, name);
10684 if (!*map) {
10685 pr_warn("failed to find skeleton map '%s'\n", name);
10686 return libbpf_err(-ESRCH);
10687 }
10688
10689
10690 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)
10691 *mmaped = (*map)->mmaped;
10692 }
10693
10694 for (i = 0; i < s->prog_cnt; i++) {
10695 struct bpf_program **prog = s->progs[i].prog;
10696 const char *name = s->progs[i].name;
10697
10698 *prog = bpf_object__find_program_by_name(obj, name);
10699 if (!*prog) {
10700 pr_warn("failed to find skeleton program '%s'\n", name);
10701 return libbpf_err(-ESRCH);
10702 }
10703 }
10704
10705 return 0;
10706}
10707
10708int bpf_object__load_skeleton(struct bpf_object_skeleton *s)
10709{
10710 int i, err;
10711
10712 err = bpf_object__load(*s->obj);
10713 if (err) {
10714 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);
10715 return libbpf_err(err);
10716 }
10717
10718 for (i = 0; i < s->map_cnt; i++) {
10719 struct bpf_map *map = *s->maps[i].map;
10720 size_t mmap_sz = bpf_map_mmap_sz(map);
10721 int prot, map_fd = bpf_map__fd(map);
10722 void **mmaped = s->maps[i].mmaped;
10723
10724 if (!mmaped)
10725 continue;
10726
10727 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {
10728 *mmaped = NULL;
10729 continue;
10730 }
10731
10732 if (map->def.map_flags & BPF_F_RDONLY_PROG)
10733 prot = PROT_READ;
10734 else
10735 prot = PROT_READ | PROT_WRITE;
10736
10737
10738
10739
10740
10741
10742
10743
10744
10745
10746
10747 *mmaped = mmap(map->mmaped, mmap_sz, prot,
10748 MAP_SHARED | MAP_FIXED, map_fd, 0);
10749 if (*mmaped == MAP_FAILED) {
10750 err = -errno;
10751 *mmaped = NULL;
10752 pr_warn("failed to re-mmap() map '%s': %d\n",
10753 bpf_map__name(map), err);
10754 return libbpf_err(err);
10755 }
10756 }
10757
10758 return 0;
10759}
10760
10761int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
10762{
10763 int i, err;
10764
10765 for (i = 0; i < s->prog_cnt; i++) {
10766 struct bpf_program *prog = *s->progs[i].prog;
10767 struct bpf_link **link = s->progs[i].link;
10768 const struct bpf_sec_def *sec_def;
10769
10770 if (!prog->load)
10771 continue;
10772
10773 sec_def = find_sec_def(prog->sec_name);
10774 if (!sec_def || !sec_def->attach_fn)
10775 continue;
10776
10777 *link = sec_def->attach_fn(sec_def, prog);
10778 err = libbpf_get_error(*link);
10779 if (err) {
10780 pr_warn("failed to auto-attach program '%s': %d\n",
10781 bpf_program__name(prog), err);
10782 return libbpf_err(err);
10783 }
10784 }
10785
10786 return 0;
10787}
10788
10789void bpf_object__detach_skeleton(struct bpf_object_skeleton *s)
10790{
10791 int i;
10792
10793 for (i = 0; i < s->prog_cnt; i++) {
10794 struct bpf_link **link = s->progs[i].link;
10795
10796 bpf_link__destroy(*link);
10797 *link = NULL;
10798 }
10799}
10800
10801void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s)
10802{
10803 if (s->progs)
10804 bpf_object__detach_skeleton(s);
10805 if (s->obj)
10806 bpf_object__close(*s->obj);
10807 free(s->maps);
10808 free(s->progs);
10809 free(s);
10810}
10811