1
2
3
4
5
6
7
8
9#ifndef __LIBBPF_LIBBPF_INTERNAL_H
10#define __LIBBPF_LIBBPF_INTERNAL_H
11
12#include <stdlib.h>
13#include <limits.h>
14#include <errno.h>
15#include <linux/err.h>
16#include <fcntl.h>
17#include <unistd.h>
18#include "libbpf_legacy.h"
19#include "relo_core.h"
20
21
22#pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64
23
24
25#pragma GCC poison reallocarray
26
27#include "libbpf.h"
28#include "btf.h"
29
30#ifndef EM_BPF
31#define EM_BPF 247
32#endif
33
34#ifndef R_BPF_64_64
35#define R_BPF_64_64 1
36#endif
37#ifndef R_BPF_64_ABS64
38#define R_BPF_64_ABS64 2
39#endif
40#ifndef R_BPF_64_ABS32
41#define R_BPF_64_ABS32 3
42#endif
43#ifndef R_BPF_64_32
44#define R_BPF_64_32 10
45#endif
46
47#ifndef SHT_LLVM_ADDRSIG
48#define SHT_LLVM_ADDRSIG 0x6FFF4C03
49#endif
50
51
52#ifndef ELF_C_READ_MMAP
53#define ELF_C_READ_MMAP ELF_C_READ
54#endif
55
56
57#ifndef ELF64_ST_VISIBILITY
58#define ELF64_ST_VISIBILITY(o) ((o) & 0x03)
59#endif
60
61#define BTF_INFO_ENC(kind, kind_flag, vlen) \
62 ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN))
63#define BTF_TYPE_ENC(name, info, size_or_type) (name), (info), (size_or_type)
64#define BTF_INT_ENC(encoding, bits_offset, nr_bits) \
65 ((encoding) << 24 | (bits_offset) << 16 | (nr_bits))
66#define BTF_TYPE_INT_ENC(name, encoding, bits_offset, bits, sz) \
67 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_INT, 0, 0), sz), \
68 BTF_INT_ENC(encoding, bits_offset, bits)
69#define BTF_MEMBER_ENC(name, type, bits_offset) (name), (type), (bits_offset)
70#define BTF_PARAM_ENC(name, type) (name), (type)
71#define BTF_VAR_SECINFO_ENC(type, offset, size) (type), (offset), (size)
72#define BTF_TYPE_FLOAT_ENC(name, sz) \
73 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz)
74#define BTF_TYPE_DECL_TAG_ENC(value, type, component_idx) \
75 BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 0, 0), type), (component_idx)
76#define BTF_TYPE_TYPE_TAG_ENC(value, type) \
77 BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TYPE_TAG, 0, 0), type)
78
79#ifndef likely
80#define likely(x) __builtin_expect(!!(x), 1)
81#endif
82#ifndef unlikely
83#define unlikely(x) __builtin_expect(!!(x), 0)
84#endif
85#ifndef min
86# define min(x, y) ((x) < (y) ? (x) : (y))
87#endif
88#ifndef max
89# define max(x, y) ((x) < (y) ? (y) : (x))
90#endif
91#ifndef offsetofend
92# define offsetofend(TYPE, FIELD) \
93 (offsetof(TYPE, FIELD) + sizeof(((TYPE *)0)->FIELD))
94#endif
95#ifndef __alias
96#define __alias(symbol) __attribute__((alias(#symbol)))
97#endif
98
99
100
101
102
103#define str_has_pfx(str, pfx) \
104 (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0)
105
106
107
108
109
110
111
112#if defined(SHARED) && defined(__GNUC__) && __GNUC__ >= 10
113
114#define DEFAULT_VERSION(internal_name, api_name, version) \
115 __attribute__((symver(#api_name "@@" #version)))
116#define COMPAT_VERSION(internal_name, api_name, version) \
117 __attribute__((symver(#api_name "@" #version)))
118
119#elif defined(SHARED)
120
121#define COMPAT_VERSION(internal_name, api_name, version) \
122 asm(".symver " #internal_name "," #api_name "@" #version);
123#define DEFAULT_VERSION(internal_name, api_name, version) \
124 asm(".symver " #internal_name "," #api_name "@@" #version);
125
126#else
127
128#define COMPAT_VERSION(internal_name, api_name, version)
129#define DEFAULT_VERSION(internal_name, api_name, version) \
130 extern typeof(internal_name) api_name \
131 __attribute__((alias(#internal_name)));
132
133#endif
134
135extern void libbpf_print(enum libbpf_print_level level,
136 const char *format, ...)
137 __attribute__((format(printf, 2, 3)));
138
139#define __pr(level, fmt, ...) \
140do { \
141 libbpf_print(level, "libbpf: " fmt, ##__VA_ARGS__); \
142} while (0)
143
144#define pr_warn(fmt, ...) __pr(LIBBPF_WARN, fmt, ##__VA_ARGS__)
145#define pr_info(fmt, ...) __pr(LIBBPF_INFO, fmt, ##__VA_ARGS__)
146#define pr_debug(fmt, ...) __pr(LIBBPF_DEBUG, fmt, ##__VA_ARGS__)
147
148#ifndef __has_builtin
149#define __has_builtin(x) 0
150#endif
151
152
153
154
155
156
157
158
159
160static inline void *libbpf_reallocarray(void *ptr, size_t nmemb, size_t size)
161{
162 size_t total;
163
164#if __has_builtin(__builtin_mul_overflow)
165 if (unlikely(__builtin_mul_overflow(nmemb, size, &total)))
166 return NULL;
167#else
168 if (size == 0 || nmemb > ULONG_MAX / size)
169 return NULL;
170 total = nmemb * size;
171#endif
172 return realloc(ptr, total);
173}
174
175
176
177
178
179
180
181static inline void libbpf_strlcpy(char *dst, const char *src, size_t sz)
182{
183 size_t i;
184
185 if (sz == 0)
186 return;
187
188 sz--;
189 for (i = 0; i < sz && src[i]; i++)
190 dst[i] = src[i];
191 dst[i] = '\0';
192}
193
194__u32 get_kernel_version(void);
195
196struct btf;
197struct btf_type;
198
199struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id);
200const char *btf_kind_str(const struct btf_type *t);
201const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);
202
203static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t)
204{
205 return (enum btf_func_linkage)(int)btf_vlen(t);
206}
207
208static inline __u32 btf_type_info(int kind, int vlen, int kflag)
209{
210 return (kflag << 31) | (kind << 24) | vlen;
211}
212
213enum map_def_parts {
214 MAP_DEF_MAP_TYPE = 0x001,
215 MAP_DEF_KEY_TYPE = 0x002,
216 MAP_DEF_KEY_SIZE = 0x004,
217 MAP_DEF_VALUE_TYPE = 0x008,
218 MAP_DEF_VALUE_SIZE = 0x010,
219 MAP_DEF_MAX_ENTRIES = 0x020,
220 MAP_DEF_MAP_FLAGS = 0x040,
221 MAP_DEF_NUMA_NODE = 0x080,
222 MAP_DEF_PINNING = 0x100,
223 MAP_DEF_INNER_MAP = 0x200,
224 MAP_DEF_MAP_EXTRA = 0x400,
225
226 MAP_DEF_ALL = 0x7ff,
227};
228
229struct btf_map_def {
230 enum map_def_parts parts;
231 __u32 map_type;
232 __u32 key_type_id;
233 __u32 key_size;
234 __u32 value_type_id;
235 __u32 value_size;
236 __u32 max_entries;
237 __u32 map_flags;
238 __u32 numa_node;
239 __u32 pinning;
240 __u64 map_extra;
241};
242
243int parse_btf_map_def(const char *map_name, struct btf *btf,
244 const struct btf_type *def_t, bool strict,
245 struct btf_map_def *map_def, struct btf_map_def *inner_def);
246
247void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz,
248 size_t cur_cnt, size_t max_cnt, size_t add_cnt);
249int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt);
250
251static inline bool libbpf_is_mem_zeroed(const char *p, ssize_t len)
252{
253 while (len > 0) {
254 if (*p)
255 return false;
256 p++;
257 len--;
258 }
259 return true;
260}
261
262static inline bool libbpf_validate_opts(const char *opts,
263 size_t opts_sz, size_t user_sz,
264 const char *type_name)
265{
266 if (user_sz < sizeof(size_t)) {
267 pr_warn("%s size (%zu) is too small\n", type_name, user_sz);
268 return false;
269 }
270 if (!libbpf_is_mem_zeroed(opts + opts_sz, (ssize_t)user_sz - opts_sz)) {
271 pr_warn("%s has non-zero extra bytes\n", type_name);
272 return false;
273 }
274 return true;
275}
276
277#define OPTS_VALID(opts, type) \
278 (!(opts) || libbpf_validate_opts((const char *)opts, \
279 offsetofend(struct type, \
280 type##__last_field), \
281 (opts)->sz, #type))
282#define OPTS_HAS(opts, field) \
283 ((opts) && opts->sz >= offsetofend(typeof(*(opts)), field))
284#define OPTS_GET(opts, field, fallback_value) \
285 (OPTS_HAS(opts, field) ? (opts)->field : fallback_value)
286#define OPTS_SET(opts, field, value) \
287 do { \
288 if (OPTS_HAS(opts, field)) \
289 (opts)->field = value; \
290 } while (0)
291
292#define OPTS_ZEROED(opts, last_nonzero_field) \
293({ \
294 ssize_t __off = offsetofend(typeof(*(opts)), last_nonzero_field); \
295 !(opts) || libbpf_is_mem_zeroed((const void *)opts + __off, \
296 (opts)->sz - __off); \
297})
298
299enum kern_feature_id {
300
301 FEAT_PROG_NAME,
302
303 FEAT_GLOBAL_DATA,
304
305 FEAT_BTF,
306
307 FEAT_BTF_FUNC,
308
309 FEAT_BTF_DATASEC,
310
311 FEAT_BTF_GLOBAL_FUNC,
312
313 FEAT_ARRAY_MMAP,
314
315 FEAT_EXP_ATTACH_TYPE,
316
317 FEAT_PROBE_READ_KERN,
318
319 FEAT_PROG_BIND_MAP,
320
321 FEAT_MODULE_BTF,
322
323 FEAT_BTF_FLOAT,
324
325 FEAT_PERF_LINK,
326
327 FEAT_BTF_DECL_TAG,
328
329 FEAT_BTF_TYPE_TAG,
330
331 FEAT_MEMCG_ACCOUNT,
332 __FEAT_CNT,
333};
334
335int probe_memcg_account(void);
336bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id);
337int bump_rlimit_memlock(void);
338
339int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz);
340int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz);
341int libbpf__load_raw_btf(const char *raw_types, size_t types_len,
342 const char *str_sec, size_t str_len);
343int btf_load_into_kernel(struct btf *btf, char *log_buf, size_t log_sz, __u32 log_level);
344
345struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf);
346void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,
347 const char **prefix, int *kind);
348
349struct btf_ext_info {
350
351
352
353
354 void *info;
355 __u32 rec_size;
356 __u32 len;
357};
358
359#define for_each_btf_ext_sec(seg, sec) \
360 for (sec = (seg)->info; \
361 (void *)sec < (seg)->info + (seg)->len; \
362 sec = (void *)sec + sizeof(struct btf_ext_info_sec) + \
363 (seg)->rec_size * sec->num_info)
364
365#define for_each_btf_ext_rec(seg, sec, i, rec) \
366 for (i = 0, rec = (void *)&(sec)->data; \
367 i < (sec)->num_info; \
368 i++, rec = (void *)rec + (seg)->rec_size)
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391struct btf_ext_header {
392 __u16 magic;
393 __u8 version;
394 __u8 flags;
395 __u32 hdr_len;
396
397
398 __u32 func_info_off;
399 __u32 func_info_len;
400 __u32 line_info_off;
401 __u32 line_info_len;
402
403
404 __u32 core_relo_off;
405 __u32 core_relo_len;
406};
407
408struct btf_ext {
409 union {
410 struct btf_ext_header *hdr;
411 void *data;
412 };
413 struct btf_ext_info func_info;
414 struct btf_ext_info line_info;
415 struct btf_ext_info core_relo_info;
416 __u32 data_size;
417};
418
419struct btf_ext_info_sec {
420 __u32 sec_name_off;
421 __u32 num_info;
422
423 __u8 data[];
424};
425
426
427struct bpf_func_info_min {
428 __u32 insn_off;
429 __u32 type_id;
430};
431
432
433struct bpf_line_info_min {
434 __u32 insn_off;
435 __u32 file_name_off;
436 __u32 line_off;
437 __u32 line_col;
438};
439
440
441typedef int (*type_id_visit_fn)(__u32 *type_id, void *ctx);
442typedef int (*str_off_visit_fn)(__u32 *str_off, void *ctx);
443int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ctx);
444int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ctx);
445int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx);
446int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx);
447__s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name,
448 __u32 kind);
449
450extern enum libbpf_strict_mode libbpf_mode;
451
452typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type,
453 const char *sym_name, void *ctx);
454
455int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *arg);
456
457
458static inline int libbpf_err(int ret)
459{
460 if (ret < 0)
461 errno = -ret;
462 return ret;
463}
464
465
466
467
468static inline int libbpf_err_errno(int ret)
469{
470 if (libbpf_mode & LIBBPF_STRICT_DIRECT_ERRS)
471
472 return ret < 0 ? -errno : ret;
473
474
475 return ret;
476}
477
478
479static inline void *libbpf_err_ptr(int err)
480{
481
482 errno = -err;
483
484 if (libbpf_mode & LIBBPF_STRICT_CLEAN_PTRS)
485 return NULL;
486
487
488 return ERR_PTR(err);
489}
490
491
492static inline void *libbpf_ptr(void *ret)
493{
494
495 if (IS_ERR(ret))
496 errno = -PTR_ERR(ret);
497
498 if (libbpf_mode & LIBBPF_STRICT_CLEAN_PTRS)
499 return IS_ERR(ret) ? NULL : ret;
500
501
502 return ret;
503}
504
505static inline bool str_is_empty(const char *s)
506{
507 return !s || !s[0];
508}
509
510static inline bool is_ldimm64_insn(struct bpf_insn *insn)
511{
512 return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
513}
514
515
516
517
518
519static inline int ensure_good_fd(int fd)
520{
521 int old_fd = fd, saved_errno;
522
523 if (fd < 0)
524 return fd;
525 if (fd < 3) {
526 fd = fcntl(fd, F_DUPFD_CLOEXEC, 3);
527 saved_errno = errno;
528 close(old_fd);
529 if (fd < 0) {
530 pr_warn("failed to dup FD %d to FD > 2: %d\n", old_fd, -saved_errno);
531 errno = saved_errno;
532 }
533 }
534 return fd;
535}
536
537
538int bpf_core_add_cands(struct bpf_core_cand *local_cand,
539 size_t local_essent_len,
540 const struct btf *targ_btf,
541 const char *targ_btf_name,
542 int targ_start_id,
543 struct bpf_core_cand_list *cands);
544void bpf_core_free_cands(struct bpf_core_cand_list *cands);
545
546#endif
547