// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)

/*
 * Common eBPF ELF object loading operations.
 *
 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
 * Copyright (C) 2015 Huawei Inc.
 * Copyright (C) 2017 Nicira, Inc.
 * Copyright (C) 2019 Isovalent, Inc.
 */

#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
#include <libgen.h>
#include <inttypes.h>
#include <limits.h>
#include <string.h>
#include <unistd.h>
#include <endian.h>
#include <fcntl.h>
#include <errno.h>
#include <ctype.h>
#include <asm/unistd.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/filter.h>
#include <linux/list.h>
#include <linux/limits.h>
#include <linux/perf_event.h>
#include <linux/ring_buffer.h>
#include <linux/version.h>
#include <sys/epoll.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/vfs.h>
#include <sys/utsname.h>
#include <sys/resource.h>
#include <libelf.h>
#include <gelf.h>
#include <zlib.h>

#include "libbpf.h"
#include "bpf.h"
#include "btf.h"
#include "str_error.h"
#include "libbpf_internal.h"
#include "hashmap.h"

#ifndef EM_BPF
#define EM_BPF 247
#endif

#ifndef BPF_FS_MAGIC
#define BPF_FS_MAGIC            0xcafe4a11
#endif

#define BPF_INSN_SZ (sizeof(struct bpf_insn))

/* vsprintf() in __base_pr() uses nonliteral format string. It may break
 * compilation if user enables corresponding warning. Disable it explicitly.
 */
#pragma GCC diagnostic ignored "-Wformat-nonliteral"

#define __printf(a, b)  __attribute__((format(printf, a, b)))

static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
static const struct btf_type *
skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id);

static int __base_pr(enum libbpf_print_level level, const char *format,
                     va_list args)
{
        if (level == LIBBPF_DEBUG)
                return 0;

        return vfprintf(stderr, format, args);
}

static libbpf_print_fn_t __libbpf_pr = __base_pr;

libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
{
        libbpf_print_fn_t old_print_fn = __libbpf_pr;

        __libbpf_pr = fn;
        return old_print_fn;
}

__printf(2, 3)
void libbpf_print(enum libbpf_print_level level, const char *format, ...)
{
        va_list args;

        if (!__libbpf_pr)
                return;

        va_start(args, format);
        __libbpf_pr(level, format, args);
        va_end(args);
}

static void pr_perm_msg(int err)
{
        struct rlimit limit;
        char buf[100];

        if (err != -EPERM || geteuid() != 0)
                return;

        err = getrlimit(RLIMIT_MEMLOCK, &limit);
        if (err)
                return;

        if (limit.rlim_cur == RLIM_INFINITY)
                return;

        if (limit.rlim_cur < 1024)
                snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
        else if (limit.rlim_cur < 1024*1024)
                snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
        else
                snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));

        pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
                buf);
}

#define STRERR_BUFSIZE  128

/* Copied from tools/perf/util/util.h */
#ifndef zfree
# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
#endif

#ifndef zclose
# define zclose(fd) ({                  \
        int ___err = 0;                 \
        if ((fd) >= 0)                  \
                ___err = close((fd));   \
        fd = -1;                        \
        ___err; })
#endif

static inline __u64 ptr_to_u64(const void *ptr)
{
        return (__u64) (unsigned long) ptr;
}

enum kern_feature_id {
        /* v4.14: kernel support for program & map names. */
        FEAT_PROG_NAME,
        /* v5.2: kernel support for global data sections. */
        FEAT_GLOBAL_DATA,
        /* BTF support */
        FEAT_BTF,
        /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
        FEAT_BTF_FUNC,
        /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
        FEAT_BTF_DATASEC,
        /* BTF_FUNC_GLOBAL is supported */
        FEAT_BTF_GLOBAL_FUNC,
        /* BPF_F_MMAPABLE is supported for arrays */
        FEAT_ARRAY_MMAP,
        /* kernel support for expected_attach_type in BPF_PROG_LOAD */
        FEAT_EXP_ATTACH_TYPE,
        /* bpf_probe_read_{kernel,user}[_str] helpers */
        FEAT_PROBE_READ_KERN,
        /* BPF_PROG_BIND_MAP is supported */
        FEAT_PROG_BIND_MAP,
        __FEAT_CNT,
};

static bool kernel_supports(enum kern_feature_id feat_id);

enum reloc_type {
        RELO_LD64,
        RELO_CALL,
        RELO_DATA,
        RELO_EXTERN,
};

struct reloc_desc {
        enum reloc_type type;
        int insn_idx;
        int map_idx;
        int sym_off;
        bool processed;
};

struct bpf_sec_def;

typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec,
                                        struct bpf_program *prog);

struct bpf_sec_def {
        const char *sec;
        size_t len;
        enum bpf_prog_type prog_type;
        enum bpf_attach_type expected_attach_type;
        bool is_exp_attach_type_optional;
        bool is_attachable;
        bool is_attach_btf;
        bool is_sleepable;
        attach_fn_t attach_fn;
};

/*
 * bpf_prog should be a better name but it has been used in
 * linux/filter.h.
 */
struct bpf_program {
        const struct bpf_sec_def *sec_def;
        char *sec_name;
        size_t sec_idx;
        /* this program's instruction offset (in number of instructions)
         * within its containing ELF section
         */
        size_t sec_insn_off;
        /* number of original instructions in ELF section belonging to this
         * program, not taking into account subprogram instructions possible
         * appended later during relocation
         */
        size_t sec_insn_cnt;
        /* Offset (in number of instructions) of the start of instruction
         * belonging to this BPF program  within its containing main BPF
         * program. For the entry-point (main) BPF program, this is always
         * zero. For a sub-program, this gets reset before each of main BPF
         * programs are processed and relocated and is used to determined
         * whether sub-program was already appended to the main program, and
         * if yes, at which instruction offset.
         */
        size_t sub_insn_off;

        char *name;
        /* sec_name with / replaced by _; makes recursive pinning
         * in bpf_object__pin_programs easier
         */
        char *pin_name;

        /* instructions that belong to BPF program; insns[0] is located at
         * sec_insn_off instruction within its ELF section in ELF file, so
         * when mapping ELF file instruction index to the local instruction,
         * one needs to subtract sec_insn_off; and vice versa.
         */
        struct bpf_insn *insns;
        /* actual number of instruction in this BPF program's image; for
         * entry-point BPF programs this includes the size of main program
         * itself plus all the used sub-programs, appended at the end
         */
        size_t insns_cnt;

        struct reloc_desc *reloc_desc;
        int nr_reloc;
        int log_level;

        struct {
                int nr;
                int *fds;
        } instances;
        bpf_program_prep_t preprocessor;

        struct bpf_object *obj;
        void *priv;
        bpf_program_clear_priv_t clear_priv;

        bool load;
        enum bpf_prog_type type;
        enum bpf_attach_type expected_attach_type;
        int prog_ifindex;
        __u32 attach_btf_id;
        __u32 attach_prog_fd;
        void *func_info;
        __u32 func_info_rec_size;
        __u32 func_info_cnt;

        void *line_info;
        __u32 line_info_rec_size;
        __u32 line_info_cnt;
        __u32 prog_flags;
};

struct bpf_struct_ops {
        const char *tname;
        const struct btf_type *type;
        struct bpf_program **progs;
        __u32 *kern_func_off;
        /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
        void *data;
        /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
         *      btf_vmlinux's format.
         * struct bpf_struct_ops_tcp_congestion_ops {
         *      [... some other kernel fields ...]
         *      struct tcp_congestion_ops data;
         * }
         * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
         * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
         * from "data".
         */
        void *kern_vdata;
        __u32 type_id;
};

#define DATA_SEC ".data"
#define BSS_SEC ".bss"
#define RODATA_SEC ".rodata"
#define KCONFIG_SEC ".kconfig"
#define KSYMS_SEC ".ksyms"
#define STRUCT_OPS_SEC ".struct_ops"

enum libbpf_map_type {
        LIBBPF_MAP_UNSPEC,
        LIBBPF_MAP_DATA,
        LIBBPF_MAP_BSS,
        LIBBPF_MAP_RODATA,
        LIBBPF_MAP_KCONFIG,
};

static const char * const libbpf_type_to_btf_name[] = {
        [LIBBPF_MAP_DATA]       = DATA_SEC,
        [LIBBPF_MAP_BSS]        = BSS_SEC,
        [LIBBPF_MAP_RODATA]     = RODATA_SEC,
        [LIBBPF_MAP_KCONFIG]    = KCONFIG_SEC,
};

struct bpf_map {
        char *name;
        int fd;
        int sec_idx;
        size_t sec_offset;
        int map_ifindex;
        int inner_map_fd;
        struct bpf_map_def def;
        __u32 numa_node;
        __u32 btf_var_idx;
        __u32 btf_key_type_id;
        __u32 btf_value_type_id;
        __u32 btf_vmlinux_value_type_id;
        void *priv;
        bpf_map_clear_priv_t clear_priv;
        enum libbpf_map_type libbpf_type;
        void *mmaped;
        struct bpf_struct_ops *st_ops;
        struct bpf_map *inner_map;
        void **init_slots;
        int init_slots_sz;
        char *pin_path;
        bool pinned;
        bool reused;
};

enum extern_type {
        EXT_UNKNOWN,
        EXT_KCFG,
        EXT_KSYM,
};

enum kcfg_type {
        KCFG_UNKNOWN,
        KCFG_CHAR,
        KCFG_BOOL,
        KCFG_INT,
        KCFG_TRISTATE,
        KCFG_CHAR_ARR,
};

struct extern_desc {
        enum extern_type type;
        int sym_idx;
        int btf_id;
        int sec_btf_id;
        const char *name;
        bool is_set;
        bool is_weak;
        union {
                struct {
                        enum kcfg_type type;
                        int sz;
                        int align;
                        int data_off;
                        bool is_signed;
                } kcfg;
                struct {
                        unsigned long long addr;

                        /* target btf_id of the corresponding kernel var. */
                        int vmlinux_btf_id;

                        /* local btf_id of the ksym extern's type. */
                        __u32 type_id;
                } ksym;
        };
};

static LIST_HEAD(bpf_objects_list);

struct bpf_object {
        char name[BPF_OBJ_NAME_LEN];
        char license[64];
        __u32 kern_version;

        struct bpf_program *programs;
        size_t nr_programs;
        struct bpf_map *maps;
        size_t nr_maps;
        size_t maps_cap;

        char *kconfig;
        struct extern_desc *externs;
        int nr_extern;
        int kconfig_map_idx;
        int rodata_map_idx;

        bool loaded;
        bool has_subcalls;

        /*
         * Information when doing elf related work. Only valid if fd
         * is valid.
         */
        struct {
                int fd;
                const void *obj_buf;
                size_t obj_buf_sz;
                Elf *elf;
                GElf_Ehdr ehdr;
                Elf_Data *symbols;
                Elf_Data *data;
                Elf_Data *rodata;
                Elf_Data *bss;
                Elf_Data *st_ops_data;
                size_t shstrndx; /* section index for section name strings */
                size_t strtabidx;
                struct {
                        GElf_Shdr shdr;
                        Elf_Data *data;
                } *reloc_sects;
                int nr_reloc_sects;
                int maps_shndx;
                int btf_maps_shndx;
                __u32 btf_maps_sec_btf_id;
                int text_shndx;
                int symbols_shndx;
                int data_shndx;
                int rodata_shndx;
                int bss_shndx;
                int st_ops_shndx;
        } efile;
        /*
         * All loaded bpf_object is linked in a list, which is
         * hidden to caller. bpf_objects__<func> handlers deal with
         * all objects.
         */
        struct list_head list;

        struct btf *btf;
        /* Parse and load BTF vmlinux if any of the programs in the object need
         * it at load time.
         */
        struct btf *btf_vmlinux;
        struct btf_ext *btf_ext;

        void *priv;
        bpf_object_clear_priv_t clear_priv;

        char path[];
};
#define obj_elf_valid(o)        ((o)->efile.elf)

static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr);
static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
static int elf_sym_by_sec_off(const struct bpf_object *obj, size_t sec_idx,
                              size_t off, __u32 sym_type, GElf_Sym *sym);

void bpf_program__unload(struct bpf_program *prog)
{
        int i;

        if (!prog)
                return;

        /*
         * If the object is opened but the program was never loaded,
         * it is possible that prog->instances.nr == -1.
         */
        if (prog->instances.nr > 0) {
                for (i = 0; i < prog->instances.nr; i++)
                        zclose(prog->instances.fds[i]);
        } else if (prog->instances.nr != -1) {
                pr_warn("Internal error: instances.nr is %d\n",
                        prog->instances.nr);
        }

        prog->instances.nr = -1;
        zfree(&prog->instances.fds);

        zfree(&prog->func_info);
        zfree(&prog->line_info);
}

static void bpf_program__exit(struct bpf_program *prog)
{
        if (!prog)
                return;

        if (prog->clear_priv)
                prog->clear_priv(prog, prog->priv);

        prog->priv = NULL;
        prog->clear_priv = NULL;

        bpf_program__unload(prog);
        zfree(&prog->name);
        zfree(&prog->sec_name);
        zfree(&prog->pin_name);
        zfree(&prog->insns);
        zfree(&prog->reloc_desc);

        prog->nr_reloc = 0;
        prog->insns_cnt = 0;
        prog->sec_idx = -1;
}

static char *__bpf_program__pin_name(struct bpf_program *prog)
{
        char *name, *p;

        name = p = strdup(prog->sec_name);
        while ((p = strchr(p, '/')))
                *p = '_';

        return name;
}

static bool insn_is_subprog_call(const struct bpf_insn *insn)
{
        return BPF_CLASS(insn->code) == BPF_JMP &&
               BPF_OP(insn->code) == BPF_CALL &&
               BPF_SRC(insn->code) == BPF_K &&
               insn->src_reg == BPF_PSEUDO_CALL &&
               insn->dst_reg == 0 &&
               insn->off == 0;
}

static int
bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
                      const char *name, size_t sec_idx, const char *sec_name,
                      size_t sec_off, void *insn_data, size_t insn_data_sz)
{
        if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
                pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
                        sec_name, name, sec_off, insn_data_sz);
                return -EINVAL;
        }

        memset(prog, 0, sizeof(*prog));
        prog->obj = obj;

        prog->sec_idx = sec_idx;
        prog->sec_insn_off = sec_off / BPF_INSN_SZ;
        prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
        /* insns_cnt can later be increased by appending used subprograms */
        prog->insns_cnt = prog->sec_insn_cnt;

        prog->type = BPF_PROG_TYPE_UNSPEC;
        prog->load = true;

        prog->instances.fds = NULL;
        prog->instances.nr = -1;

        prog->sec_name = strdup(sec_name);
        if (!prog->sec_name)
                goto errout;

        prog->name = strdup(name);
        if (!prog->name)
                goto errout;

        prog->pin_name = __bpf_program__pin_name(prog);
        if (!prog->pin_name)
                goto errout;

        prog->insns = malloc(insn_data_sz);
        if (!prog->insns)
                goto errout;
        memcpy(prog->insns, insn_data, insn_data_sz);

        return 0;
errout:
        pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
        bpf_program__exit(prog);
        return -ENOMEM;
}

static int
bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
                         const char *sec_name, int sec_idx)
{
        struct bpf_program *prog, *progs;
        void *data = sec_data->d_buf;
        size_t sec_sz = sec_data->d_size, sec_off, prog_sz;
        int nr_progs, err;
        const char *name;
        GElf_Sym sym;

        progs = obj->programs;
        nr_progs = obj->nr_programs;
        sec_off = 0;

        while (sec_off < sec_sz) {
                if (elf_sym_by_sec_off(obj, sec_idx, sec_off, STT_FUNC, &sym)) {
                        pr_warn("sec '%s': failed to find program symbol at offset %zu\n",
                                sec_name, sec_off);
                        return -LIBBPF_ERRNO__FORMAT;
                }

                prog_sz = sym.st_size;

                name = elf_sym_str(obj, sym.st_name);
                if (!name) {
                        pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
                                sec_name, sec_off);
                        return -LIBBPF_ERRNO__FORMAT;
                }

                if (sec_off + prog_sz > sec_sz) {
                        pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
                                sec_name, sec_off);
                        return -LIBBPF_ERRNO__FORMAT;
                }

                pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
                         sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);

                progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
                if (!progs) {
                        /*
                         * In this case the original obj->programs
                         * is still valid, so don't need special treat for
                         * bpf_close_object().
                         */
                        pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
                                sec_name, name);
                        return -ENOMEM;
                }
                obj->programs = progs;

                prog = &progs[nr_progs];

                err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
                                            sec_off, data + sec_off, prog_sz);
                if (err)
                        return err;

                nr_progs++;
                obj->nr_programs = nr_progs;

                sec_off += prog_sz;
        }

        return 0;
}

static __u32 get_kernel_version(void)
{
        __u32 major, minor, patch;
        struct utsname info;

        uname(&info);
        if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3)
                return 0;
        return KERNEL_VERSION(major, minor, patch);
}

static const struct btf_member *
find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
{
        struct btf_member *m;
        int i;

        for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
                if (btf_member_bit_offset(t, i) == bit_offset)
                        return m;
        }

        return NULL;
}

static const struct btf_member *
find_member_by_name(const struct btf *btf, const struct btf_type *t,
                    const char *name)
{
        struct btf_member *m;
        int i;

        for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
                if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
                        return m;
        }

        return NULL;
}

#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
                                   const char *name, __u32 kind);

static int
find_struct_ops_kern_types(const struct btf *btf, const char *tname,
                           const struct btf_type **type, __u32 *type_id,
                           const struct btf_type **vtype, __u32 *vtype_id,
                           const struct btf_member **data_member)
{
        const struct btf_type *kern_type, *kern_vtype;
        const struct btf_member *kern_data_member;
        __s32 kern_vtype_id, kern_type_id;
        __u32 i;

        kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
        if (kern_type_id < 0) {
                pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
                        tname);
                return kern_type_id;
        }
        kern_type = btf__type_by_id(btf, kern_type_id);

        /* Find the corresponding "map_value" type that will be used
         * in map_update(BPF_MAP_TYPE_STRUCT_OPS).  For example,
         * find "struct bpf_struct_ops_tcp_congestion_ops" from the
         * btf_vmlinux.
         */
        kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
                                                tname, BTF_KIND_STRUCT);
        if (kern_vtype_id < 0) {
                pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
                        STRUCT_OPS_VALUE_PREFIX, tname);
                return kern_vtype_id;
        }
        kern_vtype = btf__type_by_id(btf, kern_vtype_id);

        /* Find "struct tcp_congestion_ops" from
         * struct bpf_struct_ops_tcp_congestion_ops {
         *      [ ... ]
         *      struct tcp_congestion_ops data;
         * }
         */
        kern_data_member = btf_members(kern_vtype);
        for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
                if (kern_data_member->type == kern_type_id)
                        break;
        }
        if (i == btf_vlen(kern_vtype)) {
                pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
                        tname, STRUCT_OPS_VALUE_PREFIX, tname);
                return -EINVAL;
        }

        *type = kern_type;
        *type_id = kern_type_id;
        *vtype = kern_vtype;
        *vtype_id = kern_vtype_id;
        *data_member = kern_data_member;

        return 0;
}

static bool bpf_map__is_struct_ops(const struct bpf_map *map)
{
        return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
}

/* Init the map's fields that depend on kern_btf */
static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
                                         const struct btf *btf,
                                         const struct btf *kern_btf)
{
        const struct btf_member *member, *kern_member, *kern_data_member;
        const struct btf_type *type, *kern_type, *kern_vtype;
        __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
        struct bpf_struct_ops *st_ops;
        void *data, *kern_data;
        const char *tname;
        int err;

        st_ops = map->st_ops;
        type = st_ops->type;
        tname = st_ops->tname;
        err = find_struct_ops_kern_types(kern_btf, tname,
                                         &kern_type, &kern_type_id,
                                         &kern_vtype, &kern_vtype_id,
                                         &kern_data_member);
        if (err)
                return err;

        pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
                 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);

        map->def.value_size = kern_vtype->size;
        map->btf_vmlinux_value_type_id = kern_vtype_id;

        st_ops->kern_vdata = calloc(1, kern_vtype->size);
        if (!st_ops->kern_vdata)
                return -ENOMEM;

        data = st_ops->data;
        kern_data_off = kern_data_member->offset / 8;
        kern_data = st_ops->kern_vdata + kern_data_off;

        member = btf_members(type);
        for (i = 0; i < btf_vlen(type); i++, member++) {
                const struct btf_type *mtype, *kern_mtype;
                __u32 mtype_id, kern_mtype_id;
                void *mdata, *kern_mdata;
                __s64 msize, kern_msize;
                __u32 moff, kern_moff;
                __u32 kern_member_idx;
                const char *mname;

                mname = btf__name_by_offset(btf, member->name_off);
                kern_member = find_member_by_name(kern_btf, kern_type, mname);
                if (!kern_member) {
                        pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
                                map->name, mname);
                        return -ENOTSUP;
                }

                kern_member_idx = kern_member - btf_members(kern_type);
                if (btf_member_bitfield_size(type, i) ||
                    btf_member_bitfield_size(kern_type, kern_member_idx)) {
                        pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
                                map->name, mname);
                        return -ENOTSUP;
                }

                moff = member->offset / 8;
                kern_moff = kern_member->offset / 8;

                mdata = data + moff;
                kern_mdata = kern_data + kern_moff;

                mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
                kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
                                                    &kern_mtype_id);
                if (BTF_INFO_KIND(mtype->info) !=
                    BTF_INFO_KIND(kern_mtype->info)) {
                        pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
                                map->name, mname, BTF_INFO_KIND(mtype->info),
                                BTF_INFO_KIND(kern_mtype->info));
                        return -ENOTSUP;
                }

                if (btf_is_ptr(mtype)) {
                        struct bpf_program *prog;

                        mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id);
                        kern_mtype = skip_mods_and_typedefs(kern_btf,
                                                            kern_mtype->type,
                                                            &kern_mtype_id);
                        if (!btf_is_func_proto(mtype) ||
                            !btf_is_func_proto(kern_mtype)) {
                                pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n",
                                        map->name, mname);
                                return -ENOTSUP;
                        }

                        prog = st_ops->progs[i];
                        if (!prog) {
                                pr_debug("struct_ops init_kern %s: func ptr %s is not set\n",
                                         map->name, mname);
                                continue;
                        }

                        prog->attach_btf_id = kern_type_id;
                        prog->expected_attach_type = kern_member_idx;

                        st_ops->kern_func_off[i] = kern_data_off + kern_moff;

                        pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
                                 map->name, mname, prog->name, moff,
                                 kern_moff);

                        continue;
                }

                msize = btf__resolve_size(btf, mtype_id);
                kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
                if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
                        pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
                                map->name, mname, (ssize_t)msize,
                                (ssize_t)kern_msize);
                        return -ENOTSUP;
                }

                pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
                         map->name, mname, (unsigned int)msize,
                         moff, kern_moff);
                memcpy(kern_mdata, mdata, msize);
        }

        return 0;
}

static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
{
        struct bpf_map *map;
        size_t i;
        int err;

        for (i = 0; i < obj->nr_maps; i++) {
                map = &obj->maps[i];

                if (!bpf_map__is_struct_ops(map))
                        continue;

                err = bpf_map__init_kern_struct_ops(map, obj->btf,
                                                    obj->btf_vmlinux);
                if (err)
                        return err;
        }

        return 0;
}

static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
{
        const struct btf_type *type, *datasec;
        const struct btf_var_secinfo *vsi;
        struct bpf_struct_ops *st_ops;
        const char *tname, *var_name;
        __s32 type_id, datasec_id;
        const struct btf *btf;
        struct bpf_map *map;
        __u32 i;

        if (obj->efile.st_ops_shndx == -1)
                return 0;

        btf = obj->btf;
        datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
                                            BTF_KIND_DATASEC);
        if (datasec_id < 0) {
                pr_warn("struct_ops init: DATASEC %s not found\n",
                        STRUCT_OPS_SEC);
                return -EINVAL;
        }

        datasec = btf__type_by_id(btf, datasec_id);
        vsi = btf_var_secinfos(datasec);
        for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
                type = btf__type_by_id(obj->btf, vsi->type);
                var_name = btf__name_by_offset(obj->btf, type->name_off);

                type_id = btf__resolve_type(obj->btf, vsi->type);
                if (type_id < 0) {
                        pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
                                vsi->type, STRUCT_OPS_SEC);
                        return -EINVAL;
                }

                type = btf__type_by_id(obj->btf, type_id);
                tname = btf__name_by_offset(obj->btf, type->name_off);
                if (!tname[0]) {
                        pr_warn("struct_ops init: anonymous type is not supported\n");
                        return -ENOTSUP;
                }
                if (!btf_is_struct(type)) {
                        pr_warn("struct_ops init: %s is not a struct\n", tname);
                        return -EINVAL;
                }

                map = bpf_object__add_map(obj);
                if (IS_ERR(map))
                        return PTR_ERR(map);

                map->sec_idx = obj->efile.st_ops_shndx;
                map->sec_offset = vsi->offset;
                map->name = strdup(var_name);
                if (!map->name)
                        return -ENOMEM;

                map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
                map->def.key_size = sizeof(int);
                map->def.value_size = type->size;
                map->def.max_entries = 1;

                map->st_ops = calloc(1, sizeof(*map->st_ops));
                if (!map->st_ops)

                        return -ENOMEM;
                st_ops = map->st_ops;
                st_ops->data = malloc(type->size);
                st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
                st_ops->kern_func_off = malloc(btf_vlen(type) *
                                               sizeof(*st_ops->kern_func_off));
                if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
                        return -ENOMEM;

                if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
                        pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
                                var_name, STRUCT_OPS_SEC);
                        return -EINVAL;
                }

                memcpy(st_ops->data,
                       obj->efile.st_ops_data->d_buf + vsi->offset,
                       type->size);
                st_ops->tname = tname;
                st_ops->type = type;
                st_ops->type_id = type_id;

                pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
                         tname, type_id, var_name, vsi->offset);
        }

        return 0;
}

static struct bpf_object *bpf_object__new(const char *path,
                                          const void *obj_buf,
                                          size_t obj_buf_sz,
                                          const char *obj_name)
{
        struct bpf_object *obj;
        char *end;

        obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
        if (!obj) {
                pr_warn("alloc memory failed for %s\n", path);
                return ERR_PTR(-ENOMEM);
        }

        strcpy(obj->path, path);
        if (obj_name) {
                strncpy(obj->name, obj_name, sizeof(obj->name) - 1);
                obj->name[sizeof(obj->name) - 1] = 0;
        } else {
                /* Using basename() GNU version which doesn't modify arg. */
                strncpy(obj->name, basename((void *)path),
                        sizeof(obj->name) - 1);
                end = strchr(obj->name, '.');
                if (end)
                        *end = 0;
        }

        obj->efile.fd = -1;
        /*
         * Caller of this function should also call
         * bpf_object__elf_finish() after data collection to return
         * obj_buf to user. If not, we should duplicate the buffer to
         * avoid user freeing them before elf finish.
         */
        obj->efile.obj_buf = obj_buf;
        obj->efile.obj_buf_sz = obj_buf_sz;
        obj->efile.maps_shndx = -1;
        obj->efile.btf_maps_shndx = -1;
        obj->efile.data_shndx = -1;
        obj->efile.rodata_shndx = -1;
        obj->efile.bss_shndx = -1;
        obj->efile.st_ops_shndx = -1;
        obj->kconfig_map_idx = -1;
        obj->rodata_map_idx = -1;

        obj->kern_version = get_kernel_version();
        obj->loaded = false;

        INIT_LIST_HEAD(&obj->list);
        list_add(&obj->list, &bpf_objects_list);
        return obj;
}

static void bpf_object__elf_finish(struct bpf_object *obj)
{
        if (!obj_elf_valid(obj))
                return;

        if (obj->efile.elf) {
                elf_end(obj->efile.elf);
                obj->efile.elf = NULL;
        }
        obj->efile.symbols = NULL;
        obj->efile.data = NULL;
        obj->efile.rodata = NULL;
        obj->efile.bss = NULL;
        obj->efile.st_ops_data = NULL;

        zfree(&obj->efile.reloc_sects);
        obj->efile.nr_reloc_sects = 0;
        zclose(obj->efile.fd);
        obj->efile.obj_buf = NULL;
        obj->efile.obj_buf_sz = 0;
}

/* if libelf is old and doesn't support mmap(), fall back to read() */
#ifndef ELF_C_READ_MMAP
#define ELF_C_READ_MMAP ELF_C_READ
#endif

static int bpf_object__elf_init(struct bpf_object *obj)
{
        int err = 0;
        GElf_Ehdr *ep;

        if (obj_elf_valid(obj)) {
                pr_warn("elf: init internal error\n");
                return -LIBBPF_ERRNO__LIBELF;
        }

        if (obj->efile.obj_buf_sz > 0) {
                /*
                 * obj_buf should have been validated by
                 * bpf_object__open_buffer().
                 */
                obj->efile.elf = elf_memory((char *)obj->efile.obj_buf,
                                            obj->efile.obj_buf_sz);
        } else {
                obj->efile.fd = open(obj->path, O_RDONLY);
                if (obj->efile.fd < 0) {
                        char errmsg[STRERR_BUFSIZE], *cp;

                        err = -errno;
                        cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
                        pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
                        return err;
                }

                obj->efile.elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
        }

        if (!obj->efile.elf) {
                pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
                err = -LIBBPF_ERRNO__LIBELF;
                goto errout;
        }

        if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
                pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
                err = -LIBBPF_ERRNO__FORMAT;
                goto errout;
        }
        ep = &obj->efile.ehdr;

        if (elf_getshdrstrndx(obj->efile.elf, &obj->efile.shstrndx)) {
                pr_warn("elf: failed to get section names section index for %s: %s\n",
                        obj->path, elf_errmsg(-1));
                err = -LIBBPF_ERRNO__FORMAT;
                goto errout;
        }

        /* Elf is corrupted/truncated, avoid calling elf_strptr. */
        if (!elf_rawdata(elf_getscn(obj->efile.elf, obj->efile.shstrndx), NULL)) {
                pr_warn("elf: failed to get section names strings from %s: %s\n",
                        obj->path, elf_errmsg(-1));
                return -LIBBPF_ERRNO__FORMAT;
        }

        /* Old LLVM set e_machine to EM_NONE */
        if (ep->e_type != ET_REL ||
            (ep->e_machine && ep->e_machine != EM_BPF)) {
                pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
                err = -LIBBPF_ERRNO__FORMAT;
                goto errout;
        }

        return 0;
errout:
        bpf_object__elf_finish(obj);
        return err;
}

static int bpf_object__check_endianness(struct bpf_object *obj)
{
#if __BYTE_ORDER == __LITTLE_ENDIAN
        if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB)
                return 0;
#elif __BYTE_ORDER == __BIG_ENDIAN
        if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB)
                return 0;
#else
# error "Unrecognized __BYTE_ORDER__"
#endif
        pr_warn("elf: endianness mismatch in %s.\n", obj->path);
        return -LIBBPF_ERRNO__ENDIAN;
}

static int
bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
{
        memcpy(obj->license, data, min(size, sizeof(obj->license) - 1));
        pr_debug("license of %s is %s\n", obj->path, obj->license);
        return 0;
}

static int
bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
{
        __u32 kver;

        if (size != sizeof(kver)) {
                pr_warn("invalid kver section in %s\n", obj->path);
                return -LIBBPF_ERRNO__FORMAT;
        }
        memcpy(&kver, data, sizeof(kver));
        obj->kern_version = kver;
        pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
        return 0;
}

static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
{
        if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
            type == BPF_MAP_TYPE_HASH_OF_MAPS)
                return true;
        return false;
}

int bpf_object__section_size(const struct bpf_object *obj, const char *name,
                             __u32 *size)
{
        int ret = -ENOENT;

        *size = 0;
        if (!name) {
                return -EINVAL;
        } else if (!strcmp(name, DATA_SEC)) {
                if (obj->efile.data)
                        *size = obj->efile.data->d_size;
        } else if (!strcmp(name, BSS_SEC)) {
                if (obj->efile.bss)
                        *size = obj->efile.bss->d_size;
        } else if (!strcmp(name, RODATA_SEC)) {
                if (obj->efile.rodata)
                        *size = obj->efile.rodata->d_size;
        } else if (!strcmp(name, STRUCT_OPS_SEC)) {
                if (obj->efile.st_ops_data)
                        *size = obj->efile.st_ops_data->d_size;
        } else {
                Elf_Scn *scn = elf_sec_by_name(obj, name);
                Elf_Data *data = elf_sec_data(obj, scn);

                if (data) {
                        ret = 0; /* found it */
                        *size = data->d_size;
                }
        }

        return *size ? 0 : ret;
}

int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
                                __u32 *off)
{
        Elf_Data *symbols = obj->efile.symbols;
        const char *sname;
        size_t si;

        if (!name || !off)
                return -EINVAL;

        for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) {
                GElf_Sym sym;

                if (!gelf_getsym(symbols, si, &sym))
                        continue;
                if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
                    GELF_ST_TYPE(sym.st_info) != STT_OBJECT)
                        continue;

                sname = elf_sym_str(obj, sym.st_name);
                if (!sname) {
                        pr_warn("failed to get sym name string for var %s\n",
                                name);
                        return -EIO;
                }
                if (strcmp(name, sname) == 0) {
                        *off = sym.st_value;
                        return 0;
                }
        }

        return -ENOENT;
}

static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
{
        struct bpf_map *new_maps;
        size_t new_cap;
        int i;

        if (obj->nr_maps < obj->maps_cap)
                return &obj->maps[obj->nr_maps++];

        new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
        new_maps = libbpf_reallocarray(obj->maps, new_cap, sizeof(*obj->maps));
        if (!new_maps) {
                pr_warn("alloc maps for object failed\n");
                return ERR_PTR(-ENOMEM);
        }

        obj->maps_cap = new_cap;
        obj->maps = new_maps;

        /* zero out new maps */
        memset(obj->maps + obj->nr_maps, 0,
               (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps));
        /*
         * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin)
         * when failure (zclose won't close negative fd)).
         */
        for (i = obj->nr_maps; i < obj->maps_cap; i++) {
                obj->maps[i].fd = -1;
                obj->maps[i].inner_map_fd = -1;
        }

        return &obj->maps[obj->nr_maps++];
}

static size_t bpf_map_mmap_sz(const struct bpf_map *map)
{
        long page_sz = sysconf(_SC_PAGE_SIZE);
        size_t map_sz;

        map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries;
        map_sz = roundup(map_sz, page_sz);
        return map_sz;
}

static char *internal_map_name(struct bpf_object *obj,
                               enum libbpf_map_type type)
{
        char map_name[BPF_OBJ_NAME_LEN], *p;
        const char *sfx = libbpf_type_to_btf_name[type];
        int sfx_len = max((size_t)7, strlen(sfx));
        int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1,
                          strlen(obj->name));

        snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
                 sfx_len, libbpf_type_to_btf_name[type]);

        /* sanitise map name to characters allowed by kernel */
        for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
                if (!isalnum(*p) && *p != '_' && *p != '.')
                        *p = '_';

        return strdup(map_name);
}

static int
bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
                              int sec_idx, void *data, size_t data_sz)
{
        struct bpf_map_def *def;
        struct bpf_map *map;
        int err;

        map = bpf_object__add_map(obj);
        if (IS_ERR(map))
                return PTR_ERR(map);

        map->libbpf_type = type;
        map->sec_idx = sec_idx;
        map->sec_offset = 0;
        map->name = internal_map_name(obj, type);
        if (!map->name) {
                pr_warn("failed to alloc map name\n");
                return -ENOMEM;
        }

        def = &map->def;
        def->type = BPF_MAP_TYPE_ARRAY;
        def->key_size = sizeof(int);
        def->value_size = data_sz;
        def->max_entries = 1;
        def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
                         ? BPF_F_RDONLY_PROG : 0;
        def->map_flags |= BPF_F_MMAPABLE;

        pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
                 map->name, map->sec_idx, map->sec_offset, def->map_flags);

        map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
                           MAP_SHARED | MAP_ANONYMOUS, -1, 0);
        if (map->mmaped == MAP_FAILED) {
                err = -errno;
                map->mmaped = NULL;
                pr_warn("failed to alloc map '%s' content buffer: %d\n",
                        map->name, err);
                zfree(&map->name);
                return err;
        }

        if (data)
                memcpy(map->mmaped, data, data_sz);

        pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
        return 0;
}

static int bpf_object__init_global_data_maps(struct bpf_object *obj)
{
        int err;

        /*
         * Populate obj->maps with libbpf internal maps.
         */
        if (obj->efile.data_shndx >= 0) {
                err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
                                                    obj->efile.data_shndx,
                                                    obj->efile.data->d_buf,
                                                    obj->efile.data->d_size);
                if (err)
                        return err;
        }
        if (obj->efile.rodata_shndx >= 0) {
                err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
                                                    obj->efile.rodata_shndx,
                                                    obj->efile.rodata->d_buf,
                                                    obj->efile.rodata->d_size);
                if (err)
                        return err;

                obj->rodata_map_idx = obj->nr_maps - 1;
        }
        if (obj->efile.bss_shndx >= 0) {
                err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
                                                    obj->efile.bss_shndx,
                                                    NULL,
                                                    obj->efile.bss->d_size);
                if (err)
                        return err;
        }
        return 0;
}


static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
                                               const void *name)
{
        int i;

        for (i = 0; i < obj->nr_extern; i++) {
                if (strcmp(obj->externs[i].name, name) == 0)
                        return &obj->externs[i];
        }
        return NULL;
}

static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
                              char value)
{
        switch (ext->kcfg.type) {
        case KCFG_BOOL:
                if (value == 'm') {
                        pr_warn("extern (kcfg) %s=%c should be tristate or char\n",
                                ext->name, value);
                        return -EINVAL;
                }
                *(bool *)ext_val = value == 'y' ? true : false;
                break;
        case KCFG_TRISTATE:
                if (value == 'y')
                        *(enum libbpf_tristate *)ext_val = TRI_YES;
                else if (value == 'm')
                        *(enum libbpf_tristate *)ext_val = TRI_MODULE;
                else /* value == 'n' */
                        *(enum libbpf_tristate *)ext_val = TRI_NO;
                break;
        case KCFG_CHAR:
                *(char *)ext_val = value;
                break;
        case KCFG_UNKNOWN:
        case KCFG_INT:
        case KCFG_CHAR_ARR:
        default:
                pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n",
                        ext->name, value);
                return -EINVAL;
        }
        ext->is_set = true;
        return 0;
}

static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
                              const char *value)
{
        size_t len;

        if (ext->kcfg.type != KCFG_CHAR_ARR) {
                pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value);
                return -EINVAL;
        }

        len = strlen(value);
        if (value[len - 1] != '"') {
                pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
                        ext->name, value);
                return -EINVAL;
        }

        /* strip quotes */
        len -= 2;
        if (len >= ext->kcfg.sz) {
                pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n",
                        ext->name, value, len, ext->kcfg.sz - 1);
                len = ext->kcfg.sz - 1;
        }
        memcpy(ext_val, value + 1, len);
        ext_val[len] = '\0';
        ext->is_set = true;
        return 0;
}

static int parse_u64(const char *value, __u64 *res)
{
        char *value_end;
        int err;

        errno = 0;
        *res = strtoull(value, &value_end, 0);
        if (errno) {
                err = -errno;
                pr_warn("failed to parse '%s' as integer: %d\n", value, err);
                return err;
        }
        if (*value_end) {
                pr_warn("failed to parse '%s' as integer completely\n", value);
                return -EINVAL;
        }
        return 0;
}

static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
{
        int bit_sz = ext->kcfg.sz * 8;

        if (ext->kcfg.sz == 8)
                return true;

        /* Validate that value stored in u64 fits in integer of `ext->sz`
         * bytes size without any loss of information. If the target integer
         * is signed, we rely on the following limits of integer type of
         * Y bits and subsequent transformation:
         *
         *     -2^(Y-1) <= X           <= 2^(Y-1) - 1
         *            0 <= X + 2^(Y-1) <= 2^Y - 1
         *            0 <= X + 2^(Y-1) <  2^Y
         *
         *  For unsigned target integer, check that all the (64 - Y) bits are
         *  zero.
         */
        if (ext->kcfg.is_signed)
                return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
        else
                return (v >> bit_sz) == 0;
}

static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
                              __u64 value)
{
        if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
                pr_warn("extern (kcfg) %s=%llu should be integer\n",
                        ext->name, (unsigned long long)value);
                return -EINVAL;
        }
        if (!is_kcfg_value_in_range(ext, value)) {
                pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n",
                        ext->name, (unsigned long long)value, ext->kcfg.sz);
                return -ERANGE;
        }
        switch (ext->kcfg.sz) {
                case 1: *(__u8 *)ext_val = value; break;
                case 2: *(__u16 *)ext_val = value; break;
                case 4: *(__u32 *)ext_val = value; break;
                case 8: *(__u64 *)ext_val = value; break;
                default:
                        return -EINVAL;
        }
        ext->is_set = true;
        return 0;
}

static int bpf_object__process_kconfig_line(struct bpf_object *obj,
                                            char *buf, void *data)
{
        struct extern_desc *ext;
        char *sep, *value;
        int len, err = 0;
        void *ext_val;
        __u64 num;

        if (strncmp(buf, "CONFIG_", 7))
                return 0;

        sep = strchr(buf, '=');
        if (!sep) {
                pr_warn("failed to parse '%s': no separator\n", buf);
                return -EINVAL;
        }

        /* Trim ending '\n' */
        len = strlen(buf);
        if (buf[len - 1] == '\n')
                buf[len - 1] = '\0';
        /* Split on '=' and ensure that a value is present. */
        *sep = '\0';
        if (!sep[1]) {
                *sep = '=';
                pr_warn("failed to parse '%s': no value\n", buf);
                return -EINVAL;
        }

        ext = find_extern_by_name(obj, buf);
        if (!ext || ext->is_set)
                return 0;

        ext_val = data + ext->kcfg.data_off;
        value = sep + 1;

        switch (*value) {
        case 'y': case 'n': case 'm':
                err = set_kcfg_value_tri(ext, ext_val, *value);
                break;
        case '"':
                err = set_kcfg_value_str(ext, ext_val, value);
                break;
        default:
                /* assume integer */
                err = parse_u64(value, &num);
                if (err) {
                        pr_warn("extern (kcfg) %s=%s should be integer\n",
                                ext->name, value);
                        return err;
                }
                err = set_kcfg_value_num(ext, ext_val, num);
                break;
        }
        if (err)
                return err;
        pr_debug("extern (kcfg) %s=%s\n", ext->name, value);
        return 0;
}

static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
{
        char buf[PATH_MAX];
        struct utsname uts;
        int len, err = 0;
        gzFile file;

        uname(&uts);
        len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
        if (len < 0)
                return -EINVAL;
        else if (len >= PATH_MAX)
                return -ENAMETOOLONG;

        /* gzopen also accepts uncompressed files. */
        file = gzopen(buf, "r");
        if (!file)
                file = gzopen("/proc/config.gz", "r");

        if (!file) {
                pr_warn("failed to open system Kconfig\n");
                return -ENOENT;
        }

        while (gzgets(file, buf, sizeof(buf))) {
                err = bpf_object__process_kconfig_line(obj, buf, data);
                if (err) {
                        pr_warn("error parsing system Kconfig line '%s': %d\n",
                                buf, err);
                        goto out;
                }
        }

out:
        gzclose(file);
        return err;
}

static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
                                        const char *config, void *data)
{
        char buf[PATH_MAX];
        int err = 0;
        FILE *file;

        file = fmemopen((void *)config, strlen(config), "r");
        if (!file) {
                err = -errno;
                pr_warn("failed to open in-memory Kconfig: %d\n", err);
                return err;
        }

        while (fgets(buf, sizeof(buf), file)) {
                err = bpf_object__process_kconfig_line(obj, buf, data);
                if (err) {
                        pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
                                buf, err);
                        break;
                }
        }

        fclose(file);
        return err;
}

static int bpf_object__init_kconfig_map(struct bpf_object *obj)
{
        struct extern_desc *last_ext = NULL, *ext;
        size_t map_sz;
        int i, err;

        for (i = 0; i < obj->nr_extern; i++) {
                ext = &obj->externs[i];
                if (ext->type == EXT_KCFG)
                        last_ext = ext;
        }

        if (!last_ext)
                return 0;

        map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
        err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
                                            obj->efile.symbols_shndx,
                                            NULL, map_sz);
        if (err)
                return err;

        obj->kconfig_map_idx = obj->nr_maps - 1;

        return 0;
}

static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
{
        Elf_Data *symbols = obj->efile.symbols;
        int i, map_def_sz = 0, nr_maps = 0, nr_syms;
        Elf_Data *data = NULL;
        Elf_Scn *scn;

        if (obj->efile.maps_shndx < 0)
                return 0;

        if (!symbols)
                return -EINVAL;


        scn = elf_sec_by_idx(obj, obj->efile.maps_shndx);
        data = elf_sec_data(obj, scn);
        if (!scn || !data) {
                pr_warn("elf: failed to get legacy map definitions for %s\n",
                        obj->path);
                return -EINVAL;
        }

        /*
         * Count number of maps. Each map has a name.
         * Array of maps is not supported: only the first element is
         * considered.
         *
         * TODO: Detect array of map and report error.
         */
        nr_syms = symbols->d_size / sizeof(GElf_Sym);
        for (i = 0; i < nr_syms; i++) {
                GElf_Sym sym;

                if (!gelf_getsym(symbols, i, &sym))
                        continue;
                if (sym.st_shndx != obj->efile.maps_shndx)
                        continue;
                nr_maps++;
        }
        /* Assume equally sized map definitions */
        pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n",
                 nr_maps, data->d_size, obj->path);

        if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {
                pr_warn("elf: unable to determine legacy map definition size in %s\n",
                        obj->path);
                return -EINVAL;
        }
        map_def_sz = data->d_size / nr_maps;

        /* Fill obj->maps using data in "maps" section.  */
        for (i = 0; i < nr_syms; i++) {
                GElf_Sym sym;
                const char *map_name;
                struct bpf_map_def *def;
                struct bpf_map *map;

                if (!gelf_getsym(symbols, i, &sym))
                        continue;
                if (sym.st_shndx != obj->efile.maps_shndx)
                        continue;

                map = bpf_object__add_map(obj);
                if (IS_ERR(map))
                        return PTR_ERR(map);

                map_name = elf_sym_str(obj, sym.st_name);
                if (!map_name) {
                        pr_warn("failed to get map #%d name sym string for obj %s\n",
                                i, obj->path);
                        return -LIBBPF_ERRNO__FORMAT;
                }

                map->libbpf_type = LIBBPF_MAP_UNSPEC;
                map->sec_idx = sym.st_shndx;
                map->sec_offset = sym.st_value;
                pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
                         map_name, map->sec_idx, map->sec_offset);
                if (sym.st_value + map_def_sz > data->d_size) {
                        pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
                                obj->path, map_name);
                        return -EINVAL;
                }

                map->name = strdup(map_name);
                if (!map->name) {
                        pr_warn("failed to alloc map name\n");
                        return -ENOMEM;
                }
                pr_debug("map %d is \"%s\"\n", i, map->name);
                def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
                /*
                 * If the definition of the map in the object file fits in
                 * bpf_map_def, copy it.  Any extra fields in our version
                 * of bpf_map_def will default to zero as a result of the
                 * calloc above.
                 */
                if (map_def_sz <= sizeof(struct bpf_map_def)) {
                        memcpy(&map->def, def, map_def_sz);
                } else {
                        /*
                         * Here the map structure being read is bigger than what
                         * we expect, truncate if the excess bits are all zero.
                         * If they are not zero, reject this map as
                         * incompatible.
                         */
                        char *b;

                        for (b = ((char *)def) + sizeof(struct bpf_map_def);
                             b < ((char *)def) + map_def_sz; b++) {
                                if (*b != 0) {
                                        pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",
                                                obj->path, map_name);
                                        if (strict)
                                                return -EINVAL;
                                }
                        }
                        memcpy(&map->def, def, sizeof(struct bpf_map_def));
                }
        }
        return 0;
}

static const struct btf_type *
skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
{
        const struct btf_type *t = btf__type_by_id(btf, id);

        if (res_id)
                *res_id = id;

        while (btf_is_mod(t) || btf_is_typedef(t)) {
                if (res_id)
                        *res_id = t->type;
                t = btf__type_by_id(btf, t->type);
        }

        return t;
}

static const struct btf_type *
resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
{
        const struct btf_type *t;

        t = skip_mods_and_typedefs(btf, id, NULL);
        if (!btf_is_ptr(t))
                return NULL;

        t = skip_mods_and_typedefs(btf, t->type, res_id);

        return btf_is_func_proto(t) ? t : NULL;
}

static const char *btf_kind_str(const struct btf_type *t)
{
        switch (btf_kind(t)) {
        case BTF_KIND_UNKN: return "void";
        case BTF_KIND_INT: return "int";
        case BTF_KIND_PTR: return "ptr";
        case BTF_KIND_ARRAY: return "array";
        case BTF_KIND_STRUCT: return "struct";
        case BTF_KIND_UNION: return "union";
        case BTF_KIND_ENUM: return "enum";
        case BTF_KIND_FWD: return "fwd";
        case BTF_KIND_TYPEDEF: return "typedef";
        case BTF_KIND_VOLATILE: return "volatile";
        case BTF_KIND_CONST: return "const";
        case BTF_KIND_RESTRICT: return "restrict";
        case BTF_KIND_FUNC: return "func";
        case BTF_KIND_FUNC_PROTO: return "func_proto";
        case BTF_KIND_VAR: return "var";
        case BTF_KIND_DATASEC: return "datasec";
        default: return "unknown";
        }
}

/*
 * Fetch integer attribute of BTF map definition. Such attributes are
 * represented using a pointer to an array, in which dimensionality of array
 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
 * type definition, while using only sizeof(void *) space in ELF data section.
 */
static bool get_map_field_int(const char *map_name, const struct btf *btf,
                              const struct btf_member *m, __u32 *res)
{
        const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
        const char *name = btf__name_by_offset(btf, m->name_off);
        const struct btf_array *arr_info;
        const struct btf_type *arr_t;

        if (!btf_is_ptr(t)) {
                pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
                        map_name, name, btf_kind_str(t));
                return false;
        }

        arr_t = btf__type_by_id(btf, t->type);
        if (!arr_t) {
                pr_warn("map '%s': attr '%s': type [%u] not found.\n",
                        map_name, name, t->type);
                return false;
        }
        if (!btf_is_array(arr_t)) {
                pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
                        map_name, name, btf_kind_str(arr_t));
                return false;
        }
        arr_info = btf_array(arr_t);
        *res = arr_info->nelems;
        return true;
}

static int build_map_pin_path(struct bpf_map *map, const char *path)
{
        char buf[PATH_MAX];
        int len;

        if (!path)
                path = "/sys/fs/bpf";

        len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
        if (len < 0)
                return -EINVAL;
        else if (len >= PATH_MAX)
                return -ENAMETOOLONG;

        return bpf_map__set_pin_path(map, buf);
}


static int parse_btf_map_def(struct bpf_object *obj,
                             struct bpf_map *map,
                             const struct btf_type *def,
                             bool strict, bool is_inner,
                             const char *pin_root_path)
{
        const struct btf_type *t;
        const struct btf_member *m;
        int vlen, i;

        vlen = btf_vlen(def);
        m = btf_members(def);
        for (i = 0; i < vlen; i++, m++) {
                const char *name = btf__name_by_offset(obj->btf, m->name_off);

                if (!name) {
                        pr_warn("map '%s': invalid field #%d.\n", map->name, i);
                        return -EINVAL;
                }
                if (strcmp(name, "type") == 0) {
                        if (!get_map_field_int(map->name, obj->btf, m,
                                               &map->def.type))
                                return -EINVAL;

                        pr_debug("map '%s': found type = %u.\n",
                                 map->name, map->def.type);
                } else if (strcmp(name, "max_entries") == 0) {
                        if (!get_map_field_int(map->name, obj->btf, m,
                                               &map->def.max_entries))
                                return -EINVAL;
                        pr_debug("map '%s': found max_entries = %u.\n",
                                 map->name, map->def.max_entries);
                } else if (strcmp(name, "map_flags") == 0) {
                        if (!get_map_field_int(map->name, obj->btf, m,
                                               &map->def.map_flags))
                                return -EINVAL;
                        pr_debug("map '%s': found map_flags = %u.\n",
                                 map->name, map->def.map_flags);
                } else if (strcmp(name, "numa_node") == 0) {
                        if (!get_map_field_int(map->name, obj->btf, m, &map->numa_node))
                                return -EINVAL;
                        pr_debug("map '%s': found numa_node = %u.\n", map->name, map->numa_node);
                } else if (strcmp(name, "key_size") == 0) {
                        __u32 sz;

                        if (!get_map_field_int(map->name, obj->btf, m, &sz))
                                return -EINVAL;
                        pr_debug("map '%s': found key_size = %u.\n",
                                 map->name, sz);
                        if (map->def.key_size && map->def.key_size != sz) {
                                pr_warn("map '%s': conflicting key size %u != %u.\n",
                                        map->name, map->def.key_size, sz);
                                return -EINVAL;
                        }
                        map->def.key_size = sz;
                } else if (strcmp(name, "key") == 0) {
                        __s64 sz;

                        t = btf__type_by_id(obj->btf, m->type);
                        if (!t) {
                                pr_warn("map '%s': key type [%d] not found.\n",
                                        map->name, m->type);
                                return -EINVAL;
                        }
                        if (!btf_is_ptr(t)) {
                                pr_warn("map '%s': key spec is not PTR: %s.\n",
                                        map->name, btf_kind_str(t));
                                return -EINVAL;
                        }
                        sz = btf__resolve_size(obj->btf, t->type);
                        if (sz < 0) {
                                pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
                                        map->name, t->type, (ssize_t)sz);
                                return sz;
                        }
                        pr_debug("map '%s': found key [%u], sz = %zd.\n",
                                 map->name, t->type, (ssize_t)sz);
                        if (map->def.key_size && map->def.key_size != sz) {
                                pr_warn("map '%s': conflicting key size %u != %zd.\n",
                                        map->name, map->def.key_size, (ssize_t)sz);
                                return -EINVAL;
                        }
                        map->def.key_size = sz;
                        map->btf_key_type_id = t->type;
                } else if (strcmp(name, "value_size") == 0) {
                        __u32 sz;

                        if (!get_map_field_int(map->name, obj->btf, m, &sz))
                                return -EINVAL;
                        pr_debug("map '%s': found value_size = %u.\n",
                                 map->name, sz);
                        if (map->def.value_size && map->def.value_size != sz) {
                                pr_warn("map '%s': conflicting value size %u != %u.\n",
                                        map->name, map->def.value_size, sz);
                                return -EINVAL;
                        }
                        map->def.value_size = sz;
                } else if (strcmp(name, "value") == 0) {
                        __s64 sz;

                        t = btf__type_by_id(obj->btf, m->type);
                        if (!t) {
                                pr_warn("map '%s': value type [%d] not found.\n",
                                        map->name, m->type);
                                return -EINVAL;
                        }
                        if (!btf_is_ptr(t)) {
                                pr_warn("map '%s': value spec is not PTR: %s.\n",
                                        map->name, btf_kind_str(t));
                                return -EINVAL;
                        }
                        sz = btf__resolve_size(obj->btf, t->type);
                        if (sz < 0) {
                                pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
                                        map->name, t->type, (ssize_t)sz);
                                return sz;
                        }
                        pr_debug("map '%s': found value [%u], sz = %zd.\n",
                                 map->name, t->type, (ssize_t)sz);
                        if (map->def.value_size && map->def.value_size != sz) {
                                pr_warn("map '%s': conflicting value size %u != %zd.\n",
                                        map->name, map->def.value_size, (ssize_t)sz);
                                return -EINVAL;
                        }
                        map->def.value_size = sz;
                        map->btf_value_type_id = t->type;
                }
                else if (strcmp(name, "values") == 0) {
                        int err;

                        if (is_inner) {
                                pr_warn("map '%s': multi-level inner maps not supported.\n",
                                        map->name);
                                return -ENOTSUP;
                        }
                        if (i != vlen - 1) {
                                pr_warn("map '%s': '%s' member should be last.\n",
                                        map->name, name);
                                return -EINVAL;
                        }
                        if (!bpf_map_type__is_map_in_map(map->def.type)) {
                                pr_warn("map '%s': should be map-in-map.\n",
                                        map->name);
                                return -ENOTSUP;
                        }
                        if (map->def.value_size && map->def.value_size != 4) {
                                pr_warn("map '%s': conflicting value size %u != 4.\n",
                                        map->name, map->def.value_size);
                                return -EINVAL;
                        }
                        map->def.value_size = 4;
                        t = btf__type_by_id(obj->btf, m->type);
                        if (!t) {
                                pr_warn("map '%s': map-in-map inner type [%d] not found.\n",
                                        map->name, m->type);
                                return -EINVAL;
                        }
                        if (!btf_is_array(t) || btf_array(t)->nelems) {
                                pr_warn("map '%s': map-in-map inner spec is not a zero-sized array.\n",
                                        map->name);
                                return -EINVAL;
                        }
                        t = skip_mods_and_typedefs(obj->btf, btf_array(t)->type,
                                                   NULL);
                        if (!btf_is_ptr(t)) {
                                pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
                                        map->name, btf_kind_str(t));
                                return -EINVAL;
                        }
                        t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
                        if (!btf_is_struct(t)) {
                                pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
                                        map->name, btf_kind_str(t));
                                return -EINVAL;
                        }

                        map->inner_map = calloc(1, sizeof(*map->inner_map));
                        if (!map->inner_map)
                                return -ENOMEM;
                        map->inner_map->sec_idx = obj->efile.btf_maps_shndx;
                        map->inner_map->name = malloc(strlen(map->name) +
                                                      sizeof(".inner") + 1);
                        if (!map->inner_map->name)
                                return -ENOMEM;
                        sprintf(map->inner_map->name, "%s.inner", map->name);

                        err = parse_btf_map_def(obj, map->inner_map, t, strict,
                                                true /* is_inner */, NULL);
                        if (err)
                                return err;
                } else if (strcmp(name, "pinning") == 0) {
                        __u32 val;
                        int err;

                        if (is_inner) {
                                pr_debug("map '%s': inner def can't be pinned.\n",
                                         map->name);
                                return -EINVAL;
                        }
                        if (!get_map_field_int(map->name, obj->btf, m, &val))
                                return -EINVAL;
                        pr_debug("map '%s': found pinning = %u.\n",
                                 map->name, val);

                        if (val != LIBBPF_PIN_NONE &&
                            val != LIBBPF_PIN_BY_NAME) {
                                pr_warn("map '%s': invalid pinning value %u.\n",
                                        map->name, val);
                                return -EINVAL;
                        }
                        if (val == LIBBPF_PIN_BY_NAME) {
                                err = build_map_pin_path(map, pin_root_path);
                                if (err) {
                                        pr_warn("map '%s': couldn't build pin path.\n",
                                                map->name);
                                        return err;
                                }
                        }
                } else {
                        if (strict) {
                                pr_warn("map '%s': unknown field '%s'.\n",
                                        map->name, name);
                                return -ENOTSUP;
                        }
                        pr_debug("map '%s': ignoring unknown field '%s'.\n",
                                 map->name, name);
                }
        }

        if (map->def.type == BPF_MAP_TYPE_UNSPEC) {
                pr_warn("map '%s': map type isn't specified.\n", map->name);
                return -EINVAL;
        }

        return 0;
}

static int bpf_object__init_user_btf_map(struct bpf_object *obj,
                                         const struct btf_type *sec,
                                         int var_idx, int sec_idx,
                                         const Elf_Data *data, bool strict,
                                         const char *pin_root_path)
{
        const struct btf_type *var, *def;
        const struct btf_var_secinfo *vi;
        const struct btf_var *var_extra;
        const char *map_name;
        struct bpf_map *map;

        vi = btf_var_secinfos(sec) + var_idx;
        var = btf__type_by_id(obj->btf, vi->type);
        var_extra = btf_var(var);
        map_name = btf__name_by_offset(obj->btf, var->name_off);

        if (map_name == NULL || map_name[0] == '\0') {
                pr_warn("map #%d: empty name.\n", var_idx);
                return -EINVAL;
        }
        if ((__u64)vi->offset + vi->size > data->d_size) {
                pr_warn("map '%s' BTF data is corrupted.\n", map_name);
                return -EINVAL;
        }
        if (!btf_is_var(var)) {
                pr_warn("map '%s': unexpected var kind %s.\n",
                        map_name, btf_kind_str(var));
                return -EINVAL;
        }
        if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&
            var_extra->linkage != BTF_VAR_STATIC) {
                pr_warn("map '%s': unsupported var linkage %u.\n",
                        map_name, var_extra->linkage);
                return -EOPNOTSUPP;
        }

        def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
        if (!btf_is_struct(def)) {
                pr_warn("map '%s': unexpected def kind %s.\n",
                        map_name, btf_kind_str(var));
                return -EINVAL;
        }
        if (def->size > vi->size) {
                pr_warn("map '%s': invalid def size.\n", map_name);
                return -EINVAL;
        }

        map = bpf_object__add_map(obj);
        if (IS_ERR(map))
                return PTR_ERR(map);
        map->name = strdup(map_name);
        if (!map->name) {
                pr_warn("map '%s': failed to alloc map name.\n", map_name);
                return -ENOMEM;
        }
        map->libbpf_type = LIBBPF_MAP_UNSPEC;
        map->def.type = BPF_MAP_TYPE_UNSPEC;
        map->sec_idx = sec_idx;
        map->sec_offset = vi->offset;
        map->btf_var_idx = var_idx;
        pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
                 map_name, map->sec_idx, map->sec_offset);

        return parse_btf_map_def(obj, map, def, strict, false, pin_root_path);
}

static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
                                          const char *pin_root_path)
{
        const struct btf_type *sec = NULL;
        int nr_types, i, vlen, err;
        const struct btf_type *t;
        const char *name;
        Elf_Data *data;
        Elf_Scn *scn;

        if (obj->efile.btf_maps_shndx < 0)
                return 0;

        scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
        data = elf_sec_data(obj, scn);
        if (!scn || !data) {
                pr_warn("elf: failed to get %s map definitions for %s\n",
                        MAPS_ELF_SEC, obj->path);
                return -EINVAL;
        }

        nr_types = btf__get_nr_types(obj->btf);
        for (i = 1; i <= nr_types; i++) {
                t = btf__type_by_id(obj->btf, i);
                if (!btf_is_datasec(t))
                        continue;
                name = btf__name_by_offset(obj->btf, t->name_off);
                if (strcmp(name, MAPS_ELF_SEC) == 0) {
                        sec = t;
                        obj->efile.btf_maps_sec_btf_id = i;
                        break;
                }
        }

        if (!sec) {
                pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
                return -ENOENT;
        }

        vlen = btf_vlen(sec);
        for (i = 0; i < vlen; i++) {
                err = bpf_object__init_user_btf_map(obj, sec, i,
                                                    obj->efile.btf_maps_shndx,
                                                    data, strict,
                                                    pin_root_path);
                if (err)
                        return err;
        }

        return 0;
}

static int bpf_object__init_maps(struct bpf_object *obj,
                                 const struct bpf_object_open_opts *opts)
{
        const char *pin_root_path;
        bool strict;
        int err;

        strict = !OPTS_GET(opts, relaxed_maps, false);
        pin_root_path = OPTS_GET(opts, pin_root_path, NULL);

        err = bpf_object__init_user_maps(obj, strict);
        err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
        err = err ?: bpf_object__init_global_data_maps(obj);
        err = err ?: bpf_object__init_kconfig_map(obj);
        err = err ?: bpf_object__init_struct_ops_maps(obj);
        if (err)
                return err;

        return 0;
}

static bool section_have_execinstr(struct bpf_object *obj, int idx)
{
        GElf_Shdr sh;

        if (elf_sec_hdr(obj, elf_sec_by_idx(obj, idx), &sh))
                return false;

        return sh.sh_flags & SHF_EXECINSTR;
}

static bool btf_needs_sanitization(struct bpf_object *obj)
{
        bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC);
        bool has_datasec = kernel_supports(FEAT_BTF_DATASEC);
        bool has_func = kernel_supports(FEAT_BTF_FUNC);

        return !has_func || !has_datasec || !has_func_global;
}

static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
{
        bool has_func_global = kernel_supports(FEAT_BTF_GLOBAL_FUNC);
        bool has_datasec = kernel_supports(FEAT_BTF_DATASEC);
        bool has_func = kernel_supports(FEAT_BTF_FUNC);
        struct btf_type *t;
        int i, j, vlen;

        for (i = 1; i <= btf__get_nr_types(btf); i++) {
                t = (struct btf_type *)btf__type_by_id(btf, i);

                if (!has_datasec && btf_is_var(t)) {
                        /* replace VAR with INT */
                        t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
                        /*
                         * using size = 1 is the safest choice, 4 will be too
                         * big and cause kernel BTF validation failure if
                         * original variable took less than 4 bytes
                         */
                        t->size = 1;
                        *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
                } else if (!has_datasec && btf_is_datasec(t)) {
                        /* replace DATASEC with STRUCT */
                        const struct btf_var_secinfo *v = btf_var_secinfos(t);
                        struct btf_member *m = btf_members(t);
                        struct btf_type *vt;
                        char *name;

                        name = (char *)btf__name_by_offset(btf, t->name_off);
                        while (*name) {
                                if (*name == '.')
                                        *name = '_';
                                name++;
                        }

                        vlen = btf_vlen(t);
                        t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
                        for (j = 0; j < vlen; j++, v++, m++) {
                                /* order of field assignments is important */
                                m->offset = v->offset * 8;
                                m->type = v->type;
                                /* preserve variable name as member name */
                                vt = (void *)btf__type_by_id(btf, v->type);
                                m->name_off = vt->name_off;
                        }
                } else if (!has_func && btf_is_func_proto(t)) {
                        /* replace FUNC_PROTO with ENUM */
                        vlen = btf_vlen(t);
                        t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
                        t->size = sizeof(__u32); /* kernel enforced */
                } else if (!has_func && btf_is_func(t)) {
                        /* replace FUNC with TYPEDEF */
                        t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
                } else if (!has_func_global && btf_is_func(t)) {
                        /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
                        t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
                }
        }
}

static bool libbpf_needs_btf(const struct bpf_object *obj)
{
        return obj->efile.btf_maps_shndx >= 0 ||
               obj->efile.st_ops_shndx >= 0 ||
               obj->nr_extern > 0;
}

static bool kernel_needs_btf(const struct bpf_object *obj)
{
        return obj->efile.st_ops_shndx >= 0;
}

static int bpf_object__init_btf(struct bpf_object *obj,
                                Elf_Data *btf_data,
                                Elf_Data *btf_ext_data)
{
        int err = -ENOENT;

        if (btf_data) {
                obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
                if (IS_ERR(obj->btf)) {
                        err = PTR_ERR(obj->btf);
                        obj->btf = NULL;
                        pr_warn("Error loading ELF section %s: %d.\n",
                                BTF_ELF_SEC, err);
                        goto out;
                }
                /* enforce 8-byte pointers for BPF-targeted BTFs */
                btf__set_pointer_size(obj->btf, 8);
                err = 0;
        }
        if (btf_ext_data) {
                if (!obj->btf) {
                        pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
                                 BTF_EXT_ELF_SEC, BTF_ELF_SEC);
                        goto out;
                }
                obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
                                            btf_ext_data->d_size);
                if (IS_ERR(obj->btf_ext)) {
                        pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n",
                                BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));
                        obj->btf_ext = NULL;
                        goto out;
                }
        }
out:
        if (err && libbpf_needs_btf(obj)) {
                pr_warn("BTF is required, but is missing or corrupted.\n");
                return err;
        }
        return 0;
}

static int bpf_object__finalize_btf(struct bpf_object *obj)
{
        int err;

        if (!obj->btf)
                return 0;

        err = btf__finalize_data(obj, obj->btf);
        if (err) {
                pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
                return err;
        }

        return 0;
}

static inline bool libbpf_prog_needs_vmlinux_btf(struct bpf_program *prog)
{
        if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
            prog->type == BPF_PROG_TYPE_LSM)
                return true;

        /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
         * also need vmlinux BTF
         */
        if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
                return true;

        return false;
}

static int bpf_object__load_vmlinux_btf(struct bpf_object *obj)
{
        bool need_vmlinux_btf = false;
        struct bpf_program *prog;
        int i, err;

        /* CO-RE relocations need kernel BTF */
        if (obj->btf_ext && obj->btf_ext->core_relo_info.len)
                need_vmlinux_btf = true;

        /* Support for typed ksyms needs kernel BTF */
        for (i = 0; i < obj->nr_extern; i++) {
                const struct extern_desc *ext;

                ext = &obj->externs[i];
                if (ext->type == EXT_KSYM && ext->ksym.type_id) {
                        need_vmlinux_btf = true;
                        break;
                }
        }

        bpf_object__for_each_program(prog, obj) {
                if (!prog->load)
                        continue;
                if (libbpf_prog_needs_vmlinux_btf(prog)) {
                        need_vmlinux_btf = true;
                        break;
                }
        }

        if (!need_vmlinux_btf)
                return 0;

        obj->btf_vmlinux = libbpf_find_kernel_btf();
        if (IS_ERR(obj->btf_vmlinux)) {
                err = PTR_ERR(obj->btf_vmlinux);
                pr_warn("Error loading vmlinux BTF: %d\n", err);
                obj->btf_vmlinux = NULL;
                return err;
        }
        return 0;
}

static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
{
        struct btf *kern_btf = obj->btf;
        bool btf_mandatory, sanitize;
        int err = 0;

        if (!obj->btf)
                return 0;

        if (!kernel_supports(FEAT_BTF)) {
                if (kernel_needs_btf(obj)) {
                        err = -EOPNOTSUPP;
                        goto report;
                }
                pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
                return 0;
        }

        sanitize = btf_needs_sanitization(obj);
        if (sanitize) {
                const void *raw_data;
                __u32 sz;

                /* clone BTF to sanitize a copy and leave the original intact */
                raw_data = btf__get_raw_data(obj->btf, &sz);
                kern_btf = btf__new(raw_data, sz);
                if (IS_ERR(kern_btf))
                        return PTR_ERR(kern_btf);

                /* enforce 8-byte pointers for BPF-targeted BTFs */
                btf__set_pointer_size(obj->btf, 8);
                bpf_object__sanitize_btf(obj, kern_btf);
        }

        err = btf__load(kern_btf);
        if (sanitize) {
                if (!err) {
                        /* move fd to libbpf's BTF */
                        btf__set_fd(obj->btf, btf__fd(kern_btf));
                        btf__set_fd(kern_btf, -1);
                }
                btf__free(kern_btf);
        }
report:
        if (err) {
                btf_mandatory = kernel_needs_btf(obj);
                pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
                        btf_mandatory ? "BTF is mandatory, can't proceed."
                                      : "BTF is optional, ignoring.");
                if (!btf_mandatory)
                        err = 0;
        }
        return err;
}

static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
{
        const char *name;

        name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
        if (!name) {
                pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
                        off, obj->path, elf_errmsg(-1));
                return NULL;
        }

        return name;
}

static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
{
        const char *name;

        name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
        if (!name) {
                pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
                        off, obj->path, elf_errmsg(-1));
                return NULL;
        }

        return name;
}

static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
{
        Elf_Scn *scn;

        scn = elf_getscn(obj->efile.elf, idx);
        if (!scn) {
                pr_warn("elf: failed to get section(%zu) from %s: %s\n",
                        idx, obj->path, elf_errmsg(-1));
                return NULL;
        }
        return scn;
}

static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
{
        Elf_Scn *scn = NULL;
        Elf *elf = obj->efile.elf;
        const char *sec_name;

        while ((scn = elf_nextscn(elf, scn)) != NULL) {
                sec_name = elf_sec_name(obj, scn);
                if (!sec_name)
                        return NULL;

                if (strcmp(sec_name, name) != 0)
                        continue;

                return scn;
        }
        return NULL;
}

static int elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn, GElf_Shdr *hdr)
{
        if (!scn)
                return -EINVAL;

        if (gelf_getshdr(scn, hdr) != hdr) {
                pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
                        elf_ndxscn(scn), obj->path, elf_errmsg(-1));
                return -EINVAL;
        }

        return 0;
}

static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
{
        const char *name;
        GElf_Shdr sh;

        if (!scn)
                return NULL;

        if (elf_sec_hdr(obj, scn, &sh))
                return NULL;

        name = elf_sec_str(obj, sh.sh_name);
        if (!name) {
                pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
                        elf_ndxscn(scn), obj->path, elf_errmsg(-1));
                return NULL;
        }

        return name;
}

static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
{
        Elf_Data *data;

        if (!scn)
                return NULL;

        data = elf_getdata(scn, 0);
        if (!data) {
                pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
                        elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
                        obj->path, elf_errmsg(-1));
                return NULL;
        }

        return data;
}

static int elf_sym_by_sec_off(const struct bpf_object *obj, size_t sec_idx,
                              size_t off, __u32 sym_type, GElf_Sym *sym)
{
        Elf_Data *symbols = obj->efile.symbols;
        size_t n = symbols->d_size / sizeof(GElf_Sym);
        int i;

        for (i = 0; i < n; i++) {
                if (!gelf_getsym(symbols, i, sym))
                        continue;
                if (sym->st_shndx != sec_idx || sym->st_value != off)
                        continue;
                if (GELF_ST_TYPE(sym->st_info) != sym_type)
                        continue;
                return 0;
        }

        return -ENOENT;
}

static bool is_sec_name_dwarf(const char *name)
{
        /* approximation, but the actual list is too long */
        return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
}

static bool ignore_elf_section(GElf_Shdr *hdr, const char *name)
{
        /* no special handling of .strtab */
        if (hdr->sh_type == SHT_STRTAB)
                return true;

        /* ignore .llvm_addrsig section as well */
        if (hdr->sh_type == 0x6FFF4C03 /* SHT_LLVM_ADDRSIG */)
                return true;

        /* no subprograms will lead to an empty .text section, ignore it */
        if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
            strcmp(name, ".text") == 0)
                return true;

        /* DWARF sections */
        if (is_sec_name_dwarf(name))
                return true;

        if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
                name += sizeof(".rel") - 1;
                /* DWARF section relocations */
                if (is_sec_name_dwarf(name))
                        return true;

                /* .BTF and .BTF.ext don't need relocations */
                if (strcmp(name, BTF_ELF_SEC) == 0 ||
                    strcmp(name, BTF_EXT_ELF_SEC) == 0)
                        return true;
        }

        return false;
}

static int cmp_progs(const void *_a, const void *_b)
{
        const struct bpf_program *a = _a;
        const struct bpf_program *b = _b;

        if (a->sec_idx != b->sec_idx)
                return a->sec_idx < b->sec_idx ? -1 : 1;

        /* sec_insn_off can't be the same within the section */
        return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
}

static int bpf_object__elf_collect(struct bpf_object *obj)
{
        Elf *elf = obj->efile.elf;
        Elf_Data *btf_ext_data = NULL;
        Elf_Data *btf_data = NULL;
        int idx = 0, err = 0;
        const char *name;
        Elf_Data *data;
        Elf_Scn *scn;
        GElf_Shdr sh;

        /* a bunch of ELF parsing functionality depends on processing symbols,
         * so do the first pass and find the symbol table
         */
        scn = NULL;
        while ((scn = elf_nextscn(elf, scn)) != NULL) {
                if (elf_sec_hdr(obj, scn, &sh))
                        return -LIBBPF_ERRNO__FORMAT;

                if (sh.sh_type == SHT_SYMTAB) {
                        if (obj->efile.symbols) {
                                pr_warn("elf: multiple symbol tables in %s\n", obj->path);
                                return -LIBBPF_ERRNO__FORMAT;
                        }

                        data = elf_sec_data(obj, scn);
                        if (!data)
                                return -LIBBPF_ERRNO__FORMAT;

                        obj->efile.symbols = data;
                        obj->efile.symbols_shndx = elf_ndxscn(scn);
                        obj->efile.strtabidx = sh.sh_link;
                }
        }

        scn = NULL;
        while ((scn = elf_nextscn(elf, scn)) != NULL) {
                idx++;

                if (elf_sec_hdr(obj, scn, &sh))
                        return -LIBBPF_ERRNO__FORMAT;

                name = elf_sec_str(obj, sh.sh_name);
                if (!name)
                        return -LIBBPF_ERRNO__FORMAT;

                if (ignore_elf_section(&sh, name))
                        continue;

                data = elf_sec_data(obj, scn);
                if (!data)
                        return -LIBBPF_ERRNO__FORMAT;

                pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
                         idx, name, (unsigned long)data->d_size,
                         (int)sh.sh_link, (unsigned long)sh.sh_flags,
                         (int)sh.sh_type);

                if (strcmp(name, "license") == 0) {
                        err = bpf_object__init_license(obj, data->d_buf, data->d_size);
                        if (err)
                                return err;
                } else if (strcmp(name, "version") == 0) {
                        err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
                        if (err)
                                return err;
                } else if (strcmp(name, "maps") == 0) {
                        obj->efile.maps_shndx = idx;
                } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
                        obj->efile.btf_maps_shndx = idx;
                } else if (strcmp(name, BTF_ELF_SEC) == 0) {
                        btf_data = data;
                } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
                        btf_ext_data = data;
                } else if (sh.sh_type == SHT_SYMTAB) {
                        /* already processed during the first pass above */
                } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
                        if (sh.sh_flags & SHF_EXECINSTR) {
                                if (strcmp(name, ".text") == 0)
                                        obj->efile.text_shndx = idx;
                                err = bpf_object__add_programs(obj, data, name, idx);
                                if (err)
                                        return err;
                        } else if (strcmp(name, DATA_SEC) == 0) {
                                obj->efile.data = data;
                                obj->efile.data_shndx = idx;
                        } else if (strcmp(name, RODATA_SEC) == 0) {
                                obj->efile.rodata = data;
                                obj->efile.rodata_shndx = idx;
                        } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
                                obj->efile.st_ops_data = data;
                                obj->efile.st_ops_shndx = idx;
                        } else {
                                pr_info("elf: skipping unrecognized data section(%d) %s\n",
                                        idx, name);
                        }
                } else if (sh.sh_type == SHT_REL) {
                        int nr_sects = obj->efile.nr_reloc_sects;
                        void *sects = obj->efile.reloc_sects;
                        int sec = sh.sh_info; /* points to other section */

                        /* Only do relo for section with exec instructions */
                        if (!section_have_execinstr(obj, sec) &&
                            strcmp(name, ".rel" STRUCT_OPS_SEC) &&
                            strcmp(name, ".rel" MAPS_ELF_SEC)) {
                                pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
                                        idx, name, sec,
                                        elf_sec_name(obj, elf_sec_by_idx(obj, sec)) ?: "<?>");
                                continue;
                        }

                        sects = libbpf_reallocarray(sects, nr_sects + 1,
                                                    sizeof(*obj->efile.reloc_sects));
                        if (!sects)
                                return -ENOMEM;

                        obj->efile.reloc_sects = sects;
                        obj->efile.nr_reloc_sects++;

                        obj->efile.reloc_sects[nr_sects].shdr = sh;
                        obj->efile.reloc_sects[nr_sects].data = data;
                } else if (sh.sh_type == SHT_NOBITS && strcmp(name, BSS_SEC) == 0) {
                        obj->efile.bss = data;
                        obj->efile.bss_shndx = idx;
                } else {
                        pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
                                (size_t)sh.sh_size);
                }
        }

        if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
                pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
                return -LIBBPF_ERRNO__FORMAT;
        }

        /* sort BPF programs by section name and in-section instruction offset
         * for faster search */
        qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);

        return bpf_object__init_btf(obj, btf_data, btf_ext_data);
}

static bool sym_is_extern(const GElf_Sym *sym)
{
        int bind = GELF_ST_BIND(sym->st_info);
        /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
        return sym->st_shndx == SHN_UNDEF &&
               (bind == STB_GLOBAL || bind == STB_WEAK) &&
               GELF_ST_TYPE(sym->st_info) == STT_NOTYPE;
}

static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
{
        const struct btf_type *t;
        const char *var_name;
        int i, n;

        if (!btf)
                return -ESRCH;

        n = btf__get_nr_types(btf);
        for (i = 1; i <= n; i++) {
                t = btf__type_by_id(btf, i);

                if (!btf_is_var(t))
                        continue;

                var_name = btf__name_by_offset(btf, t->name_off);
                if (strcmp(var_name, ext_name))
                        continue;

                if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
                        return -EINVAL;

                return i;
        }

        return -ENOENT;
}

static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
        const struct btf_var_secinfo *vs;
        const struct btf_type *t;
        int i, j, n;

        if (!btf)
                return -ESRCH;

        n = btf__get_nr_types(btf);
        for (i = 1; i <= n; i++) {
                t = btf__type_by_id(btf, i);

                if (!btf_is_datasec(t))
                        continue;

                vs = btf_var_secinfos(t);
                for (j = 0; j < btf_vlen(t); j++, vs++) {
                        if (vs->type == ext_btf_id)
                                return i;
                }