linux/tools/perf/util/bpf-loader.c
<<
>>
Prefs
   1/*
   2 * bpf-loader.c
   3 *
   4 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   5 * Copyright (C) 2015 Huawei Inc.
   6 */
   7
   8#include <linux/bpf.h>
   9#include <bpf/libbpf.h>
  10#include <bpf/bpf.h>
  11#include <linux/err.h>
  12#include <linux/string.h>
  13#include "perf.h"
  14#include "debug.h"
  15#include "bpf-loader.h"
  16#include "bpf-prologue.h"
  17#include "llvm-utils.h"
  18#include "probe-event.h"
  19#include "probe-finder.h" // for MAX_PROBES
  20#include "parse-events.h"
  21#include "llvm-utils.h"
  22
  23#define DEFINE_PRINT_FN(name, level) \
  24static int libbpf_##name(const char *fmt, ...)  \
  25{                                               \
  26        va_list args;                           \
  27        int ret;                                \
  28                                                \
  29        va_start(args, fmt);                    \
  30        ret = veprintf(level, verbose, pr_fmt(fmt), args);\
  31        va_end(args);                           \
  32        return ret;                             \
  33}
  34
  35DEFINE_PRINT_FN(warning, 1)
  36DEFINE_PRINT_FN(info, 1)
  37DEFINE_PRINT_FN(debug, 1)
  38
  39struct bpf_prog_priv {
  40        struct perf_probe_event pev;
  41        bool need_prologue;
  42        struct bpf_insn *insns_buf;
  43        int nr_types;
  44        int *type_mapping;
  45};
  46
  47static bool libbpf_initialized;
  48
  49struct bpf_object *
  50bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name)
  51{
  52        struct bpf_object *obj;
  53
  54        if (!libbpf_initialized) {
  55                libbpf_set_print(libbpf_warning,
  56                                 libbpf_info,
  57                                 libbpf_debug);
  58                libbpf_initialized = true;
  59        }
  60
  61        obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, name);
  62        if (IS_ERR(obj)) {
  63                pr_debug("bpf: failed to load buffer\n");
  64                return ERR_PTR(-EINVAL);
  65        }
  66
  67        return obj;
  68}
  69
  70struct bpf_object *bpf__prepare_load(const char *filename, bool source)
  71{
  72        struct bpf_object *obj;
  73
  74        if (!libbpf_initialized) {
  75                libbpf_set_print(libbpf_warning,
  76                                 libbpf_info,
  77                                 libbpf_debug);
  78                libbpf_initialized = true;
  79        }
  80
  81        if (source) {
  82                int err;
  83                void *obj_buf;
  84                size_t obj_buf_sz;
  85
  86                err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz);
  87                if (err)
  88                        return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE);
  89                obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, filename);
  90                free(obj_buf);
  91        } else
  92                obj = bpf_object__open(filename);
  93
  94        if (IS_ERR(obj)) {
  95                pr_debug("bpf: failed to load %s\n", filename);
  96                return obj;
  97        }
  98
  99        return obj;
 100}
 101
 102void bpf__clear(void)
 103{
 104        struct bpf_object *obj, *tmp;
 105
 106        bpf_object__for_each_safe(obj, tmp) {
 107                bpf__unprobe(obj);
 108                bpf_object__close(obj);
 109        }
 110}
 111
 112static void
 113clear_prog_priv(struct bpf_program *prog __maybe_unused,
 114                void *_priv)
 115{
 116        struct bpf_prog_priv *priv = _priv;
 117
 118        cleanup_perf_probe_events(&priv->pev, 1);
 119        zfree(&priv->insns_buf);
 120        zfree(&priv->type_mapping);
 121        free(priv);
 122}
 123
 124static int
 125prog_config__exec(const char *value, struct perf_probe_event *pev)
 126{
 127        pev->uprobes = true;
 128        pev->target = strdup(value);
 129        if (!pev->target)
 130                return -ENOMEM;
 131        return 0;
 132}
 133
 134static int
 135prog_config__module(const char *value, struct perf_probe_event *pev)
 136{
 137        pev->uprobes = false;
 138        pev->target = strdup(value);
 139        if (!pev->target)
 140                return -ENOMEM;
 141        return 0;
 142}
 143
 144static int
 145prog_config__bool(const char *value, bool *pbool, bool invert)
 146{
 147        int err;
 148        bool bool_value;
 149
 150        if (!pbool)
 151                return -EINVAL;
 152
 153        err = strtobool(value, &bool_value);
 154        if (err)
 155                return err;
 156
 157        *pbool = invert ? !bool_value : bool_value;
 158        return 0;
 159}
 160
 161static int
 162prog_config__inlines(const char *value,
 163                     struct perf_probe_event *pev __maybe_unused)
 164{
 165        return prog_config__bool(value, &probe_conf.no_inlines, true);
 166}
 167
 168static int
 169prog_config__force(const char *value,
 170                   struct perf_probe_event *pev __maybe_unused)
 171{
 172        return prog_config__bool(value, &probe_conf.force_add, false);
 173}
 174
 175static struct {
 176        const char *key;
 177        const char *usage;
 178        const char *desc;
 179        int (*func)(const char *, struct perf_probe_event *);
 180} bpf_prog_config_terms[] = {
 181        {
 182                .key    = "exec",
 183                .usage  = "exec=<full path of file>",
 184                .desc   = "Set uprobe target",
 185                .func   = prog_config__exec,
 186        },
 187        {
 188                .key    = "module",
 189                .usage  = "module=<module name>    ",
 190                .desc   = "Set kprobe module",
 191                .func   = prog_config__module,
 192        },
 193        {
 194                .key    = "inlines",
 195                .usage  = "inlines=[yes|no]        ",
 196                .desc   = "Probe at inline symbol",
 197                .func   = prog_config__inlines,
 198        },
 199        {
 200                .key    = "force",
 201                .usage  = "force=[yes|no]          ",
 202                .desc   = "Forcibly add events with existing name",
 203                .func   = prog_config__force,
 204        },
 205};
 206
 207static int
 208do_prog_config(const char *key, const char *value,
 209               struct perf_probe_event *pev)
 210{
 211        unsigned int i;
 212
 213        pr_debug("config bpf program: %s=%s\n", key, value);
 214        for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++)
 215                if (strcmp(key, bpf_prog_config_terms[i].key) == 0)
 216                        return bpf_prog_config_terms[i].func(value, pev);
 217
 218        pr_debug("BPF: ERROR: invalid program config option: %s=%s\n",
 219                 key, value);
 220
 221        pr_debug("\nHint: Valid options are:\n");
 222        for (i = 0; i < ARRAY_SIZE(bpf_prog_config_terms); i++)
 223                pr_debug("\t%s:\t%s\n", bpf_prog_config_terms[i].usage,
 224                         bpf_prog_config_terms[i].desc);
 225        pr_debug("\n");
 226
 227        return -BPF_LOADER_ERRNO__PROGCONF_TERM;
 228}
 229
 230static const char *
 231parse_prog_config_kvpair(const char *config_str, struct perf_probe_event *pev)
 232{
 233        char *text = strdup(config_str);
 234        char *sep, *line;
 235        const char *main_str = NULL;
 236        int err = 0;
 237
 238        if (!text) {
 239                pr_debug("No enough memory: dup config_str failed\n");
 240                return ERR_PTR(-ENOMEM);
 241        }
 242
 243        line = text;
 244        while ((sep = strchr(line, ';'))) {
 245                char *equ;
 246
 247                *sep = '\0';
 248                equ = strchr(line, '=');
 249                if (!equ) {
 250                        pr_warning("WARNING: invalid config in BPF object: %s\n",
 251                                   line);
 252                        pr_warning("\tShould be 'key=value'.\n");
 253                        goto nextline;
 254                }
 255                *equ = '\0';
 256
 257                err = do_prog_config(line, equ + 1, pev);
 258                if (err)
 259                        break;
 260nextline:
 261                line = sep + 1;
 262        }
 263
 264        if (!err)
 265                main_str = config_str + (line - text);
 266        free(text);
 267
 268        return err ? ERR_PTR(err) : main_str;
 269}
 270
 271static int
 272parse_prog_config(const char *config_str, struct perf_probe_event *pev)
 273{
 274        int err;
 275        const char *main_str = parse_prog_config_kvpair(config_str, pev);
 276
 277        if (IS_ERR(main_str))
 278                return PTR_ERR(main_str);
 279
 280        err = parse_perf_probe_command(main_str, pev);
 281        if (err < 0) {
 282                pr_debug("bpf: '%s' is not a valid config string\n",
 283                         config_str);
 284                /* parse failed, don't need clear pev. */
 285                return -BPF_LOADER_ERRNO__CONFIG;
 286        }
 287        return 0;
 288}
 289
 290static int
 291config_bpf_program(struct bpf_program *prog)
 292{
 293        struct perf_probe_event *pev = NULL;
 294        struct bpf_prog_priv *priv = NULL;
 295        const char *config_str;
 296        int err;
 297
 298        /* Initialize per-program probing setting */
 299        probe_conf.no_inlines = false;
 300        probe_conf.force_add = false;
 301
 302        config_str = bpf_program__title(prog, false);
 303        if (IS_ERR(config_str)) {
 304                pr_debug("bpf: unable to get title for program\n");
 305                return PTR_ERR(config_str);
 306        }
 307
 308        priv = calloc(sizeof(*priv), 1);
 309        if (!priv) {
 310                pr_debug("bpf: failed to alloc priv\n");
 311                return -ENOMEM;
 312        }
 313        pev = &priv->pev;
 314
 315        pr_debug("bpf: config program '%s'\n", config_str);
 316        err = parse_prog_config(config_str, pev);
 317        if (err)
 318                goto errout;
 319
 320        if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) {
 321                pr_debug("bpf: '%s': group for event is set and not '%s'.\n",
 322                         config_str, PERF_BPF_PROBE_GROUP);
 323                err = -BPF_LOADER_ERRNO__GROUP;
 324                goto errout;
 325        } else if (!pev->group)
 326                pev->group = strdup(PERF_BPF_PROBE_GROUP);
 327
 328        if (!pev->group) {
 329                pr_debug("bpf: strdup failed\n");
 330                err = -ENOMEM;
 331                goto errout;
 332        }
 333
 334        if (!pev->event) {
 335                pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n",
 336                         config_str);
 337                err = -BPF_LOADER_ERRNO__EVENTNAME;
 338                goto errout;
 339        }
 340        pr_debug("bpf: config '%s' is ok\n", config_str);
 341
 342        err = bpf_program__set_private(prog, priv, clear_prog_priv);
 343        if (err) {
 344                pr_debug("Failed to set priv for program '%s'\n", config_str);
 345                goto errout;
 346        }
 347
 348        return 0;
 349
 350errout:
 351        if (pev)
 352                clear_perf_probe_event(pev);
 353        free(priv);
 354        return err;
 355}
 356
 357static int bpf__prepare_probe(void)
 358{
 359        static int err = 0;
 360        static bool initialized = false;
 361
 362        /*
 363         * Make err static, so if init failed the first, bpf__prepare_probe()
 364         * fails each time without calling init_probe_symbol_maps multiple
 365         * times.
 366         */
 367        if (initialized)
 368                return err;
 369
 370        initialized = true;
 371        err = init_probe_symbol_maps(false);
 372        if (err < 0)
 373                pr_debug("Failed to init_probe_symbol_maps\n");
 374        probe_conf.max_probes = MAX_PROBES;
 375        return err;
 376}
 377
 378static int
 379preproc_gen_prologue(struct bpf_program *prog, int n,
 380                     struct bpf_insn *orig_insns, int orig_insns_cnt,
 381                     struct bpf_prog_prep_result *res)
 382{
 383        struct probe_trace_event *tev;
 384        struct perf_probe_event *pev;
 385        struct bpf_prog_priv *priv;
 386        struct bpf_insn *buf;
 387        size_t prologue_cnt = 0;
 388        int i, err;
 389
 390        err = bpf_program__get_private(prog, (void **)&priv);
 391        if (err || !priv)
 392                goto errout;
 393
 394        pev = &priv->pev;
 395
 396        if (n < 0 || n >= priv->nr_types)
 397                goto errout;
 398
 399        /* Find a tev belongs to that type */
 400        for (i = 0; i < pev->ntevs; i++) {
 401                if (priv->type_mapping[i] == n)
 402                        break;
 403        }
 404
 405        if (i >= pev->ntevs) {
 406                pr_debug("Internal error: prologue type %d not found\n", n);
 407                return -BPF_LOADER_ERRNO__PROLOGUE;
 408        }
 409
 410        tev = &pev->tevs[i];
 411
 412        buf = priv->insns_buf;
 413        err = bpf__gen_prologue(tev->args, tev->nargs,
 414                                buf, &prologue_cnt,
 415                                BPF_MAXINSNS - orig_insns_cnt);
 416        if (err) {
 417                const char *title;
 418
 419                title = bpf_program__title(prog, false);
 420                if (!title)
 421                        title = "[unknown]";
 422
 423                pr_debug("Failed to generate prologue for program %s\n",
 424                         title);
 425                return err;
 426        }
 427
 428        memcpy(&buf[prologue_cnt], orig_insns,
 429               sizeof(struct bpf_insn) * orig_insns_cnt);
 430
 431        res->new_insn_ptr = buf;
 432        res->new_insn_cnt = prologue_cnt + orig_insns_cnt;
 433        res->pfd = NULL;
 434        return 0;
 435
 436errout:
 437        pr_debug("Internal error in preproc_gen_prologue\n");
 438        return -BPF_LOADER_ERRNO__PROLOGUE;
 439}
 440
 441/*
 442 * compare_tev_args is reflexive, transitive and antisymmetric.
 443 * I can proof it but this margin is too narrow to contain.
 444 */
 445static int compare_tev_args(const void *ptev1, const void *ptev2)
 446{
 447        int i, ret;
 448        const struct probe_trace_event *tev1 =
 449                *(const struct probe_trace_event **)ptev1;
 450        const struct probe_trace_event *tev2 =
 451                *(const struct probe_trace_event **)ptev2;
 452
 453        ret = tev2->nargs - tev1->nargs;
 454        if (ret)
 455                return ret;
 456
 457        for (i = 0; i < tev1->nargs; i++) {
 458                struct probe_trace_arg *arg1, *arg2;
 459                struct probe_trace_arg_ref *ref1, *ref2;
 460
 461                arg1 = &tev1->args[i];
 462                arg2 = &tev2->args[i];
 463
 464                ret = strcmp(arg1->value, arg2->value);
 465                if (ret)
 466                        return ret;
 467
 468                ref1 = arg1->ref;
 469                ref2 = arg2->ref;
 470
 471                while (ref1 && ref2) {
 472                        ret = ref2->offset - ref1->offset;
 473                        if (ret)
 474                                return ret;
 475
 476                        ref1 = ref1->next;
 477                        ref2 = ref2->next;
 478                }
 479
 480                if (ref1 || ref2)
 481                        return ref2 ? 1 : -1;
 482        }
 483
 484        return 0;
 485}
 486
 487/*
 488 * Assign a type number to each tevs in a pev.
 489 * mapping is an array with same slots as tevs in that pev.
 490 * nr_types will be set to number of types.
 491 */
 492static int map_prologue(struct perf_probe_event *pev, int *mapping,
 493                        int *nr_types)
 494{
 495        int i, type = 0;
 496        struct probe_trace_event **ptevs;
 497
 498        size_t array_sz = sizeof(*ptevs) * pev->ntevs;
 499
 500        ptevs = malloc(array_sz);
 501        if (!ptevs) {
 502                pr_debug("No ehough memory: alloc ptevs failed\n");
 503                return -ENOMEM;
 504        }
 505
 506        pr_debug("In map_prologue, ntevs=%d\n", pev->ntevs);
 507        for (i = 0; i < pev->ntevs; i++)
 508                ptevs[i] = &pev->tevs[i];
 509
 510        qsort(ptevs, pev->ntevs, sizeof(*ptevs),
 511              compare_tev_args);
 512
 513        for (i = 0; i < pev->ntevs; i++) {
 514                int n;
 515
 516                n = ptevs[i] - pev->tevs;
 517                if (i == 0) {
 518                        mapping[n] = type;
 519                        pr_debug("mapping[%d]=%d\n", n, type);
 520                        continue;
 521                }
 522
 523                if (compare_tev_args(ptevs + i, ptevs + i - 1) == 0)
 524                        mapping[n] = type;
 525                else
 526                        mapping[n] = ++type;
 527
 528                pr_debug("mapping[%d]=%d\n", n, mapping[n]);
 529        }
 530        free(ptevs);
 531        *nr_types = type + 1;
 532
 533        return 0;
 534}
 535
 536static int hook_load_preprocessor(struct bpf_program *prog)
 537{
 538        struct perf_probe_event *pev;
 539        struct bpf_prog_priv *priv;
 540        bool need_prologue = false;
 541        int err, i;
 542
 543        err = bpf_program__get_private(prog, (void **)&priv);
 544        if (err || !priv) {
 545                pr_debug("Internal error when hook preprocessor\n");
 546                return -BPF_LOADER_ERRNO__INTERNAL;
 547        }
 548
 549        pev = &priv->pev;
 550        for (i = 0; i < pev->ntevs; i++) {
 551                struct probe_trace_event *tev = &pev->tevs[i];
 552
 553                if (tev->nargs > 0) {
 554                        need_prologue = true;
 555                        break;
 556                }
 557        }
 558
 559        /*
 560         * Since all tevs don't have argument, we don't need generate
 561         * prologue.
 562         */
 563        if (!need_prologue) {
 564                priv->need_prologue = false;
 565                return 0;
 566        }
 567
 568        priv->need_prologue = true;
 569        priv->insns_buf = malloc(sizeof(struct bpf_insn) * BPF_MAXINSNS);
 570        if (!priv->insns_buf) {
 571                pr_debug("No enough memory: alloc insns_buf failed\n");
 572                return -ENOMEM;
 573        }
 574
 575        priv->type_mapping = malloc(sizeof(int) * pev->ntevs);
 576        if (!priv->type_mapping) {
 577                pr_debug("No enough memory: alloc type_mapping failed\n");
 578                return -ENOMEM;
 579        }
 580        memset(priv->type_mapping, -1,
 581               sizeof(int) * pev->ntevs);
 582
 583        err = map_prologue(pev, priv->type_mapping, &priv->nr_types);
 584        if (err)
 585                return err;
 586
 587        err = bpf_program__set_prep(prog, priv->nr_types,
 588                                    preproc_gen_prologue);
 589        return err;
 590}
 591
 592int bpf__probe(struct bpf_object *obj)
 593{
 594        int err = 0;
 595        struct bpf_program *prog;
 596        struct bpf_prog_priv *priv;
 597        struct perf_probe_event *pev;
 598
 599        err = bpf__prepare_probe();
 600        if (err) {
 601                pr_debug("bpf__prepare_probe failed\n");
 602                return err;
 603        }
 604
 605        bpf_object__for_each_program(prog, obj) {
 606                err = config_bpf_program(prog);
 607                if (err)
 608                        goto out;
 609
 610                err = bpf_program__get_private(prog, (void **)&priv);
 611                if (err || !priv)
 612                        goto out;
 613                pev = &priv->pev;
 614
 615                err = convert_perf_probe_events(pev, 1);
 616                if (err < 0) {
 617                        pr_debug("bpf_probe: failed to convert perf probe events");
 618                        goto out;
 619                }
 620
 621                err = apply_perf_probe_events(pev, 1);
 622                if (err < 0) {
 623                        pr_debug("bpf_probe: failed to apply perf probe events");
 624                        goto out;
 625                }
 626
 627                /*
 628                 * After probing, let's consider prologue, which
 629                 * adds program fetcher to BPF programs.
 630                 *
 631                 * hook_load_preprocessorr() hooks pre-processor
 632                 * to bpf_program, let it generate prologue
 633                 * dynamically during loading.
 634                 */
 635                err = hook_load_preprocessor(prog);
 636                if (err)
 637                        goto out;
 638        }
 639out:
 640        return err < 0 ? err : 0;
 641}
 642
 643#define EVENTS_WRITE_BUFSIZE  4096
 644int bpf__unprobe(struct bpf_object *obj)
 645{
 646        int err, ret = 0;
 647        struct bpf_program *prog;
 648        struct bpf_prog_priv *priv;
 649
 650        bpf_object__for_each_program(prog, obj) {
 651                int i;
 652
 653                err = bpf_program__get_private(prog, (void **)&priv);
 654                if (err || !priv)
 655                        continue;
 656
 657                for (i = 0; i < priv->pev.ntevs; i++) {
 658                        struct probe_trace_event *tev = &priv->pev.tevs[i];
 659                        char name_buf[EVENTS_WRITE_BUFSIZE];
 660                        struct strfilter *delfilter;
 661
 662                        snprintf(name_buf, EVENTS_WRITE_BUFSIZE,
 663                                 "%s:%s", tev->group, tev->event);
 664                        name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0';
 665
 666                        delfilter = strfilter__new(name_buf, NULL);
 667                        if (!delfilter) {
 668                                pr_debug("Failed to create filter for unprobing\n");
 669                                ret = -ENOMEM;
 670                                continue;
 671                        }
 672
 673                        err = del_perf_probe_events(delfilter);
 674                        strfilter__delete(delfilter);
 675                        if (err) {
 676                                pr_debug("Failed to delete %s\n", name_buf);
 677                                ret = err;
 678                                continue;
 679                        }
 680                }
 681        }
 682        return ret;
 683}
 684
 685int bpf__load(struct bpf_object *obj)
 686{
 687        int err;
 688
 689        err = bpf_object__load(obj);
 690        if (err) {
 691                pr_debug("bpf: load objects failed\n");
 692                return err;
 693        }
 694        return 0;
 695}
 696
 697int bpf__foreach_tev(struct bpf_object *obj,
 698                     bpf_prog_iter_callback_t func,
 699                     void *arg)
 700{
 701        struct bpf_program *prog;
 702        int err;
 703
 704        bpf_object__for_each_program(prog, obj) {
 705                struct probe_trace_event *tev;
 706                struct perf_probe_event *pev;
 707                struct bpf_prog_priv *priv;
 708                int i, fd;
 709
 710                err = bpf_program__get_private(prog,
 711                                (void **)&priv);
 712                if (err || !priv) {
 713                        pr_debug("bpf: failed to get private field\n");
 714                        return -BPF_LOADER_ERRNO__INTERNAL;
 715                }
 716
 717                pev = &priv->pev;
 718                for (i = 0; i < pev->ntevs; i++) {
 719                        tev = &pev->tevs[i];
 720
 721                        if (priv->need_prologue) {
 722                                int type = priv->type_mapping[i];
 723
 724                                fd = bpf_program__nth_fd(prog, type);
 725                        } else {
 726                                fd = bpf_program__fd(prog);
 727                        }
 728
 729                        if (fd < 0) {
 730                                pr_debug("bpf: failed to get file descriptor\n");
 731                                return fd;
 732                        }
 733
 734                        err = (*func)(tev, fd, arg);
 735                        if (err) {
 736                                pr_debug("bpf: call back failed, stop iterate\n");
 737                                return err;
 738                        }
 739                }
 740        }
 741        return 0;
 742}
 743
 744enum bpf_map_op_type {
 745        BPF_MAP_OP_SET_VALUE,
 746        BPF_MAP_OP_SET_EVSEL,
 747};
 748
 749enum bpf_map_key_type {
 750        BPF_MAP_KEY_ALL,
 751        BPF_MAP_KEY_RANGES,
 752};
 753
 754struct bpf_map_op {
 755        struct list_head list;
 756        enum bpf_map_op_type op_type;
 757        enum bpf_map_key_type key_type;
 758        union {
 759                struct parse_events_array array;
 760        } k;
 761        union {
 762                u64 value;
 763                struct perf_evsel *evsel;
 764        } v;
 765};
 766
 767struct bpf_map_priv {
 768        struct list_head ops_list;
 769};
 770
 771static void
 772bpf_map_op__delete(struct bpf_map_op *op)
 773{
 774        if (!list_empty(&op->list))
 775                list_del(&op->list);
 776        if (op->key_type == BPF_MAP_KEY_RANGES)
 777                parse_events__clear_array(&op->k.array);
 778        free(op);
 779}
 780
 781static void
 782bpf_map_priv__purge(struct bpf_map_priv *priv)
 783{
 784        struct bpf_map_op *pos, *n;
 785
 786        list_for_each_entry_safe(pos, n, &priv->ops_list, list) {
 787                list_del_init(&pos->list);
 788                bpf_map_op__delete(pos);
 789        }
 790}
 791
 792static void
 793bpf_map_priv__clear(struct bpf_map *map __maybe_unused,
 794                    void *_priv)
 795{
 796        struct bpf_map_priv *priv = _priv;
 797
 798        bpf_map_priv__purge(priv);
 799        free(priv);
 800}
 801
 802static int
 803bpf_map_op_setkey(struct bpf_map_op *op, struct parse_events_term *term)
 804{
 805        op->key_type = BPF_MAP_KEY_ALL;
 806        if (!term)
 807                return 0;
 808
 809        if (term->array.nr_ranges) {
 810                size_t memsz = term->array.nr_ranges *
 811                                sizeof(op->k.array.ranges[0]);
 812
 813                op->k.array.ranges = memdup(term->array.ranges, memsz);
 814                if (!op->k.array.ranges) {
 815                        pr_debug("No enough memory to alloc indices for map\n");
 816                        return -ENOMEM;
 817                }
 818                op->key_type = BPF_MAP_KEY_RANGES;
 819                op->k.array.nr_ranges = term->array.nr_ranges;
 820        }
 821        return 0;
 822}
 823
 824static struct bpf_map_op *
 825bpf_map_op__new(struct parse_events_term *term)
 826{
 827        struct bpf_map_op *op;
 828        int err;
 829
 830        op = zalloc(sizeof(*op));
 831        if (!op) {
 832                pr_debug("Failed to alloc bpf_map_op\n");
 833                return ERR_PTR(-ENOMEM);
 834        }
 835        INIT_LIST_HEAD(&op->list);
 836
 837        err = bpf_map_op_setkey(op, term);
 838        if (err) {
 839                free(op);
 840                return ERR_PTR(err);
 841        }
 842        return op;
 843}
 844
 845static int
 846bpf_map__add_op(struct bpf_map *map, struct bpf_map_op *op)
 847{
 848        struct bpf_map_priv *priv;
 849        const char *map_name;
 850        int err;
 851
 852        map_name = bpf_map__get_name(map);
 853        err = bpf_map__get_private(map, (void **)&priv);
 854        if (err) {
 855                pr_debug("Failed to get private from map %s\n", map_name);
 856                return err;
 857        }
 858
 859        if (!priv) {
 860                priv = zalloc(sizeof(*priv));
 861                if (!priv) {
 862                        pr_debug("No enough memory to alloc map private\n");
 863                        return -ENOMEM;
 864                }
 865                INIT_LIST_HEAD(&priv->ops_list);
 866
 867                if (bpf_map__set_private(map, priv, bpf_map_priv__clear)) {
 868                        free(priv);
 869                        return -BPF_LOADER_ERRNO__INTERNAL;
 870                }
 871        }
 872
 873        list_add_tail(&op->list, &priv->ops_list);
 874        return 0;
 875}
 876
 877static struct bpf_map_op *
 878bpf_map__add_newop(struct bpf_map *map, struct parse_events_term *term)
 879{
 880        struct bpf_map_op *op;
 881        int err;
 882
 883        op = bpf_map_op__new(term);
 884        if (IS_ERR(op))
 885                return op;
 886
 887        err = bpf_map__add_op(map, op);
 888        if (err) {
 889                bpf_map_op__delete(op);
 890                return ERR_PTR(err);
 891        }
 892        return op;
 893}
 894
 895static int
 896__bpf_map__config_value(struct bpf_map *map,
 897                        struct parse_events_term *term)
 898{
 899        struct bpf_map_def def;
 900        struct bpf_map_op *op;
 901        const char *map_name;
 902        int err;
 903
 904        map_name = bpf_map__get_name(map);
 905
 906        err = bpf_map__get_def(map, &def);
 907        if (err) {
 908                pr_debug("Unable to get map definition from '%s'\n",
 909                         map_name);
 910                return -BPF_LOADER_ERRNO__INTERNAL;
 911        }
 912
 913        if (def.type != BPF_MAP_TYPE_ARRAY) {
 914                pr_debug("Map %s type is not BPF_MAP_TYPE_ARRAY\n",
 915                         map_name);
 916                return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
 917        }
 918        if (def.key_size < sizeof(unsigned int)) {
 919                pr_debug("Map %s has incorrect key size\n", map_name);
 920                return -BPF_LOADER_ERRNO__OBJCONF_MAP_KEYSIZE;
 921        }
 922        switch (def.value_size) {
 923        case 1:
 924        case 2:
 925        case 4:
 926        case 8:
 927                break;
 928        default:
 929                pr_debug("Map %s has incorrect value size\n", map_name);
 930                return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE;
 931        }
 932
 933        op = bpf_map__add_newop(map, term);
 934        if (IS_ERR(op))
 935                return PTR_ERR(op);
 936        op->op_type = BPF_MAP_OP_SET_VALUE;
 937        op->v.value = term->val.num;
 938        return 0;
 939}
 940
 941static int
 942bpf_map__config_value(struct bpf_map *map,
 943                      struct parse_events_term *term,
 944                      struct perf_evlist *evlist __maybe_unused)
 945{
 946        if (!term->err_val) {
 947                pr_debug("Config value not set\n");
 948                return -BPF_LOADER_ERRNO__OBJCONF_CONF;
 949        }
 950
 951        if (term->type_val != PARSE_EVENTS__TERM_TYPE_NUM) {
 952                pr_debug("ERROR: wrong value type for 'value'\n");
 953                return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE;
 954        }
 955
 956        return __bpf_map__config_value(map, term);
 957}
 958
 959static int
 960__bpf_map__config_event(struct bpf_map *map,
 961                        struct parse_events_term *term,
 962                        struct perf_evlist *evlist)
 963{
 964        struct perf_evsel *evsel;
 965        struct bpf_map_def def;
 966        struct bpf_map_op *op;
 967        const char *map_name;
 968        int err;
 969
 970        map_name = bpf_map__get_name(map);
 971        evsel = perf_evlist__find_evsel_by_str(evlist, term->val.str);
 972        if (!evsel) {
 973                pr_debug("Event (for '%s') '%s' doesn't exist\n",
 974                         map_name, term->val.str);
 975                return -BPF_LOADER_ERRNO__OBJCONF_MAP_NOEVT;
 976        }
 977
 978        err = bpf_map__get_def(map, &def);
 979        if (err) {
 980                pr_debug("Unable to get map definition from '%s'\n",
 981                         map_name);
 982                return err;
 983        }
 984
 985        /*
 986         * No need to check key_size and value_size:
 987         * kernel has already checked them.
 988         */
 989        if (def.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
 990                pr_debug("Map %s type is not BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
 991                         map_name);
 992                return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
 993        }
 994
 995        op = bpf_map__add_newop(map, term);
 996        if (IS_ERR(op))
 997                return PTR_ERR(op);
 998        op->op_type = BPF_MAP_OP_SET_EVSEL;
 999        op->v.evsel = evsel;
1000        return 0;
1001}
1002
1003static int
1004bpf_map__config_event(struct bpf_map *map,
1005                      struct parse_events_term *term,
1006                      struct perf_evlist *evlist)
1007{
1008        if (!term->err_val) {
1009                pr_debug("Config value not set\n");
1010                return -BPF_LOADER_ERRNO__OBJCONF_CONF;
1011        }
1012
1013        if (term->type_val != PARSE_EVENTS__TERM_TYPE_STR) {
1014                pr_debug("ERROR: wrong value type for 'event'\n");
1015                return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUE;
1016        }
1017
1018        return __bpf_map__config_event(map, term, evlist);
1019}
1020
1021struct bpf_obj_config__map_func {
1022        const char *config_opt;
1023        int (*config_func)(struct bpf_map *, struct parse_events_term *,
1024                           struct perf_evlist *);
1025};
1026
1027struct bpf_obj_config__map_func bpf_obj_config__map_funcs[] = {
1028        {"value", bpf_map__config_value},
1029        {"event", bpf_map__config_event},
1030};
1031
1032static int
1033config_map_indices_range_check(struct parse_events_term *term,
1034                               struct bpf_map *map,
1035                               const char *map_name)
1036{
1037        struct parse_events_array *array = &term->array;
1038        struct bpf_map_def def;
1039        unsigned int i;
1040        int err;
1041
1042        if (!array->nr_ranges)
1043                return 0;
1044        if (!array->ranges) {
1045                pr_debug("ERROR: map %s: array->nr_ranges is %d but range array is NULL\n",
1046                         map_name, (int)array->nr_ranges);
1047                return -BPF_LOADER_ERRNO__INTERNAL;
1048        }
1049
1050        err = bpf_map__get_def(map, &def);
1051        if (err) {
1052                pr_debug("ERROR: Unable to get map definition from '%s'\n",
1053                         map_name);
1054                return -BPF_LOADER_ERRNO__INTERNAL;
1055        }
1056
1057        for (i = 0; i < array->nr_ranges; i++) {
1058                unsigned int start = array->ranges[i].start;
1059                size_t length = array->ranges[i].length;
1060                unsigned int idx = start + length - 1;
1061
1062                if (idx >= def.max_entries) {
1063                        pr_debug("ERROR: index %d too large\n", idx);
1064                        return -BPF_LOADER_ERRNO__OBJCONF_MAP_IDX2BIG;
1065                }
1066        }
1067        return 0;
1068}
1069
1070static int
1071bpf__obj_config_map(struct bpf_object *obj,
1072                    struct parse_events_term *term,
1073                    struct perf_evlist *evlist,
1074                    int *key_scan_pos)
1075{
1076        /* key is "map:<mapname>.<config opt>" */
1077        char *map_name = strdup(term->config + sizeof("map:") - 1);
1078        struct bpf_map *map;
1079        int err = -BPF_LOADER_ERRNO__OBJCONF_OPT;
1080        char *map_opt;
1081        size_t i;
1082
1083        if (!map_name)
1084                return -ENOMEM;
1085
1086        map_opt = strchr(map_name, '.');
1087        if (!map_opt) {
1088                pr_debug("ERROR: Invalid map config: %s\n", map_name);
1089                goto out;
1090        }
1091
1092        *map_opt++ = '\0';
1093        if (*map_opt == '\0') {
1094                pr_debug("ERROR: Invalid map option: %s\n", term->config);
1095                goto out;
1096        }
1097
1098        map = bpf_object__get_map_by_name(obj, map_name);
1099        if (!map) {
1100                pr_debug("ERROR: Map %s doesn't exist\n", map_name);
1101                err = -BPF_LOADER_ERRNO__OBJCONF_MAP_NOTEXIST;
1102                goto out;
1103        }
1104
1105        *key_scan_pos += strlen(map_opt);
1106        err = config_map_indices_range_check(term, map, map_name);
1107        if (err)
1108                goto out;
1109        *key_scan_pos -= strlen(map_opt);
1110
1111        for (i = 0; i < ARRAY_SIZE(bpf_obj_config__map_funcs); i++) {
1112                struct bpf_obj_config__map_func *func =
1113                                &bpf_obj_config__map_funcs[i];
1114
1115                if (strcmp(map_opt, func->config_opt) == 0) {
1116                        err = func->config_func(map, term, evlist);
1117                        goto out;
1118                }
1119        }
1120
1121        pr_debug("ERROR: Invalid map config option '%s'\n", map_opt);
1122        err = -BPF_LOADER_ERRNO__OBJCONF_MAP_OPT;
1123out:
1124        free(map_name);
1125        if (!err)
1126                key_scan_pos += strlen(map_opt);
1127        return err;
1128}
1129
1130int bpf__config_obj(struct bpf_object *obj,
1131                    struct parse_events_term *term,
1132                    struct perf_evlist *evlist,
1133                    int *error_pos)
1134{
1135        int key_scan_pos = 0;
1136        int err;
1137
1138        if (!obj || !term || !term->config)
1139                return -EINVAL;
1140
1141        if (!prefixcmp(term->config, "map:")) {
1142                key_scan_pos = sizeof("map:") - 1;
1143                err = bpf__obj_config_map(obj, term, evlist, &key_scan_pos);
1144                goto out;
1145        }
1146        err = -BPF_LOADER_ERRNO__OBJCONF_OPT;
1147out:
1148        if (error_pos)
1149                *error_pos = key_scan_pos;
1150        return err;
1151
1152}
1153
1154typedef int (*map_config_func_t)(const char *name, int map_fd,
1155                                 struct bpf_map_def *pdef,
1156                                 struct bpf_map_op *op,
1157                                 void *pkey, void *arg);
1158
1159static int
1160foreach_key_array_all(map_config_func_t func,
1161                      void *arg, const char *name,
1162                      int map_fd, struct bpf_map_def *pdef,
1163                      struct bpf_map_op *op)
1164{
1165        unsigned int i;
1166        int err;
1167
1168        for (i = 0; i < pdef->max_entries; i++) {
1169                err = func(name, map_fd, pdef, op, &i, arg);
1170                if (err) {
1171                        pr_debug("ERROR: failed to insert value to %s[%u]\n",
1172                                 name, i);
1173                        return err;
1174                }
1175        }
1176        return 0;
1177}
1178
1179static int
1180foreach_key_array_ranges(map_config_func_t func, void *arg,
1181                         const char *name, int map_fd,
1182                         struct bpf_map_def *pdef,
1183                         struct bpf_map_op *op)
1184{
1185        unsigned int i, j;
1186        int err;
1187
1188        for (i = 0; i < op->k.array.nr_ranges; i++) {
1189                unsigned int start = op->k.array.ranges[i].start;
1190                size_t length = op->k.array.ranges[i].length;
1191
1192                for (j = 0; j < length; j++) {
1193                        unsigned int idx = start + j;
1194
1195                        err = func(name, map_fd, pdef, op, &idx, arg);
1196                        if (err) {
1197                                pr_debug("ERROR: failed to insert value to %s[%u]\n",
1198                                         name, idx);
1199                                return err;
1200                        }
1201                }
1202        }
1203        return 0;
1204}
1205
1206static int
1207bpf_map_config_foreach_key(struct bpf_map *map,
1208                           map_config_func_t func,
1209                           void *arg)
1210{
1211        int err, map_fd;
1212        const char *name;
1213        struct bpf_map_op *op;
1214        struct bpf_map_def def;
1215        struct bpf_map_priv *priv;
1216
1217        name = bpf_map__get_name(map);
1218
1219        err = bpf_map__get_private(map, (void **)&priv);
1220        if (err) {
1221                pr_debug("ERROR: failed to get private from map %s\n", name);
1222                return -BPF_LOADER_ERRNO__INTERNAL;
1223        }
1224        if (!priv || list_empty(&priv->ops_list)) {
1225                pr_debug("INFO: nothing to config for map %s\n", name);
1226                return 0;
1227        }
1228
1229        err = bpf_map__get_def(map, &def);
1230        if (err) {
1231                pr_debug("ERROR: failed to get definition from map %s\n", name);
1232                return -BPF_LOADER_ERRNO__INTERNAL;
1233        }
1234        map_fd = bpf_map__get_fd(map);
1235        if (map_fd < 0) {
1236                pr_debug("ERROR: failed to get fd from map %s\n", name);
1237                return map_fd;
1238        }
1239
1240        list_for_each_entry(op, &priv->ops_list, list) {
1241                switch (def.type) {
1242                case BPF_MAP_TYPE_ARRAY:
1243                case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
1244                        switch (op->key_type) {
1245                        case BPF_MAP_KEY_ALL:
1246                                err = foreach_key_array_all(func, arg, name,
1247                                                            map_fd, &def, op);
1248                                break;
1249                        case BPF_MAP_KEY_RANGES:
1250                                err = foreach_key_array_ranges(func, arg, name,
1251                                                               map_fd, &def,
1252                                                               op);
1253                                break;
1254                        default:
1255                                pr_debug("ERROR: keytype for map '%s' invalid\n",
1256                                         name);
1257                                return -BPF_LOADER_ERRNO__INTERNAL;
1258                        }
1259                        if (err)
1260                                return err;
1261                        break;
1262                default:
1263                        pr_debug("ERROR: type of '%s' incorrect\n", name);
1264                        return -BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE;
1265                }
1266        }
1267
1268        return 0;
1269}
1270
1271static int
1272apply_config_value_for_key(int map_fd, void *pkey,
1273                           size_t val_size, u64 val)
1274{
1275        int err = 0;
1276
1277        switch (val_size) {
1278        case 1: {
1279                u8 _val = (u8)(val);
1280                err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1281                break;
1282        }
1283        case 2: {
1284                u16 _val = (u16)(val);
1285                err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1286                break;
1287        }
1288        case 4: {
1289                u32 _val = (u32)(val);
1290                err = bpf_map_update_elem(map_fd, pkey, &_val, BPF_ANY);
1291                break;
1292        }
1293        case 8: {
1294                err = bpf_map_update_elem(map_fd, pkey, &val, BPF_ANY);
1295                break;
1296        }
1297        default:
1298                pr_debug("ERROR: invalid value size\n");
1299                return -BPF_LOADER_ERRNO__OBJCONF_MAP_VALUESIZE;
1300        }
1301        if (err && errno)
1302                err = -errno;
1303        return err;
1304}
1305
1306static int
1307apply_config_evsel_for_key(const char *name, int map_fd, void *pkey,
1308                           struct perf_evsel *evsel)
1309{
1310        struct xyarray *xy = evsel->fd;
1311        struct perf_event_attr *attr;
1312        unsigned int key, events;
1313        bool check_pass = false;
1314        int *evt_fd;
1315        int err;
1316
1317        if (!xy) {
1318                pr_debug("ERROR: evsel not ready for map %s\n", name);
1319                return -BPF_LOADER_ERRNO__INTERNAL;
1320        }
1321
1322        if (xy->row_size / xy->entry_size != 1) {
1323                pr_debug("ERROR: Dimension of target event is incorrect for map %s\n",
1324                         name);
1325                return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM;
1326        }
1327
1328        attr = &evsel->attr;
1329        if (attr->inherit) {
1330                pr_debug("ERROR: Can't put inherit event into map %s\n", name);
1331                return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH;
1332        }
1333
1334        if (perf_evsel__is_bpf_output(evsel))
1335                check_pass = true;
1336        if (attr->type == PERF_TYPE_RAW)
1337                check_pass = true;
1338        if (attr->type == PERF_TYPE_HARDWARE)
1339                check_pass = true;
1340        if (!check_pass) {
1341                pr_debug("ERROR: Event type is wrong for map %s\n", name);
1342                return -BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE;
1343        }
1344
1345        events = xy->entries / (xy->row_size / xy->entry_size);
1346        key = *((unsigned int *)pkey);
1347        if (key >= events) {
1348                pr_debug("ERROR: there is no event %d for map %s\n",
1349                         key, name);
1350                return -BPF_LOADER_ERRNO__OBJCONF_MAP_MAPSIZE;
1351        }
1352        evt_fd = xyarray__entry(xy, key, 0);
1353        err = bpf_map_update_elem(map_fd, pkey, evt_fd, BPF_ANY);
1354        if (err && errno)
1355                err = -errno;
1356        return err;
1357}
1358
1359static int
1360apply_obj_config_map_for_key(const char *name, int map_fd,
1361                             struct bpf_map_def *pdef __maybe_unused,
1362                             struct bpf_map_op *op,
1363                             void *pkey, void *arg __maybe_unused)
1364{
1365        int err;
1366
1367        switch (op->op_type) {
1368        case BPF_MAP_OP_SET_VALUE:
1369                err = apply_config_value_for_key(map_fd, pkey,
1370                                                 pdef->value_size,
1371                                                 op->v.value);
1372                break;
1373        case BPF_MAP_OP_SET_EVSEL:
1374                err = apply_config_evsel_for_key(name, map_fd, pkey,
1375                                                 op->v.evsel);
1376                break;
1377        default:
1378                pr_debug("ERROR: unknown value type for '%s'\n", name);
1379                err = -BPF_LOADER_ERRNO__INTERNAL;
1380        }
1381        return err;
1382}
1383
1384static int
1385apply_obj_config_map(struct bpf_map *map)
1386{
1387        return bpf_map_config_foreach_key(map,
1388                                          apply_obj_config_map_for_key,
1389                                          NULL);
1390}
1391
1392static int
1393apply_obj_config_object(struct bpf_object *obj)
1394{
1395        struct bpf_map *map;
1396        int err;
1397
1398        bpf_map__for_each(map, obj) {
1399                err = apply_obj_config_map(map);
1400                if (err)
1401                        return err;
1402        }
1403        return 0;
1404}
1405
1406int bpf__apply_obj_config(void)
1407{
1408        struct bpf_object *obj, *tmp;
1409        int err;
1410
1411        bpf_object__for_each_safe(obj, tmp) {
1412                err = apply_obj_config_object(obj);
1413                if (err)
1414                        return err;
1415        }
1416
1417        return 0;
1418}
1419
1420#define ERRNO_OFFSET(e)         ((e) - __BPF_LOADER_ERRNO__START)
1421#define ERRCODE_OFFSET(c)       ERRNO_OFFSET(BPF_LOADER_ERRNO__##c)
1422#define NR_ERRNO        (__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
1423
1424static const char *bpf_loader_strerror_table[NR_ERRNO] = {
1425        [ERRCODE_OFFSET(CONFIG)]        = "Invalid config string",
1426        [ERRCODE_OFFSET(GROUP)]         = "Invalid group name",
1427        [ERRCODE_OFFSET(EVENTNAME)]     = "No event name found in config string",
1428        [ERRCODE_OFFSET(INTERNAL)]      = "BPF loader internal error",
1429        [ERRCODE_OFFSET(COMPILE)]       = "Error when compiling BPF scriptlet",
1430        [ERRCODE_OFFSET(PROGCONF_TERM)] = "Invalid program config term in config string",
1431        [ERRCODE_OFFSET(PROLOGUE)]      = "Failed to generate prologue",
1432        [ERRCODE_OFFSET(PROLOGUE2BIG)]  = "Prologue too big for program",
1433        [ERRCODE_OFFSET(PROLOGUEOOB)]   = "Offset out of bound for prologue",
1434        [ERRCODE_OFFSET(OBJCONF_OPT)]   = "Invalid object config option",
1435        [ERRCODE_OFFSET(OBJCONF_CONF)]  = "Config value not set (missing '=')",
1436        [ERRCODE_OFFSET(OBJCONF_MAP_OPT)]       = "Invalid object map config option",
1437        [ERRCODE_OFFSET(OBJCONF_MAP_NOTEXIST)]  = "Target map doesn't exist",
1438        [ERRCODE_OFFSET(OBJCONF_MAP_VALUE)]     = "Incorrect value type for map",
1439        [ERRCODE_OFFSET(OBJCONF_MAP_TYPE)]      = "Incorrect map type",
1440        [ERRCODE_OFFSET(OBJCONF_MAP_KEYSIZE)]   = "Incorrect map key size",
1441        [ERRCODE_OFFSET(OBJCONF_MAP_VALUESIZE)] = "Incorrect map value size",
1442        [ERRCODE_OFFSET(OBJCONF_MAP_NOEVT)]     = "Event not found for map setting",
1443        [ERRCODE_OFFSET(OBJCONF_MAP_MAPSIZE)]   = "Invalid map size for event setting",
1444        [ERRCODE_OFFSET(OBJCONF_MAP_EVTDIM)]    = "Event dimension too large",
1445        [ERRCODE_OFFSET(OBJCONF_MAP_EVTINH)]    = "Doesn't support inherit event",
1446        [ERRCODE_OFFSET(OBJCONF_MAP_EVTTYPE)]   = "Wrong event type for map",
1447        [ERRCODE_OFFSET(OBJCONF_MAP_IDX2BIG)]   = "Index too large",
1448};
1449
1450static int
1451bpf_loader_strerror(int err, char *buf, size_t size)
1452{
1453        char sbuf[STRERR_BUFSIZE];
1454        const char *msg;
1455
1456        if (!buf || !size)
1457                return -1;
1458
1459        err = err > 0 ? err : -err;
1460
1461        if (err >= __LIBBPF_ERRNO__START)
1462                return libbpf_strerror(err, buf, size);
1463
1464        if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) {
1465                msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)];
1466                snprintf(buf, size, "%s", msg);
1467                buf[size - 1] = '\0';
1468                return 0;
1469        }
1470
1471        if (err >= __BPF_LOADER_ERRNO__END)
1472                snprintf(buf, size, "Unknown bpf loader error %d", err);
1473        else
1474                snprintf(buf, size, "%s",
1475                         strerror_r(err, sbuf, sizeof(sbuf)));
1476
1477        buf[size - 1] = '\0';
1478        return -1;
1479}
1480
1481#define bpf__strerror_head(err, buf, size) \
1482        char sbuf[STRERR_BUFSIZE], *emsg;\
1483        if (!size)\
1484                return 0;\
1485        if (err < 0)\
1486                err = -err;\
1487        bpf_loader_strerror(err, sbuf, sizeof(sbuf));\
1488        emsg = sbuf;\
1489        switch (err) {\
1490        default:\
1491                scnprintf(buf, size, "%s", emsg);\
1492                break;
1493
1494#define bpf__strerror_entry(val, fmt...)\
1495        case val: {\
1496                scnprintf(buf, size, fmt);\
1497                break;\
1498        }
1499
1500#define bpf__strerror_end(buf, size)\
1501        }\
1502        buf[size - 1] = '\0';
1503
1504int bpf__strerror_prepare_load(const char *filename, bool source,
1505                               int err, char *buf, size_t size)
1506{
1507        size_t n;
1508        int ret;
1509
1510        n = snprintf(buf, size, "Failed to load %s%s: ",
1511                         filename, source ? " from source" : "");
1512        if (n >= size) {
1513                buf[size - 1] = '\0';
1514                return 0;
1515        }
1516        buf += n;
1517        size -= n;
1518
1519        ret = bpf_loader_strerror(err, buf, size);
1520        buf[size - 1] = '\0';
1521        return ret;
1522}
1523
1524int bpf__strerror_probe(struct bpf_object *obj __maybe_unused,
1525                        int err, char *buf, size_t size)
1526{
1527        bpf__strerror_head(err, buf, size);
1528        case BPF_LOADER_ERRNO__PROGCONF_TERM: {
1529                scnprintf(buf, size, "%s (add -v to see detail)", emsg);
1530                break;
1531        }
1532        bpf__strerror_entry(EEXIST, "Probe point exist. Try 'perf probe -d \"*\"' and set 'force=yes'");
1533        bpf__strerror_entry(EACCES, "You need to be root");
1534        bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0");
1535        bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file");
1536        bpf__strerror_end(buf, size);
1537        return 0;
1538}
1539
1540int bpf__strerror_load(struct bpf_object *obj,
1541                       int err, char *buf, size_t size)
1542{
1543        bpf__strerror_head(err, buf, size);
1544        case LIBBPF_ERRNO__KVER: {
1545                unsigned int obj_kver = bpf_object__get_kversion(obj);
1546                unsigned int real_kver;
1547
1548                if (fetch_kernel_version(&real_kver, NULL, 0)) {
1549                        scnprintf(buf, size, "Unable to fetch kernel version");
1550                        break;
1551                }
1552
1553                if (obj_kver != real_kver) {
1554                        scnprintf(buf, size,
1555                                  "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")",
1556                                  KVER_PARAM(obj_kver),
1557                                  KVER_PARAM(real_kver));
1558                        break;
1559                }
1560
1561                scnprintf(buf, size, "Failed to load program for unknown reason");
1562                break;
1563        }
1564        bpf__strerror_end(buf, size);
1565        return 0;
1566}
1567
1568int bpf__strerror_config_obj(struct bpf_object *obj __maybe_unused,
1569                             struct parse_events_term *term __maybe_unused,
1570                             struct perf_evlist *evlist __maybe_unused,
1571                             int *error_pos __maybe_unused, int err,
1572                             char *buf, size_t size)
1573{
1574        bpf__strerror_head(err, buf, size);
1575        bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_TYPE,
1576                            "Can't use this config term with this map type");
1577        bpf__strerror_end(buf, size);
1578        return 0;
1579}
1580
1581int bpf__strerror_apply_obj_config(int err, char *buf, size_t size)
1582{
1583        bpf__strerror_head(err, buf, size);
1584        bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTDIM,
1585                            "Cannot set event to BPF map in multi-thread tracing");
1586        bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTINH,
1587                            "%s (Hint: use -i to turn off inherit)", emsg);
1588        bpf__strerror_entry(BPF_LOADER_ERRNO__OBJCONF_MAP_EVTTYPE,
1589                            "Can only put raw, hardware and BPF output event into a BPF map");
1590        bpf__strerror_end(buf, size);
1591        return 0;
1592}
1593