LXR linux/tools/testing/selftests/bpf/prog

   1// SPDX-License-Identifier: GPL-2.0
   2#include <test_progs.h>
   3#include <network_helpers.h>
   4
   5/* test_tailcall_1 checks basic functionality by patching multiple locations
   6 * in a single program for a single tail call slot with nop->jmp, jmp->nop
   7 * and jmp->jmp rewrites. Also checks for nop->nop.
   8 */
   9static void test_tailcall_1(void)
  10{
  11        int err, map_fd, prog_fd, main_fd, i, j;
  12        struct bpf_map *prog_array;
  13        struct bpf_program *prog;
  14        struct bpf_object *obj;
  15        char prog_name[32];
  16        char buff[128] = {};
  17        LIBBPF_OPTS(bpf_test_run_opts, topts,
  18                .data_in = buff,
  19                .data_size_in = sizeof(buff),
  20                .repeat = 1,
  21        );
  22
  23        err = bpf_prog_test_load("tailcall1.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
  24                            &prog_fd);
  25        if (CHECK_FAIL(err))
  26                return;
  27
  28        prog = bpf_object__find_program_by_name(obj, "entry");
  29        if (CHECK_FAIL(!prog))
  30                goto out;
  31
  32        main_fd = bpf_program__fd(prog);
  33        if (CHECK_FAIL(main_fd < 0))
  34                goto out;
  35
  36        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
  37        if (CHECK_FAIL(!prog_array))
  38                goto out;
  39
  40        map_fd = bpf_map__fd(prog_array);
  41        if (CHECK_FAIL(map_fd < 0))
  42                goto out;
  43
  44        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
  45                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
  46
  47                prog = bpf_object__find_program_by_name(obj, prog_name);
  48                if (CHECK_FAIL(!prog))
  49                        goto out;
  50
  51                prog_fd = bpf_program__fd(prog);
  52                if (CHECK_FAIL(prog_fd < 0))
  53                        goto out;
  54
  55                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
  56                if (CHECK_FAIL(err))
  57                        goto out;
  58        }
  59
  60        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
  61                err = bpf_prog_test_run_opts(main_fd, &topts);
  62                ASSERT_OK(err, "tailcall");
  63                ASSERT_EQ(topts.retval, i, "tailcall retval");
  64
  65                err = bpf_map_delete_elem(map_fd, &i);
  66                if (CHECK_FAIL(err))
  67                        goto out;
  68        }
  69
  70        err = bpf_prog_test_run_opts(main_fd, &topts);
  71        ASSERT_OK(err, "tailcall");
  72        ASSERT_EQ(topts.retval, 3, "tailcall retval");
  73
  74        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
  75                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
  76
  77                prog = bpf_object__find_program_by_name(obj, prog_name);
  78                if (CHECK_FAIL(!prog))
  79                        goto out;
  80
  81                prog_fd = bpf_program__fd(prog);
  82                if (CHECK_FAIL(prog_fd < 0))
  83                        goto out;
  84
  85                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
  86                if (CHECK_FAIL(err))
  87                        goto out;
  88        }
  89
  90        err = bpf_prog_test_run_opts(main_fd, &topts);
  91        ASSERT_OK(err, "tailcall");
  92        ASSERT_OK(topts.retval, "tailcall retval");
  93
  94        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
  95                j = bpf_map__max_entries(prog_array) - 1 - i;
  96                snprintf(prog_name, sizeof(prog_name), "classifier_%d", j);
  97
  98                prog = bpf_object__find_program_by_name(obj, prog_name);
  99                if (CHECK_FAIL(!prog))
 100                        goto out;
 101
 102                prog_fd = bpf_program__fd(prog);
 103                if (CHECK_FAIL(prog_fd < 0))
 104                        goto out;
 105
 106                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 107                if (CHECK_FAIL(err))
 108                        goto out;
 109        }
 110
 111        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
 112                j = bpf_map__max_entries(prog_array) - 1 - i;
 113
 114                err = bpf_prog_test_run_opts(main_fd, &topts);
 115                ASSERT_OK(err, "tailcall");
 116                ASSERT_EQ(topts.retval, j, "tailcall retval");
 117
 118                err = bpf_map_delete_elem(map_fd, &i);
 119                if (CHECK_FAIL(err))
 120                        goto out;
 121        }
 122
 123        err = bpf_prog_test_run_opts(main_fd, &topts);
 124        ASSERT_OK(err, "tailcall");
 125        ASSERT_EQ(topts.retval, 3, "tailcall retval");
 126
 127        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
 128                err = bpf_map_delete_elem(map_fd, &i);
 129                if (CHECK_FAIL(err >= 0 || errno != ENOENT))
 130                        goto out;
 131
 132                err = bpf_prog_test_run_opts(main_fd, &topts);
 133                ASSERT_OK(err, "tailcall");
 134                ASSERT_EQ(topts.retval, 3, "tailcall retval");
 135        }
 136
 137out:
 138        bpf_object__close(obj);
 139}
 140
 141/* test_tailcall_2 checks that patching multiple programs for a single
 142 * tail call slot works. It also jumps through several programs and tests
 143 * the tail call limit counter.
 144 */
 145static void test_tailcall_2(void)
 146{
 147        int err, map_fd, prog_fd, main_fd, i;
 148        struct bpf_map *prog_array;
 149        struct bpf_program *prog;
 150        struct bpf_object *obj;
 151        char prog_name[32];
 152        char buff[128] = {};
 153        LIBBPF_OPTS(bpf_test_run_opts, topts,
 154                .data_in = buff,
 155                .data_size_in = sizeof(buff),
 156                .repeat = 1,
 157        );
 158
 159        err = bpf_prog_test_load("tailcall2.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 160                            &prog_fd);
 161        if (CHECK_FAIL(err))
 162                return;
 163
 164        prog = bpf_object__find_program_by_name(obj, "entry");
 165        if (CHECK_FAIL(!prog))
 166                goto out;
 167
 168        main_fd = bpf_program__fd(prog);
 169        if (CHECK_FAIL(main_fd < 0))
 170                goto out;
 171
 172        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 173        if (CHECK_FAIL(!prog_array))
 174                goto out;
 175
 176        map_fd = bpf_map__fd(prog_array);
 177        if (CHECK_FAIL(map_fd < 0))
 178                goto out;
 179
 180        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
 181                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
 182
 183                prog = bpf_object__find_program_by_name(obj, prog_name);
 184                if (CHECK_FAIL(!prog))
 185                        goto out;
 186
 187                prog_fd = bpf_program__fd(prog);
 188                if (CHECK_FAIL(prog_fd < 0))
 189                        goto out;
 190
 191                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 192                if (CHECK_FAIL(err))
 193                        goto out;
 194        }
 195
 196        err = bpf_prog_test_run_opts(main_fd, &topts);
 197        ASSERT_OK(err, "tailcall");
 198        ASSERT_EQ(topts.retval, 2, "tailcall retval");
 199
 200        i = 2;
 201        err = bpf_map_delete_elem(map_fd, &i);
 202        if (CHECK_FAIL(err))
 203                goto out;
 204
 205        err = bpf_prog_test_run_opts(main_fd, &topts);
 206        ASSERT_OK(err, "tailcall");
 207        ASSERT_EQ(topts.retval, 1, "tailcall retval");
 208
 209        i = 0;
 210        err = bpf_map_delete_elem(map_fd, &i);
 211        if (CHECK_FAIL(err))
 212                goto out;
 213
 214        err = bpf_prog_test_run_opts(main_fd, &topts);
 215        ASSERT_OK(err, "tailcall");
 216        ASSERT_EQ(topts.retval, 3, "tailcall retval");
 217out:
 218        bpf_object__close(obj);
 219}
 220
 221static void test_tailcall_count(const char *which)
 222{
 223        int err, map_fd, prog_fd, main_fd, data_fd, i, val;
 224        struct bpf_map *prog_array, *data_map;
 225        struct bpf_program *prog;
 226        struct bpf_object *obj;
 227        char buff[128] = {};
 228        LIBBPF_OPTS(bpf_test_run_opts, topts,
 229                .data_in = buff,
 230                .data_size_in = sizeof(buff),
 231                .repeat = 1,
 232        );
 233
 234        err = bpf_prog_test_load(which, BPF_PROG_TYPE_SCHED_CLS, &obj,
 235                            &prog_fd);
 236        if (CHECK_FAIL(err))
 237                return;
 238
 239        prog = bpf_object__find_program_by_name(obj, "entry");
 240        if (CHECK_FAIL(!prog))
 241                goto out;
 242
 243        main_fd = bpf_program__fd(prog);
 244        if (CHECK_FAIL(main_fd < 0))
 245                goto out;
 246
 247        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 248        if (CHECK_FAIL(!prog_array))
 249                goto out;
 250
 251        map_fd = bpf_map__fd(prog_array);
 252        if (CHECK_FAIL(map_fd < 0))
 253                goto out;
 254
 255        prog = bpf_object__find_program_by_name(obj, "classifier_0");
 256        if (CHECK_FAIL(!prog))
 257                goto out;
 258
 259        prog_fd = bpf_program__fd(prog);
 260        if (CHECK_FAIL(prog_fd < 0))
 261                goto out;
 262
 263        i = 0;
 264        err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 265        if (CHECK_FAIL(err))
 266                goto out;
 267
 268        err = bpf_prog_test_run_opts(main_fd, &topts);
 269        ASSERT_OK(err, "tailcall");
 270        ASSERT_EQ(topts.retval, 1, "tailcall retval");
 271
 272        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 273        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 274                return;
 275
 276        data_fd = bpf_map__fd(data_map);
 277        if (CHECK_FAIL(map_fd < 0))
 278                return;
 279
 280        i = 0;
 281        err = bpf_map_lookup_elem(data_fd, &i, &val);
 282        ASSERT_OK(err, "tailcall count");
 283        ASSERT_EQ(val, 33, "tailcall count");
 284
 285        i = 0;
 286        err = bpf_map_delete_elem(map_fd, &i);
 287        if (CHECK_FAIL(err))
 288                goto out;
 289
 290        err = bpf_prog_test_run_opts(main_fd, &topts);
 291        ASSERT_OK(err, "tailcall");
 292        ASSERT_OK(topts.retval, "tailcall retval");
 293out:
 294        bpf_object__close(obj);
 295}
 296
 297/* test_tailcall_3 checks that the count value of the tail call limit
 298 * enforcement matches with expectations. JIT uses direct jump.
 299 */
 300static void test_tailcall_3(void)
 301{
 302        test_tailcall_count("tailcall3.o");
 303}
 304
 305/* test_tailcall_6 checks that the count value of the tail call limit
 306 * enforcement matches with expectations. JIT uses indirect jump.
 307 */
 308static void test_tailcall_6(void)
 309{
 310        test_tailcall_count("tailcall6.o");
 311}
 312
 313/* test_tailcall_4 checks that the kernel properly selects indirect jump
 314 * for the case where the key is not known. Latter is passed via global
 315 * data to select different targets we can compare return value of.
 316 */
 317static void test_tailcall_4(void)
 318{
 319        int err, map_fd, prog_fd, main_fd, data_fd, i;
 320        struct bpf_map *prog_array, *data_map;
 321        struct bpf_program *prog;
 322        struct bpf_object *obj;
 323        static const int zero = 0;
 324        char buff[128] = {};
 325        char prog_name[32];
 326        LIBBPF_OPTS(bpf_test_run_opts, topts,
 327                .data_in = buff,
 328                .data_size_in = sizeof(buff),
 329                .repeat = 1,
 330        );
 331
 332        err = bpf_prog_test_load("tailcall4.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 333                            &prog_fd);
 334        if (CHECK_FAIL(err))
 335                return;
 336
 337        prog = bpf_object__find_program_by_name(obj, "entry");
 338        if (CHECK_FAIL(!prog))
 339                goto out;
 340
 341        main_fd = bpf_program__fd(prog);
 342        if (CHECK_FAIL(main_fd < 0))
 343                goto out;
 344
 345        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 346        if (CHECK_FAIL(!prog_array))
 347                goto out;
 348
 349        map_fd = bpf_map__fd(prog_array);
 350        if (CHECK_FAIL(map_fd < 0))
 351                goto out;
 352
 353        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 354        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 355                return;
 356
 357        data_fd = bpf_map__fd(data_map);
 358        if (CHECK_FAIL(map_fd < 0))
 359                return;
 360
 361        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
 362                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
 363
 364                prog = bpf_object__find_program_by_name(obj, prog_name);
 365                if (CHECK_FAIL(!prog))
 366                        goto out;
 367
 368                prog_fd = bpf_program__fd(prog);
 369                if (CHECK_FAIL(prog_fd < 0))
 370                        goto out;
 371
 372                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 373                if (CHECK_FAIL(err))
 374                        goto out;
 375        }
 376
 377        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
 378                err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY);
 379                if (CHECK_FAIL(err))
 380                        goto out;
 381
 382                err = bpf_prog_test_run_opts(main_fd, &topts);
 383                ASSERT_OK(err, "tailcall");
 384                ASSERT_EQ(topts.retval, i, "tailcall retval");
 385        }
 386
 387        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
 388                err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY);
 389                if (CHECK_FAIL(err))
 390                        goto out;
 391
 392                err = bpf_map_delete_elem(map_fd, &i);
 393                if (CHECK_FAIL(err))
 394                        goto out;
 395
 396                err = bpf_prog_test_run_opts(main_fd, &topts);
 397                ASSERT_OK(err, "tailcall");
 398                ASSERT_EQ(topts.retval, 3, "tailcall retval");
 399        }
 400out:
 401        bpf_object__close(obj);
 402}
 403
 404/* test_tailcall_5 probes similarly to test_tailcall_4 that the kernel generates
 405 * an indirect jump when the keys are const but different from different branches.
 406 */
 407static void test_tailcall_5(void)
 408{
 409        int err, map_fd, prog_fd, main_fd, data_fd, i, key[] = { 1111, 1234, 5678 };
 410        struct bpf_map *prog_array, *data_map;
 411        struct bpf_program *prog;
 412        struct bpf_object *obj;
 413        static const int zero = 0;
 414        char buff[128] = {};
 415        char prog_name[32];
 416        LIBBPF_OPTS(bpf_test_run_opts, topts,
 417                .data_in = buff,
 418                .data_size_in = sizeof(buff),
 419                .repeat = 1,
 420        );
 421
 422        err = bpf_prog_test_load("tailcall5.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 423                            &prog_fd);
 424        if (CHECK_FAIL(err))
 425                return;
 426
 427        prog = bpf_object__find_program_by_name(obj, "entry");
 428        if (CHECK_FAIL(!prog))
 429                goto out;
 430
 431        main_fd = bpf_program__fd(prog);
 432        if (CHECK_FAIL(main_fd < 0))
 433                goto out;
 434
 435        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 436        if (CHECK_FAIL(!prog_array))
 437                goto out;
 438
 439        map_fd = bpf_map__fd(prog_array);
 440        if (CHECK_FAIL(map_fd < 0))
 441                goto out;
 442
 443        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 444        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 445                return;
 446
 447        data_fd = bpf_map__fd(data_map);
 448        if (CHECK_FAIL(map_fd < 0))
 449                return;
 450
 451        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
 452                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
 453
 454                prog = bpf_object__find_program_by_name(obj, prog_name);
 455                if (CHECK_FAIL(!prog))
 456                        goto out;
 457
 458                prog_fd = bpf_program__fd(prog);
 459                if (CHECK_FAIL(prog_fd < 0))
 460                        goto out;
 461
 462                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 463                if (CHECK_FAIL(err))
 464                        goto out;
 465        }
 466
 467        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
 468                err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY);
 469                if (CHECK_FAIL(err))
 470                        goto out;
 471
 472                err = bpf_prog_test_run_opts(main_fd, &topts);
 473                ASSERT_OK(err, "tailcall");
 474                ASSERT_EQ(topts.retval, i, "tailcall retval");
 475        }
 476
 477        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
 478                err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY);
 479                if (CHECK_FAIL(err))
 480                        goto out;
 481
 482                err = bpf_map_delete_elem(map_fd, &i);
 483                if (CHECK_FAIL(err))
 484                        goto out;
 485
 486                err = bpf_prog_test_run_opts(main_fd, &topts);
 487                ASSERT_OK(err, "tailcall");
 488                ASSERT_EQ(topts.retval, 3, "tailcall retval");
 489        }
 490out:
 491        bpf_object__close(obj);
 492}
 493
 494/* test_tailcall_bpf2bpf_1 purpose is to make sure that tailcalls are working
 495 * correctly in correlation with BPF subprograms
 496 */
 497static void test_tailcall_bpf2bpf_1(void)
 498{
 499        int err, map_fd, prog_fd, main_fd, i;
 500        struct bpf_map *prog_array;
 501        struct bpf_program *prog;
 502        struct bpf_object *obj;
 503        char prog_name[32];
 504        LIBBPF_OPTS(bpf_test_run_opts, topts,
 505                .data_in = &pkt_v4,
 506                .data_size_in = sizeof(pkt_v4),
 507                .repeat = 1,
 508        );
 509
 510        err = bpf_prog_test_load("tailcall_bpf2bpf1.o", BPF_PROG_TYPE_SCHED_CLS,
 511                            &obj, &prog_fd);
 512        if (CHECK_FAIL(err))
 513                return;
 514
 515        prog = bpf_object__find_program_by_name(obj, "entry");
 516        if (CHECK_FAIL(!prog))
 517                goto out;
 518
 519        main_fd = bpf_program__fd(prog);
 520        if (CHECK_FAIL(main_fd < 0))
 521                goto out;
 522
 523        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 524        if (CHECK_FAIL(!prog_array))
 525                goto out;
 526
 527        map_fd = bpf_map__fd(prog_array);
 528        if (CHECK_FAIL(map_fd < 0))
 529                goto out;
 530
 531        /* nop -> jmp */
 532        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
 533                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
 534
 535                prog = bpf_object__find_program_by_name(obj, prog_name);
 536                if (CHECK_FAIL(!prog))
 537                        goto out;
 538
 539                prog_fd = bpf_program__fd(prog);
 540                if (CHECK_FAIL(prog_fd < 0))
 541                        goto out;
 542
 543                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 544                if (CHECK_FAIL(err))
 545                        goto out;
 546        }
 547
 548        err = bpf_prog_test_run_opts(main_fd, &topts);
 549        ASSERT_OK(err, "tailcall");
 550        ASSERT_EQ(topts.retval, 1, "tailcall retval");
 551
 552        /* jmp -> nop, call subprog that will do tailcall */
 553        i = 1;
 554        err = bpf_map_delete_elem(map_fd, &i);
 555        if (CHECK_FAIL(err))
 556                goto out;
 557
 558        err = bpf_prog_test_run_opts(main_fd, &topts);
 559        ASSERT_OK(err, "tailcall");
 560        ASSERT_OK(topts.retval, "tailcall retval");
 561
 562        /* make sure that subprog can access ctx and entry prog that
 563         * called this subprog can properly return
 564         */
 565        i = 0;
 566        err = bpf_map_delete_elem(map_fd, &i);
 567        if (CHECK_FAIL(err))
 568                goto out;
 569
 570        err = bpf_prog_test_run_opts(main_fd, &topts);
 571        ASSERT_OK(err, "tailcall");
 572        ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 2, "tailcall retval");
 573out:
 574        bpf_object__close(obj);
 575}
 576
 577/* test_tailcall_bpf2bpf_2 checks that the count value of the tail call limit
 578 * enforcement matches with expectations when tailcall is preceded with
 579 * bpf2bpf call.
 580 */
 581static void test_tailcall_bpf2bpf_2(void)
 582{
 583        int err, map_fd, prog_fd, main_fd, data_fd, i, val;
 584        struct bpf_map *prog_array, *data_map;
 585        struct bpf_program *prog;
 586        struct bpf_object *obj;
 587        char buff[128] = {};
 588        LIBBPF_OPTS(bpf_test_run_opts, topts,
 589                .data_in = buff,
 590                .data_size_in = sizeof(buff),
 591                .repeat = 1,
 592        );
 593
 594        err = bpf_prog_test_load("tailcall_bpf2bpf2.o", BPF_PROG_TYPE_SCHED_CLS,
 595                            &obj, &prog_fd);
 596        if (CHECK_FAIL(err))
 597                return;
 598
 599        prog = bpf_object__find_program_by_name(obj, "entry");
 600        if (CHECK_FAIL(!prog))
 601                goto out;
 602
 603        main_fd = bpf_program__fd(prog);
 604        if (CHECK_FAIL(main_fd < 0))
 605                goto out;
 606
 607        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 608        if (CHECK_FAIL(!prog_array))
 609                goto out;
 610
 611        map_fd = bpf_map__fd(prog_array);
 612        if (CHECK_FAIL(map_fd < 0))
 613                goto out;
 614
 615        prog = bpf_object__find_program_by_name(obj, "classifier_0");
 616        if (CHECK_FAIL(!prog))
 617                goto out;
 618
 619        prog_fd = bpf_program__fd(prog);
 620        if (CHECK_FAIL(prog_fd < 0))
 621                goto out;
 622
 623        i = 0;
 624        err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 625        if (CHECK_FAIL(err))
 626                goto out;
 627
 628        err = bpf_prog_test_run_opts(main_fd, &topts);
 629        ASSERT_OK(err, "tailcall");
 630        ASSERT_EQ(topts.retval, 1, "tailcall retval");
 631
 632        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 633        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 634                return;
 635
 636        data_fd = bpf_map__fd(data_map);
 637        if (CHECK_FAIL(map_fd < 0))
 638                return;
 639
 640        i = 0;
 641        err = bpf_map_lookup_elem(data_fd, &i, &val);
 642        ASSERT_OK(err, "tailcall count");
 643        ASSERT_EQ(val, 33, "tailcall count");
 644
 645        i = 0;
 646        err = bpf_map_delete_elem(map_fd, &i);
 647        if (CHECK_FAIL(err))
 648                goto out;
 649
 650        err = bpf_prog_test_run_opts(main_fd, &topts);
 651        ASSERT_OK(err, "tailcall");
 652        ASSERT_OK(topts.retval, "tailcall retval");
 653out:
 654        bpf_object__close(obj);
 655}
 656
 657/* test_tailcall_bpf2bpf_3 checks that non-trivial amount of stack (up to
 658 * 256 bytes) can be used within bpf subprograms that have the tailcalls
 659 * in them
 660 */
 661static void test_tailcall_bpf2bpf_3(void)
 662{
 663        int err, map_fd, prog_fd, main_fd, i;
 664        struct bpf_map *prog_array;
 665        struct bpf_program *prog;
 666        struct bpf_object *obj;
 667        char prog_name[32];
 668        LIBBPF_OPTS(bpf_test_run_opts, topts,
 669                .data_in = &pkt_v4,
 670                .data_size_in = sizeof(pkt_v4),
 671                .repeat = 1,
 672        );
 673
 674        err = bpf_prog_test_load("tailcall_bpf2bpf3.o", BPF_PROG_TYPE_SCHED_CLS,
 675                            &obj, &prog_fd);
 676        if (CHECK_FAIL(err))
 677                return;
 678
 679        prog = bpf_object__find_program_by_name(obj, "entry");
 680        if (CHECK_FAIL(!prog))
 681                goto out;
 682
 683        main_fd = bpf_program__fd(prog);
 684        if (CHECK_FAIL(main_fd < 0))
 685                goto out;
 686
 687        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 688        if (CHECK_FAIL(!prog_array))
 689                goto out;
 690
 691        map_fd = bpf_map__fd(prog_array);
 692        if (CHECK_FAIL(map_fd < 0))
 693                goto out;
 694
 695        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
 696                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
 697
 698                prog = bpf_object__find_program_by_name(obj, prog_name);
 699                if (CHECK_FAIL(!prog))
 700                        goto out;
 701
 702                prog_fd = bpf_program__fd(prog);
 703                if (CHECK_FAIL(prog_fd < 0))
 704                        goto out;
 705
 706                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 707                if (CHECK_FAIL(err))
 708                        goto out;
 709        }
 710
 711        err = bpf_prog_test_run_opts(main_fd, &topts);
 712        ASSERT_OK(err, "tailcall");
 713        ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 3, "tailcall retval");
 714
 715        i = 1;
 716        err = bpf_map_delete_elem(map_fd, &i);
 717        if (CHECK_FAIL(err))
 718                goto out;
 719
 720        err = bpf_prog_test_run_opts(main_fd, &topts);
 721        ASSERT_OK(err, "tailcall");
 722        ASSERT_EQ(topts.retval, sizeof(pkt_v4), "tailcall retval");
 723
 724        i = 0;
 725        err = bpf_map_delete_elem(map_fd, &i);
 726        if (CHECK_FAIL(err))
 727                goto out;
 728
 729        err = bpf_prog_test_run_opts(main_fd, &topts);
 730        ASSERT_OK(err, "tailcall");
 731        ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 2, "tailcall retval");
 732out:
 733        bpf_object__close(obj);
 734}
 735
 736#include "tailcall_bpf2bpf4.skel.h"
 737
 738/* test_tailcall_bpf2bpf_4 checks that tailcall counter is correctly preserved
 739 * across tailcalls combined with bpf2bpf calls. for making sure that tailcall
 740 * counter behaves correctly, bpf program will go through following flow:
 741 *
 742 * entry -> entry_subprog -> tailcall0 -> bpf_func0 -> subprog0 ->
 743 * -> tailcall1 -> bpf_func1 -> subprog1 -> tailcall2 -> bpf_func2 ->
 744 * subprog2 [here bump global counter] --------^
 745 *
 746 * We go through first two tailcalls and start counting from the subprog2 where
 747 * the loop begins. At the end of the test make sure that the global counter is
 748 * equal to 31, because tailcall counter includes the first two tailcalls
 749 * whereas global counter is incremented only on loop presented on flow above.
 750 *
 751 * The noise parameter is used to insert bpf_map_update calls into the logic
 752 * to force verifier to patch instructions. This allows us to ensure jump
 753 * logic remains correct with instruction movement.
 754 */
 755static void test_tailcall_bpf2bpf_4(bool noise)
 756{
 757        int err, map_fd, prog_fd, main_fd, data_fd, i;
 758        struct tailcall_bpf2bpf4__bss val;
 759        struct bpf_map *prog_array, *data_map;
 760        struct bpf_program *prog;
 761        struct bpf_object *obj;
 762        char prog_name[32];
 763        LIBBPF_OPTS(bpf_test_run_opts, topts,
 764                .data_in = &pkt_v4,
 765                .data_size_in = sizeof(pkt_v4),
 766                .repeat = 1,
 767        );
 768
 769        err = bpf_prog_test_load("tailcall_bpf2bpf4.o", BPF_PROG_TYPE_SCHED_CLS,
 770                            &obj, &prog_fd);
 771        if (CHECK_FAIL(err))
 772                return;
 773
 774        prog = bpf_object__find_program_by_name(obj, "entry");
 775        if (CHECK_FAIL(!prog))
 776                goto out;
 777
 778        main_fd = bpf_program__fd(prog);
 779        if (CHECK_FAIL(main_fd < 0))
 780                goto out;
 781
 782        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 783        if (CHECK_FAIL(!prog_array))
 784                goto out;
 785
 786        map_fd = bpf_map__fd(prog_array);
 787        if (CHECK_FAIL(map_fd < 0))
 788                goto out;
 789
 790        for (i = 0; i < bpf_map__max_entries(prog_array); i++) {
 791                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
 792
 793                prog = bpf_object__find_program_by_name(obj, prog_name);
 794                if (CHECK_FAIL(!prog))
 795                        goto out;
 796
 797                prog_fd = bpf_program__fd(prog);
 798                if (CHECK_FAIL(prog_fd < 0))
 799                        goto out;
 800
 801                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 802                if (CHECK_FAIL(err))
 803                        goto out;
 804        }
 805
 806        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 807        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 808                return;
 809
 810        data_fd = bpf_map__fd(data_map);
 811        if (CHECK_FAIL(map_fd < 0))
 812                return;
 813
 814        i = 0;
 815        val.noise = noise;
 816        val.count = 0;
 817        err = bpf_map_update_elem(data_fd, &i, &val, BPF_ANY);
 818        if (CHECK_FAIL(err))
 819                goto out;
 820
 821        err = bpf_prog_test_run_opts(main_fd, &topts);
 822        ASSERT_OK(err, "tailcall");
 823        ASSERT_EQ(topts.retval, sizeof(pkt_v4) * 3, "tailcall retval");
 824
 825        i = 0;
 826        err = bpf_map_lookup_elem(data_fd, &i, &val);
 827        ASSERT_OK(err, "tailcall count");
 828        ASSERT_EQ(val.count, 31, "tailcall count");
 829
 830out:
 831        bpf_object__close(obj);
 832}
 833
 834void test_tailcalls(void)
 835{
 836        if (test__start_subtest("tailcall_1"))
 837                test_tailcall_1();
 838        if (test__start_subtest("tailcall_2"))
 839                test_tailcall_2();
 840        if (test__start_subtest("tailcall_3"))
 841                test_tailcall_3();
 842        if (test__start_subtest("tailcall_4"))
 843                test_tailcall_4();
 844        if (test__start_subtest("tailcall_5"))
 845                test_tailcall_5();
 846        if (test__start_subtest("tailcall_6"))
 847                test_tailcall_6();
 848        if (test__start_subtest("tailcall_bpf2bpf_1"))
 849                test_tailcall_bpf2bpf_1();
 850        if (test__start_subtest("tailcall_bpf2bpf_2"))
 851                test_tailcall_bpf2bpf_2();
 852        if (test__start_subtest("tailcall_bpf2bpf_3"))
 853                test_tailcall_bpf2bpf_3();
 854        if (test__start_subtest("tailcall_bpf2bpf_4"))
 855                test_tailcall_bpf2bpf_4(false);
 856        if (test__start_subtest("tailcall_bpf2bpf_5"))
 857                test_tailcall_bpf2bpf_4(true);
 858}
 859