LXR linux/tools/testing/selftests/bpf/prog

   1// SPDX-License-Identifier: GPL-2.0
   2#include <test_progs.h>
   3#include <network_helpers.h>
   4
   5/* test_tailcall_1 checks basic functionality by patching multiple locations
   6 * in a single program for a single tail call slot with nop->jmp, jmp->nop
   7 * and jmp->jmp rewrites. Also checks for nop->nop.
   8 */
   9static void test_tailcall_1(void)
  10{
  11        int err, map_fd, prog_fd, main_fd, i, j;
  12        struct bpf_map *prog_array;
  13        struct bpf_program *prog;
  14        struct bpf_object *obj;
  15        __u32 retval, duration;
  16        char prog_name[32];
  17        char buff[128] = {};
  18
  19        err = bpf_prog_test_load("tailcall1.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
  20                            &prog_fd);
  21        if (CHECK_FAIL(err))
  22                return;
  23
  24        prog = bpf_object__find_program_by_name(obj, "entry");
  25        if (CHECK_FAIL(!prog))
  26                goto out;
  27
  28        main_fd = bpf_program__fd(prog);
  29        if (CHECK_FAIL(main_fd < 0))
  30                goto out;
  31
  32        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
  33        if (CHECK_FAIL(!prog_array))
  34                goto out;
  35
  36        map_fd = bpf_map__fd(prog_array);
  37        if (CHECK_FAIL(map_fd < 0))
  38                goto out;
  39
  40        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
  41                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
  42
  43                prog = bpf_object__find_program_by_name(obj, prog_name);
  44                if (CHECK_FAIL(!prog))
  45                        goto out;
  46
  47                prog_fd = bpf_program__fd(prog);
  48                if (CHECK_FAIL(prog_fd < 0))
  49                        goto out;
  50
  51                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
  52                if (CHECK_FAIL(err))
  53                        goto out;
  54        }
  55
  56        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
  57                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
  58                                        &duration, &retval, NULL);
  59                CHECK(err || retval != i, "tailcall",
  60                      "err %d errno %d retval %d\n", err, errno, retval);
  61
  62                err = bpf_map_delete_elem(map_fd, &i);
  63                if (CHECK_FAIL(err))
  64                        goto out;
  65        }
  66
  67        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
  68                                &duration, &retval, NULL);
  69        CHECK(err || retval != 3, "tailcall", "err %d errno %d retval %d\n",
  70              err, errno, retval);
  71
  72        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
  73                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
  74
  75                prog = bpf_object__find_program_by_name(obj, prog_name);
  76                if (CHECK_FAIL(!prog))
  77                        goto out;
  78
  79                prog_fd = bpf_program__fd(prog);
  80                if (CHECK_FAIL(prog_fd < 0))
  81                        goto out;
  82
  83                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
  84                if (CHECK_FAIL(err))
  85                        goto out;
  86        }
  87
  88        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
  89                                &duration, &retval, NULL);
  90        CHECK(err || retval != 0, "tailcall", "err %d errno %d retval %d\n",
  91              err, errno, retval);
  92
  93        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
  94                j = bpf_map__def(prog_array)->max_entries - 1 - i;
  95                snprintf(prog_name, sizeof(prog_name), "classifier_%d", j);
  96
  97                prog = bpf_object__find_program_by_name(obj, prog_name);
  98                if (CHECK_FAIL(!prog))
  99                        goto out;
 100
 101                prog_fd = bpf_program__fd(prog);
 102                if (CHECK_FAIL(prog_fd < 0))
 103                        goto out;
 104
 105                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 106                if (CHECK_FAIL(err))
 107                        goto out;
 108        }
 109
 110        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 111                j = bpf_map__def(prog_array)->max_entries - 1 - i;
 112
 113                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 114                                        &duration, &retval, NULL);
 115                CHECK(err || retval != j, "tailcall",
 116                      "err %d errno %d retval %d\n", err, errno, retval);
 117
 118                err = bpf_map_delete_elem(map_fd, &i);
 119                if (CHECK_FAIL(err))
 120                        goto out;
 121        }
 122
 123        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 124                                &duration, &retval, NULL);
 125        CHECK(err || retval != 3, "tailcall", "err %d errno %d retval %d\n",
 126              err, errno, retval);
 127
 128        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 129                err = bpf_map_delete_elem(map_fd, &i);
 130                if (CHECK_FAIL(err >= 0 || errno != ENOENT))
 131                        goto out;
 132
 133                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 134                                        &duration, &retval, NULL);
 135                CHECK(err || retval != 3, "tailcall",
 136                      "err %d errno %d retval %d\n", err, errno, retval);
 137        }
 138
 139out:
 140        bpf_object__close(obj);
 141}
 142
 143/* test_tailcall_2 checks that patching multiple programs for a single
 144 * tail call slot works. It also jumps through several programs and tests
 145 * the tail call limit counter.
 146 */
 147static void test_tailcall_2(void)
 148{
 149        int err, map_fd, prog_fd, main_fd, i;
 150        struct bpf_map *prog_array;
 151        struct bpf_program *prog;
 152        struct bpf_object *obj;
 153        __u32 retval, duration;
 154        char prog_name[32];
 155        char buff[128] = {};
 156
 157        err = bpf_prog_test_load("tailcall2.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 158                            &prog_fd);
 159        if (CHECK_FAIL(err))
 160                return;
 161
 162        prog = bpf_object__find_program_by_name(obj, "entry");
 163        if (CHECK_FAIL(!prog))
 164                goto out;
 165
 166        main_fd = bpf_program__fd(prog);
 167        if (CHECK_FAIL(main_fd < 0))
 168                goto out;
 169
 170        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 171        if (CHECK_FAIL(!prog_array))
 172                goto out;
 173
 174        map_fd = bpf_map__fd(prog_array);
 175        if (CHECK_FAIL(map_fd < 0))
 176                goto out;
 177
 178        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 179                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
 180
 181                prog = bpf_object__find_program_by_name(obj, prog_name);
 182                if (CHECK_FAIL(!prog))
 183                        goto out;
 184
 185                prog_fd = bpf_program__fd(prog);
 186                if (CHECK_FAIL(prog_fd < 0))
 187                        goto out;
 188
 189                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 190                if (CHECK_FAIL(err))
 191                        goto out;
 192        }
 193
 194        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 195                                &duration, &retval, NULL);
 196        CHECK(err || retval != 2, "tailcall", "err %d errno %d retval %d\n",
 197              err, errno, retval);
 198
 199        i = 2;
 200        err = bpf_map_delete_elem(map_fd, &i);
 201        if (CHECK_FAIL(err))
 202                goto out;
 203
 204        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 205                                &duration, &retval, NULL);
 206        CHECK(err || retval != 1, "tailcall", "err %d errno %d retval %d\n",
 207              err, errno, retval);
 208
 209        i = 0;
 210        err = bpf_map_delete_elem(map_fd, &i);
 211        if (CHECK_FAIL(err))
 212                goto out;
 213
 214        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 215                                &duration, &retval, NULL);
 216        CHECK(err || retval != 3, "tailcall", "err %d errno %d retval %d\n",
 217              err, errno, retval);
 218out:
 219        bpf_object__close(obj);
 220}
 221
 222static void test_tailcall_count(const char *which)
 223{
 224        int err, map_fd, prog_fd, main_fd, data_fd, i, val;
 225        struct bpf_map *prog_array, *data_map;
 226        struct bpf_program *prog;
 227        struct bpf_object *obj;
 228        __u32 retval, duration;
 229        char buff[128] = {};
 230
 231        err = bpf_prog_test_load(which, BPF_PROG_TYPE_SCHED_CLS, &obj,
 232                            &prog_fd);
 233        if (CHECK_FAIL(err))
 234                return;
 235
 236        prog = bpf_object__find_program_by_name(obj, "entry");
 237        if (CHECK_FAIL(!prog))
 238                goto out;
 239
 240        main_fd = bpf_program__fd(prog);
 241        if (CHECK_FAIL(main_fd < 0))
 242                goto out;
 243
 244        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 245        if (CHECK_FAIL(!prog_array))
 246                goto out;
 247
 248        map_fd = bpf_map__fd(prog_array);
 249        if (CHECK_FAIL(map_fd < 0))
 250                goto out;
 251
 252        prog = bpf_object__find_program_by_name(obj, "classifier_0");
 253        if (CHECK_FAIL(!prog))
 254                goto out;
 255
 256        prog_fd = bpf_program__fd(prog);
 257        if (CHECK_FAIL(prog_fd < 0))
 258                goto out;
 259
 260        i = 0;
 261        err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 262        if (CHECK_FAIL(err))
 263                goto out;
 264
 265        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 266                                &duration, &retval, NULL);
 267        CHECK(err || retval != 1, "tailcall", "err %d errno %d retval %d\n",
 268              err, errno, retval);
 269
 270        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 271        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 272                return;
 273
 274        data_fd = bpf_map__fd(data_map);
 275        if (CHECK_FAIL(map_fd < 0))
 276                return;
 277
 278        i = 0;
 279        err = bpf_map_lookup_elem(data_fd, &i, &val);
 280        CHECK(err || val != 33, "tailcall count", "err %d errno %d count %d\n",
 281              err, errno, val);
 282
 283        i = 0;
 284        err = bpf_map_delete_elem(map_fd, &i);
 285        if (CHECK_FAIL(err))
 286                goto out;
 287
 288        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 289                                &duration, &retval, NULL);
 290        CHECK(err || retval != 0, "tailcall", "err %d errno %d retval %d\n",
 291              err, errno, retval);
 292out:
 293        bpf_object__close(obj);
 294}
 295
 296/* test_tailcall_3 checks that the count value of the tail call limit
 297 * enforcement matches with expectations. JIT uses direct jump.
 298 */
 299static void test_tailcall_3(void)
 300{
 301        test_tailcall_count("tailcall3.o");
 302}
 303
 304/* test_tailcall_6 checks that the count value of the tail call limit
 305 * enforcement matches with expectations. JIT uses indirect jump.
 306 */
 307static void test_tailcall_6(void)
 308{
 309        test_tailcall_count("tailcall6.o");
 310}
 311
 312/* test_tailcall_4 checks that the kernel properly selects indirect jump
 313 * for the case where the key is not known. Latter is passed via global
 314 * data to select different targets we can compare return value of.
 315 */
 316static void test_tailcall_4(void)
 317{
 318        int err, map_fd, prog_fd, main_fd, data_fd, i;
 319        struct bpf_map *prog_array, *data_map;
 320        struct bpf_program *prog;
 321        struct bpf_object *obj;
 322        __u32 retval, duration;
 323        static const int zero = 0;
 324        char buff[128] = {};
 325        char prog_name[32];
 326
 327        err = bpf_prog_test_load("tailcall4.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 328                            &prog_fd);
 329        if (CHECK_FAIL(err))
 330                return;
 331
 332        prog = bpf_object__find_program_by_name(obj, "entry");
 333        if (CHECK_FAIL(!prog))
 334                goto out;
 335
 336        main_fd = bpf_program__fd(prog);
 337        if (CHECK_FAIL(main_fd < 0))
 338                goto out;
 339
 340        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 341        if (CHECK_FAIL(!prog_array))
 342                goto out;
 343
 344        map_fd = bpf_map__fd(prog_array);
 345        if (CHECK_FAIL(map_fd < 0))
 346                goto out;
 347
 348        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 349        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 350                return;
 351
 352        data_fd = bpf_map__fd(data_map);
 353        if (CHECK_FAIL(map_fd < 0))
 354                return;
 355
 356        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 357                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
 358
 359                prog = bpf_object__find_program_by_name(obj, prog_name);
 360                if (CHECK_FAIL(!prog))
 361                        goto out;
 362
 363                prog_fd = bpf_program__fd(prog);
 364                if (CHECK_FAIL(prog_fd < 0))
 365                        goto out;
 366
 367                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 368                if (CHECK_FAIL(err))
 369                        goto out;
 370        }
 371
 372        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 373                err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY);
 374                if (CHECK_FAIL(err))
 375                        goto out;
 376
 377                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 378                                        &duration, &retval, NULL);
 379                CHECK(err || retval != i, "tailcall",
 380                      "err %d errno %d retval %d\n", err, errno, retval);
 381        }
 382
 383        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 384                err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY);
 385                if (CHECK_FAIL(err))
 386                        goto out;
 387
 388                err = bpf_map_delete_elem(map_fd, &i);
 389                if (CHECK_FAIL(err))
 390                        goto out;
 391
 392                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 393                                        &duration, &retval, NULL);
 394                CHECK(err || retval != 3, "tailcall",
 395                      "err %d errno %d retval %d\n", err, errno, retval);
 396        }
 397out:
 398        bpf_object__close(obj);
 399}
 400
 401/* test_tailcall_5 probes similarly to test_tailcall_4 that the kernel generates
 402 * an indirect jump when the keys are const but different from different branches.
 403 */
 404static void test_tailcall_5(void)
 405{
 406        int err, map_fd, prog_fd, main_fd, data_fd, i, key[] = { 1111, 1234, 5678 };
 407        struct bpf_map *prog_array, *data_map;
 408        struct bpf_program *prog;
 409        struct bpf_object *obj;
 410        __u32 retval, duration;
 411        static const int zero = 0;
 412        char buff[128] = {};
 413        char prog_name[32];
 414
 415        err = bpf_prog_test_load("tailcall5.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 416                            &prog_fd);
 417        if (CHECK_FAIL(err))
 418                return;
 419
 420        prog = bpf_object__find_program_by_name(obj, "entry");
 421        if (CHECK_FAIL(!prog))
 422                goto out;
 423
 424        main_fd = bpf_program__fd(prog);
 425        if (CHECK_FAIL(main_fd < 0))
 426                goto out;
 427
 428        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 429        if (CHECK_FAIL(!prog_array))
 430                goto out;
 431
 432        map_fd = bpf_map__fd(prog_array);
 433        if (CHECK_FAIL(map_fd < 0))
 434                goto out;
 435
 436        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 437        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 438                return;
 439
 440        data_fd = bpf_map__fd(data_map);
 441        if (CHECK_FAIL(map_fd < 0))
 442                return;
 443
 444        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 445                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
 446
 447                prog = bpf_object__find_program_by_name(obj, prog_name);
 448                if (CHECK_FAIL(!prog))
 449                        goto out;
 450
 451                prog_fd = bpf_program__fd(prog);
 452                if (CHECK_FAIL(prog_fd < 0))
 453                        goto out;
 454
 455                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 456                if (CHECK_FAIL(err))
 457                        goto out;
 458        }
 459
 460        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 461                err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY);
 462                if (CHECK_FAIL(err))
 463                        goto out;
 464
 465                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 466                                        &duration, &retval, NULL);
 467                CHECK(err || retval != i, "tailcall",
 468                      "err %d errno %d retval %d\n", err, errno, retval);
 469        }
 470
 471        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 472                err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY);
 473                if (CHECK_FAIL(err))
 474                        goto out;
 475
 476                err = bpf_map_delete_elem(map_fd, &i);
 477                if (CHECK_FAIL(err))
 478                        goto out;
 479
 480                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 481                                        &duration, &retval, NULL);
 482                CHECK(err || retval != 3, "tailcall",
 483                      "err %d errno %d retval %d\n", err, errno, retval);
 484        }
 485out:
 486        bpf_object__close(obj);
 487}
 488
 489/* test_tailcall_bpf2bpf_1 purpose is to make sure that tailcalls are working
 490 * correctly in correlation with BPF subprograms
 491 */
 492static void test_tailcall_bpf2bpf_1(void)
 493{
 494        int err, map_fd, prog_fd, main_fd, i;
 495        struct bpf_map *prog_array;
 496        struct bpf_program *prog;
 497        struct bpf_object *obj;
 498        __u32 retval, duration;
 499        char prog_name[32];
 500
 501        err = bpf_prog_test_load("tailcall_bpf2bpf1.o", BPF_PROG_TYPE_SCHED_CLS,
 502                            &obj, &prog_fd);
 503        if (CHECK_FAIL(err))
 504                return;
 505
 506        prog = bpf_object__find_program_by_name(obj, "entry");
 507        if (CHECK_FAIL(!prog))
 508                goto out;
 509
 510        main_fd = bpf_program__fd(prog);
 511        if (CHECK_FAIL(main_fd < 0))
 512                goto out;
 513
 514        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 515        if (CHECK_FAIL(!prog_array))
 516                goto out;
 517
 518        map_fd = bpf_map__fd(prog_array);
 519        if (CHECK_FAIL(map_fd < 0))
 520                goto out;
 521
 522        /* nop -> jmp */
 523        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 524                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
 525
 526                prog = bpf_object__find_program_by_name(obj, prog_name);
 527                if (CHECK_FAIL(!prog))
 528                        goto out;
 529
 530                prog_fd = bpf_program__fd(prog);
 531                if (CHECK_FAIL(prog_fd < 0))
 532                        goto out;
 533
 534                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 535                if (CHECK_FAIL(err))
 536                        goto out;
 537        }
 538
 539        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 540                                0, &retval, &duration);
 541        CHECK(err || retval != 1, "tailcall",
 542              "err %d errno %d retval %d\n", err, errno, retval);
 543
 544        /* jmp -> nop, call subprog that will do tailcall */
 545        i = 1;
 546        err = bpf_map_delete_elem(map_fd, &i);
 547        if (CHECK_FAIL(err))
 548                goto out;
 549
 550        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 551                                0, &retval, &duration);
 552        CHECK(err || retval != 0, "tailcall", "err %d errno %d retval %d\n",
 553              err, errno, retval);
 554
 555        /* make sure that subprog can access ctx and entry prog that
 556         * called this subprog can properly return
 557         */
 558        i = 0;
 559        err = bpf_map_delete_elem(map_fd, &i);
 560        if (CHECK_FAIL(err))
 561                goto out;
 562
 563        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 564                                0, &retval, &duration);
 565        CHECK(err || retval != sizeof(pkt_v4) * 2,
 566              "tailcall", "err %d errno %d retval %d\n",
 567              err, errno, retval);
 568out:
 569        bpf_object__close(obj);
 570}
 571
 572/* test_tailcall_bpf2bpf_2 checks that the count value of the tail call limit
 573 * enforcement matches with expectations when tailcall is preceded with
 574 * bpf2bpf call.
 575 */
 576static void test_tailcall_bpf2bpf_2(void)
 577{
 578        int err, map_fd, prog_fd, main_fd, data_fd, i, val;
 579        struct bpf_map *prog_array, *data_map;
 580        struct bpf_program *prog;
 581        struct bpf_object *obj;
 582        __u32 retval, duration;
 583        char buff[128] = {};
 584
 585        err = bpf_prog_test_load("tailcall_bpf2bpf2.o", BPF_PROG_TYPE_SCHED_CLS,
 586                            &obj, &prog_fd);
 587        if (CHECK_FAIL(err))
 588                return;
 589
 590        prog = bpf_object__find_program_by_name(obj, "entry");
 591        if (CHECK_FAIL(!prog))
 592                goto out;
 593
 594        main_fd = bpf_program__fd(prog);
 595        if (CHECK_FAIL(main_fd < 0))
 596                goto out;
 597
 598        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 599        if (CHECK_FAIL(!prog_array))
 600                goto out;
 601
 602        map_fd = bpf_map__fd(prog_array);
 603        if (CHECK_FAIL(map_fd < 0))
 604                goto out;
 605
 606        prog = bpf_object__find_program_by_name(obj, "classifier_0");
 607        if (CHECK_FAIL(!prog))
 608                goto out;
 609
 610        prog_fd = bpf_program__fd(prog);
 611        if (CHECK_FAIL(prog_fd < 0))
 612                goto out;
 613
 614        i = 0;
 615        err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 616        if (CHECK_FAIL(err))
 617                goto out;
 618
 619        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 620                                &duration, &retval, NULL);
 621        CHECK(err || retval != 1, "tailcall", "err %d errno %d retval %d\n",
 622              err, errno, retval);
 623
 624        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 625        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 626                return;
 627
 628        data_fd = bpf_map__fd(data_map);
 629        if (CHECK_FAIL(map_fd < 0))
 630                return;
 631
 632        i = 0;
 633        err = bpf_map_lookup_elem(data_fd, &i, &val);
 634        CHECK(err || val != 33, "tailcall count", "err %d errno %d count %d\n",
 635              err, errno, val);
 636
 637        i = 0;
 638        err = bpf_map_delete_elem(map_fd, &i);
 639        if (CHECK_FAIL(err))
 640                goto out;
 641
 642        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 643                                &duration, &retval, NULL);
 644        CHECK(err || retval != 0, "tailcall", "err %d errno %d retval %d\n",
 645              err, errno, retval);
 646out:
 647        bpf_object__close(obj);
 648}
 649
 650/* test_tailcall_bpf2bpf_3 checks that non-trivial amount of stack (up to
 651 * 256 bytes) can be used within bpf subprograms that have the tailcalls
 652 * in them
 653 */
 654static void test_tailcall_bpf2bpf_3(void)
 655{
 656        int err, map_fd, prog_fd, main_fd, i;
 657        struct bpf_map *prog_array;
 658        struct bpf_program *prog;
 659        struct bpf_object *obj;
 660        __u32 retval, duration;
 661        char prog_name[32];
 662
 663        err = bpf_prog_test_load("tailcall_bpf2bpf3.o", BPF_PROG_TYPE_SCHED_CLS,
 664                            &obj, &prog_fd);
 665        if (CHECK_FAIL(err))
 666                return;
 667
 668        prog = bpf_object__find_program_by_name(obj, "entry");
 669        if (CHECK_FAIL(!prog))
 670                goto out;
 671
 672        main_fd = bpf_program__fd(prog);
 673        if (CHECK_FAIL(main_fd < 0))
 674                goto out;
 675
 676        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 677        if (CHECK_FAIL(!prog_array))
 678                goto out;
 679
 680        map_fd = bpf_map__fd(prog_array);
 681        if (CHECK_FAIL(map_fd < 0))
 682                goto out;
 683
 684        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 685                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
 686
 687                prog = bpf_object__find_program_by_name(obj, prog_name);
 688                if (CHECK_FAIL(!prog))
 689                        goto out;
 690
 691                prog_fd = bpf_program__fd(prog);
 692                if (CHECK_FAIL(prog_fd < 0))
 693                        goto out;
 694
 695                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 696                if (CHECK_FAIL(err))
 697                        goto out;
 698        }
 699
 700        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 701                                &duration, &retval, NULL);
 702        CHECK(err || retval != sizeof(pkt_v4) * 3,
 703              "tailcall", "err %d errno %d retval %d\n",
 704              err, errno, retval);
 705
 706        i = 1;
 707        err = bpf_map_delete_elem(map_fd, &i);
 708        if (CHECK_FAIL(err))
 709                goto out;
 710
 711        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 712                                &duration, &retval, NULL);
 713        CHECK(err || retval != sizeof(pkt_v4),
 714              "tailcall", "err %d errno %d retval %d\n",
 715              err, errno, retval);
 716
 717        i = 0;
 718        err = bpf_map_delete_elem(map_fd, &i);
 719        if (CHECK_FAIL(err))
 720                goto out;
 721
 722        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 723                                &duration, &retval, NULL);
 724        CHECK(err || retval != sizeof(pkt_v4) * 2,
 725              "tailcall", "err %d errno %d retval %d\n",
 726              err, errno, retval);
 727out:
 728        bpf_object__close(obj);
 729}
 730
 731#include "tailcall_bpf2bpf4.skel.h"
 732
 733/* test_tailcall_bpf2bpf_4 checks that tailcall counter is correctly preserved
 734 * across tailcalls combined with bpf2bpf calls. for making sure that tailcall
 735 * counter behaves correctly, bpf program will go through following flow:
 736 *
 737 * entry -> entry_subprog -> tailcall0 -> bpf_func0 -> subprog0 ->
 738 * -> tailcall1 -> bpf_func1 -> subprog1 -> tailcall2 -> bpf_func2 ->
 739 * subprog2 [here bump global counter] --------^
 740 *
 741 * We go through first two tailcalls and start counting from the subprog2 where
 742 * the loop begins. At the end of the test make sure that the global counter is
 743 * equal to 31, because tailcall counter includes the first two tailcalls
 744 * whereas global counter is incremented only on loop presented on flow above.
 745 *
 746 * The noise parameter is used to insert bpf_map_update calls into the logic
 747 * to force verifier to patch instructions. This allows us to ensure jump
 748 * logic remains correct with instruction movement.
 749 */
 750static void test_tailcall_bpf2bpf_4(bool noise)
 751{
 752        int err, map_fd, prog_fd, main_fd, data_fd, i;
 753        struct tailcall_bpf2bpf4__bss val;
 754        struct bpf_map *prog_array, *data_map;
 755        struct bpf_program *prog;
 756        struct bpf_object *obj;
 757        __u32 retval, duration;
 758        char prog_name[32];
 759
 760        err = bpf_prog_test_load("tailcall_bpf2bpf4.o", BPF_PROG_TYPE_SCHED_CLS,
 761                            &obj, &prog_fd);
 762        if (CHECK_FAIL(err))
 763                return;
 764
 765        prog = bpf_object__find_program_by_name(obj, "entry");
 766        if (CHECK_FAIL(!prog))
 767                goto out;
 768
 769        main_fd = bpf_program__fd(prog);
 770        if (CHECK_FAIL(main_fd < 0))
 771                goto out;
 772
 773        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 774        if (CHECK_FAIL(!prog_array))
 775                goto out;
 776
 777        map_fd = bpf_map__fd(prog_array);
 778        if (CHECK_FAIL(map_fd < 0))
 779                goto out;
 780
 781        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 782                snprintf(prog_name, sizeof(prog_name), "classifier_%d", i);
 783
 784                prog = bpf_object__find_program_by_name(obj, prog_name);
 785                if (CHECK_FAIL(!prog))
 786                        goto out;
 787
 788                prog_fd = bpf_program__fd(prog);
 789                if (CHECK_FAIL(prog_fd < 0))
 790                        goto out;
 791
 792                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 793                if (CHECK_FAIL(err))
 794                        goto out;
 795        }
 796
 797        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 798        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 799                return;
 800
 801        data_fd = bpf_map__fd(data_map);
 802        if (CHECK_FAIL(map_fd < 0))
 803                return;
 804
 805        i = 0;
 806        val.noise = noise;
 807        val.count = 0;
 808        err = bpf_map_update_elem(data_fd, &i, &val, BPF_ANY);
 809        if (CHECK_FAIL(err))
 810                goto out;
 811
 812        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 813                                &duration, &retval, NULL);
 814        CHECK(err || retval != sizeof(pkt_v4) * 3, "tailcall", "err %d errno %d retval %d\n",
 815              err, errno, retval);
 816
 817        i = 0;
 818        err = bpf_map_lookup_elem(data_fd, &i, &val);
 819        CHECK(err || val.count != 31, "tailcall count", "err %d errno %d count %d\n",
 820              err, errno, val.count);
 821
 822out:
 823        bpf_object__close(obj);
 824}
 825
 826void test_tailcalls(void)
 827{
 828        if (test__start_subtest("tailcall_1"))
 829                test_tailcall_1();
 830        if (test__start_subtest("tailcall_2"))
 831                test_tailcall_2();
 832        if (test__start_subtest("tailcall_3"))
 833                test_tailcall_3();
 834        if (test__start_subtest("tailcall_4"))
 835                test_tailcall_4();
 836        if (test__start_subtest("tailcall_5"))
 837                test_tailcall_5();
 838        if (test__start_subtest("tailcall_6"))
 839                test_tailcall_6();
 840        if (test__start_subtest("tailcall_bpf2bpf_1"))
 841                test_tailcall_bpf2bpf_1();
 842        if (test__start_subtest("tailcall_bpf2bpf_2"))
 843                test_tailcall_bpf2bpf_2();
 844        if (test__start_subtest("tailcall_bpf2bpf_3"))
 845                test_tailcall_bpf2bpf_3();
 846        if (test__start_subtest("tailcall_bpf2bpf_4"))
 847                test_tailcall_bpf2bpf_4(false);
 848        if (test__start_subtest("tailcall_bpf2bpf_5"))
 849                test_tailcall_bpf2bpf_4(true);
 850}
 851