LXR linux/tools/testing/selftests/bpf/prog

   1// SPDX-License-Identifier: GPL-2.0
   2#include <test_progs.h>
   3#include <network_helpers.h>
   4
   5/* test_tailcall_1 checks basic functionality by patching multiple locations
   6 * in a single program for a single tail call slot with nop->jmp, jmp->nop
   7 * and jmp->jmp rewrites. Also checks for nop->nop.
   8 */
   9static void test_tailcall_1(void)
  10{
  11        int err, map_fd, prog_fd, main_fd, i, j;
  12        struct bpf_map *prog_array;
  13        struct bpf_program *prog;
  14        struct bpf_object *obj;
  15        __u32 retval, duration;
  16        char prog_name[32];
  17        char buff[128] = {};
  18
  19        err = bpf_prog_load("tailcall1.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
  20                            &prog_fd);
  21        if (CHECK_FAIL(err))
  22                return;
  23
  24        prog = bpf_object__find_program_by_title(obj, "classifier");
  25        if (CHECK_FAIL(!prog))
  26                goto out;
  27
  28        main_fd = bpf_program__fd(prog);
  29        if (CHECK_FAIL(main_fd < 0))
  30                goto out;
  31
  32        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
  33        if (CHECK_FAIL(!prog_array))
  34                goto out;
  35
  36        map_fd = bpf_map__fd(prog_array);
  37        if (CHECK_FAIL(map_fd < 0))
  38                goto out;
  39
  40        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
  41                snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
  42
  43                prog = bpf_object__find_program_by_title(obj, prog_name);
  44                if (CHECK_FAIL(!prog))
  45                        goto out;
  46
  47                prog_fd = bpf_program__fd(prog);
  48                if (CHECK_FAIL(prog_fd < 0))
  49                        goto out;
  50
  51                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
  52                if (CHECK_FAIL(err))
  53                        goto out;
  54        }
  55
  56        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
  57                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
  58                                        &duration, &retval, NULL);
  59                CHECK(err || retval != i, "tailcall",
  60                      "err %d errno %d retval %d\n", err, errno, retval);
  61
  62                err = bpf_map_delete_elem(map_fd, &i);
  63                if (CHECK_FAIL(err))
  64                        goto out;
  65        }
  66
  67        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
  68                                &duration, &retval, NULL);
  69        CHECK(err || retval != 3, "tailcall", "err %d errno %d retval %d\n",
  70              err, errno, retval);
  71
  72        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
  73                snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
  74
  75                prog = bpf_object__find_program_by_title(obj, prog_name);
  76                if (CHECK_FAIL(!prog))
  77                        goto out;
  78
  79                prog_fd = bpf_program__fd(prog);
  80                if (CHECK_FAIL(prog_fd < 0))
  81                        goto out;
  82
  83                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
  84                if (CHECK_FAIL(err))
  85                        goto out;
  86        }
  87
  88        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
  89                                &duration, &retval, NULL);
  90        CHECK(err || retval != 0, "tailcall", "err %d errno %d retval %d\n",
  91              err, errno, retval);
  92
  93        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
  94                j = bpf_map__def(prog_array)->max_entries - 1 - i;
  95                snprintf(prog_name, sizeof(prog_name), "classifier/%i", j);
  96
  97                prog = bpf_object__find_program_by_title(obj, prog_name);
  98                if (CHECK_FAIL(!prog))
  99                        goto out;
 100
 101                prog_fd = bpf_program__fd(prog);
 102                if (CHECK_FAIL(prog_fd < 0))
 103                        goto out;
 104
 105                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 106                if (CHECK_FAIL(err))
 107                        goto out;
 108        }
 109
 110        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 111                j = bpf_map__def(prog_array)->max_entries - 1 - i;
 112
 113                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 114                                        &duration, &retval, NULL);
 115                CHECK(err || retval != j, "tailcall",
 116                      "err %d errno %d retval %d\n", err, errno, retval);
 117
 118                err = bpf_map_delete_elem(map_fd, &i);
 119                if (CHECK_FAIL(err))
 120                        goto out;
 121        }
 122
 123        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 124                                &duration, &retval, NULL);
 125        CHECK(err || retval != 3, "tailcall", "err %d errno %d retval %d\n",
 126              err, errno, retval);
 127
 128        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 129                err = bpf_map_delete_elem(map_fd, &i);
 130                if (CHECK_FAIL(err >= 0 || errno != ENOENT))
 131                        goto out;
 132
 133                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 134                                        &duration, &retval, NULL);
 135                CHECK(err || retval != 3, "tailcall",
 136                      "err %d errno %d retval %d\n", err, errno, retval);
 137        }
 138
 139out:
 140        bpf_object__close(obj);
 141}
 142
 143/* test_tailcall_2 checks that patching multiple programs for a single
 144 * tail call slot works. It also jumps through several programs and tests
 145 * the tail call limit counter.
 146 */
 147static void test_tailcall_2(void)
 148{
 149        int err, map_fd, prog_fd, main_fd, i;
 150        struct bpf_map *prog_array;
 151        struct bpf_program *prog;
 152        struct bpf_object *obj;
 153        __u32 retval, duration;
 154        char prog_name[32];
 155        char buff[128] = {};
 156
 157        err = bpf_prog_load("tailcall2.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 158                            &prog_fd);
 159        if (CHECK_FAIL(err))
 160                return;
 161
 162        prog = bpf_object__find_program_by_title(obj, "classifier");
 163        if (CHECK_FAIL(!prog))
 164                goto out;
 165
 166        main_fd = bpf_program__fd(prog);
 167        if (CHECK_FAIL(main_fd < 0))
 168                goto out;
 169
 170        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 171        if (CHECK_FAIL(!prog_array))
 172                goto out;
 173
 174        map_fd = bpf_map__fd(prog_array);
 175        if (CHECK_FAIL(map_fd < 0))
 176                goto out;
 177
 178        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 179                snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
 180
 181                prog = bpf_object__find_program_by_title(obj, prog_name);
 182                if (CHECK_FAIL(!prog))
 183                        goto out;
 184
 185                prog_fd = bpf_program__fd(prog);
 186                if (CHECK_FAIL(prog_fd < 0))
 187                        goto out;
 188
 189                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 190                if (CHECK_FAIL(err))
 191                        goto out;
 192        }
 193
 194        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 195                                &duration, &retval, NULL);
 196        CHECK(err || retval != 2, "tailcall", "err %d errno %d retval %d\n",
 197              err, errno, retval);
 198
 199        i = 2;
 200        err = bpf_map_delete_elem(map_fd, &i);
 201        if (CHECK_FAIL(err))
 202                goto out;
 203
 204        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 205                                &duration, &retval, NULL);
 206        CHECK(err || retval != 1, "tailcall", "err %d errno %d retval %d\n",
 207              err, errno, retval);
 208
 209        i = 0;
 210        err = bpf_map_delete_elem(map_fd, &i);
 211        if (CHECK_FAIL(err))
 212                goto out;
 213
 214        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 215                                &duration, &retval, NULL);
 216        CHECK(err || retval != 3, "tailcall", "err %d errno %d retval %d\n",
 217              err, errno, retval);
 218out:
 219        bpf_object__close(obj);
 220}
 221
 222/* test_tailcall_3 checks that the count value of the tail call limit
 223 * enforcement matches with expectations.
 224 */
 225static void test_tailcall_3(void)
 226{
 227        int err, map_fd, prog_fd, main_fd, data_fd, i, val;
 228        struct bpf_map *prog_array, *data_map;
 229        struct bpf_program *prog;
 230        struct bpf_object *obj;
 231        __u32 retval, duration;
 232        char buff[128] = {};
 233
 234        err = bpf_prog_load("tailcall3.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 235                            &prog_fd);
 236        if (CHECK_FAIL(err))
 237                return;
 238
 239        prog = bpf_object__find_program_by_title(obj, "classifier");
 240        if (CHECK_FAIL(!prog))
 241                goto out;
 242
 243        main_fd = bpf_program__fd(prog);
 244        if (CHECK_FAIL(main_fd < 0))
 245                goto out;
 246
 247        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 248        if (CHECK_FAIL(!prog_array))
 249                goto out;
 250
 251        map_fd = bpf_map__fd(prog_array);
 252        if (CHECK_FAIL(map_fd < 0))
 253                goto out;
 254
 255        prog = bpf_object__find_program_by_title(obj, "classifier/0");
 256        if (CHECK_FAIL(!prog))
 257                goto out;
 258
 259        prog_fd = bpf_program__fd(prog);
 260        if (CHECK_FAIL(prog_fd < 0))
 261                goto out;
 262
 263        i = 0;
 264        err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 265        if (CHECK_FAIL(err))
 266                goto out;
 267
 268        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 269                                &duration, &retval, NULL);
 270        CHECK(err || retval != 1, "tailcall", "err %d errno %d retval %d\n",
 271              err, errno, retval);
 272
 273        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 274        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 275                return;
 276
 277        data_fd = bpf_map__fd(data_map);
 278        if (CHECK_FAIL(map_fd < 0))
 279                return;
 280
 281        i = 0;
 282        err = bpf_map_lookup_elem(data_fd, &i, &val);
 283        CHECK(err || val != 33, "tailcall count", "err %d errno %d count %d\n",
 284              err, errno, val);
 285
 286        i = 0;
 287        err = bpf_map_delete_elem(map_fd, &i);
 288        if (CHECK_FAIL(err))
 289                goto out;
 290
 291        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 292                                &duration, &retval, NULL);
 293        CHECK(err || retval != 0, "tailcall", "err %d errno %d retval %d\n",
 294              err, errno, retval);
 295out:
 296        bpf_object__close(obj);
 297}
 298
 299/* test_tailcall_4 checks that the kernel properly selects indirect jump
 300 * for the case where the key is not known. Latter is passed via global
 301 * data to select different targets we can compare return value of.
 302 */
 303static void test_tailcall_4(void)
 304{
 305        int err, map_fd, prog_fd, main_fd, data_fd, i;
 306        struct bpf_map *prog_array, *data_map;
 307        struct bpf_program *prog;
 308        struct bpf_object *obj;
 309        __u32 retval, duration;
 310        static const int zero = 0;
 311        char buff[128] = {};
 312        char prog_name[32];
 313
 314        err = bpf_prog_load("tailcall4.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 315                            &prog_fd);
 316        if (CHECK_FAIL(err))
 317                return;
 318
 319        prog = bpf_object__find_program_by_title(obj, "classifier");
 320        if (CHECK_FAIL(!prog))
 321                goto out;
 322
 323        main_fd = bpf_program__fd(prog);
 324        if (CHECK_FAIL(main_fd < 0))
 325                goto out;
 326
 327        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 328        if (CHECK_FAIL(!prog_array))
 329                goto out;
 330
 331        map_fd = bpf_map__fd(prog_array);
 332        if (CHECK_FAIL(map_fd < 0))
 333                goto out;
 334
 335        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 336        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 337                return;
 338
 339        data_fd = bpf_map__fd(data_map);
 340        if (CHECK_FAIL(map_fd < 0))
 341                return;
 342
 343        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 344                snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
 345
 346                prog = bpf_object__find_program_by_title(obj, prog_name);
 347                if (CHECK_FAIL(!prog))
 348                        goto out;
 349
 350                prog_fd = bpf_program__fd(prog);
 351                if (CHECK_FAIL(prog_fd < 0))
 352                        goto out;
 353
 354                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 355                if (CHECK_FAIL(err))
 356                        goto out;
 357        }
 358
 359        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 360                err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY);
 361                if (CHECK_FAIL(err))
 362                        goto out;
 363
 364                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 365                                        &duration, &retval, NULL);
 366                CHECK(err || retval != i, "tailcall",
 367                      "err %d errno %d retval %d\n", err, errno, retval);
 368        }
 369
 370        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 371                err = bpf_map_update_elem(data_fd, &zero, &i, BPF_ANY);
 372                if (CHECK_FAIL(err))
 373                        goto out;
 374
 375                err = bpf_map_delete_elem(map_fd, &i);
 376                if (CHECK_FAIL(err))
 377                        goto out;
 378
 379                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 380                                        &duration, &retval, NULL);
 381                CHECK(err || retval != 3, "tailcall",
 382                      "err %d errno %d retval %d\n", err, errno, retval);
 383        }
 384out:
 385        bpf_object__close(obj);
 386}
 387
 388/* test_tailcall_5 probes similarly to test_tailcall_4 that the kernel generates
 389 * an indirect jump when the keys are const but different from different branches.
 390 */
 391static void test_tailcall_5(void)
 392{
 393        int err, map_fd, prog_fd, main_fd, data_fd, i, key[] = { 1111, 1234, 5678 };
 394        struct bpf_map *prog_array, *data_map;
 395        struct bpf_program *prog;
 396        struct bpf_object *obj;
 397        __u32 retval, duration;
 398        static const int zero = 0;
 399        char buff[128] = {};
 400        char prog_name[32];
 401
 402        err = bpf_prog_load("tailcall5.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
 403                            &prog_fd);
 404        if (CHECK_FAIL(err))
 405                return;
 406
 407        prog = bpf_object__find_program_by_title(obj, "classifier");
 408        if (CHECK_FAIL(!prog))
 409                goto out;
 410
 411        main_fd = bpf_program__fd(prog);
 412        if (CHECK_FAIL(main_fd < 0))
 413                goto out;
 414
 415        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 416        if (CHECK_FAIL(!prog_array))
 417                goto out;
 418
 419        map_fd = bpf_map__fd(prog_array);
 420        if (CHECK_FAIL(map_fd < 0))
 421                goto out;
 422
 423        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 424        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 425                return;
 426
 427        data_fd = bpf_map__fd(data_map);
 428        if (CHECK_FAIL(map_fd < 0))
 429                return;
 430
 431        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 432                snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
 433
 434                prog = bpf_object__find_program_by_title(obj, prog_name);
 435                if (CHECK_FAIL(!prog))
 436                        goto out;
 437
 438                prog_fd = bpf_program__fd(prog);
 439                if (CHECK_FAIL(prog_fd < 0))
 440                        goto out;
 441
 442                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 443                if (CHECK_FAIL(err))
 444                        goto out;
 445        }
 446
 447        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 448                err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY);
 449                if (CHECK_FAIL(err))
 450                        goto out;
 451
 452                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 453                                        &duration, &retval, NULL);
 454                CHECK(err || retval != i, "tailcall",
 455                      "err %d errno %d retval %d\n", err, errno, retval);
 456        }
 457
 458        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 459                err = bpf_map_update_elem(data_fd, &zero, &key[i], BPF_ANY);
 460                if (CHECK_FAIL(err))
 461                        goto out;
 462
 463                err = bpf_map_delete_elem(map_fd, &i);
 464                if (CHECK_FAIL(err))
 465                        goto out;
 466
 467                err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 468                                        &duration, &retval, NULL);
 469                CHECK(err || retval != 3, "tailcall",
 470                      "err %d errno %d retval %d\n", err, errno, retval);
 471        }
 472out:
 473        bpf_object__close(obj);
 474}
 475
 476/* test_tailcall_bpf2bpf_1 purpose is to make sure that tailcalls are working
 477 * correctly in correlation with BPF subprograms
 478 */
 479static void test_tailcall_bpf2bpf_1(void)
 480{
 481        int err, map_fd, prog_fd, main_fd, i;
 482        struct bpf_map *prog_array;
 483        struct bpf_program *prog;
 484        struct bpf_object *obj;
 485        __u32 retval, duration;
 486        char prog_name[32];
 487
 488        err = bpf_prog_load("tailcall_bpf2bpf1.o", BPF_PROG_TYPE_SCHED_CLS,
 489                            &obj, &prog_fd);
 490        if (CHECK_FAIL(err))
 491                return;
 492
 493        prog = bpf_object__find_program_by_title(obj, "classifier");
 494        if (CHECK_FAIL(!prog))
 495                goto out;
 496
 497        main_fd = bpf_program__fd(prog);
 498        if (CHECK_FAIL(main_fd < 0))
 499                goto out;
 500
 501        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 502        if (CHECK_FAIL(!prog_array))
 503                goto out;
 504
 505        map_fd = bpf_map__fd(prog_array);
 506        if (CHECK_FAIL(map_fd < 0))
 507                goto out;
 508
 509        /* nop -> jmp */
 510        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 511                snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
 512
 513                prog = bpf_object__find_program_by_title(obj, prog_name);
 514                if (CHECK_FAIL(!prog))
 515                        goto out;
 516
 517                prog_fd = bpf_program__fd(prog);
 518                if (CHECK_FAIL(prog_fd < 0))
 519                        goto out;
 520
 521                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 522                if (CHECK_FAIL(err))
 523                        goto out;
 524        }
 525
 526        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 527                                0, &retval, &duration);
 528        CHECK(err || retval != 1, "tailcall",
 529              "err %d errno %d retval %d\n", err, errno, retval);
 530
 531        /* jmp -> nop, call subprog that will do tailcall */
 532        i = 1;
 533        err = bpf_map_delete_elem(map_fd, &i);
 534        if (CHECK_FAIL(err))
 535                goto out;
 536
 537        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 538                                0, &retval, &duration);
 539        CHECK(err || retval != 0, "tailcall", "err %d errno %d retval %d\n",
 540              err, errno, retval);
 541
 542        /* make sure that subprog can access ctx and entry prog that
 543         * called this subprog can properly return
 544         */
 545        i = 0;
 546        err = bpf_map_delete_elem(map_fd, &i);
 547        if (CHECK_FAIL(err))
 548                goto out;
 549
 550        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 551                                0, &retval, &duration);
 552        CHECK(err || retval != sizeof(pkt_v4) * 2,
 553              "tailcall", "err %d errno %d retval %d\n",
 554              err, errno, retval);
 555out:
 556        bpf_object__close(obj);
 557}
 558
 559/* test_tailcall_bpf2bpf_2 checks that the count value of the tail call limit
 560 * enforcement matches with expectations when tailcall is preceded with
 561 * bpf2bpf call.
 562 */
 563static void test_tailcall_bpf2bpf_2(void)
 564{
 565        int err, map_fd, prog_fd, main_fd, data_fd, i, val;
 566        struct bpf_map *prog_array, *data_map;
 567        struct bpf_program *prog;
 568        struct bpf_object *obj;
 569        __u32 retval, duration;
 570        char buff[128] = {};
 571
 572        err = bpf_prog_load("tailcall_bpf2bpf2.o", BPF_PROG_TYPE_SCHED_CLS,
 573                            &obj, &prog_fd);
 574        if (CHECK_FAIL(err))
 575                return;
 576
 577        prog = bpf_object__find_program_by_title(obj, "classifier");
 578        if (CHECK_FAIL(!prog))
 579                goto out;
 580
 581        main_fd = bpf_program__fd(prog);
 582        if (CHECK_FAIL(main_fd < 0))
 583                goto out;
 584
 585        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 586        if (CHECK_FAIL(!prog_array))
 587                goto out;
 588
 589        map_fd = bpf_map__fd(prog_array);
 590        if (CHECK_FAIL(map_fd < 0))
 591                goto out;
 592
 593        prog = bpf_object__find_program_by_title(obj, "classifier/0");
 594        if (CHECK_FAIL(!prog))
 595                goto out;
 596
 597        prog_fd = bpf_program__fd(prog);
 598        if (CHECK_FAIL(prog_fd < 0))
 599                goto out;
 600
 601        i = 0;
 602        err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 603        if (CHECK_FAIL(err))
 604                goto out;
 605
 606        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 607                                &duration, &retval, NULL);
 608        CHECK(err || retval != 1, "tailcall", "err %d errno %d retval %d\n",
 609              err, errno, retval);
 610
 611        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 612        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 613                return;
 614
 615        data_fd = bpf_map__fd(data_map);
 616        if (CHECK_FAIL(map_fd < 0))
 617                return;
 618
 619        i = 0;
 620        err = bpf_map_lookup_elem(data_fd, &i, &val);
 621        CHECK(err || val != 33, "tailcall count", "err %d errno %d count %d\n",
 622              err, errno, val);
 623
 624        i = 0;
 625        err = bpf_map_delete_elem(map_fd, &i);
 626        if (CHECK_FAIL(err))
 627                goto out;
 628
 629        err = bpf_prog_test_run(main_fd, 1, buff, sizeof(buff), 0,
 630                                &duration, &retval, NULL);
 631        CHECK(err || retval != 0, "tailcall", "err %d errno %d retval %d\n",
 632              err, errno, retval);
 633out:
 634        bpf_object__close(obj);
 635}
 636
 637/* test_tailcall_bpf2bpf_3 checks that non-trivial amount of stack (up to
 638 * 256 bytes) can be used within bpf subprograms that have the tailcalls
 639 * in them
 640 */
 641static void test_tailcall_bpf2bpf_3(void)
 642{
 643        int err, map_fd, prog_fd, main_fd, i;
 644        struct bpf_map *prog_array;
 645        struct bpf_program *prog;
 646        struct bpf_object *obj;
 647        __u32 retval, duration;
 648        char prog_name[32];
 649
 650        err = bpf_prog_load("tailcall_bpf2bpf3.o", BPF_PROG_TYPE_SCHED_CLS,
 651                            &obj, &prog_fd);
 652        if (CHECK_FAIL(err))
 653                return;
 654
 655        prog = bpf_object__find_program_by_title(obj, "classifier");
 656        if (CHECK_FAIL(!prog))
 657                goto out;
 658
 659        main_fd = bpf_program__fd(prog);
 660        if (CHECK_FAIL(main_fd < 0))
 661                goto out;
 662
 663        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 664        if (CHECK_FAIL(!prog_array))
 665                goto out;
 666
 667        map_fd = bpf_map__fd(prog_array);
 668        if (CHECK_FAIL(map_fd < 0))
 669                goto out;
 670
 671        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 672                snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
 673
 674                prog = bpf_object__find_program_by_title(obj, prog_name);
 675                if (CHECK_FAIL(!prog))
 676                        goto out;
 677
 678                prog_fd = bpf_program__fd(prog);
 679                if (CHECK_FAIL(prog_fd < 0))
 680                        goto out;
 681
 682                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 683                if (CHECK_FAIL(err))
 684                        goto out;
 685        }
 686
 687        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 688                                &duration, &retval, NULL);
 689        CHECK(err || retval != sizeof(pkt_v4) * 3,
 690              "tailcall", "err %d errno %d retval %d\n",
 691              err, errno, retval);
 692
 693        i = 1;
 694        err = bpf_map_delete_elem(map_fd, &i);
 695        if (CHECK_FAIL(err))
 696                goto out;
 697
 698        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 699                                &duration, &retval, NULL);
 700        CHECK(err || retval != sizeof(pkt_v4),
 701              "tailcall", "err %d errno %d retval %d\n",
 702              err, errno, retval);
 703
 704        i = 0;
 705        err = bpf_map_delete_elem(map_fd, &i);
 706        if (CHECK_FAIL(err))
 707                goto out;
 708
 709        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 710                                &duration, &retval, NULL);
 711        CHECK(err || retval != sizeof(pkt_v4) * 2,
 712              "tailcall", "err %d errno %d retval %d\n",
 713              err, errno, retval);
 714out:
 715        bpf_object__close(obj);
 716}
 717
 718#include "tailcall_bpf2bpf4.skel.h"
 719
 720/* test_tailcall_bpf2bpf_4 checks that tailcall counter is correctly preserved
 721 * across tailcalls combined with bpf2bpf calls. for making sure that tailcall
 722 * counter behaves correctly, bpf program will go through following flow:
 723 *
 724 * entry -> entry_subprog -> tailcall0 -> bpf_func0 -> subprog0 ->
 725 * -> tailcall1 -> bpf_func1 -> subprog1 -> tailcall2 -> bpf_func2 ->
 726 * subprog2 [here bump global counter] --------^
 727 *
 728 * We go through first two tailcalls and start counting from the subprog2 where
 729 * the loop begins. At the end of the test make sure that the global counter is
 730 * equal to 31, because tailcall counter includes the first two tailcalls
 731 * whereas global counter is incremented only on loop presented on flow above.
 732 *
 733 * The noise parameter is used to insert bpf_map_update calls into the logic
 734 * to force verifier to patch instructions. This allows us to ensure jump
 735 * logic remains correct with instruction movement.
 736 */
 737static void test_tailcall_bpf2bpf_4(bool noise)
 738{
 739        int err, map_fd, prog_fd, main_fd, data_fd, i;
 740        struct tailcall_bpf2bpf4__bss val;
 741        struct bpf_map *prog_array, *data_map;
 742        struct bpf_program *prog;
 743        struct bpf_object *obj;
 744        __u32 retval, duration;
 745        char prog_name[32];
 746
 747        err = bpf_prog_load("tailcall_bpf2bpf4.o", BPF_PROG_TYPE_SCHED_CLS,
 748                            &obj, &prog_fd);
 749        if (CHECK_FAIL(err))
 750                return;
 751
 752        prog = bpf_object__find_program_by_title(obj, "classifier");
 753        if (CHECK_FAIL(!prog))
 754                goto out;
 755
 756        main_fd = bpf_program__fd(prog);
 757        if (CHECK_FAIL(main_fd < 0))
 758                goto out;
 759
 760        prog_array = bpf_object__find_map_by_name(obj, "jmp_table");
 761        if (CHECK_FAIL(!prog_array))
 762                goto out;
 763
 764        map_fd = bpf_map__fd(prog_array);
 765        if (CHECK_FAIL(map_fd < 0))
 766                goto out;
 767
 768        for (i = 0; i < bpf_map__def(prog_array)->max_entries; i++) {
 769                snprintf(prog_name, sizeof(prog_name), "classifier/%i", i);
 770
 771                prog = bpf_object__find_program_by_title(obj, prog_name);
 772                if (CHECK_FAIL(!prog))
 773                        goto out;
 774
 775                prog_fd = bpf_program__fd(prog);
 776                if (CHECK_FAIL(prog_fd < 0))
 777                        goto out;
 778
 779                err = bpf_map_update_elem(map_fd, &i, &prog_fd, BPF_ANY);
 780                if (CHECK_FAIL(err))
 781                        goto out;
 782        }
 783
 784        data_map = bpf_object__find_map_by_name(obj, "tailcall.bss");
 785        if (CHECK_FAIL(!data_map || !bpf_map__is_internal(data_map)))
 786                return;
 787
 788        data_fd = bpf_map__fd(data_map);
 789        if (CHECK_FAIL(map_fd < 0))
 790                return;
 791
 792        i = 0;
 793        val.noise = noise;
 794        val.count = 0;
 795        err = bpf_map_update_elem(data_fd, &i, &val, BPF_ANY);
 796        if (CHECK_FAIL(err))
 797                goto out;
 798
 799        err = bpf_prog_test_run(main_fd, 1, &pkt_v4, sizeof(pkt_v4), 0,
 800                                &duration, &retval, NULL);
 801        CHECK(err || retval != sizeof(pkt_v4) * 3, "tailcall", "err %d errno %d retval %d\n",
 802              err, errno, retval);
 803
 804        i = 0;
 805        err = bpf_map_lookup_elem(data_fd, &i, &val);
 806        CHECK(err || val.count != 31, "tailcall count", "err %d errno %d count %d\n",
 807              err, errno, val.count);
 808
 809out:
 810        bpf_object__close(obj);
 811}
 812
 813void test_tailcalls(void)
 814{
 815        if (test__start_subtest("tailcall_1"))
 816                test_tailcall_1();
 817        if (test__start_subtest("tailcall_2"))
 818                test_tailcall_2();
 819        if (test__start_subtest("tailcall_3"))
 820                test_tailcall_3();
 821        if (test__start_subtest("tailcall_4"))
 822                test_tailcall_4();
 823        if (test__start_subtest("tailcall_5"))
 824                test_tailcall_5();
 825        if (test__start_subtest("tailcall_bpf2bpf_1"))
 826                test_tailcall_bpf2bpf_1();
 827        if (test__start_subtest("tailcall_bpf2bpf_2"))
 828                test_tailcall_bpf2bpf_2();
 829        if (test__start_subtest("tailcall_bpf2bpf_3"))
 830                test_tailcall_bpf2bpf_3();
 831        if (test__start_subtest("tailcall_bpf2bpf_4"))
 832                test_tailcall_bpf2bpf_4(false);
 833        if (test__start_subtest("tailcall_bpf2bpf_5"))
 834                test_tailcall_bpf2bpf_4(true);
 835}
 836