LXR linux/arch/riscv/net/bpf_jit

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Common functionality for RV32 and RV64 BPF JIT compilers
   4 *
   5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
   6 *
   7 */
   8
   9#include <linux/bpf.h>
  10#include <linux/filter.h>
  11#include "bpf_jit.h"
  12
  13/* Number of iterations to try until offsets converge. */
  14#define NR_JIT_ITERATIONS       16
  15
  16static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset)
  17{
  18        const struct bpf_prog *prog = ctx->prog;
  19        int i;
  20
  21        for (i = 0; i < prog->len; i++) {
  22                const struct bpf_insn *insn = &prog->insnsi[i];
  23                int ret;
  24
  25                ret = bpf_jit_emit_insn(insn, ctx, extra_pass);
  26                /* BPF_LD | BPF_IMM | BPF_DW: skip the next instruction. */
  27                if (ret > 0)
  28                        i++;
  29                if (offset)
  30                        offset[i] = ctx->ninsns;
  31                if (ret < 0)
  32                        return ret;
  33        }
  34        return 0;
  35}
  36
  37bool bpf_jit_needs_zext(void)
  38{
  39        return true;
  40}
  41
  42struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
  43{
  44        bool tmp_blinded = false, extra_pass = false;
  45        struct bpf_prog *tmp, *orig_prog = prog;
  46        int pass = 0, prev_ninsns = 0, i;
  47        struct rv_jit_data *jit_data;
  48        struct rv_jit_context *ctx;
  49        unsigned int image_size = 0;
  50
  51        if (!prog->jit_requested)
  52                return orig_prog;
  53
  54        tmp = bpf_jit_blind_constants(prog);
  55        if (IS_ERR(tmp))
  56                return orig_prog;
  57        if (tmp != prog) {
  58                tmp_blinded = true;
  59                prog = tmp;
  60        }
  61
  62        jit_data = prog->aux->jit_data;
  63        if (!jit_data) {
  64                jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL);
  65                if (!jit_data) {
  66                        prog = orig_prog;
  67                        goto out;
  68                }
  69                prog->aux->jit_data = jit_data;
  70        }
  71
  72        ctx = &jit_data->ctx;
  73
  74        if (ctx->offset) {
  75                extra_pass = true;
  76                image_size = sizeof(*ctx->insns) * ctx->ninsns;
  77                goto skip_init_ctx;
  78        }
  79
  80        ctx->prog = prog;
  81        ctx->offset = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
  82        if (!ctx->offset) {
  83                prog = orig_prog;
  84                goto out_offset;
  85        }
  86        for (i = 0; i < prog->len; i++) {
  87                prev_ninsns += 32;
  88                ctx->offset[i] = prev_ninsns;
  89        }
  90
  91        for (i = 0; i < NR_JIT_ITERATIONS; i++) {
  92                pass++;
  93                ctx->ninsns = 0;
  94                if (build_body(ctx, extra_pass, ctx->offset)) {
  95                        prog = orig_prog;
  96                        goto out_offset;
  97                }
  98                bpf_jit_build_prologue(ctx);
  99                ctx->epilogue_offset = ctx->ninsns;
 100                bpf_jit_build_epilogue(ctx);
 101
 102                if (ctx->ninsns == prev_ninsns) {
 103                        if (jit_data->header)
 104                                break;
 105
 106                        image_size = sizeof(*ctx->insns) * ctx->ninsns;
 107                        jit_data->header =
 108                                bpf_jit_binary_alloc(image_size,
 109                                                     &jit_data->image,
 110                                                     sizeof(u32),
 111                                                     bpf_fill_ill_insns);
 112                        if (!jit_data->header) {
 113                                prog = orig_prog;
 114                                goto out_offset;
 115                        }
 116
 117                        ctx->insns = (u16 *)jit_data->image;
 118                        /*
 119                         * Now, when the image is allocated, the image can
 120                         * potentially shrink more (auipc/jalr -> jal).
 121                         */
 122                }
 123                prev_ninsns = ctx->ninsns;
 124        }
 125
 126        if (i == NR_JIT_ITERATIONS) {
 127                pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
 128                bpf_jit_binary_free(jit_data->header);
 129                prog = orig_prog;
 130                goto out_offset;
 131        }
 132
 133skip_init_ctx:
 134        pass++;
 135        ctx->ninsns = 0;
 136
 137        bpf_jit_build_prologue(ctx);
 138        if (build_body(ctx, extra_pass, NULL)) {
 139                bpf_jit_binary_free(jit_data->header);
 140                prog = orig_prog;
 141                goto out_offset;
 142        }
 143        bpf_jit_build_epilogue(ctx);
 144
 145        if (bpf_jit_enable > 1)
 146                bpf_jit_dump(prog->len, image_size, pass, ctx->insns);
 147
 148        prog->bpf_func = (void *)ctx->insns;
 149        prog->jited = 1;
 150        prog->jited_len = image_size;
 151
 152        bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
 153
 154        if (!prog->is_func || extra_pass) {
 155out_offset:
 156                kfree(ctx->offset);
 157                kfree(jit_data);
 158                prog->aux->jit_data = NULL;
 159        }
 160out:
 161
 162        if (tmp_blinded)
 163                bpf_jit_prog_release_other(prog, prog == orig_prog ?
 164                                           tmp : orig_prog);
 165        return prog;
 166}
 167