LXR linux/arch/x86/net/bpf

   1/* bpf_jit.S : BPF JIT helper functions
   2 *
   3 * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
   4 *
   5 * This program is free software; you can redistribute it and/or
   6 * modify it under the terms of the GNU General Public License
   7 * as published by the Free Software Foundation; version 2
   8 * of the License.
   9 */
  10#include <linux/linkage.h>
  11#include <asm/dwarf2.h>
  12
  13/*
  14 * Calling convention :
  15 * rbx : skb pointer (callee saved)
  16 * esi : offset of byte(s) to fetch in skb (can be scratched)
  17 * r10 : copy of skb->data
  18 * r9d : hlen = skb->len - skb->data_len
  19 */
  20#define SKBDATA %r10
  21#define SKF_MAX_NEG_OFF    $(-0x200000) /* SKF_LL_OFF from filter.h */
  22#define MAX_BPF_STACK (512 /* from filter.h */ + \
  23        32 /* space for rbx,r13,r14,r15 */ + \
  24        8 /* space for skb_copy_bits */)
  25
  26sk_load_word:
  27        .globl  sk_load_word
  28
  29        test    %esi,%esi
  30        js      bpf_slow_path_word_neg
  31
  32sk_load_word_positive_offset:
  33        .globl  sk_load_word_positive_offset
  34
  35        mov     %r9d,%eax               # hlen
  36        sub     %esi,%eax               # hlen - offset
  37        cmp     $3,%eax
  38        jle     bpf_slow_path_word
  39        mov     (SKBDATA,%rsi),%eax
  40        bswap   %eax                    /* ntohl() */
  41        ret
  42
  43sk_load_half:
  44        .globl  sk_load_half
  45
  46        test    %esi,%esi
  47        js      bpf_slow_path_half_neg
  48
  49sk_load_half_positive_offset:
  50        .globl  sk_load_half_positive_offset
  51
  52        mov     %r9d,%eax
  53        sub     %esi,%eax               #       hlen - offset
  54        cmp     $1,%eax
  55        jle     bpf_slow_path_half
  56        movzwl  (SKBDATA,%rsi),%eax
  57        rol     $8,%ax                  # ntohs()
  58        ret
  59
  60sk_load_byte:
  61        .globl  sk_load_byte
  62
  63        test    %esi,%esi
  64        js      bpf_slow_path_byte_neg
  65
  66sk_load_byte_positive_offset:
  67        .globl  sk_load_byte_positive_offset
  68
  69        cmp     %esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
  70        jle     bpf_slow_path_byte
  71        movzbl  (SKBDATA,%rsi),%eax
  72        ret
  73
  74/* rsi contains offset and can be scratched */
  75#define bpf_slow_path_common(LEN)               \
  76        mov     %rbx, %rdi; /* arg1 == skb */   \
  77        push    %r9;                            \
  78        push    SKBDATA;                        \
  79/* rsi already has offset */                    \
  80        mov     $LEN,%ecx;      /* len */       \
  81        lea     - MAX_BPF_STACK + 32(%rbp),%rdx;                        \
  82        call    skb_copy_bits;                  \
  83        test    %eax,%eax;                      \
  84        pop     SKBDATA;                        \
  85        pop     %r9;
  86
  87
  88bpf_slow_path_word:
  89        bpf_slow_path_common(4)
  90        js      bpf_error
  91        mov     - MAX_BPF_STACK + 32(%rbp),%eax
  92        bswap   %eax
  93        ret
  94
  95bpf_slow_path_half:
  96        bpf_slow_path_common(2)
  97        js      bpf_error
  98        mov     - MAX_BPF_STACK + 32(%rbp),%ax
  99        rol     $8,%ax
 100        movzwl  %ax,%eax
 101        ret
 102
 103bpf_slow_path_byte:
 104        bpf_slow_path_common(1)
 105        js      bpf_error
 106        movzbl  - MAX_BPF_STACK + 32(%rbp),%eax
 107        ret
 108
 109#define sk_negative_common(SIZE)                                \
 110        mov     %rbx, %rdi; /* arg1 == skb */                   \
 111        push    %r9;                                            \
 112        push    SKBDATA;                                        \
 113/* rsi already has offset */                                    \
 114        mov     $SIZE,%edx;     /* size */                      \
 115        call    bpf_internal_load_pointer_neg_helper;           \
 116        test    %rax,%rax;                                      \
 117        pop     SKBDATA;                                        \
 118        pop     %r9;                                            \
 119        jz      bpf_error
 120
 121bpf_slow_path_word_neg:
 122        cmp     SKF_MAX_NEG_OFF, %esi   /* test range */
 123        jl      bpf_error       /* offset lower -> error  */
 124sk_load_word_negative_offset:
 125        .globl  sk_load_word_negative_offset
 126        sk_negative_common(4)
 127        mov     (%rax), %eax
 128        bswap   %eax
 129        ret
 130
 131bpf_slow_path_half_neg:
 132        cmp     SKF_MAX_NEG_OFF, %esi
 133        jl      bpf_error
 134sk_load_half_negative_offset:
 135        .globl  sk_load_half_negative_offset
 136        sk_negative_common(2)
 137        mov     (%rax),%ax
 138        rol     $8,%ax
 139        movzwl  %ax,%eax
 140        ret
 141
 142bpf_slow_path_byte_neg:
 143        cmp     SKF_MAX_NEG_OFF, %esi
 144        jl      bpf_error
 145sk_load_byte_negative_offset:
 146        .globl  sk_load_byte_negative_offset
 147        sk_negative_common(1)
 148        movzbl  (%rax), %eax
 149        ret
 150
 151bpf_error:
 152# force a return 0 from jit handler
 153        xor     %eax,%eax
 154        mov     - MAX_BPF_STACK(%rbp),%rbx
 155        mov     - MAX_BPF_STACK + 8(%rbp),%r13
 156        mov     - MAX_BPF_STACK + 16(%rbp),%r14
 157        mov     - MAX_BPF_STACK + 24(%rbp),%r15
 158        leaveq
 159        ret
 160