LXR linux/include/linux/bpf_local

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 * Copyright (c) 2019 Facebook
   4 * Copyright 2020 Google LLC.
   5 */
   6
   7#ifndef _BPF_LOCAL_STORAGE_H
   8#define _BPF_LOCAL_STORAGE_H
   9
  10#include <linux/bpf.h>
  11#include <linux/rculist.h>
  12#include <linux/list.h>
  13#include <linux/hash.h>
  14#include <linux/types.h>
  15#include <uapi/linux/btf.h>
  16
  17#define BPF_LOCAL_STORAGE_CACHE_SIZE    16
  18
  19struct bpf_local_storage_map_bucket {
  20        struct hlist_head list;
  21        raw_spinlock_t lock;
  22};
  23
  24/* Thp map is not the primary owner of a bpf_local_storage_elem.
  25 * Instead, the container object (eg. sk->sk_bpf_storage) is.
  26 *
  27 * The map (bpf_local_storage_map) is for two purposes
  28 * 1. Define the size of the "local storage".  It is
  29 *    the map's value_size.
  30 *
  31 * 2. Maintain a list to keep track of all elems such
  32 *    that they can be cleaned up during the map destruction.
  33 *
  34 * When a bpf local storage is being looked up for a
  35 * particular object,  the "bpf_map" pointer is actually used
  36 * as the "key" to search in the list of elem in
  37 * the respective bpf_local_storage owned by the object.
  38 *
  39 * e.g. sk->sk_bpf_storage is the mini-map with the "bpf_map" pointer
  40 * as the searching key.
  41 */
  42struct bpf_local_storage_map {
  43        struct bpf_map map;
  44        /* Lookup elem does not require accessing the map.
  45         *
  46         * Updating/Deleting requires a bucket lock to
  47         * link/unlink the elem from the map.  Having
  48         * multiple buckets to improve contention.
  49         */
  50        struct bpf_local_storage_map_bucket *buckets;
  51        u32 bucket_log;
  52        u16 elem_size;
  53        u16 cache_idx;
  54};
  55
  56struct bpf_local_storage_data {
  57        /* smap is used as the searching key when looking up
  58         * from the object's bpf_local_storage.
  59         *
  60         * Put it in the same cacheline as the data to minimize
  61         * the number of cachelines accessed during the cache hit case.
  62         */
  63        struct bpf_local_storage_map __rcu *smap;
  64        u8 data[] __aligned(8);
  65};
  66
  67/* Linked to bpf_local_storage and bpf_local_storage_map */
  68struct bpf_local_storage_elem {
  69        struct hlist_node map_node;     /* Linked to bpf_local_storage_map */
  70        struct hlist_node snode;        /* Linked to bpf_local_storage */
  71        struct bpf_local_storage __rcu *local_storage;
  72        struct rcu_head rcu;
  73        /* 8 bytes hole */
  74        /* The data is stored in another cacheline to minimize
  75         * the number of cachelines access during a cache hit.
  76         */
  77        struct bpf_local_storage_data sdata ____cacheline_aligned;
  78};
  79
  80struct bpf_local_storage {
  81        struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE];
  82        struct hlist_head list; /* List of bpf_local_storage_elem */
  83        void *owner;            /* The object that owns the above "list" of
  84                                 * bpf_local_storage_elem.
  85                                 */
  86        struct rcu_head rcu;
  87        raw_spinlock_t lock;    /* Protect adding/removing from the "list" */
  88};
  89
  90/* U16_MAX is much more than enough for sk local storage
  91 * considering a tcp_sock is ~2k.
  92 */
  93#define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE                                       \
  94        min_t(u32,                                                             \
  95              (KMALLOC_MAX_SIZE - MAX_BPF_STACK -                              \
  96               sizeof(struct bpf_local_storage_elem)),                         \
  97              (U16_MAX - sizeof(struct bpf_local_storage_elem)))
  98
  99#define SELEM(_SDATA)                                                          \
 100        container_of((_SDATA), struct bpf_local_storage_elem, sdata)
 101#define SDATA(_SELEM) (&(_SELEM)->sdata)
 102
 103#define BPF_LOCAL_STORAGE_CACHE_SIZE    16
 104
 105struct bpf_local_storage_cache {
 106        spinlock_t idx_lock;
 107        u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE];
 108};
 109
 110#define DEFINE_BPF_STORAGE_CACHE(name)                          \
 111static struct bpf_local_storage_cache name = {                  \
 112        .idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock),        \
 113}
 114
 115u16 bpf_local_storage_cache_idx_get(struct bpf_local_storage_cache *cache);
 116void bpf_local_storage_cache_idx_free(struct bpf_local_storage_cache *cache,
 117                                      u16 idx);
 118
 119/* Helper functions for bpf_local_storage */
 120int bpf_local_storage_map_alloc_check(union bpf_attr *attr);
 121
 122struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr);
 123
 124struct bpf_local_storage_data *
 125bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
 126                         struct bpf_local_storage_map *smap,
 127                         bool cacheit_lockit);
 128
 129void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,
 130                                int __percpu *busy_counter);
 131
 132int bpf_local_storage_map_check_btf(const struct bpf_map *map,
 133                                    const struct btf *btf,
 134                                    const struct btf_type *key_type,
 135                                    const struct btf_type *value_type);
 136
 137void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
 138                                   struct bpf_local_storage_elem *selem);
 139
 140bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
 141                                     struct bpf_local_storage_elem *selem,
 142                                     bool uncharge_omem);
 143
 144void bpf_selem_unlink(struct bpf_local_storage_elem *selem);
 145
 146void bpf_selem_link_map(struct bpf_local_storage_map *smap,
 147                        struct bpf_local_storage_elem *selem);
 148
 149void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem);
 150
 151struct bpf_local_storage_elem *
 152bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
 153                bool charge_mem);
 154
 155int
 156bpf_local_storage_alloc(void *owner,
 157                        struct bpf_local_storage_map *smap,
 158                        struct bpf_local_storage_elem *first_selem);
 159
 160struct bpf_local_storage_data *
 161bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
 162                         void *value, u64 map_flags);
 163
 164#endif /* _BPF_LOCAL_STORAGE_H */
 165