1
2
3
4
5
6#include <linux/bpf.h>
7#include <linux/capability.h>
8#include <net/xdp_sock.h>
9#include <linux/slab.h>
10#include <linux/sched.h>
11
12struct xsk_map {
13 struct bpf_map map;
14 struct xdp_sock **xsk_map;
15 struct list_head __percpu *flush_list;
16};
17
18static struct bpf_map *xsk_map_alloc(union bpf_attr *attr)
19{
20 struct xsk_map *m;
21 int cpu, err;
22 u64 cost;
23
24 if (!capable(CAP_NET_ADMIN))
25 return ERR_PTR(-EPERM);
26
27 if (attr->max_entries == 0 || attr->key_size != 4 ||
28 attr->value_size != 4 ||
29 attr->map_flags & ~(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY))
30 return ERR_PTR(-EINVAL);
31
32 m = kzalloc(sizeof(*m), GFP_USER);
33 if (!m)
34 return ERR_PTR(-ENOMEM);
35
36 bpf_map_init_from_attr(&m->map, attr);
37
38 cost = (u64)m->map.max_entries * sizeof(struct xdp_sock *);
39 cost += sizeof(struct list_head) * num_possible_cpus();
40
41
42 err = bpf_map_charge_init(&m->map.memory, cost);
43 if (err)
44 goto free_m;
45
46 err = -ENOMEM;
47
48 m->flush_list = alloc_percpu(struct list_head);
49 if (!m->flush_list)
50 goto free_charge;
51
52 for_each_possible_cpu(cpu)
53 INIT_LIST_HEAD(per_cpu_ptr(m->flush_list, cpu));
54
55 m->xsk_map = bpf_map_area_alloc(m->map.max_entries *
56 sizeof(struct xdp_sock *),
57 m->map.numa_node);
58 if (!m->xsk_map)
59 goto free_percpu;
60 return &m->map;
61
62free_percpu:
63 free_percpu(m->flush_list);
64free_charge:
65 bpf_map_charge_finish(&m->map.memory);
66free_m:
67 kfree(m);
68 return ERR_PTR(err);
69}
70
71static void xsk_map_free(struct bpf_map *map)
72{
73 struct xsk_map *m = container_of(map, struct xsk_map, map);
74 int i;
75
76 bpf_clear_redirect_map(map);
77 synchronize_net();
78
79 for (i = 0; i < map->max_entries; i++) {
80 struct xdp_sock *xs;
81
82 xs = m->xsk_map[i];
83 if (!xs)
84 continue;
85
86 sock_put((struct sock *)xs);
87 }
88
89 free_percpu(m->flush_list);
90 bpf_map_area_free(m->xsk_map);
91 kfree(m);
92}
93
94static int xsk_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
95{
96 struct xsk_map *m = container_of(map, struct xsk_map, map);
97 u32 index = key ? *(u32 *)key : U32_MAX;
98 u32 *next = next_key;
99
100 if (index >= m->map.max_entries) {
101 *next = 0;
102 return 0;
103 }
104
105 if (index == m->map.max_entries - 1)
106 return -ENOENT;
107 *next = index + 1;
108 return 0;
109}
110
111struct xdp_sock *__xsk_map_lookup_elem(struct bpf_map *map, u32 key)
112{
113 struct xsk_map *m = container_of(map, struct xsk_map, map);
114 struct xdp_sock *xs;
115
116 if (key >= map->max_entries)
117 return NULL;
118
119 xs = READ_ONCE(m->xsk_map[key]);
120 return xs;
121}
122
123int __xsk_map_redirect(struct bpf_map *map, struct xdp_buff *xdp,
124 struct xdp_sock *xs)
125{
126 struct xsk_map *m = container_of(map, struct xsk_map, map);
127 struct list_head *flush_list = this_cpu_ptr(m->flush_list);
128 int err;
129
130 err = xsk_rcv(xs, xdp);
131 if (err)
132 return err;
133
134 if (!xs->flush_node.prev)
135 list_add(&xs->flush_node, flush_list);
136
137 return 0;
138}
139
140void __xsk_map_flush(struct bpf_map *map)
141{
142 struct xsk_map *m = container_of(map, struct xsk_map, map);
143 struct list_head *flush_list = this_cpu_ptr(m->flush_list);
144 struct xdp_sock *xs, *tmp;
145
146 list_for_each_entry_safe(xs, tmp, flush_list, flush_node) {
147 xsk_flush(xs);
148 __list_del_clearprev(&xs->flush_node);
149 }
150}
151
152static void *xsk_map_lookup_elem(struct bpf_map *map, void *key)
153{
154 WARN_ON_ONCE(!rcu_read_lock_held());
155 return __xsk_map_lookup_elem(map, *(u32 *)key);
156}
157
158static void *xsk_map_lookup_elem_sys_only(struct bpf_map *map, void *key)
159{
160 return ERR_PTR(-EOPNOTSUPP);
161}
162
163static int xsk_map_update_elem(struct bpf_map *map, void *key, void *value,
164 u64 map_flags)
165{
166 struct xsk_map *m = container_of(map, struct xsk_map, map);
167 u32 i = *(u32 *)key, fd = *(u32 *)value;
168 struct xdp_sock *xs, *old_xs;
169 struct socket *sock;
170 int err;
171
172 if (unlikely(map_flags > BPF_EXIST))
173 return -EINVAL;
174 if (unlikely(i >= m->map.max_entries))
175 return -E2BIG;
176 if (unlikely(map_flags == BPF_NOEXIST))
177 return -EEXIST;
178
179 sock = sockfd_lookup(fd, &err);
180 if (!sock)
181 return err;
182
183 if (sock->sk->sk_family != PF_XDP) {
184 sockfd_put(sock);
185 return -EOPNOTSUPP;
186 }
187
188 xs = (struct xdp_sock *)sock->sk;
189
190 if (!xsk_is_setup_for_bpf_map(xs)) {
191 sockfd_put(sock);
192 return -EOPNOTSUPP;
193 }
194
195 sock_hold(sock->sk);
196
197 old_xs = xchg(&m->xsk_map[i], xs);
198 if (old_xs)
199 sock_put((struct sock *)old_xs);
200
201 sockfd_put(sock);
202 return 0;
203}
204
205static int xsk_map_delete_elem(struct bpf_map *map, void *key)
206{
207 struct xsk_map *m = container_of(map, struct xsk_map, map);
208 struct xdp_sock *old_xs;
209 int k = *(u32 *)key;
210
211 if (k >= map->max_entries)
212 return -EINVAL;
213
214 old_xs = xchg(&m->xsk_map[k], NULL);
215 if (old_xs)
216 sock_put((struct sock *)old_xs);
217
218 return 0;
219}
220
221const struct bpf_map_ops xsk_map_ops = {
222 .map_alloc = xsk_map_alloc,
223 .map_free = xsk_map_free,
224 .map_get_next_key = xsk_map_get_next_key,
225 .map_lookup_elem = xsk_map_lookup_elem,
226 .map_lookup_elem_sys_only = xsk_map_lookup_elem_sys_only,
227 .map_update_elem = xsk_map_update_elem,
228 .map_delete_elem = xsk_map_delete_elem,
229 .map_check_btf = map_check_no_btf,
230};
231
