1
2
3
4
5
6
7
8
9
10
11
12
13
14#include "qemu/osdep.h"
15#include "qemu/error-report.h"
16#include "qapi/qapi-types-misc.h"
17#include "qapi/qapi-commands-ebpf.h"
18
19#include <bpf/libbpf.h>
20#include <bpf/bpf.h>
21
22#include "hw/virtio/virtio-net.h"
23
24#include "ebpf/ebpf_rss.h"
25#include "ebpf/rss.bpf.skeleton.h"
26#include "ebpf/ebpf.h"
27
28#include "trace.h"
29
30void ebpf_rss_init(struct EBPFRSSContext *ctx)
31{
32 if (ctx != NULL) {
33 ctx->obj = NULL;
34 ctx->program_fd = -1;
35 ctx->map_configuration = -1;
36 ctx->map_toeplitz_key = -1;
37 ctx->map_indirections_table = -1;
38
39 ctx->mmap_configuration = NULL;
40 ctx->mmap_toeplitz_key = NULL;
41 ctx->mmap_indirections_table = NULL;
42 }
43}
44
45bool ebpf_rss_is_loaded(struct EBPFRSSContext *ctx)
46{
47 return ctx != NULL && (ctx->obj != NULL || ctx->program_fd != -1);
48}
49
50static bool ebpf_rss_mmap(struct EBPFRSSContext *ctx, Error **errp)
51{
52 ctx->mmap_configuration = mmap(NULL, qemu_real_host_page_size(),
53 PROT_READ | PROT_WRITE, MAP_SHARED,
54 ctx->map_configuration, 0);
55 if (ctx->mmap_configuration == MAP_FAILED) {
56 trace_ebpf_rss_mmap_error(ctx, "configuration");
57 error_setg(errp, "Unable to map eBPF configuration array");
58 return false;
59 }
60 ctx->mmap_toeplitz_key = mmap(NULL, qemu_real_host_page_size(),
61 PROT_READ | PROT_WRITE, MAP_SHARED,
62 ctx->map_toeplitz_key, 0);
63 if (ctx->mmap_toeplitz_key == MAP_FAILED) {
64 trace_ebpf_rss_mmap_error(ctx, "toeplitz key");
65 error_setg(errp, "Unable to map eBPF toeplitz array");
66 goto toeplitz_fail;
67 }
68 ctx->mmap_indirections_table = mmap(NULL, qemu_real_host_page_size(),
69 PROT_READ | PROT_WRITE, MAP_SHARED,
70 ctx->map_indirections_table, 0);
71 if (ctx->mmap_indirections_table == MAP_FAILED) {
72 trace_ebpf_rss_mmap_error(ctx, "indirections table");
73 error_setg(errp, "Unable to map eBPF indirection array");
74 goto indirection_fail;
75 }
76
77 trace_ebpf_rss_mmap(ctx,
78 ctx->mmap_configuration,
79 ctx->mmap_toeplitz_key,
80 ctx->mmap_indirections_table);
81 return true;
82
83indirection_fail:
84 munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
85 ctx->mmap_toeplitz_key = NULL;
86toeplitz_fail:
87 munmap(ctx->mmap_configuration, qemu_real_host_page_size());
88 ctx->mmap_configuration = NULL;
89
90 ctx->mmap_indirections_table = NULL;
91 return false;
92}
93
94static void ebpf_rss_munmap(struct EBPFRSSContext *ctx)
95{
96 munmap(ctx->mmap_indirections_table, qemu_real_host_page_size());
97 munmap(ctx->mmap_toeplitz_key, qemu_real_host_page_size());
98 munmap(ctx->mmap_configuration, qemu_real_host_page_size());
99
100 ctx->mmap_configuration = NULL;
101 ctx->mmap_toeplitz_key = NULL;
102 ctx->mmap_indirections_table = NULL;
103}
104
105bool ebpf_rss_load(struct EBPFRSSContext *ctx, Error **errp)
106{
107 struct rss_bpf *rss_bpf_ctx;
108
109 if (ebpf_rss_is_loaded(ctx)) {
110 return false;
111 }
112
113 rss_bpf_ctx = rss_bpf__open();
114 if (rss_bpf_ctx == NULL) {
115 trace_ebpf_rss_open_error(ctx);
116 error_setg(errp, "Unable to open eBPF RSS object");
117 goto error;
118 }
119
120 bpf_program__set_type(rss_bpf_ctx->progs.tun_rss_steering_prog, BPF_PROG_TYPE_SOCKET_FILTER);
121
122 if (rss_bpf__load(rss_bpf_ctx)) {
123 trace_ebpf_rss_load_error(ctx);
124 error_setg(errp, "Unable to load eBPF program");
125 goto error;
126 }
127
128 ctx->obj = rss_bpf_ctx;
129 ctx->program_fd = bpf_program__fd(
130 rss_bpf_ctx->progs.tun_rss_steering_prog);
131 ctx->map_configuration = bpf_map__fd(
132 rss_bpf_ctx->maps.tap_rss_map_configurations);
133 ctx->map_indirections_table = bpf_map__fd(
134 rss_bpf_ctx->maps.tap_rss_map_indirection_table);
135 ctx->map_toeplitz_key = bpf_map__fd(
136 rss_bpf_ctx->maps.tap_rss_map_toeplitz_key);
137
138 trace_ebpf_rss_load(ctx,
139 ctx->program_fd,
140 ctx->map_configuration,
141 ctx->map_indirections_table,
142 ctx->map_toeplitz_key);
143 if (!ebpf_rss_mmap(ctx, errp)) {
144 goto error;
145 }
146
147 return true;
148error:
149 rss_bpf__destroy(rss_bpf_ctx);
150 ctx->obj = NULL;
151 ctx->program_fd = -1;
152 ctx->map_configuration = -1;
153 ctx->map_toeplitz_key = -1;
154 ctx->map_indirections_table = -1;
155
156 return false;
157}
158
159bool ebpf_rss_load_fds(struct EBPFRSSContext *ctx, int program_fd,
160 int config_fd, int toeplitz_fd, int table_fd,
161 Error **errp)
162{
163 if (ebpf_rss_is_loaded(ctx)) {
164 error_setg(errp, "eBPF program is already loaded");
165 return false;
166 }
167
168 if (program_fd < 0) {
169 error_setg(errp, "eBPF program FD is not open");
170 return false;
171 }
172 if (config_fd < 0) {
173 error_setg(errp, "eBPF config FD is not open");
174 return false;
175 }
176 if (toeplitz_fd < 0) {
177 error_setg(errp, "eBPF toeplitz FD is not open");
178 return false;
179 }
180 if (table_fd < 0) {
181 error_setg(errp, "eBPF indirection FD is not open");
182 return false;
183 }
184
185 ctx->program_fd = program_fd;
186 ctx->map_configuration = config_fd;
187 ctx->map_toeplitz_key = toeplitz_fd;
188 ctx->map_indirections_table = table_fd;
189
190 trace_ebpf_rss_load(ctx,
191 ctx->program_fd,
192 ctx->map_configuration,
193 ctx->map_indirections_table,
194 ctx->map_toeplitz_key);
195
196 if (!ebpf_rss_mmap(ctx, errp)) {
197 ctx->program_fd = -1;
198 ctx->map_configuration = -1;
199 ctx->map_toeplitz_key = -1;
200 ctx->map_indirections_table = -1;
201 return false;
202 }
203
204 return true;
205}
206
207static void ebpf_rss_set_config(struct EBPFRSSContext *ctx,
208 struct EBPFRSSConfig *config)
209{
210 memcpy(ctx->mmap_configuration, config, sizeof(*config));
211}
212
213static bool ebpf_rss_set_indirections_table(struct EBPFRSSContext *ctx,
214 uint16_t *indirections_table,
215 size_t len,
216 Error **errp)
217{
218 char *cursor = ctx->mmap_indirections_table;
219
220 if (len > VIRTIO_NET_RSS_MAX_TABLE_LEN) {
221 error_setg(errp, "Indirections table length %zu exceeds limit %d",
222 len, VIRTIO_NET_RSS_MAX_TABLE_LEN);
223 return false;
224 }
225
226 for (size_t i = 0; i < len; i++) {
227 *(uint16_t *)cursor = indirections_table[i];
228 cursor += 8;
229 }
230
231 return true;
232}
233
234static void ebpf_rss_set_toepliz_key(struct EBPFRSSContext *ctx,
235 uint8_t *toeplitz_key)
236{
237
238 uint8_t toe[VIRTIO_NET_RSS_MAX_KEY_SIZE] = {};
239
240 memcpy(toe, toeplitz_key, VIRTIO_NET_RSS_MAX_KEY_SIZE);
241 *(uint32_t *)toe = ntohl(*(uint32_t *)toe);
242
243 memcpy(ctx->mmap_toeplitz_key, toe, VIRTIO_NET_RSS_MAX_KEY_SIZE);
244}
245
246bool ebpf_rss_set_all(struct EBPFRSSContext *ctx, struct EBPFRSSConfig *config,
247 uint16_t *indirections_table, uint8_t *toeplitz_key,
248 Error **errp)
249{
250 if (!ebpf_rss_is_loaded(ctx)) {
251 error_setg(errp, "eBPF program is not loaded");
252 return false;
253 }
254 if (config == NULL) {
255 error_setg(errp, "eBPF config table is NULL");
256 return false;
257 }
258 if (indirections_table == NULL) {
259 error_setg(errp, "eBPF indirections table is NULL");
260 return false;
261 }
262 if (toeplitz_key == NULL) {
263 error_setg(errp, "eBPF toeplitz key is NULL");
264 return false;
265 }
266
267 ebpf_rss_set_config(ctx, config);
268
269 if (!ebpf_rss_set_indirections_table(ctx, indirections_table,
270 config->indirections_len,
271 errp)) {
272 return false;
273 }
274
275 ebpf_rss_set_toepliz_key(ctx, toeplitz_key);
276
277 trace_ebpf_rss_set_data(ctx, config, indirections_table, toeplitz_key);
278
279 return true;
280}
281
282void ebpf_rss_unload(struct EBPFRSSContext *ctx)
283{
284 if (!ebpf_rss_is_loaded(ctx)) {
285 return;
286 }
287
288 trace_ebpf_rss_unload(ctx);
289
290 ebpf_rss_munmap(ctx);
291
292 if (ctx->obj) {
293 rss_bpf__destroy(ctx->obj);
294 } else {
295 close(ctx->program_fd);
296 close(ctx->map_configuration);
297 close(ctx->map_toeplitz_key);
298 close(ctx->map_indirections_table);
299 }
300
301 ctx->obj = NULL;
302 ctx->program_fd = -1;
303 ctx->map_configuration = -1;
304 ctx->map_toeplitz_key = -1;
305 ctx->map_indirections_table = -1;
306}
307
308ebpf_binary_init(EBPF_PROGRAM_ID_RSS, rss_bpf__elf_bytes)
309
