1
2
3
4
5
6#include <uapi/linux/btf.h>
7#include <linux/kernel.h>
8#include <linux/types.h>
9#include <linux/slab.h>
10#include <linux/bpf.h>
11#include <linux/btf.h>
12#include <linux/bpf_verifier.h>
13#include <linux/filter.h>
14#include <net/netlink.h>
15#include <linux/file.h>
16#include <linux/vmalloc.h>
17#include <linux/stringify.h>
18#include <linux/bsearch.h>
19#include <linux/sort.h>
20#include <linux/perf_event.h>
21#include <linux/ctype.h>
22#include <linux/error-injection.h>
23#include <linux/bpf_lsm.h>
24#include <linux/btf_ids.h>
25
26#include "disasm.h"
27
28static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
29#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
30 [_id] = & _name ## _verifier_ops,
31#define BPF_MAP_TYPE(_id, _ops)
32#define BPF_LINK_TYPE(_id, _name)
33#include <linux/bpf_types.h>
34#undef BPF_PROG_TYPE
35#undef BPF_MAP_TYPE
36#undef BPF_LINK_TYPE
37};
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165struct bpf_verifier_stack_elem {
166
167
168
169
170 struct bpf_verifier_state st;
171 int insn_idx;
172 int prev_insn_idx;
173 struct bpf_verifier_stack_elem *next;
174
175 u32 log_pos;
176};
177
178#define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192
179#define BPF_COMPLEXITY_LIMIT_STATES 64
180
181#define BPF_MAP_KEY_POISON (1ULL << 63)
182#define BPF_MAP_KEY_SEEN (1ULL << 62)
183
184#define BPF_MAP_PTR_UNPRIV 1UL
185#define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \
186 POISON_POINTER_DELTA))
187#define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV))
188
189static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
190{
191 return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON;
192}
193
194static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
195{
196 return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV;
197}
198
199static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
200 const struct bpf_map *map, bool unpriv)
201{
202 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV);
203 unpriv |= bpf_map_ptr_unpriv(aux);
204 aux->map_ptr_state = (unsigned long)map |
205 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL);
206}
207
208static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux)
209{
210 return aux->map_key_state & BPF_MAP_KEY_POISON;
211}
212
213static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux)
214{
215 return !(aux->map_key_state & BPF_MAP_KEY_SEEN);
216}
217
218static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux)
219{
220 return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON);
221}
222
223static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state)
224{
225 bool poisoned = bpf_map_key_poisoned(aux);
226
227 aux->map_key_state = state | BPF_MAP_KEY_SEEN |
228 (poisoned ? BPF_MAP_KEY_POISON : 0ULL);
229}
230
231static bool bpf_pseudo_call(const struct bpf_insn *insn)
232{
233 return insn->code == (BPF_JMP | BPF_CALL) &&
234 insn->src_reg == BPF_PSEUDO_CALL;
235}
236
237static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
238{
239 return insn->code == (BPF_JMP | BPF_CALL) &&
240 insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
241}
242
243static bool bpf_pseudo_func(const struct bpf_insn *insn)
244{
245 return insn->code == (BPF_LD | BPF_IMM | BPF_DW) &&
246 insn->src_reg == BPF_PSEUDO_FUNC;
247}
248
249struct bpf_call_arg_meta {
250 struct bpf_map *map_ptr;
251 bool raw_mode;
252 bool pkt_access;
253 int regno;
254 int access_size;
255 int mem_size;
256 u64 msize_max_value;
257 int ref_obj_id;
258 int func_id;
259 struct btf *btf;
260 u32 btf_id;
261 struct btf *ret_btf;
262 u32 ret_btf_id;
263 u32 subprogno;
264};
265
266struct btf *btf_vmlinux;
267
268static DEFINE_MUTEX(bpf_verifier_lock);
269
270static const struct bpf_line_info *
271find_linfo(const struct bpf_verifier_env *env, u32 insn_off)
272{
273 const struct bpf_line_info *linfo;
274 const struct bpf_prog *prog;
275 u32 i, nr_linfo;
276
277 prog = env->prog;
278 nr_linfo = prog->aux->nr_linfo;
279
280 if (!nr_linfo || insn_off >= prog->len)
281 return NULL;
282
283 linfo = prog->aux->linfo;
284 for (i = 1; i < nr_linfo; i++)
285 if (insn_off < linfo[i].insn_off)
286 break;
287
288 return &linfo[i - 1];
289}
290
291void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,
292 va_list args)
293{
294 unsigned int n;
295
296 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);
297
298 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1,
299 "verifier log line truncated - local buffer too short\n");
300
301 n = min(log->len_total - log->len_used - 1, n);
302 log->kbuf[n] = '\0';
303
304 if (log->level == BPF_LOG_KERNEL) {
305 pr_err("BPF:%s\n", log->kbuf);
306 return;
307 }
308 if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1))
309 log->len_used += n;
310 else
311 log->ubuf = NULL;
312}
313
314static void bpf_vlog_reset(struct bpf_verifier_log *log, u32 new_pos)
315{
316 char zero = 0;
317
318 if (!bpf_verifier_log_needed(log))
319 return;
320
321 log->len_used = new_pos;
322 if (put_user(zero, log->ubuf + new_pos))
323 log->ubuf = NULL;
324}
325
326
327
328
329
330__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
331 const char *fmt, ...)
332{
333 va_list args;
334
335 if (!bpf_verifier_log_needed(&env->log))
336 return;
337
338 va_start(args, fmt);
339 bpf_verifier_vlog(&env->log, fmt, args);
340 va_end(args);
341}
342EXPORT_SYMBOL_GPL(bpf_verifier_log_write);
343
344__printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
345{
346 struct bpf_verifier_env *env = private_data;
347 va_list args;
348
349 if (!bpf_verifier_log_needed(&env->log))
350 return;
351
352 va_start(args, fmt);
353 bpf_verifier_vlog(&env->log, fmt, args);
354 va_end(args);
355}
356
357__printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
358 const char *fmt, ...)
359{
360 va_list args;
361
362 if (!bpf_verifier_log_needed(log))
363 return;
364
365 va_start(args, fmt);
366 bpf_verifier_vlog(log, fmt, args);
367 va_end(args);
368}
369
370static const char *ltrim(const char *s)
371{
372 while (isspace(*s))
373 s++;
374
375 return s;
376}
377
378__printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env,
379 u32 insn_off,
380 const char *prefix_fmt, ...)
381{
382 const struct bpf_line_info *linfo;
383
384 if (!bpf_verifier_log_needed(&env->log))
385 return;
386
387 linfo = find_linfo(env, insn_off);
388 if (!linfo || linfo == env->prev_linfo)
389 return;
390
391 if (prefix_fmt) {
392 va_list args;
393
394 va_start(args, prefix_fmt);
395 bpf_verifier_vlog(&env->log, prefix_fmt, args);
396 va_end(args);
397 }
398
399 verbose(env, "%s\n",
400 ltrim(btf_name_by_offset(env->prog->aux->btf,
401 linfo->line_off)));
402
403 env->prev_linfo = linfo;
404}
405
406static void verbose_invalid_scalar(struct bpf_verifier_env *env,
407 struct bpf_reg_state *reg,
408 struct tnum *range, const char *ctx,
409 const char *reg_name)
410{
411 char tn_buf[48];
412
413 verbose(env, "At %s the register %s ", ctx, reg_name);
414 if (!tnum_is_unknown(reg->var_off)) {
415 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
416 verbose(env, "has value %s", tn_buf);
417 } else {
418 verbose(env, "has unknown scalar value");
419 }
420 tnum_strn(tn_buf, sizeof(tn_buf), *range);
421 verbose(env, " should have been in %s\n", tn_buf);
422}
423
424static bool type_is_pkt_pointer(enum bpf_reg_type type)
425{
426 return type == PTR_TO_PACKET ||
427 type == PTR_TO_PACKET_META;
428}
429
430static bool type_is_sk_pointer(enum bpf_reg_type type)
431{
432 return type == PTR_TO_SOCKET ||
433 type == PTR_TO_SOCK_COMMON ||
434 type == PTR_TO_TCP_SOCK ||
435 type == PTR_TO_XDP_SOCK;
436}
437
438static bool reg_type_not_null(enum bpf_reg_type type)
439{
440 return type == PTR_TO_SOCKET ||
441 type == PTR_TO_TCP_SOCK ||
442 type == PTR_TO_MAP_VALUE ||
443 type == PTR_TO_MAP_KEY ||
444 type == PTR_TO_SOCK_COMMON;
445}
446
447static bool reg_type_may_be_null(enum bpf_reg_type type)
448{
449 return type == PTR_TO_MAP_VALUE_OR_NULL ||
450 type == PTR_TO_SOCKET_OR_NULL ||
451 type == PTR_TO_SOCK_COMMON_OR_NULL ||
452 type == PTR_TO_TCP_SOCK_OR_NULL ||
453 type == PTR_TO_BTF_ID_OR_NULL ||
454 type == PTR_TO_MEM_OR_NULL ||
455 type == PTR_TO_RDONLY_BUF_OR_NULL ||
456 type == PTR_TO_RDWR_BUF_OR_NULL;
457}
458
459static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
460{
461 return reg->type == PTR_TO_MAP_VALUE &&
462 map_value_has_spin_lock(reg->map_ptr);
463}
464
465static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type)
466{
467 return type == PTR_TO_SOCKET ||
468 type == PTR_TO_SOCKET_OR_NULL ||
469 type == PTR_TO_TCP_SOCK ||
470 type == PTR_TO_TCP_SOCK_OR_NULL ||
471 type == PTR_TO_MEM ||
472 type == PTR_TO_MEM_OR_NULL;
473}
474
475static bool arg_type_may_be_refcounted(enum bpf_arg_type type)
476{
477 return type == ARG_PTR_TO_SOCK_COMMON;
478}
479
480static bool arg_type_may_be_null(enum bpf_arg_type type)
481{
482 return type == ARG_PTR_TO_MAP_VALUE_OR_NULL ||
483 type == ARG_PTR_TO_MEM_OR_NULL ||
484 type == ARG_PTR_TO_CTX_OR_NULL ||
485 type == ARG_PTR_TO_SOCKET_OR_NULL ||
486 type == ARG_PTR_TO_ALLOC_MEM_OR_NULL ||
487 type == ARG_PTR_TO_STACK_OR_NULL;
488}
489
490
491
492
493
494static bool is_release_function(enum bpf_func_id func_id)
495{
496 return func_id == BPF_FUNC_sk_release ||
497 func_id == BPF_FUNC_ringbuf_submit ||
498 func_id == BPF_FUNC_ringbuf_discard;
499}
500
501static bool may_be_acquire_function(enum bpf_func_id func_id)
502{
503 return func_id == BPF_FUNC_sk_lookup_tcp ||
504 func_id == BPF_FUNC_sk_lookup_udp ||
505 func_id == BPF_FUNC_skc_lookup_tcp ||
506 func_id == BPF_FUNC_map_lookup_elem ||
507 func_id == BPF_FUNC_ringbuf_reserve;
508}
509
510static bool is_acquire_function(enum bpf_func_id func_id,
511 const struct bpf_map *map)
512{
513 enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC;
514
515 if (func_id == BPF_FUNC_sk_lookup_tcp ||
516 func_id == BPF_FUNC_sk_lookup_udp ||
517 func_id == BPF_FUNC_skc_lookup_tcp ||
518 func_id == BPF_FUNC_ringbuf_reserve)
519 return true;
520
521 if (func_id == BPF_FUNC_map_lookup_elem &&
522 (map_type == BPF_MAP_TYPE_SOCKMAP ||
523 map_type == BPF_MAP_TYPE_SOCKHASH))
524 return true;
525
526 return false;
527}
528
529static bool is_ptr_cast_function(enum bpf_func_id func_id)
530{
531 return func_id == BPF_FUNC_tcp_sock ||
532 func_id == BPF_FUNC_sk_fullsock ||
533 func_id == BPF_FUNC_skc_to_tcp_sock ||
534 func_id == BPF_FUNC_skc_to_tcp6_sock ||
535 func_id == BPF_FUNC_skc_to_udp6_sock ||
536 func_id == BPF_FUNC_skc_to_tcp_timewait_sock ||
537 func_id == BPF_FUNC_skc_to_tcp_request_sock;
538}
539
540static bool is_cmpxchg_insn(const struct bpf_insn *insn)
541{
542 return BPF_CLASS(insn->code) == BPF_STX &&
543 BPF_MODE(insn->code) == BPF_ATOMIC &&
544 insn->imm == BPF_CMPXCHG;
545}
546
547
548static const char * const reg_type_str[] = {
549 [NOT_INIT] = "?",
550 [SCALAR_VALUE] = "inv",
551 [PTR_TO_CTX] = "ctx",
552 [CONST_PTR_TO_MAP] = "map_ptr",
553 [PTR_TO_MAP_VALUE] = "map_value",
554 [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null",
555 [PTR_TO_STACK] = "fp",
556 [PTR_TO_PACKET] = "pkt",
557 [PTR_TO_PACKET_META] = "pkt_meta",
558 [PTR_TO_PACKET_END] = "pkt_end",
559 [PTR_TO_FLOW_KEYS] = "flow_keys",
560 [PTR_TO_SOCKET] = "sock",
561 [PTR_TO_SOCKET_OR_NULL] = "sock_or_null",
562 [PTR_TO_SOCK_COMMON] = "sock_common",
563 [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null",
564 [PTR_TO_TCP_SOCK] = "tcp_sock",
565 [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",
566 [PTR_TO_TP_BUFFER] = "tp_buffer",
567 [PTR_TO_XDP_SOCK] = "xdp_sock",
568 [PTR_TO_BTF_ID] = "ptr_",
569 [PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_",
570 [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",
571 [PTR_TO_MEM] = "mem",
572 [PTR_TO_MEM_OR_NULL] = "mem_or_null",
573 [PTR_TO_RDONLY_BUF] = "rdonly_buf",
574 [PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null",
575 [PTR_TO_RDWR_BUF] = "rdwr_buf",
576 [PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null",
577 [PTR_TO_FUNC] = "func",
578 [PTR_TO_MAP_KEY] = "map_key",
579};
580
581static char slot_type_char[] = {
582 [STACK_INVALID] = '?',
583 [STACK_SPILL] = 'r',
584 [STACK_MISC] = 'm',
585 [STACK_ZERO] = '0',
586};
587
588static void print_liveness(struct bpf_verifier_env *env,
589 enum bpf_reg_liveness live)
590{
591 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE))
592 verbose(env, "_");
593 if (live & REG_LIVE_READ)
594 verbose(env, "r");
595 if (live & REG_LIVE_WRITTEN)
596 verbose(env, "w");
597 if (live & REG_LIVE_DONE)
598 verbose(env, "D");
599}
600
601static struct bpf_func_state *func(struct bpf_verifier_env *env,
602 const struct bpf_reg_state *reg)
603{
604 struct bpf_verifier_state *cur = env->cur_state;
605
606 return cur->frame[reg->frameno];
607}
608
609static const char *kernel_type_name(const struct btf* btf, u32 id)
610{
611 return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
612}
613
614static void print_verifier_state(struct bpf_verifier_env *env,
615 const struct bpf_func_state *state)
616{
617 const struct bpf_reg_state *reg;
618 enum bpf_reg_type t;
619 int i;
620
621 if (state->frameno)
622 verbose(env, " frame%d:", state->frameno);
623 for (i = 0; i < MAX_BPF_REG; i++) {
624 reg = &state->regs[i];
625 t = reg->type;
626 if (t == NOT_INIT)
627 continue;
628 verbose(env, " R%d", i);
629 print_liveness(env, reg->live);
630 verbose(env, "=%s", reg_type_str[t]);
631 if (t == SCALAR_VALUE && reg->precise)
632 verbose(env, "P");
633 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) &&
634 tnum_is_const(reg->var_off)) {
635
636 verbose(env, "%lld", reg->var_off.value + reg->off);
637 } else {
638 if (t == PTR_TO_BTF_ID ||
639 t == PTR_TO_BTF_ID_OR_NULL ||
640 t == PTR_TO_PERCPU_BTF_ID)
641 verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id));
642 verbose(env, "(id=%d", reg->id);
643 if (reg_type_may_be_refcounted_or_null(t))
644 verbose(env, ",ref_obj_id=%d", reg->ref_obj_id);
645 if (t != SCALAR_VALUE)
646 verbose(env, ",off=%d", reg->off);
647 if (type_is_pkt_pointer(t))
648 verbose(env, ",r=%d", reg->range);
649 else if (t == CONST_PTR_TO_MAP ||
650 t == PTR_TO_MAP_KEY ||
651 t == PTR_TO_MAP_VALUE ||
652 t == PTR_TO_MAP_VALUE_OR_NULL)
653 verbose(env, ",ks=%d,vs=%d",
654 reg->map_ptr->key_size,
655 reg->map_ptr->value_size);
656 if (tnum_is_const(reg->var_off)) {
657
658
659
660
661 verbose(env, ",imm=%llx", reg->var_off.value);
662 } else {
663 if (reg->smin_value != reg->umin_value &&
664 reg->smin_value != S64_MIN)
665 verbose(env, ",smin_value=%lld",
666 (long long)reg->smin_value);
667 if (reg->smax_value != reg->umax_value &&
668 reg->smax_value != S64_MAX)
669 verbose(env, ",smax_value=%lld",
670 (long long)reg->smax_value);
671 if (reg->umin_value != 0)
672 verbose(env, ",umin_value=%llu",
673 (unsigned long long)reg->umin_value);
674 if (reg->umax_value != U64_MAX)
675 verbose(env, ",umax_value=%llu",
676 (unsigned long long)reg->umax_value);
677 if (!tnum_is_unknown(reg->var_off)) {
678 char tn_buf[48];
679
680 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
681 verbose(env, ",var_off=%s", tn_buf);
682 }
683 if (reg->s32_min_value != reg->smin_value &&
684 reg->s32_min_value != S32_MIN)
685 verbose(env, ",s32_min_value=%d",
686 (int)(reg->s32_min_value));
687 if (reg->s32_max_value != reg->smax_value &&
688 reg->s32_max_value != S32_MAX)
689 verbose(env, ",s32_max_value=%d",
690 (int)(reg->s32_max_value));
691 if (reg->u32_min_value != reg->umin_value &&
692 reg->u32_min_value != U32_MIN)
693 verbose(env, ",u32_min_value=%d",
694 (int)(reg->u32_min_value));
695 if (reg->u32_max_value != reg->umax_value &&
696 reg->u32_max_value != U32_MAX)
697 verbose(env, ",u32_max_value=%d",
698 (int)(reg->u32_max_value));
699 }
700 verbose(env, ")");
701 }
702 }
703 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
704 char types_buf[BPF_REG_SIZE + 1];
705 bool valid = false;
706 int j;
707
708 for (j = 0; j < BPF_REG_SIZE; j++) {
709 if (state->stack[i].slot_type[j] != STACK_INVALID)
710 valid = true;
711 types_buf[j] = slot_type_char[
712 state->stack[i].slot_type[j]];
713 }
714 types_buf[BPF_REG_SIZE] = 0;
715 if (!valid)
716 continue;
717 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
718 print_liveness(env, state->stack[i].spilled_ptr.live);
719 if (state->stack[i].slot_type[0] == STACK_SPILL) {
720 reg = &state->stack[i].spilled_ptr;
721 t = reg->type;
722 verbose(env, "=%s", reg_type_str[t]);
723 if (t == SCALAR_VALUE && reg->precise)
724 verbose(env, "P");
725 if (t == SCALAR_VALUE && tnum_is_const(reg->var_off))
726 verbose(env, "%lld", reg->var_off.value + reg->off);
727 } else {
728 verbose(env, "=%s", types_buf);
729 }
730 }
731 if (state->acquired_refs && state->refs[0].id) {
732 verbose(env, " refs=%d", state->refs[0].id);
733 for (i = 1; i < state->acquired_refs; i++)
734 if (state->refs[i].id)
735 verbose(env, ",%d", state->refs[i].id);
736 }
737 verbose(env, "\n");
738}
739
740
741
742
743
744
745
746
747static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags)
748{
749 size_t bytes;
750
751 if (ZERO_OR_NULL_PTR(src))
752 goto out;
753
754 if (unlikely(check_mul_overflow(n, size, &bytes)))
755 return NULL;
756
757 if (ksize(dst) < bytes) {
758 kfree(dst);
759 dst = kmalloc_track_caller(bytes, flags);
760 if (!dst)
761 return NULL;
762 }
763
764 memcpy(dst, src, bytes);
765out:
766 return dst ? dst : ZERO_SIZE_PTR;
767}
768
769
770
771
772
773
774static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size)
775{
776 if (!new_n || old_n == new_n)
777 goto out;
778
779 arr = krealloc_array(arr, new_n, size, GFP_KERNEL);
780 if (!arr)
781 return NULL;
782
783 if (new_n > old_n)
784 memset(arr + old_n * size, 0, (new_n - old_n) * size);
785
786out:
787 return arr ? arr : ZERO_SIZE_PTR;
788}
789
790static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
791{
792 dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs,
793 sizeof(struct bpf_reference_state), GFP_KERNEL);
794 if (!dst->refs)
795 return -ENOMEM;
796
797 dst->acquired_refs = src->acquired_refs;
798 return 0;
799}
800
801static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
802{
803 size_t n = src->allocated_stack / BPF_REG_SIZE;
804
805 dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state),
806 GFP_KERNEL);
807 if (!dst->stack)
808 return -ENOMEM;
809
810 dst->allocated_stack = src->allocated_stack;
811 return 0;
812}
813
814static int resize_reference_state(struct bpf_func_state *state, size_t n)
815{
816 state->refs = realloc_array(state->refs, state->acquired_refs, n,
817 sizeof(struct bpf_reference_state));
818 if (!state->refs)
819 return -ENOMEM;
820
821 state->acquired_refs = n;
822 return 0;
823}
824
825static int grow_stack_state(struct bpf_func_state *state, int size)
826{
827 size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE;
828
829 if (old_n >= n)
830 return 0;
831
832 state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state));
833 if (!state->stack)
834 return -ENOMEM;
835
836 state->allocated_stack = size;
837 return 0;
838}
839
840
841
842
843
844
845static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
846{
847 struct bpf_func_state *state = cur_func(env);
848 int new_ofs = state->acquired_refs;
849 int id, err;
850
851 err = resize_reference_state(state, state->acquired_refs + 1);
852 if (err)
853 return err;
854 id = ++env->id_gen;
855 state->refs[new_ofs].id = id;
856 state->refs[new_ofs].insn_idx = insn_idx;
857
858 return id;
859}
860
861
862static int release_reference_state(struct bpf_func_state *state, int ptr_id)
863{
864 int i, last_idx;
865
866 last_idx = state->acquired_refs - 1;
867 for (i = 0; i < state->acquired_refs; i++) {
868 if (state->refs[i].id == ptr_id) {
869 if (last_idx && i != last_idx)
870 memcpy(&state->refs[i], &state->refs[last_idx],
871 sizeof(*state->refs));
872 memset(&state->refs[last_idx], 0, sizeof(*state->refs));
873 state->acquired_refs--;
874 return 0;
875 }
876 }
877 return -EINVAL;
878}
879
880static void free_func_state(struct bpf_func_state *state)
881{
882 if (!state)
883 return;
884 kfree(state->refs);
885 kfree(state->stack);
886 kfree(state);
887}
888
889static void clear_jmp_history(struct bpf_verifier_state *state)
890{
891 kfree(state->jmp_history);
892 state->jmp_history = NULL;
893 state->jmp_history_cnt = 0;
894}
895
896static void free_verifier_state(struct bpf_verifier_state *state,
897 bool free_self)
898{
899 int i;
900
901 for (i = 0; i <= state->curframe; i++) {
902 free_func_state(state->frame[i]);
903 state->frame[i] = NULL;
904 }
905 clear_jmp_history(state);
906 if (free_self)
907 kfree(state);
908}
909
910
911
912
913static int copy_func_state(struct bpf_func_state *dst,
914 const struct bpf_func_state *src)
915{
916 int err;
917
918 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs));
919 err = copy_reference_state(dst, src);
920 if (err)
921 return err;
922 return copy_stack_state(dst, src);
923}
924
925static int copy_verifier_state(struct bpf_verifier_state *dst_state,
926 const struct bpf_verifier_state *src)
927{
928 struct bpf_func_state *dst;
929 int i, err;
930
931 dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history,
932 src->jmp_history_cnt, sizeof(struct bpf_idx_pair),
933 GFP_USER);
934 if (!dst_state->jmp_history)
935 return -ENOMEM;
936 dst_state->jmp_history_cnt = src->jmp_history_cnt;
937
938
939 for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
940 free_func_state(dst_state->frame[i]);
941 dst_state->frame[i] = NULL;
942 }
943 dst_state->speculative = src->speculative;
944 dst_state->curframe = src->curframe;
945 dst_state->active_spin_lock = src->active_spin_lock;
946 dst_state->branches = src->branches;
947 dst_state->parent = src->parent;
948 dst_state->first_insn_idx = src->first_insn_idx;
949 dst_state->last_insn_idx = src->last_insn_idx;
950 for (i = 0; i <= src->curframe; i++) {
951 dst = dst_state->frame[i];
952 if (!dst) {
953 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
954 if (!dst)
955 return -ENOMEM;
956 dst_state->frame[i] = dst;
957 }
958 err = copy_func_state(dst, src->frame[i]);
959 if (err)
960 return err;
961 }
962 return 0;
963}
964
965static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
966{
967 while (st) {
968 u32 br = --st->branches;
969
970
971
972
973 WARN_ONCE((int)br < 0,
974 "BUG update_branch_counts:branches_to_explore=%d\n",
975 br);
976 if (br)
977 break;
978 st = st->parent;
979 }
980}
981
982static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
983 int *insn_idx, bool pop_log)
984{
985 struct bpf_verifier_state *cur = env->cur_state;
986 struct bpf_verifier_stack_elem *elem, *head = env->head;
987 int err;
988
989 if (env->head == NULL)
990 return -ENOENT;
991
992 if (cur) {
993 err = copy_verifier_state(cur, &head->st);
994 if (err)
995 return err;
996 }
997 if (pop_log)
998 bpf_vlog_reset(&env->log, head->log_pos);
999 if (insn_idx)
1000 *insn_idx = head->insn_idx;
1001 if (prev_insn_idx)
1002 *prev_insn_idx = head->prev_insn_idx;
1003 elem = head->next;
1004 free_verifier_state(&head->st, false);
1005 kfree(head);
1006 env->head = elem;
1007 env->stack_size--;
1008 return 0;
1009}
1010
1011static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
1012 int insn_idx, int prev_insn_idx,
1013 bool speculative)
1014{
1015 struct bpf_verifier_state *cur = env->cur_state;
1016 struct bpf_verifier_stack_elem *elem;
1017 int err;
1018
1019 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
1020 if (!elem)
1021 goto err;
1022
1023 elem->insn_idx = insn_idx;
1024 elem->prev_insn_idx = prev_insn_idx;
1025 elem->next = env->head;
1026 elem->log_pos = env->log.len_used;
1027 env->head = elem;
1028 env->stack_size++;
1029 err = copy_verifier_state(&elem->st, cur);
1030 if (err)
1031 goto err;
1032 elem->st.speculative |= speculative;
1033 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
1034 verbose(env, "The sequence of %d jumps is too complex.\n",
1035 env->stack_size);
1036 goto err;
1037 }
1038 if (elem->st.parent) {
1039 ++elem->st.parent->branches;
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049 }
1050 return &elem->st;
1051err:
1052 free_verifier_state(env->cur_state, true);
1053 env->cur_state = NULL;
1054
1055 while (!pop_stack(env, NULL, NULL, false));
1056 return NULL;
1057}
1058
1059#define CALLER_SAVED_REGS 6
1060static const int caller_saved[CALLER_SAVED_REGS] = {
1061 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
1062};
1063
1064static void __mark_reg_not_init(const struct bpf_verifier_env *env,
1065 struct bpf_reg_state *reg);
1066
1067
1068static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)
1069{
1070 reg->var_off = tnum_const(imm);
1071 reg->smin_value = (s64)imm;
1072 reg->smax_value = (s64)imm;
1073 reg->umin_value = imm;
1074 reg->umax_value = imm;
1075
1076 reg->s32_min_value = (s32)imm;
1077 reg->s32_max_value = (s32)imm;
1078 reg->u32_min_value = (u32)imm;
1079 reg->u32_max_value = (u32)imm;
1080}
1081
1082
1083
1084
1085static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
1086{
1087
1088 memset(((u8 *)reg) + sizeof(reg->type), 0,
1089 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
1090 ___mark_reg_known(reg, imm);
1091}
1092
1093static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm)
1094{
1095 reg->var_off = tnum_const_subreg(reg->var_off, imm);
1096 reg->s32_min_value = (s32)imm;
1097 reg->s32_max_value = (s32)imm;
1098 reg->u32_min_value = (u32)imm;
1099 reg->u32_max_value = (u32)imm;
1100}
1101
1102
1103
1104
1105static void __mark_reg_known_zero(struct bpf_reg_state *reg)
1106{
1107 __mark_reg_known(reg, 0);
1108}
1109
1110static void __mark_reg_const_zero(struct bpf_reg_state *reg)
1111{
1112 __mark_reg_known(reg, 0);
1113 reg->type = SCALAR_VALUE;
1114}
1115
1116static void mark_reg_known_zero(struct bpf_verifier_env *env,
1117 struct bpf_reg_state *regs, u32 regno)
1118{
1119 if (WARN_ON(regno >= MAX_BPF_REG)) {
1120 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
1121
1122 for (regno = 0; regno < MAX_BPF_REG; regno++)
1123 __mark_reg_not_init(env, regs + regno);
1124 return;
1125 }
1126 __mark_reg_known_zero(regs + regno);
1127}
1128
1129static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
1130{
1131 switch (reg->type) {
1132 case PTR_TO_MAP_VALUE_OR_NULL: {
1133 const struct bpf_map *map = reg->map_ptr;
1134
1135 if (map->inner_map_meta) {
1136 reg->type = CONST_PTR_TO_MAP;
1137 reg->map_ptr = map->inner_map_meta;
1138 } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
1139 reg->type = PTR_TO_XDP_SOCK;
1140 } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
1141 map->map_type == BPF_MAP_TYPE_SOCKHASH) {
1142 reg->type = PTR_TO_SOCKET;
1143 } else {
1144 reg->type = PTR_TO_MAP_VALUE;
1145 }
1146 break;
1147 }
1148 case PTR_TO_SOCKET_OR_NULL:
1149 reg->type = PTR_TO_SOCKET;
1150 break;
1151 case PTR_TO_SOCK_COMMON_OR_NULL:
1152 reg->type = PTR_TO_SOCK_COMMON;
1153 break;
1154 case PTR_TO_TCP_SOCK_OR_NULL:
1155 reg->type = PTR_TO_TCP_SOCK;
1156 break;
1157 case PTR_TO_BTF_ID_OR_NULL:
1158 reg->type = PTR_TO_BTF_ID;
1159 break;
1160 case PTR_TO_MEM_OR_NULL:
1161 reg->type = PTR_TO_MEM;
1162 break;
1163 case PTR_TO_RDONLY_BUF_OR_NULL:
1164 reg->type = PTR_TO_RDONLY_BUF;
1165 break;
1166 case PTR_TO_RDWR_BUF_OR_NULL:
1167 reg->type = PTR_TO_RDWR_BUF;
1168 break;
1169 default:
1170 WARN_ONCE(1, "unknown nullable register type");
1171 }
1172}
1173
1174static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
1175{
1176 return type_is_pkt_pointer(reg->type);
1177}
1178
1179static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
1180{
1181 return reg_is_pkt_pointer(reg) ||
1182 reg->type == PTR_TO_PACKET_END;
1183}
1184
1185
1186static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
1187 enum bpf_reg_type which)
1188{
1189
1190
1191
1192
1193 return reg->type == which &&
1194 reg->id == 0 &&
1195 reg->off == 0 &&
1196 tnum_equals_const(reg->var_off, 0);
1197}
1198
1199
1200static void __mark_reg_unbounded(struct bpf_reg_state *reg)
1201{
1202 reg->smin_value = S64_MIN;
1203 reg->smax_value = S64_MAX;
1204 reg->umin_value = 0;
1205 reg->umax_value = U64_MAX;
1206
1207 reg->s32_min_value = S32_MIN;
1208 reg->s32_max_value = S32_MAX;
1209 reg->u32_min_value = 0;
1210 reg->u32_max_value = U32_MAX;
1211}
1212
1213static void __mark_reg64_unbounded(struct bpf_reg_state *reg)
1214{
1215 reg->smin_value = S64_MIN;
1216 reg->smax_value = S64_MAX;
1217 reg->umin_value = 0;
1218 reg->umax_value = U64_MAX;
1219}
1220
1221static void __mark_reg32_unbounded(struct bpf_reg_state *reg)
1222{
1223 reg->s32_min_value = S32_MIN;
1224 reg->s32_max_value = S32_MAX;
1225 reg->u32_min_value = 0;
1226 reg->u32_max_value = U32_MAX;
1227}
1228
1229static void __update_reg32_bounds(struct bpf_reg_state *reg)
1230{
1231 struct tnum var32_off = tnum_subreg(reg->var_off);
1232
1233
1234 reg->s32_min_value = max_t(s32, reg->s32_min_value,
1235 var32_off.value | (var32_off.mask & S32_MIN));
1236
1237 reg->s32_max_value = min_t(s32, reg->s32_max_value,
1238 var32_off.value | (var32_off.mask & S32_MAX));
1239 reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value);
1240 reg->u32_max_value = min(reg->u32_max_value,
1241 (u32)(var32_off.value | var32_off.mask));
1242}
1243
1244static void __update_reg64_bounds(struct bpf_reg_state *reg)
1245{
1246
1247 reg->smin_value = max_t(s64, reg->smin_value,
1248 reg->var_off.value | (reg->var_off.mask & S64_MIN));
1249
1250 reg->smax_value = min_t(s64, reg->smax_value,
1251 reg->var_off.value | (reg->var_off.mask & S64_MAX));
1252 reg->umin_value = max(reg->umin_value, reg->var_off.value);
1253 reg->umax_value = min(reg->umax_value,
1254 reg->var_off.value | reg->var_off.mask);
1255}
1256
1257static void __update_reg_bounds(struct bpf_reg_state *reg)
1258{
1259 __update_reg32_bounds(reg);
1260 __update_reg64_bounds(reg);
1261}
1262
1263
1264static void __reg32_deduce_bounds(struct bpf_reg_state *reg)
1265{
1266
1267
1268
1269
1270
1271 if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) {
1272 reg->s32_min_value = reg->u32_min_value =
1273 max_t(u32, reg->s32_min_value, reg->u32_min_value);
1274 reg->s32_max_value = reg->u32_max_value =
1275 min_t(u32, reg->s32_max_value, reg->u32_max_value);
1276 return;
1277 }
1278
1279
1280
1281 if ((s32)reg->u32_max_value >= 0) {
1282
1283
1284
1285 reg->s32_min_value = reg->u32_min_value;
1286 reg->s32_max_value = reg->u32_max_value =
1287 min_t(u32, reg->s32_max_value, reg->u32_max_value);
1288 } else if ((s32)reg->u32_min_value < 0) {
1289
1290
1291
1292 reg->s32_min_value = reg->u32_min_value =
1293 max_t(u32, reg->s32_min_value, reg->u32_min_value);
1294 reg->s32_max_value = reg->u32_max_value;
1295 }
1296}
1297
1298static void __reg64_deduce_bounds(struct bpf_reg_state *reg)
1299{
1300
1301
1302
1303
1304
1305 if (reg->smin_value >= 0 || reg->smax_value < 0) {
1306 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
1307 reg->umin_value);
1308 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
1309 reg->umax_value);
1310 return;
1311 }
1312
1313
1314
1315 if ((s64)reg->umax_value >= 0) {
1316
1317
1318
1319 reg->smin_value = reg->umin_value;
1320 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
1321 reg->umax_value);
1322 } else if ((s64)reg->umin_value < 0) {
1323
1324
1325
1326 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
1327 reg->umin_value);
1328 reg->smax_value = reg->umax_value;
1329 }
1330}
1331
1332static void __reg_deduce_bounds(struct bpf_reg_state *reg)
1333{
1334 __reg32_deduce_bounds(reg);
1335 __reg64_deduce_bounds(reg);
1336}
1337
1338
1339static void __reg_bound_offset(struct bpf_reg_state *reg)
1340{
1341 struct tnum var64_off = tnum_intersect(reg->var_off,
1342 tnum_range(reg->umin_value,
1343 reg->umax_value));
1344 struct tnum var32_off = tnum_intersect(tnum_subreg(reg->var_off),
1345 tnum_range(reg->u32_min_value,
1346 reg->u32_max_value));
1347
1348 reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off);
1349}
1350
1351static void __reg_assign_32_into_64(struct bpf_reg_state *reg)
1352{
1353 reg->umin_value = reg->u32_min_value;
1354 reg->umax_value = reg->u32_max_value;
1355
1356
1357
1358
1359 if (reg->s32_min_value >= 0 && reg->s32_max_value >= 0)
1360 reg->smax_value = reg->s32_max_value;
1361 else
1362 reg->smax_value = U32_MAX;
1363 if (reg->s32_min_value >= 0)
1364 reg->smin_value = reg->s32_min_value;
1365 else
1366 reg->smin_value = 0;
1367}
1368
1369static void __reg_combine_32_into_64(struct bpf_reg_state *reg)
1370{
1371
1372
1373
1374
1375 if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) {
1376 __reg_assign_32_into_64(reg);
1377 } else {
1378
1379
1380
1381
1382
1383
1384
1385 __mark_reg64_unbounded(reg);
1386 __update_reg_bounds(reg);
1387 }
1388
1389
1390
1391
1392
1393 __reg_deduce_bounds(reg);
1394 __reg_bound_offset(reg);
1395 __update_reg_bounds(reg);
1396}
1397
1398static bool __reg64_bound_s32(s64 a)
1399{
1400 return a > S32_MIN && a < S32_MAX;
1401}
1402
1403static bool __reg64_bound_u32(u64 a)
1404{
1405 return a > U32_MIN && a < U32_MAX;
1406}
1407
1408static void __reg_combine_64_into_32(struct bpf_reg_state *reg)
1409{
1410 __mark_reg32_unbounded(reg);
1411
1412 if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) {
1413 reg->s32_min_value = (s32)reg->smin_value;
1414 reg->s32_max_value = (s32)reg->smax_value;
1415 }
1416 if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) {
1417 reg->u32_min_value = (u32)reg->umin_value;
1418 reg->u32_max_value = (u32)reg->umax_value;
1419 }
1420
1421
1422
1423
1424
1425 __reg_deduce_bounds(reg);
1426 __reg_bound_offset(reg);
1427 __update_reg_bounds(reg);
1428}
1429
1430
1431static void __mark_reg_unknown(const struct bpf_verifier_env *env,
1432 struct bpf_reg_state *reg)
1433{
1434
1435
1436
1437
1438 memset(reg, 0, offsetof(struct bpf_reg_state, var_off));
1439 reg->type = SCALAR_VALUE;
1440 reg->var_off = tnum_unknown;
1441 reg->frameno = 0;
1442 reg->precise = env->subprog_cnt > 1 || !env->bpf_capable;
1443 __mark_reg_unbounded(reg);
1444}
1445
1446static void mark_reg_unknown(struct bpf_verifier_env *env,
1447 struct bpf_reg_state *regs, u32 regno)
1448{
1449 if (WARN_ON(regno >= MAX_BPF_REG)) {
1450 verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
1451
1452 for (regno = 0; regno < BPF_REG_FP; regno++)
1453 __mark_reg_not_init(env, regs + regno);
1454 return;
1455 }
1456 __mark_reg_unknown(env, regs + regno);
1457}
1458
1459static void __mark_reg_not_init(const struct bpf_verifier_env *env,
1460 struct bpf_reg_state *reg)
1461{
1462 __mark_reg_unknown(env, reg);
1463 reg->type = NOT_INIT;
1464}
1465
1466static void mark_reg_not_init(struct bpf_verifier_env *env,
1467 struct bpf_reg_state *regs, u32 regno)
1468{
1469 if (WARN_ON(regno >= MAX_BPF_REG)) {
1470 verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
1471
1472 for (regno = 0; regno < BPF_REG_FP; regno++)
1473 __mark_reg_not_init(env, regs + regno);
1474 return;
1475 }
1476 __mark_reg_not_init(env, regs + regno);
1477}
1478
1479static void mark_btf_ld_reg(struct bpf_verifier_env *env,
1480 struct bpf_reg_state *regs, u32 regno,
1481 enum bpf_reg_type reg_type,
1482 struct btf *btf, u32 btf_id)
1483{
1484 if (reg_type == SCALAR_VALUE) {
1485 mark_reg_unknown(env, regs, regno);
1486 return;
1487 }
1488 mark_reg_known_zero(env, regs, regno);
1489 regs[regno].type = PTR_TO_BTF_ID;
1490 regs[regno].btf = btf;
1491 regs[regno].btf_id = btf_id;
1492}
1493
1494#define DEF_NOT_SUBREG (0)
1495static void init_reg_state(struct bpf_verifier_env *env,
1496 struct bpf_func_state *state)
1497{
1498 struct bpf_reg_state *regs = state->regs;
1499 int i;
1500
1501 for (i = 0; i < MAX_BPF_REG; i++) {
1502 mark_reg_not_init(env, regs, i);
1503 regs[i].live = REG_LIVE_NONE;
1504 regs[i].parent = NULL;
1505 regs[i].subreg_def = DEF_NOT_SUBREG;
1506 }
1507
1508
1509 regs[BPF_REG_FP].type = PTR_TO_STACK;
1510 mark_reg_known_zero(env, regs, BPF_REG_FP);
1511 regs[BPF_REG_FP].frameno = state->frameno;
1512}
1513
1514#define BPF_MAIN_FUNC (-1)
1515static void init_func_state(struct bpf_verifier_env *env,
1516 struct bpf_func_state *state,
1517 int callsite, int frameno, int subprogno)
1518{
1519 state->callsite = callsite;
1520 state->frameno = frameno;
1521 state->subprogno = subprogno;
1522 init_reg_state(env, state);
1523}
1524
1525enum reg_arg_type {
1526 SRC_OP,
1527 DST_OP,
1528 DST_OP_NO_MARK
1529};
1530
1531static int cmp_subprogs(const void *a, const void *b)
1532{
1533 return ((struct bpf_subprog_info *)a)->start -
1534 ((struct bpf_subprog_info *)b)->start;
1535}
1536
1537static int find_subprog(struct bpf_verifier_env *env, int off)
1538{
1539 struct bpf_subprog_info *p;
1540
1541 p = bsearch(&off, env->subprog_info, env->subprog_cnt,
1542 sizeof(env->subprog_info[0]), cmp_subprogs);
1543 if (!p)
1544 return -ENOENT;
1545 return p - env->subprog_info;
1546
1547}
1548
1549static int add_subprog(struct bpf_verifier_env *env, int off)
1550{
1551 int insn_cnt = env->prog->len;
1552 int ret;
1553
1554 if (off >= insn_cnt || off < 0) {
1555 verbose(env, "call to invalid destination\n");
1556 return -EINVAL;
1557 }
1558 ret = find_subprog(env, off);
1559 if (ret >= 0)
1560 return ret;
1561 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
1562 verbose(env, "too many subprograms\n");
1563 return -E2BIG;
1564 }
1565
1566 env->subprog_info[env->subprog_cnt++].start = off;
1567 sort(env->subprog_info, env->subprog_cnt,
1568 sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
1569 return env->subprog_cnt - 1;
1570}
1571
1572struct bpf_kfunc_desc {
1573 struct btf_func_model func_model;
1574 u32 func_id;
1575 s32 imm;
1576};
1577
1578#define MAX_KFUNC_DESCS 256
1579struct bpf_kfunc_desc_tab {
1580 struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
1581 u32 nr_descs;
1582};
1583
1584static int kfunc_desc_cmp_by_id(const void *a, const void *b)
1585{
1586 const struct bpf_kfunc_desc *d0 = a;
1587 const struct bpf_kfunc_desc *d1 = b;
1588
1589
1590 return d0->func_id - d1->func_id;
1591}
1592
1593static const struct bpf_kfunc_desc *
1594find_kfunc_desc(const struct bpf_prog *prog, u32 func_id)
1595{
1596 struct bpf_kfunc_desc desc = {
1597 .func_id = func_id,
1598 };
1599 struct bpf_kfunc_desc_tab *tab;
1600
1601 tab = prog->aux->kfunc_tab;
1602 return bsearch(&desc, tab->descs, tab->nr_descs,
1603 sizeof(tab->descs[0]), kfunc_desc_cmp_by_id);
1604}
1605
1606static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id)
1607{
1608 const struct btf_type *func, *func_proto;
1609 struct bpf_kfunc_desc_tab *tab;
1610 struct bpf_prog_aux *prog_aux;
1611 struct bpf_kfunc_desc *desc;
1612 const char *func_name;
1613 unsigned long addr;
1614 int err;
1615
1616 prog_aux = env->prog->aux;
1617 tab = prog_aux->kfunc_tab;
1618 if (!tab) {
1619 if (!btf_vmlinux) {
1620 verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
1621 return -ENOTSUPP;
1622 }
1623
1624 if (!env->prog->jit_requested) {
1625 verbose(env, "JIT is required for calling kernel function\n");
1626 return -ENOTSUPP;
1627 }
1628
1629 if (!bpf_jit_supports_kfunc_call()) {
1630 verbose(env, "JIT does not support calling kernel function\n");
1631 return -ENOTSUPP;
1632 }
1633
1634 if (!env->prog->gpl_compatible) {
1635 verbose(env, "cannot call kernel function from non-GPL compatible program\n");
1636 return -EINVAL;
1637 }
1638
1639 tab = kzalloc(sizeof(*tab), GFP_KERNEL);
1640 if (!tab)
1641 return -ENOMEM;
1642 prog_aux->kfunc_tab = tab;
1643 }
1644
1645 if (find_kfunc_desc(env->prog, func_id))
1646 return 0;
1647
1648 if (tab->nr_descs == MAX_KFUNC_DESCS) {
1649 verbose(env, "too many different kernel function calls\n");
1650 return -E2BIG;
1651 }
1652
1653 func = btf_type_by_id(btf_vmlinux, func_id);
1654 if (!func || !btf_type_is_func(func)) {
1655 verbose(env, "kernel btf_id %u is not a function\n",
1656 func_id);
1657 return -EINVAL;
1658 }
1659 func_proto = btf_type_by_id(btf_vmlinux, func->type);
1660 if (!func_proto || !btf_type_is_func_proto(func_proto)) {
1661 verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
1662 func_id);
1663 return -EINVAL;
1664 }
1665
1666 func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
1667 addr = kallsyms_lookup_name(func_name);
1668 if (!addr) {
1669 verbose(env, "cannot find address for kernel function %s\n",
1670 func_name);
1671 return -EINVAL;
1672 }
1673
1674 desc = &tab->descs[tab->nr_descs++];
1675 desc->func_id = func_id;
1676 desc->imm = BPF_CAST_CALL(addr) - __bpf_call_base;
1677 err = btf_distill_func_proto(&env->log, btf_vmlinux,
1678 func_proto, func_name,
1679 &desc->func_model);
1680 if (!err)
1681 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
1682 kfunc_desc_cmp_by_id, NULL);
1683 return err;
1684}
1685
1686static int kfunc_desc_cmp_by_imm(const void *a, const void *b)
1687{
1688 const struct bpf_kfunc_desc *d0 = a;
1689 const struct bpf_kfunc_desc *d1 = b;
1690
1691 if (d0->imm > d1->imm)
1692 return 1;
1693 else if (d0->imm < d1->imm)
1694 return -1;
1695 return 0;
1696}
1697
1698static void sort_kfunc_descs_by_imm(struct bpf_prog *prog)
1699{
1700 struct bpf_kfunc_desc_tab *tab;
1701
1702 tab = prog->aux->kfunc_tab;
1703 if (!tab)
1704 return;
1705
1706 sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
1707 kfunc_desc_cmp_by_imm, NULL);
1708}
1709
1710bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
1711{
1712 return !!prog->aux->kfunc_tab;
1713}
1714
1715const struct btf_func_model *
1716bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
1717 const struct bpf_insn *insn)
1718{
1719 const struct bpf_kfunc_desc desc = {
1720 .imm = insn->imm,
1721 };
1722 const struct bpf_kfunc_desc *res;
1723 struct bpf_kfunc_desc_tab *tab;
1724
1725 tab = prog->aux->kfunc_tab;
1726 res = bsearch(&desc, tab->descs, tab->nr_descs,
1727 sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm);
1728
1729 return res ? &res->func_model : NULL;
1730}
1731
1732static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
1733{
1734 struct bpf_subprog_info *subprog = env->subprog_info;
1735 struct bpf_insn *insn = env->prog->insnsi;
1736 int i, ret, insn_cnt = env->prog->len;
1737
1738
1739 ret = add_subprog(env, 0);
1740 if (ret)
1741 return ret;
1742
1743 for (i = 0; i < insn_cnt; i++, insn++) {
1744 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
1745 !bpf_pseudo_kfunc_call(insn))
1746 continue;
1747
1748 if (!env->bpf_capable) {
1749 verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1750 return -EPERM;
1751 }
1752
1753 if (bpf_pseudo_func(insn)) {
1754 ret = add_subprog(env, i + insn->imm + 1);
1755 if (ret >= 0)
1756
1757 insn[1].imm = ret;
1758 } else if (bpf_pseudo_call(insn)) {
1759 ret = add_subprog(env, i + insn->imm + 1);
1760 } else {
1761 ret = add_kfunc_call(env, insn->imm);
1762 }
1763
1764 if (ret < 0)
1765 return ret;
1766 }
1767
1768
1769
1770
1771 subprog[env->subprog_cnt].start = insn_cnt;
1772
1773 if (env->log.level & BPF_LOG_LEVEL2)
1774 for (i = 0; i < env->subprog_cnt; i++)
1775 verbose(env, "func#%d @%d\n", i, subprog[i].start);
1776
1777 return 0;
1778}
1779
1780static int check_subprogs(struct bpf_verifier_env *env)
1781{
1782 int i, subprog_start, subprog_end, off, cur_subprog = 0;
1783 struct bpf_subprog_info *subprog = env->subprog_info;
1784 struct bpf_insn *insn = env->prog->insnsi;
1785 int insn_cnt = env->prog->len;
1786
1787
1788 subprog_start = subprog[cur_subprog].start;
1789 subprog_end = subprog[cur_subprog + 1].start;
1790 for (i = 0; i < insn_cnt; i++) {
1791 u8 code = insn[i].code;
1792
1793 if (code == (BPF_JMP | BPF_CALL) &&
1794 insn[i].imm == BPF_FUNC_tail_call &&
1795 insn[i].src_reg != BPF_PSEUDO_CALL)
1796 subprog[cur_subprog].has_tail_call = true;
1797 if (BPF_CLASS(code) == BPF_LD &&
1798 (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND))
1799 subprog[cur_subprog].has_ld_abs = true;
1800 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
1801 goto next;
1802 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
1803 goto next;
1804 off = i + insn[i].off + 1;
1805 if (off < subprog_start || off >= subprog_end) {
1806 verbose(env, "jump out of range from insn %d to %d\n", i, off);
1807 return -EINVAL;
1808 }
1809next:
1810 if (i == subprog_end - 1) {
1811
1812
1813
1814
1815 if (code != (BPF_JMP | BPF_EXIT) &&
1816 code != (BPF_JMP | BPF_JA)) {
1817 verbose(env, "last insn is not an exit or jmp\n");
1818 return -EINVAL;
1819 }
1820 subprog_start = subprog_end;
1821 cur_subprog++;
1822 if (cur_subprog < env->subprog_cnt)
1823 subprog_end = subprog[cur_subprog + 1].start;
1824 }
1825 }
1826 return 0;
1827}
1828
1829
1830
1831
1832static int mark_reg_read(struct bpf_verifier_env *env,
1833 const struct bpf_reg_state *state,
1834 struct bpf_reg_state *parent, u8 flag)
1835{
1836 bool writes = parent == state->parent;
1837 int cnt = 0;
1838
1839 while (parent) {
1840
1841 if (writes && state->live & REG_LIVE_WRITTEN)
1842 break;
1843 if (parent->live & REG_LIVE_DONE) {
1844 verbose(env, "verifier BUG type %s var_off %lld off %d\n",
1845 reg_type_str[parent->type],
1846 parent->var_off.value, parent->off);
1847 return -EFAULT;
1848 }
1849
1850
1851
1852 if ((parent->live & REG_LIVE_READ) == flag ||
1853 parent->live & REG_LIVE_READ64)
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863 break;
1864
1865 parent->live |= flag;
1866
1867 if (flag == REG_LIVE_READ64)
1868 parent->live &= ~REG_LIVE_READ32;
1869 state = parent;
1870 parent = state->parent;
1871 writes = true;
1872 cnt++;
1873 }
1874
1875 if (env->longest_mark_read_walk < cnt)
1876 env->longest_mark_read_walk = cnt;
1877 return 0;
1878}
1879
1880
1881
1882
1883
1884static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,
1885 u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t)
1886{
1887 u8 code, class, op;
1888
1889 code = insn->code;
1890 class = BPF_CLASS(code);
1891 op = BPF_OP(code);
1892 if (class == BPF_JMP) {
1893
1894
1895
1896 if (op == BPF_EXIT)
1897 return true;
1898 if (op == BPF_CALL) {
1899
1900
1901
1902
1903
1904 if (insn->src_reg == BPF_PSEUDO_CALL)
1905 return false;
1906
1907
1908
1909 if (t == SRC_OP)
1910 return true;
1911
1912 return false;
1913 }
1914 }
1915
1916 if (class == BPF_ALU64 || class == BPF_JMP ||
1917
1918 (class == BPF_ALU && op == BPF_END && insn->imm == 64))
1919 return true;
1920
1921 if (class == BPF_ALU || class == BPF_JMP32)
1922 return false;
1923
1924 if (class == BPF_LDX) {
1925 if (t != SRC_OP)
1926 return BPF_SIZE(code) == BPF_DW;
1927
1928 return true;
1929 }
1930
1931 if (class == BPF_STX) {
1932
1933
1934
1935
1936 if (t == SRC_OP && reg->type != SCALAR_VALUE)
1937 return true;
1938 return BPF_SIZE(code) == BPF_DW;
1939 }
1940
1941 if (class == BPF_LD) {
1942 u8 mode = BPF_MODE(code);
1943
1944
1945 if (mode == BPF_IMM)
1946 return true;
1947
1948
1949 if (t != SRC_OP)
1950 return false;
1951
1952
1953 if (regno == BPF_REG_6)
1954 return true;
1955
1956
1957 return true;
1958 }
1959
1960 if (class == BPF_ST)
1961
1962 return true;
1963
1964
1965 return true;
1966}
1967
1968
1969static int insn_def_regno(const struct bpf_insn *insn)
1970{
1971 switch (BPF_CLASS(insn->code)) {
1972 case BPF_JMP:
1973 case BPF_JMP32:
1974 case BPF_ST:
1975 return -1;
1976 case BPF_STX:
1977 if (BPF_MODE(insn->code) == BPF_ATOMIC &&
1978 (insn->imm & BPF_FETCH)) {
1979 if (insn->imm == BPF_CMPXCHG)
1980 return BPF_REG_0;
1981 else
1982 return insn->src_reg;
1983 } else {
1984 return -1;
1985 }
1986 default:
1987 return insn->dst_reg;
1988 }
1989}
1990
1991
1992static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)
1993{
1994 int dst_reg = insn_def_regno(insn);
1995
1996 if (dst_reg == -1)
1997 return false;
1998
1999 return !is_reg64(env, insn, dst_reg, NULL, DST_OP);
2000}
2001
2002static void mark_insn_zext(struct bpf_verifier_env *env,
2003 struct bpf_reg_state *reg)
2004{
2005 s32 def_idx = reg->subreg_def;
2006
2007 if (def_idx == DEF_NOT_SUBREG)
2008 return;
2009
2010 env->insn_aux_data[def_idx - 1].zext_dst = true;
2011
2012 reg->subreg_def = DEF_NOT_SUBREG;
2013}
2014
2015static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
2016 enum reg_arg_type t)
2017{
2018 struct bpf_verifier_state *vstate = env->cur_state;
2019 struct bpf_func_state *state = vstate->frame[vstate->curframe];
2020 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;
2021 struct bpf_reg_state *reg, *regs = state->regs;
2022 bool rw64;
2023
2024 if (regno >= MAX_BPF_REG) {
2025 verbose(env, "R%d is invalid\n", regno);
2026 return -EINVAL;
2027 }
2028
2029 reg = ®s[regno];
2030 rw64 = is_reg64(env, insn, regno, reg, t);
2031 if (t == SRC_OP) {
2032
2033 if (reg->type == NOT_INIT) {
2034 verbose(env, "R%d !read_ok\n", regno);
2035 return -EACCES;
2036 }
2037
2038 if (regno == BPF_REG_FP)
2039 return 0;
2040
2041 if (rw64)
2042 mark_insn_zext(env, reg);
2043
2044 return mark_reg_read(env, reg, reg->parent,
2045 rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32);
2046 } else {
2047
2048 if (regno == BPF_REG_FP) {
2049 verbose(env, "frame pointer is read only\n");
2050 return -EACCES;
2051 }
2052 reg->live |= REG_LIVE_WRITTEN;
2053 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;
2054 if (t == DST_OP)
2055 mark_reg_unknown(env, regs, regno);
2056 }
2057 return 0;
2058}
2059
2060
2061static int push_jmp_history(struct bpf_verifier_env *env,
2062 struct bpf_verifier_state *cur)
2063{
2064 u32 cnt = cur->jmp_history_cnt;
2065 struct bpf_idx_pair *p;
2066
2067 cnt++;
2068 p = krealloc(cur->jmp_history, cnt * sizeof(*p), GFP_USER);
2069 if (!p)
2070 return -ENOMEM;
2071 p[cnt - 1].idx = env->insn_idx;
2072 p[cnt - 1].prev_idx = env->prev_insn_idx;
2073 cur->jmp_history = p;
2074 cur->jmp_history_cnt = cnt;
2075 return 0;
2076}
2077
2078
2079
2080
2081static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
2082 u32 *history)
2083{
2084 u32 cnt = *history;
2085
2086 if (cnt && st->jmp_history[cnt - 1].idx == i) {
2087 i = st->jmp_history[cnt - 1].prev_idx;
2088 (*history)--;
2089 } else {
2090 i--;
2091 }
2092 return i;
2093}
2094
2095static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
2096{
2097 const struct btf_type *func;
2098
2099 if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
2100 return NULL;
2101
2102 func = btf_type_by_id(btf_vmlinux, insn->imm);
2103 return btf_name_by_offset(btf_vmlinux, func->name_off);
2104}
2105
2106
2107
2108
2109
2110static int backtrack_insn(struct bpf_verifier_env *env, int idx,
2111 u32 *reg_mask, u64 *stack_mask)
2112{
2113 const struct bpf_insn_cbs cbs = {
2114 .cb_call = disasm_kfunc_name,
2115 .cb_print = verbose,
2116 .private_data = env,
2117 };
2118 struct bpf_insn *insn = env->prog->insnsi + idx;
2119 u8 class = BPF_CLASS(insn->code);
2120 u8 opcode = BPF_OP(insn->code);
2121 u8 mode = BPF_MODE(insn->code);
2122 u32 dreg = 1u << insn->dst_reg;
2123 u32 sreg = 1u << insn->src_reg;
2124 u32 spi;
2125
2126 if (insn->code == 0)
2127 return 0;
2128 if (env->log.level & BPF_LOG_LEVEL) {
2129 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask);
2130 verbose(env, "%d: ", idx);
2131 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
2132 }
2133
2134 if (class == BPF_ALU || class == BPF_ALU64) {
2135 if (!(*reg_mask & dreg))
2136 return 0;
2137 if (opcode == BPF_MOV) {
2138 if (BPF_SRC(insn->code) == BPF_X) {
2139
2140
2141
2142
2143 *reg_mask &= ~dreg;
2144 *reg_mask |= sreg;
2145 } else {
2146
2147
2148
2149
2150
2151
2152 *reg_mask &= ~dreg;
2153 }
2154 } else {
2155 if (BPF_SRC(insn->code) == BPF_X) {
2156
2157
2158
2159
2160 *reg_mask |= sreg;
2161 }
2162
2163
2164 }
2165 } else if (class == BPF_LDX) {
2166 if (!(*reg_mask & dreg))
2167 return 0;
2168 *reg_mask &= ~dreg;
2169
2170
2171
2172
2173
2174
2175
2176 if (insn->src_reg != BPF_REG_FP)
2177 return 0;
2178 if (BPF_SIZE(insn->code) != BPF_DW)
2179 return 0;
2180
2181
2182
2183
2184
2185 spi = (-insn->off - 1) / BPF_REG_SIZE;
2186 if (spi >= 64) {
2187 verbose(env, "BUG spi %d\n", spi);
2188 WARN_ONCE(1, "verifier backtracking bug");
2189 return -EFAULT;
2190 }
2191 *stack_mask |= 1ull << spi;
2192 } else if (class == BPF_STX || class == BPF_ST) {
2193 if (*reg_mask & dreg)
2194
2195
2196
2197
2198 return -ENOTSUPP;
2199
2200 if (insn->dst_reg != BPF_REG_FP)
2201 return 0;
2202 if (BPF_SIZE(insn->code) != BPF_DW)
2203 return 0;
2204 spi = (-insn->off - 1) / BPF_REG_SIZE;
2205 if (spi >= 64) {
2206 verbose(env, "BUG spi %d\n", spi);
2207 WARN_ONCE(1, "verifier backtracking bug");
2208 return -EFAULT;
2209 }
2210 if (!(*stack_mask & (1ull << spi)))
2211 return 0;
2212 *stack_mask &= ~(1ull << spi);
2213 if (class == BPF_STX)
2214 *reg_mask |= sreg;
2215 } else if (class == BPF_JMP || class == BPF_JMP32) {
2216 if (opcode == BPF_CALL) {
2217 if (insn->src_reg == BPF_PSEUDO_CALL)
2218 return -ENOTSUPP;
2219
2220 *reg_mask &= ~1;
2221 if (*reg_mask & 0x3f) {
2222
2223
2224
2225 verbose(env, "BUG regs %x\n", *reg_mask);
2226 WARN_ONCE(1, "verifier backtracking bug");
2227 return -EFAULT;
2228 }
2229 } else if (opcode == BPF_EXIT) {
2230 return -ENOTSUPP;
2231 }
2232 } else if (class == BPF_LD) {
2233 if (!(*reg_mask & dreg))
2234 return 0;
2235 *reg_mask &= ~dreg;
2236
2237
2238
2239
2240 if (mode == BPF_IND || mode == BPF_ABS)
2241
2242 return -ENOTSUPP;
2243 }
2244 return 0;
2245}
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299static void mark_all_scalars_precise(struct bpf_verifier_env *env,
2300 struct bpf_verifier_state *st)
2301{
2302 struct bpf_func_state *func;
2303 struct bpf_reg_state *reg;
2304 int i, j;
2305
2306
2307
2308
2309 for (; st; st = st->parent)
2310 for (i = 0; i <= st->curframe; i++) {
2311 func = st->frame[i];
2312 for (j = 0; j < BPF_REG_FP; j++) {
2313 reg = &func->regs[j];
2314 if (reg->type != SCALAR_VALUE)
2315 continue;
2316 reg->precise = true;
2317 }
2318 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
2319 if (func->stack[j].slot_type[0] != STACK_SPILL)
2320 continue;
2321 reg = &func->stack[j].spilled_ptr;
2322 if (reg->type != SCALAR_VALUE)
2323 continue;
2324 reg->precise = true;
2325 }
2326 }
2327}
2328
2329static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,
2330 int spi)
2331{
2332 struct bpf_verifier_state *st = env->cur_state;
2333 int first_idx = st->first_insn_idx;
2334 int last_idx = env->insn_idx;
2335 struct bpf_func_state *func;
2336 struct bpf_reg_state *reg;
2337 u32 reg_mask = regno >= 0 ? 1u << regno : 0;
2338 u64 stack_mask = spi >= 0 ? 1ull << spi : 0;
2339 bool skip_first = true;
2340 bool new_marks = false;
2341 int i, err;
2342
2343 if (!env->bpf_capable)
2344 return 0;
2345
2346 func = st->frame[st->curframe];
2347 if (regno >= 0) {
2348 reg = &func->regs[regno];
2349 if (reg->type != SCALAR_VALUE) {
2350 WARN_ONCE(1, "backtracing misuse");
2351 return -EFAULT;
2352 }
2353 if (!reg->precise)
2354 new_marks = true;
2355 else
2356 reg_mask = 0;
2357 reg->precise = true;
2358 }
2359
2360 while (spi >= 0) {
2361 if (func->stack[spi].slot_type[0] != STACK_SPILL) {
2362 stack_mask = 0;
2363 break;
2364 }
2365 reg = &func->stack[spi].spilled_ptr;
2366 if (reg->type != SCALAR_VALUE) {
2367 stack_mask = 0;
2368 break;
2369 }
2370 if (!reg->precise)
2371 new_marks = true;
2372 else
2373 stack_mask = 0;
2374 reg->precise = true;
2375 break;
2376 }
2377
2378 if (!new_marks)
2379 return 0;
2380 if (!reg_mask && !stack_mask)
2381 return 0;
2382 for (;;) {
2383 DECLARE_BITMAP(mask, 64);
2384 u32 history = st->jmp_history_cnt;
2385
2386 if (env->log.level & BPF_LOG_LEVEL)
2387 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);
2388 for (i = last_idx;;) {
2389 if (skip_first) {
2390 err = 0;
2391 skip_first = false;
2392 } else {
2393 err = backtrack_insn(env, i, ®_mask, &stack_mask);
2394 }
2395 if (err == -ENOTSUPP) {
2396 mark_all_scalars_precise(env, st);
2397 return 0;
2398 } else if (err) {
2399 return err;
2400 }
2401 if (!reg_mask && !stack_mask)
2402
2403
2404
2405
2406 return 0;
2407 if (i == first_idx)
2408 break;
2409 i = get_prev_insn_idx(st, i, &history);
2410 if (i >= env->prog->len) {
2411
2412
2413
2414
2415
2416
2417 verbose(env, "BUG backtracking idx %d\n", i);
2418 WARN_ONCE(1, "verifier backtracking bug");
2419 return -EFAULT;
2420 }
2421 }
2422 st = st->parent;
2423 if (!st)
2424 break;
2425
2426 new_marks = false;
2427 func = st->frame[st->curframe];
2428 bitmap_from_u64(mask, reg_mask);
2429 for_each_set_bit(i, mask, 32) {
2430 reg = &func->regs[i];
2431 if (reg->type != SCALAR_VALUE) {
2432 reg_mask &= ~(1u << i);
2433 continue;
2434 }
2435 if (!reg->precise)
2436 new_marks = true;
2437 reg->precise = true;
2438 }
2439
2440 bitmap_from_u64(mask, stack_mask);
2441 for_each_set_bit(i, mask, 64) {
2442 if (i >= func->allocated_stack / BPF_REG_SIZE) {
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456 mark_all_scalars_precise(env, st);
2457 return 0;
2458 }
2459
2460 if (func->stack[i].slot_type[0] != STACK_SPILL) {
2461 stack_mask &= ~(1ull << i);
2462 continue;
2463 }
2464 reg = &func->stack[i].spilled_ptr;
2465 if (reg->type != SCALAR_VALUE) {
2466 stack_mask &= ~(1ull << i);
2467 continue;
2468 }
2469 if (!reg->precise)
2470 new_marks = true;
2471 reg->precise = true;
2472 }
2473 if (env->log.level & BPF_LOG_LEVEL) {
2474 print_verifier_state(env, func);
2475 verbose(env, "parent %s regs=%x stack=%llx marks\n",
2476 new_marks ? "didn't have" : "already had",
2477 reg_mask, stack_mask);
2478 }
2479
2480 if (!reg_mask && !stack_mask)
2481 break;
2482 if (!new_marks)
2483 break;
2484
2485 last_idx = st->last_insn_idx;
2486 first_idx = st->first_insn_idx;
2487 }
2488 return 0;
2489}
2490
2491static int mark_chain_precision(struct bpf_verifier_env *env, int regno)
2492{
2493 return __mark_chain_precision(env, regno, -1);
2494}
2495
2496static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi)
2497{
2498 return __mark_chain_precision(env, -1, spi);
2499}
2500
2501static bool is_spillable_regtype(enum bpf_reg_type type)
2502{
2503 switch (type) {
2504 case PTR_TO_MAP_VALUE:
2505 case PTR_TO_MAP_VALUE_OR_NULL:
2506 case PTR_TO_STACK:
2507 case PTR_TO_CTX:
2508 case PTR_TO_PACKET:
2509 case PTR_TO_PACKET_META:
2510 case PTR_TO_PACKET_END:
2511 case PTR_TO_FLOW_KEYS:
2512 case CONST_PTR_TO_MAP:
2513 case PTR_TO_SOCKET:
2514 case PTR_TO_SOCKET_OR_NULL:
2515 case PTR_TO_SOCK_COMMON:
2516 case PTR_TO_SOCK_COMMON_OR_NULL:
2517 case PTR_TO_TCP_SOCK:
2518 case PTR_TO_TCP_SOCK_OR_NULL:
2519 case PTR_TO_XDP_SOCK:
2520 case PTR_TO_BTF_ID:
2521 case PTR_TO_BTF_ID_OR_NULL:
2522 case PTR_TO_RDONLY_BUF:
2523 case PTR_TO_RDONLY_BUF_OR_NULL:
2524 case PTR_TO_RDWR_BUF:
2525 case PTR_TO_RDWR_BUF_OR_NULL:
2526 case PTR_TO_PERCPU_BTF_ID:
2527 case PTR_TO_MEM:
2528 case PTR_TO_MEM_OR_NULL:
2529 case PTR_TO_FUNC:
2530 case PTR_TO_MAP_KEY:
2531 return true;
2532 default:
2533 return false;
2534 }
2535}
2536
2537
2538static bool register_is_null(struct bpf_reg_state *reg)
2539{
2540 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
2541}
2542
2543static bool register_is_const(struct bpf_reg_state *reg)
2544{
2545 return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);
2546}
2547
2548static bool __is_scalar_unbounded(struct bpf_reg_state *reg)
2549{
2550 return tnum_is_unknown(reg->var_off) &&
2551 reg->smin_value == S64_MIN && reg->smax_value == S64_MAX &&
2552 reg->umin_value == 0 && reg->umax_value == U64_MAX &&
2553 reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX &&
2554 reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX;
2555}
2556
2557static bool register_is_bounded(struct bpf_reg_state *reg)
2558{
2559 return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg);
2560}
2561
2562static bool __is_pointer_value(bool allow_ptr_leaks,
2563 const struct bpf_reg_state *reg)
2564{
2565 if (allow_ptr_leaks)
2566 return false;
2567
2568 return reg->type != SCALAR_VALUE;
2569}
2570
2571static void save_register_state(struct bpf_func_state *state,
2572 int spi, struct bpf_reg_state *reg)
2573{
2574 int i;
2575
2576 state->stack[spi].spilled_ptr = *reg;
2577 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
2578
2579 for (i = 0; i < BPF_REG_SIZE; i++)
2580 state->stack[spi].slot_type[i] = STACK_SPILL;
2581}
2582
2583
2584
2585
2586static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
2587
2588 struct bpf_func_state *state,
2589 int off, int size, int value_regno,
2590 int insn_idx)
2591{
2592 struct bpf_func_state *cur;
2593 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
2594 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg;
2595 struct bpf_reg_state *reg = NULL;
2596
2597 err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE));
2598 if (err)
2599 return err;
2600
2601
2602
2603 if (!env->allow_ptr_leaks &&
2604 state->stack[spi].slot_type[0] == STACK_SPILL &&
2605 size != BPF_REG_SIZE) {
2606 verbose(env, "attempt to corrupt spilled pointer on stack\n");
2607 return -EACCES;
2608 }
2609
2610 cur = env->cur_state->frame[env->cur_state->curframe];
2611 if (value_regno >= 0)
2612 reg = &cur->regs[value_regno];
2613 if (!env->bypass_spec_v4) {
2614 bool sanitize = reg && is_spillable_regtype(reg->type);
2615
2616 for (i = 0; i < size; i++) {
2617 if (state->stack[spi].slot_type[i] == STACK_INVALID) {
2618 sanitize = true;
2619 break;
2620 }
2621 }
2622
2623 if (sanitize)
2624 env->insn_aux_data[insn_idx].sanitize_stack_spill = true;
2625 }
2626
2627 if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) &&
2628 !register_is_null(reg) && env->bpf_capable) {
2629 if (dst_reg != BPF_REG_FP) {
2630
2631
2632
2633
2634
2635
2636 err = mark_chain_precision(env, value_regno);
2637 if (err)
2638 return err;
2639 }
2640 save_register_state(state, spi, reg);
2641 } else if (reg && is_spillable_regtype(reg->type)) {
2642
2643 if (size != BPF_REG_SIZE) {
2644 verbose_linfo(env, insn_idx, "; ");
2645 verbose(env, "invalid size of register spill\n");
2646 return -EACCES;
2647 }
2648 if (state != cur && reg->type == PTR_TO_STACK) {
2649 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
2650 return -EINVAL;
2651 }
2652 save_register_state(state, spi, reg);
2653 } else {
2654 u8 type = STACK_MISC;
2655
2656
2657 state->stack[spi].spilled_ptr.type = NOT_INIT;
2658
2659 if (state->stack[spi].slot_type[0] == STACK_SPILL)
2660 for (i = 0; i < BPF_REG_SIZE; i++)
2661 state->stack[spi].slot_type[i] = STACK_MISC;
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671 if (size == BPF_REG_SIZE)
2672 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
2673
2674
2675 if (reg && register_is_null(reg)) {
2676
2677 err = mark_chain_precision(env, value_regno);
2678 if (err)
2679 return err;
2680 type = STACK_ZERO;
2681 }
2682
2683
2684 for (i = 0; i < size; i++)
2685 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] =
2686 type;
2687 }
2688 return 0;
2689}
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710static int check_stack_write_var_off(struct bpf_verifier_env *env,
2711
2712 struct bpf_func_state *state,
2713 int ptr_regno, int off, int size,
2714 int value_regno, int insn_idx)
2715{
2716 struct bpf_func_state *cur;
2717 int min_off, max_off;
2718 int i, err;
2719 struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;
2720 bool writing_zero = false;
2721
2722
2723
2724 bool zero_used = false;
2725
2726 cur = env->cur_state->frame[env->cur_state->curframe];
2727 ptr_reg = &cur->regs[ptr_regno];
2728 min_off = ptr_reg->smin_value + off;
2729 max_off = ptr_reg->smax_value + off + size;
2730 if (value_regno >= 0)
2731 value_reg = &cur->regs[value_regno];
2732 if (value_reg && register_is_null(value_reg))
2733 writing_zero = true;
2734
2735 err = grow_stack_state(state, round_up(-min_off, BPF_REG_SIZE));
2736 if (err)
2737 return err;
2738
2739
2740
2741 for (i = min_off; i < max_off; i++) {
2742 u8 new_type, *stype;
2743 int slot, spi;
2744
2745 slot = -i - 1;
2746 spi = slot / BPF_REG_SIZE;
2747 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
2748
2749 if (!env->allow_ptr_leaks
2750 && *stype != NOT_INIT
2751 && *stype != SCALAR_VALUE) {
2752
2753
2754
2755
2756
2757
2758 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",
2759 insn_idx, i);
2760 return -EINVAL;
2761 }
2762
2763
2764 state->stack[spi].spilled_ptr.type = NOT_INIT;
2765
2766
2767 new_type = STACK_MISC;
2768 if (writing_zero && *stype == STACK_ZERO) {
2769 new_type = STACK_ZERO;
2770 zero_used = true;
2771 }
2772
2773
2774
2775
2776
2777
2778
2779
2780 if (*stype == STACK_INVALID && !env->allow_uninit_stack) {
2781 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",
2782 insn_idx, i);
2783 return -EINVAL;
2784 }
2785 *stype = new_type;
2786 }
2787 if (zero_used) {
2788
2789 err = mark_chain_precision(env, value_regno);
2790 if (err)
2791 return err;
2792 }
2793 return 0;
2794}
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804static void mark_reg_stack_read(struct bpf_verifier_env *env,
2805
2806 struct bpf_func_state *ptr_state,
2807 int min_off, int max_off, int dst_regno)
2808{
2809 struct bpf_verifier_state *vstate = env->cur_state;
2810 struct bpf_func_state *state = vstate->frame[vstate->curframe];
2811 int i, slot, spi;
2812 u8 *stype;
2813 int zeros = 0;
2814
2815 for (i = min_off; i < max_off; i++) {
2816 slot = -i - 1;
2817 spi = slot / BPF_REG_SIZE;
2818 stype = ptr_state->stack[spi].slot_type;
2819 if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
2820 break;
2821 zeros++;
2822 }
2823 if (zeros == max_off - min_off) {
2824
2825
2826
2827 __mark_reg_const_zero(&state->regs[dst_regno]);
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838 state->regs[dst_regno].precise = true;
2839 } else {
2840
2841 mark_reg_unknown(env, state->regs, dst_regno);
2842 }
2843 state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
2844}
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
2856
2857 struct bpf_func_state *reg_state,
2858 int off, int size, int dst_regno)
2859{
2860 struct bpf_verifier_state *vstate = env->cur_state;
2861 struct bpf_func_state *state = vstate->frame[vstate->curframe];
2862 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
2863 struct bpf_reg_state *reg;
2864 u8 *stype;
2865
2866 stype = reg_state->stack[spi].slot_type;
2867 reg = ®_state->stack[spi].spilled_ptr;
2868
2869 if (stype[0] == STACK_SPILL) {
2870 if (size != BPF_REG_SIZE) {
2871 if (reg->type != SCALAR_VALUE) {
2872 verbose_linfo(env, env->insn_idx, "; ");
2873 verbose(env, "invalid size of register fill\n");
2874 return -EACCES;
2875 }
2876 if (dst_regno >= 0) {
2877 mark_reg_unknown(env, state->regs, dst_regno);
2878 state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
2879 }
2880 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2881 return 0;
2882 }
2883 for (i = 1; i < BPF_REG_SIZE; i++) {
2884 if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) {
2885 verbose(env, "corrupted spill memory\n");
2886 return -EACCES;
2887 }
2888 }
2889
2890 if (dst_regno >= 0) {
2891
2892 state->regs[dst_regno] = *reg;
2893
2894
2895
2896
2897 state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
2898 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
2899
2900
2901
2902
2903
2904
2905 verbose(env, "leaking pointer from stack off %d\n",
2906 off);
2907 return -EACCES;
2908 }
2909 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2910 } else {
2911 u8 type;
2912
2913 for (i = 0; i < size; i++) {
2914 type = stype[(slot - i) % BPF_REG_SIZE];
2915 if (type == STACK_MISC)
2916 continue;
2917 if (type == STACK_ZERO)
2918 continue;
2919 verbose(env, "invalid read from stack off %d+%d size %d\n",
2920 off, i, size);
2921 return -EACCES;
2922 }
2923 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2924 if (dst_regno >= 0)
2925 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
2926 }
2927 return 0;
2928}
2929
2930enum stack_access_src {
2931 ACCESS_DIRECT = 1,
2932 ACCESS_HELPER = 2,
2933};
2934
2935static int check_stack_range_initialized(struct bpf_verifier_env *env,
2936 int regno, int off, int access_size,
2937 bool zero_size_allowed,
2938 enum stack_access_src type,
2939 struct bpf_call_arg_meta *meta);
2940
2941static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
2942{
2943 return cur_regs(env) + regno;
2944}
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959static int check_stack_read_var_off(struct bpf_verifier_env *env,
2960 int ptr_regno, int off, int size, int dst_regno)
2961{
2962
2963 struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2964 struct bpf_func_state *ptr_state = func(env, reg);
2965 int err;
2966 int min_off, max_off;
2967
2968
2969
2970 err = check_stack_range_initialized(env, ptr_regno, off, size,
2971 false, ACCESS_DIRECT, NULL);
2972 if (err)
2973 return err;
2974
2975 min_off = reg->smin_value + off;
2976 max_off = reg->smax_value + off;
2977 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
2978 return 0;
2979}
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990static int check_stack_read(struct bpf_verifier_env *env,
2991 int ptr_regno, int off, int size,
2992 int dst_regno)
2993{
2994 struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2995 struct bpf_func_state *state = func(env, reg);
2996 int err;
2997
2998 bool var_off = !tnum_is_const(reg->var_off);
2999
3000
3001
3002
3003
3004 if (dst_regno < 0 && var_off) {
3005 char tn_buf[48];
3006
3007 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3008 verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
3009 tn_buf, off, size);
3010 return -EACCES;
3011 }
3012
3013
3014
3015
3016 if (!env->bypass_spec_v1 && var_off) {
3017 char tn_buf[48];
3018
3019 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3020 verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n",
3021 ptr_regno, tn_buf);
3022 return -EACCES;
3023 }
3024
3025 if (!var_off) {
3026 off += reg->var_off.value;
3027 err = check_stack_read_fixed_off(env, state, off, size,
3028 dst_regno);
3029 } else {
3030
3031
3032
3033
3034 err = check_stack_read_var_off(env, ptr_regno, off, size,
3035 dst_regno);
3036 }
3037 return err;
3038}
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051static int check_stack_write(struct bpf_verifier_env *env,
3052 int ptr_regno, int off, int size,
3053 int value_regno, int insn_idx)
3054{
3055 struct bpf_reg_state *reg = reg_state(env, ptr_regno);
3056 struct bpf_func_state *state = func(env, reg);
3057 int err;
3058
3059 if (tnum_is_const(reg->var_off)) {
3060 off += reg->var_off.value;
3061 err = check_stack_write_fixed_off(env, state, off, size,
3062 value_regno, insn_idx);
3063 } else {
3064
3065
3066
3067 err = check_stack_write_var_off(env, state,
3068 ptr_regno, off, size,
3069 value_regno, insn_idx);
3070 }
3071 return err;
3072}
3073
3074static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
3075 int off, int size, enum bpf_access_type type)
3076{
3077 struct bpf_reg_state *regs = cur_regs(env);
3078 struct bpf_map *map = regs[regno].map_ptr;
3079 u32 cap = bpf_map_flags_to_cap(map);
3080
3081 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) {
3082 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n",
3083 map->value_size, off, size);
3084 return -EACCES;
3085 }
3086
3087 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) {
3088 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n",
3089 map->value_size, off, size);
3090 return -EACCES;
3091 }
3092
3093 return 0;
3094}
3095
3096
3097static int __check_mem_access(struct bpf_verifier_env *env, int regno,
3098 int off, int size, u32 mem_size,
3099 bool zero_size_allowed)
3100{
3101 bool size_ok = size > 0 || (size == 0 && zero_size_allowed);
3102 struct bpf_reg_state *reg;
3103
3104 if (off >= 0 && size_ok && (u64)off + size <= mem_size)
3105 return 0;
3106
3107 reg = &cur_regs(env)[regno];
3108 switch (reg->type) {
3109 case PTR_TO_MAP_KEY:
3110 verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",
3111 mem_size, off, size);
3112 break;
3113 case PTR_TO_MAP_VALUE:
3114 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",
3115 mem_size, off, size);
3116 break;
3117 case PTR_TO_PACKET:
3118 case PTR_TO_PACKET_META:
3119 case PTR_TO_PACKET_END:
3120 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",
3121 off, size, regno, reg->id, off, mem_size);
3122 break;
3123 case PTR_TO_MEM:
3124 default:
3125 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n",
3126 mem_size, off, size);
3127 }
3128
3129 return -EACCES;
3130}
3131
3132
3133static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,
3134 int off, int size, u32 mem_size,
3135 bool zero_size_allowed)
3136{
3137 struct bpf_verifier_state *vstate = env->cur_state;
3138 struct bpf_func_state *state = vstate->frame[vstate->curframe];
3139 struct bpf_reg_state *reg = &state->regs[regno];
3140 int err;
3141
3142
3143
3144
3145
3146 if (env->log.level & BPF_LOG_LEVEL)
3147 print_verifier_state(env, state);
3148
3149
3150
3151
3152
3153
3154
3155 if (reg->smin_value < 0 &&
3156 (reg->smin_value == S64_MIN ||
3157 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) ||
3158 reg->smin_value + off < 0)) {
3159 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
3160 regno);
3161 return -EACCES;
3162 }
3163 err = __check_mem_access(env, regno, reg->smin_value + off, size,
3164 mem_size, zero_size_allowed);
3165 if (err) {
3166 verbose(env, "R%d min value is outside of the allowed memory range\n",
3167 regno);
3168 return err;
3169 }
3170
3171
3172
3173
3174
3175 if (reg->umax_value >= BPF_MAX_VAR_OFF) {
3176 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n",
3177 regno);
3178 return -EACCES;
3179 }
3180 err = __check_mem_access(env, regno, reg->umax_value + off, size,
3181 mem_size, zero_size_allowed);
3182 if (err) {
3183 verbose(env, "R%d max value is outside of the allowed memory range\n",
3184 regno);
3185 return err;
3186 }
3187
3188 return 0;
3189}
3190
3191
3192static int check_map_access(struct bpf_verifier_env *env, u32 regno,
3193 int off, int size, bool zero_size_allowed)
3194{
3195 struct bpf_verifier_state *vstate = env->cur_state;
3196 struct bpf_func_state *state = vstate->frame[vstate->curframe];
3197 struct bpf_reg_state *reg = &state->regs[regno];
3198 struct bpf_map *map = reg->map_ptr;
3199 int err;
3200
3201 err = check_mem_region_access(env, regno, off, size, map->value_size,
3202 zero_size_allowed);
3203 if (err)
3204 return err;
3205
3206 if (map_value_has_spin_lock(map)) {
3207 u32 lock = map->spin_lock_off;
3208
3209
3210
3211
3212
3213
3214 if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) &&
3215 lock < reg->umax_value + off + size) {
3216 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n");
3217 return -EACCES;
3218 }
3219 }
3220 return err;
3221}
3222
3223#define MAX_PACKET_OFF 0xffff
3224
3225static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog)
3226{
3227 return prog->aux->dst_prog ? prog->aux->dst_prog->type : prog->type;
3228}
3229
3230static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
3231 const struct bpf_call_arg_meta *meta,
3232 enum bpf_access_type t)
3233{
3234 enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
3235
3236 switch (prog_type) {
3237
3238 case BPF_PROG_TYPE_LWT_IN:
3239 case BPF_PROG_TYPE_LWT_OUT:
3240 case BPF_PROG_TYPE_LWT_SEG6LOCAL:
3241 case BPF_PROG_TYPE_SK_REUSEPORT:
3242 case BPF_PROG_TYPE_FLOW_DISSECTOR:
3243 case BPF_PROG_TYPE_CGROUP_SKB:
3244 if (t == BPF_WRITE)
3245 return false;
3246 fallthrough;
3247
3248
3249 case BPF_PROG_TYPE_SCHED_CLS:
3250 case BPF_PROG_TYPE_SCHED_ACT:
3251 case BPF_PROG_TYPE_XDP:
3252 case BPF_PROG_TYPE_LWT_XMIT:
3253 case BPF_PROG_TYPE_SK_SKB:
3254 case BPF_PROG_TYPE_SK_MSG:
3255 if (meta)
3256 return meta->pkt_access;
3257
3258 env->seen_direct_write = true;
3259 return true;
3260
3261 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
3262 if (t == BPF_WRITE)
3263 env->seen_direct_write = true;
3264
3265 return true;
3266
3267 default:
3268 return false;
3269 }
3270}
3271
3272static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
3273 int size, bool zero_size_allowed)
3274{
3275 struct bpf_reg_state *regs = cur_regs(env);
3276 struct bpf_reg_state *reg = ®s[regno];
3277 int err;
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287 if (reg->smin_value < 0) {
3288 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
3289 regno);
3290 return -EACCES;
3291 }
3292
3293 err = reg->range < 0 ? -EINVAL :
3294 __check_mem_access(env, regno, off, size, reg->range,
3295 zero_size_allowed);
3296 if (err) {
3297 verbose(env, "R%d offset is outside of the packet\n", regno);
3298 return err;
3299 }
3300
3301
3302
3303
3304
3305
3306
3307 env->prog->aux->max_pkt_offset =
3308 max_t(u32, env->prog->aux->max_pkt_offset,
3309 off + reg->umax_value + size - 1);
3310
3311 return err;
3312}
3313
3314
3315static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
3316 enum bpf_access_type t, enum bpf_reg_type *reg_type,
3317 struct btf **btf, u32 *btf_id)
3318{
3319 struct bpf_insn_access_aux info = {
3320 .reg_type = *reg_type,
3321 .log = &env->log,
3322 };
3323
3324 if (env->ops->is_valid_access &&
3325 env->ops->is_valid_access(off, size, t, env->prog, &info)) {
3326
3327
3328
3329
3330
3331
3332
3333 *reg_type = info.reg_type;
3334
3335 if (*reg_type == PTR_TO_BTF_ID || *reg_type == PTR_TO_BTF_ID_OR_NULL) {
3336 *btf = info.btf;
3337 *btf_id = info.btf_id;
3338 } else {
3339 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
3340 }
3341
3342 if (env->prog->aux->max_ctx_offset < off + size)
3343 env->prog->aux->max_ctx_offset = off + size;
3344 return 0;
3345 }
3346
3347 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
3348 return -EACCES;
3349}
3350
3351static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
3352 int size)
3353{
3354 if (size < 0 || off < 0 ||
3355 (u64)off + size > sizeof(struct bpf_flow_keys)) {
3356 verbose(env, "invalid access to flow keys off=%d size=%d\n",
3357 off, size);
3358 return -EACCES;
3359 }
3360 return 0;
3361}
3362
3363static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
3364 u32 regno, int off, int size,
3365 enum bpf_access_type t)
3366{
3367 struct bpf_reg_state *regs = cur_regs(env);
3368 struct bpf_reg_state *reg = ®s[regno];
3369 struct bpf_insn_access_aux info = {};
3370 bool valid;
3371
3372 if (reg->smin_value < 0) {
3373 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
3374 regno);
3375 return -EACCES;
3376 }
3377
3378 switch (reg->type) {
3379 case PTR_TO_SOCK_COMMON:
3380 valid = bpf_sock_common_is_valid_access(off, size, t, &info);
3381 break;
3382 case PTR_TO_SOCKET:
3383 valid = bpf_sock_is_valid_access(off, size, t, &info);
3384 break;
3385 case PTR_TO_TCP_SOCK:
3386 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
3387 break;
3388 case PTR_TO_XDP_SOCK:
3389 valid = bpf_xdp_sock_is_valid_access(off, size, t, &info);
3390 break;
3391 default:
3392 valid = false;
3393 }
3394
3395
3396 if (valid) {
3397 env->insn_aux_data[insn_idx].ctx_field_size =
3398 info.ctx_field_size;
3399 return 0;
3400 }
3401
3402 verbose(env, "R%d invalid %s access off=%d size=%d\n",
3403 regno, reg_type_str[reg->type], off, size);
3404
3405 return -EACCES;
3406}
3407
3408static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
3409{
3410 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
3411}
3412
3413static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
3414{
3415 const struct bpf_reg_state *reg = reg_state(env, regno);
3416
3417 return reg->type == PTR_TO_CTX;
3418}
3419
3420static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
3421{
3422 const struct bpf_reg_state *reg = reg_state(env, regno);
3423
3424 return type_is_sk_pointer(reg->type);
3425}
3426
3427static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
3428{
3429 const struct bpf_reg_state *reg = reg_state(env, regno);
3430
3431 return type_is_pkt_pointer(reg->type);
3432}
3433
3434static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
3435{
3436 const struct bpf_reg_state *reg = reg_state(env, regno);
3437
3438
3439 return reg->type == PTR_TO_FLOW_KEYS;
3440}
3441
3442static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
3443 const struct bpf_reg_state *reg,
3444 int off, int size, bool strict)
3445{
3446 struct tnum reg_off;
3447 int ip_align;
3448
3449
3450 if (!strict || size == 1)
3451 return 0;
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461 ip_align = 2;
3462
3463 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off));
3464 if (!tnum_is_aligned(reg_off, size)) {
3465 char tn_buf[48];
3466
3467 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3468 verbose(env,
3469 "misaligned packet access off %d+%s+%d+%d size %d\n",
3470 ip_align, tn_buf, reg->off, off, size);
3471 return -EACCES;
3472 }
3473
3474 return 0;
3475}
3476
3477static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
3478 const struct bpf_reg_state *reg,
3479 const char *pointer_desc,
3480 int off, int size, bool strict)
3481{
3482 struct tnum reg_off;
3483
3484
3485 if (!strict || size == 1)
3486 return 0;
3487
3488 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off));
3489 if (!tnum_is_aligned(reg_off, size)) {
3490 char tn_buf[48];
3491
3492 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3493 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",
3494 pointer_desc, tn_buf, reg->off, off, size);
3495 return -EACCES;
3496 }
3497
3498 return 0;
3499}
3500
3501static int check_ptr_alignment(struct bpf_verifier_env *env,
3502 const struct bpf_reg_state *reg, int off,
3503 int size, bool strict_alignment_once)
3504{
3505 bool strict = env->strict_alignment || strict_alignment_once;
3506 const char *pointer_desc = "";
3507
3508 switch (reg->type) {
3509 case PTR_TO_PACKET:
3510 case PTR_TO_PACKET_META:
3511
3512
3513
3514 return check_pkt_ptr_alignment(env, reg, off, size, strict);
3515 case PTR_TO_FLOW_KEYS:
3516 pointer_desc = "flow keys ";
3517 break;
3518 case PTR_TO_MAP_KEY:
3519 pointer_desc = "key ";
3520 break;
3521 case PTR_TO_MAP_VALUE:
3522 pointer_desc = "value ";
3523 break;
3524 case PTR_TO_CTX:
3525 pointer_desc = "context ";
3526 break;
3527 case PTR_TO_STACK:
3528 pointer_desc = "stack ";
3529
3530
3531
3532
3533 strict = true;
3534 break;
3535 case PTR_TO_SOCKET:
3536 pointer_desc = "sock ";
3537 break;
3538 case PTR_TO_SOCK_COMMON:
3539 pointer_desc = "sock_common ";
3540 break;
3541 case PTR_TO_TCP_SOCK:
3542 pointer_desc = "tcp_sock ";
3543 break;
3544 case PTR_TO_XDP_SOCK:
3545 pointer_desc = "xdp_sock ";
3546 break;
3547 default:
3548 break;
3549 }
3550 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,
3551 strict);
3552}
3553
3554static int update_stack_depth(struct bpf_verifier_env *env,
3555 const struct bpf_func_state *func,
3556 int off)
3557{
3558 u16 stack = env->subprog_info[func->subprogno].stack_depth;
3559
3560 if (stack >= -off)
3561 return 0;
3562
3563
3564 env->subprog_info[func->subprogno].stack_depth = -off;
3565 return 0;
3566}
3567
3568
3569
3570
3571
3572
3573
3574static int check_max_stack_depth(struct bpf_verifier_env *env)
3575{
3576 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end;
3577 struct bpf_subprog_info *subprog = env->subprog_info;
3578 struct bpf_insn *insn = env->prog->insnsi;
3579 bool tail_call_reachable = false;
3580 int ret_insn[MAX_CALL_FRAMES];
3581 int ret_prog[MAX_CALL_FRAMES];
3582 int j;
3583
3584process_func:
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604 if (idx && subprog[idx].has_tail_call && depth >= 256) {
3605 verbose(env,
3606 "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n",
3607 depth);
3608 return -EACCES;
3609 }
3610
3611
3612
3613 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
3614 if (depth > MAX_BPF_STACK) {
3615 verbose(env, "combined stack size of %d calls is %d. Too large\n",
3616 frame + 1, depth);
3617 return -EACCES;
3618 }
3619continue_func:
3620 subprog_end = subprog[idx + 1].start;
3621 for (; i < subprog_end; i++) {
3622 if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i))
3623 continue;
3624
3625 ret_insn[frame] = i + 1;
3626 ret_prog[frame] = idx;
3627
3628
3629 i = i + insn[i].imm + 1;
3630 idx = find_subprog(env, i);
3631 if (idx < 0) {
3632 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
3633 i);
3634 return -EFAULT;
3635 }
3636
3637 if (subprog[idx].has_tail_call)
3638 tail_call_reachable = true;
3639
3640 frame++;
3641 if (frame >= MAX_CALL_FRAMES) {
3642 verbose(env, "the call stack of %d frames is too deep !\n",
3643 frame);
3644 return -E2BIG;
3645 }
3646 goto process_func;
3647 }
3648
3649
3650
3651
3652
3653 if (tail_call_reachable)
3654 for (j = 0; j < frame; j++)
3655 subprog[ret_prog[j]].tail_call_reachable = true;
3656 if (subprog[0].tail_call_reachable)
3657 env->prog->aux->tail_call_reachable = true;
3658
3659
3660
3661
3662 if (frame == 0)
3663 return 0;
3664 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
3665 frame--;
3666 i = ret_insn[frame];
3667 idx = ret_prog[frame];
3668 goto continue_func;
3669}
3670
3671#ifndef CONFIG_BPF_JIT_ALWAYS_ON
3672static int get_callee_stack_depth(struct bpf_verifier_env *env,
3673 const struct bpf_insn *insn, int idx)
3674{
3675 int start = idx + insn->imm + 1, subprog;
3676
3677 subprog = find_subprog(env, start);
3678 if (subprog < 0) {
3679 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
3680 start);
3681 return -EFAULT;
3682 }
3683 return env->subprog_info[subprog].stack_depth;
3684}
3685#endif
3686
3687int check_ctx_reg(struct bpf_verifier_env *env,
3688 const struct bpf_reg_state *reg, int regno)
3689{
3690
3691
3692
3693
3694 if (reg->off) {
3695 verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n",
3696 regno, reg->off);
3697 return -EACCES;
3698 }
3699
3700 if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
3701 char tn_buf[48];
3702
3703 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3704 verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf);
3705 return -EACCES;
3706 }
3707
3708 return 0;
3709}
3710
3711static int __check_buffer_access(struct bpf_verifier_env *env,
3712 const char *buf_info,
3713 const struct bpf_reg_state *reg,
3714 int regno, int off, int size)
3715{
3716 if (off < 0) {
3717 verbose(env,
3718 "R%d invalid %s buffer access: off=%d, size=%d\n",
3719 regno, buf_info, off, size);
3720 return -EACCES;
3721 }
3722 if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
3723 char tn_buf[48];
3724
3725 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3726 verbose(env,
3727 "R%d invalid variable buffer offset: off=%d, var_off=%s\n",
3728 regno, off, tn_buf);
3729 return -EACCES;
3730 }
3731
3732 return 0;
3733}
3734
3735static int check_tp_buffer_access(struct bpf_verifier_env *env,
3736 const struct bpf_reg_state *reg,
3737 int regno, int off, int size)
3738{
3739 int err;
3740
3741 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);
3742 if (err)
3743 return err;
3744
3745 if (off + size > env->prog->aux->max_tp_access)
3746 env->prog->aux->max_tp_access = off + size;
3747
3748 return 0;
3749}
3750
3751static int check_buffer_access(struct bpf_verifier_env *env,
3752 const struct bpf_reg_state *reg,
3753 int regno, int off, int size,
3754 bool zero_size_allowed,
3755 const char *buf_info,
3756 u32 *max_access)
3757{
3758 int err;
3759
3760 err = __check_buffer_access(env, buf_info, reg, regno, off, size);
3761 if (err)
3762 return err;
3763
3764 if (off + size > *max_access)
3765 *max_access = off + size;
3766
3767 return 0;
3768}
3769
3770
3771static void zext_32_to_64(struct bpf_reg_state *reg)
3772{
3773 reg->var_off = tnum_subreg(reg->var_off);
3774 __reg_assign_32_into_64(reg);
3775}
3776
3777
3778
3779
3780static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
3781{
3782 u64 mask;
3783
3784
3785 reg->var_off = tnum_cast(reg->var_off, size);
3786
3787
3788 mask = ((u64)1 << (size * 8)) - 1;
3789 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) {
3790 reg->umin_value &= mask;
3791 reg->umax_value &= mask;
3792 } else {
3793 reg->umin_value = 0;
3794 reg->umax_value = mask;
3795 }
3796 reg->smin_value = reg->umin_value;
3797 reg->smax_value = reg->umax_value;
3798
3799
3800
3801
3802
3803 if (size >= 4)
3804 return;
3805 __reg_combine_64_into_32(reg);
3806}
3807
3808static bool bpf_map_is_rdonly(const struct bpf_map *map)
3809{
3810 return (map->map_flags & BPF_F_RDONLY_PROG) && map->frozen;
3811}
3812
3813static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val)
3814{
3815 void *ptr;
3816 u64 addr;
3817 int err;
3818
3819 err = map->ops->map_direct_value_addr(map, &addr, off);
3820 if (err)
3821 return err;
3822 ptr = (void *)(long)addr + off;
3823
3824 switch (size) {
3825 case sizeof(u8):
3826 *val = (u64)*(u8 *)ptr;
3827 break;
3828 case sizeof(u16):
3829 *val = (u64)*(u16 *)ptr;
3830 break;
3831 case sizeof(u32):
3832 *val = (u64)*(u32 *)ptr;
3833 break;
3834 case sizeof(u64):
3835 *val = *(u64 *)ptr;
3836 break;
3837 default:
3838 return -EINVAL;
3839 }
3840 return 0;
3841}
3842
3843static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
3844 struct bpf_reg_state *regs,
3845 int regno, int off, int size,
3846 enum bpf_access_type atype,
3847 int value_regno)
3848{
3849 struct bpf_reg_state *reg = regs + regno;
3850 const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
3851 const char *tname = btf_name_by_offset(reg->btf, t->name_off);
3852 u32 btf_id;
3853 int ret;
3854
3855 if (off < 0) {
3856 verbose(env,
3857 "R%d is ptr_%s invalid negative access: off=%d\n",
3858 regno, tname, off);
3859 return -EACCES;
3860 }
3861 if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
3862 char tn_buf[48];
3863
3864 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3865 verbose(env,
3866 "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n",
3867 regno, tname, off, tn_buf);
3868 return -EACCES;
3869 }
3870
3871 if (env->ops->btf_struct_access) {
3872 ret = env->ops->btf_struct_access(&env->log, reg->btf, t,
3873 off, size, atype, &btf_id);
3874 } else {
3875 if (atype != BPF_READ) {
3876 verbose(env, "only read is supported\n");
3877 return -EACCES;
3878 }
3879
3880 ret = btf_struct_access(&env->log, reg->btf, t, off, size,
3881 atype, &btf_id);
3882 }
3883
3884 if (ret < 0)
3885 return ret;
3886
3887 if (atype == BPF_READ && value_regno >= 0)
3888 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id);
3889
3890 return 0;
3891}
3892
3893static int check_ptr_to_map_access(struct bpf_verifier_env *env,
3894 struct bpf_reg_state *regs,
3895 int regno, int off, int size,
3896 enum bpf_access_type atype,
3897 int value_regno)
3898{
3899 struct bpf_reg_state *reg = regs + regno;
3900 struct bpf_map *map = reg->map_ptr;
3901 const struct btf_type *t;
3902 const char *tname;
3903 u32 btf_id;
3904 int ret;
3905
3906 if (!btf_vmlinux) {
3907 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");
3908 return -ENOTSUPP;
3909 }
3910
3911 if (!map->ops->map_btf_id || !*map->ops->map_btf_id) {
3912 verbose(env, "map_ptr access not supported for map type %d\n",
3913 map->map_type);
3914 return -ENOTSUPP;
3915 }
3916
3917 t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id);
3918 tname = btf_name_by_offset(btf_vmlinux, t->name_off);
3919
3920 if (!env->allow_ptr_to_map_access) {
3921 verbose(env,
3922 "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
3923 tname);
3924 return -EPERM;
3925 }
3926
3927 if (off < 0) {
3928 verbose(env, "R%d is %s invalid negative access: off=%d\n",
3929 regno, tname, off);
3930 return -EACCES;
3931 }
3932
3933 if (atype != BPF_READ) {
3934 verbose(env, "only read from %s is supported\n", tname);
3935 return -EACCES;
3936 }
3937
3938 ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id);
3939 if (ret < 0)
3940 return ret;
3941
3942 if (value_regno >= 0)
3943 mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id);
3944
3945 return 0;
3946}
3947
3948
3949
3950
3951
3952
3953
3954static int check_stack_slot_within_bounds(int off,
3955 struct bpf_func_state *state,
3956 enum bpf_access_type t)
3957{
3958 int min_valid_off;
3959
3960 if (t == BPF_WRITE)
3961 min_valid_off = -MAX_BPF_STACK;
3962 else
3963 min_valid_off = -state->allocated_stack;
3964
3965 if (off < min_valid_off || off > -1)
3966 return -EACCES;
3967 return 0;
3968}
3969
3970
3971
3972
3973
3974
3975static int check_stack_access_within_bounds(
3976 struct bpf_verifier_env *env,
3977 int regno, int off, int access_size,
3978 enum stack_access_src src, enum bpf_access_type type)
3979{
3980 struct bpf_reg_state *regs = cur_regs(env);
3981 struct bpf_reg_state *reg = regs + regno;
3982 struct bpf_func_state *state = func(env, reg);
3983 int min_off, max_off;
3984 int err;
3985 char *err_extra;
3986
3987 if (src == ACCESS_HELPER)
3988
3989 err_extra = " indirect access to";
3990 else if (type == BPF_READ)
3991 err_extra = " read from";
3992 else
3993 err_extra = " write to";
3994
3995 if (tnum_is_const(reg->var_off)) {
3996 min_off = reg->var_off.value + off;
3997 if (access_size > 0)
3998 max_off = min_off + access_size - 1;
3999 else
4000 max_off = min_off;
4001 } else {
4002 if (reg->smax_value >= BPF_MAX_VAR_OFF ||
4003 reg->smin_value <= -BPF_MAX_VAR_OFF) {
4004 verbose(env, "invalid unbounded variable-offset%s stack R%d\n",
4005 err_extra, regno);
4006 return -EACCES;
4007 }
4008 min_off = reg->smin_value + off;
4009 if (access_size > 0)
4010 max_off = reg->smax_value + off + access_size - 1;
4011 else
4012 max_off = min_off;
4013 }
4014
4015 err = check_stack_slot_within_bounds(min_off, state, type);
4016 if (!err)
4017 err = check_stack_slot_within_bounds(max_off, state, type);
4018
4019 if (err) {
4020 if (tnum_is_const(reg->var_off)) {
4021 verbose(env, "invalid%s stack R%d off=%d size=%d\n",
4022 err_extra, regno, off, access_size);
4023 } else {
4024 char tn_buf[48];
4025
4026 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
4027 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n",
4028 err_extra, regno, tn_buf, access_size);
4029 }
4030 }
4031 return err;
4032}
4033
4034
4035
4036
4037
4038
4039
4040static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
4041 int off, int bpf_size, enum bpf_access_type t,
4042 int value_regno, bool strict_alignment_once)
4043{
4044 struct bpf_reg_state *regs = cur_regs(env);
4045 struct bpf_reg_state *reg = regs + regno;
4046 struct bpf_func_state *state;
4047 int size, err = 0;
4048
4049 size = bpf_size_to_bytes(bpf_size);
4050 if (size < 0)
4051 return size;
4052
4053
4054 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
4055 if (err)
4056 return err;
4057
4058
4059 off += reg->off;
4060
4061 if (reg->type == PTR_TO_MAP_KEY) {
4062 if (t == BPF_WRITE) {
4063 verbose(env, "write to change key R%d not allowed\n", regno);
4064 return -EACCES;
4065 }
4066
4067 err = check_mem_region_access(env, regno, off, size,
4068 reg->map_ptr->key_size, false);
4069 if (err)
4070 return err;
4071 if (value_regno >= 0)
4072 mark_reg_unknown(env, regs, value_regno);
4073 } else if (reg->type == PTR_TO_MAP_VALUE) {
4074 if (t == BPF_WRITE && value_regno >= 0 &&
4075 is_pointer_value(env, value_regno)) {
4076 verbose(env, "R%d leaks addr into map\n", value_regno);
4077 return -EACCES;
4078 }
4079 err = check_map_access_type(env, regno, off, size, t);
4080 if (err)
4081 return err;
4082 err = check_map_access(env, regno, off, size, false);
4083 if (!err && t == BPF_READ && value_regno >= 0) {
4084 struct bpf_map *map = reg->map_ptr;
4085
4086
4087 if (tnum_is_const(reg->var_off) &&
4088 bpf_map_is_rdonly(map) &&
4089 map->ops->map_direct_value_addr) {
4090 int map_off = off + reg->var_off.value;
4091 u64 val = 0;
4092
4093 err = bpf_map_direct_read(map, map_off, size,
4094 &val);
4095 if (err)
4096 return err;
4097
4098 regs[value_regno].type = SCALAR_VALUE;
4099 __mark_reg_known(®s[value_regno], val);
4100 } else {
4101 mark_reg_unknown(env, regs, value_regno);
4102 }
4103 }
4104 } else if (reg->type == PTR_TO_MEM) {
4105 if (t == BPF_WRITE && value_regno >= 0 &&
4106 is_pointer_value(env, value_regno)) {
4107 verbose(env, "R%d leaks addr into mem\n", value_regno);
4108 return -EACCES;
4109 }
4110 err = check_mem_region_access(env, regno, off, size,
4111 reg->mem_size, false);
4112 if (!err && t == BPF_READ && value_regno >= 0)
4113 mark_reg_unknown(env, regs, value_regno);
4114 } else if (reg->type == PTR_TO_CTX) {
4115 enum bpf_reg_type reg_type = SCALAR_VALUE;
4116 struct btf *btf = NULL;
4117 u32 btf_id = 0;
4118
4119 if (t == BPF_WRITE && value_regno >= 0 &&
4120 is_pointer_value(env, value_regno)) {
4121 verbose(env, "R%d leaks addr into ctx\n", value_regno);
4122 return -EACCES;
4123 }
4124
4125 err = check_ctx_reg(env, reg, regno);
4126 if (err < 0)
4127 return err;
4128
4129 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, &btf_id);
4130 if (err)
4131 verbose_linfo(env, insn_idx, "; ");
4132 if (!err && t == BPF_READ && value_regno >= 0) {
4133
4134
4135
4136
4137 if (reg_type == SCALAR_VALUE) {
4138 mark_reg_unknown(env, regs, value_regno);
4139 } else {
4140 mark_reg_known_zero(env, regs,
4141 value_regno);
4142 if (reg_type_may_be_null(reg_type))
4143 regs[value_regno].id = ++env->id_gen;
4144
4145
4146
4147
4148
4149 regs[value_regno].subreg_def = DEF_NOT_SUBREG;
4150 if (reg_type == PTR_TO_BTF_ID ||
4151 reg_type == PTR_TO_BTF_ID_OR_NULL) {
4152 regs[value_regno].btf = btf;
4153 regs[value_regno].btf_id = btf_id;
4154 }
4155 }
4156 regs[value_regno].type = reg_type;
4157 }
4158
4159 } else if (reg->type == PTR_TO_STACK) {
4160
4161 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t);
4162 if (err)
4163 return err;
4164
4165 state = func(env, reg);
4166 err = update_stack_depth(env, state, off);
4167 if (err)
4168 return err;
4169
4170 if (t == BPF_READ)
4171 err = check_stack_read(env, regno, off, size,
4172 value_regno);
4173 else
4174 err = check_stack_write(env, regno, off, size,
4175 value_regno, insn_idx);
4176 } else if (reg_is_pkt_pointer(reg)) {
4177 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
4178 verbose(env, "cannot write into packet\n");
4179 return -EACCES;
4180 }
4181 if (t == BPF_WRITE && value_regno >= 0 &&
4182 is_pointer_value(env, value_regno)) {
4183 verbose(env, "R%d leaks addr into packet\n",
4184 value_regno);
4185 return -EACCES;
4186 }
4187 err = check_packet_access(env, regno, off, size, false);
4188 if (!err && t == BPF_READ && value_regno >= 0)
4189 mark_reg_unknown(env, regs, value_regno);
4190 } else if (reg->type == PTR_TO_FLOW_KEYS) {
4191 if (t == BPF_WRITE && value_regno >= 0 &&
4192 is_pointer_value(env, value_regno)) {
4193 verbose(env, "R%d leaks addr into flow keys\n",
4194 value_regno);
4195 return -EACCES;
4196 }
4197
4198 err = check_flow_keys_access(env, off, size);
4199 if (!err && t == BPF_READ && value_regno >= 0)
4200 mark_reg_unknown(env, regs, value_regno);
4201 } else if (type_is_sk_pointer(reg->type)) {
4202 if (t == BPF_WRITE) {
4203 verbose(env, "R%d cannot write into %s\n",
4204 regno, reg_type_str[reg->type]);
4205 return -EACCES;
4206 }
4207 err = check_sock_access(env, insn_idx, regno, off, size, t);
4208 if (!err && value_regno >= 0)
4209 mark_reg_unknown(env, regs, value_regno);
4210 } else if (reg->type == PTR_TO_TP_BUFFER) {
4211 err = check_tp_buffer_access(env, reg, regno, off, size);
4212 if (!err && t == BPF_READ && value_regno >= 0)
4213 mark_reg_unknown(env, regs, value_regno);
4214 } else if (reg->type == PTR_TO_BTF_ID) {
4215 err = check_ptr_to_btf_access(env, regs, regno, off, size, t,
4216 value_regno);
4217 } else if (reg->type == CONST_PTR_TO_MAP) {
4218 err = check_ptr_to_map_access(env, regs, regno, off, size, t,
4219 value_regno);
4220 } else if (reg->type == PTR_TO_RDONLY_BUF) {
4221 if (t == BPF_WRITE) {
4222 verbose(env, "R%d cannot write into %s\n",
4223 regno, reg_type_str[reg->type]);
4224 return -EACCES;
4225 }
4226 err = check_buffer_access(env, reg, regno, off, size, false,
4227 "rdonly",
4228 &env->prog->aux->max_rdonly_access);
4229 if (!err && value_regno >= 0)
4230 mark_reg_unknown(env, regs, value_regno);
4231 } else if (reg->type == PTR_TO_RDWR_BUF) {
4232 err = check_buffer_access(env, reg, regno, off, size, false,
4233 "rdwr",
4234 &env->prog->aux->max_rdwr_access);
4235 if (!err && t == BPF_READ && value_regno >= 0)
4236 mark_reg_unknown(env, regs, value_regno);
4237 } else {
4238 verbose(env, "R%d invalid mem access '%s'\n", regno,
4239 reg_type_str[reg->type]);
4240 return -EACCES;
4241 }
4242
4243 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
4244 regs[value_regno].type == SCALAR_VALUE) {
4245
4246 coerce_reg_to_size(®s[value_regno], size);
4247 }
4248 return err;
4249}
4250
4251static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)
4252{
4253 int load_reg;
4254 int err;
4255
4256 switch (insn->imm) {
4257 case BPF_ADD:
4258 case BPF_ADD | BPF_FETCH:
4259 case BPF_AND:
4260 case BPF_AND | BPF_FETCH:
4261 case BPF_OR:
4262 case BPF_OR | BPF_FETCH:
4263 case BPF_XOR:
4264 case BPF_XOR | BPF_FETCH:
4265 case BPF_XCHG:
4266 case BPF_CMPXCHG:
4267 break;
4268 default:
4269 verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm);
4270 return -EINVAL;
4271 }
4272
4273 if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) {
4274 verbose(env, "invalid atomic operand size\n");
4275 return -EINVAL;
4276 }
4277
4278
4279 err = check_reg_arg(env, insn->src_reg, SRC_OP);
4280 if (err)
4281 return err;
4282
4283
4284 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
4285 if (err)
4286 return err;
4287
4288 if (insn->imm == BPF_CMPXCHG) {
4289
4290 err = check_reg_arg(env, BPF_REG_0, SRC_OP);
4291 if (err)
4292 return err;
4293 }
4294
4295 if (is_pointer_value(env, insn->src_reg)) {
4296 verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
4297 return -EACCES;
4298 }
4299
4300 if (is_ctx_reg(env, insn->dst_reg) ||
4301 is_pkt_reg(env, insn->dst_reg) ||
4302 is_flow_key_reg(env, insn->dst_reg) ||
4303 is_sk_reg(env, insn->dst_reg)) {
4304 verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
4305 insn->dst_reg,
4306 reg_type_str[reg_state(env, insn->dst_reg)->type]);
4307 return -EACCES;
4308 }
4309
4310 if (insn->imm & BPF_FETCH) {
4311 if (insn->imm == BPF_CMPXCHG)
4312 load_reg = BPF_REG_0;
4313 else
4314 load_reg = insn->src_reg;
4315
4316
4317 err = check_reg_arg(env, load_reg, DST_OP);
4318 if (err)
4319 return err;
4320 } else {
4321
4322
4323
4324 load_reg = -1;
4325 }
4326
4327
4328 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
4329 BPF_SIZE(insn->code), BPF_READ, load_reg, true);
4330 if (err)
4331 return err;
4332
4333
4334 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
4335 BPF_SIZE(insn->code), BPF_WRITE, -1, true);
4336 if (err)
4337 return err;
4338
4339 return 0;
4340}
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351static int check_stack_range_initialized(
4352 struct bpf_verifier_env *env, int regno, int off,
4353 int access_size, bool zero_size_allowed,
4354 enum stack_access_src type, struct bpf_call_arg_meta *meta)
4355{
4356 struct bpf_reg_state *reg = reg_state(env, regno);
4357 struct bpf_func_state *state = func(env, reg);
4358 int err, min_off, max_off, i, j, slot, spi;
4359 char *err_extra = type == ACCESS_HELPER ? " indirect" : "";
4360 enum bpf_access_type bounds_check_type;
4361
4362
4363
4364 bool clobber = false;
4365
4366 if (access_size == 0 && !zero_size_allowed) {
4367 verbose(env, "invalid zero-sized read\n");
4368 return -EACCES;
4369 }
4370
4371 if (type == ACCESS_HELPER) {
4372
4373
4374
4375
4376 bounds_check_type = BPF_WRITE;
4377 clobber = true;
4378 } else {
4379 bounds_check_type = BPF_READ;
4380 }
4381 err = check_stack_access_within_bounds(env, regno, off, access_size,
4382 type, bounds_check_type);
4383 if (err)
4384 return err;
4385
4386
4387 if (tnum_is_const(reg->var_off)) {
4388 min_off = max_off = reg->var_off.value + off;
4389 } else {
4390
4391
4392
4393
4394
4395 if (!env->bypass_spec_v1) {
4396 char tn_buf[48];
4397
4398 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
4399 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n",
4400 regno, err_extra, tn_buf);
4401 return -EACCES;
4402 }
4403
4404
4405
4406
4407
4408
4409 if (meta && meta->raw_mode)
4410 meta = NULL;
4411
4412 min_off = reg->smin_value + off;
4413 max_off = reg->smax_value + off;
4414 }
4415
4416 if (meta && meta->raw_mode) {
4417 meta->access_size = access_size;
4418 meta->regno = regno;
4419 return 0;
4420 }
4421
4422 for (i = min_off; i < max_off + access_size; i++) {
4423 u8 *stype;
4424
4425 slot = -i - 1;
4426 spi = slot / BPF_REG_SIZE;
4427 if (state->allocated_stack <= slot)
4428 goto err;
4429 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
4430 if (*stype == STACK_MISC)
4431 goto mark;
4432 if (*stype == STACK_ZERO) {
4433 if (clobber) {
4434
4435 *stype = STACK_MISC;
4436 }
4437 goto mark;
4438 }
4439
4440 if (state->stack[spi].slot_type[0] == STACK_SPILL &&
4441 state->stack[spi].spilled_ptr.type == PTR_TO_BTF_ID)
4442 goto mark;
4443
4444 if (state->stack[spi].slot_type[0] == STACK_SPILL &&
4445 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||
4446 env->allow_ptr_leaks)) {
4447 if (clobber) {
4448 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
4449 for (j = 0; j < BPF_REG_SIZE; j++)
4450 state->stack[spi].slot_type[j] = STACK_MISC;
4451 }
4452 goto mark;
4453 }
4454
4455err:
4456 if (tnum_is_const(reg->var_off)) {
4457 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n",
4458 err_extra, regno, min_off, i - min_off, access_size);
4459 } else {
4460 char tn_buf[48];
4461
4462 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
4463 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n",
4464 err_extra, regno, tn_buf, i - min_off, access_size);
4465 }
4466 return -EACCES;
4467mark:
4468
4469
4470
4471 mark_reg_read(env, &state->stack[spi].spilled_ptr,
4472 state->stack[spi].spilled_ptr.parent,
4473 REG_LIVE_READ64);
4474 }
4475 return update_stack_depth(env, state, min_off);
4476}
4477
4478static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
4479 int access_size, bool zero_size_allowed,
4480 struct bpf_call_arg_meta *meta)
4481{
4482 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
4483
4484 switch (reg->type) {
4485 case PTR_TO_PACKET:
4486 case PTR_TO_PACKET_META:
4487 return check_packet_access(env, regno, reg->off, access_size,
4488 zero_size_allowed);
4489 case PTR_TO_MAP_KEY:
4490 return check_mem_region_access(env, regno, reg->off, access_size,
4491 reg->map_ptr->key_size, false);
4492 case PTR_TO_MAP_VALUE:
4493 if (check_map_access_type(env, regno, reg->off, access_size,
4494 meta && meta->raw_mode ? BPF_WRITE :
4495 BPF_READ))
4496 return -EACCES;
4497 return check_map_access(env, regno, reg->off, access_size,
4498 zero_size_allowed);
4499 case PTR_TO_MEM:
4500 return check_mem_region_access(env, regno, reg->off,
4501 access_size, reg->mem_size,
4502 zero_size_allowed);
4503 case PTR_TO_RDONLY_BUF:
4504 if (meta && meta->raw_mode)
4505 return -EACCES;
4506 return check_buffer_access(env, reg, regno, reg->off,
4507 access_size, zero_size_allowed,
4508 "rdonly",
4509 &env->prog->aux->max_rdonly_access);
4510 case PTR_TO_RDWR_BUF:
4511 return check_buffer_access(env, reg, regno, reg->off,
4512 access_size, zero_size_allowed,
4513 "rdwr",
4514 &env->prog->aux->max_rdwr_access);
4515 case PTR_TO_STACK:
4516 return check_stack_range_initialized(
4517 env,
4518 regno, reg->off, access_size,
4519 zero_size_allowed, ACCESS_HELPER, meta);
4520 default:
4521
4522 if (zero_size_allowed && access_size == 0 &&
4523 register_is_null(reg))
4524 return 0;
4525
4526 verbose(env, "R%d type=%s expected=%s\n", regno,
4527 reg_type_str[reg->type],
4528 reg_type_str[PTR_TO_STACK]);
4529 return -EACCES;
4530 }
4531}
4532
4533int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
4534 u32 regno, u32 mem_size)
4535{
4536 if (register_is_null(reg))
4537 return 0;
4538
4539 if (reg_type_may_be_null(reg->type)) {
4540
4541
4542
4543
4544 const struct bpf_reg_state saved_reg = *reg;
4545 int rv;
4546
4547 mark_ptr_not_null_reg(reg);
4548 rv = check_helper_mem_access(env, regno, mem_size, true, NULL);
4549 *reg = saved_reg;
4550 return rv;
4551 }
4552
4553 return check_helper_mem_access(env, regno, mem_size, true, NULL);
4554}
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575static int process_spin_lock(struct bpf_verifier_env *env, int regno,
4576 bool is_lock)
4577{
4578 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
4579 struct bpf_verifier_state *cur = env->cur_state;
4580 bool is_const = tnum_is_const(reg->var_off);
4581 struct bpf_map *map = reg->map_ptr;
4582 u64 val = reg->var_off.value;
4583
4584 if (!is_const) {
4585 verbose(env,
4586 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n",
4587 regno);
4588 return -EINVAL;
4589 }
4590 if (!map->btf) {
4591 verbose(env,
4592 "map '%s' has to have BTF in order to use bpf_spin_lock\n",
4593 map->name);
4594 return -EINVAL;
4595 }
4596 if (!map_value_has_spin_lock(map)) {
4597 if (map->spin_lock_off == -E2BIG)
4598 verbose(env,
4599 "map '%s' has more than one 'struct bpf_spin_lock'\n",
4600 map->name);
4601 else if (map->spin_lock_off == -ENOENT)
4602 verbose(env,
4603 "map '%s' doesn't have 'struct bpf_spin_lock'\n",
4604 map->name);
4605 else
4606 verbose(env,
4607 "map '%s' is not a struct type or bpf_spin_lock is mangled\n",
4608 map->name);
4609 return -EINVAL;
4610 }
4611 if (map->spin_lock_off != val + reg->off) {
4612 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n",
4613 val + reg->off);
4614 return -EINVAL;
4615 }
4616 if (is_lock) {
4617 if (cur->active_spin_lock) {
4618 verbose(env,
4619 "Locking two bpf_spin_locks are not allowed\n");
4620 return -EINVAL;
4621 }
4622 cur->active_spin_lock = reg->id;
4623 } else {
4624 if (!cur->active_spin_lock) {
4625 verbose(env, "bpf_spin_unlock without taking a lock\n");
4626 return -EINVAL;
4627 }
4628 if (cur->active_spin_lock != reg->id) {
4629 verbose(env, "bpf_spin_unlock of different lock\n");
4630 return -EINVAL;
4631 }
4632 cur->active_spin_lock = 0;
4633 }
4634 return 0;
4635}
4636
4637static bool arg_type_is_mem_ptr(enum bpf_arg_type type)
4638{
4639 return type == ARG_PTR_TO_MEM ||
4640 type == ARG_PTR_TO_MEM_OR_NULL ||
4641 type == ARG_PTR_TO_UNINIT_MEM;
4642}
4643
4644static bool arg_type_is_mem_size(enum bpf_arg_type type)
4645{
4646 return type == ARG_CONST_SIZE ||
4647 type == ARG_CONST_SIZE_OR_ZERO;
4648}
4649
4650static bool arg_type_is_alloc_size(enum bpf_arg_type type)
4651{
4652 return type == ARG_CONST_ALLOC_SIZE_OR_ZERO;
4653}
4654
4655static bool arg_type_is_int_ptr(enum bpf_arg_type type)
4656{
4657 return type == ARG_PTR_TO_INT ||
4658 type == ARG_PTR_TO_LONG;
4659}
4660
4661static int int_ptr_type_to_size(enum bpf_arg_type type)
4662{
4663 if (type == ARG_PTR_TO_INT)
4664 return sizeof(u32);
4665 else if (type == ARG_PTR_TO_LONG)
4666 return sizeof(u64);
4667
4668 return -EINVAL;
4669}
4670
4671static int resolve_map_arg_type(struct bpf_verifier_env *env,
4672 const struct bpf_call_arg_meta *meta,
4673 enum bpf_arg_type *arg_type)
4674{
4675 if (!meta->map_ptr) {
4676
4677 verbose(env, "invalid map_ptr to access map->type\n");
4678 return -EACCES;
4679 }
4680
4681 switch (meta->map_ptr->map_type) {
4682 case BPF_MAP_TYPE_SOCKMAP:
4683 case BPF_MAP_TYPE_SOCKHASH:
4684 if (*arg_type == ARG_PTR_TO_MAP_VALUE) {
4685 *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON;
4686 } else {
4687 verbose(env, "invalid arg_type for sockmap/sockhash\n");
4688 return -EINVAL;
4689 }
4690 break;
4691
4692 default:
4693 break;
4694 }
4695 return 0;
4696}
4697
4698struct bpf_reg_types {
4699 const enum bpf_reg_type types[10];
4700 u32 *btf_id;
4701};
4702
4703static const struct bpf_reg_types map_key_value_types = {
4704 .types = {
4705 PTR_TO_STACK,
4706 PTR_TO_PACKET,
4707 PTR_TO_PACKET_META,
4708 PTR_TO_MAP_KEY,
4709 PTR_TO_MAP_VALUE,
4710 },
4711};
4712
4713static const struct bpf_reg_types sock_types = {
4714 .types = {
4715 PTR_TO_SOCK_COMMON,
4716 PTR_TO_SOCKET,
4717 PTR_TO_TCP_SOCK,
4718 PTR_TO_XDP_SOCK,
4719 },
4720};
4721
4722#ifdef CONFIG_NET
4723static const struct bpf_reg_types btf_id_sock_common_types = {
4724 .types = {
4725 PTR_TO_SOCK_COMMON,
4726 PTR_TO_SOCKET,
4727 PTR_TO_TCP_SOCK,
4728 PTR_TO_XDP_SOCK,
4729 PTR_TO_BTF_ID,
4730 },
4731 .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
4732};
4733#endif
4734
4735static const struct bpf_reg_types mem_types = {
4736 .types = {
4737 PTR_TO_STACK,
4738 PTR_TO_PACKET,
4739 PTR_TO_PACKET_META,
4740 PTR_TO_MAP_KEY,
4741 PTR_TO_MAP_VALUE,
4742 PTR_TO_MEM,
4743 PTR_TO_RDONLY_BUF,
4744 PTR_TO_RDWR_BUF,
4745 },
4746};
4747
4748static const struct bpf_reg_types int_ptr_types = {
4749 .types = {
4750 PTR_TO_STACK,
4751 PTR_TO_PACKET,
4752 PTR_TO_PACKET_META,
4753 PTR_TO_MAP_KEY,
4754 PTR_TO_MAP_VALUE,
4755 },
4756};
4757
4758static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } };
4759static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } };
4760static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } };
4761static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM } };
4762static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
4763static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } };
4764static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } };
4765static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } };
4766static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } };
4767static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } };
4768static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } };
4769
4770static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
4771 [ARG_PTR_TO_MAP_KEY] = &map_key_value_types,
4772 [ARG_PTR_TO_MAP_VALUE] = &map_key_value_types,
4773 [ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types,
4774 [ARG_PTR_TO_MAP_VALUE_OR_NULL] = &map_key_value_types,
4775 [ARG_CONST_SIZE] = &scalar_types,
4776 [ARG_CONST_SIZE_OR_ZERO] = &scalar_types,
4777 [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types,
4778 [ARG_CONST_MAP_PTR] = &const_map_ptr_types,
4779 [ARG_PTR_TO_CTX] = &context_types,
4780 [ARG_PTR_TO_CTX_OR_NULL] = &context_types,
4781 [ARG_PTR_TO_SOCK_COMMON] = &sock_types,
4782#ifdef CONFIG_NET
4783 [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,
4784#endif
4785 [ARG_PTR_TO_SOCKET] = &fullsock_types,
4786 [ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,
4787 [ARG_PTR_TO_BTF_ID] = &btf_ptr_types,
4788 [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types,
4789 [ARG_PTR_TO_MEM] = &mem_types,
4790 [ARG_PTR_TO_MEM_OR_NULL] = &mem_types,
4791 [ARG_PTR_TO_UNINIT_MEM] = &mem_types,
4792 [ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types,
4793 [ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types,
4794 [ARG_PTR_TO_INT] = &int_ptr_types,
4795 [ARG_PTR_TO_LONG] = &int_ptr_types,
4796 [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types,
4797 [ARG_PTR_TO_FUNC] = &func_ptr_types,
4798 [ARG_PTR_TO_STACK_OR_NULL] = &stack_ptr_types,
4799 [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types,
4800};
4801
4802static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
4803 enum bpf_arg_type arg_type,
4804 const u32 *arg_btf_id)
4805{
4806 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
4807 enum bpf_reg_type expected, type = reg->type;
4808 const struct bpf_reg_types *compatible;
4809 int i, j;
4810
4811 compatible = compatible_reg_types[arg_type];
4812 if (!compatible) {
4813 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
4814 return -EFAULT;
4815 }
4816
4817 for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
4818 expected = compatible->types[i];
4819 if (expected == NOT_INIT)
4820 break;
4821
4822 if (type == expected)
4823 goto found;
4824 }
4825
4826 verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]);
4827 for (j = 0; j + 1 < i; j++)
4828 verbose(env, "%s, ", reg_type_str[compatible->types[j]]);
4829 verbose(env, "%s\n", reg_type_str[compatible->types[j]]);
4830 return -EACCES;
4831
4832found:
4833 if (type == PTR_TO_BTF_ID) {
4834 if (!arg_btf_id) {
4835 if (!compatible->btf_id) {
4836 verbose(env, "verifier internal error: missing arg compatible BTF ID\n");
4837 return -EFAULT;
4838 }
4839 arg_btf_id = compatible->btf_id;
4840 }
4841
4842 if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
4843 btf_vmlinux, *arg_btf_id)) {
4844 verbose(env, "R%d is of type %s but %s is expected\n",
4845 regno, kernel_type_name(reg->btf, reg->btf_id),
4846 kernel_type_name(btf_vmlinux, *arg_btf_id));
4847 return -EACCES;
4848 }
4849
4850 if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
4851 verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n",
4852 regno);
4853 return -EACCES;
4854 }
4855 }
4856
4857 return 0;
4858}
4859
4860static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
4861 struct bpf_call_arg_meta *meta,
4862 const struct bpf_func_proto *fn)
4863{
4864 u32 regno = BPF_REG_1 + arg;
4865 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
4866 enum bpf_arg_type arg_type = fn->arg_type[arg];
4867 enum bpf_reg_type type = reg->type;
4868 int err = 0;
4869
4870 if (arg_type == ARG_DONTCARE)
4871 return 0;
4872
4873 err = check_reg_arg(env, regno, SRC_OP);
4874 if (err)
4875 return err;
4876
4877 if (arg_type == ARG_ANYTHING) {
4878 if (is_pointer_value(env, regno)) {
4879 verbose(env, "R%d leaks addr into helper function\n",
4880 regno);
4881 return -EACCES;
4882 }
4883 return 0;
4884 }
4885
4886 if (type_is_pkt_pointer(type) &&
4887 !may_access_direct_pkt_data(env, meta, BPF_READ)) {
4888 verbose(env, "helper access to the packet is not allowed\n");
4889 return -EACCES;
4890 }
4891
4892 if (arg_type == ARG_PTR_TO_MAP_VALUE ||
4893 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE ||
4894 arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) {
4895 err = resolve_map_arg_type(env, meta, &arg_type);
4896 if (err)
4897 return err;
4898 }
4899
4900 if (register_is_null(reg) && arg_type_may_be_null(arg_type))
4901
4902
4903
4904 goto skip_type_check;
4905
4906 err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg]);
4907 if (err)
4908 return err;
4909
4910 if (type == PTR_TO_CTX) {
4911 err = check_ctx_reg(env, reg, regno);
4912 if (err < 0)
4913 return err;
4914 }
4915
4916skip_type_check:
4917 if (reg->ref_obj_id) {
4918 if (meta->ref_obj_id) {
4919 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
4920 regno, reg->ref_obj_id,
4921 meta->ref_obj_id);
4922 return -EFAULT;
4923 }
4924 meta->ref_obj_id = reg->ref_obj_id;
4925 }
4926
4927 if (arg_type == ARG_CONST_MAP_PTR) {
4928
4929 meta->map_ptr = reg->map_ptr;
4930 } else if (arg_type == ARG_PTR_TO_MAP_KEY) {
4931
4932
4933
4934
4935 if (!meta->map_ptr) {
4936
4937
4938
4939
4940
4941 verbose(env, "invalid map_ptr to access map->key\n");
4942 return -EACCES;
4943 }
4944 err = check_helper_mem_access(env, regno,
4945 meta->map_ptr->key_size, false,
4946 NULL);
4947 } else if (arg_type == ARG_PTR_TO_MAP_VALUE ||
4948 (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL &&
4949 !register_is_null(reg)) ||
4950 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
4951
4952
4953
4954 if (!meta->map_ptr) {
4955
4956 verbose(env, "invalid map_ptr to access map->value\n");
4957 return -EACCES;
4958 }
4959 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE);
4960 err = check_helper_mem_access(env, regno,
4961 meta->map_ptr->value_size, false,
4962 meta);
4963 } else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) {
4964 if (!reg->btf_id) {
4965 verbose(env, "Helper has invalid btf_id in R%d\n", regno);
4966 return -EACCES;
4967 }
4968 meta->ret_btf = reg->btf;
4969 meta->ret_btf_id = reg->btf_id;
4970 } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
4971 if (meta->func_id == BPF_FUNC_spin_lock) {
4972 if (process_spin_lock(env, regno, true))
4973 return -EACCES;
4974 } else if (meta->func_id == BPF_FUNC_spin_unlock) {
4975 if (process_spin_lock(env, regno, false))
4976 return -EACCES;
4977 } else {
4978 verbose(env, "verifier internal error\n");
4979 return -EFAULT;
4980 }
4981 } else if (arg_type == ARG_PTR_TO_FUNC) {
4982 meta->subprogno = reg->subprogno;
4983 } else if (arg_type_is_mem_ptr(arg_type)) {
4984
4985
4986
4987 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MEM);
4988 } else if (arg_type_is_mem_size(arg_type)) {
4989 bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO);
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999 meta->msize_max_value = reg->umax_value;
5000
5001
5002
5003
5004 if (!tnum_is_const(reg->var_off))
5005
5006
5007
5008
5009
5010 meta = NULL;
5011
5012 if (reg->smin_value < 0) {
5013 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",
5014 regno);
5015 return -EACCES;
5016 }
5017
5018 if (reg->umin_value == 0) {
5019 err = check_helper_mem_access(env, regno - 1, 0,
5020 zero_size_allowed,
5021 meta);
5022 if (err)
5023 return err;
5024 }
5025
5026 if (reg->umax_value >= BPF_MAX_VAR_SIZ) {
5027 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
5028 regno);
5029 return -EACCES;
5030 }
5031 err = check_helper_mem_access(env, regno - 1,
5032 reg->umax_value,
5033 zero_size_allowed, meta);
5034 if (!err)
5035 err = mark_chain_precision(env, regno);
5036 } else if (arg_type_is_alloc_size(arg_type)) {
5037 if (!tnum_is_const(reg->var_off)) {
5038 verbose(env, "R%d is not a known constant'\n",
5039 regno);
5040 return -EACCES;
5041 }
5042 meta->mem_size = reg->var_off.value;
5043 } else if (arg_type_is_int_ptr(arg_type)) {
5044 int size = int_ptr_type_to_size(arg_type);
5045
5046 err = check_helper_mem_access(env, regno, size, false, meta);
5047 if (err)
5048 return err;
5049 err = check_ptr_alignment(env, reg, 0, size, true);
5050 } else if (arg_type == ARG_PTR_TO_CONST_STR) {
5051 struct bpf_map *map = reg->map_ptr;
5052 int map_off;
5053 u64 map_addr;
5054 char *str_ptr;
5055
5056 if (!bpf_map_is_rdonly(map)) {
5057 verbose(env, "R%d does not point to a readonly map'\n", regno);
5058 return -EACCES;
5059 }
5060
5061 if (!tnum_is_const(reg->var_off)) {
5062 verbose(env, "R%d is not a constant address'\n", regno);
5063 return -EACCES;
5064 }
5065
5066 if (!map->ops->map_direct_value_addr) {
5067 verbose(env, "no direct value access support for this map type\n");
5068 return -EACCES;
5069 }
5070
5071 err = check_map_access(env, regno, reg->off,
5072 map->value_size - reg->off, false);
5073 if (err)
5074 return err;
5075
5076 map_off = reg->off + reg->var_off.value;
5077 err = map->ops->map_direct_value_addr(map, &map_addr, map_off);
5078 if (err) {
5079 verbose(env, "direct value access on string failed\n");
5080 return err;
5081 }
5082
5083 str_ptr = (char *)(long)(map_addr);
5084 if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) {
5085 verbose(env, "string is not zero-terminated\n");
5086 return -EINVAL;
5087 }
5088 }
5089
5090 return err;
5091}
5092
5093static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
5094{
5095 enum bpf_attach_type eatype = env->prog->expected_attach_type;
5096 enum bpf_prog_type type = resolve_prog_type(env->prog);
5097
5098 if (func_id != BPF_FUNC_map_update_elem)
5099 return false;
5100
5101
5102
5103
5104 switch (type) {
5105 case BPF_PROG_TYPE_TRACING:
5106 if (eatype == BPF_TRACE_ITER)
5107 return true;
5108 break;
5109 case BPF_PROG_TYPE_SOCKET_FILTER:
5110 case BPF_PROG_TYPE_SCHED_CLS:
5111 case BPF_PROG_TYPE_SCHED_ACT:
5112 case BPF_PROG_TYPE_XDP:
5113 case BPF_PROG_TYPE_SK_REUSEPORT:
5114 case BPF_PROG_TYPE_FLOW_DISSECTOR:
5115 case BPF_PROG_TYPE_SK_LOOKUP:
5116 return true;
5117 default:
5118 break;
5119 }
5120
5121 verbose(env, "cannot update sockmap in this context\n");
5122 return false;
5123}
5124
5125static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env)
5126{
5127 return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64);
5128}
5129
5130static int check_map_func_compatibility(struct bpf_verifier_env *env,
5131 struct bpf_map *map, int func_id)
5132{
5133 if (!map)
5134 return 0;
5135
5136
5137 switch (map->map_type) {
5138 case BPF_MAP_TYPE_PROG_ARRAY:
5139 if (func_id != BPF_FUNC_tail_call)
5140 goto error;
5141 break;
5142 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
5143 if (func_id != BPF_FUNC_perf_event_read &&
5144 func_id != BPF_FUNC_perf_event_output &&
5145 func_id != BPF_FUNC_skb_output &&
5146 func_id != BPF_FUNC_perf_event_read_value &&
5147 func_id != BPF_FUNC_xdp_output)
5148 goto error;
5149 break;
5150 case BPF_MAP_TYPE_RINGBUF:
5151 if (func_id != BPF_FUNC_ringbuf_output &&
5152 func_id != BPF_FUNC_ringbuf_reserve &&
5153 func_id != BPF_FUNC_ringbuf_query)
5154 goto error;
5155 break;
5156 case BPF_MAP_TYPE_STACK_TRACE:
5157 if (func_id != BPF_FUNC_get_stackid)
5158 goto error;
5159 break;
5160 case BPF_MAP_TYPE_CGROUP_ARRAY:
5161 if (func_id != BPF_FUNC_skb_under_cgroup &&
5162 func_id != BPF_FUNC_current_task_under_cgroup)
5163 goto error;
5164 break;
5165 case BPF_MAP_TYPE_CGROUP_STORAGE:
5166 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
5167 if (func_id != BPF_FUNC_get_local_storage)
5168 goto error;
5169 break;
5170 case BPF_MAP_TYPE_DEVMAP:
5171 case BPF_MAP_TYPE_DEVMAP_HASH:
5172 if (func_id != BPF_FUNC_redirect_map &&
5173 func_id != BPF_FUNC_map_lookup_elem)
5174 goto error;
5175 break;
5176
5177
5178
5179 case BPF_MAP_TYPE_CPUMAP:
5180 if (func_id != BPF_FUNC_redirect_map)
5181 goto error;
5182 break;
5183 case BPF_MAP_TYPE_XSKMAP:
5184 if (func_id != BPF_FUNC_redirect_map &&
5185 func_id != BPF_FUNC_map_lookup_elem)
5186 goto error;
5187 break;
5188 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
5189 case BPF_MAP_TYPE_HASH_OF_MAPS:
5190 if (func_id != BPF_FUNC_map_lookup_elem)
5191 goto error;
5192 break;
5193 case BPF_MAP_TYPE_SOCKMAP:
5194 if (func_id != BPF_FUNC_sk_redirect_map &&
5195 func_id != BPF_FUNC_sock_map_update &&
5196 func_id != BPF_FUNC_map_delete_elem &&
5197 func_id != BPF_FUNC_msg_redirect_map &&
5198 func_id != BPF_FUNC_sk_select_reuseport &&
5199 func_id != BPF_FUNC_map_lookup_elem &&
5200 !may_update_sockmap(env, func_id))
5201 goto error;
5202 break;
5203 case BPF_MAP_TYPE_SOCKHASH:
5204 if (func_id != BPF_FUNC_sk_redirect_hash &&
5205 func_id != BPF_FUNC_sock_hash_update &&
5206 func_id != BPF_FUNC_map_delete_elem &&
5207 func_id != BPF_FUNC_msg_redirect_hash &&
5208 func_id != BPF_FUNC_sk_select_reuseport &&
5209 func_id != BPF_FUNC_map_lookup_elem &&
5210 !may_update_sockmap(env, func_id))
5211 goto error;
5212 break;
5213 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
5214 if (func_id != BPF_FUNC_sk_select_reuseport)
5215 goto error;
5216 break;
5217 case BPF_MAP_TYPE_QUEUE:
5218 case BPF_MAP_TYPE_STACK:
5219 if (func_id != BPF_FUNC_map_peek_elem &&
5220 func_id != BPF_FUNC_map_pop_elem &&
5221 func_id != BPF_FUNC_map_push_elem)
5222 goto error;
5223 break;
5224 case BPF_MAP_TYPE_SK_STORAGE:
5225 if (func_id != BPF_FUNC_sk_storage_get &&
5226 func_id != BPF_FUNC_sk_storage_delete)
5227 goto error;
5228 break;
5229 case BPF_MAP_TYPE_INODE_STORAGE:
5230 if (func_id != BPF_FUNC_inode_storage_get &&
5231 func_id != BPF_FUNC_inode_storage_delete)
5232 goto error;
5233 break;
5234 case BPF_MAP_TYPE_TASK_STORAGE:
5235 if (func_id != BPF_FUNC_task_storage_get &&
5236 func_id != BPF_FUNC_task_storage_delete)
5237 goto error;
5238 break;
5239 default:
5240 break;
5241 }
5242
5243
5244 switch (func_id) {
5245 case BPF_FUNC_tail_call:
5246 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
5247 goto error;
5248 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {
5249 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
5250 return -EINVAL;
5251 }
5252 break;
5253 case BPF_FUNC_perf_event_read:
5254 case BPF_FUNC_perf_event_output:
5255 case BPF_FUNC_perf_event_read_value:
5256 case BPF_FUNC_skb_output:
5257 case BPF_FUNC_xdp_output:
5258 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
5259 goto error;
5260 break;
5261 case BPF_FUNC_ringbuf_output:
5262 case BPF_FUNC_ringbuf_reserve:
5263 case BPF_FUNC_ringbuf_query:
5264 if (map->map_type != BPF_MAP_TYPE_RINGBUF)
5265 goto error;
5266 break;
5267 case BPF_FUNC_get_stackid:
5268 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
5269 goto error;
5270 break;
5271 case BPF_FUNC_current_task_under_cgroup:
5272 case BPF_FUNC_skb_under_cgroup:
5273 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY)
5274 goto error;
5275 break;
5276 case BPF_FUNC_redirect_map:
5277 if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
5278 map->map_type != BPF_MAP_TYPE_DEVMAP_HASH &&
5279 map->map_type != BPF_MAP_TYPE_CPUMAP &&
5280 map->map_type != BPF_MAP_TYPE_XSKMAP)
5281 goto error;
5282 break;
5283 case BPF_FUNC_sk_redirect_map:
5284 case BPF_FUNC_msg_redirect_map:
5285 case BPF_FUNC_sock_map_update:
5286 if (map->map_type != BPF_MAP_TYPE_SOCKMAP)
5287 goto error;
5288 break;
5289 case BPF_FUNC_sk_redirect_hash:
5290 case BPF_FUNC_msg_redirect_hash:
5291 case BPF_FUNC_sock_hash_update:
5292 if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
5293 goto error;
5294 break;
5295 case BPF_FUNC_get_local_storage:
5296 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
5297 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
5298 goto error;
5299 break;
5300 case BPF_FUNC_sk_select_reuseport:
5301 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY &&
5302 map->map_type != BPF_MAP_TYPE_SOCKMAP &&
5303 map->map_type != BPF_MAP_TYPE_SOCKHASH)
5304 goto error;
5305 break;
5306 case BPF_FUNC_map_peek_elem:
5307 case BPF_FUNC_map_pop_elem:
5308 case BPF_FUNC_map_push_elem:
5309 if (map->map_type != BPF_MAP_TYPE_QUEUE &&
5310 map->map_type != BPF_MAP_TYPE_STACK)
5311 goto error;
5312 break;
5313 case BPF_FUNC_sk_storage_get:
5314 case BPF_FUNC_sk_storage_delete:
5315 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
5316 goto error;
5317 break;
5318 case BPF_FUNC_inode_storage_get:
5319 case BPF_FUNC_inode_storage_delete:
5320 if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
5321 goto error;
5322 break;
5323 case BPF_FUNC_task_storage_get:
5324 case BPF_FUNC_task_storage_delete:
5325 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
5326 goto error;
5327 break;
5328 default:
5329 break;
5330 }
5331
5332 return 0;
5333error:
5334 verbose(env, "cannot pass map_type %d into func %s#%d\n",
5335 map->map_type, func_id_name(func_id), func_id);
5336 return -EINVAL;
5337}
5338
5339static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
5340{
5341 int count = 0;
5342
5343 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM)
5344 count++;
5345 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM)
5346 count++;
5347 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM)
5348 count++;
5349 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM)
5350 count++;
5351 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM)
5352 count++;
5353
5354
5355
5356
5357
5358 return count <= 1;
5359}
5360
5361static bool check_args_pair_invalid(enum bpf_arg_type arg_curr,
5362 enum bpf_arg_type arg_next)
5363{
5364 return (arg_type_is_mem_ptr(arg_curr) &&
5365 !arg_type_is_mem_size(arg_next)) ||
5366 (!arg_type_is_mem_ptr(arg_curr) &&
5367 arg_type_is_mem_size(arg_next));
5368}
5369
5370static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
5371{
5372
5373
5374
5375
5376
5377 if (arg_type_is_mem_size(fn->arg1_type) ||
5378 arg_type_is_mem_ptr(fn->arg5_type) ||
5379 check_args_pair_invalid(fn->arg1_type, fn->arg2_type) ||
5380 check_args_pair_invalid(fn->arg2_type, fn->arg3_type) ||
5381 check_args_pair_invalid(fn->arg3_type, fn->arg4_type) ||
5382 check_args_pair_invalid(fn->arg4_type, fn->arg5_type))
5383 return false;
5384
5385 return true;
5386}
5387
5388static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id)
5389{
5390 int count = 0;
5391
5392 if (arg_type_may_be_refcounted(fn->arg1_type))
5393 count++;
5394 if (arg_type_may_be_refcounted(fn->arg2_type))
5395 count++;
5396 if (arg_type_may_be_refcounted(fn->arg3_type))
5397 count++;
5398 if (arg_type_may_be_refcounted(fn->arg4_type))
5399 count++;
5400 if (arg_type_may_be_refcounted(fn->arg5_type))
5401 count++;
5402
5403
5404
5405
5406 if (may_be_acquire_function(func_id) && count)
5407 return false;
5408
5409
5410
5411
5412 return count <= 1;
5413}
5414
5415static bool check_btf_id_ok(const struct bpf_func_proto *fn)
5416{
5417 int i;
5418
5419 for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
5420 if (fn->arg_type[i] == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i])
5421 return false;
5422
5423 if (fn->arg_type[i] != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i])
5424 return false;
5425 }
5426
5427 return true;
5428}
5429
5430static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
5431{
5432 return check_raw_mode_ok(fn) &&
5433 check_arg_pair_ok(fn) &&
5434 check_btf_id_ok(fn) &&
5435 check_refcount_ok(fn, func_id) ? 0 : -EINVAL;
5436}
5437
5438
5439
5440
5441static void __clear_all_pkt_pointers(struct bpf_verifier_env *env,
5442 struct bpf_func_state *state)
5443{
5444 struct bpf_reg_state *regs = state->regs, *reg;
5445 int i;
5446
5447 for (i = 0; i < MAX_BPF_REG; i++)
5448 if (reg_is_pkt_pointer_any(®s[i]))
5449 mark_reg_unknown(env, regs, i);
5450
5451 bpf_for_each_spilled_reg(i, state, reg) {
5452 if (!reg)
5453 continue;
5454 if (reg_is_pkt_pointer_any(reg))
5455 __mark_reg_unknown(env, reg);
5456 }
5457}
5458
5459static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
5460{
5461 struct bpf_verifier_state *vstate = env->cur_state;
5462 int i;
5463
5464 for (i = 0; i <= vstate->curframe; i++)
5465 __clear_all_pkt_pointers(env, vstate->frame[i]);
5466}
5467
5468enum {
5469 AT_PKT_END = -1,
5470 BEYOND_PKT_END = -2,
5471};
5472
5473static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open)
5474{
5475 struct bpf_func_state *state = vstate->frame[vstate->curframe];
5476 struct bpf_reg_state *reg = &state->regs[regn];
5477
5478 if (reg->type != PTR_TO_PACKET)
5479
5480 return;
5481
5482
5483
5484
5485
5486
5487
5488 if (range_open)
5489 reg->range = BEYOND_PKT_END;
5490 else
5491 reg->range = AT_PKT_END;
5492}
5493
5494static void release_reg_references(struct bpf_verifier_env *env,
5495 struct bpf_func_state *state,
5496 int ref_obj_id)
5497{
5498 struct bpf_reg_state *regs = state->regs, *reg;
5499 int i;
5500
5501 for (i = 0; i < MAX_BPF_REG; i++)
5502 if (regs[i].ref_obj_id == ref_obj_id)
5503 mark_reg_unknown(env, regs, i);
5504
5505 bpf_for_each_spilled_reg(i, state, reg) {
5506 if (!reg)
5507 continue;
5508 if (reg->ref_obj_id == ref_obj_id)
5509 __mark_reg_unknown(env, reg);
5510 }
5511}
5512
5513
5514
5515
5516static int release_reference(struct bpf_verifier_env *env,
5517 int ref_obj_id)
5518{
5519 struct bpf_verifier_state *vstate = env->cur_state;
5520 int err;
5521 int i;
5522
5523 err = release_reference_state(cur_func(env), ref_obj_id);
5524 if (err)
5525 return err;
5526
5527 for (i = 0; i <= vstate->curframe; i++)
5528 release_reg_references(env, vstate->frame[i], ref_obj_id);
5529
5530 return 0;
5531}
5532
5533static void clear_caller_saved_regs(struct bpf_verifier_env *env,
5534 struct bpf_reg_state *regs)
5535{
5536 int i;
5537
5538
5539 for (i = 0; i < CALLER_SAVED_REGS; i++) {
5540 mark_reg_not_init(env, regs, caller_saved[i]);
5541 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
5542 }
5543}
5544
5545typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
5546 struct bpf_func_state *caller,
5547 struct bpf_func_state *callee,
5548 int insn_idx);
5549
5550static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
5551 int *insn_idx, int subprog,
5552 set_callee_state_fn set_callee_state_cb)
5553{
5554 struct bpf_verifier_state *state = env->cur_state;
5555 struct bpf_func_info_aux *func_info_aux;
5556 struct bpf_func_state *caller, *callee;
5557 int err;
5558 bool is_global = false;
5559
5560 if (state->curframe + 1 >= MAX_CALL_FRAMES) {
5561 verbose(env, "the call stack of %d frames is too deep\n",
5562 state->curframe + 2);
5563 return -E2BIG;
5564 }
5565
5566 caller = state->frame[state->curframe];
5567 if (state->frame[state->curframe + 1]) {
5568 verbose(env, "verifier bug. Frame %d already allocated\n",
5569 state->curframe + 1);
5570 return -EFAULT;
5571 }
5572
5573 func_info_aux = env->prog->aux->func_info_aux;
5574 if (func_info_aux)
5575 is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5576 err = btf_check_subprog_arg_match(env, subprog, caller->regs);
5577 if (err == -EFAULT)
5578 return err;
5579 if (is_global) {
5580 if (err) {
5581 verbose(env, "Caller passes invalid args into func#%d\n",
5582 subprog);
5583 return err;
5584 } else {
5585 if (env->log.level & BPF_LOG_LEVEL)
5586 verbose(env,
5587 "Func#%d is global and valid. Skipping.\n",
5588 subprog);
5589 clear_caller_saved_regs(env, caller->regs);
5590
5591
5592 mark_reg_unknown(env, caller->regs, BPF_REG_0);
5593 caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
5594
5595
5596 return 0;
5597 }
5598 }
5599
5600 callee = kzalloc(sizeof(*callee), GFP_KERNEL);
5601 if (!callee)
5602 return -ENOMEM;
5603 state->frame[state->curframe + 1] = callee;
5604
5605
5606
5607
5608
5609 init_func_state(env, callee,
5610
5611 *insn_idx ,
5612 state->curframe + 1 ,
5613 subprog );
5614
5615
5616 err = copy_reference_state(callee, caller);
5617 if (err)
5618 return err;
5619
5620 err = set_callee_state_cb(env, caller, callee, *insn_idx);
5621 if (err)
5622 return err;
5623
5624 clear_caller_saved_regs(env, caller->regs);
5625
5626
5627 state->curframe++;
5628
5629
5630 *insn_idx = env->subprog_info[subprog].start - 1;
5631
5632 if (env->log.level & BPF_LOG_LEVEL) {
5633 verbose(env, "caller:\n");
5634 print_verifier_state(env, caller);
5635 verbose(env, "callee:\n");
5636 print_verifier_state(env, callee);
5637 }
5638 return 0;
5639}
5640
5641int map_set_for_each_callback_args(struct bpf_verifier_env *env,
5642 struct bpf_func_state *caller,
5643 struct bpf_func_state *callee)
5644{
5645
5646
5647
5648
5649
5650 callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
5651
5652 callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
5653 __mark_reg_known_zero(&callee->regs[BPF_REG_2]);
5654 callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr;
5655
5656 callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
5657 __mark_reg_known_zero(&callee->regs[BPF_REG_3]);
5658 callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr;
5659
5660
5661 callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];
5662
5663
5664 __mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
5665 return 0;
5666}
5667
5668static int set_callee_state(struct bpf_verifier_env *env,
5669 struct bpf_func_state *caller,
5670 struct bpf_func_state *callee, int insn_idx)
5671{
5672 int i;
5673
5674
5675
5676
5677 for (i = BPF_REG_1; i <= BPF_REG_5; i++)
5678 callee->regs[i] = caller->regs[i];
5679 return 0;
5680}
5681
5682static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
5683 int *insn_idx)
5684{
5685 int subprog, target_insn;
5686
5687 target_insn = *insn_idx + insn->imm + 1;
5688 subprog = find_subprog(env, target_insn);
5689 if (subprog < 0) {
5690 verbose(env, "verifier bug. No program starts at insn %d\n",
5691 target_insn);
5692 return -EFAULT;
5693 }
5694
5695 return __check_func_call(env, insn, insn_idx, subprog, set_callee_state);
5696}
5697
5698static int set_map_elem_callback_state(struct bpf_verifier_env *env,
5699 struct bpf_func_state *caller,
5700 struct bpf_func_state *callee,
5701 int insn_idx)
5702{
5703 struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];
5704 struct bpf_map *map;
5705 int err;
5706
5707 if (bpf_map_ptr_poisoned(insn_aux)) {
5708 verbose(env, "tail_call abusing map_ptr\n");
5709 return -EINVAL;
5710 }
5711
5712 map = BPF_MAP_PTR(insn_aux->map_ptr_state);
5713 if (!map->ops->map_set_for_each_callback_args ||
5714 !map->ops->map_for_each_callback) {
5715 verbose(env, "callback function not allowed for map\n");
5716 return -ENOTSUPP;
5717 }
5718
5719 err = map->ops->map_set_for_each_callback_args(env, caller, callee);
5720 if (err)
5721 return err;
5722
5723 callee->in_callback_fn = true;
5724 return 0;
5725}
5726
5727static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
5728{
5729 struct bpf_verifier_state *state = env->cur_state;
5730 struct bpf_func_state *caller, *callee;
5731 struct bpf_reg_state *r0;
5732 int err;
5733
5734 callee = state->frame[state->curframe];
5735 r0 = &callee->regs[BPF_REG_0];
5736 if (r0->type == PTR_TO_STACK) {
5737
5738
5739
5740
5741
5742
5743 verbose(env, "cannot return stack pointer to the caller\n");
5744 return -EINVAL;
5745 }
5746
5747 state->curframe--;
5748 caller = state->frame[state->curframe];
5749 if (callee->in_callback_fn) {
5750
5751 struct tnum range = tnum_range(0, 1);
5752
5753 if (r0->type != SCALAR_VALUE) {
5754 verbose(env, "R0 not a scalar value\n");
5755 return -EACCES;
5756 }
5757 if (!tnum_in(range, r0->var_off)) {
5758 verbose_invalid_scalar(env, r0, &range, "callback return", "R0");
5759 return -EINVAL;
5760 }
5761 } else {
5762
5763 caller->regs[BPF_REG_0] = *r0;
5764 }
5765
5766
5767 err = copy_reference_state(caller, callee);
5768 if (err)
5769 return err;
5770
5771 *insn_idx = callee->callsite + 1;
5772 if (env->log.level & BPF_LOG_LEVEL) {
5773 verbose(env, "returning from callee:\n");
5774 print_verifier_state(env, callee);
5775 verbose(env, "to caller at %d:\n", *insn_idx);
5776 print_verifier_state(env, caller);
5777 }
5778
5779 free_func_state(callee);
5780 state->frame[state->curframe + 1] = NULL;
5781 return 0;
5782}
5783
5784static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,
5785 int func_id,
5786 struct bpf_call_arg_meta *meta)
5787{
5788 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0];
5789
5790 if (ret_type != RET_INTEGER ||
5791 (func_id != BPF_FUNC_get_stack &&
5792 func_id != BPF_FUNC_get_task_stack &&
5793 func_id != BPF_FUNC_probe_read_str &&
5794 func_id != BPF_FUNC_probe_read_kernel_str &&
5795 func_id != BPF_FUNC_probe_read_user_str))
5796 return;
5797
5798 ret_reg->smax_value = meta->msize_max_value;
5799 ret_reg->s32_max_value = meta->msize_max_value;
5800 ret_reg->smin_value = -MAX_ERRNO;
5801 ret_reg->s32_min_value = -MAX_ERRNO;
5802 __reg_deduce_bounds(ret_reg);
5803 __reg_bound_offset(ret_reg);
5804 __update_reg_bounds(ret_reg);
5805}
5806
5807static int
5808record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
5809 int func_id, int insn_idx)
5810{
5811 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
5812 struct bpf_map *map = meta->map_ptr;
5813
5814 if (func_id != BPF_FUNC_tail_call &&
5815 func_id != BPF_FUNC_map_lookup_elem &&
5816 func_id != BPF_FUNC_map_update_elem &&
5817 func_id != BPF_FUNC_map_delete_elem &&
5818 func_id != BPF_FUNC_map_push_elem &&
5819 func_id != BPF_FUNC_map_pop_elem &&
5820 func_id != BPF_FUNC_map_peek_elem &&
5821 func_id != BPF_FUNC_for_each_map_elem &&
5822 func_id != BPF_FUNC_redirect_map)
5823 return 0;
5824
5825 if (map == NULL) {
5826 verbose(env, "kernel subsystem misconfigured verifier\n");
5827 return -EINVAL;
5828 }
5829
5830
5831
5832
5833
5834 if ((map->map_flags & BPF_F_RDONLY_PROG) &&
5835 (func_id == BPF_FUNC_map_delete_elem ||
5836 func_id == BPF_FUNC_map_update_elem ||
5837 func_id == BPF_FUNC_map_push_elem ||
5838 func_id == BPF_FUNC_map_pop_elem)) {
5839 verbose(env, "write into map forbidden\n");
5840 return -EACCES;
5841 }
5842
5843 if (!BPF_MAP_PTR(aux->map_ptr_state))
5844 bpf_map_ptr_store(aux, meta->map_ptr,
5845 !meta->map_ptr->bypass_spec_v1);
5846 else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr)
5847 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON,
5848 !meta->map_ptr->bypass_spec_v1);
5849 return 0;
5850}
5851
5852static int
5853record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
5854 int func_id, int insn_idx)
5855{
5856 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
5857 struct bpf_reg_state *regs = cur_regs(env), *reg;
5858 struct bpf_map *map = meta->map_ptr;
5859 struct tnum range;
5860 u64 val;
5861 int err;
5862
5863 if (func_id != BPF_FUNC_tail_call)
5864 return 0;
5865 if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) {
5866 verbose(env, "kernel subsystem misconfigured verifier\n");
5867 return -EINVAL;
5868 }
5869
5870 range = tnum_range(0, map->max_entries - 1);
5871 reg = ®s[BPF_REG_3];
5872
5873 if (!register_is_const(reg) || !tnum_in(range, reg->var_off)) {
5874 bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
5875 return 0;
5876 }
5877
5878 err = mark_chain_precision(env, BPF_REG_3);
5879 if (err)
5880 return err;
5881
5882 val = reg->var_off.value;
5883 if (bpf_map_key_unseen(aux))
5884 bpf_map_key_store(aux, val);
5885 else if (!bpf_map_key_poisoned(aux) &&
5886 bpf_map_key_immediate(aux) != val)
5887 bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
5888 return 0;
5889}
5890
5891static int check_reference_leak(struct bpf_verifier_env *env)
5892{
5893 struct bpf_func_state *state = cur_func(env);
5894 int i;
5895
5896 for (i = 0; i < state->acquired_refs; i++) {
5897 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
5898 state->refs[i].id, state->refs[i].insn_idx);
5899 }
5900 return state->acquired_refs ? -EINVAL : 0;
5901}
5902
5903static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
5904 struct bpf_reg_state *regs)
5905{
5906 struct bpf_reg_state *fmt_reg = ®s[BPF_REG_3];
5907 struct bpf_reg_state *data_len_reg = ®s[BPF_REG_5];
5908 struct bpf_map *fmt_map = fmt_reg->map_ptr;
5909 int err, fmt_map_off, num_args;
5910 u64 fmt_addr;
5911 char *fmt;
5912
5913
5914 if (data_len_reg->var_off.value % 8)
5915 return -EINVAL;
5916 num_args = data_len_reg->var_off.value / 8;
5917
5918
5919
5920
5921 fmt_map_off = fmt_reg->off + fmt_reg->var_off.value;
5922 err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr,
5923 fmt_map_off);
5924 if (err) {
5925 verbose(env, "verifier bug\n");
5926 return -EFAULT;
5927 }
5928 fmt = (char *)(long)fmt_addr + fmt_map_off;
5929
5930
5931
5932
5933 err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, NULL, num_args);
5934 if (err < 0)
5935 verbose(env, "Invalid format string\n");
5936
5937 return err;
5938}
5939
5940static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
5941 int *insn_idx_p)
5942{
5943 const struct bpf_func_proto *fn = NULL;
5944 struct bpf_reg_state *regs;
5945 struct bpf_call_arg_meta meta;
5946 int insn_idx = *insn_idx_p;
5947 bool changes_data;
5948 int i, err, func_id;
5949
5950
5951 func_id = insn->imm;
5952 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) {
5953 verbose(env, "invalid func %s#%d\n", func_id_name(func_id),
5954 func_id);
5955 return -EINVAL;
5956 }
5957
5958 if (env->ops->get_func_proto)
5959 fn = env->ops->get_func_proto(func_id, env->prog);
5960 if (!fn) {
5961 verbose(env, "unknown func %s#%d\n", func_id_name(func_id),
5962 func_id);
5963 return -EINVAL;
5964 }
5965
5966
5967 if (!env->prog->gpl_compatible && fn->gpl_only) {
5968 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
5969 return -EINVAL;
5970 }
5971
5972 if (fn->allowed && !fn->allowed(env->prog)) {
5973 verbose(env, "helper call is not allowed in probe\n");
5974 return -EINVAL;
5975 }
5976
5977
5978 changes_data = bpf_helper_changes_pkt_data(fn->func);
5979 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
5980 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",
5981 func_id_name(func_id), func_id);
5982 return -EINVAL;
5983 }
5984
5985 memset(&meta, 0, sizeof(meta));
5986 meta.pkt_access = fn->pkt_access;
5987
5988 err = check_func_proto(fn, func_id);
5989 if (err) {
5990 verbose(env, "kernel subsystem misconfigured func %s#%d\n",
5991 func_id_name(func_id), func_id);
5992 return err;
5993 }
5994
5995 meta.func_id = func_id;
5996
5997 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
5998 err = check_func_arg(env, i, &meta, fn);
5999 if (err)
6000 return err;
6001 }
6002
6003 err = record_func_map(env, &meta, func_id, insn_idx);
6004 if (err)
6005 return err;
6006
6007 err = record_func_key(env, &meta, func_id, insn_idx);
6008 if (err)
6009 return err;
6010
6011
6012
6013
6014 for (i = 0; i < meta.access_size; i++) {
6015 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,
6016 BPF_WRITE, -1, false);
6017 if (err)
6018 return err;
6019 }
6020
6021 if (func_id == BPF_FUNC_tail_call) {
6022 err = check_reference_leak(env);
6023 if (err) {
6024 verbose(env, "tail_call would lead to reference leak\n");
6025 return err;
6026 }
6027 } else if (is_release_function(func_id)) {
6028 err = release_reference(env, meta.ref_obj_id);
6029 if (err) {
6030 verbose(env, "func %s#%d reference has not been acquired before\n",
6031 func_id_name(func_id), func_id);
6032 return err;
6033 }
6034 }
6035
6036 regs = cur_regs(env);
6037
6038
6039
6040
6041 if (func_id == BPF_FUNC_get_local_storage &&
6042 !register_is_null(®s[BPF_REG_2])) {
6043 verbose(env, "get_local_storage() doesn't support non-zero flags\n");
6044 return -EINVAL;
6045 }
6046
6047 if (func_id == BPF_FUNC_for_each_map_elem) {
6048 err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
6049 set_map_elem_callback_state);
6050 if (err < 0)
6051 return -EINVAL;
6052 }
6053
6054 if (func_id == BPF_FUNC_snprintf) {
6055 err = check_bpf_snprintf_call(env, regs);
6056 if (err < 0)
6057 return err;
6058 }
6059
6060
6061 for (i = 0; i < CALLER_SAVED_REGS; i++) {
6062 mark_reg_not_init(env, regs, caller_saved[i]);
6063 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
6064 }
6065
6066
6067 regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
6068
6069
6070 if (fn->ret_type == RET_INTEGER) {
6071
6072 mark_reg_unknown(env, regs, BPF_REG_0);
6073 } else if (fn->ret_type == RET_VOID) {
6074 regs[BPF_REG_0].type = NOT_INIT;
6075 } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL ||
6076 fn->ret_type == RET_PTR_TO_MAP_VALUE) {
6077
6078 mark_reg_known_zero(env, regs, BPF_REG_0);
6079
6080
6081
6082
6083 if (meta.map_ptr == NULL) {
6084 verbose(env,
6085 "kernel subsystem misconfigured verifier\n");
6086 return -EINVAL;
6087 }
6088 regs[BPF_REG_0].map_ptr = meta.map_ptr;
6089 if (fn->ret_type == RET_PTR_TO_MAP_VALUE) {
6090 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE;
6091 if (map_value_has_spin_lock(meta.map_ptr))
6092 regs[BPF_REG_0].id = ++env->id_gen;
6093 } else {
6094 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
6095 }
6096 } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) {
6097 mark_reg_known_zero(env, regs, BPF_REG_0);
6098 regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL;
6099 } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) {
6100 mark_reg_known_zero(env, regs, BPF_REG_0);
6101 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL;
6102 } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) {
6103 mark_reg_known_zero(env, regs, BPF_REG_0);
6104 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL;
6105 } else if (fn->ret_type == RET_PTR_TO_ALLOC_MEM_OR_NULL) {
6106 mark_reg_known_zero(env, regs, BPF_REG_0);
6107 regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;
6108 regs[BPF_REG_0].mem_size = meta.mem_size;
6109 } else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL ||
6110 fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) {
6111 const struct btf_type *t;
6112
6113 mark_reg_known_zero(env, regs, BPF_REG_0);
6114 t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL);
6115 if (!btf_type_is_struct(t)) {
6116 u32 tsize;
6117 const struct btf_type *ret;
6118 const char *tname;
6119
6120
6121 ret = btf_resolve_size(meta.ret_btf, t, &tsize);
6122 if (IS_ERR(ret)) {
6123 tname = btf_name_by_offset(meta.ret_btf, t->name_off);
6124 verbose(env, "unable to resolve the size of type '%s': %ld\n",
6125 tname, PTR_ERR(ret));
6126 return -EINVAL;
6127 }
6128 regs[BPF_REG_0].type =
6129 fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
6130 PTR_TO_MEM : PTR_TO_MEM_OR_NULL;
6131 regs[BPF_REG_0].mem_size = tsize;
6132 } else {
6133 regs[BPF_REG_0].type =
6134 fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
6135 PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL;
6136 regs[BPF_REG_0].btf = meta.ret_btf;
6137 regs[BPF_REG_0].btf_id = meta.ret_btf_id;
6138 }
6139 } else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL ||
6140 fn->ret_type == RET_PTR_TO_BTF_ID) {
6141 int ret_btf_id;
6142
6143 mark_reg_known_zero(env, regs, BPF_REG_0);
6144 regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ?
6145 PTR_TO_BTF_ID :
6146 PTR_TO_BTF_ID_OR_NULL;
6147 ret_btf_id = *fn->ret_btf_id;
6148 if (ret_btf_id == 0) {
6149 verbose(env, "invalid return type %d of func %s#%d\n",
6150 fn->ret_type, func_id_name(func_id), func_id);
6151 return -EINVAL;
6152 }
6153
6154
6155
6156 regs[BPF_REG_0].btf = btf_vmlinux;
6157 regs[BPF_REG_0].btf_id = ret_btf_id;
6158 } else {
6159 verbose(env, "unknown return type %d of func %s#%d\n",
6160 fn->ret_type, func_id_name(func_id), func_id);
6161 return -EINVAL;
6162 }
6163
6164 if (reg_type_may_be_null(regs[BPF_REG_0].type))
6165 regs[BPF_REG_0].id = ++env->id_gen;
6166
6167 if (is_ptr_cast_function(func_id)) {
6168
6169 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
6170 } else if (is_acquire_function(func_id, meta.map_ptr)) {
6171 int id = acquire_reference_state(env, insn_idx);
6172
6173 if (id < 0)
6174 return id;
6175
6176 regs[BPF_REG_0].id = id;
6177
6178 regs[BPF_REG_0].ref_obj_id = id;
6179 }
6180
6181 do_refine_retval_range(regs, fn->ret_type, func_id, &meta);
6182
6183 err = check_map_func_compatibility(env, meta.map_ptr, func_id);
6184 if (err)
6185 return err;
6186
6187 if ((func_id == BPF_FUNC_get_stack ||
6188 func_id == BPF_FUNC_get_task_stack) &&
6189 !env->prog->has_callchain_buf) {
6190 const char *err_str;
6191
6192#ifdef CONFIG_PERF_EVENTS
6193 err = get_callchain_buffers(sysctl_perf_event_max_stack);
6194 err_str = "cannot get callchain buffer for func %s#%d\n";
6195#else
6196 err = -ENOTSUPP;
6197 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";
6198#endif
6199 if (err) {
6200 verbose(env, err_str, func_id_name(func_id), func_id);
6201 return err;
6202 }
6203
6204 env->prog->has_callchain_buf = true;
6205 }
6206
6207 if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack)
6208 env->prog->call_get_stack = true;
6209
6210 if (changes_data)
6211 clear_all_pkt_pointers(env);
6212 return 0;
6213}
6214
6215
6216
6217
6218static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
6219 size_t reg_size)
6220{
6221 struct bpf_reg_state *reg = &cur_regs(env)[regno];
6222
6223 if (regno == BPF_REG_0) {
6224
6225 reg->live |= REG_LIVE_WRITTEN;
6226 reg->subreg_def = reg_size == sizeof(u64) ?
6227 DEF_NOT_SUBREG : env->insn_idx + 1;
6228 } else {
6229
6230 if (reg_size == sizeof(u64)) {
6231 mark_insn_zext(env, reg);
6232 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
6233 } else {
6234 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);
6235 }
6236 }
6237}
6238
6239static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn)
6240{
6241 const struct btf_type *t, *func, *func_proto, *ptr_type;
6242 struct bpf_reg_state *regs = cur_regs(env);
6243 const char *func_name, *ptr_type_name;
6244 u32 i, nargs, func_id, ptr_type_id;
6245 const struct btf_param *args;
6246 int err;
6247
6248 func_id = insn->imm;
6249 func = btf_type_by_id(btf_vmlinux, func_id);
6250 func_name = btf_name_by_offset(btf_vmlinux, func->name_off);
6251 func_proto = btf_type_by_id(btf_vmlinux, func->type);
6252
6253 if (!env->ops->check_kfunc_call ||
6254 !env->ops->check_kfunc_call(func_id)) {
6255 verbose(env, "calling kernel function %s is not allowed\n",
6256 func_name);
6257 return -EACCES;
6258 }
6259
6260
6261 err = btf_check_kfunc_arg_match(env, btf_vmlinux, func_id, regs);
6262 if (err)
6263 return err;
6264
6265 for (i = 0; i < CALLER_SAVED_REGS; i++)
6266 mark_reg_not_init(env, regs, caller_saved[i]);
6267
6268
6269 t = btf_type_skip_modifiers(btf_vmlinux, func_proto->type, NULL);
6270 if (btf_type_is_scalar(t)) {
6271 mark_reg_unknown(env, regs, BPF_REG_0);
6272 mark_btf_func_reg_size(env, BPF_REG_0, t->size);
6273 } else if (btf_type_is_ptr(t)) {
6274 ptr_type = btf_type_skip_modifiers(btf_vmlinux, t->type,
6275 &ptr_type_id);
6276 if (!btf_type_is_struct(ptr_type)) {
6277 ptr_type_name = btf_name_by_offset(btf_vmlinux,
6278 ptr_type->name_off);
6279 verbose(env, "kernel function %s returns pointer type %s %s is not supported\n",
6280 func_name, btf_type_str(ptr_type),
6281 ptr_type_name);
6282 return -EINVAL;
6283 }
6284 mark_reg_known_zero(env, regs, BPF_REG_0);
6285 regs[BPF_REG_0].btf = btf_vmlinux;
6286 regs[BPF_REG_0].type = PTR_TO_BTF_ID;
6287 regs[BPF_REG_0].btf_id = ptr_type_id;
6288 mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
6289 }
6290
6291 nargs = btf_type_vlen(func_proto);
6292 args = (const struct btf_param *)(func_proto + 1);
6293 for (i = 0; i < nargs; i++) {
6294 u32 regno = i + 1;
6295
6296 t = btf_type_skip_modifiers(btf_vmlinux, args[i].type, NULL);
6297 if (btf_type_is_ptr(t))
6298 mark_btf_func_reg_size(env, regno, sizeof(void *));
6299 else
6300
6301 mark_btf_func_reg_size(env, regno, t->size);
6302 }
6303
6304 return 0;
6305}
6306
6307static bool signed_add_overflows(s64 a, s64 b)
6308{
6309
6310 s64 res = (s64)((u64)a + (u64)b);
6311
6312 if (b < 0)
6313 return res > a;
6314 return res < a;
6315}
6316
6317static bool signed_add32_overflows(s32 a, s32 b)
6318{
6319
6320 s32 res = (s32)((u32)a + (u32)b);
6321
6322 if (b < 0)
6323 return res > a;
6324 return res < a;
6325}
6326
6327static bool signed_sub_overflows(s64 a, s64 b)
6328{
6329
6330 s64 res = (s64)((u64)a - (u64)b);
6331
6332 if (b < 0)
6333 return res < a;
6334 return res > a;
6335}
6336
6337static bool signed_sub32_overflows(s32 a, s32 b)
6338{
6339
6340 s32 res = (s32)((u32)a - (u32)b);
6341
6342 if (b < 0)
6343 return res < a;
6344 return res > a;
6345}
6346
6347static bool check_reg_sane_offset(struct bpf_verifier_env *env,
6348 const struct bpf_reg_state *reg,
6349 enum bpf_reg_type type)
6350{
6351 bool known = tnum_is_const(reg->var_off);
6352 s64 val = reg->var_off.value;
6353 s64 smin = reg->smin_value;
6354
6355 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
6356 verbose(env, "math between %s pointer and %lld is not allowed\n",
6357 reg_type_str[type], val);
6358 return false;
6359 }
6360
6361 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
6362 verbose(env, "%s pointer offset %d is not allowed\n",
6363 reg_type_str[type], reg->off);
6364 return false;
6365 }
6366
6367 if (smin == S64_MIN) {
6368 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
6369 reg_type_str[type]);
6370 return false;
6371 }
6372
6373 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
6374 verbose(env, "value %lld makes %s pointer be out of bounds\n",
6375 smin, reg_type_str[type]);
6376 return false;
6377 }
6378
6379 return true;
6380}
6381
6382static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)
6383{
6384 return &env->insn_aux_data[env->insn_idx];
6385}
6386
6387enum {
6388 REASON_BOUNDS = -1,
6389 REASON_TYPE = -2,
6390 REASON_PATHS = -3,
6391 REASON_LIMIT = -4,
6392 REASON_STACK = -5,
6393};
6394
6395static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
6396 u32 *alu_limit, bool mask_to_left)
6397{
6398 u32 max = 0, ptr_limit = 0;
6399
6400 switch (ptr_reg->type) {
6401 case PTR_TO_STACK:
6402
6403
6404
6405
6406
6407 max = MAX_BPF_STACK + mask_to_left;
6408 ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off);
6409 break;
6410 case PTR_TO_MAP_VALUE:
6411 max = ptr_reg->map_ptr->value_size;
6412 ptr_limit = (mask_to_left ?
6413 ptr_reg->smin_value :
6414 ptr_reg->umax_value) + ptr_reg->off;
6415 break;
6416 default:
6417 return REASON_TYPE;
6418 }
6419
6420 if (ptr_limit >= max)
6421 return REASON_LIMIT;
6422 *alu_limit = ptr_limit;
6423 return 0;
6424}
6425
6426static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
6427 const struct bpf_insn *insn)
6428{
6429 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K;
6430}
6431
6432static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
6433 u32 alu_state, u32 alu_limit)
6434{
6435
6436
6437
6438 if (aux->alu_state &&
6439 (aux->alu_state != alu_state ||
6440 aux->alu_limit != alu_limit))
6441 return REASON_PATHS;
6442
6443
6444 aux->alu_state = alu_state;
6445 aux->alu_limit = alu_limit;
6446 return 0;
6447}
6448
6449static int sanitize_val_alu(struct bpf_verifier_env *env,
6450 struct bpf_insn *insn)
6451{
6452 struct bpf_insn_aux_data *aux = cur_aux(env);
6453
6454 if (can_skip_alu_sanitation(env, insn))
6455 return 0;
6456
6457 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
6458}
6459
6460static bool sanitize_needed(u8 opcode)
6461{
6462 return opcode == BPF_ADD || opcode == BPF_SUB;
6463}
6464
6465struct bpf_sanitize_info {
6466 struct bpf_insn_aux_data aux;
6467 bool mask_to_left;
6468};
6469
6470static struct bpf_verifier_state *
6471sanitize_speculative_path(struct bpf_verifier_env *env,
6472 const struct bpf_insn *insn,
6473 u32 next_idx, u32 curr_idx)
6474{
6475 struct bpf_verifier_state *branch;
6476 struct bpf_reg_state *regs;
6477
6478 branch = push_stack(env, next_idx, curr_idx, true);
6479 if (branch && insn) {
6480 regs = branch->frame[branch->curframe]->regs;
6481 if (BPF_SRC(insn->code) == BPF_K) {
6482 mark_reg_unknown(env, regs, insn->dst_reg);
6483 } else if (BPF_SRC(insn->code) == BPF_X) {
6484 mark_reg_unknown(env, regs, insn->dst_reg);
6485 mark_reg_unknown(env, regs, insn->src_reg);
6486 }
6487 }
6488 return branch;
6489}
6490
6491static int sanitize_ptr_alu(struct bpf_verifier_env *env,
6492 struct bpf_insn *insn,
6493 const struct bpf_reg_state *ptr_reg,
6494 const struct bpf_reg_state *off_reg,
6495 struct bpf_reg_state *dst_reg,
6496 struct bpf_sanitize_info *info,
6497 const bool commit_window)
6498{
6499 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;
6500 struct bpf_verifier_state *vstate = env->cur_state;
6501 bool off_is_imm = tnum_is_const(off_reg->var_off);
6502 bool off_is_neg = off_reg->smin_value < 0;
6503 bool ptr_is_dst_reg = ptr_reg == dst_reg;
6504 u8 opcode = BPF_OP(insn->code);
6505 u32 alu_state, alu_limit;
6506 struct bpf_reg_state tmp;
6507 bool ret;
6508 int err;
6509
6510 if (can_skip_alu_sanitation(env, insn))
6511 return 0;
6512
6513
6514
6515
6516
6517 if (vstate->speculative)
6518 goto do_sim;
6519
6520 if (!commit_window) {
6521 if (!tnum_is_const(off_reg->var_off) &&
6522 (off_reg->smin_value < 0) != (off_reg->smax_value < 0))
6523 return REASON_BOUNDS;
6524
6525 info->mask_to_left = (opcode == BPF_ADD && off_is_neg) ||
6526 (opcode == BPF_SUB && !off_is_neg);
6527 }
6528
6529 err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left);
6530 if (err < 0)
6531 return err;
6532
6533 if (commit_window) {
6534
6535
6536
6537 alu_state = info->aux.alu_state;
6538 alu_limit = abs(info->aux.alu_limit - alu_limit);
6539 } else {
6540 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
6541 alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0;
6542 alu_state |= ptr_is_dst_reg ?
6543 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
6544
6545
6546
6547
6548 if (!off_is_imm)
6549 env->explore_alu_limits = true;
6550 }
6551
6552 err = update_alu_sanitation_state(aux, alu_state, alu_limit);
6553 if (err < 0)
6554 return err;
6555do_sim:
6556
6557
6558
6559
6560
6561
6562
6563
6564 if (commit_window || off_is_imm)
6565 return 0;
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576 if (!ptr_is_dst_reg) {
6577 tmp = *dst_reg;
6578 *dst_reg = *ptr_reg;
6579 }
6580 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1,
6581 env->insn_idx);
6582 if (!ptr_is_dst_reg && ret)
6583 *dst_reg = tmp;
6584 return !ret ? REASON_STACK : 0;
6585}
6586
6587static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)
6588{
6589 struct bpf_verifier_state *vstate = env->cur_state;
6590
6591
6592
6593
6594
6595
6596 if (!vstate->speculative)
6597 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;
6598}
6599
6600static int sanitize_err(struct bpf_verifier_env *env,
6601 const struct bpf_insn *insn, int reason,
6602 const struct bpf_reg_state *off_reg,
6603 const struct bpf_reg_state *dst_reg)
6604{
6605 static const char *err = "pointer arithmetic with it prohibited for !root";
6606 const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub";
6607 u32 dst = insn->dst_reg, src = insn->src_reg;
6608
6609 switch (reason) {
6610 case REASON_BOUNDS:
6611 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",
6612 off_reg == dst_reg ? dst : src, err);
6613 break;
6614 case REASON_TYPE:
6615 verbose(env, "R%d has pointer with unsupported alu operation, %s\n",
6616 off_reg == dst_reg ? src : dst, err);
6617 break;
6618 case REASON_PATHS:
6619 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",
6620 dst, op, err);
6621 break;
6622 case REASON_LIMIT:
6623 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",
6624 dst, op, err);
6625 break;
6626 case REASON_STACK:
6627 verbose(env, "R%d could not be pushed for speculative verification, %s\n",
6628 dst, err);
6629 break;
6630 default:
6631 verbose(env, "verifier internal error: unknown reason (%d)\n",
6632 reason);
6633 break;
6634 }
6635
6636 return -EACCES;
6637}
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649static int check_stack_access_for_ptr_arithmetic(
6650 struct bpf_verifier_env *env,
6651 int regno,
6652 const struct bpf_reg_state *reg,
6653 int off)
6654{
6655 if (!tnum_is_const(reg->var_off)) {
6656 char tn_buf[48];
6657
6658 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
6659 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",
6660 regno, tn_buf, off);
6661 return -EACCES;
6662 }
6663
6664 if (off >= 0 || off < -MAX_BPF_STACK) {
6665 verbose(env, "R%d stack pointer arithmetic goes out of range, "
6666 "prohibited for !root; off=%d\n", regno, off);
6667 return -EACCES;
6668 }
6669
6670 return 0;
6671}
6672
6673static int sanitize_check_bounds(struct bpf_verifier_env *env,
6674 const struct bpf_insn *insn,
6675 const struct bpf_reg_state *dst_reg)
6676{
6677 u32 dst = insn->dst_reg;
6678
6679
6680
6681
6682 if (env->bypass_spec_v1)
6683 return 0;
6684
6685 switch (dst_reg->type) {
6686 case PTR_TO_STACK:
6687 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,
6688 dst_reg->off + dst_reg->var_off.value))
6689 return -EACCES;
6690 break;
6691 case PTR_TO_MAP_VALUE:
6692 if (check_map_access(env, dst, dst_reg->off, 1, false)) {
6693 verbose(env, "R%d pointer arithmetic of map value goes out of range, "
6694 "prohibited for !root\n", dst);
6695 return -EACCES;
6696 }
6697 break;
6698 default:
6699 break;
6700 }
6701
6702 return 0;
6703}
6704
6705
6706
6707
6708
6709
6710static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
6711 struct bpf_insn *insn,
6712 const struct bpf_reg_state *ptr_reg,
6713 const struct bpf_reg_state *off_reg)
6714{
6715 struct bpf_verifier_state *vstate = env->cur_state;
6716 struct bpf_func_state *state = vstate->frame[vstate->curframe];
6717 struct bpf_reg_state *regs = state->regs, *dst_reg;
6718 bool known = tnum_is_const(off_reg->var_off);
6719 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value,
6720 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value;
6721 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value,
6722 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value;
6723 struct bpf_sanitize_info info = {};
6724 u8 opcode = BPF_OP(insn->code);
6725 u32 dst = insn->dst_reg;
6726 int ret;
6727
6728 dst_reg = ®s[dst];
6729
6730 if ((known && (smin_val != smax_val || umin_val != umax_val)) ||
6731 smin_val > smax_val || umin_val > umax_val) {
6732
6733
6734
6735 __mark_reg_unknown(env, dst_reg);
6736 return 0;
6737 }
6738
6739 if (BPF_CLASS(insn->code) != BPF_ALU64) {
6740
6741 if (opcode == BPF_SUB && env->allow_ptr_leaks) {
6742 __mark_reg_unknown(env, dst_reg);
6743 return 0;
6744 }
6745
6746 verbose(env,
6747 "R%d 32-bit pointer arithmetic prohibited\n",
6748 dst);
6749 return -EACCES;
6750 }
6751
6752 switch (ptr_reg->type) {
6753 case PTR_TO_MAP_VALUE_OR_NULL:
6754 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
6755 dst, reg_type_str[ptr_reg->type]);
6756 return -EACCES;
6757 case CONST_PTR_TO_MAP:
6758
6759 if (known && smin_val == 0 && opcode == BPF_ADD)
6760 break;
6761 fallthrough;
6762 case PTR_TO_PACKET_END:
6763 case PTR_TO_SOCKET:
6764 case PTR_TO_SOCKET_OR_NULL:
6765 case PTR_TO_SOCK_COMMON:
6766 case PTR_TO_SOCK_COMMON_OR_NULL:
6767 case PTR_TO_TCP_SOCK:
6768 case PTR_TO_TCP_SOCK_OR_NULL:
6769 case PTR_TO_XDP_SOCK:
6770 verbose(env, "R%d pointer arithmetic on %s prohibited\n",
6771 dst, reg_type_str[ptr_reg->type]);
6772 return -EACCES;
6773 default:
6774 break;
6775 }
6776
6777
6778
6779
6780 dst_reg->type = ptr_reg->type;
6781 dst_reg->id = ptr_reg->id;
6782
6783 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||
6784 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))
6785 return -EINVAL;
6786
6787
6788 __mark_reg32_unbounded(dst_reg);
6789
6790 if (sanitize_needed(opcode)) {
6791 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
6792 &info, false);
6793 if (ret < 0)
6794 return sanitize_err(env, insn, ret, off_reg, dst_reg);
6795 }
6796
6797 switch (opcode) {
6798 case BPF_ADD:
6799
6800
6801
6802 if (known && (ptr_reg->off + smin_val ==
6803 (s64)(s32)(ptr_reg->off + smin_val))) {
6804
6805 dst_reg->smin_value = smin_ptr;
6806 dst_reg->smax_value = smax_ptr;
6807 dst_reg->umin_value = umin_ptr;
6808 dst_reg->umax_value = umax_ptr;
6809 dst_reg->var_off = ptr_reg->var_off;
6810 dst_reg->off = ptr_reg->off + smin_val;
6811 dst_reg->raw = ptr_reg->raw;
6812 break;
6813 }
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823 if (signed_add_overflows(smin_ptr, smin_val) ||
6824 signed_add_overflows(smax_ptr, smax_val)) {
6825 dst_reg->smin_value = S64_MIN;
6826 dst_reg->smax_value = S64_MAX;
6827 } else {
6828 dst_reg->smin_value = smin_ptr + smin_val;
6829 dst_reg->smax_value = smax_ptr + smax_val;
6830 }
6831 if (umin_ptr + umin_val < umin_ptr ||
6832 umax_ptr + umax_val < umax_ptr) {
6833 dst_reg->umin_value = 0;
6834 dst_reg->umax_value = U64_MAX;
6835 } else {
6836 dst_reg->umin_value = umin_ptr + umin_val;
6837 dst_reg->umax_value = umax_ptr + umax_val;
6838 }
6839 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
6840 dst_reg->off = ptr_reg->off;
6841 dst_reg->raw = ptr_reg->raw;
6842 if (reg_is_pkt_pointer(ptr_reg)) {
6843 dst_reg->id = ++env->id_gen;
6844
6845 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
6846 }
6847 break;
6848 case BPF_SUB:
6849 if (dst_reg == off_reg) {
6850
6851 verbose(env, "R%d tried to subtract pointer from scalar\n",
6852 dst);
6853 return -EACCES;
6854 }
6855
6856
6857
6858
6859 if (ptr_reg->type == PTR_TO_STACK) {
6860 verbose(env, "R%d subtraction from stack pointer prohibited\n",
6861 dst);
6862 return -EACCES;
6863 }
6864 if (known && (ptr_reg->off - smin_val ==
6865 (s64)(s32)(ptr_reg->off - smin_val))) {
6866
6867 dst_reg->smin_value = smin_ptr;
6868 dst_reg->smax_value = smax_ptr;
6869 dst_reg->umin_value = umin_ptr;
6870 dst_reg->umax_value = umax_ptr;
6871 dst_reg->var_off = ptr_reg->var_off;
6872 dst_reg->id = ptr_reg->id;
6873 dst_reg->off = ptr_reg->off - smin_val;
6874 dst_reg->raw = ptr_reg->raw;
6875 break;
6876 }
6877
6878
6879
6880 if (signed_sub_overflows(smin_ptr, smax_val) ||
6881 signed_sub_overflows(smax_ptr, smin_val)) {
6882
6883 dst_reg->smin_value = S64_MIN;
6884 dst_reg->smax_value = S64_MAX;
6885 } else {
6886 dst_reg->smin_value = smin_ptr - smax_val;
6887 dst_reg->smax_value = smax_ptr - smin_val;
6888 }
6889 if (umin_ptr < umax_val) {
6890
6891 dst_reg->umin_value = 0;
6892 dst_reg->umax_value = U64_MAX;
6893 } else {
6894
6895 dst_reg->umin_value = umin_ptr - umax_val;
6896 dst_reg->umax_value = umax_ptr - umin_val;
6897 }
6898 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
6899 dst_reg->off = ptr_reg->off;
6900 dst_reg->raw = ptr_reg->raw;
6901 if (reg_is_pkt_pointer(ptr_reg)) {
6902 dst_reg->id = ++env->id_gen;
6903
6904 if (smin_val < 0)
6905 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
6906 }
6907 break;
6908 case BPF_AND:
6909 case BPF_OR:
6910 case BPF_XOR:
6911
6912 verbose(env, "R%d bitwise operator %s on pointer prohibited\n",
6913 dst, bpf_alu_string[opcode >> 4]);
6914 return -EACCES;
6915 default:
6916
6917 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",
6918 dst, bpf_alu_string[opcode >> 4]);
6919 return -EACCES;
6920 }
6921
6922 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))
6923 return -EINVAL;
6924
6925 __update_reg_bounds(dst_reg);
6926 __reg_deduce_bounds(dst_reg);
6927 __reg_bound_offset(dst_reg);
6928
6929 if (sanitize_check_bounds(env, insn, dst_reg) < 0)
6930 return -EACCES;
6931 if (sanitize_needed(opcode)) {
6932 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,
6933 &info, true);
6934 if (ret < 0)
6935 return sanitize_err(env, insn, ret, off_reg, dst_reg);
6936 }
6937
6938 return 0;
6939}
6940
6941static void scalar32_min_max_add(struct bpf_reg_state *dst_reg,
6942 struct bpf_reg_state *src_reg)
6943{
6944 s32 smin_val = src_reg->s32_min_value;
6945 s32 smax_val = src_reg->s32_max_value;
6946 u32 umin_val = src_reg->u32_min_value;
6947 u32 umax_val = src_reg->u32_max_value;
6948
6949 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) ||
6950 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) {
6951 dst_reg->s32_min_value = S32_MIN;
6952 dst_reg->s32_max_value = S32_MAX;
6953 } else {
6954 dst_reg->s32_min_value += smin_val;
6955 dst_reg->s32_max_value += smax_val;
6956 }
6957 if (dst_reg->u32_min_value + umin_val < umin_val ||
6958 dst_reg->u32_max_value + umax_val < umax_val) {
6959 dst_reg->u32_min_value = 0;
6960 dst_reg->u32_max_value = U32_MAX;
6961 } else {
6962 dst_reg->u32_min_value += umin_val;
6963 dst_reg->u32_max_value += umax_val;
6964 }
6965}
6966
6967static void scalar_min_max_add(struct bpf_reg_state *dst_reg,
6968 struct bpf_reg_state *src_reg)
6969{
6970 s64 smin_val = src_reg->smin_value;
6971 s64 smax_val = src_reg->smax_value;
6972 u64 umin_val = src_reg->umin_value;
6973 u64 umax_val = src_reg->umax_value;
6974
6975 if (signed_add_overflows(dst_reg->smin_value, smin_val) ||
6976 signed_add_overflows(dst_reg->smax_value, smax_val)) {
6977 dst_reg->smin_value = S64_MIN;
6978 dst_reg->smax_value = S64_MAX;
6979 } else {
6980 dst_reg->smin_value += smin_val;
6981 dst_reg->smax_value += smax_val;
6982 }
6983 if (dst_reg->umin_value + umin_val < umin_val ||
6984 dst_reg->umax_value + umax_val < umax_val) {
6985 dst_reg->umin_value = 0;
6986 dst_reg->umax_value = U64_MAX;
6987 } else {
6988 dst_reg->umin_value += umin_val;
6989 dst_reg->umax_value += umax_val;
6990 }
6991}
6992
6993static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg,
6994 struct bpf_reg_state *src_reg)
6995{
6996 s32 smin_val = src_reg->s32_min_value;
6997 s32 smax_val = src_reg->s32_max_value;
6998 u32 umin_val = src_reg->u32_min_value;
6999 u32 umax_val = src_reg->u32_max_value;
7000
7001 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) ||
7002 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) {
7003
7004 dst_reg->s32_min_value = S32_MIN;
7005 dst_reg->s32_max_value = S32_MAX;
7006 } else {
7007 dst_reg->s32_min_value -= smax_val;
7008 dst_reg->s32_max_value -= smin_val;
7009 }
7010 if (dst_reg->u32_min_value < umax_val) {
7011
7012 dst_reg->u32_min_value = 0;
7013 dst_reg->u32_max_value = U32_MAX;
7014 } else {
7015
7016 dst_reg->u32_min_value -= umax_val;
7017 dst_reg->u32_max_value -= umin_val;
7018 }
7019}
7020
7021static void scalar_min_max_sub(struct bpf_reg_state *dst_reg,
7022 struct bpf_reg_state *src_reg)
7023{
7024 s64 smin_val = src_reg->smin_value;
7025 s64 smax_val = src_reg->smax_value;
7026 u64 umin_val = src_reg->umin_value;
7027 u64 umax_val = src_reg->umax_value;
7028
7029 if (signed_sub_overflows(dst_reg->smin_value, smax_val) ||
7030 signed_sub_overflows(dst_reg->smax_value, smin_val)) {
7031
7032 dst_reg->smin_value = S64_MIN;
7033 dst_reg->smax_value = S64_MAX;
7034 } else {
7035 dst_reg->smin_value -= smax_val;
7036 dst_reg->smax_value -= smin_val;
7037 }
7038 if (dst_reg->umin_value < umax_val) {
7039
7040 dst_reg->umin_value = 0;
7041 dst_reg->umax_value = U64_MAX;
7042 } else {
7043
7044 dst_reg->umin_value -= umax_val;
7045 dst_reg->umax_value -= umin_val;
7046 }
7047}
7048
7049static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg,
7050 struct bpf_reg_state *src_reg)
7051{
7052 s32 smin_val = src_reg->s32_min_value;
7053 u32 umin_val = src_reg->u32_min_value;
7054 u32 umax_val = src_reg->u32_max_value;
7055
7056 if (smin_val < 0 || dst_reg->s32_min_value < 0) {
7057
7058 __mark_reg32_unbounded(dst_reg);
7059 return;
7060 }
7061
7062
7063
7064 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) {
7065
7066 __mark_reg32_unbounded(dst_reg);
7067 return;
7068 }
7069 dst_reg->u32_min_value *= umin_val;
7070 dst_reg->u32_max_value *= umax_val;
7071 if (dst_reg->u32_max_value > S32_MAX) {
7072
7073 dst_reg->s32_min_value = S32_MIN;
7074 dst_reg->s32_max_value = S32_MAX;
7075 } else {
7076 dst_reg->s32_min_value = dst_reg->u32_min_value;
7077 dst_reg->s32_max_value = dst_reg->u32_max_value;
7078 }
7079}
7080
7081static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,
7082 struct bpf_reg_state *src_reg)
7083{
7084 s64 smin_val = src_reg->smin_value;
7085 u64 umin_val = src_reg->umin_value;
7086 u64 umax_val = src_reg->umax_value;
7087
7088 if (smin_val < 0 || dst_reg->smin_value < 0) {
7089
7090 __mark_reg64_unbounded(dst_reg);
7091 return;
7092 }
7093
7094
7095
7096 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) {
7097
7098 __mark_reg64_unbounded(dst_reg);
7099 return;
7100 }
7101 dst_reg->umin_value *= umin_val;
7102 dst_reg->umax_value *= umax_val;
7103 if (dst_reg->umax_value > S64_MAX) {
7104
7105 dst_reg->smin_value = S64_MIN;
7106 dst_reg->smax_value = S64_MAX;
7107 } else {
7108 dst_reg->smin_value = dst_reg->umin_value;
7109 dst_reg->smax_value = dst_reg->umax_value;
7110 }
7111}
7112
7113static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,
7114 struct bpf_reg_state *src_reg)
7115{
7116 bool src_known = tnum_subreg_is_const(src_reg->var_off);
7117 bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
7118 struct tnum var32_off = tnum_subreg(dst_reg->var_off);
7119 s32 smin_val = src_reg->s32_min_value;
7120 u32 umax_val = src_reg->u32_max_value;
7121
7122 if (src_known && dst_known) {
7123 __mark_reg32_known(dst_reg, var32_off.value);
7124 return;
7125 }
7126
7127
7128
7129
7130 dst_reg->u32_min_value = var32_off.value;
7131 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val);
7132 if (dst_reg->s32_min_value < 0 || smin_val < 0) {
7133
7134
7135
7136 dst_reg->s32_min_value = S32_MIN;
7137 dst_reg->s32_max_value = S32_MAX;
7138 } else {
7139
7140
7141
7142 dst_reg->s32_min_value = dst_reg->u32_min_value;
7143 dst_reg->s32_max_value = dst_reg->u32_max_value;
7144 }
7145}
7146
7147static void scalar_min_max_and(struct bpf_reg_state *dst_reg,
7148 struct bpf_reg_state *src_reg)
7149{
7150 bool src_known = tnum_is_const(src_reg->var_off);
7151 bool dst_known = tnum_is_const(dst_reg->var_off);
7152 s64 smin_val = src_reg->smin_value;
7153 u64 umax_val = src_reg->umax_value;
7154
7155 if (src_known && dst_known) {
7156 __mark_reg_known(dst_reg, dst_reg->var_off.value);
7157 return;
7158 }
7159
7160
7161
7162
7163 dst_reg->umin_value = dst_reg->var_off.value;
7164 dst_reg->umax_value = min(dst_reg->umax_value, umax_val);
7165 if (dst_reg->smin_value < 0 || smin_val < 0) {
7166
7167
7168
7169 dst_reg->smin_value = S64_MIN;
7170 dst_reg->smax_value = S64_MAX;
7171 } else {
7172
7173
7174
7175 dst_reg->smin_value = dst_reg->umin_value;
7176 dst_reg->smax_value = dst_reg->umax_value;
7177 }
7178
7179 __update_reg_bounds(dst_reg);
7180}
7181
7182static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,
7183 struct bpf_reg_state *src_reg)
7184{
7185 bool src_known = tnum_subreg_is_const(src_reg->var_off);
7186 bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
7187 struct tnum var32_off = tnum_subreg(dst_reg->var_off);
7188 s32 smin_val = src_reg->s32_min_value;
7189 u32 umin_val = src_reg->u32_min_value;
7190
7191 if (src_known && dst_known) {
7192 __mark_reg32_known(dst_reg, var32_off.value);
7193 return;
7194 }
7195
7196
7197
7198
7199 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val);
7200 dst_reg->u32_max_value = var32_off.value | var32_off.mask;
7201 if (dst_reg->s32_min_value < 0 || smin_val < 0) {
7202
7203
7204
7205 dst_reg->s32_min_value = S32_MIN;
7206 dst_reg->s32_max_value = S32_MAX;
7207 } else {
7208
7209
7210
7211 dst_reg->s32_min_value = dst_reg->u32_min_value;
7212 dst_reg->s32_max_value = dst_reg->u32_max_value;
7213 }
7214}
7215
7216static void scalar_min_max_or(struct bpf_reg_state *dst_reg,
7217 struct bpf_reg_state *src_reg)
7218{
7219 bool src_known = tnum_is_const(src_reg->var_off);
7220 bool dst_known = tnum_is_const(dst_reg->var_off);
7221 s64 smin_val = src_reg->smin_value;
7222 u64 umin_val = src_reg->umin_value;
7223
7224 if (src_known && dst_known) {
7225 __mark_reg_known(dst_reg, dst_reg->var_off.value);
7226 return;
7227 }
7228
7229
7230
7231
7232 dst_reg->umin_value = max(dst_reg->umin_value, umin_val);
7233 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
7234 if (dst_reg->smin_value < 0 || smin_val < 0) {
7235
7236
7237
7238 dst_reg->smin_value = S64_MIN;
7239 dst_reg->smax_value = S64_MAX;
7240 } else {
7241
7242
7243
7244 dst_reg->smin_value = dst_reg->umin_value;
7245 dst_reg->smax_value = dst_reg->umax_value;
7246 }
7247
7248 __update_reg_bounds(dst_reg);
7249}
7250
7251static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,
7252 struct bpf_reg_state *src_reg)
7253{
7254 bool src_known = tnum_subreg_is_const(src_reg->var_off);
7255 bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
7256 struct tnum var32_off = tnum_subreg(dst_reg->var_off);
7257 s32 smin_val = src_reg->s32_min_value;
7258
7259 if (src_known && dst_known) {
7260 __mark_reg32_known(dst_reg, var32_off.value);
7261 return;
7262 }
7263
7264
7265 dst_reg->u32_min_value = var32_off.value;
7266 dst_reg->u32_max_value = var32_off.value | var32_off.mask;
7267
7268 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) {
7269
7270
7271
7272 dst_reg->s32_min_value = dst_reg->u32_min_value;
7273 dst_reg->s32_max_value = dst_reg->u32_max_value;
7274 } else {
7275 dst_reg->s32_min_value = S32_MIN;
7276 dst_reg->s32_max_value = S32_MAX;
7277 }
7278}
7279
7280static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,
7281 struct bpf_reg_state *src_reg)
7282{
7283 bool src_known = tnum_is_const(src_reg->var_off);
7284 bool dst_known = tnum_is_const(dst_reg->var_off);
7285 s64 smin_val = src_reg->smin_value;
7286
7287 if (src_known && dst_known) {
7288
7289 __mark_reg_known(dst_reg, dst_reg->var_off.value);
7290 return;
7291 }
7292
7293
7294 dst_reg->umin_value = dst_reg->var_off.value;
7295 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
7296
7297 if (dst_reg->smin_value >= 0 && smin_val >= 0) {
7298
7299
7300
7301 dst_reg->smin_value = dst_reg->umin_value;
7302 dst_reg->smax_value = dst_reg->umax_value;
7303 } else {
7304 dst_reg->smin_value = S64_MIN;
7305 dst_reg->smax_value = S64_MAX;
7306 }
7307
7308 __update_reg_bounds(dst_reg);
7309}
7310
7311static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
7312 u64 umin_val, u64 umax_val)
7313{
7314
7315
7316
7317 dst_reg->s32_min_value = S32_MIN;
7318 dst_reg->s32_max_value = S32_MAX;
7319
7320 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) {
7321 dst_reg->u32_min_value = 0;
7322 dst_reg->u32_max_value = U32_MAX;
7323 } else {
7324 dst_reg->u32_min_value <<= umin_val;
7325 dst_reg->u32_max_value <<= umax_val;
7326 }
7327}
7328
7329static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
7330 struct bpf_reg_state *src_reg)
7331{
7332 u32 umax_val = src_reg->u32_max_value;
7333 u32 umin_val = src_reg->u32_min_value;
7334
7335 struct tnum subreg = tnum_subreg(dst_reg->var_off);
7336
7337 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
7338 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val));
7339
7340
7341
7342
7343 __mark_reg64_unbounded(dst_reg);
7344 __update_reg32_bounds(dst_reg);
7345}
7346
7347static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,
7348 u64 umin_val, u64 umax_val)
7349{
7350
7351
7352
7353
7354
7355
7356
7357 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0)
7358 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32;
7359 else
7360 dst_reg->smax_value = S64_MAX;
7361
7362 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0)
7363 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32;
7364 else
7365 dst_reg->smin_value = S64_MIN;
7366
7367
7368 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
7369 dst_reg->umin_value = 0;
7370 dst_reg->umax_value = U64_MAX;
7371 } else {
7372 dst_reg->umin_value <<= umin_val;
7373 dst_reg->umax_value <<= umax_val;
7374 }
7375}
7376
7377static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg,
7378 struct bpf_reg_state *src_reg)
7379{
7380 u64 umax_val = src_reg->umax_value;
7381 u64 umin_val = src_reg->umin_value;
7382
7383
7384 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val);
7385 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
7386
7387 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val);
7388
7389 __update_reg_bounds(dst_reg);
7390}
7391
7392static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,
7393 struct bpf_reg_state *src_reg)
7394{
7395 struct tnum subreg = tnum_subreg(dst_reg->var_off);
7396 u32 umax_val = src_reg->u32_max_value;
7397 u32 umin_val = src_reg->u32_min_value;
7398
7399
7400
7401
7402
7403
7404
7405
7406
7407
7408
7409
7410
7411
7412
7413 dst_reg->s32_min_value = S32_MIN;
7414 dst_reg->s32_max_value = S32_MAX;
7415
7416 dst_reg->var_off = tnum_rshift(subreg, umin_val);
7417 dst_reg->u32_min_value >>= umax_val;
7418 dst_reg->u32_max_value >>= umin_val;
7419
7420 __mark_reg64_unbounded(dst_reg);
7421 __update_reg32_bounds(dst_reg);
7422}
7423
7424static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,
7425 struct bpf_reg_state *src_reg)
7426{
7427 u64 umax_val = src_reg->umax_value;
7428 u64 umin_val = src_reg->umin_value;
7429
7430
7431
7432
7433
7434
7435
7436
7437
7438
7439
7440
7441
7442
7443
7444 dst_reg->smin_value = S64_MIN;
7445 dst_reg->smax_value = S64_MAX;
7446 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
7447 dst_reg->umin_value >>= umax_val;
7448 dst_reg->umax_value >>= umin_val;
7449
7450
7451
7452
7453
7454 __mark_reg32_unbounded(dst_reg);
7455 __update_reg_bounds(dst_reg);
7456}
7457
7458static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,
7459 struct bpf_reg_state *src_reg)
7460{
7461 u64 umin_val = src_reg->u32_min_value;
7462
7463
7464
7465
7466 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val);
7467 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val);
7468
7469 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32);
7470
7471
7472
7473
7474 dst_reg->u32_min_value = 0;
7475 dst_reg->u32_max_value = U32_MAX;
7476
7477 __mark_reg64_unbounded(dst_reg);
7478 __update_reg32_bounds(dst_reg);
7479}
7480
7481static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg,
7482 struct bpf_reg_state *src_reg)
7483{
7484 u64 umin_val = src_reg->umin_value;
7485
7486
7487
7488
7489 dst_reg->smin_value >>= umin_val;
7490 dst_reg->smax_value >>= umin_val;
7491
7492 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64);
7493
7494
7495
7496
7497 dst_reg->umin_value = 0;
7498 dst_reg->umax_value = U64_MAX;
7499
7500
7501
7502
7503
7504 __mark_reg32_unbounded(dst_reg);
7505 __update_reg_bounds(dst_reg);
7506}
7507
7508
7509
7510
7511
7512static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
7513 struct bpf_insn *insn,
7514 struct bpf_reg_state *dst_reg,
7515 struct bpf_reg_state src_reg)
7516{
7517 struct bpf_reg_state *regs = cur_regs(env);
7518 u8 opcode = BPF_OP(insn->code);
7519 bool src_known;
7520 s64 smin_val, smax_val;
7521 u64 umin_val, umax_val;
7522 s32 s32_min_val, s32_max_val;
7523 u32 u32_min_val, u32_max_val;
7524 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
7525 bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
7526 int ret;
7527
7528 smin_val = src_reg.smin_value;
7529 smax_val = src_reg.smax_value;
7530 umin_val = src_reg.umin_value;
7531 umax_val = src_reg.umax_value;
7532
7533 s32_min_val = src_reg.s32_min_value;
7534 s32_max_val = src_reg.s32_max_value;
7535 u32_min_val = src_reg.u32_min_value;
7536 u32_max_val = src_reg.u32_max_value;
7537
7538 if (alu32) {
7539 src_known = tnum_subreg_is_const(src_reg.var_off);
7540 if ((src_known &&
7541 (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) ||
7542 s32_min_val > s32_max_val || u32_min_val > u32_max_val) {
7543
7544
7545
7546 __mark_reg_unknown(env, dst_reg);
7547 return 0;
7548 }
7549 } else {
7550 src_known = tnum_is_const(src_reg.var_off);
7551 if ((src_known &&
7552 (smin_val != smax_val || umin_val != umax_val)) ||
7553 smin_val > smax_val || umin_val > umax_val) {
7554
7555
7556
7557 __mark_reg_unknown(env, dst_reg);
7558 return 0;
7559 }
7560 }
7561
7562 if (!src_known &&
7563 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) {
7564 __mark_reg_unknown(env, dst_reg);
7565 return 0;
7566 }
7567
7568 if (sanitize_needed(opcode)) {
7569 ret = sanitize_val_alu(env, insn);
7570 if (ret < 0)
7571 return sanitize_err(env, insn, ret, NULL, NULL);
7572 }
7573
7574
7575
7576
7577
7578
7579
7580
7581
7582
7583
7584
7585
7586
7587
7588 switch (opcode) {
7589 case BPF_ADD:
7590 scalar32_min_max_add(dst_reg, &src_reg);
7591 scalar_min_max_add(dst_reg, &src_reg);
7592 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
7593 break;
7594 case BPF_SUB:
7595 scalar32_min_max_sub(dst_reg, &src_reg);
7596 scalar_min_max_sub(dst_reg, &src_reg);
7597 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off);
7598 break;
7599 case BPF_MUL:
7600 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
7601 scalar32_min_max_mul(dst_reg, &src_reg);
7602 scalar_min_max_mul(dst_reg, &src_reg);
7603 break;
7604 case BPF_AND:
7605 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off);
7606 scalar32_min_max_and(dst_reg, &src_reg);
7607 scalar_min_max_and(dst_reg, &src_reg);
7608 break;
7609 case BPF_OR:
7610 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off);
7611 scalar32_min_max_or(dst_reg, &src_reg);
7612 scalar_min_max_or(dst_reg, &src_reg);
7613 break;
7614 case BPF_XOR:
7615 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off);
7616 scalar32_min_max_xor(dst_reg, &src_reg);
7617 scalar_min_max_xor(dst_reg, &src_reg);
7618 break;
7619 case BPF_LSH:
7620 if (umax_val >= insn_bitness) {
7621
7622
7623
7624 mark_reg_unknown(env, regs, insn->dst_reg);
7625 break;
7626 }
7627 if (alu32)
7628 scalar32_min_max_lsh(dst_reg, &src_reg);
7629 else
7630 scalar_min_max_lsh(dst_reg, &src_reg);
7631 break;
7632 case BPF_RSH:
7633 if (umax_val >= insn_bitness) {
7634
7635
7636
7637 mark_reg_unknown(env, regs, insn->dst_reg);
7638 break;
7639 }
7640 if (alu32)
7641 scalar32_min_max_rsh(dst_reg, &src_reg);
7642 else
7643 scalar_min_max_rsh(dst_reg, &src_reg);
7644 break;
7645 case BPF_ARSH:
7646 if (umax_val >= insn_bitness) {
7647
7648
7649
7650 mark_reg_unknown(env, regs, insn->dst_reg);
7651 break;
7652 }
7653 if (alu32)
7654 scalar32_min_max_arsh(dst_reg, &src_reg);
7655 else
7656 scalar_min_max_arsh(dst_reg, &src_reg);
7657 break;
7658 default:
7659 mark_reg_unknown(env, regs, insn->dst_reg);
7660 break;
7661 }
7662
7663
7664 if (alu32)
7665 zext_32_to_64(dst_reg);
7666
7667 __update_reg_bounds(dst_reg);
7668 __reg_deduce_bounds(dst_reg);
7669 __reg_bound_offset(dst_reg);
7670 return 0;
7671}
7672
7673
7674
7675
7676static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
7677 struct bpf_insn *insn)
7678{
7679 struct bpf_verifier_state *vstate = env->cur_state;
7680 struct bpf_func_state *state = vstate->frame[vstate->curframe];
7681 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
7682 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
7683 u8 opcode = BPF_OP(insn->code);
7684 int err;
7685
7686 dst_reg = ®s[insn->dst_reg];
7687 src_reg = NULL;
7688 if (dst_reg->type != SCALAR_VALUE)
7689 ptr_reg = dst_reg;
7690 else
7691
7692
7693
7694 dst_reg->id = 0;
7695 if (BPF_SRC(insn->code) == BPF_X) {
7696 src_reg = ®s[insn->src_reg];
7697 if (src_reg->type != SCALAR_VALUE) {
7698 if (dst_reg->type != SCALAR_VALUE) {
7699
7700
7701
7702
7703 if (opcode == BPF_SUB && env->allow_ptr_leaks) {
7704 mark_reg_unknown(env, regs, insn->dst_reg);
7705 return 0;
7706 }
7707 verbose(env, "R%d pointer %s pointer prohibited\n",
7708 insn->dst_reg,
7709 bpf_alu_string[opcode >> 4]);
7710 return -EACCES;
7711 } else {
7712
7713
7714
7715
7716 err = mark_chain_precision(env, insn->dst_reg);
7717 if (err)
7718 return err;
7719 return adjust_ptr_min_max_vals(env, insn,
7720 src_reg, dst_reg);
7721 }
7722 } else if (ptr_reg) {
7723
7724 err = mark_chain_precision(env, insn->src_reg);
7725 if (err)
7726 return err;
7727 return adjust_ptr_min_max_vals(env, insn,
7728 dst_reg, src_reg);
7729 }
7730 } else {
7731
7732
7733
7734 off_reg.type = SCALAR_VALUE;
7735 __mark_reg_known(&off_reg, insn->imm);
7736 src_reg = &off_reg;
7737 if (ptr_reg)
7738 return adjust_ptr_min_max_vals(env, insn,
7739 ptr_reg, src_reg);
7740 }
7741
7742
7743 if (WARN_ON_ONCE(ptr_reg)) {
7744 print_verifier_state(env, state);
7745 verbose(env, "verifier internal error: unexpected ptr_reg\n");
7746 return -EINVAL;
7747 }
7748 if (WARN_ON(!src_reg)) {
7749 print_verifier_state(env, state);
7750 verbose(env, "verifier internal error: no src_reg\n");
7751 return -EINVAL;
7752 }
7753 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
7754}
7755
7756
7757static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
7758{
7759 struct bpf_reg_state *regs = cur_regs(env);
7760 u8 opcode = BPF_OP(insn->code);
7761 int err;
7762
7763 if (opcode == BPF_END || opcode == BPF_NEG) {
7764 if (opcode == BPF_NEG) {
7765 if (BPF_SRC(insn->code) != 0 ||
7766 insn->src_reg != BPF_REG_0 ||
7767 insn->off != 0 || insn->imm != 0) {
7768 verbose(env, "BPF_NEG uses reserved fields\n");
7769 return -EINVAL;
7770 }
7771 } else {
7772 if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||
7773 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||
7774 BPF_CLASS(insn->code) == BPF_ALU64) {
7775 verbose(env, "BPF_END uses reserved fields\n");
7776 return -EINVAL;
7777 }
7778 }
7779
7780
7781 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
7782 if (err)
7783 return err;
7784
7785 if (is_pointer_value(env, insn->dst_reg)) {
7786 verbose(env, "R%d pointer arithmetic prohibited\n",
7787 insn->dst_reg);
7788 return -EACCES;
7789 }
7790
7791
7792 err = check_reg_arg(env, insn->dst_reg, DST_OP);
7793 if (err)
7794 return err;
7795
7796 } else if (opcode == BPF_MOV) {
7797
7798 if (BPF_SRC(insn->code) == BPF_X) {
7799 if (insn->imm != 0 || insn->off != 0) {
7800 verbose(env, "BPF_MOV uses reserved fields\n");
7801 return -EINVAL;
7802 }
7803
7804
7805 err = check_reg_arg(env, insn->src_reg, SRC_OP);
7806 if (err)
7807 return err;
7808 } else {
7809 if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
7810 verbose(env, "BPF_MOV uses reserved fields\n");
7811 return -EINVAL;
7812 }
7813 }
7814
7815
7816 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
7817 if (err)
7818 return err;
7819
7820 if (BPF_SRC(insn->code) == BPF_X) {
7821 struct bpf_reg_state *src_reg = regs + insn->src_reg;
7822 struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
7823
7824 if (BPF_CLASS(insn->code) == BPF_ALU64) {
7825
7826
7827
7828 if (src_reg->type == SCALAR_VALUE && !src_reg->id)
7829
7830
7831
7832
7833 src_reg->id = ++env->id_gen;
7834 *dst_reg = *src_reg;
7835 dst_reg->live |= REG_LIVE_WRITTEN;
7836 dst_reg->subreg_def = DEF_NOT_SUBREG;
7837 } else {
7838
7839 if (is_pointer_value(env, insn->src_reg)) {
7840 verbose(env,
7841 "R%d partial copy of pointer\n",
7842 insn->src_reg);
7843 return -EACCES;
7844 } else if (src_reg->type == SCALAR_VALUE) {
7845 *dst_reg = *src_reg;
7846
7847
7848
7849
7850 dst_reg->id = 0;
7851 dst_reg->live |= REG_LIVE_WRITTEN;
7852 dst_reg->subreg_def = env->insn_idx + 1;
7853 } else {
7854 mark_reg_unknown(env, regs,
7855 insn->dst_reg);
7856 }
7857 zext_32_to_64(dst_reg);
7858 }
7859 } else {
7860
7861
7862
7863
7864 mark_reg_unknown(env, regs, insn->dst_reg);
7865 regs[insn->dst_reg].type = SCALAR_VALUE;
7866 if (BPF_CLASS(insn->code) == BPF_ALU64) {
7867 __mark_reg_known(regs + insn->dst_reg,
7868 insn->imm);
7869 } else {
7870 __mark_reg_known(regs + insn->dst_reg,
7871 (u32)insn->imm);
7872 }
7873 }
7874
7875 } else if (opcode > BPF_END) {
7876 verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
7877 return -EINVAL;
7878
7879 } else {
7880
7881 if (BPF_SRC(insn->code) == BPF_X) {
7882 if (insn->imm != 0 || insn->off != 0) {
7883 verbose(env, "BPF_ALU uses reserved fields\n");
7884 return -EINVAL;
7885 }
7886
7887 err = check_reg_arg(env, insn->src_reg, SRC_OP);
7888 if (err)
7889 return err;
7890 } else {
7891 if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
7892 verbose(env, "BPF_ALU uses reserved fields\n");
7893 return -EINVAL;
7894 }
7895 }
7896
7897
7898 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
7899 if (err)
7900 return err;
7901
7902 if ((opcode == BPF_MOD || opcode == BPF_DIV) &&
7903 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {
7904 verbose(env, "div by zero\n");
7905 return -EINVAL;
7906 }
7907
7908 if ((opcode == BPF_LSH || opcode == BPF_RSH ||
7909 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
7910 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
7911
7912 if (insn->imm < 0 || insn->imm >= size) {
7913 verbose(env, "invalid shift %d\n", insn->imm);
7914 return -EINVAL;
7915 }
7916 }
7917
7918
7919 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
7920 if (err)
7921 return err;
7922
7923 return adjust_reg_min_max_vals(env, insn);
7924 }
7925
7926 return 0;
7927}
7928
7929static void __find_good_pkt_pointers(struct bpf_func_state *state,
7930 struct bpf_reg_state *dst_reg,
7931 enum bpf_reg_type type, int new_range)
7932{
7933 struct bpf_reg_state *reg;
7934 int i;
7935
7936 for (i = 0; i < MAX_BPF_REG; i++) {
7937 reg = &state->regs[i];
7938 if (reg->type == type && reg->id == dst_reg->id)
7939
7940 reg->range = max(reg->range, new_range);
7941 }
7942
7943 bpf_for_each_spilled_reg(i, state, reg) {
7944 if (!reg)
7945 continue;
7946 if (reg->type == type && reg->id == dst_reg->id)
7947 reg->range = max(reg->range, new_range);
7948 }
7949}
7950
7951static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
7952 struct bpf_reg_state *dst_reg,
7953 enum bpf_reg_type type,
7954 bool range_right_open)
7955{
7956 int new_range, i;
7957
7958 if (dst_reg->off < 0 ||
7959 (dst_reg->off == 0 && range_right_open))
7960
7961 return;
7962
7963 if (dst_reg->umax_value > MAX_PACKET_OFF ||
7964 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF)
7965
7966
7967
7968 return;
7969
7970 new_range = dst_reg->off;
7971 if (range_right_open)
7972 new_range--;
7973
7974
7975
7976
7977
7978
7979
7980
7981
7982
7983
7984
7985
7986
7987
7988
7989
7990
7991
7992
7993
7994
7995
7996
7997
7998
7999
8000
8001
8002
8003
8004
8005
8006
8007
8008
8009
8010
8011
8012
8013
8014
8015
8016
8017
8018
8019
8020
8021 for (i = 0; i <= vstate->curframe; i++)
8022 __find_good_pkt_pointers(vstate->frame[i], dst_reg, type,
8023 new_range);
8024}
8025
8026static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode)
8027{
8028 struct tnum subreg = tnum_subreg(reg->var_off);
8029 s32 sval = (s32)val;
8030
8031 switch (opcode) {
8032 case BPF_JEQ:
8033 if (tnum_is_const(subreg))
8034 return !!tnum_equals_const(subreg, val);
8035 break;
8036 case BPF_JNE:
8037 if (tnum_is_const(subreg))
8038 return !tnum_equals_const(subreg, val);
8039 break;
8040 case BPF_JSET:
8041 if ((~subreg.mask & subreg.value) & val)
8042 return 1;
8043 if (!((subreg.mask | subreg.value) & val))
8044 return 0;
8045 break;
8046 case BPF_JGT:
8047 if (reg->u32_min_value > val)
8048 return 1;
8049 else if (reg->u32_max_value <= val)
8050 return 0;
8051 break;
8052 case BPF_JSGT:
8053 if (reg->s32_min_value > sval)
8054 return 1;
8055 else if (reg->s32_max_value <= sval)
8056 return 0;
8057 break;
8058 case BPF_JLT:
8059 if (reg->u32_max_value < val)
8060 return 1;
8061 else if (reg->u32_min_value >= val)
8062 return 0;
8063 break;
8064 case BPF_JSLT:
8065 if (reg->s32_max_value < sval)
8066 return 1;
8067 else if (reg->s32_min_value >= sval)
8068 return 0;
8069 break;
8070 case BPF_JGE:
8071 if (reg->u32_min_value >= val)
8072 return 1;
8073 else if (reg->u32_max_value < val)
8074 return 0;
8075 break;
8076 case BPF_JSGE:
8077 if (reg->s32_min_value >= sval)
8078 return 1;
8079 else if (reg->s32_max_value < sval)
8080 return 0;
8081 break;
8082 case BPF_JLE:
8083 if (reg->u32_max_value <= val)
8084 return 1;
8085 else if (reg->u32_min_value > val)
8086 return 0;
8087 break;
8088 case BPF_JSLE:
8089 if (reg->s32_max_value <= sval)
8090 return 1;
8091 else if (reg->s32_min_value > sval)
8092 return 0;
8093 break;
8094 }
8095
8096 return -1;
8097}
8098
8099
8100static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode)
8101{
8102 s64 sval = (s64)val;
8103
8104 switch (opcode) {
8105 case BPF_JEQ:
8106 if (tnum_is_const(reg->var_off))
8107 return !!tnum_equals_const(reg->var_off, val);
8108 break;
8109 case BPF_JNE:
8110 if (tnum_is_const(reg->var_off))
8111 return !tnum_equals_const(reg->var_off, val);
8112 break;
8113 case BPF_JSET:
8114 if ((~reg->var_off.mask & reg->var_off.value) & val)
8115 return 1;
8116 if (!((reg->var_off.mask | reg->var_off.value) & val))
8117 return 0;
8118 break;
8119 case BPF_JGT:
8120 if (reg->umin_value > val)
8121 return 1;
8122 else if (reg->umax_value <= val)
8123 return 0;
8124 break;
8125 case BPF_JSGT:
8126 if (reg->smin_value > sval)
8127 return 1;
8128 else if (reg->smax_value <= sval)
8129 return 0;
8130 break;
8131 case BPF_JLT:
8132 if (reg->umax_value < val)
8133 return 1;
8134 else if (reg->umin_value >= val)
8135 return 0;
8136 break;
8137 case BPF_JSLT:
8138 if (reg->smax_value < sval)
8139 return 1;
8140 else if (reg->smin_value >= sval)
8141 return 0;
8142 break;
8143 case BPF_JGE:
8144 if (reg->umin_value >= val)
8145 return 1;
8146 else if (reg->umax_value < val)
8147 return 0;
8148 break;
8149 case BPF_JSGE:
8150 if (reg->smin_value >= sval)
8151 return 1;
8152 else if (reg->smax_value < sval)
8153 return 0;
8154 break;
8155 case BPF_JLE:
8156 if (reg->umax_value <= val)
8157 return 1;
8158 else if (reg->umin_value > val)
8159 return 0;
8160 break;
8161 case BPF_JSLE:
8162 if (reg->smax_value <= sval)
8163 return 1;
8164 else if (reg->smin_value > sval)
8165 return 0;
8166 break;
8167 }
8168
8169 return -1;
8170}
8171
8172
8173
8174
8175
8176
8177
8178
8179static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,
8180 bool is_jmp32)
8181{
8182 if (__is_pointer_value(false, reg)) {
8183 if (!reg_type_not_null(reg->type))
8184 return -1;
8185
8186
8187
8188
8189 if (val != 0)
8190 return -1;
8191
8192 switch (opcode) {
8193 case BPF_JEQ:
8194 return 0;
8195 case BPF_JNE:
8196 return 1;
8197 default:
8198 return -1;
8199 }
8200 }
8201
8202 if (is_jmp32)
8203 return is_branch32_taken(reg, val, opcode);
8204 return is_branch64_taken(reg, val, opcode);
8205}
8206
8207static int flip_opcode(u32 opcode)
8208{
8209
8210 static const u8 opcode_flip[16] = {
8211
8212 [BPF_JEQ >> 4] = BPF_JEQ,
8213 [BPF_JNE >> 4] = BPF_JNE,
8214 [BPF_JSET >> 4] = BPF_JSET,
8215
8216 [BPF_JGE >> 4] = BPF_JLE,
8217 [BPF_JGT >> 4] = BPF_JLT,
8218 [BPF_JLE >> 4] = BPF_JGE,
8219 [BPF_JLT >> 4] = BPF_JGT,
8220 [BPF_JSGE >> 4] = BPF_JSLE,
8221 [BPF_JSGT >> 4] = BPF_JSLT,
8222 [BPF_JSLE >> 4] = BPF_JSGE,
8223 [BPF_JSLT >> 4] = BPF_JSGT
8224 };
8225 return opcode_flip[opcode >> 4];
8226}
8227
8228static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg,
8229 struct bpf_reg_state *src_reg,
8230 u8 opcode)
8231{
8232 struct bpf_reg_state *pkt;
8233
8234 if (src_reg->type == PTR_TO_PACKET_END) {
8235 pkt = dst_reg;
8236 } else if (dst_reg->type == PTR_TO_PACKET_END) {
8237 pkt = src_reg;
8238 opcode = flip_opcode(opcode);
8239 } else {
8240 return -1;
8241 }
8242
8243 if (pkt->range >= 0)
8244 return -1;
8245
8246 switch (opcode) {
8247 case BPF_JLE:
8248
8249 fallthrough;
8250 case BPF_JGT:
8251
8252 if (pkt->range == BEYOND_PKT_END)
8253
8254 return opcode == BPF_JGT;
8255 break;
8256 case BPF_JLT:
8257
8258 fallthrough;
8259 case BPF_JGE:
8260
8261 if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END)
8262 return opcode == BPF_JGE;
8263 break;
8264 }
8265 return -1;
8266}
8267
8268
8269
8270
8271
8272
8273static void reg_set_min_max(struct bpf_reg_state *true_reg,
8274 struct bpf_reg_state *false_reg,
8275 u64 val, u32 val32,
8276 u8 opcode, bool is_jmp32)
8277{
8278 struct tnum false_32off = tnum_subreg(false_reg->var_off);
8279 struct tnum false_64off = false_reg->var_off;
8280 struct tnum true_32off = tnum_subreg(true_reg->var_off);
8281 struct tnum true_64off = true_reg->var_off;
8282 s64 sval = (s64)val;
8283 s32 sval32 = (s32)val32;
8284
8285
8286
8287
8288
8289
8290
8291 if (__is_pointer_value(false, false_reg))
8292 return;
8293
8294 switch (opcode) {
8295 case BPF_JEQ:
8296 case BPF_JNE:
8297 {
8298 struct bpf_reg_state *reg =
8299 opcode == BPF_JEQ ? true_reg : false_reg;
8300
8301
8302
8303
8304
8305
8306
8307
8308
8309 if (is_jmp32)
8310 __mark_reg32_known(reg, val32);
8311 else
8312 ___mark_reg_known(reg, val);
8313 break;
8314 }
8315 case BPF_JSET:
8316 if (is_jmp32) {
8317 false_32off = tnum_and(false_32off, tnum_const(~val32));
8318 if (is_power_of_2(val32))
8319 true_32off = tnum_or(true_32off,
8320 tnum_const(val32));
8321 } else {
8322 false_64off = tnum_and(false_64off, tnum_const(~val));
8323 if (is_power_of_2(val))
8324 true_64off = tnum_or(true_64off,
8325 tnum_const(val));
8326 }
8327 break;
8328 case BPF_JGE:
8329 case BPF_JGT:
8330 {
8331 if (is_jmp32) {
8332 u32 false_umax = opcode == BPF_JGT ? val32 : val32 - 1;
8333 u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32;
8334
8335 false_reg->u32_max_value = min(false_reg->u32_max_value,
8336 false_umax);
8337 true_reg->u32_min_value = max(true_reg->u32_min_value,
8338 true_umin);
8339 } else {
8340 u64 false_umax = opcode == BPF_JGT ? val : val - 1;
8341 u64 true_umin = opcode == BPF_JGT ? val + 1 : val;
8342
8343 false_reg->umax_value = min(false_reg->umax_value, false_umax);
8344 true_reg->umin_value = max(true_reg->umin_value, true_umin);
8345 }
8346 break;
8347 }
8348 case BPF_JSGE:
8349 case BPF_JSGT:
8350 {
8351 if (is_jmp32) {
8352 s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1;
8353 s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32;
8354
8355 false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax);
8356 true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin);
8357 } else {
8358 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1;
8359 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval;
8360
8361 false_reg->smax_value = min(false_reg->smax_value, false_smax);
8362 true_reg->smin_value = max(true_reg->smin_value, true_smin);
8363 }
8364 break;
8365 }
8366 case BPF_JLE:
8367 case BPF_JLT:
8368 {
8369 if (is_jmp32) {
8370 u32 false_umin = opcode == BPF_JLT ? val32 : val32 + 1;
8371 u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32;
8372
8373 false_reg->u32_min_value = max(false_reg->u32_min_value,
8374 false_umin);
8375 true_reg->u32_max_value = min(true_reg->u32_max_value,
8376 true_umax);
8377 } else {
8378 u64 false_umin = opcode == BPF_JLT ? val : val + 1;
8379 u64 true_umax = opcode == BPF_JLT ? val - 1 : val;
8380
8381 false_reg->umin_value = max(false_reg->umin_value, false_umin);
8382 true_reg->umax_value = min(true_reg->umax_value, true_umax);
8383 }
8384 break;
8385 }
8386 case BPF_JSLE:
8387 case BPF_JSLT:
8388 {
8389 if (is_jmp32) {
8390 s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1;
8391 s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32;
8392
8393 false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin);
8394 true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax);
8395 } else {
8396 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1;
8397 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval;
8398
8399 false_reg->smin_value = max(false_reg->smin_value, false_smin);
8400 true_reg->smax_value = min(true_reg->smax_value, true_smax);
8401 }
8402 break;
8403 }
8404 default:
8405 return;
8406 }
8407
8408 if (is_jmp32) {
8409 false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off),
8410 tnum_subreg(false_32off));
8411 true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off),
8412 tnum_subreg(true_32off));
8413 __reg_combine_32_into_64(false_reg);
8414 __reg_combine_32_into_64(true_reg);
8415 } else {
8416 false_reg->var_off = false_64off;
8417 true_reg->var_off = true_64off;
8418 __reg_combine_64_into_32(false_reg);
8419 __reg_combine_64_into_32(true_reg);
8420 }
8421}
8422
8423
8424
8425
8426static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,
8427 struct bpf_reg_state *false_reg,
8428 u64 val, u32 val32,
8429 u8 opcode, bool is_jmp32)
8430{
8431 opcode = flip_opcode(opcode);
8432
8433
8434
8435 if (opcode)
8436 reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32);
8437}
8438
8439
8440static void __reg_combine_min_max(struct bpf_reg_state *src_reg,
8441 struct bpf_reg_state *dst_reg)
8442{
8443 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value,
8444 dst_reg->umin_value);
8445 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value,
8446 dst_reg->umax_value);
8447 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value,
8448 dst_reg->smin_value);
8449 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value,
8450 dst_reg->smax_value);
8451 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off,
8452 dst_reg->var_off);
8453
8454 __update_reg_bounds(src_reg);
8455 __update_reg_bounds(dst_reg);
8456
8457 __reg_deduce_bounds(src_reg);
8458 __reg_deduce_bounds(dst_reg);
8459
8460 __reg_bound_offset(src_reg);
8461 __reg_bound_offset(dst_reg);
8462
8463
8464
8465
8466 __update_reg_bounds(src_reg);
8467 __update_reg_bounds(dst_reg);
8468}
8469
8470static void reg_combine_min_max(struct bpf_reg_state *true_src,
8471 struct bpf_reg_state *true_dst,
8472 struct bpf_reg_state *false_src,
8473 struct bpf_reg_state *false_dst,
8474 u8 opcode)
8475{
8476 switch (opcode) {
8477 case BPF_JEQ:
8478 __reg_combine_min_max(true_src, true_dst);
8479 break;
8480 case BPF_JNE:
8481 __reg_combine_min_max(false_src, false_dst);
8482 break;
8483 }
8484}
8485
8486static void mark_ptr_or_null_reg(struct bpf_func_state *state,
8487 struct bpf_reg_state *reg, u32 id,
8488 bool is_null)
8489{
8490 if (reg_type_may_be_null(reg->type) && reg->id == id &&
8491 !WARN_ON_ONCE(!reg->id)) {
8492
8493
8494
8495
8496 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value ||
8497 !tnum_equals_const(reg->var_off, 0) ||
8498 reg->off)) {
8499 __mark_reg_known_zero(reg);
8500 reg->off = 0;
8501 }
8502 if (is_null) {
8503 reg->type = SCALAR_VALUE;
8504
8505
8506
8507
8508 reg->id = 0;
8509 reg->ref_obj_id = 0;
8510
8511 return;
8512 }
8513
8514 mark_ptr_not_null_reg(reg);
8515
8516 if (!reg_may_point_to_spin_lock(reg)) {
8517
8518
8519
8520
8521
8522
8523 reg->id = 0;
8524 }
8525 }
8526}
8527
8528static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id,
8529 bool is_null)
8530{
8531 struct bpf_reg_state *reg;
8532 int i;
8533
8534 for (i = 0; i < MAX_BPF_REG; i++)
8535 mark_ptr_or_null_reg(state, &state->regs[i], id, is_null);
8536
8537 bpf_for_each_spilled_reg(i, state, reg) {
8538 if (!reg)
8539 continue;
8540 mark_ptr_or_null_reg(state, reg, id, is_null);
8541 }
8542}
8543
8544
8545
8546
8547static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
8548 bool is_null)
8549{
8550 struct bpf_func_state *state = vstate->frame[vstate->curframe];
8551 struct bpf_reg_state *regs = state->regs;
8552 u32 ref_obj_id = regs[regno].ref_obj_id;
8553 u32 id = regs[regno].id;
8554 int i;
8555
8556 if (ref_obj_id && ref_obj_id == id && is_null)
8557
8558
8559
8560
8561 WARN_ON_ONCE(release_reference_state(state, id));
8562
8563 for (i = 0; i <= vstate->curframe; i++)
8564 __mark_ptr_or_null_regs(vstate->frame[i], id, is_null);
8565}
8566
8567static bool try_match_pkt_pointers(const struct bpf_insn *insn,
8568 struct bpf_reg_state *dst_reg,
8569 struct bpf_reg_state *src_reg,
8570 struct bpf_verifier_state *this_branch,
8571 struct bpf_verifier_state *other_branch)
8572{
8573 if (BPF_SRC(insn->code) != BPF_X)
8574 return false;
8575
8576
8577 if (BPF_CLASS(insn->code) == BPF_JMP32)
8578 return false;
8579
8580 switch (BPF_OP(insn->code)) {
8581 case BPF_JGT:
8582 if ((dst_reg->type == PTR_TO_PACKET &&
8583 src_reg->type == PTR_TO_PACKET_END) ||
8584 (dst_reg->type == PTR_TO_PACKET_META &&
8585 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
8586
8587 find_good_pkt_pointers(this_branch, dst_reg,
8588 dst_reg->type, false);
8589 mark_pkt_end(other_branch, insn->dst_reg, true);
8590 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
8591 src_reg->type == PTR_TO_PACKET) ||
8592 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
8593 src_reg->type == PTR_TO_PACKET_META)) {
8594
8595 find_good_pkt_pointers(other_branch, src_reg,
8596 src_reg->type, true);
8597 mark_pkt_end(this_branch, insn->src_reg, false);
8598 } else {
8599 return false;
8600 }
8601 break;
8602 case BPF_JLT:
8603 if ((dst_reg->type == PTR_TO_PACKET &&
8604 src_reg->type == PTR_TO_PACKET_END) ||
8605 (dst_reg->type == PTR_TO_PACKET_META &&
8606 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
8607
8608 find_good_pkt_pointers(other_branch, dst_reg,
8609 dst_reg->type, true);
8610 mark_pkt_end(this_branch, insn->dst_reg, false);
8611 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
8612 src_reg->type == PTR_TO_PACKET) ||
8613 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
8614 src_reg->type == PTR_TO_PACKET_META)) {
8615
8616 find_good_pkt_pointers(this_branch, src_reg,
8617 src_reg->type, false);
8618 mark_pkt_end(other_branch, insn->src_reg, true);
8619 } else {
8620 return false;
8621 }
8622 break;
8623 case BPF_JGE:
8624 if ((dst_reg->type == PTR_TO_PACKET &&
8625 src_reg->type == PTR_TO_PACKET_END) ||
8626 (dst_reg->type == PTR_TO_PACKET_META &&
8627 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
8628
8629 find_good_pkt_pointers(this_branch, dst_reg,
8630 dst_reg->type, true);
8631 mark_pkt_end(other_branch, insn->dst_reg, false);
8632 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
8633 src_reg->type == PTR_TO_PACKET) ||
8634 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
8635 src_reg->type == PTR_TO_PACKET_META)) {
8636
8637 find_good_pkt_pointers(other_branch, src_reg,
8638 src_reg->type, false);
8639 mark_pkt_end(this_branch, insn->src_reg, true);
8640 } else {
8641 return false;
8642 }
8643 break;
8644 case BPF_JLE:
8645 if ((dst_reg->type == PTR_TO_PACKET &&
8646 src_reg->type == PTR_TO_PACKET_END) ||
8647 (dst_reg->type == PTR_TO_PACKET_META &&
8648 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
8649
8650 find_good_pkt_pointers(other_branch, dst_reg,
8651 dst_reg->type, false);
8652 mark_pkt_end(this_branch, insn->dst_reg, true);
8653 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
8654 src_reg->type == PTR_TO_PACKET) ||
8655 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
8656 src_reg->type == PTR_TO_PACKET_META)) {
8657
8658 find_good_pkt_pointers(this_branch, src_reg,
8659 src_reg->type, true);
8660 mark_pkt_end(other_branch, insn->src_reg, false);
8661 } else {
8662 return false;
8663 }
8664 break;
8665 default:
8666 return false;
8667 }
8668
8669 return true;
8670}
8671
8672static void find_equal_scalars(struct bpf_verifier_state *vstate,
8673 struct bpf_reg_state *known_reg)
8674{
8675 struct bpf_func_state *state;
8676 struct bpf_reg_state *reg;
8677 int i, j;
8678
8679 for (i = 0; i <= vstate->curframe; i++) {
8680 state = vstate->frame[i];
8681 for (j = 0; j < MAX_BPF_REG; j++) {
8682 reg = &state->regs[j];
8683 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
8684 *reg = *known_reg;
8685 }
8686
8687 bpf_for_each_spilled_reg(j, state, reg) {
8688 if (!reg)
8689 continue;
8690 if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
8691 *reg = *known_reg;
8692 }
8693 }
8694}
8695
8696static int check_cond_jmp_op(struct bpf_verifier_env *env,
8697 struct bpf_insn *insn, int *insn_idx)
8698{
8699 struct bpf_verifier_state *this_branch = env->cur_state;
8700 struct bpf_verifier_state *other_branch;
8701 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
8702 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;
8703 u8 opcode = BPF_OP(insn->code);
8704 bool is_jmp32;
8705 int pred = -1;
8706 int err;
8707
8708
8709 if (opcode == BPF_JA || opcode > BPF_JSLE) {
8710 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
8711 return -EINVAL;
8712 }
8713
8714 if (BPF_SRC(insn->code) == BPF_X) {
8715 if (insn->imm != 0) {
8716 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
8717 return -EINVAL;
8718 }
8719
8720
8721 err = check_reg_arg(env, insn->src_reg, SRC_OP);
8722 if (err)
8723 return err;
8724
8725 if (is_pointer_value(env, insn->src_reg)) {
8726 verbose(env, "R%d pointer comparison prohibited\n",
8727 insn->src_reg);
8728 return -EACCES;
8729 }
8730 src_reg = ®s[insn->src_reg];
8731 } else {
8732 if (insn->src_reg != BPF_REG_0) {
8733 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
8734 return -EINVAL;
8735 }
8736 }
8737
8738
8739 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
8740 if (err)
8741 return err;
8742
8743 dst_reg = ®s[insn->dst_reg];
8744 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
8745
8746 if (BPF_SRC(insn->code) == BPF_K) {
8747 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32);
8748 } else if (src_reg->type == SCALAR_VALUE &&
8749 is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) {
8750 pred = is_branch_taken(dst_reg,
8751 tnum_subreg(src_reg->var_off).value,
8752 opcode,
8753 is_jmp32);
8754 } else if (src_reg->type == SCALAR_VALUE &&
8755 !is_jmp32 && tnum_is_const(src_reg->var_off)) {
8756 pred = is_branch_taken(dst_reg,
8757 src_reg->var_off.value,
8758 opcode,
8759 is_jmp32);
8760 } else if (reg_is_pkt_pointer_any(dst_reg) &&
8761 reg_is_pkt_pointer_any(src_reg) &&
8762 !is_jmp32) {
8763 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode);
8764 }
8765
8766 if (pred >= 0) {
8767
8768
8769
8770 if (!__is_pointer_value(false, dst_reg))
8771 err = mark_chain_precision(env, insn->dst_reg);
8772 if (BPF_SRC(insn->code) == BPF_X && !err &&
8773 !__is_pointer_value(false, src_reg))
8774 err = mark_chain_precision(env, insn->src_reg);
8775 if (err)
8776 return err;
8777 }
8778
8779 if (pred == 1) {
8780
8781
8782
8783
8784 if (!env->bypass_spec_v1 &&
8785 !sanitize_speculative_path(env, insn, *insn_idx + 1,
8786 *insn_idx))
8787 return -EFAULT;
8788 *insn_idx += insn->off;
8789 return 0;
8790 } else if (pred == 0) {
8791
8792
8793
8794
8795 if (!env->bypass_spec_v1 &&
8796 !sanitize_speculative_path(env, insn,
8797 *insn_idx + insn->off + 1,
8798 *insn_idx))
8799 return -EFAULT;
8800 return 0;
8801 }
8802
8803 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,
8804 false);
8805 if (!other_branch)
8806 return -EFAULT;
8807 other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
8808
8809
8810
8811
8812
8813
8814
8815
8816 if (BPF_SRC(insn->code) == BPF_X) {
8817 struct bpf_reg_state *src_reg = ®s[insn->src_reg];
8818
8819 if (dst_reg->type == SCALAR_VALUE &&
8820 src_reg->type == SCALAR_VALUE) {
8821 if (tnum_is_const(src_reg->var_off) ||
8822 (is_jmp32 &&
8823 tnum_is_const(tnum_subreg(src_reg->var_off))))
8824 reg_set_min_max(&other_branch_regs[insn->dst_reg],
8825 dst_reg,
8826 src_reg->var_off.value,
8827 tnum_subreg(src_reg->var_off).value,
8828 opcode, is_jmp32);
8829 else if (tnum_is_const(dst_reg->var_off) ||
8830 (is_jmp32 &&
8831 tnum_is_const(tnum_subreg(dst_reg->var_off))))
8832 reg_set_min_max_inv(&other_branch_regs[insn->src_reg],
8833 src_reg,
8834 dst_reg->var_off.value,
8835 tnum_subreg(dst_reg->var_off).value,
8836 opcode, is_jmp32);
8837 else if (!is_jmp32 &&
8838 (opcode == BPF_JEQ || opcode == BPF_JNE))
8839
8840 reg_combine_min_max(&other_branch_regs[insn->src_reg],
8841 &other_branch_regs[insn->dst_reg],
8842 src_reg, dst_reg, opcode);
8843 if (src_reg->id &&
8844 !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) {
8845 find_equal_scalars(this_branch, src_reg);
8846 find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]);
8847 }
8848
8849 }
8850 } else if (dst_reg->type == SCALAR_VALUE) {
8851 reg_set_min_max(&other_branch_regs[insn->dst_reg],
8852 dst_reg, insn->imm, (u32)insn->imm,
8853 opcode, is_jmp32);
8854 }
8855
8856 if (dst_reg->type == SCALAR_VALUE && dst_reg->id &&
8857 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) {
8858 find_equal_scalars(this_branch, dst_reg);
8859 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]);
8860 }
8861
8862
8863
8864
8865
8866 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
8867 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
8868 reg_type_may_be_null(dst_reg->type)) {
8869
8870
8871
8872 mark_ptr_or_null_regs(this_branch, insn->dst_reg,
8873 opcode == BPF_JNE);
8874 mark_ptr_or_null_regs(other_branch, insn->dst_reg,
8875 opcode == BPF_JEQ);
8876 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg],
8877 this_branch, other_branch) &&
8878 is_pointer_value(env, insn->dst_reg)) {
8879 verbose(env, "R%d pointer comparison prohibited\n",
8880 insn->dst_reg);
8881 return -EACCES;
8882 }
8883 if (env->log.level & BPF_LOG_LEVEL)
8884 print_verifier_state(env, this_branch->frame[this_branch->curframe]);
8885 return 0;
8886}
8887
8888
8889static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
8890{
8891 struct bpf_insn_aux_data *aux = cur_aux(env);
8892 struct bpf_reg_state *regs = cur_regs(env);
8893 struct bpf_reg_state *dst_reg;
8894 struct bpf_map *map;
8895 int err;
8896
8897 if (BPF_SIZE(insn->code) != BPF_DW) {
8898 verbose(env, "invalid BPF_LD_IMM insn\n");
8899 return -EINVAL;
8900 }
8901 if (insn->off != 0) {
8902 verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
8903 return -EINVAL;
8904 }
8905
8906 err = check_reg_arg(env, insn->dst_reg, DST_OP);
8907 if (err)
8908 return err;
8909
8910 dst_reg = ®s[insn->dst_reg];
8911 if (insn->src_reg == 0) {
8912 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
8913
8914 dst_reg->type = SCALAR_VALUE;
8915 __mark_reg_known(®s[insn->dst_reg], imm);
8916 return 0;
8917 }
8918
8919 if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
8920 mark_reg_known_zero(env, regs, insn->dst_reg);
8921
8922 dst_reg->type = aux->btf_var.reg_type;
8923 switch (dst_reg->type) {
8924 case PTR_TO_MEM:
8925 dst_reg->mem_size = aux->btf_var.mem_size;
8926 break;
8927 case PTR_TO_BTF_ID:
8928 case PTR_TO_PERCPU_BTF_ID:
8929 dst_reg->btf = aux->btf_var.btf;
8930 dst_reg->btf_id = aux->btf_var.btf_id;
8931 break;
8932 default:
8933 verbose(env, "bpf verifier is misconfigured\n");
8934 return -EFAULT;
8935 }
8936 return 0;
8937 }
8938
8939 if (insn->src_reg == BPF_PSEUDO_FUNC) {
8940 struct bpf_prog_aux *aux = env->prog->aux;
8941 u32 subprogno = insn[1].imm;
8942
8943 if (!aux->func_info) {
8944 verbose(env, "missing btf func_info\n");
8945 return -EINVAL;
8946 }
8947 if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) {
8948 verbose(env, "callback function not static\n");
8949 return -EINVAL;
8950 }
8951
8952 dst_reg->type = PTR_TO_FUNC;
8953 dst_reg->subprogno = subprogno;
8954 return 0;
8955 }
8956
8957 map = env->used_maps[aux->map_index];
8958 mark_reg_known_zero(env, regs, insn->dst_reg);
8959 dst_reg->map_ptr = map;
8960
8961 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
8962 insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) {
8963 dst_reg->type = PTR_TO_MAP_VALUE;
8964 dst_reg->off = aux->map_off;
8965 if (map_value_has_spin_lock(map))
8966 dst_reg->id = ++env->id_gen;
8967 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD ||
8968 insn->src_reg == BPF_PSEUDO_MAP_IDX) {
8969 dst_reg->type = CONST_PTR_TO_MAP;
8970 } else {
8971 verbose(env, "bpf verifier is misconfigured\n");
8972 return -EINVAL;
8973 }
8974
8975 return 0;
8976}
8977
8978static bool may_access_skb(enum bpf_prog_type type)
8979{
8980 switch (type) {
8981 case BPF_PROG_TYPE_SOCKET_FILTER:
8982 case BPF_PROG_TYPE_SCHED_CLS:
8983 case BPF_PROG_TYPE_SCHED_ACT:
8984 return true;
8985 default:
8986 return false;
8987 }
8988}
8989
8990
8991
8992
8993
8994
8995
8996
8997
8998
8999
9000
9001
9002
9003
9004
9005static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
9006{
9007 struct bpf_reg_state *regs = cur_regs(env);
9008 static const int ctx_reg = BPF_REG_6;
9009 u8 mode = BPF_MODE(insn->code);
9010 int i, err;
9011
9012 if (!may_access_skb(resolve_prog_type(env->prog))) {
9013 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
9014 return -EINVAL;
9015 }
9016
9017 if (!env->ops->gen_ld_abs) {
9018 verbose(env, "bpf verifier is misconfigured\n");
9019 return -EINVAL;
9020 }
9021
9022 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||
9023 BPF_SIZE(insn->code) == BPF_DW ||
9024 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {
9025 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
9026 return -EINVAL;
9027 }
9028
9029
9030 err = check_reg_arg(env, ctx_reg, SRC_OP);
9031 if (err)
9032 return err;
9033
9034
9035
9036
9037
9038 err = check_reference_leak(env);
9039 if (err) {
9040 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");
9041 return err;
9042 }
9043
9044 if (env->cur_state->active_spin_lock) {
9045 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");
9046 return -EINVAL;
9047 }
9048
9049 if (regs[ctx_reg].type != PTR_TO_CTX) {
9050 verbose(env,
9051 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
9052 return -EINVAL;
9053 }
9054
9055 if (mode == BPF_IND) {
9056
9057 err = check_reg_arg(env, insn->src_reg, SRC_OP);
9058 if (err)
9059 return err;
9060 }
9061
9062 err = check_ctx_reg(env, ®s[ctx_reg], ctx_reg);
9063 if (err < 0)
9064 return err;
9065
9066
9067 for (i = 0; i < CALLER_SAVED_REGS; i++) {
9068 mark_reg_not_init(env, regs, caller_saved[i]);
9069 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
9070 }
9071
9072
9073
9074
9075
9076 mark_reg_unknown(env, regs, BPF_REG_0);
9077
9078 regs[BPF_REG_0].subreg_def = env->insn_idx + 1;
9079 return 0;
9080}
9081
9082static int check_return_code(struct bpf_verifier_env *env)
9083{
9084 struct tnum enforce_attach_type_range = tnum_unknown;
9085 const struct bpf_prog *prog = env->prog;
9086 struct bpf_reg_state *reg;
9087 struct tnum range = tnum_range(0, 1);
9088 enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
9089 int err;
9090 const bool is_subprog = env->cur_state->frame[0]->subprogno;
9091
9092
9093 if (!is_subprog &&
9094 (prog_type == BPF_PROG_TYPE_STRUCT_OPS ||
9095 prog_type == BPF_PROG_TYPE_LSM) &&
9096 !prog->aux->attach_func_proto->type)
9097 return 0;
9098
9099
9100
9101
9102
9103
9104
9105 err = check_reg_arg(env, BPF_REG_0, SRC_OP);
9106 if (err)
9107 return err;
9108
9109 if (is_pointer_value(env, BPF_REG_0)) {
9110 verbose(env, "R0 leaks addr as return value\n");
9111 return -EACCES;
9112 }
9113
9114 reg = cur_regs(env) + BPF_REG_0;
9115 if (is_subprog) {
9116 if (reg->type != SCALAR_VALUE) {
9117 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",
9118 reg_type_str[reg->type]);
9119 return -EINVAL;
9120 }
9121 return 0;
9122 }
9123
9124 switch (prog_type) {
9125 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
9126 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||
9127 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||
9128 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||
9129 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||
9130 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
9131 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME)
9132 range = tnum_range(1, 1);
9133 if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||
9134 env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)
9135 range = tnum_range(0, 3);
9136 break;
9137 case BPF_PROG_TYPE_CGROUP_SKB:
9138 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
9139 range = tnum_range(0, 3);
9140 enforce_attach_type_range = tnum_range(2, 3);
9141 }
9142 break;
9143 case BPF_PROG_TYPE_CGROUP_SOCK:
9144 case BPF_PROG_TYPE_SOCK_OPS:
9145 case BPF_PROG_TYPE_CGROUP_DEVICE:
9146 case BPF_PROG_TYPE_CGROUP_SYSCTL:
9147 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
9148 break;
9149 case BPF_PROG_TYPE_RAW_TRACEPOINT:
9150 if (!env->prog->aux->attach_btf_id)
9151 return 0;
9152 range = tnum_const(0);
9153 break;
9154 case BPF_PROG_TYPE_TRACING:
9155 switch (env->prog->expected_attach_type) {
9156 case BPF_TRACE_FENTRY:
9157 case BPF_TRACE_FEXIT:
9158 range = tnum_const(0);
9159 break;
9160 case BPF_TRACE_RAW_TP:
9161 case BPF_MODIFY_RETURN:
9162 return 0;
9163 case BPF_TRACE_ITER:
9164 break;
9165 default:
9166 return -ENOTSUPP;
9167 }
9168 break;
9169 case BPF_PROG_TYPE_SK_LOOKUP:
9170 range = tnum_range(SK_DROP, SK_PASS);
9171 break;
9172 case BPF_PROG_TYPE_EXT:
9173
9174
9175
9176 default:
9177 return 0;
9178 }
9179
9180 if (reg->type != SCALAR_VALUE) {
9181 verbose(env, "At program exit the register R0 is not a known value (%s)\n",
9182 reg_type_str[reg->type]);
9183 return -EINVAL;
9184 }
9185
9186 if (!tnum_in(range, reg->var_off)) {
9187 verbose_invalid_scalar(env, reg, &range, "program exit", "R0");
9188 return -EINVAL;
9189 }
9190
9191 if (!tnum_is_unknown(enforce_attach_type_range) &&
9192 tnum_in(enforce_attach_type_range, reg->var_off))
9193 env->prog->enforce_expected_attach_type = 1;
9194 return 0;
9195}
9196
9197
9198
9199
9200
9201
9202
9203
9204
9205
9206
9207
9208
9209
9210
9211
9212
9213
9214
9215
9216
9217
9218
9219
9220
9221
9222
9223
9224
9225
9226
9227
9228
9229
9230enum {
9231 DISCOVERED = 0x10,
9232 EXPLORED = 0x20,
9233 FALLTHROUGH = 1,
9234 BRANCH = 2,
9235};
9236
9237static u32 state_htab_size(struct bpf_verifier_env *env)
9238{
9239 return env->prog->len;
9240}
9241
9242static struct bpf_verifier_state_list **explored_state(
9243 struct bpf_verifier_env *env,
9244 int idx)
9245{
9246 struct bpf_verifier_state *cur = env->cur_state;
9247 struct bpf_func_state *state = cur->frame[cur->curframe];
9248
9249 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];
9250}
9251
9252static void init_explored_state(struct bpf_verifier_env *env, int idx)
9253{
9254 env->insn_aux_data[idx].prune_point = true;
9255}
9256
9257enum {
9258 DONE_EXPLORING = 0,
9259 KEEP_EXPLORING = 1,
9260};
9261
9262
9263
9264
9265
9266
9267static int push_insn(int t, int w, int e, struct bpf_verifier_env *env,
9268 bool loop_ok)
9269{
9270 int *insn_stack = env->cfg.insn_stack;
9271 int *insn_state = env->cfg.insn_state;
9272
9273 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
9274 return DONE_EXPLORING;
9275
9276 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
9277 return DONE_EXPLORING;
9278
9279 if (w < 0 || w >= env->prog->len) {
9280 verbose_linfo(env, t, "%d: ", t);
9281 verbose(env, "jump out of range from insn %d to %d\n", t, w);
9282 return -EINVAL;
9283 }
9284
9285 if (e == BRANCH)
9286
9287 init_explored_state(env, w);
9288
9289 if (insn_state[w] == 0) {
9290
9291 insn_state[t] = DISCOVERED | e;
9292 insn_state[w] = DISCOVERED;
9293 if (env->cfg.cur_stack >= env->prog->len)
9294 return -E2BIG;
9295 insn_stack[env->cfg.cur_stack++] = w;
9296 return KEEP_EXPLORING;
9297 } else if ((insn_state[w] & 0xF0) == DISCOVERED) {
9298 if (loop_ok && env->bpf_capable)
9299 return DONE_EXPLORING;
9300 verbose_linfo(env, t, "%d: ", t);
9301 verbose_linfo(env, w, "%d: ", w);
9302 verbose(env, "back-edge from insn %d to %d\n", t, w);
9303 return -EINVAL;
9304 } else if (insn_state[w] == EXPLORED) {
9305
9306 insn_state[t] = DISCOVERED | e;
9307 } else {
9308 verbose(env, "insn state internal bug\n");
9309 return -EFAULT;
9310 }
9311 return DONE_EXPLORING;
9312}
9313
9314static int visit_func_call_insn(int t, int insn_cnt,
9315 struct bpf_insn *insns,
9316 struct bpf_verifier_env *env,
9317 bool visit_callee)
9318{
9319 int ret;
9320
9321 ret = push_insn(t, t + 1, FALLTHROUGH, env, false);
9322 if (ret)
9323 return ret;
9324
9325 if (t + 1 < insn_cnt)
9326 init_explored_state(env, t + 1);
9327 if (visit_callee) {
9328 init_explored_state(env, t);
9329 ret = push_insn(t, t + insns[t].imm + 1, BRANCH,
9330 env, false);
9331 }
9332 return ret;
9333}
9334
9335
9336
9337
9338
9339
9340static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env)
9341{
9342 struct bpf_insn *insns = env->prog->insnsi;
9343 int ret;
9344
9345 if (bpf_pseudo_func(insns + t))
9346 return visit_func_call_insn(t, insn_cnt, insns, env, true);
9347
9348
9349 if (BPF_CLASS(insns[t].code) != BPF_JMP &&
9350 BPF_CLASS(insns[t].code) != BPF_JMP32)
9351 return push_insn(t, t + 1, FALLTHROUGH, env, false);
9352
9353 switch (BPF_OP(insns[t].code)) {
9354 case BPF_EXIT:
9355 return DONE_EXPLORING;
9356
9357 case BPF_CALL:
9358 return visit_func_call_insn(t, insn_cnt, insns, env,
9359 insns[t].src_reg == BPF_PSEUDO_CALL);
9360
9361 case BPF_JA:
9362 if (BPF_SRC(insns[t].code) != BPF_K)
9363 return -EINVAL;
9364
9365
9366 ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env,
9367 true);
9368 if (ret)
9369 return ret;
9370
9371
9372
9373
9374
9375 init_explored_state(env, t + insns[t].off + 1);
9376
9377
9378
9379 if (t + 1 < insn_cnt)
9380 init_explored_state(env, t + 1);
9381
9382 return ret;
9383
9384 default:
9385
9386 init_explored_state(env, t);
9387 ret = push_insn(t, t + 1, FALLTHROUGH, env, true);
9388 if (ret)
9389 return ret;
9390
9391 return push_insn(t, t + insns[t].off + 1, BRANCH, env, true);
9392 }
9393}
9394
9395
9396
9397
9398static int check_cfg(struct bpf_verifier_env *env)
9399{
9400 int insn_cnt = env->prog->len;
9401 int *insn_stack, *insn_state;
9402 int ret = 0;
9403 int i;
9404
9405 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
9406 if (!insn_state)
9407 return -ENOMEM;
9408
9409 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
9410 if (!insn_stack) {
9411 kvfree(insn_state);
9412 return -ENOMEM;
9413 }
9414
9415 insn_state[0] = DISCOVERED;
9416 insn_stack[0] = 0;
9417 env->cfg.cur_stack = 1;
9418
9419 while (env->cfg.cur_stack > 0) {
9420 int t = insn_stack[env->cfg.cur_stack - 1];
9421
9422 ret = visit_insn(t, insn_cnt, env);
9423 switch (ret) {
9424 case DONE_EXPLORING:
9425 insn_state[t] = EXPLORED;
9426 env->cfg.cur_stack--;
9427 break;
9428 case KEEP_EXPLORING:
9429 break;
9430 default:
9431 if (ret > 0) {
9432 verbose(env, "visit_insn internal bug\n");
9433 ret = -EFAULT;
9434 }
9435 goto err_free;
9436 }
9437 }
9438
9439 if (env->cfg.cur_stack < 0) {
9440 verbose(env, "pop stack internal bug\n");
9441 ret = -EFAULT;
9442 goto err_free;
9443 }
9444
9445 for (i = 0; i < insn_cnt; i++) {
9446 if (insn_state[i] != EXPLORED) {
9447 verbose(env, "unreachable insn %d\n", i);
9448 ret = -EINVAL;
9449 goto err_free;
9450 }
9451 }
9452 ret = 0;
9453
9454err_free:
9455 kvfree(insn_state);
9456 kvfree(insn_stack);
9457 env->cfg.insn_state = env->cfg.insn_stack = NULL;
9458 return ret;
9459}
9460
9461static int check_abnormal_return(struct bpf_verifier_env *env)
9462{
9463 int i;
9464
9465 for (i = 1; i < env->subprog_cnt; i++) {
9466 if (env->subprog_info[i].has_ld_abs) {
9467 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");
9468 return -EINVAL;
9469 }
9470 if (env->subprog_info[i].has_tail_call) {
9471 verbose(env, "tail_call is not allowed in subprogs without BTF\n");
9472 return -EINVAL;
9473 }
9474 }
9475 return 0;
9476}
9477
9478
9479#define MIN_BPF_FUNCINFO_SIZE 8
9480#define MAX_FUNCINFO_REC_SIZE 252
9481
9482static int check_btf_func(struct bpf_verifier_env *env,
9483 const union bpf_attr *attr,
9484 bpfptr_t uattr)
9485{
9486 const struct btf_type *type, *func_proto, *ret_type;
9487 u32 i, nfuncs, urec_size, min_size;
9488 u32 krec_size = sizeof(struct bpf_func_info);
9489 struct bpf_func_info *krecord;
9490 struct bpf_func_info_aux *info_aux = NULL;
9491 struct bpf_prog *prog;
9492 const struct btf *btf;
9493 bpfptr_t urecord;
9494 u32 prev_offset = 0;
9495 bool scalar_return;
9496 int ret = -ENOMEM;
9497
9498 nfuncs = attr->func_info_cnt;
9499 if (!nfuncs) {
9500 if (check_abnormal_return(env))
9501 return -EINVAL;
9502 return 0;
9503 }
9504
9505 if (nfuncs != env->subprog_cnt) {
9506 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
9507 return -EINVAL;
9508 }
9509
9510 urec_size = attr->func_info_rec_size;
9511 if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
9512 urec_size > MAX_FUNCINFO_REC_SIZE ||
9513 urec_size % sizeof(u32)) {
9514 verbose(env, "invalid func info rec size %u\n", urec_size);
9515 return -EINVAL;
9516 }
9517
9518 prog = env->prog;
9519 btf = prog->aux->btf;
9520
9521 urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
9522 min_size = min_t(u32, krec_size, urec_size);
9523
9524 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
9525 if (!krecord)
9526 return -ENOMEM;
9527 info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN);
9528 if (!info_aux)
9529 goto err_free;
9530
9531 for (i = 0; i < nfuncs; i++) {
9532 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
9533 if (ret) {
9534 if (ret == -E2BIG) {
9535 verbose(env, "nonzero tailing record in func info");
9536
9537
9538
9539 if (copy_to_bpfptr_offset(uattr,
9540 offsetof(union bpf_attr, func_info_rec_size),
9541 &min_size, sizeof(min_size)))
9542 ret = -EFAULT;
9543 }
9544 goto err_free;
9545 }
9546
9547 if (copy_from_bpfptr(&krecord[i], urecord, min_size)) {
9548 ret = -EFAULT;
9549 goto err_free;
9550 }
9551
9552
9553 ret = -EINVAL;
9554 if (i == 0) {
9555 if (krecord[i].insn_off) {
9556 verbose(env,
9557 "nonzero insn_off %u for the first func info record",
9558 krecord[i].insn_off);
9559 goto err_free;
9560 }
9561 } else if (krecord[i].insn_off <= prev_offset) {
9562 verbose(env,
9563 "same or smaller insn offset (%u) than previous func info record (%u)",
9564 krecord[i].insn_off, prev_offset);
9565 goto err_free;
9566 }
9567
9568 if (env->subprog_info[i].start != krecord[i].insn_off) {
9569 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
9570 goto err_free;
9571 }
9572
9573
9574 type = btf_type_by_id(btf, krecord[i].type_id);
9575 if (!type || !btf_type_is_func(type)) {
9576 verbose(env, "invalid type id %d in func info",
9577 krecord[i].type_id);
9578 goto err_free;
9579 }
9580 info_aux[i].linkage = BTF_INFO_VLEN(type->info);
9581
9582 func_proto = btf_type_by_id(btf, type->type);
9583 if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto)))
9584
9585 goto err_free;
9586 ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
9587 scalar_return =
9588 btf_type_is_small_int(ret_type) || btf_type_is_enum(ret_type);
9589 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {
9590 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");
9591 goto err_free;
9592 }
9593 if (i && !scalar_return && env->subprog_info[i].has_tail_call) {
9594 verbose(env, "tail_call is only allowed in functions that return 'int'.\n");
9595 goto err_free;
9596 }
9597
9598 prev_offset = krecord[i].insn_off;
9599 bpfptr_add(&urecord, urec_size);
9600 }
9601
9602 prog->aux->func_info = krecord;
9603 prog->aux->func_info_cnt = nfuncs;
9604 prog->aux->func_info_aux = info_aux;
9605 return 0;
9606
9607err_free:
9608 kvfree(krecord);
9609 kfree(info_aux);
9610 return ret;
9611}
9612
9613static void adjust_btf_func(struct bpf_verifier_env *env)
9614{
9615 struct bpf_prog_aux *aux = env->prog->aux;
9616 int i;
9617
9618 if (!aux->func_info)
9619 return;
9620
9621 for (i = 0; i < env->subprog_cnt; i++)
9622 aux->func_info[i].insn_off = env->subprog_info[i].start;
9623}
9624
9625#define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \
9626 sizeof(((struct bpf_line_info *)(0))->line_col))
9627#define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE
9628
9629static int check_btf_line(struct bpf_verifier_env *env,
9630 const union bpf_attr *attr,
9631 bpfptr_t uattr)
9632{
9633 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
9634 struct bpf_subprog_info *sub;
9635 struct bpf_line_info *linfo;
9636 struct bpf_prog *prog;
9637 const struct btf *btf;
9638 bpfptr_t ulinfo;
9639 int err;
9640
9641 nr_linfo = attr->line_info_cnt;
9642 if (!nr_linfo)
9643 return 0;
9644
9645 rec_size = attr->line_info_rec_size;
9646 if (rec_size < MIN_BPF_LINEINFO_SIZE ||
9647 rec_size > MAX_LINEINFO_REC_SIZE ||
9648 rec_size & (sizeof(u32) - 1))
9649 return -EINVAL;
9650
9651
9652
9653
9654 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
9655 GFP_KERNEL | __GFP_NOWARN);
9656 if (!linfo)
9657 return -ENOMEM;
9658
9659 prog = env->prog;
9660 btf = prog->aux->btf;
9661
9662 s = 0;
9663 sub = env->subprog_info;
9664 ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel);
9665 expected_size = sizeof(struct bpf_line_info);
9666 ncopy = min_t(u32, expected_size, rec_size);
9667 for (i = 0; i < nr_linfo; i++) {
9668 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
9669 if (err) {
9670 if (err == -E2BIG) {
9671 verbose(env, "nonzero tailing record in line_info");
9672 if (copy_to_bpfptr_offset(uattr,
9673 offsetof(union bpf_attr, line_info_rec_size),
9674 &expected_size, sizeof(expected_size)))
9675 err = -EFAULT;
9676 }
9677 goto err_free;
9678 }
9679
9680 if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) {
9681 err = -EFAULT;
9682 goto err_free;
9683 }
9684
9685
9686
9687
9688
9689
9690
9691
9692
9693
9694
9695
9696 if ((i && linfo[i].insn_off <= prev_offset) ||
9697 linfo[i].insn_off >= prog->len) {
9698 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
9699 i, linfo[i].insn_off, prev_offset,
9700 prog->len);
9701 err = -EINVAL;
9702 goto err_free;
9703 }
9704
9705 if (!prog->insnsi[linfo[i].insn_off].code) {
9706 verbose(env,
9707 "Invalid insn code at line_info[%u].insn_off\n",
9708 i);
9709 err = -EINVAL;
9710 goto err_free;
9711 }
9712
9713 if (!btf_name_by_offset(btf, linfo[i].line_off) ||
9714 !btf_name_by_offset(btf, linfo[i].file_name_off)) {
9715 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
9716 err = -EINVAL;
9717 goto err_free;
9718 }
9719
9720 if (s != env->subprog_cnt) {
9721 if (linfo[i].insn_off == sub[s].start) {
9722 sub[s].linfo_idx = i;
9723 s++;
9724 } else if (sub[s].start < linfo[i].insn_off) {
9725 verbose(env, "missing bpf_line_info for func#%u\n", s);
9726 err = -EINVAL;
9727 goto err_free;
9728 }
9729 }
9730
9731 prev_offset = linfo[i].insn_off;
9732 bpfptr_add(&ulinfo, rec_size);
9733 }
9734
9735 if (s != env->subprog_cnt) {
9736 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
9737 env->subprog_cnt - s, s);
9738 err = -EINVAL;
9739 goto err_free;
9740 }
9741
9742 prog->aux->linfo = linfo;
9743 prog->aux->nr_linfo = nr_linfo;
9744
9745 return 0;
9746
9747err_free:
9748 kvfree(linfo);
9749 return err;
9750}
9751
9752static int check_btf_info(struct bpf_verifier_env *env,
9753 const union bpf_attr *attr,
9754 bpfptr_t uattr)
9755{
9756 struct btf *btf;
9757 int err;
9758
9759 if (!attr->func_info_cnt && !attr->line_info_cnt) {
9760 if (check_abnormal_return(env))
9761 return -EINVAL;
9762 return 0;
9763 }
9764
9765 btf = btf_get_by_fd(attr->prog_btf_fd);
9766 if (IS_ERR(btf))
9767 return PTR_ERR(btf);
9768 if (btf_is_kernel(btf)) {
9769 btf_put(btf);
9770 return -EACCES;
9771 }
9772 env->prog->aux->btf = btf;
9773
9774 err = check_btf_func(env, attr, uattr);
9775 if (err)
9776 return err;
9777
9778 err = check_btf_line(env, attr, uattr);
9779 if (err)
9780 return err;
9781
9782 return 0;
9783}
9784
9785
9786static bool range_within(struct bpf_reg_state *old,
9787 struct bpf_reg_state *cur)
9788{
9789 return old->umin_value <= cur->umin_value &&
9790 old->umax_value >= cur->umax_value &&
9791 old->smin_value <= cur->smin_value &&
9792 old->smax_value >= cur->smax_value &&
9793 old->u32_min_value <= cur->u32_min_value &&
9794 old->u32_max_value >= cur->u32_max_value &&
9795 old->s32_min_value <= cur->s32_min_value &&
9796 old->s32_max_value >= cur->s32_max_value;
9797}
9798
9799
9800
9801
9802
9803
9804
9805
9806
9807
9808
9809static bool check_ids(u32 old_id, u32 cur_id, struct bpf_id_pair *idmap)
9810{
9811 unsigned int i;
9812
9813 for (i = 0; i < BPF_ID_MAP_SIZE; i++) {
9814 if (!idmap[i].old) {
9815
9816 idmap[i].old = old_id;
9817 idmap[i].cur = cur_id;
9818 return true;
9819 }
9820 if (idmap[i].old == old_id)
9821 return idmap[i].cur == cur_id;
9822 }
9823
9824 WARN_ON_ONCE(1);
9825 return false;
9826}
9827
9828static void clean_func_state(struct bpf_verifier_env *env,
9829 struct bpf_func_state *st)
9830{
9831 enum bpf_reg_liveness live;
9832 int i, j;
9833
9834 for (i = 0; i < BPF_REG_FP; i++) {
9835 live = st->regs[i].live;
9836
9837 st->regs[i].live |= REG_LIVE_DONE;
9838 if (!(live & REG_LIVE_READ))
9839
9840
9841
9842 __mark_reg_not_init(env, &st->regs[i]);
9843 }
9844
9845 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) {
9846 live = st->stack[i].spilled_ptr.live;
9847
9848 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE;
9849 if (!(live & REG_LIVE_READ)) {
9850 __mark_reg_not_init(env, &st->stack[i].spilled_ptr);
9851 for (j = 0; j < BPF_REG_SIZE; j++)
9852 st->stack[i].slot_type[j] = STACK_INVALID;
9853 }
9854 }
9855}
9856
9857static void clean_verifier_state(struct bpf_verifier_env *env,
9858 struct bpf_verifier_state *st)
9859{
9860 int i;
9861
9862 if (st->frame[0]->regs[0].live & REG_LIVE_DONE)
9863
9864 return;
9865
9866 for (i = 0; i <= st->curframe; i++)
9867 clean_func_state(env, st->frame[i]);
9868}
9869
9870
9871
9872
9873
9874
9875
9876
9877
9878
9879
9880
9881
9882
9883
9884
9885
9886
9887
9888
9889
9890
9891
9892
9893
9894
9895
9896
9897
9898
9899
9900
9901
9902static void clean_live_states(struct bpf_verifier_env *env, int insn,
9903 struct bpf_verifier_state *cur)
9904{
9905 struct bpf_verifier_state_list *sl;
9906 int i;
9907
9908 sl = *explored_state(env, insn);
9909 while (sl) {
9910 if (sl->state.branches)
9911 goto next;
9912 if (sl->state.insn_idx != insn ||
9913 sl->state.curframe != cur->curframe)
9914 goto next;
9915 for (i = 0; i <= cur->curframe; i++)
9916 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite)
9917 goto next;
9918 clean_verifier_state(env, &sl->state);
9919next:
9920 sl = sl->next;
9921 }
9922}
9923
9924
9925static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
9926 struct bpf_reg_state *rcur, struct bpf_id_pair *idmap)
9927{
9928 bool equal;
9929
9930 if (!(rold->live & REG_LIVE_READ))
9931
9932 return true;
9933
9934 equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0;
9935
9936 if (rold->type == PTR_TO_STACK)
9937
9938
9939
9940 return equal && rold->frameno == rcur->frameno;
9941
9942 if (equal)
9943 return true;
9944
9945 if (rold->type == NOT_INIT)
9946
9947 return true;
9948 if (rcur->type == NOT_INIT)
9949 return false;
9950 switch (rold->type) {
9951 case SCALAR_VALUE:
9952 if (env->explore_alu_limits)
9953 return false;
9954 if (rcur->type == SCALAR_VALUE) {
9955 if (!rold->precise && !rcur->precise)
9956 return true;
9957
9958 return range_within(rold, rcur) &&
9959 tnum_in(rold->var_off, rcur->var_off);
9960 } else {
9961
9962
9963
9964
9965
9966
9967
9968 return false;
9969 }
9970 case PTR_TO_MAP_KEY:
9971 case PTR_TO_MAP_VALUE:
9972
9973
9974
9975
9976
9977
9978
9979
9980 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
9981 range_within(rold, rcur) &&
9982 tnum_in(rold->var_off, rcur->var_off);
9983 case PTR_TO_MAP_VALUE_OR_NULL:
9984
9985
9986
9987
9988
9989
9990
9991 if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL)
9992 return false;
9993 if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
9994 return false;
9995
9996 return check_ids(rold->id, rcur->id, idmap);
9997 case PTR_TO_PACKET_META:
9998 case PTR_TO_PACKET:
9999 if (rcur->type != rold->type)
10000 return false;
10001
10002
10003
10004
10005
10006
10007 if (rold->range > rcur->range)
10008 return false;
10009
10010
10011
10012 if (rold->off != rcur->off)
10013 return false;
10014
10015 if (rold->id && !check_ids(rold->id, rcur->id, idmap))
10016 return false;
10017
10018 return range_within(rold, rcur) &&
10019 tnum_in(rold->var_off, rcur->var_off);
10020 case PTR_TO_CTX:
10021 case CONST_PTR_TO_MAP:
10022 case PTR_TO_PACKET_END:
10023 case PTR_TO_FLOW_KEYS:
10024 case PTR_TO_SOCKET:
10025 case PTR_TO_SOCKET_OR_NULL:
10026 case PTR_TO_SOCK_COMMON:
10027 case PTR_TO_SOCK_COMMON_OR_NULL:
10028 case PTR_TO_TCP_SOCK:
10029 case PTR_TO_TCP_SOCK_OR_NULL:
10030 case PTR_TO_XDP_SOCK:
10031
10032
10033
10034 default:
10035
10036 return false;
10037 }
10038
10039
10040 WARN_ON_ONCE(1);
10041 return false;
10042}
10043
10044static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
10045 struct bpf_func_state *cur, struct bpf_id_pair *idmap)
10046{
10047 int i, spi;
10048
10049
10050
10051
10052
10053 for (i = 0; i < old->allocated_stack; i++) {
10054 spi = i / BPF_REG_SIZE;
10055
10056 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) {
10057 i += BPF_REG_SIZE - 1;
10058
10059 continue;
10060 }
10061
10062 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID)
10063 continue;
10064
10065
10066
10067
10068 if (i >= cur->allocated_stack)
10069 return false;
10070
10071
10072
10073
10074
10075 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC &&
10076 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO)
10077 continue;
10078 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
10079 cur->stack[spi].slot_type[i % BPF_REG_SIZE])
10080
10081
10082
10083
10084
10085 return false;
10086 if (i % BPF_REG_SIZE)
10087 continue;
10088 if (old->stack[spi].slot_type[0] != STACK_SPILL)
10089 continue;
10090 if (!regsafe(env, &old->stack[spi].spilled_ptr,
10091 &cur->stack[spi].spilled_ptr, idmap))
10092
10093
10094
10095
10096
10097
10098
10099
10100
10101
10102 return false;
10103 }
10104 return true;
10105}
10106
10107static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur)
10108{
10109 if (old->acquired_refs != cur->acquired_refs)
10110 return false;
10111 return !memcmp(old->refs, cur->refs,
10112 sizeof(*old->refs) * old->acquired_refs);
10113}
10114
10115
10116
10117
10118
10119
10120
10121
10122
10123
10124
10125
10126
10127
10128
10129
10130
10131
10132
10133
10134
10135
10136
10137
10138
10139
10140
10141static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,
10142 struct bpf_func_state *cur)
10143{
10144 int i;
10145
10146 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch));
10147 for (i = 0; i < MAX_BPF_REG; i++)
10148 if (!regsafe(env, &old->regs[i], &cur->regs[i],
10149 env->idmap_scratch))
10150 return false;
10151
10152 if (!stacksafe(env, old, cur, env->idmap_scratch))
10153 return false;
10154
10155 if (!refsafe(old, cur))
10156 return false;
10157
10158 return true;
10159}
10160
10161static bool states_equal(struct bpf_verifier_env *env,
10162 struct bpf_verifier_state *old,
10163 struct bpf_verifier_state *cur)
10164{
10165 int i;
10166
10167 if (old->curframe != cur->curframe)
10168 return false;
10169
10170
10171
10172
10173 if (old->speculative && !cur->speculative)
10174 return false;
10175
10176 if (old->active_spin_lock != cur->active_spin_lock)
10177 return false;
10178
10179
10180
10181
10182 for (i = 0; i <= old->curframe; i++) {
10183 if (old->frame[i]->callsite != cur->frame[i]->callsite)
10184 return false;
10185 if (!func_states_equal(env, old->frame[i], cur->frame[i]))
10186 return false;
10187 }
10188 return true;
10189}
10190
10191
10192
10193
10194static int propagate_liveness_reg(struct bpf_verifier_env *env,
10195 struct bpf_reg_state *reg,
10196 struct bpf_reg_state *parent_reg)
10197{
10198 u8 parent_flag = parent_reg->live & REG_LIVE_READ;
10199 u8 flag = reg->live & REG_LIVE_READ;
10200 int err;
10201
10202
10203
10204
10205
10206 if (parent_flag == REG_LIVE_READ64 ||
10207
10208 !flag ||
10209
10210 parent_flag == flag)
10211 return 0;
10212
10213 err = mark_reg_read(env, reg, parent_reg, flag);
10214 if (err)
10215 return err;
10216
10217 return flag;
10218}
10219
10220
10221
10222
10223
10224
10225
10226
10227static int propagate_liveness(struct bpf_verifier_env *env,
10228 const struct bpf_verifier_state *vstate,
10229 struct bpf_verifier_state *vparent)
10230{
10231 struct bpf_reg_state *state_reg, *parent_reg;
10232 struct bpf_func_state *state, *parent;
10233 int i, frame, err = 0;
10234
10235 if (vparent->curframe != vstate->curframe) {
10236 WARN(1, "propagate_live: parent frame %d current frame %d\n",
10237 vparent->curframe, vstate->curframe);
10238 return -EFAULT;
10239 }
10240
10241 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
10242 for (frame = 0; frame <= vstate->curframe; frame++) {
10243 parent = vparent->frame[frame];
10244 state = vstate->frame[frame];
10245 parent_reg = parent->regs;
10246 state_reg = state->regs;
10247
10248 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) {
10249 err = propagate_liveness_reg(env, &state_reg[i],
10250 &parent_reg[i]);
10251 if (err < 0)
10252 return err;
10253 if (err == REG_LIVE_READ64)
10254 mark_insn_zext(env, &parent_reg[i]);
10255 }
10256
10257
10258 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE &&
10259 i < parent->allocated_stack / BPF_REG_SIZE; i++) {
10260 parent_reg = &parent->stack[i].spilled_ptr;
10261 state_reg = &state->stack[i].spilled_ptr;
10262 err = propagate_liveness_reg(env, state_reg,
10263 parent_reg);
10264 if (err < 0)
10265 return err;
10266 }
10267 }
10268 return 0;
10269}
10270
10271
10272
10273
10274static int propagate_precision(struct bpf_verifier_env *env,
10275 const struct bpf_verifier_state *old)
10276{
10277 struct bpf_reg_state *state_reg;
10278 struct bpf_func_state *state;
10279 int i, err = 0;
10280
10281 state = old->frame[old->curframe];
10282 state_reg = state->regs;
10283 for (i = 0; i < BPF_REG_FP; i++, state_reg++) {
10284 if (state_reg->type != SCALAR_VALUE ||
10285 !state_reg->precise)
10286 continue;
10287 if (env->log.level & BPF_LOG_LEVEL2)
10288 verbose(env, "propagating r%d\n", i);
10289 err = mark_chain_precision(env, i);
10290 if (err < 0)
10291 return err;
10292 }
10293
10294 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
10295 if (state->stack[i].slot_type[0] != STACK_SPILL)
10296 continue;
10297 state_reg = &state->stack[i].spilled_ptr;
10298 if (state_reg->type != SCALAR_VALUE ||
10299 !state_reg->precise)
10300 continue;
10301 if (env->log.level & BPF_LOG_LEVEL2)
10302 verbose(env, "propagating fp%d\n",
10303 (-i - 1) * BPF_REG_SIZE);
10304 err = mark_chain_precision_stack(env, i);
10305 if (err < 0)
10306 return err;
10307 }
10308 return 0;
10309}
10310
10311static bool states_maybe_looping(struct bpf_verifier_state *old,
10312 struct bpf_verifier_state *cur)
10313{
10314 struct bpf_func_state *fold, *fcur;
10315 int i, fr = cur->curframe;
10316
10317 if (old->curframe != fr)
10318 return false;
10319
10320 fold = old->frame[fr];
10321 fcur = cur->frame[fr];
10322 for (i = 0; i < MAX_BPF_REG; i++)
10323 if (memcmp(&fold->regs[i], &fcur->regs[i],
10324 offsetof(struct bpf_reg_state, parent)))
10325 return false;
10326 return true;
10327}
10328
10329
10330static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
10331{
10332 struct bpf_verifier_state_list *new_sl;
10333 struct bpf_verifier_state_list *sl, **pprev;
10334 struct bpf_verifier_state *cur = env->cur_state, *new;
10335 int i, j, err, states_cnt = 0;
10336 bool add_new_state = env->test_state_freq ? true : false;
10337
10338 cur->last_insn_idx = env->prev_insn_idx;
10339 if (!env->insn_aux_data[insn_idx].prune_point)
10340
10341
10342
10343 return 0;
10344
10345
10346
10347
10348
10349
10350
10351
10352
10353 if (env->jmps_processed - env->prev_jmps_processed >= 2 &&
10354 env->insn_processed - env->prev_insn_processed >= 8)
10355 add_new_state = true;
10356
10357 pprev = explored_state(env, insn_idx);
10358 sl = *pprev;
10359
10360 clean_live_states(env, insn_idx, cur);
10361
10362 while (sl) {
10363 states_cnt++;
10364 if (sl->state.insn_idx != insn_idx)
10365 goto next;
10366 if (sl->state.branches) {
10367 if (states_maybe_looping(&sl->state, cur) &&
10368 states_equal(env, &sl->state, cur)) {
10369 verbose_linfo(env, insn_idx, "; ");
10370 verbose(env, "infinite loop detected at insn %d\n", insn_idx);
10371 return -EINVAL;
10372 }
10373
10374
10375
10376
10377
10378
10379
10380
10381
10382
10383
10384
10385 if (env->jmps_processed - env->prev_jmps_processed < 20 &&
10386 env->insn_processed - env->prev_insn_processed < 100)
10387 add_new_state = false;
10388 goto miss;
10389 }
10390 if (states_equal(env, &sl->state, cur)) {
10391 sl->hit_cnt++;
10392
10393
10394
10395
10396
10397
10398
10399
10400
10401
10402 err = propagate_liveness(env, &sl->state, cur);
10403
10404
10405
10406
10407
10408
10409 err = err ? : push_jmp_history(env, cur);
10410 err = err ? : propagate_precision(env, &sl->state);
10411 if (err)
10412 return err;
10413 return 1;
10414 }
10415miss:
10416
10417
10418
10419
10420
10421
10422 if (add_new_state)
10423 sl->miss_cnt++;
10424
10425
10426
10427
10428
10429 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) {
10430
10431
10432
10433 *pprev = sl->next;
10434 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) {
10435 u32 br = sl->state.branches;
10436
10437 WARN_ONCE(br,
10438 "BUG live_done but branches_to_explore %d\n",
10439 br);
10440 free_verifier_state(&sl->state, false);
10441 kfree(sl);
10442 env->peak_states--;
10443 } else {
10444
10445
10446
10447
10448 sl->next = env->free_list;
10449 env->free_list = sl;
10450 }
10451 sl = *pprev;
10452 continue;
10453 }
10454next:
10455 pprev = &sl->next;
10456 sl = *pprev;
10457 }
10458
10459 if (env->max_states_per_insn < states_cnt)
10460 env->max_states_per_insn = states_cnt;
10461
10462 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)
10463 return push_jmp_history(env, cur);
10464
10465 if (!add_new_state)
10466 return push_jmp_history(env, cur);
10467
10468
10469
10470
10471
10472
10473
10474
10475
10476
10477 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL);
10478 if (!new_sl)
10479 return -ENOMEM;
10480 env->total_states++;
10481 env->peak_states++;
10482 env->prev_jmps_processed = env->jmps_processed;
10483 env->prev_insn_processed = env->insn_processed;
10484
10485
10486 new = &new_sl->state;
10487 err = copy_verifier_state(new, cur);
10488 if (err) {
10489 free_verifier_state(new, false);
10490 kfree(new_sl);
10491 return err;
10492 }
10493 new->insn_idx = insn_idx;
10494 WARN_ONCE(new->branches != 1,
10495 "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx);
10496
10497 cur->parent = new;
10498 cur->first_insn_idx = insn_idx;
10499 clear_jmp_history(cur);
10500 new_sl->next = *explored_state(env, insn_idx);
10501 *explored_state(env, insn_idx) = new_sl;
10502
10503
10504
10505
10506
10507
10508
10509
10510
10511
10512
10513
10514
10515 for (j = 0; j <= cur->curframe; j++) {
10516 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++)
10517 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i];
10518 for (i = 0; i < BPF_REG_FP; i++)
10519 cur->frame[j]->regs[i].live = REG_LIVE_NONE;
10520 }
10521
10522
10523 for (j = 0; j <= cur->curframe; j++) {
10524 struct bpf_func_state *frame = cur->frame[j];
10525 struct bpf_func_state *newframe = new->frame[j];
10526
10527 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) {
10528 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE;
10529 frame->stack[i].spilled_ptr.parent =
10530 &newframe->stack[i].spilled_ptr;
10531 }
10532 }
10533 return 0;
10534}
10535
10536
10537static bool reg_type_mismatch_ok(enum bpf_reg_type type)
10538{
10539 switch (type) {
10540 case PTR_TO_CTX:
10541 case PTR_TO_SOCKET:
10542 case PTR_TO_SOCKET_OR_NULL:
10543 case PTR_TO_SOCK_COMMON:
10544 case PTR_TO_SOCK_COMMON_OR_NULL:
10545 case PTR_TO_TCP_SOCK:
10546 case PTR_TO_TCP_SOCK_OR_NULL:
10547 case PTR_TO_XDP_SOCK:
10548 case PTR_TO_BTF_ID:
10549 case PTR_TO_BTF_ID_OR_NULL:
10550 return false;
10551 default:
10552 return true;
10553 }
10554}
10555
10556
10557
10558
10559
10560
10561
10562
10563
10564
10565
10566
10567
10568static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev)
10569{
10570 return src != prev && (!reg_type_mismatch_ok(src) ||
10571 !reg_type_mismatch_ok(prev));
10572}
10573
10574static int do_check(struct bpf_verifier_env *env)
10575{
10576 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
10577 struct bpf_verifier_state *state = env->cur_state;
10578 struct bpf_insn *insns = env->prog->insnsi;
10579 struct bpf_reg_state *regs;
10580 int insn_cnt = env->prog->len;
10581 bool do_print_state = false;
10582 int prev_insn_idx = -1;
10583
10584 for (;;) {
10585 struct bpf_insn *insn;
10586 u8 class;
10587 int err;
10588
10589 env->prev_insn_idx = prev_insn_idx;
10590 if (env->insn_idx >= insn_cnt) {
10591 verbose(env, "invalid insn idx %d insn_cnt %d\n",
10592 env->insn_idx, insn_cnt);
10593 return -EFAULT;
10594 }
10595
10596 insn = &insns[env->insn_idx];
10597 class = BPF_CLASS(insn->code);
10598
10599 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
10600 verbose(env,
10601 "BPF program is too large. Processed %d insn\n",
10602 env->insn_processed);
10603 return -E2BIG;
10604 }
10605
10606 err = is_state_visited(env, env->insn_idx);
10607 if (err < 0)
10608 return err;
10609 if (err == 1) {
10610
10611 if (env->log.level & BPF_LOG_LEVEL) {
10612 if (do_print_state)
10613 verbose(env, "\nfrom %d to %d%s: safe\n",
10614 env->prev_insn_idx, env->insn_idx,
10615 env->cur_state->speculative ?
10616 " (speculative execution)" : "");
10617 else
10618 verbose(env, "%d: safe\n", env->insn_idx);
10619 }
10620 goto process_bpf_exit;
10621 }
10622
10623 if (signal_pending(current))
10624 return -EAGAIN;
10625
10626 if (need_resched())
10627 cond_resched();
10628
10629 if (env->log.level & BPF_LOG_LEVEL2 ||
10630 (env->log.level & BPF_LOG_LEVEL && do_print_state)) {
10631 if (env->log.level & BPF_LOG_LEVEL2)
10632 verbose(env, "%d:", env->insn_idx);
10633 else
10634 verbose(env, "\nfrom %d to %d%s:",
10635 env->prev_insn_idx, env->insn_idx,
10636 env->cur_state->speculative ?
10637 " (speculative execution)" : "");
10638 print_verifier_state(env, state->frame[state->curframe]);
10639 do_print_state = false;
10640 }
10641
10642 if (env->log.level & BPF_LOG_LEVEL) {
10643 const struct bpf_insn_cbs cbs = {
10644 .cb_call = disasm_kfunc_name,
10645 .cb_print = verbose,
10646 .private_data = env,
10647 };
10648
10649 verbose_linfo(env, env->insn_idx, "; ");
10650 verbose(env, "%d: ", env->insn_idx);
10651 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
10652 }
10653
10654 if (bpf_prog_is_dev_bound(env->prog->aux)) {
10655 err = bpf_prog_offload_verify_insn(env, env->insn_idx,
10656 env->prev_insn_idx);
10657 if (err)
10658 return err;
10659 }
10660
10661 regs = cur_regs(env);
10662 sanitize_mark_insn_seen(env);
10663 prev_insn_idx = env->insn_idx;
10664
10665 if (class == BPF_ALU || class == BPF_ALU64) {
10666 err = check_alu_op(env, insn);
10667 if (err)
10668 return err;
10669
10670 } else if (class == BPF_LDX) {
10671 enum bpf_reg_type *prev_src_type, src_reg_type;
10672
10673
10674
10675
10676 err = check_reg_arg(env, insn->src_reg, SRC_OP);
10677 if (err)
10678 return err;
10679
10680 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
10681 if (err)
10682 return err;
10683
10684 src_reg_type = regs[insn->src_reg].type;
10685
10686
10687
10688
10689 err = check_mem_access(env, env->insn_idx, insn->src_reg,
10690 insn->off, BPF_SIZE(insn->code),
10691 BPF_READ, insn->dst_reg, false);
10692 if (err)
10693 return err;
10694
10695 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type;
10696
10697 if (*prev_src_type == NOT_INIT) {
10698
10699
10700
10701
10702 *prev_src_type = src_reg_type;
10703
10704 } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) {
10705
10706
10707
10708
10709
10710
10711
10712 verbose(env, "same insn cannot be used with different pointers\n");
10713 return -EINVAL;
10714 }
10715
10716 } else if (class == BPF_STX) {
10717 enum bpf_reg_type *prev_dst_type, dst_reg_type;
10718
10719 if (BPF_MODE(insn->code) == BPF_ATOMIC) {
10720 err = check_atomic(env, env->insn_idx, insn);
10721 if (err)
10722 return err;
10723 env->insn_idx++;
10724 continue;
10725 }
10726
10727 if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) {
10728 verbose(env, "BPF_STX uses reserved fields\n");
10729 return -EINVAL;
10730 }
10731
10732
10733 err = check_reg_arg(env, insn->src_reg, SRC_OP);
10734 if (err)
10735 return err;
10736
10737 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
10738 if (err)
10739 return err;
10740
10741 dst_reg_type = regs[insn->dst_reg].type;
10742
10743
10744 err = check_mem_access(env, env->insn_idx, insn->dst_reg,
10745 insn->off, BPF_SIZE(insn->code),
10746 BPF_WRITE, insn->src_reg, false);
10747 if (err)
10748 return err;
10749
10750 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type;
10751
10752 if (*prev_dst_type == NOT_INIT) {
10753 *prev_dst_type = dst_reg_type;
10754 } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) {
10755 verbose(env, "same insn cannot be used with different pointers\n");
10756 return -EINVAL;
10757 }
10758
10759 } else if (class == BPF_ST) {
10760 if (BPF_MODE(insn->code) != BPF_MEM ||
10761 insn->src_reg != BPF_REG_0) {
10762 verbose(env, "BPF_ST uses reserved fields\n");
10763 return -EINVAL;
10764 }
10765
10766 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
10767 if (err)
10768 return err;
10769
10770 if (is_ctx_reg(env, insn->dst_reg)) {
10771 verbose(env, "BPF_ST stores into R%d %s is not allowed\n",
10772 insn->dst_reg,
10773 reg_type_str[reg_state(env, insn->dst_reg)->type]);
10774 return -EACCES;
10775 }
10776
10777
10778 err = check_mem_access(env, env->insn_idx, insn->dst_reg,
10779 insn->off, BPF_SIZE(insn->code),
10780 BPF_WRITE, -1, false);
10781 if (err)
10782 return err;
10783
10784 } else if (class == BPF_JMP || class == BPF_JMP32) {
10785 u8 opcode = BPF_OP(insn->code);
10786
10787 env->jmps_processed++;
10788 if (opcode == BPF_CALL) {
10789 if (BPF_SRC(insn->code) != BPF_K ||
10790 insn->off != 0 ||
10791 (insn->src_reg != BPF_REG_0 &&
10792 insn->src_reg != BPF_PSEUDO_CALL &&
10793 insn->src_reg != BPF_PSEUDO_KFUNC_CALL) ||
10794 insn->dst_reg != BPF_REG_0 ||
10795 class == BPF_JMP32) {
10796 verbose(env, "BPF_CALL uses reserved fields\n");
10797 return -EINVAL;
10798 }
10799
10800 if (env->cur_state->active_spin_lock &&
10801 (insn->src_reg == BPF_PSEUDO_CALL ||
10802 insn->imm != BPF_FUNC_spin_unlock)) {
10803 verbose(env, "function calls are not allowed while holding a lock\n");
10804 return -EINVAL;
10805 }
10806 if (insn->src_reg == BPF_PSEUDO_CALL)
10807 err = check_func_call(env, insn, &env->insn_idx);
10808 else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
10809 err = check_kfunc_call(env, insn);
10810 else
10811 err = check_helper_call(env, insn, &env->insn_idx);
10812 if (err)
10813 return err;
10814 } else if (opcode == BPF_JA) {
10815 if (BPF_SRC(insn->code) != BPF_K ||
10816 insn->imm != 0 ||
10817 insn->src_reg != BPF_REG_0 ||
10818 insn->dst_reg != BPF_REG_0 ||
10819 class == BPF_JMP32) {
10820 verbose(env, "BPF_JA uses reserved fields\n");
10821 return -EINVAL;
10822 }
10823
10824 env->insn_idx += insn->off + 1;
10825 continue;
10826
10827 } else if (opcode == BPF_EXIT) {
10828 if (BPF_SRC(insn->code) != BPF_K ||
10829 insn->imm != 0 ||
10830 insn->src_reg != BPF_REG_0 ||
10831 insn->dst_reg != BPF_REG_0 ||
10832 class == BPF_JMP32) {
10833 verbose(env, "BPF_EXIT uses reserved fields\n");
10834 return -EINVAL;
10835 }
10836
10837 if (env->cur_state->active_spin_lock) {
10838 verbose(env, "bpf_spin_unlock is missing\n");
10839 return -EINVAL;
10840 }
10841
10842 if (state->curframe) {
10843
10844 err = prepare_func_exit(env, &env->insn_idx);
10845 if (err)
10846 return err;
10847 do_print_state = true;
10848 continue;
10849 }
10850
10851 err = check_reference_leak(env);
10852 if (err)
10853 return err;
10854
10855 err = check_return_code(env);
10856 if (err)
10857 return err;
10858process_bpf_exit:
10859 update_branch_counts(env, env->cur_state);
10860 err = pop_stack(env, &prev_insn_idx,
10861 &env->insn_idx, pop_log);
10862 if (err < 0) {
10863 if (err != -ENOENT)
10864 return err;
10865 break;
10866 } else {
10867 do_print_state = true;
10868 continue;
10869 }
10870 } else {
10871 err = check_cond_jmp_op(env, insn, &env->insn_idx);
10872 if (err)
10873 return err;
10874 }
10875 } else if (class == BPF_LD) {
10876 u8 mode = BPF_MODE(insn->code);
10877
10878 if (mode == BPF_ABS || mode == BPF_IND) {
10879 err = check_ld_abs(env, insn);
10880 if (err)
10881 return err;
10882
10883 } else if (mode == BPF_IMM) {
10884 err = check_ld_imm(env, insn);
10885 if (err)
10886 return err;
10887
10888 env->insn_idx++;
10889 sanitize_mark_insn_seen(env);
10890 } else {
10891 verbose(env, "invalid BPF_LD mode\n");
10892 return -EINVAL;
10893 }
10894 } else {
10895 verbose(env, "unknown insn class %d\n", class);
10896 return -EINVAL;
10897 }
10898
10899 env->insn_idx++;
10900 }
10901
10902 return 0;
10903}
10904
10905static int find_btf_percpu_datasec(struct btf *btf)
10906{
10907 const struct btf_type *t;
10908 const char *tname;
10909 int i, n;
10910
10911
10912
10913
10914
10915
10916 n = btf_nr_types(btf);
10917 if (btf_is_module(btf))
10918 i = btf_nr_types(btf_vmlinux);
10919 else
10920 i = 1;
10921
10922 for(; i < n; i++) {
10923 t = btf_type_by_id(btf, i);
10924 if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC)
10925 continue;
10926
10927 tname = btf_name_by_offset(btf, t->name_off);
10928 if (!strcmp(tname, ".data..percpu"))
10929 return i;
10930 }
10931
10932 return -ENOENT;
10933}
10934
10935
10936static int check_pseudo_btf_id(struct bpf_verifier_env *env,
10937 struct bpf_insn *insn,
10938 struct bpf_insn_aux_data *aux)
10939{
10940 const struct btf_var_secinfo *vsi;
10941 const struct btf_type *datasec;
10942 struct btf_mod_pair *btf_mod;
10943 const struct btf_type *t;
10944 const char *sym_name;
10945 bool percpu = false;
10946 u32 type, id = insn->imm;
10947 struct btf *btf;
10948 s32 datasec_id;
10949 u64 addr;
10950 int i, btf_fd, err;
10951
10952 btf_fd = insn[1].imm;
10953 if (btf_fd) {
10954 btf = btf_get_by_fd(btf_fd);
10955 if (IS_ERR(btf)) {
10956 verbose(env, "invalid module BTF object FD specified.\n");
10957 return -EINVAL;
10958 }
10959 } else {
10960 if (!btf_vmlinux) {
10961 verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
10962 return -EINVAL;
10963 }
10964 btf = btf_vmlinux;
10965 btf_get(btf);
10966 }
10967
10968 t = btf_type_by_id(btf, id);
10969 if (!t) {
10970 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
10971 err = -ENOENT;
10972 goto err_put;
10973 }
10974
10975 if (!btf_type_is_var(t)) {
10976 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id);
10977 err = -EINVAL;
10978 goto err_put;
10979 }
10980
10981 sym_name = btf_name_by_offset(btf, t->name_off);
10982 addr = kallsyms_lookup_name(sym_name);
10983 if (!addr) {
10984 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",
10985 sym_name);
10986 err = -ENOENT;
10987 goto err_put;
10988 }
10989
10990 datasec_id = find_btf_percpu_datasec(btf);
10991 if (datasec_id > 0) {
10992 datasec = btf_type_by_id(btf, datasec_id);
10993 for_each_vsi(i, datasec, vsi) {
10994 if (vsi->type == id) {
10995 percpu = true;
10996 break;
10997 }
10998 }
10999 }
11000
11001 insn[0].imm = (u32)addr;
11002 insn[1].imm = addr >> 32;
11003
11004 type = t->type;
11005 t = btf_type_skip_modifiers(btf, type, NULL);
11006 if (percpu) {
11007 aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID;
11008 aux->btf_var.btf = btf;
11009 aux->btf_var.btf_id = type;
11010 } else if (!btf_type_is_struct(t)) {
11011 const struct btf_type *ret;
11012 const char *tname;
11013 u32 tsize;
11014
11015
11016 ret = btf_resolve_size(btf, t, &tsize);
11017 if (IS_ERR(ret)) {
11018 tname = btf_name_by_offset(btf, t->name_off);
11019 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",
11020 tname, PTR_ERR(ret));
11021 err = -EINVAL;
11022 goto err_put;
11023 }
11024 aux->btf_var.reg_type = PTR_TO_MEM;
11025 aux->btf_var.mem_size = tsize;
11026 } else {
11027 aux->btf_var.reg_type = PTR_TO_BTF_ID;
11028 aux->btf_var.btf = btf;
11029 aux->btf_var.btf_id = type;
11030 }
11031
11032
11033 for (i = 0; i < env->used_btf_cnt; i++) {
11034 if (env->used_btfs[i].btf == btf) {
11035 btf_put(btf);
11036 return 0;
11037 }
11038 }
11039
11040 if (env->used_btf_cnt >= MAX_USED_BTFS) {
11041 err = -E2BIG;
11042 goto err_put;
11043 }
11044
11045 btf_mod = &env->used_btfs[env->used_btf_cnt];
11046 btf_mod->btf = btf;
11047 btf_mod->module = NULL;
11048
11049
11050 if (btf_is_module(btf)) {
11051 btf_mod->module = btf_try_get_module(btf);
11052 if (!btf_mod->module) {
11053 err = -ENXIO;
11054 goto err_put;
11055 }
11056 }
11057
11058 env->used_btf_cnt++;
11059
11060 return 0;
11061err_put:
11062 btf_put(btf);
11063 return err;
11064}
11065
11066static int check_map_prealloc(struct bpf_map *map)
11067{
11068 return (map->map_type != BPF_MAP_TYPE_HASH &&
11069 map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
11070 map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) ||
11071 !(map->map_flags & BPF_F_NO_PREALLOC);
11072}
11073
11074static bool is_tracing_prog_type(enum bpf_prog_type type)
11075{
11076 switch (type) {
11077 case BPF_PROG_TYPE_KPROBE:
11078 case BPF_PROG_TYPE_TRACEPOINT:
11079 case BPF_PROG_TYPE_PERF_EVENT:
11080 case BPF_PROG_TYPE_RAW_TRACEPOINT:
11081 return true;
11082 default:
11083 return false;
11084 }
11085}
11086
11087static bool is_preallocated_map(struct bpf_map *map)
11088{
11089 if (!check_map_prealloc(map))
11090 return false;
11091 if (map->inner_map_meta && !check_map_prealloc(map->inner_map_meta))
11092 return false;
11093 return true;
11094}
11095
11096static int check_map_prog_compatibility(struct bpf_verifier_env *env,
11097 struct bpf_map *map,
11098 struct bpf_prog *prog)
11099
11100{
11101 enum bpf_prog_type prog_type = resolve_prog_type(prog);
11102
11103
11104
11105
11106
11107
11108
11109
11110
11111
11112
11113
11114
11115
11116
11117
11118
11119 if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) {
11120 if (prog_type == BPF_PROG_TYPE_PERF_EVENT) {
11121 verbose(env, "perf_event programs can only use preallocated hash map\n");
11122 return -EINVAL;
11123 }
11124 if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
11125 verbose(env, "trace type programs can only use preallocated hash map\n");
11126 return -EINVAL;
11127 }
11128 WARN_ONCE(1, "trace type BPF program uses run-time allocation\n");
11129 verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n");
11130 }
11131
11132 if (map_value_has_spin_lock(map)) {
11133 if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) {
11134 verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");
11135 return -EINVAL;
11136 }
11137
11138 if (is_tracing_prog_type(prog_type)) {
11139 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
11140 return -EINVAL;
11141 }
11142
11143 if (prog->aux->sleepable) {
11144 verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n");
11145 return -EINVAL;
11146 }
11147 }
11148
11149 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) &&
11150 !bpf_offload_prog_map_match(prog, map)) {
11151 verbose(env, "offload device mismatch between prog and map\n");
11152 return -EINVAL;
11153 }
11154
11155 if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
11156 verbose(env, "bpf_struct_ops map cannot be used in prog\n");
11157 return -EINVAL;
11158 }
11159
11160 if (prog->aux->sleepable)
11161 switch (map->map_type) {
11162 case BPF_MAP_TYPE_HASH:
11163 case BPF_MAP_TYPE_LRU_HASH:
11164 case BPF_MAP_TYPE_ARRAY:
11165 case BPF_MAP_TYPE_PERCPU_HASH:
11166 case BPF_MAP_TYPE_PERCPU_ARRAY:
11167 case BPF_MAP_TYPE_LRU_PERCPU_HASH:
11168 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
11169 case BPF_MAP_TYPE_HASH_OF_MAPS:
11170 if (!is_preallocated_map(map)) {
11171 verbose(env,
11172 "Sleepable programs can only use preallocated maps\n");
11173 return -EINVAL;
11174 }
11175 break;
11176 case BPF_MAP_TYPE_RINGBUF:
11177 break;
11178 default:
11179 verbose(env,
11180 "Sleepable programs can only use array, hash, and ringbuf maps\n");
11181 return -EINVAL;
11182 }
11183
11184 return 0;
11185}
11186
11187static bool bpf_map_is_cgroup_storage(struct bpf_map *map)
11188{
11189 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
11190 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
11191}
11192
11193
11194
11195
11196
11197
11198
11199
11200static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
11201{
11202 struct bpf_insn *insn = env->prog->insnsi;
11203 int insn_cnt = env->prog->len;
11204 int i, j, err;
11205
11206 err = bpf_prog_calc_tag(env->prog);
11207 if (err)
11208 return err;
11209
11210 for (i = 0; i < insn_cnt; i++, insn++) {
11211 if (BPF_CLASS(insn->code) == BPF_LDX &&
11212 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) {
11213 verbose(env, "BPF_LDX uses reserved fields\n");
11214 return -EINVAL;
11215 }
11216
11217 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
11218 struct bpf_insn_aux_data *aux;
11219 struct bpf_map *map;
11220 struct fd f;
11221 u64 addr;
11222 u32 fd;
11223
11224 if (i == insn_cnt - 1 || insn[1].code != 0 ||
11225 insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
11226 insn[1].off != 0) {
11227 verbose(env, "invalid bpf_ld_imm64 insn\n");
11228 return -EINVAL;
11229 }
11230
11231 if (insn[0].src_reg == 0)
11232
11233 goto next_insn;
11234
11235 if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) {
11236 aux = &env->insn_aux_data[i];
11237 err = check_pseudo_btf_id(env, insn, aux);
11238 if (err)
11239 return err;
11240 goto next_insn;
11241 }
11242
11243 if (insn[0].src_reg == BPF_PSEUDO_FUNC) {
11244 aux = &env->insn_aux_data[i];
11245 aux->ptr_type = PTR_TO_FUNC;
11246 goto next_insn;
11247 }
11248
11249
11250
11251
11252 switch (insn[0].src_reg) {
11253 case BPF_PSEUDO_MAP_VALUE:
11254 case BPF_PSEUDO_MAP_IDX_VALUE:
11255 break;
11256 case BPF_PSEUDO_MAP_FD:
11257 case BPF_PSEUDO_MAP_IDX:
11258 if (insn[1].imm == 0)
11259 break;
11260 fallthrough;
11261 default:
11262 verbose(env, "unrecognized bpf_ld_imm64 insn\n");
11263 return -EINVAL;
11264 }
11265
11266 switch (insn[0].src_reg) {
11267 case BPF_PSEUDO_MAP_IDX_VALUE:
11268 case BPF_PSEUDO_MAP_IDX:
11269 if (bpfptr_is_null(env->fd_array)) {
11270 verbose(env, "fd_idx without fd_array is invalid\n");
11271 return -EPROTO;
11272 }
11273 if (copy_from_bpfptr_offset(&fd, env->fd_array,
11274 insn[0].imm * sizeof(fd),
11275 sizeof(fd)))
11276 return -EFAULT;
11277 break;
11278 default:
11279 fd = insn[0].imm;
11280 break;
11281 }
11282
11283 f = fdget(fd);
11284 map = __bpf_map_get(f);
11285 if (IS_ERR(map)) {
11286 verbose(env, "fd %d is not pointing to valid bpf_map\n",
11287 insn[0].imm);
11288 return PTR_ERR(map);
11289 }
11290
11291 err = check_map_prog_compatibility(env, map, env->prog);
11292 if (err) {
11293 fdput(f);
11294 return err;
11295 }
11296
11297 aux = &env->insn_aux_data[i];
11298 if (insn[0].src_reg == BPF_PSEUDO_MAP_FD ||
11299 insn[0].src_reg == BPF_PSEUDO_MAP_IDX) {
11300 addr = (unsigned long)map;
11301 } else {
11302 u32 off = insn[1].imm;
11303
11304 if (off >= BPF_MAX_VAR_OFF) {
11305 verbose(env, "direct value offset of %u is not allowed\n", off);
11306 fdput(f);
11307 return -EINVAL;
11308 }
11309
11310 if (!map->ops->map_direct_value_addr) {
11311 verbose(env, "no direct value access support for this map type\n");
11312 fdput(f);
11313 return -EINVAL;
11314 }
11315
11316 err = map->ops->map_direct_value_addr(map, &addr, off);
11317 if (err) {
11318 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",
11319 map->value_size, off);
11320 fdput(f);
11321 return err;
11322 }
11323
11324 aux->map_off = off;
11325 addr += off;
11326 }
11327
11328 insn[0].imm = (u32)addr;
11329 insn[1].imm = addr >> 32;
11330
11331
11332 for (j = 0; j < env->used_map_cnt; j++) {
11333 if (env->used_maps[j] == map) {
11334 aux->map_index = j;
11335 fdput(f);
11336 goto next_insn;
11337 }
11338 }
11339
11340 if (env->used_map_cnt >= MAX_USED_MAPS) {
11341 fdput(f);
11342 return -E2BIG;
11343 }
11344
11345
11346
11347
11348
11349
11350 bpf_map_inc(map);
11351
11352 aux->map_index = env->used_map_cnt;
11353 env->used_maps[env->used_map_cnt++] = map;
11354
11355 if (bpf_map_is_cgroup_storage(map) &&
11356 bpf_cgroup_storage_assign(env->prog->aux, map)) {
11357 verbose(env, "only one cgroup storage of each type is allowed\n");
11358 fdput(f);
11359 return -EBUSY;
11360 }
11361
11362 fdput(f);
11363next_insn:
11364 insn++;
11365 i++;
11366 continue;
11367 }
11368
11369
11370 if (!bpf_opcode_in_insntable(insn->code)) {
11371 verbose(env, "unknown opcode %02x\n", insn->code);
11372 return -EINVAL;
11373 }
11374 }
11375
11376
11377
11378
11379
11380 return 0;
11381}
11382
11383
11384static void release_maps(struct bpf_verifier_env *env)
11385{
11386 __bpf_free_used_maps(env->prog->aux, env->used_maps,
11387 env->used_map_cnt);
11388}
11389
11390
11391static void release_btfs(struct bpf_verifier_env *env)
11392{
11393 __bpf_free_used_btfs(env->prog->aux, env->used_btfs,
11394 env->used_btf_cnt);
11395}
11396
11397
11398static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
11399{
11400 struct bpf_insn *insn = env->prog->insnsi;
11401 int insn_cnt = env->prog->len;
11402 int i;
11403
11404 for (i = 0; i < insn_cnt; i++, insn++) {
11405 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW))
11406 continue;
11407 if (insn->src_reg == BPF_PSEUDO_FUNC)
11408 continue;
11409 insn->src_reg = 0;
11410 }
11411}
11412
11413
11414
11415
11416
11417static int adjust_insn_aux_data(struct bpf_verifier_env *env,
11418 struct bpf_prog *new_prog, u32 off, u32 cnt)
11419{
11420 struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data;
11421 struct bpf_insn *insn = new_prog->insnsi;
11422 u32 old_seen = old_data[off].seen;
11423 u32 prog_len;
11424 int i;
11425
11426
11427
11428
11429
11430 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1);
11431
11432 if (cnt == 1)
11433 return 0;
11434 prog_len = new_prog->len;
11435 new_data = vzalloc(array_size(prog_len,
11436 sizeof(struct bpf_insn_aux_data)));
11437 if (!new_data)
11438 return -ENOMEM;
11439 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
11440 memcpy(new_data + off + cnt - 1, old_data + off,
11441 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
11442 for (i = off; i < off + cnt - 1; i++) {
11443
11444 new_data[i].seen = old_seen;
11445 new_data[i].zext_dst = insn_has_def32(env, insn + i);
11446 }
11447 env->insn_aux_data = new_data;
11448 vfree(old_data);
11449 return 0;
11450}
11451
11452static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
11453{
11454 int i;
11455
11456 if (len == 1)
11457 return;
11458
11459 for (i = 0; i <= env->subprog_cnt; i++) {
11460 if (env->subprog_info[i].start <= off)
11461 continue;
11462 env->subprog_info[i].start += len - 1;
11463 }
11464}
11465
11466static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
11467{
11468 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
11469 int i, sz = prog->aux->size_poke_tab;
11470 struct bpf_jit_poke_descriptor *desc;
11471
11472 for (i = 0; i < sz; i++) {
11473 desc = &tab[i];
11474 if (desc->insn_idx <= off)
11475 continue;
11476 desc->insn_idx += len - 1;
11477 }
11478}
11479
11480static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
11481 const struct bpf_insn *patch, u32 len)
11482{
11483 struct bpf_prog *new_prog;
11484
11485 new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
11486 if (IS_ERR(new_prog)) {
11487 if (PTR_ERR(new_prog) == -ERANGE)
11488 verbose(env,
11489 "insn %d cannot be patched due to 16-bit range\n",
11490 env->insn_aux_data[off].orig_idx);
11491 return NULL;
11492 }
11493 if (adjust_insn_aux_data(env, new_prog, off, len))
11494 return NULL;
11495 adjust_subprog_starts(env, off, len);
11496 adjust_poke_descs(new_prog, off, len);
11497 return new_prog;
11498}
11499
11500static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
11501 u32 off, u32 cnt)
11502{
11503 int i, j;
11504
11505
11506 for (i = 0; i < env->subprog_cnt; i++)
11507 if (env->subprog_info[i].start >= off)
11508 break;
11509
11510 for (j = i; j < env->subprog_cnt; j++)
11511 if (env->subprog_info[j].start >= off + cnt)
11512 break;
11513
11514
11515
11516 if (env->subprog_info[j].start != off + cnt)
11517 j--;
11518
11519 if (j > i) {
11520 struct bpf_prog_aux *aux = env->prog->aux;
11521 int move;
11522
11523
11524 move = env->subprog_cnt + 1 - j;
11525
11526 memmove(env->subprog_info + i,
11527 env->subprog_info + j,
11528 sizeof(*env->subprog_info) * move);
11529 env->subprog_cnt -= j - i;
11530
11531
11532 if (aux->func_info) {
11533 move = aux->func_info_cnt - j;
11534
11535 memmove(aux->func_info + i,
11536 aux->func_info + j,
11537 sizeof(*aux->func_info) * move);
11538 aux->func_info_cnt -= j - i;
11539
11540
11541
11542 }
11543 } else {
11544
11545 if (env->subprog_info[i].start == off)
11546 i++;
11547 }
11548
11549
11550 for (; i <= env->subprog_cnt; i++)
11551 env->subprog_info[i].start -= cnt;
11552
11553 return 0;
11554}
11555
11556static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
11557 u32 cnt)
11558{
11559 struct bpf_prog *prog = env->prog;
11560 u32 i, l_off, l_cnt, nr_linfo;
11561 struct bpf_line_info *linfo;
11562
11563 nr_linfo = prog->aux->nr_linfo;
11564 if (!nr_linfo)
11565 return 0;
11566
11567 linfo = prog->aux->linfo;
11568
11569
11570 for (i = 0; i < nr_linfo; i++)
11571 if (linfo[i].insn_off >= off)
11572 break;
11573
11574 l_off = i;
11575 l_cnt = 0;
11576 for (; i < nr_linfo; i++)
11577 if (linfo[i].insn_off < off + cnt)
11578 l_cnt++;
11579 else
11580 break;
11581
11582
11583
11584
11585
11586 if (prog->len != off && l_cnt &&
11587 (i == nr_linfo || linfo[i].insn_off != off + cnt)) {
11588 l_cnt--;
11589 linfo[--i].insn_off = off + cnt;
11590 }
11591
11592
11593 if (l_cnt) {
11594 memmove(linfo + l_off, linfo + i,
11595 sizeof(*linfo) * (nr_linfo - i));
11596
11597 prog->aux->nr_linfo -= l_cnt;
11598 nr_linfo = prog->aux->nr_linfo;
11599 }
11600
11601
11602 for (i = l_off; i < nr_linfo; i++)
11603 linfo[i].insn_off -= cnt;
11604
11605
11606 for (i = 0; i <= env->subprog_cnt; i++)
11607 if (env->subprog_info[i].linfo_idx > l_off) {
11608
11609
11610
11611 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)
11612 env->subprog_info[i].linfo_idx -= l_cnt;
11613 else
11614 env->subprog_info[i].linfo_idx = l_off;
11615 }
11616
11617 return 0;
11618}
11619
11620static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
11621{
11622 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
11623 unsigned int orig_prog_len = env->prog->len;
11624 int err;
11625
11626 if (bpf_prog_is_dev_bound(env->prog->aux))
11627 bpf_prog_offload_remove_insns(env, off, cnt);
11628
11629 err = bpf_remove_insns(env->prog, off, cnt);
11630 if (err)
11631 return err;
11632
11633 err = adjust_subprog_starts_after_remove(env, off, cnt);
11634 if (err)
11635 return err;
11636
11637 err = bpf_adj_linfo_after_remove(env, off, cnt);
11638 if (err)
11639 return err;
11640
11641 memmove(aux_data + off, aux_data + off + cnt,
11642 sizeof(*aux_data) * (orig_prog_len - off - cnt));
11643
11644 return 0;
11645}
11646
11647
11648
11649
11650
11651
11652
11653
11654
11655
11656
11657
11658static void sanitize_dead_code(struct bpf_verifier_env *env)
11659{
11660 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
11661 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
11662 struct bpf_insn *insn = env->prog->insnsi;
11663 const int insn_cnt = env->prog->len;
11664 int i;
11665
11666 for (i = 0; i < insn_cnt; i++) {
11667 if (aux_data[i].seen)
11668 continue;
11669 memcpy(insn + i, &trap, sizeof(trap));
11670 aux_data[i].zext_dst = false;
11671 }
11672}
11673
11674static bool insn_is_cond_jump(u8 code)
11675{
11676 u8 op;
11677
11678 if (BPF_CLASS(code) == BPF_JMP32)
11679 return true;
11680
11681 if (BPF_CLASS(code) != BPF_JMP)
11682 return false;
11683
11684 op = BPF_OP(code);
11685 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
11686}
11687
11688static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
11689{
11690 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
11691 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
11692 struct bpf_insn *insn = env->prog->insnsi;
11693 const int insn_cnt = env->prog->len;
11694 int i;
11695
11696 for (i = 0; i < insn_cnt; i++, insn++) {
11697 if (!insn_is_cond_jump(insn->code))
11698 continue;
11699
11700 if (!aux_data[i + 1].seen)
11701 ja.off = insn->off;
11702 else if (!aux_data[i + 1 + insn->off].seen)
11703 ja.off = 0;
11704 else
11705 continue;
11706
11707 if (bpf_prog_is_dev_bound(env->prog->aux))
11708 bpf_prog_offload_replace_insn(env, i, &ja);
11709
11710 memcpy(insn, &ja, sizeof(ja));
11711 }
11712}
11713
11714static int opt_remove_dead_code(struct bpf_verifier_env *env)
11715{
11716 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
11717 int insn_cnt = env->prog->len;
11718 int i, err;
11719
11720 for (i = 0; i < insn_cnt; i++) {
11721 int j;
11722
11723 j = 0;
11724 while (i + j < insn_cnt && !aux_data[i + j].seen)
11725 j++;
11726 if (!j)
11727 continue;
11728
11729 err = verifier_remove_insns(env, i, j);
11730 if (err)
11731 return err;
11732 insn_cnt = env->prog->len;
11733 }
11734
11735 return 0;
11736}
11737
11738static int opt_remove_nops(struct bpf_verifier_env *env)
11739{
11740 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
11741 struct bpf_insn *insn = env->prog->insnsi;
11742 int insn_cnt = env->prog->len;
11743 int i, err;
11744
11745 for (i = 0; i < insn_cnt; i++) {
11746 if (memcmp(&insn[i], &ja, sizeof(ja)))
11747 continue;
11748
11749 err = verifier_remove_insns(env, i, 1);
11750 if (err)
11751 return err;
11752 insn_cnt--;
11753 i--;
11754 }
11755
11756 return 0;
11757}
11758
11759static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,
11760 const union bpf_attr *attr)
11761{
11762 struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4];
11763 struct bpf_insn_aux_data *aux = env->insn_aux_data;
11764 int i, patch_len, delta = 0, len = env->prog->len;
11765 struct bpf_insn *insns = env->prog->insnsi;
11766 struct bpf_prog *new_prog;
11767 bool rnd_hi32;
11768
11769 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32;
11770 zext_patch[1] = BPF_ZEXT_REG(0);
11771 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0);
11772 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32);
11773 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX);
11774 for (i = 0; i < len; i++) {
11775 int adj_idx = i + delta;
11776 struct bpf_insn insn;
11777 int load_reg;
11778
11779 insn = insns[adj_idx];
11780 load_reg = insn_def_regno(&insn);
11781 if (!aux[adj_idx].zext_dst) {
11782 u8 code, class;
11783 u32 imm_rnd;
11784
11785 if (!rnd_hi32)
11786 continue;
11787
11788 code = insn.code;
11789 class = BPF_CLASS(code);
11790 if (load_reg == -1)
11791 continue;
11792
11793
11794
11795
11796
11797 if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) {
11798 if (class == BPF_LD &&
11799 BPF_MODE(code) == BPF_IMM)
11800 i++;
11801 continue;
11802 }
11803
11804
11805 if (class == BPF_LDX &&
11806 aux[adj_idx].ptr_type == PTR_TO_CTX)
11807 continue;
11808
11809 imm_rnd = get_random_int();
11810 rnd_hi32_patch[0] = insn;
11811 rnd_hi32_patch[1].imm = imm_rnd;
11812 rnd_hi32_patch[3].dst_reg = load_reg;
11813 patch = rnd_hi32_patch;
11814 patch_len = 4;
11815 goto apply_patch_buffer;
11816 }
11817
11818
11819
11820
11821
11822
11823
11824
11825
11826
11827
11828 if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn))
11829 continue;
11830
11831 if (WARN_ON(load_reg == -1)) {
11832 verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n");
11833 return -EFAULT;
11834 }
11835
11836 zext_patch[0] = insn;
11837 zext_patch[1].dst_reg = load_reg;
11838 zext_patch[1].src_reg = load_reg;
11839 patch = zext_patch;
11840 patch_len = 2;
11841apply_patch_buffer:
11842 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);
11843 if (!new_prog)
11844 return -ENOMEM;
11845 env->prog = new_prog;
11846 insns = new_prog->insnsi;
11847 aux = env->insn_aux_data;
11848 delta += patch_len - 1;
11849 }
11850
11851 return 0;
11852}
11853
11854
11855
11856
11857
11858
11859static int convert_ctx_accesses(struct bpf_verifier_env *env)
11860{
11861 const struct bpf_verifier_ops *ops = env->ops;
11862 int i, cnt, size, ctx_field_size, delta = 0;
11863 const int insn_cnt = env->prog->len;
11864 struct bpf_insn insn_buf[16], *insn;
11865 u32 target_size, size_default, off;
11866 struct bpf_prog *new_prog;
11867 enum bpf_access_type type;
11868 bool is_narrower_load;
11869
11870 if (ops->gen_prologue || env->seen_direct_write) {
11871 if (!ops->gen_prologue) {
11872 verbose(env, "bpf verifier is misconfigured\n");
11873 return -EINVAL;
11874 }
11875 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,
11876 env->prog);
11877 if (cnt >= ARRAY_SIZE(insn_buf)) {
11878 verbose(env, "bpf verifier is misconfigured\n");
11879 return -EINVAL;
11880 } else if (cnt) {
11881 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
11882 if (!new_prog)
11883 return -ENOMEM;
11884
11885 env->prog = new_prog;
11886 delta += cnt - 1;
11887 }
11888 }
11889
11890 if (bpf_prog_is_dev_bound(env->prog->aux))
11891 return 0;
11892
11893 insn = env->prog->insnsi + delta;
11894
11895 for (i = 0; i < insn_cnt; i++, insn++) {
11896 bpf_convert_ctx_access_t convert_ctx_access;
11897 bool ctx_access;
11898
11899 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
11900 insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
11901 insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
11902 insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) {
11903 type = BPF_READ;
11904 ctx_access = true;
11905 } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
11906 insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
11907 insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
11908 insn->code == (BPF_STX | BPF_MEM | BPF_DW) ||
11909 insn->code == (BPF_ST | BPF_MEM | BPF_B) ||
11910 insn->code == (BPF_ST | BPF_MEM | BPF_H) ||
11911 insn->code == (BPF_ST | BPF_MEM | BPF_W) ||
11912 insn->code == (BPF_ST | BPF_MEM | BPF_DW)) {
11913 type = BPF_WRITE;
11914 ctx_access = BPF_CLASS(insn->code) == BPF_STX;
11915 } else {
11916 continue;
11917 }
11918
11919 if (type == BPF_WRITE &&
11920 env->insn_aux_data[i + delta].sanitize_stack_spill) {
11921 struct bpf_insn patch[] = {
11922 *insn,
11923 BPF_ST_NOSPEC(),
11924 };
11925
11926 cnt = ARRAY_SIZE(patch);
11927 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
11928 if (!new_prog)
11929 return -ENOMEM;
11930
11931 delta += cnt - 1;
11932 env->prog = new_prog;
11933 insn = new_prog->insnsi + i + delta;
11934 continue;
11935 }
11936
11937 if (!ctx_access)
11938 continue;
11939
11940 switch (env->insn_aux_data[i + delta].ptr_type) {
11941 case PTR_TO_CTX:
11942 if (!ops->convert_ctx_access)
11943 continue;
11944 convert_ctx_access = ops->convert_ctx_access;
11945 break;
11946 case PTR_TO_SOCKET:
11947 case PTR_TO_SOCK_COMMON:
11948 convert_ctx_access = bpf_sock_convert_ctx_access;
11949 break;
11950 case PTR_TO_TCP_SOCK:
11951 convert_ctx_access = bpf_tcp_sock_convert_ctx_access;
11952 break;
11953 case PTR_TO_XDP_SOCK:
11954 convert_ctx_access = bpf_xdp_sock_convert_ctx_access;
11955 break;
11956 case PTR_TO_BTF_ID:
11957 if (type == BPF_READ) {
11958 insn->code = BPF_LDX | BPF_PROBE_MEM |
11959 BPF_SIZE((insn)->code);
11960 env->prog->aux->num_exentries++;
11961 } else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) {
11962 verbose(env, "Writes through BTF pointers are not allowed\n");
11963 return -EINVAL;
11964 }
11965 continue;
11966 default:
11967 continue;
11968 }
11969
11970 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
11971 size = BPF_LDST_BYTES(insn);
11972
11973
11974
11975
11976
11977
11978 is_narrower_load = size < ctx_field_size;
11979 size_default = bpf_ctx_off_adjust_machine(ctx_field_size);
11980 off = insn->off;
11981 if (is_narrower_load) {
11982 u8 size_code;
11983
11984 if (type == BPF_WRITE) {
11985 verbose(env, "bpf verifier narrow ctx access misconfigured\n");
11986 return -EINVAL;
11987 }
11988
11989 size_code = BPF_H;
11990 if (ctx_field_size == 4)
11991 size_code = BPF_W;
11992 else if (ctx_field_size == 8)
11993 size_code = BPF_DW;
11994
11995 insn->off = off & ~(size_default - 1);
11996 insn->code = BPF_LDX | BPF_MEM | size_code;
11997 }
11998
11999 target_size = 0;
12000 cnt = convert_ctx_access(type, insn, insn_buf, env->prog,
12001 &target_size);
12002 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) ||
12003 (ctx_field_size && !target_size)) {
12004 verbose(env, "bpf verifier is misconfigured\n");
12005 return -EINVAL;
12006 }
12007
12008 if (is_narrower_load && size < target_size) {
12009 u8 shift = bpf_ctx_narrow_access_offset(
12010 off, size, size_default) * 8;
12011 if (ctx_field_size <= 4) {
12012 if (shift)
12013 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH,
12014 insn->dst_reg,
12015 shift);
12016 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
12017 (1 << size * 8) - 1);
12018 } else {
12019 if (shift)
12020 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH,
12021 insn->dst_reg,
12022 shift);
12023 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg,
12024 (1ULL << size * 8) - 1);
12025 }
12026 }
12027
12028 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
12029 if (!new_prog)
12030 return -ENOMEM;
12031
12032 delta += cnt - 1;
12033
12034
12035 env->prog = new_prog;
12036 insn = new_prog->insnsi + i + delta;
12037 }
12038
12039 return 0;
12040}
12041
12042static int jit_subprogs(struct bpf_verifier_env *env)
12043{
12044 struct bpf_prog *prog = env->prog, **func, *tmp;
12045 int i, j, subprog_start, subprog_end = 0, len, subprog;
12046 struct bpf_map *map_ptr;
12047 struct bpf_insn *insn;
12048 void *old_bpf_func;
12049 int err, num_exentries;
12050
12051 if (env->subprog_cnt <= 1)
12052 return 0;
12053
12054 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
12055 if (bpf_pseudo_func(insn)) {
12056 env->insn_aux_data[i].call_imm = insn->imm;
12057
12058 continue;
12059 }
12060
12061 if (!bpf_pseudo_call(insn))
12062 continue;
12063
12064
12065
12066
12067 subprog = find_subprog(env, i + insn->imm + 1);
12068 if (subprog < 0) {
12069 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
12070 i + insn->imm + 1);
12071 return -EFAULT;
12072 }
12073
12074
12075
12076 insn->off = subprog;
12077
12078
12079
12080 env->insn_aux_data[i].call_imm = insn->imm;
12081
12082 insn->imm = 1;
12083 }
12084
12085 err = bpf_prog_alloc_jited_linfo(prog);
12086 if (err)
12087 goto out_undo_insn;
12088
12089 err = -ENOMEM;
12090 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
12091 if (!func)
12092 goto out_undo_insn;
12093
12094 for (i = 0; i < env->subprog_cnt; i++) {
12095 subprog_start = subprog_end;
12096 subprog_end = env->subprog_info[i + 1].start;
12097
12098 len = subprog_end - subprog_start;
12099
12100
12101
12102
12103
12104 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
12105 if (!func[i])
12106 goto out_free;
12107 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
12108 len * sizeof(struct bpf_insn));
12109 func[i]->type = prog->type;
12110 func[i]->len = len;
12111 if (bpf_prog_calc_tag(func[i]))
12112 goto out_free;
12113 func[i]->is_func = 1;
12114 func[i]->aux->func_idx = i;
12115
12116 func[i]->aux->btf = prog->aux->btf;
12117 func[i]->aux->func_info = prog->aux->func_info;
12118 func[i]->aux->poke_tab = prog->aux->poke_tab;
12119 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;
12120
12121 for (j = 0; j < prog->aux->size_poke_tab; j++) {
12122 struct bpf_jit_poke_descriptor *poke;
12123
12124 poke = &prog->aux->poke_tab[j];
12125 if (poke->insn_idx < subprog_end &&
12126 poke->insn_idx >= subprog_start)
12127 poke->aux = func[i]->aux;
12128 }
12129
12130
12131
12132
12133 func[i]->aux->name[0] = 'F';
12134 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
12135 func[i]->jit_requested = 1;
12136 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
12137 func[i]->aux->linfo = prog->aux->linfo;
12138 func[i]->aux->nr_linfo = prog->aux->nr_linfo;
12139 func[i]->aux->jited_linfo = prog->aux->jited_linfo;
12140 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
12141 num_exentries = 0;
12142 insn = func[i]->insnsi;
12143 for (j = 0; j < func[i]->len; j++, insn++) {
12144 if (BPF_CLASS(insn->code) == BPF_LDX &&
12145 BPF_MODE(insn->code) == BPF_PROBE_MEM)
12146 num_exentries++;
12147 }
12148 func[i]->aux->num_exentries = num_exentries;
12149 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
12150 func[i] = bpf_int_jit_compile(func[i]);
12151 if (!func[i]->jited) {
12152 err = -ENOTSUPP;
12153 goto out_free;
12154 }
12155 cond_resched();
12156 }
12157
12158
12159
12160
12161
12162 for (i = 0; i < env->subprog_cnt; i++) {
12163 insn = func[i]->insnsi;
12164 for (j = 0; j < func[i]->len; j++, insn++) {
12165 if (bpf_pseudo_func(insn)) {
12166 subprog = insn[1].imm;
12167 insn[0].imm = (u32)(long)func[subprog]->bpf_func;
12168 insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32;
12169 continue;
12170 }
12171 if (!bpf_pseudo_call(insn))
12172 continue;
12173 subprog = insn->off;
12174 insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) -
12175 __bpf_call_base;
12176 }
12177
12178
12179
12180
12181
12182
12183
12184
12185
12186
12187
12188
12189 func[i]->aux->func = func;
12190 func[i]->aux->func_cnt = env->subprog_cnt;
12191 }
12192 for (i = 0; i < env->subprog_cnt; i++) {
12193 old_bpf_func = func[i]->bpf_func;
12194 tmp = bpf_int_jit_compile(func[i]);
12195 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
12196 verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
12197 err = -ENOTSUPP;
12198 goto out_free;
12199 }
12200 cond_resched();
12201 }
12202
12203
12204
12205
12206 for (i = 0; i < env->subprog_cnt; i++) {
12207 bpf_prog_lock_ro(func[i]);
12208 bpf_prog_kallsyms_add(func[i]);
12209 }
12210
12211
12212
12213
12214
12215 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
12216 if (bpf_pseudo_func(insn)) {
12217 insn[0].imm = env->insn_aux_data[i].call_imm;
12218 insn[1].imm = find_subprog(env, i + insn[0].imm + 1);
12219 continue;
12220 }
12221 if (!bpf_pseudo_call(insn))
12222 continue;
12223 insn->off = env->insn_aux_data[i].call_imm;
12224 subprog = find_subprog(env, i + insn->off + 1);
12225 insn->imm = subprog;
12226 }
12227
12228 prog->jited = 1;
12229 prog->bpf_func = func[0]->bpf_func;
12230 prog->aux->func = func;
12231 prog->aux->func_cnt = env->subprog_cnt;
12232 bpf_prog_jit_attempt_done(prog);
12233 return 0;
12234out_free:
12235
12236
12237
12238
12239 for (i = 0; i < prog->aux->size_poke_tab; i++) {
12240 map_ptr = prog->aux->poke_tab[i].tail_call.map;
12241 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);
12242 }
12243
12244
12245
12246
12247 for (i = 0; i < env->subprog_cnt; i++) {
12248 if (!func[i])
12249 continue;
12250 func[i]->aux->poke_tab = NULL;
12251 bpf_jit_free(func[i]);
12252 }
12253 kfree(func);
12254out_undo_insn:
12255
12256 prog->jit_requested = 0;
12257 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
12258 if (!bpf_pseudo_call(insn))
12259 continue;
12260 insn->off = 0;
12261 insn->imm = env->insn_aux_data[i].call_imm;
12262 }
12263 bpf_prog_jit_attempt_done(prog);
12264 return err;
12265}
12266
12267static int fixup_call_args(struct bpf_verifier_env *env)
12268{
12269#ifndef CONFIG_BPF_JIT_ALWAYS_ON
12270 struct bpf_prog *prog = env->prog;
12271 struct bpf_insn *insn = prog->insnsi;
12272 bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
12273 int i, depth;
12274#endif
12275 int err = 0;
12276
12277 if (env->prog->jit_requested &&
12278 !bpf_prog_is_dev_bound(env->prog->aux)) {
12279 err = jit_subprogs(env);
12280 if (err == 0)
12281 return 0;
12282 if (err == -EFAULT)
12283 return err;
12284 }
12285#ifndef CONFIG_BPF_JIT_ALWAYS_ON
12286 if (has_kfunc_call) {
12287 verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
12288 return -EINVAL;
12289 }
12290 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
12291
12292
12293
12294 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
12295 return -EINVAL;
12296 }
12297 for (i = 0; i < prog->len; i++, insn++) {
12298 if (bpf_pseudo_func(insn)) {
12299
12300
12301
12302 verbose(env, "callbacks are not allowed in non-JITed programs\n");
12303 return -EINVAL;
12304 }
12305
12306 if (!bpf_pseudo_call(insn))
12307 continue;
12308 depth = get_callee_stack_depth(env, insn, i);
12309 if (depth < 0)
12310 return depth;
12311 bpf_patch_call_args(insn, depth);
12312 }
12313 err = 0;
12314#endif
12315 return err;
12316}
12317
12318static int fixup_kfunc_call(struct bpf_verifier_env *env,
12319 struct bpf_insn *insn)
12320{
12321 const struct bpf_kfunc_desc *desc;
12322
12323
12324
12325
12326 desc = find_kfunc_desc(env->prog, insn->imm);
12327 if (!desc) {
12328 verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
12329 insn->imm);
12330 return -EFAULT;
12331 }
12332
12333 insn->imm = desc->imm;
12334
12335 return 0;
12336}
12337
12338
12339
12340
12341static int do_misc_fixups(struct bpf_verifier_env *env)
12342{
12343 struct bpf_prog *prog = env->prog;
12344 bool expect_blinding = bpf_jit_blinding_enabled(prog);
12345 struct bpf_insn *insn = prog->insnsi;
12346 const struct bpf_func_proto *fn;
12347 const int insn_cnt = prog->len;
12348 const struct bpf_map_ops *ops;
12349 struct bpf_insn_aux_data *aux;
12350 struct bpf_insn insn_buf[16];
12351 struct bpf_prog *new_prog;
12352 struct bpf_map *map_ptr;
12353 int i, ret, cnt, delta = 0;
12354
12355 for (i = 0; i < insn_cnt; i++, insn++) {
12356
12357 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
12358 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
12359 insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
12360 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
12361 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
12362 bool isdiv = BPF_OP(insn->code) == BPF_DIV;
12363 struct bpf_insn *patchlet;
12364 struct bpf_insn chk_and_div[] = {
12365
12366 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
12367 BPF_JNE | BPF_K, insn->src_reg,
12368 0, 2, 0),
12369 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),
12370 BPF_JMP_IMM(BPF_JA, 0, 0, 1),
12371 *insn,
12372 };
12373 struct bpf_insn chk_and_mod[] = {
12374
12375 BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
12376 BPF_JEQ | BPF_K, insn->src_reg,
12377 0, 1 + (is64 ? 0 : 1), 0),
12378 *insn,
12379 BPF_JMP_IMM(BPF_JA, 0, 0, 1),
12380 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg),
12381 };
12382
12383 patchlet = isdiv ? chk_and_div : chk_and_mod;
12384 cnt = isdiv ? ARRAY_SIZE(chk_and_div) :
12385 ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0);
12386
12387 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
12388 if (!new_prog)
12389 return -ENOMEM;
12390
12391 delta += cnt - 1;
12392 env->prog = prog = new_prog;
12393 insn = new_prog->insnsi + i + delta;
12394 continue;
12395 }
12396
12397
12398 if (BPF_CLASS(insn->code) == BPF_LD &&
12399 (BPF_MODE(insn->code) == BPF_ABS ||
12400 BPF_MODE(insn->code) == BPF_IND)) {
12401 cnt = env->ops->gen_ld_abs(insn, insn_buf);
12402 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
12403 verbose(env, "bpf verifier is misconfigured\n");
12404 return -EINVAL;
12405 }
12406
12407 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
12408 if (!new_prog)
12409 return -ENOMEM;
12410
12411 delta += cnt - 1;
12412 env->prog = prog = new_prog;
12413 insn = new_prog->insnsi + i + delta;
12414 continue;
12415 }
12416
12417
12418 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
12419 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
12420 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
12421 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
12422 struct bpf_insn *patch = &insn_buf[0];
12423 bool issrc, isneg, isimm;
12424 u32 off_reg;
12425
12426 aux = &env->insn_aux_data[i + delta];
12427 if (!aux->alu_state ||
12428 aux->alu_state == BPF_ALU_NON_POINTER)
12429 continue;
12430
12431 isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
12432 issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
12433 BPF_ALU_SANITIZE_SRC;
12434 isimm = aux->alu_state & BPF_ALU_IMMEDIATE;
12435
12436 off_reg = issrc ? insn->src_reg : insn->dst_reg;
12437 if (isimm) {
12438 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
12439 } else {
12440 if (isneg)
12441 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
12442 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
12443 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
12444 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
12445 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
12446 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
12447 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg);
12448 }
12449 if (!issrc)
12450 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg);
12451 insn->src_reg = BPF_REG_AX;
12452 if (isneg)
12453 insn->code = insn->code == code_add ?
12454 code_sub : code_add;
12455 *patch++ = *insn;
12456 if (issrc && isneg && !isimm)
12457 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
12458 cnt = patch - insn_buf;
12459
12460 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
12461 if (!new_prog)
12462 return -ENOMEM;
12463
12464 delta += cnt - 1;
12465 env->prog = prog = new_prog;
12466 insn = new_prog->insnsi + i + delta;
12467 continue;
12468 }
12469
12470 if (insn->code != (BPF_JMP | BPF_CALL))
12471 continue;
12472 if (insn->src_reg == BPF_PSEUDO_CALL)
12473 continue;
12474 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
12475 ret = fixup_kfunc_call(env, insn);
12476 if (ret)
12477 return ret;
12478 continue;
12479 }
12480
12481 if (insn->imm == BPF_FUNC_get_route_realm)
12482 prog->dst_needed = 1;
12483 if (insn->imm == BPF_FUNC_get_prandom_u32)
12484 bpf_user_rnd_init_once();
12485 if (insn->imm == BPF_FUNC_override_return)
12486 prog->kprobe_override = 1;
12487 if (insn->imm == BPF_FUNC_tail_call) {
12488
12489
12490
12491
12492
12493 prog->cb_access = 1;
12494 if (!allow_tail_call_in_subprogs(env))
12495 prog->aux->stack_depth = MAX_BPF_STACK;
12496 prog->aux->max_pkt_offset = MAX_PACKET_OFF;
12497
12498
12499
12500
12501
12502
12503 insn->imm = 0;
12504 insn->code = BPF_JMP | BPF_TAIL_CALL;
12505
12506 aux = &env->insn_aux_data[i + delta];
12507 if (env->bpf_capable && !expect_blinding &&
12508 prog->jit_requested &&
12509 !bpf_map_key_poisoned(aux) &&
12510 !bpf_map_ptr_poisoned(aux) &&
12511 !bpf_map_ptr_unpriv(aux)) {
12512 struct bpf_jit_poke_descriptor desc = {
12513 .reason = BPF_POKE_REASON_TAIL_CALL,
12514 .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state),
12515 .tail_call.key = bpf_map_key_immediate(aux),
12516 .insn_idx = i + delta,
12517 };
12518
12519 ret = bpf_jit_add_poke_descriptor(prog, &desc);
12520 if (ret < 0) {
12521 verbose(env, "adding tail call poke descriptor failed\n");
12522 return ret;
12523 }
12524
12525 insn->imm = ret + 1;
12526 continue;
12527 }
12528
12529 if (!bpf_map_ptr_unpriv(aux))
12530 continue;
12531
12532
12533
12534
12535
12536
12537
12538 if (bpf_map_ptr_poisoned(aux)) {
12539 verbose(env, "tail_call abusing map_ptr\n");
12540 return -EINVAL;
12541 }
12542
12543 map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
12544 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
12545 map_ptr->max_entries, 2);
12546 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
12547 container_of(map_ptr,
12548 struct bpf_array,
12549 map)->index_mask);
12550 insn_buf[2] = *insn;
12551 cnt = 3;
12552 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
12553 if (!new_prog)
12554 return -ENOMEM;
12555
12556 delta += cnt - 1;
12557 env->prog = prog = new_prog;
12558 insn = new_prog->insnsi + i + delta;
12559 continue;
12560 }
12561
12562
12563
12564
12565
12566 if (prog->jit_requested && BITS_PER_LONG == 64 &&
12567 (insn->imm == BPF_FUNC_map_lookup_elem ||
12568 insn->imm == BPF_FUNC_map_update_elem ||
12569 insn->imm == BPF_FUNC_map_delete_elem ||
12570 insn->imm == BPF_FUNC_map_push_elem ||
12571 insn->imm == BPF_FUNC_map_pop_elem ||
12572 insn->imm == BPF_FUNC_map_peek_elem ||
12573 insn->imm == BPF_FUNC_redirect_map)) {
12574 aux = &env->insn_aux_data[i + delta];
12575 if (bpf_map_ptr_poisoned(aux))
12576 goto patch_call_imm;
12577
12578 map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
12579 ops = map_ptr->ops;
12580 if (insn->imm == BPF_FUNC_map_lookup_elem &&
12581 ops->map_gen_lookup) {
12582 cnt = ops->map_gen_lookup(map_ptr, insn_buf);
12583 if (cnt == -EOPNOTSUPP)
12584 goto patch_map_ops_generic;
12585 if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) {
12586 verbose(env, "bpf verifier is misconfigured\n");
12587 return -EINVAL;
12588 }
12589
12590 new_prog = bpf_patch_insn_data(env, i + delta,
12591 insn_buf, cnt);
12592 if (!new_prog)
12593 return -ENOMEM;
12594
12595 delta += cnt - 1;
12596 env->prog = prog = new_prog;
12597 insn = new_prog->insnsi + i + delta;
12598 continue;
12599 }
12600
12601 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
12602 (void *(*)(struct bpf_map *map, void *key))NULL));
12603 BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
12604 (int (*)(struct bpf_map *map, void *key))NULL));
12605 BUILD_BUG_ON(!__same_type(ops->map_update_elem,
12606 (int (*)(struct bpf_map *map, void *key, void *value,
12607 u64 flags))NULL));
12608 BUILD_BUG_ON(!__same_type(ops->map_push_elem,
12609 (int (*)(struct bpf_map *map, void *value,
12610 u64 flags))NULL));
12611 BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
12612 (int (*)(struct bpf_map *map, void *value))NULL));
12613 BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
12614 (int (*)(struct bpf_map *map, void *value))NULL));
12615 BUILD_BUG_ON(!__same_type(ops->map_redirect,
12616 (int (*)(struct bpf_map *map, u32 ifindex, u64 flags))NULL));
12617
12618patch_map_ops_generic:
12619 switch (insn->imm) {
12620 case BPF_FUNC_map_lookup_elem:
12621 insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -
12622 __bpf_call_base;
12623 continue;
12624 case BPF_FUNC_map_update_elem:
12625 insn->imm = BPF_CAST_CALL(ops->map_update_elem) -
12626 __bpf_call_base;
12627 continue;
12628 case BPF_FUNC_map_delete_elem:
12629 insn->imm = BPF_CAST_CALL(ops->map_delete_elem) -
12630 __bpf_call_base;
12631 continue;
12632 case BPF_FUNC_map_push_elem:
12633 insn->imm = BPF_CAST_CALL(ops->map_push_elem) -
12634 __bpf_call_base;
12635 continue;
12636 case BPF_FUNC_map_pop_elem:
12637 insn->imm = BPF_CAST_CALL(ops->map_pop_elem) -
12638 __bpf_call_base;
12639 continue;
12640 case BPF_FUNC_map_peek_elem:
12641 insn->imm = BPF_CAST_CALL(ops->map_peek_elem) -
12642 __bpf_call_base;
12643 continue;
12644 case BPF_FUNC_redirect_map:
12645 insn->imm = BPF_CAST_CALL(ops->map_redirect) -
12646 __bpf_call_base;
12647 continue;
12648 }
12649
12650 goto patch_call_imm;
12651 }
12652
12653
12654 if (prog->jit_requested && BITS_PER_LONG == 64 &&
12655 insn->imm == BPF_FUNC_jiffies64) {
12656 struct bpf_insn ld_jiffies_addr[2] = {
12657 BPF_LD_IMM64(BPF_REG_0,
12658 (unsigned long)&jiffies),
12659 };
12660
12661 insn_buf[0] = ld_jiffies_addr[0];
12662 insn_buf[1] = ld_jiffies_addr[1];
12663 insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0,
12664 BPF_REG_0, 0);
12665 cnt = 3;
12666
12667 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,
12668 cnt);
12669 if (!new_prog)
12670 return -ENOMEM;
12671
12672 delta += cnt - 1;
12673 env->prog = prog = new_prog;
12674 insn = new_prog->insnsi + i + delta;
12675 continue;
12676 }
12677
12678patch_call_imm:
12679 fn = env->ops->get_func_proto(insn->imm, env->prog);
12680
12681
12682
12683 if (!fn->func) {
12684 verbose(env,
12685 "kernel subsystem misconfigured func %s#%d\n",
12686 func_id_name(insn->imm), insn->imm);
12687 return -EFAULT;
12688 }
12689 insn->imm = fn->func - __bpf_call_base;
12690 }
12691
12692
12693 for (i = 0; i < prog->aux->size_poke_tab; i++) {
12694 map_ptr = prog->aux->poke_tab[i].tail_call.map;
12695 if (!map_ptr->ops->map_poke_track ||
12696 !map_ptr->ops->map_poke_untrack ||
12697 !map_ptr->ops->map_poke_run) {
12698 verbose(env, "bpf verifier is misconfigured\n");
12699 return -EINVAL;
12700 }
12701
12702 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);
12703 if (ret < 0) {
12704 verbose(env, "tracking tail call prog failed\n");
12705 return ret;
12706 }
12707 }
12708
12709 sort_kfunc_descs_by_imm(env->prog);
12710
12711 return 0;
12712}
12713
12714static void free_states(struct bpf_verifier_env *env)
12715{
12716 struct bpf_verifier_state_list *sl, *sln;
12717 int i;
12718
12719 sl = env->free_list;
12720 while (sl) {
12721 sln = sl->next;
12722 free_verifier_state(&sl->state, false);
12723 kfree(sl);
12724 sl = sln;
12725 }
12726 env->free_list = NULL;
12727
12728 if (!env->explored_states)
12729 return;
12730
12731 for (i = 0; i < state_htab_size(env); i++) {
12732 sl = env->explored_states[i];
12733
12734 while (sl) {
12735 sln = sl->next;
12736 free_verifier_state(&sl->state, false);
12737 kfree(sl);
12738 sl = sln;
12739 }
12740 env->explored_states[i] = NULL;
12741 }
12742}
12743
12744static int do_check_common(struct bpf_verifier_env *env, int subprog)
12745{
12746 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
12747 struct bpf_verifier_state *state;
12748 struct bpf_reg_state *regs;
12749 int ret, i;
12750
12751 env->prev_linfo = NULL;
12752 env->pass_cnt++;
12753
12754 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL);
12755 if (!state)
12756 return -ENOMEM;
12757 state->curframe = 0;
12758 state->speculative = false;
12759 state->branches = 1;
12760 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL);
12761 if (!state->frame[0]) {
12762 kfree(state);
12763 return -ENOMEM;
12764 }
12765 env->cur_state = state;
12766 init_func_state(env, state->frame[0],
12767 BPF_MAIN_FUNC ,
12768 0 ,
12769 subprog);
12770
12771 regs = state->frame[state->curframe]->regs;
12772 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {
12773 ret = btf_prepare_func_args(env, subprog, regs);
12774 if (ret)
12775 goto out;
12776 for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
12777 if (regs[i].type == PTR_TO_CTX)
12778 mark_reg_known_zero(env, regs, i);
12779 else if (regs[i].type == SCALAR_VALUE)
12780 mark_reg_unknown(env, regs, i);
12781 else if (regs[i].type == PTR_TO_MEM_OR_NULL) {
12782 const u32 mem_size = regs[i].mem_size;
12783
12784 mark_reg_known_zero(env, regs, i);
12785 regs[i].mem_size = mem_size;
12786 regs[i].id = ++env->id_gen;
12787 }
12788 }
12789 } else {
12790
12791 regs[BPF_REG_1].type = PTR_TO_CTX;
12792 mark_reg_known_zero(env, regs, BPF_REG_1);
12793 ret = btf_check_subprog_arg_match(env, subprog, regs);
12794 if (ret == -EFAULT)
12795
12796
12797
12798
12799
12800
12801
12802
12803
12804 goto out;
12805 }
12806
12807 ret = do_check(env);
12808out:
12809
12810
12811
12812 if (env->cur_state) {
12813 free_verifier_state(env->cur_state, true);
12814 env->cur_state = NULL;
12815 }
12816 while (!pop_stack(env, NULL, NULL, false));
12817 if (!ret && pop_log)
12818 bpf_vlog_reset(&env->log, 0);
12819 free_states(env);
12820 return ret;
12821}
12822
12823
12824
12825
12826
12827
12828
12829
12830
12831
12832
12833
12834
12835
12836
12837
12838
12839
12840static int do_check_subprogs(struct bpf_verifier_env *env)
12841{
12842 struct bpf_prog_aux *aux = env->prog->aux;
12843 int i, ret;
12844
12845 if (!aux->func_info)
12846 return 0;
12847
12848 for (i = 1; i < env->subprog_cnt; i++) {
12849 if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL)
12850 continue;
12851 env->insn_idx = env->subprog_info[i].start;
12852 WARN_ON_ONCE(env->insn_idx == 0);
12853 ret = do_check_common(env, i);
12854 if (ret) {
12855 return ret;
12856 } else if (env->log.level & BPF_LOG_LEVEL) {
12857 verbose(env,
12858 "Func#%d is safe for any args that match its prototype\n",
12859 i);
12860 }
12861 }
12862 return 0;
12863}
12864
12865static int do_check_main(struct bpf_verifier_env *env)
12866{
12867 int ret;
12868
12869 env->insn_idx = 0;
12870 ret = do_check_common(env, 0);
12871 if (!ret)
12872 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
12873 return ret;
12874}
12875
12876
12877static void print_verification_stats(struct bpf_verifier_env *env)
12878{
12879 int i;
12880
12881 if (env->log.level & BPF_LOG_STATS) {
12882 verbose(env, "verification time %lld usec\n",
12883 div_u64(env->verification_time, 1000));
12884 verbose(env, "stack depth ");
12885 for (i = 0; i < env->subprog_cnt; i++) {
12886 u32 depth = env->subprog_info[i].stack_depth;
12887
12888 verbose(env, "%d", depth);
12889 if (i + 1 < env->subprog_cnt)
12890 verbose(env, "+");
12891 }
12892 verbose(env, "\n");
12893 }
12894 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "
12895 "total_states %d peak_states %d mark_read %d\n",
12896 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,
12897 env->max_states_per_insn, env->total_states,
12898 env->peak_states, env->longest_mark_read_walk);
12899}
12900
12901static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
12902{
12903 const struct btf_type *t, *func_proto;
12904 const struct bpf_struct_ops *st_ops;
12905 const struct btf_member *member;
12906 struct bpf_prog *prog = env->prog;
12907 u32 btf_id, member_idx;
12908 const char *mname;
12909
12910 if (!prog->gpl_compatible) {
12911 verbose(env, "struct ops programs must have a GPL compatible license\n");
12912 return -EINVAL;
12913 }
12914
12915 btf_id = prog->aux->attach_btf_id;
12916 st_ops = bpf_struct_ops_find(btf_id);
12917 if (!st_ops) {
12918 verbose(env, "attach_btf_id %u is not a supported struct\n",
12919 btf_id);
12920 return -ENOTSUPP;
12921 }
12922
12923 t = st_ops->type;
12924 member_idx = prog->expected_attach_type;
12925 if (member_idx >= btf_type_vlen(t)) {
12926 verbose(env, "attach to invalid member idx %u of struct %s\n",
12927 member_idx, st_ops->name);
12928 return -EINVAL;
12929 }
12930
12931 member = &btf_type_member(t)[member_idx];
12932 mname = btf_name_by_offset(btf_vmlinux, member->name_off);
12933 func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type,
12934 NULL);
12935 if (!func_proto) {
12936 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",
12937 mname, member_idx, st_ops->name);
12938 return -EINVAL;
12939 }
12940
12941 if (st_ops->check_member) {
12942 int err = st_ops->check_member(t, member);
12943
12944 if (err) {
12945 verbose(env, "attach to unsupported member %s of struct %s\n",
12946 mname, st_ops->name);
12947 return err;
12948 }
12949 }
12950
12951 prog->aux->attach_func_proto = func_proto;
12952 prog->aux->attach_func_name = mname;
12953 env->ops = st_ops->verifier_ops;
12954
12955 return 0;
12956}
12957#define SECURITY_PREFIX "security_"
12958
12959static int check_attach_modify_return(unsigned long addr, const char *func_name)
12960{
12961 if (within_error_injection_list(addr) ||
12962 !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
12963 return 0;
12964
12965 return -EINVAL;
12966}
12967
12968
12969
12970
12971BTF_SET_START(btf_non_sleepable_error_inject)
12972
12973
12974
12975BTF_ID(func, __add_to_page_cache_locked)
12976BTF_ID(func, should_fail_alloc_page)
12977BTF_ID(func, should_failslab)
12978BTF_SET_END(btf_non_sleepable_error_inject)
12979
12980static int check_non_sleepable_error_inject(u32 btf_id)
12981{
12982 return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
12983}
12984
12985int bpf_check_attach_target(struct bpf_verifier_log *log,
12986 const struct bpf_prog *prog,
12987 const struct bpf_prog *tgt_prog,
12988 u32 btf_id,
12989 struct bpf_attach_target_info *tgt_info)
12990{
12991 bool prog_extension = prog->type == BPF_PROG_TYPE_EXT;
12992 const char prefix[] = "btf_trace_";
12993 int ret = 0, subprog = -1, i;
12994 const struct btf_type *t;
12995 bool conservative = true;
12996 const char *tname;
12997 struct btf *btf;
12998 long addr = 0;
12999
13000 if (!btf_id) {
13001 bpf_log(log, "Tracing programs must provide btf_id\n");
13002 return -EINVAL;
13003 }
13004 btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf;
13005 if (!btf) {
13006 bpf_log(log,
13007 "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n");
13008 return -EINVAL;
13009 }
13010 t = btf_type_by_id(btf, btf_id);
13011 if (!t) {
13012 bpf_log(log, "attach_btf_id %u is invalid\n", btf_id);
13013 return -EINVAL;
13014 }
13015 tname = btf_name_by_offset(btf, t->name_off);
13016 if (!tname) {
13017 bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id);
13018 return -EINVAL;
13019 }
13020 if (tgt_prog) {
13021 struct bpf_prog_aux *aux = tgt_prog->aux;
13022
13023 for (i = 0; i < aux->func_info_cnt; i++)
13024 if (aux->func_info[i].type_id == btf_id) {
13025 subprog = i;
13026 break;
13027 }
13028 if (subprog == -1) {
13029 bpf_log(log, "Subprog %s doesn't exist\n", tname);
13030 return -EINVAL;
13031 }
13032 conservative = aux->func_info_aux[subprog].unreliable;
13033 if (prog_extension) {
13034 if (conservative) {
13035 bpf_log(log,
13036 "Cannot replace static functions\n");
13037 return -EINVAL;
13038 }
13039 if (!prog->jit_requested) {
13040 bpf_log(log,
13041 "Extension programs should be JITed\n");
13042 return -EINVAL;
13043 }
13044 }
13045 if (!tgt_prog->jited) {
13046 bpf_log(log, "Can attach to only JITed progs\n");
13047 return -EINVAL;
13048 }
13049 if (tgt_prog->type == prog->type) {
13050
13051
13052
13053
13054 bpf_log(log, "Cannot recursively attach\n");
13055 return -EINVAL;
13056 }
13057 if (tgt_prog->type == BPF_PROG_TYPE_TRACING &&
13058 prog_extension &&
13059 (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY ||
13060 tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) {
13061
13062
13063
13064
13065
13066
13067
13068
13069
13070
13071
13072
13073
13074
13075
13076 bpf_log(log, "Cannot extend fentry/fexit\n");
13077 return -EINVAL;
13078 }
13079 } else {
13080 if (prog_extension) {
13081 bpf_log(log, "Cannot replace kernel functions\n");
13082 return -EINVAL;
13083 }
13084 }
13085
13086 switch (prog->expected_attach_type) {
13087 case BPF_TRACE_RAW_TP:
13088 if (tgt_prog) {
13089 bpf_log(log,
13090 "Only FENTRY/FEXIT progs are attachable to another BPF prog\n");
13091 return -EINVAL;
13092 }
13093 if (!btf_type_is_typedef(t)) {
13094 bpf_log(log, "attach_btf_id %u is not a typedef\n",
13095 btf_id);
13096 return -EINVAL;
13097 }
13098 if (strncmp(prefix, tname, sizeof(prefix) - 1)) {
13099 bpf_log(log, "attach_btf_id %u points to wrong type name %s\n",
13100 btf_id, tname);
13101 return -EINVAL;
13102 }
13103 tname += sizeof(prefix) - 1;
13104 t = btf_type_by_id(btf, t->type);
13105 if (!btf_type_is_ptr(t))
13106
13107 return -EINVAL;
13108 t = btf_type_by_id(btf, t->type);
13109 if (!btf_type_is_func_proto(t))
13110
13111 return -EINVAL;
13112
13113 break;
13114 case BPF_TRACE_ITER:
13115 if (!btf_type_is_func(t)) {
13116 bpf_log(log, "attach_btf_id %u is not a function\n",
13117 btf_id);
13118 return -EINVAL;
13119 }
13120 t = btf_type_by_id(btf, t->type);
13121 if (!btf_type_is_func_proto(t))
13122 return -EINVAL;
13123 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
13124 if (ret)
13125 return ret;
13126 break;
13127 default:
13128 if (!prog_extension)
13129 return -EINVAL;
13130 fallthrough;
13131 case BPF_MODIFY_RETURN:
13132 case BPF_LSM_MAC:
13133 case BPF_TRACE_FENTRY:
13134 case BPF_TRACE_FEXIT:
13135 if (!btf_type_is_func(t)) {
13136 bpf_log(log, "attach_btf_id %u is not a function\n",
13137 btf_id);
13138 return -EINVAL;
13139 }
13140 if (prog_extension &&
13141 btf_check_type_match(log, prog, btf, t))
13142 return -EINVAL;
13143 t = btf_type_by_id(btf, t->type);
13144 if (!btf_type_is_func_proto(t))
13145 return -EINVAL;
13146
13147 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) &&
13148 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type ||
13149 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type))
13150 return -EINVAL;
13151
13152 if (tgt_prog && conservative)
13153 t = NULL;
13154
13155 ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
13156 if (ret < 0)
13157 return ret;
13158
13159 if (tgt_prog) {
13160 if (subprog == 0)
13161 addr = (long) tgt_prog->bpf_func;
13162 else
13163 addr = (long) tgt_prog->aux->func[subprog]->bpf_func;
13164 } else {
13165 addr = kallsyms_lookup_name(tname);
13166 if (!addr) {
13167 bpf_log(log,
13168 "The address of function %s cannot be found\n",
13169 tname);
13170 return -ENOENT;
13171 }
13172 }
13173
13174 if (prog->aux->sleepable) {
13175 ret = -EINVAL;
13176 switch (prog->type) {
13177 case BPF_PROG_TYPE_TRACING:
13178
13179
13180
13181 if (!check_non_sleepable_error_inject(btf_id) &&
13182 within_error_injection_list(addr))
13183 ret = 0;
13184 break;
13185 case BPF_PROG_TYPE_LSM:
13186
13187
13188
13189 if (bpf_lsm_is_sleepable_hook(btf_id))
13190 ret = 0;
13191 break;
13192 default:
13193 break;
13194 }
13195 if (ret) {
13196 bpf_log(log, "%s is not sleepable\n", tname);
13197 return ret;
13198 }
13199 } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
13200 if (tgt_prog) {
13201 bpf_log(log, "can't modify return codes of BPF programs\n");
13202 return -EINVAL;
13203 }
13204 ret = check_attach_modify_return(addr, tname);
13205 if (ret) {
13206 bpf_log(log, "%s() is not modifiable\n", tname);
13207 return ret;
13208 }
13209 }
13210
13211 break;
13212 }
13213 tgt_info->tgt_addr = addr;
13214 tgt_info->tgt_name = tname;
13215 tgt_info->tgt_type = t;
13216 return 0;
13217}
13218
13219BTF_SET_START(btf_id_deny)
13220BTF_ID_UNUSED
13221#ifdef CONFIG_SMP
13222BTF_ID(func, migrate_disable)
13223BTF_ID(func, migrate_enable)
13224#endif
13225#if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU
13226BTF_ID(func, rcu_read_unlock_strict)
13227#endif
13228BTF_SET_END(btf_id_deny)
13229
13230static int check_attach_btf_id(struct bpf_verifier_env *env)
13231{
13232 struct bpf_prog *prog = env->prog;
13233 struct bpf_prog *tgt_prog = prog->aux->dst_prog;
13234 struct bpf_attach_target_info tgt_info = {};
13235 u32 btf_id = prog->aux->attach_btf_id;
13236 struct bpf_trampoline *tr;
13237 int ret;
13238 u64 key;
13239
13240 if (prog->type == BPF_PROG_TYPE_SYSCALL) {
13241 if (prog->aux->sleepable)
13242
13243 return 0;
13244 verbose(env, "Syscall programs can only be sleepable\n");
13245 return -EINVAL;
13246 }
13247
13248 if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING &&
13249 prog->type != BPF_PROG_TYPE_LSM) {
13250 verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n");
13251 return -EINVAL;
13252 }
13253
13254 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
13255 return check_struct_ops_btf_id(env);
13256
13257 if (prog->type != BPF_PROG_TYPE_TRACING &&
13258 prog->type != BPF_PROG_TYPE_LSM &&
13259 prog->type != BPF_PROG_TYPE_EXT)
13260 return 0;
13261
13262 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);
13263 if (ret)
13264 return ret;
13265
13266 if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) {
13267
13268
13269
13270
13271 env->ops = bpf_verifier_ops[tgt_prog->type];
13272 prog->expected_attach_type = tgt_prog->expected_attach_type;
13273 }
13274
13275
13276 prog->aux->attach_func_proto = tgt_info.tgt_type;
13277 prog->aux->attach_func_name = tgt_info.tgt_name;
13278
13279 if (tgt_prog) {
13280 prog->aux->saved_dst_prog_type = tgt_prog->type;
13281 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type;
13282 }
13283
13284 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {
13285 prog->aux->attach_btf_trace = true;
13286 return 0;
13287 } else if (prog->expected_attach_type == BPF_TRACE_ITER) {
13288 if (!bpf_iter_prog_supported(prog))
13289 return -EINVAL;
13290 return 0;
13291 }
13292
13293 if (prog->type == BPF_PROG_TYPE_LSM) {
13294 ret = bpf_lsm_verify_prog(&env->log, prog);
13295 if (ret < 0)
13296 return ret;
13297 } else if (prog->type == BPF_PROG_TYPE_TRACING &&
13298 btf_id_set_contains(&btf_id_deny, btf_id)) {
13299 return -EINVAL;
13300 }
13301
13302 key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
13303 tr = bpf_trampoline_get(key, &tgt_info);
13304 if (!tr)
13305 return -ENOMEM;
13306
13307 prog->aux->dst_trampoline = tr;
13308 return 0;
13309}
13310
13311struct btf *bpf_get_btf_vmlinux(void)
13312{
13313 if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
13314 mutex_lock(&bpf_verifier_lock);
13315 if (!btf_vmlinux)
13316 btf_vmlinux = btf_parse_vmlinux();
13317 mutex_unlock(&bpf_verifier_lock);
13318 }
13319 return btf_vmlinux;
13320}
13321
13322int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr)
13323{
13324 u64 start_time = ktime_get_ns();
13325 struct bpf_verifier_env *env;
13326 struct bpf_verifier_log *log;
13327 int i, len, ret = -EINVAL;
13328 bool is_priv;
13329
13330
13331 if (ARRAY_SIZE(bpf_verifier_ops) == 0)
13332 return -EINVAL;
13333
13334
13335
13336
13337 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);
13338 if (!env)
13339 return -ENOMEM;
13340 log = &env->log;
13341
13342 len = (*prog)->len;
13343 env->insn_aux_data =
13344 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
13345 ret = -ENOMEM;
13346 if (!env->insn_aux_data)
13347 goto err_free_env;
13348 for (i = 0; i < len; i++)
13349 env->insn_aux_data[i].orig_idx = i;
13350 env->prog = *prog;
13351 env->ops = bpf_verifier_ops[env->prog->type];
13352 env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);
13353 is_priv = bpf_capable();
13354
13355 bpf_get_btf_vmlinux();
13356
13357
13358 if (!is_priv)
13359 mutex_lock(&bpf_verifier_lock);
13360
13361 if (attr->log_level || attr->log_buf || attr->log_size) {
13362
13363
13364
13365 log->level = attr->log_level;
13366 log->ubuf = (char __user *) (unsigned long) attr->log_buf;
13367 log->len_total = attr->log_size;
13368
13369 ret = -EINVAL;
13370
13371 if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 ||
13372 !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK)
13373 goto err_unlock;
13374 }
13375
13376 if (IS_ERR(btf_vmlinux)) {
13377
13378 verbose(env, "in-kernel BTF is malformed\n");
13379 ret = PTR_ERR(btf_vmlinux);
13380 goto skip_full_check;
13381 }
13382
13383 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
13384 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
13385 env->strict_alignment = true;
13386 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
13387 env->strict_alignment = false;
13388
13389 env->allow_ptr_leaks = bpf_allow_ptr_leaks();
13390 env->allow_uninit_stack = bpf_allow_uninit_stack();
13391 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access();
13392 env->bypass_spec_v1 = bpf_bypass_spec_v1();
13393 env->bypass_spec_v4 = bpf_bypass_spec_v4();
13394 env->bpf_capable = bpf_capable();
13395
13396 if (is_priv)
13397 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
13398
13399 env->explored_states = kvcalloc(state_htab_size(env),
13400 sizeof(struct bpf_verifier_state_list *),
13401 GFP_USER);
13402 ret = -ENOMEM;
13403 if (!env->explored_states)
13404 goto skip_full_check;
13405
13406 ret = add_subprog_and_kfunc(env);
13407 if (ret < 0)
13408 goto skip_full_check;
13409
13410 ret = check_subprogs(env);
13411 if (ret < 0)
13412 goto skip_full_check;
13413
13414 ret = check_btf_info(env, attr, uattr);
13415 if (ret < 0)
13416 goto skip_full_check;
13417
13418 ret = check_attach_btf_id(env);
13419 if (ret)
13420 goto skip_full_check;
13421
13422 ret = resolve_pseudo_ldimm64(env);
13423 if (ret < 0)
13424 goto skip_full_check;
13425
13426 if (bpf_prog_is_dev_bound(env->prog->aux)) {
13427 ret = bpf_prog_offload_verifier_prep(env->prog);
13428 if (ret)
13429 goto skip_full_check;
13430 }
13431
13432 ret = check_cfg(env);
13433 if (ret < 0)
13434 goto skip_full_check;
13435
13436 ret = do_check_subprogs(env);
13437 ret = ret ?: do_check_main(env);
13438
13439 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux))
13440 ret = bpf_prog_offload_finalize(env);
13441
13442skip_full_check:
13443 kvfree(env->explored_states);
13444
13445 if (ret == 0)
13446 ret = check_max_stack_depth(env);
13447
13448
13449 if (is_priv) {
13450 if (ret == 0)
13451 opt_hard_wire_dead_code_branches(env);
13452 if (ret == 0)
13453 ret = opt_remove_dead_code(env);
13454 if (ret == 0)
13455 ret = opt_remove_nops(env);
13456 } else {
13457 if (ret == 0)
13458 sanitize_dead_code(env);
13459 }
13460
13461 if (ret == 0)
13462
13463 ret = convert_ctx_accesses(env);
13464
13465 if (ret == 0)
13466 ret = do_misc_fixups(env);
13467
13468
13469
13470
13471 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) {
13472 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr);
13473 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret
13474 : false;
13475 }
13476
13477 if (ret == 0)
13478 ret = fixup_call_args(env);
13479
13480 env->verification_time = ktime_get_ns() - start_time;
13481 print_verification_stats(env);
13482
13483 if (log->level && bpf_verifier_log_full(log))
13484 ret = -ENOSPC;
13485 if (log->level && !log->ubuf) {
13486 ret = -EFAULT;
13487 goto err_release_maps;
13488 }
13489
13490 if (ret)
13491 goto err_release_maps;
13492
13493 if (env->used_map_cnt) {
13494
13495 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
13496 sizeof(env->used_maps[0]),
13497 GFP_KERNEL);
13498
13499 if (!env->prog->aux->used_maps) {
13500 ret = -ENOMEM;
13501 goto err_release_maps;
13502 }
13503
13504 memcpy(env->prog->aux->used_maps, env->used_maps,
13505 sizeof(env->used_maps[0]) * env->used_map_cnt);
13506 env->prog->aux->used_map_cnt = env->used_map_cnt;
13507 }
13508 if (env->used_btf_cnt) {
13509
13510 env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt,
13511 sizeof(env->used_btfs[0]),
13512 GFP_KERNEL);
13513 if (!env->prog->aux->used_btfs) {
13514 ret = -ENOMEM;
13515 goto err_release_maps;
13516 }
13517
13518 memcpy(env->prog->aux->used_btfs, env->used_btfs,
13519 sizeof(env->used_btfs[0]) * env->used_btf_cnt);
13520 env->prog->aux->used_btf_cnt = env->used_btf_cnt;
13521 }
13522 if (env->used_map_cnt || env->used_btf_cnt) {
13523
13524
13525
13526 convert_pseudo_ld_imm64(env);
13527 }
13528
13529 adjust_btf_func(env);
13530
13531err_release_maps:
13532 if (!env->prog->aux->used_maps)
13533
13534
13535
13536 release_maps(env);
13537 if (!env->prog->aux->used_btfs)
13538 release_btfs(env);
13539
13540
13541
13542
13543 if (env->prog->type == BPF_PROG_TYPE_EXT)
13544 env->prog->expected_attach_type = 0;
13545
13546 *prog = env->prog;
13547err_unlock:
13548 if (!is_priv)
13549 mutex_unlock(&bpf_verifier_lock);
13550 vfree(env->insn_aux_data);
13551err_free_env:
13552 kfree(env);
13553 return ret;
13554}
13555