1
2
3
4
5
6#include <uapi/linux/btf.h>
7#include <linux/kernel.h>
8#include <linux/types.h>
9#include <linux/slab.h>
10#include <linux/bpf.h>
11#include <linux/btf.h>
12#include <linux/bpf_verifier.h>
13#include <linux/filter.h>
14#include <net/netlink.h>
15#include <linux/file.h>
16#include <linux/vmalloc.h>
17#include <linux/stringify.h>
18#include <linux/bsearch.h>
19#include <linux/sort.h>
20#include <linux/perf_event.h>
21#include <linux/ctype.h>
22
23#include "disasm.h"
24
25static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
26#define BPF_PROG_TYPE(_id, _name) \
27 [_id] = & _name ## _verifier_ops,
28#define BPF_MAP_TYPE(_id, _ops)
29#include <linux/bpf_types.h>
30#undef BPF_PROG_TYPE
31#undef BPF_MAP_TYPE
32};
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160struct bpf_verifier_stack_elem {
161
162
163
164
165 struct bpf_verifier_state st;
166 int insn_idx;
167 int prev_insn_idx;
168 struct bpf_verifier_stack_elem *next;
169};
170
171#define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192
172#define BPF_COMPLEXITY_LIMIT_STATES 64
173
174#define BPF_MAP_PTR_UNPRIV 1UL
175#define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \
176 POISON_POINTER_DELTA))
177#define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV))
178
179static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
180{
181 return BPF_MAP_PTR(aux->map_state) == BPF_MAP_PTR_POISON;
182}
183
184static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
185{
186 return aux->map_state & BPF_MAP_PTR_UNPRIV;
187}
188
189static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
190 const struct bpf_map *map, bool unpriv)
191{
192 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV);
193 unpriv |= bpf_map_ptr_unpriv(aux);
194 aux->map_state = (unsigned long)map |
195 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL);
196}
197
198struct bpf_call_arg_meta {
199 struct bpf_map *map_ptr;
200 bool raw_mode;
201 bool pkt_access;
202 int regno;
203 int access_size;
204 s64 msize_smax_value;
205 u64 msize_umax_value;
206 int ref_obj_id;
207 int func_id;
208};
209
210static DEFINE_MUTEX(bpf_verifier_lock);
211
212static const struct bpf_line_info *
213find_linfo(const struct bpf_verifier_env *env, u32 insn_off)
214{
215 const struct bpf_line_info *linfo;
216 const struct bpf_prog *prog;
217 u32 i, nr_linfo;
218
219 prog = env->prog;
220 nr_linfo = prog->aux->nr_linfo;
221
222 if (!nr_linfo || insn_off >= prog->len)
223 return NULL;
224
225 linfo = prog->aux->linfo;
226 for (i = 1; i < nr_linfo; i++)
227 if (insn_off < linfo[i].insn_off)
228 break;
229
230 return &linfo[i - 1];
231}
232
233void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,
234 va_list args)
235{
236 unsigned int n;
237
238 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);
239
240 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1,
241 "verifier log line truncated - local buffer too short\n");
242
243 n = min(log->len_total - log->len_used - 1, n);
244 log->kbuf[n] = '\0';
245
246 if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1))
247 log->len_used += n;
248 else
249 log->ubuf = NULL;
250}
251
252
253
254
255
256__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
257 const char *fmt, ...)
258{
259 va_list args;
260
261 if (!bpf_verifier_log_needed(&env->log))
262 return;
263
264 va_start(args, fmt);
265 bpf_verifier_vlog(&env->log, fmt, args);
266 va_end(args);
267}
268EXPORT_SYMBOL_GPL(bpf_verifier_log_write);
269
270__printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
271{
272 struct bpf_verifier_env *env = private_data;
273 va_list args;
274
275 if (!bpf_verifier_log_needed(&env->log))
276 return;
277
278 va_start(args, fmt);
279 bpf_verifier_vlog(&env->log, fmt, args);
280 va_end(args);
281}
282
283static const char *ltrim(const char *s)
284{
285 while (isspace(*s))
286 s++;
287
288 return s;
289}
290
291__printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env,
292 u32 insn_off,
293 const char *prefix_fmt, ...)
294{
295 const struct bpf_line_info *linfo;
296
297 if (!bpf_verifier_log_needed(&env->log))
298 return;
299
300 linfo = find_linfo(env, insn_off);
301 if (!linfo || linfo == env->prev_linfo)
302 return;
303
304 if (prefix_fmt) {
305 va_list args;
306
307 va_start(args, prefix_fmt);
308 bpf_verifier_vlog(&env->log, prefix_fmt, args);
309 va_end(args);
310 }
311
312 verbose(env, "%s\n",
313 ltrim(btf_name_by_offset(env->prog->aux->btf,
314 linfo->line_off)));
315
316 env->prev_linfo = linfo;
317}
318
319static bool type_is_pkt_pointer(enum bpf_reg_type type)
320{
321 return type == PTR_TO_PACKET ||
322 type == PTR_TO_PACKET_META;
323}
324
325static bool type_is_sk_pointer(enum bpf_reg_type type)
326{
327 return type == PTR_TO_SOCKET ||
328 type == PTR_TO_SOCK_COMMON ||
329 type == PTR_TO_TCP_SOCK ||
330 type == PTR_TO_XDP_SOCK;
331}
332
333static bool reg_type_may_be_null(enum bpf_reg_type type)
334{
335 return type == PTR_TO_MAP_VALUE_OR_NULL ||
336 type == PTR_TO_SOCKET_OR_NULL ||
337 type == PTR_TO_SOCK_COMMON_OR_NULL ||
338 type == PTR_TO_TCP_SOCK_OR_NULL;
339}
340
341static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
342{
343 return reg->type == PTR_TO_MAP_VALUE &&
344 map_value_has_spin_lock(reg->map_ptr);
345}
346
347static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type)
348{
349 return type == PTR_TO_SOCKET ||
350 type == PTR_TO_SOCKET_OR_NULL ||
351 type == PTR_TO_TCP_SOCK ||
352 type == PTR_TO_TCP_SOCK_OR_NULL;
353}
354
355static bool arg_type_may_be_refcounted(enum bpf_arg_type type)
356{
357 return type == ARG_PTR_TO_SOCK_COMMON;
358}
359
360
361
362
363
364static bool is_release_function(enum bpf_func_id func_id)
365{
366 return func_id == BPF_FUNC_sk_release;
367}
368
369static bool is_acquire_function(enum bpf_func_id func_id)
370{
371 return func_id == BPF_FUNC_sk_lookup_tcp ||
372 func_id == BPF_FUNC_sk_lookup_udp ||
373 func_id == BPF_FUNC_skc_lookup_tcp;
374}
375
376static bool is_ptr_cast_function(enum bpf_func_id func_id)
377{
378 return func_id == BPF_FUNC_tcp_sock ||
379 func_id == BPF_FUNC_sk_fullsock;
380}
381
382
383static const char * const reg_type_str[] = {
384 [NOT_INIT] = "?",
385 [SCALAR_VALUE] = "inv",
386 [PTR_TO_CTX] = "ctx",
387 [CONST_PTR_TO_MAP] = "map_ptr",
388 [PTR_TO_MAP_VALUE] = "map_value",
389 [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null",
390 [PTR_TO_STACK] = "fp",
391 [PTR_TO_PACKET] = "pkt",
392 [PTR_TO_PACKET_META] = "pkt_meta",
393 [PTR_TO_PACKET_END] = "pkt_end",
394 [PTR_TO_FLOW_KEYS] = "flow_keys",
395 [PTR_TO_SOCKET] = "sock",
396 [PTR_TO_SOCKET_OR_NULL] = "sock_or_null",
397 [PTR_TO_SOCK_COMMON] = "sock_common",
398 [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null",
399 [PTR_TO_TCP_SOCK] = "tcp_sock",
400 [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",
401 [PTR_TO_TP_BUFFER] = "tp_buffer",
402 [PTR_TO_XDP_SOCK] = "xdp_sock",
403};
404
405static char slot_type_char[] = {
406 [STACK_INVALID] = '?',
407 [STACK_SPILL] = 'r',
408 [STACK_MISC] = 'm',
409 [STACK_ZERO] = '0',
410};
411
412static void print_liveness(struct bpf_verifier_env *env,
413 enum bpf_reg_liveness live)
414{
415 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE))
416 verbose(env, "_");
417 if (live & REG_LIVE_READ)
418 verbose(env, "r");
419 if (live & REG_LIVE_WRITTEN)
420 verbose(env, "w");
421 if (live & REG_LIVE_DONE)
422 verbose(env, "D");
423}
424
425static struct bpf_func_state *func(struct bpf_verifier_env *env,
426 const struct bpf_reg_state *reg)
427{
428 struct bpf_verifier_state *cur = env->cur_state;
429
430 return cur->frame[reg->frameno];
431}
432
433static void print_verifier_state(struct bpf_verifier_env *env,
434 const struct bpf_func_state *state)
435{
436 const struct bpf_reg_state *reg;
437 enum bpf_reg_type t;
438 int i;
439
440 if (state->frameno)
441 verbose(env, " frame%d:", state->frameno);
442 for (i = 0; i < MAX_BPF_REG; i++) {
443 reg = &state->regs[i];
444 t = reg->type;
445 if (t == NOT_INIT)
446 continue;
447 verbose(env, " R%d", i);
448 print_liveness(env, reg->live);
449 verbose(env, "=%s", reg_type_str[t]);
450 if (t == SCALAR_VALUE && reg->precise)
451 verbose(env, "P");
452 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) &&
453 tnum_is_const(reg->var_off)) {
454
455 verbose(env, "%lld", reg->var_off.value + reg->off);
456 } else {
457 verbose(env, "(id=%d", reg->id);
458 if (reg_type_may_be_refcounted_or_null(t))
459 verbose(env, ",ref_obj_id=%d", reg->ref_obj_id);
460 if (t != SCALAR_VALUE)
461 verbose(env, ",off=%d", reg->off);
462 if (type_is_pkt_pointer(t))
463 verbose(env, ",r=%d", reg->range);
464 else if (t == CONST_PTR_TO_MAP ||
465 t == PTR_TO_MAP_VALUE ||
466 t == PTR_TO_MAP_VALUE_OR_NULL)
467 verbose(env, ",ks=%d,vs=%d",
468 reg->map_ptr->key_size,
469 reg->map_ptr->value_size);
470 if (tnum_is_const(reg->var_off)) {
471
472
473
474
475 verbose(env, ",imm=%llx", reg->var_off.value);
476 } else {
477 if (reg->smin_value != reg->umin_value &&
478 reg->smin_value != S64_MIN)
479 verbose(env, ",smin_value=%lld",
480 (long long)reg->smin_value);
481 if (reg->smax_value != reg->umax_value &&
482 reg->smax_value != S64_MAX)
483 verbose(env, ",smax_value=%lld",
484 (long long)reg->smax_value);
485 if (reg->umin_value != 0)
486 verbose(env, ",umin_value=%llu",
487 (unsigned long long)reg->umin_value);
488 if (reg->umax_value != U64_MAX)
489 verbose(env, ",umax_value=%llu",
490 (unsigned long long)reg->umax_value);
491 if (!tnum_is_unknown(reg->var_off)) {
492 char tn_buf[48];
493
494 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
495 verbose(env, ",var_off=%s", tn_buf);
496 }
497 }
498 verbose(env, ")");
499 }
500 }
501 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
502 char types_buf[BPF_REG_SIZE + 1];
503 bool valid = false;
504 int j;
505
506 for (j = 0; j < BPF_REG_SIZE; j++) {
507 if (state->stack[i].slot_type[j] != STACK_INVALID)
508 valid = true;
509 types_buf[j] = slot_type_char[
510 state->stack[i].slot_type[j]];
511 }
512 types_buf[BPF_REG_SIZE] = 0;
513 if (!valid)
514 continue;
515 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
516 print_liveness(env, state->stack[i].spilled_ptr.live);
517 if (state->stack[i].slot_type[0] == STACK_SPILL) {
518 reg = &state->stack[i].spilled_ptr;
519 t = reg->type;
520 verbose(env, "=%s", reg_type_str[t]);
521 if (t == SCALAR_VALUE && reg->precise)
522 verbose(env, "P");
523 if (t == SCALAR_VALUE && tnum_is_const(reg->var_off))
524 verbose(env, "%lld", reg->var_off.value + reg->off);
525 } else {
526 verbose(env, "=%s", types_buf);
527 }
528 }
529 if (state->acquired_refs && state->refs[0].id) {
530 verbose(env, " refs=%d", state->refs[0].id);
531 for (i = 1; i < state->acquired_refs; i++)
532 if (state->refs[i].id)
533 verbose(env, ",%d", state->refs[i].id);
534 }
535 verbose(env, "\n");
536}
537
538#define COPY_STATE_FN(NAME, COUNT, FIELD, SIZE) \
539static int copy_##NAME##_state(struct bpf_func_state *dst, \
540 const struct bpf_func_state *src) \
541{ \
542 if (!src->FIELD) \
543 return 0; \
544 if (WARN_ON_ONCE(dst->COUNT < src->COUNT)) { \
545 \
546 memset(dst, 0, sizeof(*dst)); \
547 return -EFAULT; \
548 } \
549 memcpy(dst->FIELD, src->FIELD, \
550 sizeof(*src->FIELD) * (src->COUNT / SIZE)); \
551 return 0; \
552}
553
554COPY_STATE_FN(reference, acquired_refs, refs, 1)
555
556COPY_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE)
557#undef COPY_STATE_FN
558
559#define REALLOC_STATE_FN(NAME, COUNT, FIELD, SIZE) \
560static int realloc_##NAME##_state(struct bpf_func_state *state, int size, \
561 bool copy_old) \
562{ \
563 u32 old_size = state->COUNT; \
564 struct bpf_##NAME##_state *new_##FIELD; \
565 int slot = size / SIZE; \
566 \
567 if (size <= old_size || !size) { \
568 if (copy_old) \
569 return 0; \
570 state->COUNT = slot * SIZE; \
571 if (!size && old_size) { \
572 kfree(state->FIELD); \
573 state->FIELD = NULL; \
574 } \
575 return 0; \
576 } \
577 new_##FIELD = kmalloc_array(slot, sizeof(struct bpf_##NAME##_state), \
578 GFP_KERNEL); \
579 if (!new_##FIELD) \
580 return -ENOMEM; \
581 if (copy_old) { \
582 if (state->FIELD) \
583 memcpy(new_##FIELD, state->FIELD, \
584 sizeof(*new_##FIELD) * (old_size / SIZE)); \
585 memset(new_##FIELD + old_size / SIZE, 0, \
586 sizeof(*new_##FIELD) * (size - old_size) / SIZE); \
587 } \
588 state->COUNT = slot * SIZE; \
589 kfree(state->FIELD); \
590 state->FIELD = new_##FIELD; \
591 return 0; \
592}
593
594REALLOC_STATE_FN(reference, acquired_refs, refs, 1)
595
596REALLOC_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE)
597#undef REALLOC_STATE_FN
598
599
600
601
602
603
604
605
606static int realloc_func_state(struct bpf_func_state *state, int stack_size,
607 int refs_size, bool copy_old)
608{
609 int err = realloc_reference_state(state, refs_size, copy_old);
610 if (err)
611 return err;
612 return realloc_stack_state(state, stack_size, copy_old);
613}
614
615
616
617
618
619
620static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
621{
622 struct bpf_func_state *state = cur_func(env);
623 int new_ofs = state->acquired_refs;
624 int id, err;
625
626 err = realloc_reference_state(state, state->acquired_refs + 1, true);
627 if (err)
628 return err;
629 id = ++env->id_gen;
630 state->refs[new_ofs].id = id;
631 state->refs[new_ofs].insn_idx = insn_idx;
632
633 return id;
634}
635
636
637static int release_reference_state(struct bpf_func_state *state, int ptr_id)
638{
639 int i, last_idx;
640
641 last_idx = state->acquired_refs - 1;
642 for (i = 0; i < state->acquired_refs; i++) {
643 if (state->refs[i].id == ptr_id) {
644 if (last_idx && i != last_idx)
645 memcpy(&state->refs[i], &state->refs[last_idx],
646 sizeof(*state->refs));
647 memset(&state->refs[last_idx], 0, sizeof(*state->refs));
648 state->acquired_refs--;
649 return 0;
650 }
651 }
652 return -EINVAL;
653}
654
655static int transfer_reference_state(struct bpf_func_state *dst,
656 struct bpf_func_state *src)
657{
658 int err = realloc_reference_state(dst, src->acquired_refs, false);
659 if (err)
660 return err;
661 err = copy_reference_state(dst, src);
662 if (err)
663 return err;
664 return 0;
665}
666
667static void free_func_state(struct bpf_func_state *state)
668{
669 if (!state)
670 return;
671 kfree(state->refs);
672 kfree(state->stack);
673 kfree(state);
674}
675
676static void clear_jmp_history(struct bpf_verifier_state *state)
677{
678 kfree(state->jmp_history);
679 state->jmp_history = NULL;
680 state->jmp_history_cnt = 0;
681}
682
683static void free_verifier_state(struct bpf_verifier_state *state,
684 bool free_self)
685{
686 int i;
687
688 for (i = 0; i <= state->curframe; i++) {
689 free_func_state(state->frame[i]);
690 state->frame[i] = NULL;
691 }
692 clear_jmp_history(state);
693 if (free_self)
694 kfree(state);
695}
696
697
698
699
700static int copy_func_state(struct bpf_func_state *dst,
701 const struct bpf_func_state *src)
702{
703 int err;
704
705 err = realloc_func_state(dst, src->allocated_stack, src->acquired_refs,
706 false);
707 if (err)
708 return err;
709 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs));
710 err = copy_reference_state(dst, src);
711 if (err)
712 return err;
713 return copy_stack_state(dst, src);
714}
715
716static int copy_verifier_state(struct bpf_verifier_state *dst_state,
717 const struct bpf_verifier_state *src)
718{
719 struct bpf_func_state *dst;
720 u32 jmp_sz = sizeof(struct bpf_idx_pair) * src->jmp_history_cnt;
721 int i, err;
722
723 if (dst_state->jmp_history_cnt < src->jmp_history_cnt) {
724 kfree(dst_state->jmp_history);
725 dst_state->jmp_history = kmalloc(jmp_sz, GFP_USER);
726 if (!dst_state->jmp_history)
727 return -ENOMEM;
728 }
729 memcpy(dst_state->jmp_history, src->jmp_history, jmp_sz);
730 dst_state->jmp_history_cnt = src->jmp_history_cnt;
731
732
733 for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
734 free_func_state(dst_state->frame[i]);
735 dst_state->frame[i] = NULL;
736 }
737 dst_state->speculative = src->speculative;
738 dst_state->curframe = src->curframe;
739 dst_state->active_spin_lock = src->active_spin_lock;
740 dst_state->branches = src->branches;
741 dst_state->parent = src->parent;
742 dst_state->first_insn_idx = src->first_insn_idx;
743 dst_state->last_insn_idx = src->last_insn_idx;
744 for (i = 0; i <= src->curframe; i++) {
745 dst = dst_state->frame[i];
746 if (!dst) {
747 dst = kzalloc(sizeof(*dst), GFP_KERNEL);
748 if (!dst)
749 return -ENOMEM;
750 dst_state->frame[i] = dst;
751 }
752 err = copy_func_state(dst, src->frame[i]);
753 if (err)
754 return err;
755 }
756 return 0;
757}
758
759static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
760{
761 while (st) {
762 u32 br = --st->branches;
763
764
765
766
767 WARN_ONCE((int)br < 0,
768 "BUG update_branch_counts:branches_to_explore=%d\n",
769 br);
770 if (br)
771 break;
772 st = st->parent;
773 }
774}
775
776static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
777 int *insn_idx)
778{
779 struct bpf_verifier_state *cur = env->cur_state;
780 struct bpf_verifier_stack_elem *elem, *head = env->head;
781 int err;
782
783 if (env->head == NULL)
784 return -ENOENT;
785
786 if (cur) {
787 err = copy_verifier_state(cur, &head->st);
788 if (err)
789 return err;
790 }
791 if (insn_idx)
792 *insn_idx = head->insn_idx;
793 if (prev_insn_idx)
794 *prev_insn_idx = head->prev_insn_idx;
795 elem = head->next;
796 free_verifier_state(&head->st, false);
797 kfree(head);
798 env->head = elem;
799 env->stack_size--;
800 return 0;
801}
802
803static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
804 int insn_idx, int prev_insn_idx,
805 bool speculative)
806{
807 struct bpf_verifier_state *cur = env->cur_state;
808 struct bpf_verifier_stack_elem *elem;
809 int err;
810
811 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
812 if (!elem)
813 goto err;
814
815 elem->insn_idx = insn_idx;
816 elem->prev_insn_idx = prev_insn_idx;
817 elem->next = env->head;
818 env->head = elem;
819 env->stack_size++;
820 err = copy_verifier_state(&elem->st, cur);
821 if (err)
822 goto err;
823 elem->st.speculative |= speculative;
824 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
825 verbose(env, "The sequence of %d jumps is too complex.\n",
826 env->stack_size);
827 goto err;
828 }
829 if (elem->st.parent) {
830 ++elem->st.parent->branches;
831
832
833
834
835
836
837
838
839
840 }
841 return &elem->st;
842err:
843 free_verifier_state(env->cur_state, true);
844 env->cur_state = NULL;
845
846 while (!pop_stack(env, NULL, NULL));
847 return NULL;
848}
849
850#define CALLER_SAVED_REGS 6
851static const int caller_saved[CALLER_SAVED_REGS] = {
852 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
853};
854
855static void __mark_reg_not_init(struct bpf_reg_state *reg);
856
857
858
859
860static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
861{
862
863 memset(((u8 *)reg) + sizeof(reg->type), 0,
864 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
865 reg->var_off = tnum_const(imm);
866 reg->smin_value = (s64)imm;
867 reg->smax_value = (s64)imm;
868 reg->umin_value = imm;
869 reg->umax_value = imm;
870}
871
872
873
874
875static void __mark_reg_known_zero(struct bpf_reg_state *reg)
876{
877 __mark_reg_known(reg, 0);
878}
879
880static void __mark_reg_const_zero(struct bpf_reg_state *reg)
881{
882 __mark_reg_known(reg, 0);
883 reg->type = SCALAR_VALUE;
884}
885
886static void mark_reg_known_zero(struct bpf_verifier_env *env,
887 struct bpf_reg_state *regs, u32 regno)
888{
889 if (WARN_ON(regno >= MAX_BPF_REG)) {
890 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
891
892 for (regno = 0; regno < MAX_BPF_REG; regno++)
893 __mark_reg_not_init(regs + regno);
894 return;
895 }
896 __mark_reg_known_zero(regs + regno);
897}
898
899static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
900{
901 return type_is_pkt_pointer(reg->type);
902}
903
904static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
905{
906 return reg_is_pkt_pointer(reg) ||
907 reg->type == PTR_TO_PACKET_END;
908}
909
910
911static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
912 enum bpf_reg_type which)
913{
914
915
916
917
918 return reg->type == which &&
919 reg->id == 0 &&
920 reg->off == 0 &&
921 tnum_equals_const(reg->var_off, 0);
922}
923
924
925static void __update_reg_bounds(struct bpf_reg_state *reg)
926{
927
928 reg->smin_value = max_t(s64, reg->smin_value,
929 reg->var_off.value | (reg->var_off.mask & S64_MIN));
930
931 reg->smax_value = min_t(s64, reg->smax_value,
932 reg->var_off.value | (reg->var_off.mask & S64_MAX));
933 reg->umin_value = max(reg->umin_value, reg->var_off.value);
934 reg->umax_value = min(reg->umax_value,
935 reg->var_off.value | reg->var_off.mask);
936}
937
938
939static void __reg_deduce_bounds(struct bpf_reg_state *reg)
940{
941
942
943
944
945
946 if (reg->smin_value >= 0 || reg->smax_value < 0) {
947 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
948 reg->umin_value);
949 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
950 reg->umax_value);
951 return;
952 }
953
954
955
956 if ((s64)reg->umax_value >= 0) {
957
958
959
960 reg->smin_value = reg->umin_value;
961 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
962 reg->umax_value);
963 } else if ((s64)reg->umin_value < 0) {
964
965
966
967 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
968 reg->umin_value);
969 reg->smax_value = reg->umax_value;
970 }
971}
972
973
974static void __reg_bound_offset(struct bpf_reg_state *reg)
975{
976 reg->var_off = tnum_intersect(reg->var_off,
977 tnum_range(reg->umin_value,
978 reg->umax_value));
979}
980
981
982static void __mark_reg_unbounded(struct bpf_reg_state *reg)
983{
984 reg->smin_value = S64_MIN;
985 reg->smax_value = S64_MAX;
986 reg->umin_value = 0;
987 reg->umax_value = U64_MAX;
988}
989
990
991static void __mark_reg_unknown(struct bpf_reg_state *reg)
992{
993
994
995
996
997 memset(reg, 0, offsetof(struct bpf_reg_state, var_off));
998 reg->type = SCALAR_VALUE;
999 reg->var_off = tnum_unknown;
1000 reg->frameno = 0;
1001 __mark_reg_unbounded(reg);
1002}
1003
1004static void mark_reg_unknown(struct bpf_verifier_env *env,
1005 struct bpf_reg_state *regs, u32 regno)
1006{
1007 if (WARN_ON(regno >= MAX_BPF_REG)) {
1008 verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
1009
1010 for (regno = 0; regno < BPF_REG_FP; regno++)
1011 __mark_reg_not_init(regs + regno);
1012 return;
1013 }
1014 regs += regno;
1015 __mark_reg_unknown(regs);
1016
1017 regs->precise = env->subprog_cnt > 1 || !env->allow_ptr_leaks ?
1018 true : false;
1019}
1020
1021static void __mark_reg_not_init(struct bpf_reg_state *reg)
1022{
1023 __mark_reg_unknown(reg);
1024 reg->type = NOT_INIT;
1025}
1026
1027static void mark_reg_not_init(struct bpf_verifier_env *env,
1028 struct bpf_reg_state *regs, u32 regno)
1029{
1030 if (WARN_ON(regno >= MAX_BPF_REG)) {
1031 verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
1032
1033 for (regno = 0; regno < BPF_REG_FP; regno++)
1034 __mark_reg_not_init(regs + regno);
1035 return;
1036 }
1037 __mark_reg_not_init(regs + regno);
1038}
1039
1040#define DEF_NOT_SUBREG (0)
1041static void init_reg_state(struct bpf_verifier_env *env,
1042 struct bpf_func_state *state)
1043{
1044 struct bpf_reg_state *regs = state->regs;
1045 int i;
1046
1047 for (i = 0; i < MAX_BPF_REG; i++) {
1048 mark_reg_not_init(env, regs, i);
1049 regs[i].live = REG_LIVE_NONE;
1050 regs[i].parent = NULL;
1051 regs[i].subreg_def = DEF_NOT_SUBREG;
1052 }
1053
1054
1055 regs[BPF_REG_FP].type = PTR_TO_STACK;
1056 mark_reg_known_zero(env, regs, BPF_REG_FP);
1057 regs[BPF_REG_FP].frameno = state->frameno;
1058
1059
1060 regs[BPF_REG_1].type = PTR_TO_CTX;
1061 mark_reg_known_zero(env, regs, BPF_REG_1);
1062}
1063
1064#define BPF_MAIN_FUNC (-1)
1065static void init_func_state(struct bpf_verifier_env *env,
1066 struct bpf_func_state *state,
1067 int callsite, int frameno, int subprogno)
1068{
1069 state->callsite = callsite;
1070 state->frameno = frameno;
1071 state->subprogno = subprogno;
1072 init_reg_state(env, state);
1073}
1074
1075enum reg_arg_type {
1076 SRC_OP,
1077 DST_OP,
1078 DST_OP_NO_MARK
1079};
1080
1081static int cmp_subprogs(const void *a, const void *b)
1082{
1083 return ((struct bpf_subprog_info *)a)->start -
1084 ((struct bpf_subprog_info *)b)->start;
1085}
1086
1087static int find_subprog(struct bpf_verifier_env *env, int off)
1088{
1089 struct bpf_subprog_info *p;
1090
1091 p = bsearch(&off, env->subprog_info, env->subprog_cnt,
1092 sizeof(env->subprog_info[0]), cmp_subprogs);
1093 if (!p)
1094 return -ENOENT;
1095 return p - env->subprog_info;
1096
1097}
1098
1099static int add_subprog(struct bpf_verifier_env *env, int off)
1100{
1101 int insn_cnt = env->prog->len;
1102 int ret;
1103
1104 if (off >= insn_cnt || off < 0) {
1105 verbose(env, "call to invalid destination\n");
1106 return -EINVAL;
1107 }
1108 ret = find_subprog(env, off);
1109 if (ret >= 0)
1110 return 0;
1111 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
1112 verbose(env, "too many subprograms\n");
1113 return -E2BIG;
1114 }
1115 env->subprog_info[env->subprog_cnt++].start = off;
1116 sort(env->subprog_info, env->subprog_cnt,
1117 sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
1118 return 0;
1119}
1120
1121static int check_subprogs(struct bpf_verifier_env *env)
1122{
1123 int i, ret, subprog_start, subprog_end, off, cur_subprog = 0;
1124 struct bpf_subprog_info *subprog = env->subprog_info;
1125 struct bpf_insn *insn = env->prog->insnsi;
1126 int insn_cnt = env->prog->len;
1127
1128
1129 ret = add_subprog(env, 0);
1130 if (ret < 0)
1131 return ret;
1132
1133
1134 for (i = 0; i < insn_cnt; i++) {
1135 if (insn[i].code != (BPF_JMP | BPF_CALL))
1136 continue;
1137 if (insn[i].src_reg != BPF_PSEUDO_CALL)
1138 continue;
1139 if (!env->allow_ptr_leaks) {
1140 verbose(env, "function calls to other bpf functions are allowed for root only\n");
1141 return -EPERM;
1142 }
1143 ret = add_subprog(env, i + insn[i].imm + 1);
1144 if (ret < 0)
1145 return ret;
1146 }
1147
1148
1149
1150
1151 subprog[env->subprog_cnt].start = insn_cnt;
1152
1153 if (env->log.level & BPF_LOG_LEVEL2)
1154 for (i = 0; i < env->subprog_cnt; i++)
1155 verbose(env, "func#%d @%d\n", i, subprog[i].start);
1156
1157
1158 subprog_start = subprog[cur_subprog].start;
1159 subprog_end = subprog[cur_subprog + 1].start;
1160 for (i = 0; i < insn_cnt; i++) {
1161 u8 code = insn[i].code;
1162
1163 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
1164 goto next;
1165 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
1166 goto next;
1167 off = i + insn[i].off + 1;
1168 if (off < subprog_start || off >= subprog_end) {
1169 verbose(env, "jump out of range from insn %d to %d\n", i, off);
1170 return -EINVAL;
1171 }
1172next:
1173 if (i == subprog_end - 1) {
1174
1175
1176
1177
1178 if (code != (BPF_JMP | BPF_EXIT) &&
1179 code != (BPF_JMP | BPF_JA)) {
1180 verbose(env, "last insn is not an exit or jmp\n");
1181 return -EINVAL;
1182 }
1183 subprog_start = subprog_end;
1184 cur_subprog++;
1185 if (cur_subprog < env->subprog_cnt)
1186 subprog_end = subprog[cur_subprog + 1].start;
1187 }
1188 }
1189 return 0;
1190}
1191
1192
1193
1194
1195static int mark_reg_read(struct bpf_verifier_env *env,
1196 const struct bpf_reg_state *state,
1197 struct bpf_reg_state *parent, u8 flag)
1198{
1199 bool writes = parent == state->parent;
1200 int cnt = 0;
1201
1202 while (parent) {
1203
1204 if (writes && state->live & REG_LIVE_WRITTEN)
1205 break;
1206 if (parent->live & REG_LIVE_DONE) {
1207 verbose(env, "verifier BUG type %s var_off %lld off %d\n",
1208 reg_type_str[parent->type],
1209 parent->var_off.value, parent->off);
1210 return -EFAULT;
1211 }
1212
1213
1214
1215 if ((parent->live & REG_LIVE_READ) == flag ||
1216 parent->live & REG_LIVE_READ64)
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226 break;
1227
1228 parent->live |= flag;
1229
1230 if (flag == REG_LIVE_READ64)
1231 parent->live &= ~REG_LIVE_READ32;
1232 state = parent;
1233 parent = state->parent;
1234 writes = true;
1235 cnt++;
1236 }
1237
1238 if (env->longest_mark_read_walk < cnt)
1239 env->longest_mark_read_walk = cnt;
1240 return 0;
1241}
1242
1243
1244
1245
1246
1247static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,
1248 u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t)
1249{
1250 u8 code, class, op;
1251
1252 code = insn->code;
1253 class = BPF_CLASS(code);
1254 op = BPF_OP(code);
1255 if (class == BPF_JMP) {
1256
1257
1258
1259 if (op == BPF_EXIT)
1260 return true;
1261 if (op == BPF_CALL) {
1262
1263
1264
1265
1266
1267 if (insn->src_reg == BPF_PSEUDO_CALL)
1268 return false;
1269
1270
1271
1272 if (t == SRC_OP)
1273 return true;
1274
1275 return false;
1276 }
1277 }
1278
1279 if (class == BPF_ALU64 || class == BPF_JMP ||
1280
1281 (class == BPF_ALU && op == BPF_END && insn->imm == 64))
1282 return true;
1283
1284 if (class == BPF_ALU || class == BPF_JMP32)
1285 return false;
1286
1287 if (class == BPF_LDX) {
1288 if (t != SRC_OP)
1289 return BPF_SIZE(code) == BPF_DW;
1290
1291 return true;
1292 }
1293
1294 if (class == BPF_STX) {
1295 if (reg->type != SCALAR_VALUE)
1296 return true;
1297 return BPF_SIZE(code) == BPF_DW;
1298 }
1299
1300 if (class == BPF_LD) {
1301 u8 mode = BPF_MODE(code);
1302
1303
1304 if (mode == BPF_IMM)
1305 return true;
1306
1307
1308 if (t != SRC_OP)
1309 return false;
1310
1311
1312 if (regno == BPF_REG_6)
1313 return true;
1314
1315
1316 return true;
1317 }
1318
1319 if (class == BPF_ST)
1320
1321 return true;
1322
1323
1324 return true;
1325}
1326
1327
1328static bool insn_no_def(struct bpf_insn *insn)
1329{
1330 u8 class = BPF_CLASS(insn->code);
1331
1332 return (class == BPF_JMP || class == BPF_JMP32 ||
1333 class == BPF_STX || class == BPF_ST);
1334}
1335
1336
1337static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)
1338{
1339 if (insn_no_def(insn))
1340 return false;
1341
1342 return !is_reg64(env, insn, insn->dst_reg, NULL, DST_OP);
1343}
1344
1345static void mark_insn_zext(struct bpf_verifier_env *env,
1346 struct bpf_reg_state *reg)
1347{
1348 s32 def_idx = reg->subreg_def;
1349
1350 if (def_idx == DEF_NOT_SUBREG)
1351 return;
1352
1353 env->insn_aux_data[def_idx - 1].zext_dst = true;
1354
1355 reg->subreg_def = DEF_NOT_SUBREG;
1356}
1357
1358static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
1359 enum reg_arg_type t)
1360{
1361 struct bpf_verifier_state *vstate = env->cur_state;
1362 struct bpf_func_state *state = vstate->frame[vstate->curframe];
1363 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;
1364 struct bpf_reg_state *reg, *regs = state->regs;
1365 bool rw64;
1366
1367 if (regno >= MAX_BPF_REG) {
1368 verbose(env, "R%d is invalid\n", regno);
1369 return -EINVAL;
1370 }
1371
1372 reg = ®s[regno];
1373 rw64 = is_reg64(env, insn, regno, reg, t);
1374 if (t == SRC_OP) {
1375
1376 if (reg->type == NOT_INIT) {
1377 verbose(env, "R%d !read_ok\n", regno);
1378 return -EACCES;
1379 }
1380
1381 if (regno == BPF_REG_FP)
1382 return 0;
1383
1384 if (rw64)
1385 mark_insn_zext(env, reg);
1386
1387 return mark_reg_read(env, reg, reg->parent,
1388 rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32);
1389 } else {
1390
1391 if (regno == BPF_REG_FP) {
1392 verbose(env, "frame pointer is read only\n");
1393 return -EACCES;
1394 }
1395 reg->live |= REG_LIVE_WRITTEN;
1396 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;
1397 if (t == DST_OP)
1398 mark_reg_unknown(env, regs, regno);
1399 }
1400 return 0;
1401}
1402
1403
1404static int push_jmp_history(struct bpf_verifier_env *env,
1405 struct bpf_verifier_state *cur)
1406{
1407 u32 cnt = cur->jmp_history_cnt;
1408 struct bpf_idx_pair *p;
1409
1410 cnt++;
1411 p = krealloc(cur->jmp_history, cnt * sizeof(*p), GFP_USER);
1412 if (!p)
1413 return -ENOMEM;
1414 p[cnt - 1].idx = env->insn_idx;
1415 p[cnt - 1].prev_idx = env->prev_insn_idx;
1416 cur->jmp_history = p;
1417 cur->jmp_history_cnt = cnt;
1418 return 0;
1419}
1420
1421
1422
1423
1424static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
1425 u32 *history)
1426{
1427 u32 cnt = *history;
1428
1429 if (cnt && st->jmp_history[cnt - 1].idx == i) {
1430 i = st->jmp_history[cnt - 1].prev_idx;
1431 (*history)--;
1432 } else {
1433 i--;
1434 }
1435 return i;
1436}
1437
1438
1439
1440
1441
1442static int backtrack_insn(struct bpf_verifier_env *env, int idx,
1443 u32 *reg_mask, u64 *stack_mask)
1444{
1445 const struct bpf_insn_cbs cbs = {
1446 .cb_print = verbose,
1447 .private_data = env,
1448 };
1449 struct bpf_insn *insn = env->prog->insnsi + idx;
1450 u8 class = BPF_CLASS(insn->code);
1451 u8 opcode = BPF_OP(insn->code);
1452 u8 mode = BPF_MODE(insn->code);
1453 u32 dreg = 1u << insn->dst_reg;
1454 u32 sreg = 1u << insn->src_reg;
1455 u32 spi;
1456
1457 if (insn->code == 0)
1458 return 0;
1459 if (env->log.level & BPF_LOG_LEVEL) {
1460 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask);
1461 verbose(env, "%d: ", idx);
1462 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
1463 }
1464
1465 if (class == BPF_ALU || class == BPF_ALU64) {
1466 if (!(*reg_mask & dreg))
1467 return 0;
1468 if (opcode == BPF_MOV) {
1469 if (BPF_SRC(insn->code) == BPF_X) {
1470
1471
1472
1473
1474 *reg_mask &= ~dreg;
1475 *reg_mask |= sreg;
1476 } else {
1477
1478
1479
1480
1481
1482
1483 *reg_mask &= ~dreg;
1484 }
1485 } else {
1486 if (BPF_SRC(insn->code) == BPF_X) {
1487
1488
1489
1490
1491 *reg_mask |= sreg;
1492 }
1493
1494
1495 }
1496 } else if (class == BPF_LDX) {
1497 if (!(*reg_mask & dreg))
1498 return 0;
1499 *reg_mask &= ~dreg;
1500
1501
1502
1503
1504
1505
1506
1507 if (insn->src_reg != BPF_REG_FP)
1508 return 0;
1509 if (BPF_SIZE(insn->code) != BPF_DW)
1510 return 0;
1511
1512
1513
1514
1515
1516 spi = (-insn->off - 1) / BPF_REG_SIZE;
1517 if (spi >= 64) {
1518 verbose(env, "BUG spi %d\n", spi);
1519 WARN_ONCE(1, "verifier backtracking bug");
1520 return -EFAULT;
1521 }
1522 *stack_mask |= 1ull << spi;
1523 } else if (class == BPF_STX || class == BPF_ST) {
1524 if (*reg_mask & dreg)
1525
1526
1527
1528
1529 return -ENOTSUPP;
1530
1531 if (insn->dst_reg != BPF_REG_FP)
1532 return 0;
1533 if (BPF_SIZE(insn->code) != BPF_DW)
1534 return 0;
1535 spi = (-insn->off - 1) / BPF_REG_SIZE;
1536 if (spi >= 64) {
1537 verbose(env, "BUG spi %d\n", spi);
1538 WARN_ONCE(1, "verifier backtracking bug");
1539 return -EFAULT;
1540 }
1541 if (!(*stack_mask & (1ull << spi)))
1542 return 0;
1543 *stack_mask &= ~(1ull << spi);
1544 if (class == BPF_STX)
1545 *reg_mask |= sreg;
1546 } else if (class == BPF_JMP || class == BPF_JMP32) {
1547 if (opcode == BPF_CALL) {
1548 if (insn->src_reg == BPF_PSEUDO_CALL)
1549 return -ENOTSUPP;
1550
1551 *reg_mask &= ~1;
1552 if (*reg_mask & 0x3f) {
1553
1554
1555
1556 verbose(env, "BUG regs %x\n", *reg_mask);
1557 WARN_ONCE(1, "verifier backtracking bug");
1558 return -EFAULT;
1559 }
1560 } else if (opcode == BPF_EXIT) {
1561 return -ENOTSUPP;
1562 }
1563 } else if (class == BPF_LD) {
1564 if (!(*reg_mask & dreg))
1565 return 0;
1566 *reg_mask &= ~dreg;
1567
1568
1569
1570
1571 if (mode == BPF_IND || mode == BPF_ABS)
1572
1573 return -ENOTSUPP;
1574 }
1575 return 0;
1576}
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630static void mark_all_scalars_precise(struct bpf_verifier_env *env,
1631 struct bpf_verifier_state *st)
1632{
1633 struct bpf_func_state *func;
1634 struct bpf_reg_state *reg;
1635 int i, j;
1636
1637
1638
1639
1640 for (; st; st = st->parent)
1641 for (i = 0; i <= st->curframe; i++) {
1642 func = st->frame[i];
1643 for (j = 0; j < BPF_REG_FP; j++) {
1644 reg = &func->regs[j];
1645 if (reg->type != SCALAR_VALUE)
1646 continue;
1647 reg->precise = true;
1648 }
1649 for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
1650 if (func->stack[j].slot_type[0] != STACK_SPILL)
1651 continue;
1652 reg = &func->stack[j].spilled_ptr;
1653 if (reg->type != SCALAR_VALUE)
1654 continue;
1655 reg->precise = true;
1656 }
1657 }
1658}
1659
1660static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,
1661 int spi)
1662{
1663 struct bpf_verifier_state *st = env->cur_state;
1664 int first_idx = st->first_insn_idx;
1665 int last_idx = env->insn_idx;
1666 struct bpf_func_state *func;
1667 struct bpf_reg_state *reg;
1668 u32 reg_mask = regno >= 0 ? 1u << regno : 0;
1669 u64 stack_mask = spi >= 0 ? 1ull << spi : 0;
1670 bool skip_first = true;
1671 bool new_marks = false;
1672 int i, err;
1673
1674 if (!env->allow_ptr_leaks)
1675
1676 return 0;
1677
1678 func = st->frame[st->curframe];
1679 if (regno >= 0) {
1680 reg = &func->regs[regno];
1681 if (reg->type != SCALAR_VALUE) {
1682 WARN_ONCE(1, "backtracing misuse");
1683 return -EFAULT;
1684 }
1685 if (!reg->precise)
1686 new_marks = true;
1687 else
1688 reg_mask = 0;
1689 reg->precise = true;
1690 }
1691
1692 while (spi >= 0) {
1693 if (func->stack[spi].slot_type[0] != STACK_SPILL) {
1694 stack_mask = 0;
1695 break;
1696 }
1697 reg = &func->stack[spi].spilled_ptr;
1698 if (reg->type != SCALAR_VALUE) {
1699 stack_mask = 0;
1700 break;
1701 }
1702 if (!reg->precise)
1703 new_marks = true;
1704 else
1705 stack_mask = 0;
1706 reg->precise = true;
1707 break;
1708 }
1709
1710 if (!new_marks)
1711 return 0;
1712 if (!reg_mask && !stack_mask)
1713 return 0;
1714 for (;;) {
1715 DECLARE_BITMAP(mask, 64);
1716 u32 history = st->jmp_history_cnt;
1717
1718 if (env->log.level & BPF_LOG_LEVEL)
1719 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);
1720 for (i = last_idx;;) {
1721 if (skip_first) {
1722 err = 0;
1723 skip_first = false;
1724 } else {
1725 err = backtrack_insn(env, i, ®_mask, &stack_mask);
1726 }
1727 if (err == -ENOTSUPP) {
1728 mark_all_scalars_precise(env, st);
1729 return 0;
1730 } else if (err) {
1731 return err;
1732 }
1733 if (!reg_mask && !stack_mask)
1734
1735
1736
1737
1738 return 0;
1739 if (i == first_idx)
1740 break;
1741 i = get_prev_insn_idx(st, i, &history);
1742 if (i >= env->prog->len) {
1743
1744
1745
1746
1747
1748
1749 verbose(env, "BUG backtracking idx %d\n", i);
1750 WARN_ONCE(1, "verifier backtracking bug");
1751 return -EFAULT;
1752 }
1753 }
1754 st = st->parent;
1755 if (!st)
1756 break;
1757
1758 new_marks = false;
1759 func = st->frame[st->curframe];
1760 bitmap_from_u64(mask, reg_mask);
1761 for_each_set_bit(i, mask, 32) {
1762 reg = &func->regs[i];
1763 if (reg->type != SCALAR_VALUE) {
1764 reg_mask &= ~(1u << i);
1765 continue;
1766 }
1767 if (!reg->precise)
1768 new_marks = true;
1769 reg->precise = true;
1770 }
1771
1772 bitmap_from_u64(mask, stack_mask);
1773 for_each_set_bit(i, mask, 64) {
1774 if (i >= func->allocated_stack / BPF_REG_SIZE) {
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788 mark_all_scalars_precise(env, st);
1789 return 0;
1790 }
1791
1792 if (func->stack[i].slot_type[0] != STACK_SPILL) {
1793 stack_mask &= ~(1ull << i);
1794 continue;
1795 }
1796 reg = &func->stack[i].spilled_ptr;
1797 if (reg->type != SCALAR_VALUE) {
1798 stack_mask &= ~(1ull << i);
1799 continue;
1800 }
1801 if (!reg->precise)
1802 new_marks = true;
1803 reg->precise = true;
1804 }
1805 if (env->log.level & BPF_LOG_LEVEL) {
1806 print_verifier_state(env, func);
1807 verbose(env, "parent %s regs=%x stack=%llx marks\n",
1808 new_marks ? "didn't have" : "already had",
1809 reg_mask, stack_mask);
1810 }
1811
1812 if (!reg_mask && !stack_mask)
1813 break;
1814 if (!new_marks)
1815 break;
1816
1817 last_idx = st->last_insn_idx;
1818 first_idx = st->first_insn_idx;
1819 }
1820 return 0;
1821}
1822
1823static int mark_chain_precision(struct bpf_verifier_env *env, int regno)
1824{
1825 return __mark_chain_precision(env, regno, -1);
1826}
1827
1828static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi)
1829{
1830 return __mark_chain_precision(env, -1, spi);
1831}
1832
1833static bool is_spillable_regtype(enum bpf_reg_type type)
1834{
1835 switch (type) {
1836 case PTR_TO_MAP_VALUE:
1837 case PTR_TO_MAP_VALUE_OR_NULL:
1838 case PTR_TO_STACK:
1839 case PTR_TO_CTX:
1840 case PTR_TO_PACKET:
1841 case PTR_TO_PACKET_META:
1842 case PTR_TO_PACKET_END:
1843 case PTR_TO_FLOW_KEYS:
1844 case CONST_PTR_TO_MAP:
1845 case PTR_TO_SOCKET:
1846 case PTR_TO_SOCKET_OR_NULL:
1847 case PTR_TO_SOCK_COMMON:
1848 case PTR_TO_SOCK_COMMON_OR_NULL:
1849 case PTR_TO_TCP_SOCK:
1850 case PTR_TO_TCP_SOCK_OR_NULL:
1851 case PTR_TO_XDP_SOCK:
1852 return true;
1853 default:
1854 return false;
1855 }
1856}
1857
1858
1859static bool register_is_null(struct bpf_reg_state *reg)
1860{
1861 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
1862}
1863
1864static bool register_is_const(struct bpf_reg_state *reg)
1865{
1866 return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);
1867}
1868
1869static void save_register_state(struct bpf_func_state *state,
1870 int spi, struct bpf_reg_state *reg)
1871{
1872 int i;
1873
1874 state->stack[spi].spilled_ptr = *reg;
1875 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
1876
1877 for (i = 0; i < BPF_REG_SIZE; i++)
1878 state->stack[spi].slot_type[i] = STACK_SPILL;
1879}
1880
1881
1882
1883
1884static int check_stack_write(struct bpf_verifier_env *env,
1885 struct bpf_func_state *state,
1886 int off, int size, int value_regno, int insn_idx)
1887{
1888 struct bpf_func_state *cur;
1889 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
1890 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg;
1891 struct bpf_reg_state *reg = NULL;
1892
1893 err = realloc_func_state(state, round_up(slot + 1, BPF_REG_SIZE),
1894 state->acquired_refs, true);
1895 if (err)
1896 return err;
1897
1898
1899
1900 if (!env->allow_ptr_leaks &&
1901 state->stack[spi].slot_type[0] == STACK_SPILL &&
1902 size != BPF_REG_SIZE) {
1903 verbose(env, "attempt to corrupt spilled pointer on stack\n");
1904 return -EACCES;
1905 }
1906
1907 cur = env->cur_state->frame[env->cur_state->curframe];
1908 if (value_regno >= 0)
1909 reg = &cur->regs[value_regno];
1910
1911 if (reg && size == BPF_REG_SIZE && register_is_const(reg) &&
1912 !register_is_null(reg) && env->allow_ptr_leaks) {
1913 if (dst_reg != BPF_REG_FP) {
1914
1915
1916
1917
1918
1919
1920 err = mark_chain_precision(env, value_regno);
1921 if (err)
1922 return err;
1923 }
1924 save_register_state(state, spi, reg);
1925 } else if (reg && is_spillable_regtype(reg->type)) {
1926
1927 if (size != BPF_REG_SIZE) {
1928 verbose_linfo(env, insn_idx, "; ");
1929 verbose(env, "invalid size of register spill\n");
1930 return -EACCES;
1931 }
1932
1933 if (state != cur && reg->type == PTR_TO_STACK) {
1934 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
1935 return -EINVAL;
1936 }
1937
1938 if (!env->allow_ptr_leaks) {
1939 bool sanitize = false;
1940
1941 if (state->stack[spi].slot_type[0] == STACK_SPILL &&
1942 register_is_const(&state->stack[spi].spilled_ptr))
1943 sanitize = true;
1944 for (i = 0; i < BPF_REG_SIZE; i++)
1945 if (state->stack[spi].slot_type[i] == STACK_MISC) {
1946 sanitize = true;
1947 break;
1948 }
1949 if (sanitize) {
1950 int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off;
1951 int soff = (-spi - 1) * BPF_REG_SIZE;
1952
1953
1954
1955
1956
1957
1958
1959
1960 if (*poff && *poff != soff) {
1961
1962
1963
1964
1965 verbose(env,
1966 "insn %d cannot access two stack slots fp%d and fp%d",
1967 insn_idx, *poff, soff);
1968 return -EINVAL;
1969 }
1970 *poff = soff;
1971 }
1972 }
1973 save_register_state(state, spi, reg);
1974 } else {
1975 u8 type = STACK_MISC;
1976
1977
1978 state->stack[spi].spilled_ptr.type = NOT_INIT;
1979
1980 if (state->stack[spi].slot_type[0] == STACK_SPILL)
1981 for (i = 0; i < BPF_REG_SIZE; i++)
1982 state->stack[spi].slot_type[i] = STACK_MISC;
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992 if (size == BPF_REG_SIZE)
1993 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
1994
1995
1996 if (reg && register_is_null(reg)) {
1997
1998 err = mark_chain_precision(env, value_regno);
1999 if (err)
2000 return err;
2001 type = STACK_ZERO;
2002 }
2003
2004
2005 for (i = 0; i < size; i++)
2006 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] =
2007 type;
2008 }
2009 return 0;
2010}
2011
2012static int check_stack_read(struct bpf_verifier_env *env,
2013 struct bpf_func_state *reg_state ,
2014 int off, int size, int value_regno)
2015{
2016 struct bpf_verifier_state *vstate = env->cur_state;
2017 struct bpf_func_state *state = vstate->frame[vstate->curframe];
2018 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
2019 struct bpf_reg_state *reg;
2020 u8 *stype;
2021
2022 if (reg_state->allocated_stack <= slot) {
2023 verbose(env, "invalid read from stack off %d+0 size %d\n",
2024 off, size);
2025 return -EACCES;
2026 }
2027 stype = reg_state->stack[spi].slot_type;
2028 reg = ®_state->stack[spi].spilled_ptr;
2029
2030 if (stype[0] == STACK_SPILL) {
2031 if (size != BPF_REG_SIZE) {
2032 if (reg->type != SCALAR_VALUE) {
2033 verbose_linfo(env, env->insn_idx, "; ");
2034 verbose(env, "invalid size of register fill\n");
2035 return -EACCES;
2036 }
2037 if (value_regno >= 0) {
2038 mark_reg_unknown(env, state->regs, value_regno);
2039 state->regs[value_regno].live |= REG_LIVE_WRITTEN;
2040 }
2041 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2042 return 0;
2043 }
2044 for (i = 1; i < BPF_REG_SIZE; i++) {
2045 if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) {
2046 verbose(env, "corrupted spill memory\n");
2047 return -EACCES;
2048 }
2049 }
2050
2051 if (value_regno >= 0) {
2052
2053 state->regs[value_regno] = *reg;
2054
2055
2056
2057
2058 state->regs[value_regno].live |= REG_LIVE_WRITTEN;
2059 }
2060 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2061 } else {
2062 int zeros = 0;
2063
2064 for (i = 0; i < size; i++) {
2065 if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC)
2066 continue;
2067 if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) {
2068 zeros++;
2069 continue;
2070 }
2071 verbose(env, "invalid read from stack off %d+%d size %d\n",
2072 off, i, size);
2073 return -EACCES;
2074 }
2075 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2076 if (value_regno >= 0) {
2077 if (zeros == size) {
2078
2079
2080
2081 __mark_reg_const_zero(&state->regs[value_regno]);
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092 state->regs[value_regno].precise = true;
2093 } else {
2094
2095 mark_reg_unknown(env, state->regs, value_regno);
2096 }
2097 state->regs[value_regno].live |= REG_LIVE_WRITTEN;
2098 }
2099 }
2100 return 0;
2101}
2102
2103static int check_stack_access(struct bpf_verifier_env *env,
2104 const struct bpf_reg_state *reg,
2105 int off, int size)
2106{
2107
2108
2109
2110
2111 if (!tnum_is_const(reg->var_off)) {
2112 char tn_buf[48];
2113
2114 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2115 verbose(env, "variable stack access var_off=%s off=%d size=%d\n",
2116 tn_buf, off, size);
2117 return -EACCES;
2118 }
2119
2120 if (off >= 0 || off < -MAX_BPF_STACK) {
2121 verbose(env, "invalid stack off=%d size=%d\n", off, size);
2122 return -EACCES;
2123 }
2124
2125 return 0;
2126}
2127
2128static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
2129 int off, int size, enum bpf_access_type type)
2130{
2131 struct bpf_reg_state *regs = cur_regs(env);
2132 struct bpf_map *map = regs[regno].map_ptr;
2133 u32 cap = bpf_map_flags_to_cap(map);
2134
2135 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) {
2136 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n",
2137 map->value_size, off, size);
2138 return -EACCES;
2139 }
2140
2141 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) {
2142 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n",
2143 map->value_size, off, size);
2144 return -EACCES;
2145 }
2146
2147 return 0;
2148}
2149
2150
2151static int __check_map_access(struct bpf_verifier_env *env, u32 regno, int off,
2152 int size, bool zero_size_allowed)
2153{
2154 struct bpf_reg_state *regs = cur_regs(env);
2155 struct bpf_map *map = regs[regno].map_ptr;
2156
2157 if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) ||
2158 off + size > map->value_size) {
2159 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",
2160 map->value_size, off, size);
2161 return -EACCES;
2162 }
2163 return 0;
2164}
2165
2166
2167static int check_map_access(struct bpf_verifier_env *env, u32 regno,
2168 int off, int size, bool zero_size_allowed)
2169{
2170 struct bpf_verifier_state *vstate = env->cur_state;
2171 struct bpf_func_state *state = vstate->frame[vstate->curframe];
2172 struct bpf_reg_state *reg = &state->regs[regno];
2173 int err;
2174
2175
2176
2177
2178
2179 if (env->log.level & BPF_LOG_LEVEL)
2180 print_verifier_state(env, state);
2181
2182
2183
2184
2185
2186
2187
2188 if (reg->smin_value < 0 &&
2189 (reg->smin_value == S64_MIN ||
2190 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) ||
2191 reg->smin_value + off < 0)) {
2192 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
2193 regno);
2194 return -EACCES;
2195 }
2196 err = __check_map_access(env, regno, reg->smin_value + off, size,
2197 zero_size_allowed);
2198 if (err) {
2199 verbose(env, "R%d min value is outside of the array range\n",
2200 regno);
2201 return err;
2202 }
2203
2204
2205
2206
2207
2208 if (reg->umax_value >= BPF_MAX_VAR_OFF) {
2209 verbose(env, "R%d unbounded memory access, make sure to bounds check any array access into a map\n",
2210 regno);
2211 return -EACCES;
2212 }
2213 err = __check_map_access(env, regno, reg->umax_value + off, size,
2214 zero_size_allowed);
2215 if (err)
2216 verbose(env, "R%d max value is outside of the array range\n",
2217 regno);
2218
2219 if (map_value_has_spin_lock(reg->map_ptr)) {
2220 u32 lock = reg->map_ptr->spin_lock_off;
2221
2222
2223
2224
2225
2226
2227 if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) &&
2228 lock < reg->umax_value + off + size) {
2229 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n");
2230 return -EACCES;
2231 }
2232 }
2233 return err;
2234}
2235
2236#define MAX_PACKET_OFF 0xffff
2237
2238static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
2239 const struct bpf_call_arg_meta *meta,
2240 enum bpf_access_type t)
2241{
2242 switch (env->prog->type) {
2243
2244 case BPF_PROG_TYPE_LWT_IN:
2245 case BPF_PROG_TYPE_LWT_OUT:
2246 case BPF_PROG_TYPE_LWT_SEG6LOCAL:
2247 case BPF_PROG_TYPE_SK_REUSEPORT:
2248 case BPF_PROG_TYPE_FLOW_DISSECTOR:
2249 case BPF_PROG_TYPE_CGROUP_SKB:
2250 if (t == BPF_WRITE)
2251 return false;
2252
2253
2254
2255 case BPF_PROG_TYPE_SCHED_CLS:
2256 case BPF_PROG_TYPE_SCHED_ACT:
2257 case BPF_PROG_TYPE_XDP:
2258 case BPF_PROG_TYPE_LWT_XMIT:
2259 case BPF_PROG_TYPE_SK_SKB:
2260 case BPF_PROG_TYPE_SK_MSG:
2261 if (meta)
2262 return meta->pkt_access;
2263
2264 env->seen_direct_write = true;
2265 return true;
2266
2267 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2268 if (t == BPF_WRITE)
2269 env->seen_direct_write = true;
2270
2271 return true;
2272
2273 default:
2274 return false;
2275 }
2276}
2277
2278static int __check_packet_access(struct bpf_verifier_env *env, u32 regno,
2279 int off, int size, bool zero_size_allowed)
2280{
2281 struct bpf_reg_state *regs = cur_regs(env);
2282 struct bpf_reg_state *reg = ®s[regno];
2283
2284 if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) ||
2285 (u64)off + size > reg->range) {
2286 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",
2287 off, size, regno, reg->id, reg->off, reg->range);
2288 return -EACCES;
2289 }
2290 return 0;
2291}
2292
2293static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
2294 int size, bool zero_size_allowed)
2295{
2296 struct bpf_reg_state *regs = cur_regs(env);
2297 struct bpf_reg_state *reg = ®s[regno];
2298 int err;
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308 if (reg->smin_value < 0) {
2309 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
2310 regno);
2311 return -EACCES;
2312 }
2313 err = __check_packet_access(env, regno, off, size, zero_size_allowed);
2314 if (err) {
2315 verbose(env, "R%d offset is outside of the packet\n", regno);
2316 return err;
2317 }
2318
2319
2320
2321
2322
2323
2324
2325 env->prog->aux->max_pkt_offset =
2326 max_t(u32, env->prog->aux->max_pkt_offset,
2327 off + reg->umax_value + size - 1);
2328
2329 return err;
2330}
2331
2332
2333static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
2334 enum bpf_access_type t, enum bpf_reg_type *reg_type)
2335{
2336 struct bpf_insn_access_aux info = {
2337 .reg_type = *reg_type,
2338 };
2339
2340 if (env->ops->is_valid_access &&
2341 env->ops->is_valid_access(off, size, t, env->prog, &info)) {
2342
2343
2344
2345
2346
2347
2348
2349 *reg_type = info.reg_type;
2350
2351 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
2352
2353 if (env->prog->aux->max_ctx_offset < off + size)
2354 env->prog->aux->max_ctx_offset = off + size;
2355 return 0;
2356 }
2357
2358 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
2359 return -EACCES;
2360}
2361
2362static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
2363 int size)
2364{
2365 if (size < 0 || off < 0 ||
2366 (u64)off + size > sizeof(struct bpf_flow_keys)) {
2367 verbose(env, "invalid access to flow keys off=%d size=%d\n",
2368 off, size);
2369 return -EACCES;
2370 }
2371 return 0;
2372}
2373
2374static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
2375 u32 regno, int off, int size,
2376 enum bpf_access_type t)
2377{
2378 struct bpf_reg_state *regs = cur_regs(env);
2379 struct bpf_reg_state *reg = ®s[regno];
2380 struct bpf_insn_access_aux info = {};
2381 bool valid;
2382
2383 if (reg->smin_value < 0) {
2384 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
2385 regno);
2386 return -EACCES;
2387 }
2388
2389 switch (reg->type) {
2390 case PTR_TO_SOCK_COMMON:
2391 valid = bpf_sock_common_is_valid_access(off, size, t, &info);
2392 break;
2393 case PTR_TO_SOCKET:
2394 valid = bpf_sock_is_valid_access(off, size, t, &info);
2395 break;
2396 case PTR_TO_TCP_SOCK:
2397 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
2398 break;
2399 case PTR_TO_XDP_SOCK:
2400 valid = bpf_xdp_sock_is_valid_access(off, size, t, &info);
2401 break;
2402 default:
2403 valid = false;
2404 }
2405
2406
2407 if (valid) {
2408 env->insn_aux_data[insn_idx].ctx_field_size =
2409 info.ctx_field_size;
2410 return 0;
2411 }
2412
2413 verbose(env, "R%d invalid %s access off=%d size=%d\n",
2414 regno, reg_type_str[reg->type], off, size);
2415
2416 return -EACCES;
2417}
2418
2419static bool __is_pointer_value(bool allow_ptr_leaks,
2420 const struct bpf_reg_state *reg)
2421{
2422 if (allow_ptr_leaks)
2423 return false;
2424
2425 return reg->type != SCALAR_VALUE;
2426}
2427
2428static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
2429{
2430 return cur_regs(env) + regno;
2431}
2432
2433static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
2434{
2435 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
2436}
2437
2438static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
2439{
2440 const struct bpf_reg_state *reg = reg_state(env, regno);
2441
2442 return reg->type == PTR_TO_CTX;
2443}
2444
2445static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
2446{
2447 const struct bpf_reg_state *reg = reg_state(env, regno);
2448
2449 return type_is_sk_pointer(reg->type);
2450}
2451
2452static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
2453{
2454 const struct bpf_reg_state *reg = reg_state(env, regno);
2455
2456 return type_is_pkt_pointer(reg->type);
2457}
2458
2459static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
2460{
2461 const struct bpf_reg_state *reg = reg_state(env, regno);
2462
2463
2464 return reg->type == PTR_TO_FLOW_KEYS;
2465}
2466
2467static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
2468 const struct bpf_reg_state *reg,
2469 int off, int size, bool strict)
2470{
2471 struct tnum reg_off;
2472 int ip_align;
2473
2474
2475 if (!strict || size == 1)
2476 return 0;
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486 ip_align = 2;
2487
2488 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off));
2489 if (!tnum_is_aligned(reg_off, size)) {
2490 char tn_buf[48];
2491
2492 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2493 verbose(env,
2494 "misaligned packet access off %d+%s+%d+%d size %d\n",
2495 ip_align, tn_buf, reg->off, off, size);
2496 return -EACCES;
2497 }
2498
2499 return 0;
2500}
2501
2502static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
2503 const struct bpf_reg_state *reg,
2504 const char *pointer_desc,
2505 int off, int size, bool strict)
2506{
2507 struct tnum reg_off;
2508
2509
2510 if (!strict || size == 1)
2511 return 0;
2512
2513 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off));
2514 if (!tnum_is_aligned(reg_off, size)) {
2515 char tn_buf[48];
2516
2517 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2518 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",
2519 pointer_desc, tn_buf, reg->off, off, size);
2520 return -EACCES;
2521 }
2522
2523 return 0;
2524}
2525
2526static int check_ptr_alignment(struct bpf_verifier_env *env,
2527 const struct bpf_reg_state *reg, int off,
2528 int size, bool strict_alignment_once)
2529{
2530 bool strict = env->strict_alignment || strict_alignment_once;
2531 const char *pointer_desc = "";
2532
2533 switch (reg->type) {
2534 case PTR_TO_PACKET:
2535 case PTR_TO_PACKET_META:
2536
2537
2538
2539 return check_pkt_ptr_alignment(env, reg, off, size, strict);
2540 case PTR_TO_FLOW_KEYS:
2541 pointer_desc = "flow keys ";
2542 break;
2543 case PTR_TO_MAP_VALUE:
2544 pointer_desc = "value ";
2545 break;
2546 case PTR_TO_CTX:
2547 pointer_desc = "context ";
2548 break;
2549 case PTR_TO_STACK:
2550 pointer_desc = "stack ";
2551
2552
2553
2554
2555 strict = true;
2556 break;
2557 case PTR_TO_SOCKET:
2558 pointer_desc = "sock ";
2559 break;
2560 case PTR_TO_SOCK_COMMON:
2561 pointer_desc = "sock_common ";
2562 break;
2563 case PTR_TO_TCP_SOCK:
2564 pointer_desc = "tcp_sock ";
2565 break;
2566 case PTR_TO_XDP_SOCK:
2567 pointer_desc = "xdp_sock ";
2568 break;
2569 default:
2570 break;
2571 }
2572 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,
2573 strict);
2574}
2575
2576static int update_stack_depth(struct bpf_verifier_env *env,
2577 const struct bpf_func_state *func,
2578 int off)
2579{
2580 u16 stack = env->subprog_info[func->subprogno].stack_depth;
2581
2582 if (stack >= -off)
2583 return 0;
2584
2585
2586 env->subprog_info[func->subprogno].stack_depth = -off;
2587 return 0;
2588}
2589
2590
2591
2592
2593
2594
2595
2596static int check_max_stack_depth(struct bpf_verifier_env *env)
2597{
2598 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end;
2599 struct bpf_subprog_info *subprog = env->subprog_info;
2600 struct bpf_insn *insn = env->prog->insnsi;
2601 int ret_insn[MAX_CALL_FRAMES];
2602 int ret_prog[MAX_CALL_FRAMES];
2603
2604process_func:
2605
2606
2607
2608 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
2609 if (depth > MAX_BPF_STACK) {
2610 verbose(env, "combined stack size of %d calls is %d. Too large\n",
2611 frame + 1, depth);
2612 return -EACCES;
2613 }
2614continue_func:
2615 subprog_end = subprog[idx + 1].start;
2616 for (; i < subprog_end; i++) {
2617 if (insn[i].code != (BPF_JMP | BPF_CALL))
2618 continue;
2619 if (insn[i].src_reg != BPF_PSEUDO_CALL)
2620 continue;
2621
2622 ret_insn[frame] = i + 1;
2623 ret_prog[frame] = idx;
2624
2625
2626 i = i + insn[i].imm + 1;
2627 idx = find_subprog(env, i);
2628 if (idx < 0) {
2629 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
2630 i);
2631 return -EFAULT;
2632 }
2633 frame++;
2634 if (frame >= MAX_CALL_FRAMES) {
2635 verbose(env, "the call stack of %d frames is too deep !\n",
2636 frame);
2637 return -E2BIG;
2638 }
2639 goto process_func;
2640 }
2641
2642
2643
2644 if (frame == 0)
2645 return 0;
2646 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
2647 frame--;
2648 i = ret_insn[frame];
2649 idx = ret_prog[frame];
2650 goto continue_func;
2651}
2652
2653#ifndef CONFIG_BPF_JIT_ALWAYS_ON
2654static int get_callee_stack_depth(struct bpf_verifier_env *env,
2655 const struct bpf_insn *insn, int idx)
2656{
2657 int start = idx + insn->imm + 1, subprog;
2658
2659 subprog = find_subprog(env, start);
2660 if (subprog < 0) {
2661 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
2662 start);
2663 return -EFAULT;
2664 }
2665 return env->subprog_info[subprog].stack_depth;
2666}
2667#endif
2668
2669static int check_ctx_reg(struct bpf_verifier_env *env,
2670 const struct bpf_reg_state *reg, int regno)
2671{
2672
2673
2674
2675
2676 if (reg->off) {
2677 verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n",
2678 regno, reg->off);
2679 return -EACCES;
2680 }
2681
2682 if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
2683 char tn_buf[48];
2684
2685 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2686 verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf);
2687 return -EACCES;
2688 }
2689
2690 return 0;
2691}
2692
2693static int check_tp_buffer_access(struct bpf_verifier_env *env,
2694 const struct bpf_reg_state *reg,
2695 int regno, int off, int size)
2696{
2697 if (off < 0) {
2698 verbose(env,
2699 "R%d invalid tracepoint buffer access: off=%d, size=%d",
2700 regno, off, size);
2701 return -EACCES;
2702 }
2703 if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
2704 char tn_buf[48];
2705
2706 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2707 verbose(env,
2708 "R%d invalid variable buffer offset: off=%d, var_off=%s",
2709 regno, off, tn_buf);
2710 return -EACCES;
2711 }
2712 if (off + size > env->prog->aux->max_tp_access)
2713 env->prog->aux->max_tp_access = off + size;
2714
2715 return 0;
2716}
2717
2718
2719
2720
2721
2722static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
2723{
2724 u64 mask;
2725
2726
2727 reg->var_off = tnum_cast(reg->var_off, size);
2728
2729
2730 mask = ((u64)1 << (size * 8)) - 1;
2731 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) {
2732 reg->umin_value &= mask;
2733 reg->umax_value &= mask;
2734 } else {
2735 reg->umin_value = 0;
2736 reg->umax_value = mask;
2737 }
2738 reg->smin_value = reg->umin_value;
2739 reg->smax_value = reg->umax_value;
2740}
2741
2742
2743
2744
2745
2746
2747
2748static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
2749 int off, int bpf_size, enum bpf_access_type t,
2750 int value_regno, bool strict_alignment_once)
2751{
2752 struct bpf_reg_state *regs = cur_regs(env);
2753 struct bpf_reg_state *reg = regs + regno;
2754 struct bpf_func_state *state;
2755 int size, err = 0;
2756
2757 size = bpf_size_to_bytes(bpf_size);
2758 if (size < 0)
2759 return size;
2760
2761
2762 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
2763 if (err)
2764 return err;
2765
2766
2767 off += reg->off;
2768
2769 if (reg->type == PTR_TO_MAP_VALUE) {
2770 if (t == BPF_WRITE && value_regno >= 0 &&
2771 is_pointer_value(env, value_regno)) {
2772 verbose(env, "R%d leaks addr into map\n", value_regno);
2773 return -EACCES;
2774 }
2775 err = check_map_access_type(env, regno, off, size, t);
2776 if (err)
2777 return err;
2778 err = check_map_access(env, regno, off, size, false);
2779 if (!err && t == BPF_READ && value_regno >= 0)
2780 mark_reg_unknown(env, regs, value_regno);
2781
2782 } else if (reg->type == PTR_TO_CTX) {
2783 enum bpf_reg_type reg_type = SCALAR_VALUE;
2784
2785 if (t == BPF_WRITE && value_regno >= 0 &&
2786 is_pointer_value(env, value_regno)) {
2787 verbose(env, "R%d leaks addr into ctx\n", value_regno);
2788 return -EACCES;
2789 }
2790
2791 err = check_ctx_reg(env, reg, regno);
2792 if (err < 0)
2793 return err;
2794
2795 err = check_ctx_access(env, insn_idx, off, size, t, ®_type);
2796 if (!err && t == BPF_READ && value_regno >= 0) {
2797
2798
2799
2800
2801 if (reg_type == SCALAR_VALUE) {
2802 mark_reg_unknown(env, regs, value_regno);
2803 } else {
2804 mark_reg_known_zero(env, regs,
2805 value_regno);
2806 if (reg_type_may_be_null(reg_type))
2807 regs[value_regno].id = ++env->id_gen;
2808
2809
2810
2811
2812
2813 regs[value_regno].subreg_def = DEF_NOT_SUBREG;
2814 }
2815 regs[value_regno].type = reg_type;
2816 }
2817
2818 } else if (reg->type == PTR_TO_STACK) {
2819 off += reg->var_off.value;
2820 err = check_stack_access(env, reg, off, size);
2821 if (err)
2822 return err;
2823
2824 state = func(env, reg);
2825 err = update_stack_depth(env, state, off);
2826 if (err)
2827 return err;
2828
2829 if (t == BPF_WRITE)
2830 err = check_stack_write(env, state, off, size,
2831 value_regno, insn_idx);
2832 else
2833 err = check_stack_read(env, state, off, size,
2834 value_regno);
2835 } else if (reg_is_pkt_pointer(reg)) {
2836 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
2837 verbose(env, "cannot write into packet\n");
2838 return -EACCES;
2839 }
2840 if (t == BPF_WRITE && value_regno >= 0 &&
2841 is_pointer_value(env, value_regno)) {
2842 verbose(env, "R%d leaks addr into packet\n",
2843 value_regno);
2844 return -EACCES;
2845 }
2846 err = check_packet_access(env, regno, off, size, false);
2847 if (!err && t == BPF_READ && value_regno >= 0)
2848 mark_reg_unknown(env, regs, value_regno);
2849 } else if (reg->type == PTR_TO_FLOW_KEYS) {
2850 if (t == BPF_WRITE && value_regno >= 0 &&
2851 is_pointer_value(env, value_regno)) {
2852 verbose(env, "R%d leaks addr into flow keys\n",
2853 value_regno);
2854 return -EACCES;
2855 }
2856
2857 err = check_flow_keys_access(env, off, size);
2858 if (!err && t == BPF_READ && value_regno >= 0)
2859 mark_reg_unknown(env, regs, value_regno);
2860 } else if (type_is_sk_pointer(reg->type)) {
2861 if (t == BPF_WRITE) {
2862 verbose(env, "R%d cannot write into %s\n",
2863 regno, reg_type_str[reg->type]);
2864 return -EACCES;
2865 }
2866 err = check_sock_access(env, insn_idx, regno, off, size, t);
2867 if (!err && value_regno >= 0)
2868 mark_reg_unknown(env, regs, value_regno);
2869 } else if (reg->type == PTR_TO_TP_BUFFER) {
2870 err = check_tp_buffer_access(env, reg, regno, off, size);
2871 if (!err && t == BPF_READ && value_regno >= 0)
2872 mark_reg_unknown(env, regs, value_regno);
2873 } else {
2874 verbose(env, "R%d invalid mem access '%s'\n", regno,
2875 reg_type_str[reg->type]);
2876 return -EACCES;
2877 }
2878
2879 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
2880 regs[value_regno].type == SCALAR_VALUE) {
2881
2882 coerce_reg_to_size(®s[value_regno], size);
2883 }
2884 return err;
2885}
2886
2887static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)
2888{
2889 int err;
2890
2891 if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) ||
2892 insn->imm != 0) {
2893 verbose(env, "BPF_XADD uses reserved fields\n");
2894 return -EINVAL;
2895 }
2896
2897
2898 err = check_reg_arg(env, insn->src_reg, SRC_OP);
2899 if (err)
2900 return err;
2901
2902
2903 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
2904 if (err)
2905 return err;
2906
2907 if (is_pointer_value(env, insn->src_reg)) {
2908 verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
2909 return -EACCES;
2910 }
2911
2912 if (is_ctx_reg(env, insn->dst_reg) ||
2913 is_pkt_reg(env, insn->dst_reg) ||
2914 is_flow_key_reg(env, insn->dst_reg) ||
2915 is_sk_reg(env, insn->dst_reg)) {
2916 verbose(env, "BPF_XADD stores into R%d %s is not allowed\n",
2917 insn->dst_reg,
2918 reg_type_str[reg_state(env, insn->dst_reg)->type]);
2919 return -EACCES;
2920 }
2921
2922
2923 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
2924 BPF_SIZE(insn->code), BPF_READ, -1, true);
2925 if (err)
2926 return err;
2927
2928
2929 return check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
2930 BPF_SIZE(insn->code), BPF_WRITE, -1, true);
2931}
2932
2933static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno,
2934 int off, int access_size,
2935 bool zero_size_allowed)
2936{
2937 struct bpf_reg_state *reg = reg_state(env, regno);
2938
2939 if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 ||
2940 access_size < 0 || (access_size == 0 && !zero_size_allowed)) {
2941 if (tnum_is_const(reg->var_off)) {
2942 verbose(env, "invalid stack type R%d off=%d access_size=%d\n",
2943 regno, off, access_size);
2944 } else {
2945 char tn_buf[48];
2946
2947 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2948 verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n",
2949 regno, tn_buf, access_size);
2950 }
2951 return -EACCES;
2952 }
2953 return 0;
2954}
2955
2956
2957
2958
2959
2960
2961
2962static int check_stack_boundary(struct bpf_verifier_env *env, int regno,
2963 int access_size, bool zero_size_allowed,
2964 struct bpf_call_arg_meta *meta)
2965{
2966 struct bpf_reg_state *reg = reg_state(env, regno);
2967 struct bpf_func_state *state = func(env, reg);
2968 int err, min_off, max_off, i, j, slot, spi;
2969
2970 if (reg->type != PTR_TO_STACK) {
2971
2972 if (zero_size_allowed && access_size == 0 &&
2973 register_is_null(reg))
2974 return 0;
2975
2976 verbose(env, "R%d type=%s expected=%s\n", regno,
2977 reg_type_str[reg->type],
2978 reg_type_str[PTR_TO_STACK]);
2979 return -EACCES;
2980 }
2981
2982 if (tnum_is_const(reg->var_off)) {
2983 min_off = max_off = reg->var_off.value + reg->off;
2984 err = __check_stack_boundary(env, regno, min_off, access_size,
2985 zero_size_allowed);
2986 if (err)
2987 return err;
2988 } else {
2989
2990
2991
2992
2993
2994 if (!env->allow_ptr_leaks) {
2995 char tn_buf[48];
2996
2997 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2998 verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n",
2999 regno, tn_buf);
3000 return -EACCES;
3001 }
3002
3003
3004
3005
3006
3007
3008 if (meta && meta->raw_mode)
3009 meta = NULL;
3010
3011 if (reg->smax_value >= BPF_MAX_VAR_OFF ||
3012 reg->smax_value <= -BPF_MAX_VAR_OFF) {
3013 verbose(env, "R%d unbounded indirect variable offset stack access\n",
3014 regno);
3015 return -EACCES;
3016 }
3017 min_off = reg->smin_value + reg->off;
3018 max_off = reg->smax_value + reg->off;
3019 err = __check_stack_boundary(env, regno, min_off, access_size,
3020 zero_size_allowed);
3021 if (err) {
3022 verbose(env, "R%d min value is outside of stack bound\n",
3023 regno);
3024 return err;
3025 }
3026 err = __check_stack_boundary(env, regno, max_off, access_size,
3027 zero_size_allowed);
3028 if (err) {
3029 verbose(env, "R%d max value is outside of stack bound\n",
3030 regno);
3031 return err;
3032 }
3033 }
3034
3035 if (meta && meta->raw_mode) {
3036 meta->access_size = access_size;
3037 meta->regno = regno;
3038 return 0;
3039 }
3040
3041 for (i = min_off; i < max_off + access_size; i++) {
3042 u8 *stype;
3043
3044 slot = -i - 1;
3045 spi = slot / BPF_REG_SIZE;
3046 if (state->allocated_stack <= slot)
3047 goto err;
3048 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
3049 if (*stype == STACK_MISC)
3050 goto mark;
3051 if (*stype == STACK_ZERO) {
3052
3053 *stype = STACK_MISC;
3054 goto mark;
3055 }
3056 if (state->stack[spi].slot_type[0] == STACK_SPILL &&
3057 state->stack[spi].spilled_ptr.type == SCALAR_VALUE) {
3058 __mark_reg_unknown(&state->stack[spi].spilled_ptr);
3059 for (j = 0; j < BPF_REG_SIZE; j++)
3060 state->stack[spi].slot_type[j] = STACK_MISC;
3061 goto mark;
3062 }
3063
3064err:
3065 if (tnum_is_const(reg->var_off)) {
3066 verbose(env, "invalid indirect read from stack off %d+%d size %d\n",
3067 min_off, i - min_off, access_size);
3068 } else {
3069 char tn_buf[48];
3070
3071 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
3072 verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n",
3073 tn_buf, i - min_off, access_size);
3074 }
3075 return -EACCES;
3076mark:
3077
3078
3079
3080 mark_reg_read(env, &state->stack[spi].spilled_ptr,
3081 state->stack[spi].spilled_ptr.parent,
3082 REG_LIVE_READ64);
3083 }
3084 return update_stack_depth(env, state, min_off);
3085}
3086
3087static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
3088 int access_size, bool zero_size_allowed,
3089 struct bpf_call_arg_meta *meta)
3090{
3091 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
3092
3093 switch (reg->type) {
3094 case PTR_TO_PACKET:
3095 case PTR_TO_PACKET_META:
3096 return check_packet_access(env, regno, reg->off, access_size,
3097 zero_size_allowed);
3098 case PTR_TO_MAP_VALUE:
3099 if (check_map_access_type(env, regno, reg->off, access_size,
3100 meta && meta->raw_mode ? BPF_WRITE :
3101 BPF_READ))
3102 return -EACCES;
3103 return check_map_access(env, regno, reg->off, access_size,
3104 zero_size_allowed);
3105 default:
3106 return check_stack_boundary(env, regno, access_size,
3107 zero_size_allowed, meta);
3108 }
3109}
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130static int process_spin_lock(struct bpf_verifier_env *env, int regno,
3131 bool is_lock)
3132{
3133 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
3134 struct bpf_verifier_state *cur = env->cur_state;
3135 bool is_const = tnum_is_const(reg->var_off);
3136 struct bpf_map *map = reg->map_ptr;
3137 u64 val = reg->var_off.value;
3138
3139 if (reg->type != PTR_TO_MAP_VALUE) {
3140 verbose(env, "R%d is not a pointer to map_value\n", regno);
3141 return -EINVAL;
3142 }
3143 if (!is_const) {
3144 verbose(env,
3145 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n",
3146 regno);
3147 return -EINVAL;
3148 }
3149 if (!map->btf) {
3150 verbose(env,
3151 "map '%s' has to have BTF in order to use bpf_spin_lock\n",
3152 map->name);
3153 return -EINVAL;
3154 }
3155 if (!map_value_has_spin_lock(map)) {
3156 if (map->spin_lock_off == -E2BIG)
3157 verbose(env,
3158 "map '%s' has more than one 'struct bpf_spin_lock'\n",
3159 map->name);
3160 else if (map->spin_lock_off == -ENOENT)
3161 verbose(env,
3162 "map '%s' doesn't have 'struct bpf_spin_lock'\n",
3163 map->name);
3164 else
3165 verbose(env,
3166 "map '%s' is not a struct type or bpf_spin_lock is mangled\n",
3167 map->name);
3168 return -EINVAL;
3169 }
3170 if (map->spin_lock_off != val + reg->off) {
3171 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n",
3172 val + reg->off);
3173 return -EINVAL;
3174 }
3175 if (is_lock) {
3176 if (cur->active_spin_lock) {
3177 verbose(env,
3178 "Locking two bpf_spin_locks are not allowed\n");
3179 return -EINVAL;
3180 }
3181 cur->active_spin_lock = reg->id;
3182 } else {
3183 if (!cur->active_spin_lock) {
3184 verbose(env, "bpf_spin_unlock without taking a lock\n");
3185 return -EINVAL;
3186 }
3187 if (cur->active_spin_lock != reg->id) {
3188 verbose(env, "bpf_spin_unlock of different lock\n");
3189 return -EINVAL;
3190 }
3191 cur->active_spin_lock = 0;
3192 }
3193 return 0;
3194}
3195
3196static bool arg_type_is_mem_ptr(enum bpf_arg_type type)
3197{
3198 return type == ARG_PTR_TO_MEM ||
3199 type == ARG_PTR_TO_MEM_OR_NULL ||
3200 type == ARG_PTR_TO_UNINIT_MEM;
3201}
3202
3203static bool arg_type_is_mem_size(enum bpf_arg_type type)
3204{
3205 return type == ARG_CONST_SIZE ||
3206 type == ARG_CONST_SIZE_OR_ZERO;
3207}
3208
3209static bool arg_type_is_int_ptr(enum bpf_arg_type type)
3210{
3211 return type == ARG_PTR_TO_INT ||
3212 type == ARG_PTR_TO_LONG;
3213}
3214
3215static int int_ptr_type_to_size(enum bpf_arg_type type)
3216{
3217 if (type == ARG_PTR_TO_INT)
3218 return sizeof(u32);
3219 else if (type == ARG_PTR_TO_LONG)
3220 return sizeof(u64);
3221
3222 return -EINVAL;
3223}
3224
3225static int check_func_arg(struct bpf_verifier_env *env, u32 regno,
3226 enum bpf_arg_type arg_type,
3227 struct bpf_call_arg_meta *meta)
3228{
3229 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
3230 enum bpf_reg_type expected_type, type = reg->type;
3231 int err = 0;
3232
3233 if (arg_type == ARG_DONTCARE)
3234 return 0;
3235
3236 err = check_reg_arg(env, regno, SRC_OP);
3237 if (err)
3238 return err;
3239
3240 if (arg_type == ARG_ANYTHING) {
3241 if (is_pointer_value(env, regno)) {
3242 verbose(env, "R%d leaks addr into helper function\n",
3243 regno);
3244 return -EACCES;
3245 }
3246 return 0;
3247 }
3248
3249 if (type_is_pkt_pointer(type) &&
3250 !may_access_direct_pkt_data(env, meta, BPF_READ)) {
3251 verbose(env, "helper access to the packet is not allowed\n");
3252 return -EACCES;
3253 }
3254
3255 if (arg_type == ARG_PTR_TO_MAP_KEY ||
3256 arg_type == ARG_PTR_TO_MAP_VALUE ||
3257 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE ||
3258 arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) {
3259 expected_type = PTR_TO_STACK;
3260 if (register_is_null(reg) &&
3261 arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL)
3262 ;
3263 else if (!type_is_pkt_pointer(type) &&
3264 type != PTR_TO_MAP_VALUE &&
3265 type != expected_type)
3266 goto err_type;
3267 } else if (arg_type == ARG_CONST_SIZE ||
3268 arg_type == ARG_CONST_SIZE_OR_ZERO) {
3269 expected_type = SCALAR_VALUE;
3270 if (type != expected_type)
3271 goto err_type;
3272 } else if (arg_type == ARG_CONST_MAP_PTR) {
3273 expected_type = CONST_PTR_TO_MAP;
3274 if (type != expected_type)
3275 goto err_type;
3276 } else if (arg_type == ARG_PTR_TO_CTX) {
3277 expected_type = PTR_TO_CTX;
3278 if (type != expected_type)
3279 goto err_type;
3280 err = check_ctx_reg(env, reg, regno);
3281 if (err < 0)
3282 return err;
3283 } else if (arg_type == ARG_PTR_TO_SOCK_COMMON) {
3284 expected_type = PTR_TO_SOCK_COMMON;
3285
3286 if (!type_is_sk_pointer(type))
3287 goto err_type;
3288 if (reg->ref_obj_id) {
3289 if (meta->ref_obj_id) {
3290 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
3291 regno, reg->ref_obj_id,
3292 meta->ref_obj_id);
3293 return -EFAULT;
3294 }
3295 meta->ref_obj_id = reg->ref_obj_id;
3296 }
3297 } else if (arg_type == ARG_PTR_TO_SOCKET) {
3298 expected_type = PTR_TO_SOCKET;
3299 if (type != expected_type)
3300 goto err_type;
3301 } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
3302 if (meta->func_id == BPF_FUNC_spin_lock) {
3303 if (process_spin_lock(env, regno, true))
3304 return -EACCES;
3305 } else if (meta->func_id == BPF_FUNC_spin_unlock) {
3306 if (process_spin_lock(env, regno, false))
3307 return -EACCES;
3308 } else {
3309 verbose(env, "verifier internal error\n");
3310 return -EFAULT;
3311 }
3312 } else if (arg_type_is_mem_ptr(arg_type)) {
3313 expected_type = PTR_TO_STACK;
3314
3315
3316
3317
3318 if (register_is_null(reg) &&
3319 arg_type == ARG_PTR_TO_MEM_OR_NULL)
3320 ;
3321 else if (!type_is_pkt_pointer(type) &&
3322 type != PTR_TO_MAP_VALUE &&
3323 type != expected_type)
3324 goto err_type;
3325 meta->raw_mode = arg_type == ARG_PTR_TO_UNINIT_MEM;
3326 } else if (arg_type_is_int_ptr(arg_type)) {
3327 expected_type = PTR_TO_STACK;
3328 if (!type_is_pkt_pointer(type) &&
3329 type != PTR_TO_MAP_VALUE &&
3330 type != expected_type)
3331 goto err_type;
3332 } else {
3333 verbose(env, "unsupported arg_type %d\n", arg_type);
3334 return -EFAULT;
3335 }
3336
3337 if (arg_type == ARG_CONST_MAP_PTR) {
3338
3339 meta->map_ptr = reg->map_ptr;
3340 } else if (arg_type == ARG_PTR_TO_MAP_KEY) {
3341
3342
3343
3344
3345 if (!meta->map_ptr) {
3346
3347
3348
3349
3350
3351 verbose(env, "invalid map_ptr to access map->key\n");
3352 return -EACCES;
3353 }
3354 err = check_helper_mem_access(env, regno,
3355 meta->map_ptr->key_size, false,
3356 NULL);
3357 } else if (arg_type == ARG_PTR_TO_MAP_VALUE ||
3358 (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL &&
3359 !register_is_null(reg)) ||
3360 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
3361
3362
3363
3364 if (!meta->map_ptr) {
3365
3366 verbose(env, "invalid map_ptr to access map->value\n");
3367 return -EACCES;
3368 }
3369 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE);
3370 err = check_helper_mem_access(env, regno,
3371 meta->map_ptr->value_size, false,
3372 meta);
3373 } else if (arg_type_is_mem_size(arg_type)) {
3374 bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO);
3375
3376
3377
3378
3379 meta->msize_smax_value = reg->smax_value;
3380 meta->msize_umax_value = reg->umax_value;
3381
3382
3383
3384
3385 if (!tnum_is_const(reg->var_off))
3386
3387
3388
3389
3390
3391 meta = NULL;
3392
3393 if (reg->smin_value < 0) {
3394 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",
3395 regno);
3396 return -EACCES;
3397 }
3398
3399 if (reg->umin_value == 0) {
3400 err = check_helper_mem_access(env, regno - 1, 0,
3401 zero_size_allowed,
3402 meta);
3403 if (err)
3404 return err;
3405 }
3406
3407 if (reg->umax_value >= BPF_MAX_VAR_SIZ) {
3408 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
3409 regno);
3410 return -EACCES;
3411 }
3412 err = check_helper_mem_access(env, regno - 1,
3413 reg->umax_value,
3414 zero_size_allowed, meta);
3415 if (!err)
3416 err = mark_chain_precision(env, regno);
3417 } else if (arg_type_is_int_ptr(arg_type)) {
3418 int size = int_ptr_type_to_size(arg_type);
3419
3420 err = check_helper_mem_access(env, regno, size, false, meta);
3421 if (err)
3422 return err;
3423 err = check_ptr_alignment(env, reg, 0, size, true);
3424 }
3425
3426 return err;
3427err_type:
3428 verbose(env, "R%d type=%s expected=%s\n", regno,
3429 reg_type_str[type], reg_type_str[expected_type]);
3430 return -EACCES;
3431}
3432
3433static int check_map_func_compatibility(struct bpf_verifier_env *env,
3434 struct bpf_map *map, int func_id)
3435{
3436 if (!map)
3437 return 0;
3438
3439
3440 switch (map->map_type) {
3441 case BPF_MAP_TYPE_PROG_ARRAY:
3442 if (func_id != BPF_FUNC_tail_call)
3443 goto error;
3444 break;
3445 case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
3446 if (func_id != BPF_FUNC_perf_event_read &&
3447 func_id != BPF_FUNC_perf_event_output &&
3448 func_id != BPF_FUNC_perf_event_read_value)
3449 goto error;
3450 break;
3451 case BPF_MAP_TYPE_STACK_TRACE:
3452 if (func_id != BPF_FUNC_get_stackid)
3453 goto error;
3454 break;
3455 case BPF_MAP_TYPE_CGROUP_ARRAY:
3456 if (func_id != BPF_FUNC_skb_under_cgroup &&
3457 func_id != BPF_FUNC_current_task_under_cgroup)
3458 goto error;
3459 break;
3460 case BPF_MAP_TYPE_CGROUP_STORAGE:
3461 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
3462 if (func_id != BPF_FUNC_get_local_storage)
3463 goto error;
3464 break;
3465 case BPF_MAP_TYPE_DEVMAP:
3466 if (func_id != BPF_FUNC_redirect_map &&
3467 func_id != BPF_FUNC_map_lookup_elem)
3468 goto error;
3469 break;
3470
3471
3472
3473 case BPF_MAP_TYPE_CPUMAP:
3474 if (func_id != BPF_FUNC_redirect_map)
3475 goto error;
3476 break;
3477 case BPF_MAP_TYPE_XSKMAP:
3478 if (func_id != BPF_FUNC_redirect_map &&
3479 func_id != BPF_FUNC_map_lookup_elem)
3480 goto error;
3481 break;
3482 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
3483 case BPF_MAP_TYPE_HASH_OF_MAPS:
3484 if (func_id != BPF_FUNC_map_lookup_elem)
3485 goto error;
3486 break;
3487 case BPF_MAP_TYPE_SOCKMAP:
3488 if (func_id != BPF_FUNC_sk_redirect_map &&
3489 func_id != BPF_FUNC_sock_map_update &&
3490 func_id != BPF_FUNC_map_delete_elem &&
3491 func_id != BPF_FUNC_msg_redirect_map)
3492 goto error;
3493 break;
3494 case BPF_MAP_TYPE_SOCKHASH:
3495 if (func_id != BPF_FUNC_sk_redirect_hash &&
3496 func_id != BPF_FUNC_sock_hash_update &&
3497 func_id != BPF_FUNC_map_delete_elem &&
3498 func_id != BPF_FUNC_msg_redirect_hash)
3499 goto error;
3500 break;
3501 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
3502 if (func_id != BPF_FUNC_sk_select_reuseport)
3503 goto error;
3504 break;
3505 case BPF_MAP_TYPE_QUEUE:
3506 case BPF_MAP_TYPE_STACK:
3507 if (func_id != BPF_FUNC_map_peek_elem &&
3508 func_id != BPF_FUNC_map_pop_elem &&
3509 func_id != BPF_FUNC_map_push_elem)
3510 goto error;
3511 break;
3512 case BPF_MAP_TYPE_SK_STORAGE:
3513 if (func_id != BPF_FUNC_sk_storage_get &&
3514 func_id != BPF_FUNC_sk_storage_delete)
3515 goto error;
3516 break;
3517 default:
3518 break;
3519 }
3520
3521
3522 switch (func_id) {
3523 case BPF_FUNC_tail_call:
3524 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
3525 goto error;
3526 if (env->subprog_cnt > 1) {
3527 verbose(env, "tail_calls are not allowed in programs with bpf-to-bpf calls\n");
3528 return -EINVAL;
3529 }
3530 break;
3531 case BPF_FUNC_perf_event_read:
3532 case BPF_FUNC_perf_event_output:
3533 case BPF_FUNC_perf_event_read_value:
3534 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
3535 goto error;
3536 break;
3537 case BPF_FUNC_get_stackid:
3538 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
3539 goto error;
3540 break;
3541 case BPF_FUNC_current_task_under_cgroup:
3542 case BPF_FUNC_skb_under_cgroup:
3543 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY)
3544 goto error;
3545 break;
3546 case BPF_FUNC_redirect_map:
3547 if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
3548 map->map_type != BPF_MAP_TYPE_CPUMAP &&
3549 map->map_type != BPF_MAP_TYPE_XSKMAP)
3550 goto error;
3551 break;
3552 case BPF_FUNC_sk_redirect_map:
3553 case BPF_FUNC_msg_redirect_map:
3554 case BPF_FUNC_sock_map_update:
3555 if (map->map_type != BPF_MAP_TYPE_SOCKMAP)
3556 goto error;
3557 break;
3558 case BPF_FUNC_sk_redirect_hash:
3559 case BPF_FUNC_msg_redirect_hash:
3560 case BPF_FUNC_sock_hash_update:
3561 if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
3562 goto error;
3563 break;
3564 case BPF_FUNC_get_local_storage:
3565 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
3566 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
3567 goto error;
3568 break;
3569 case BPF_FUNC_sk_select_reuseport:
3570 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY)
3571 goto error;
3572 break;
3573 case BPF_FUNC_map_peek_elem:
3574 case BPF_FUNC_map_pop_elem:
3575 case BPF_FUNC_map_push_elem:
3576 if (map->map_type != BPF_MAP_TYPE_QUEUE &&
3577 map->map_type != BPF_MAP_TYPE_STACK)
3578 goto error;
3579 break;
3580 case BPF_FUNC_sk_storage_get:
3581 case BPF_FUNC_sk_storage_delete:
3582 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
3583 goto error;
3584 break;
3585 default:
3586 break;
3587 }
3588
3589 return 0;
3590error:
3591 verbose(env, "cannot pass map_type %d into func %s#%d\n",
3592 map->map_type, func_id_name(func_id), func_id);
3593 return -EINVAL;
3594}
3595
3596static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
3597{
3598 int count = 0;
3599
3600 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM)
3601 count++;
3602 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM)
3603 count++;
3604 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM)
3605 count++;
3606 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM)
3607 count++;
3608 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM)
3609 count++;
3610
3611
3612
3613
3614
3615 return count <= 1;
3616}
3617
3618static bool check_args_pair_invalid(enum bpf_arg_type arg_curr,
3619 enum bpf_arg_type arg_next)
3620{
3621 return (arg_type_is_mem_ptr(arg_curr) &&
3622 !arg_type_is_mem_size(arg_next)) ||
3623 (!arg_type_is_mem_ptr(arg_curr) &&
3624 arg_type_is_mem_size(arg_next));
3625}
3626
3627static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
3628{
3629
3630
3631
3632
3633
3634 if (arg_type_is_mem_size(fn->arg1_type) ||
3635 arg_type_is_mem_ptr(fn->arg5_type) ||
3636 check_args_pair_invalid(fn->arg1_type, fn->arg2_type) ||
3637 check_args_pair_invalid(fn->arg2_type, fn->arg3_type) ||
3638 check_args_pair_invalid(fn->arg3_type, fn->arg4_type) ||
3639 check_args_pair_invalid(fn->arg4_type, fn->arg5_type))
3640 return false;
3641
3642 return true;
3643}
3644
3645static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id)
3646{
3647 int count = 0;
3648
3649 if (arg_type_may_be_refcounted(fn->arg1_type))
3650 count++;
3651 if (arg_type_may_be_refcounted(fn->arg2_type))
3652 count++;
3653 if (arg_type_may_be_refcounted(fn->arg3_type))
3654 count++;
3655 if (arg_type_may_be_refcounted(fn->arg4_type))
3656 count++;
3657 if (arg_type_may_be_refcounted(fn->arg5_type))
3658 count++;
3659
3660
3661
3662
3663 if (is_acquire_function(func_id) && count)
3664 return false;
3665
3666
3667
3668
3669 return count <= 1;
3670}
3671
3672static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
3673{
3674 return check_raw_mode_ok(fn) &&
3675 check_arg_pair_ok(fn) &&
3676 check_refcount_ok(fn, func_id) ? 0 : -EINVAL;
3677}
3678
3679
3680
3681
3682static void __clear_all_pkt_pointers(struct bpf_verifier_env *env,
3683 struct bpf_func_state *state)
3684{
3685 struct bpf_reg_state *regs = state->regs, *reg;
3686 int i;
3687
3688 for (i = 0; i < MAX_BPF_REG; i++)
3689 if (reg_is_pkt_pointer_any(®s[i]))
3690 mark_reg_unknown(env, regs, i);
3691
3692 bpf_for_each_spilled_reg(i, state, reg) {
3693 if (!reg)
3694 continue;
3695 if (reg_is_pkt_pointer_any(reg))
3696 __mark_reg_unknown(reg);
3697 }
3698}
3699
3700static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
3701{
3702 struct bpf_verifier_state *vstate = env->cur_state;
3703 int i;
3704
3705 for (i = 0; i <= vstate->curframe; i++)
3706 __clear_all_pkt_pointers(env, vstate->frame[i]);
3707}
3708
3709static void release_reg_references(struct bpf_verifier_env *env,
3710 struct bpf_func_state *state,
3711 int ref_obj_id)
3712{
3713 struct bpf_reg_state *regs = state->regs, *reg;
3714 int i;
3715
3716 for (i = 0; i < MAX_BPF_REG; i++)
3717 if (regs[i].ref_obj_id == ref_obj_id)
3718 mark_reg_unknown(env, regs, i);
3719
3720 bpf_for_each_spilled_reg(i, state, reg) {
3721 if (!reg)
3722 continue;
3723 if (reg->ref_obj_id == ref_obj_id)
3724 __mark_reg_unknown(reg);
3725 }
3726}
3727
3728
3729
3730
3731static int release_reference(struct bpf_verifier_env *env,
3732 int ref_obj_id)
3733{
3734 struct bpf_verifier_state *vstate = env->cur_state;
3735 int err;
3736 int i;
3737
3738 err = release_reference_state(cur_func(env), ref_obj_id);
3739 if (err)
3740 return err;
3741
3742 for (i = 0; i <= vstate->curframe; i++)
3743 release_reg_references(env, vstate->frame[i], ref_obj_id);
3744
3745 return 0;
3746}
3747
3748static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
3749 int *insn_idx)
3750{
3751 struct bpf_verifier_state *state = env->cur_state;
3752 struct bpf_func_state *caller, *callee;
3753 int i, err, subprog, target_insn;
3754
3755 if (state->curframe + 1 >= MAX_CALL_FRAMES) {
3756 verbose(env, "the call stack of %d frames is too deep\n",
3757 state->curframe + 2);
3758 return -E2BIG;
3759 }
3760
3761 target_insn = *insn_idx + insn->imm;
3762 subprog = find_subprog(env, target_insn + 1);
3763 if (subprog < 0) {
3764 verbose(env, "verifier bug. No program starts at insn %d\n",
3765 target_insn + 1);
3766 return -EFAULT;
3767 }
3768
3769 caller = state->frame[state->curframe];
3770 if (state->frame[state->curframe + 1]) {
3771 verbose(env, "verifier bug. Frame %d already allocated\n",
3772 state->curframe + 1);
3773 return -EFAULT;
3774 }
3775
3776 callee = kzalloc(sizeof(*callee), GFP_KERNEL);
3777 if (!callee)
3778 return -ENOMEM;
3779 state->frame[state->curframe + 1] = callee;
3780
3781
3782
3783
3784
3785 init_func_state(env, callee,
3786
3787 *insn_idx ,
3788 state->curframe + 1 ,
3789 subprog );
3790
3791
3792 err = transfer_reference_state(callee, caller);
3793 if (err)
3794 return err;
3795
3796
3797
3798
3799 for (i = BPF_REG_1; i <= BPF_REG_5; i++)
3800 callee->regs[i] = caller->regs[i];
3801
3802
3803 for (i = 0; i < CALLER_SAVED_REGS; i++) {
3804 mark_reg_not_init(env, caller->regs, caller_saved[i]);
3805 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
3806 }
3807
3808
3809 state->curframe++;
3810
3811
3812 *insn_idx = target_insn;
3813
3814 if (env->log.level & BPF_LOG_LEVEL) {
3815 verbose(env, "caller:\n");
3816 print_verifier_state(env, caller);
3817 verbose(env, "callee:\n");
3818 print_verifier_state(env, callee);
3819 }
3820 return 0;
3821}
3822
3823static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
3824{
3825 struct bpf_verifier_state *state = env->cur_state;
3826 struct bpf_func_state *caller, *callee;
3827 struct bpf_reg_state *r0;
3828 int err;
3829
3830 callee = state->frame[state->curframe];
3831 r0 = &callee->regs[BPF_REG_0];
3832 if (r0->type == PTR_TO_STACK) {
3833
3834
3835
3836
3837
3838
3839 verbose(env, "cannot return stack pointer to the caller\n");
3840 return -EINVAL;
3841 }
3842
3843 state->curframe--;
3844 caller = state->frame[state->curframe];
3845
3846 caller->regs[BPF_REG_0] = *r0;
3847
3848
3849 err = transfer_reference_state(caller, callee);
3850 if (err)
3851 return err;
3852
3853 *insn_idx = callee->callsite + 1;
3854 if (env->log.level & BPF_LOG_LEVEL) {
3855 verbose(env, "returning from callee:\n");
3856 print_verifier_state(env, callee);
3857 verbose(env, "to caller at %d:\n", *insn_idx);
3858 print_verifier_state(env, caller);
3859 }
3860
3861 free_func_state(callee);
3862 state->frame[state->curframe + 1] = NULL;
3863 return 0;
3864}
3865
3866static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,
3867 int func_id,
3868 struct bpf_call_arg_meta *meta)
3869{
3870 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0];
3871
3872 if (ret_type != RET_INTEGER ||
3873 (func_id != BPF_FUNC_get_stack &&
3874 func_id != BPF_FUNC_probe_read_str))
3875 return;
3876
3877 ret_reg->smax_value = meta->msize_smax_value;
3878 ret_reg->umax_value = meta->msize_umax_value;
3879 __reg_deduce_bounds(ret_reg);
3880 __reg_bound_offset(ret_reg);
3881}
3882
3883static int
3884record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
3885 int func_id, int insn_idx)
3886{
3887 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
3888 struct bpf_map *map = meta->map_ptr;
3889
3890 if (func_id != BPF_FUNC_tail_call &&
3891 func_id != BPF_FUNC_map_lookup_elem &&
3892 func_id != BPF_FUNC_map_update_elem &&
3893 func_id != BPF_FUNC_map_delete_elem &&
3894 func_id != BPF_FUNC_map_push_elem &&
3895 func_id != BPF_FUNC_map_pop_elem &&
3896 func_id != BPF_FUNC_map_peek_elem)
3897 return 0;
3898
3899 if (map == NULL) {
3900 verbose(env, "kernel subsystem misconfigured verifier\n");
3901 return -EINVAL;
3902 }
3903
3904
3905
3906
3907
3908 if ((map->map_flags & BPF_F_RDONLY_PROG) &&
3909 (func_id == BPF_FUNC_map_delete_elem ||
3910 func_id == BPF_FUNC_map_update_elem ||
3911 func_id == BPF_FUNC_map_push_elem ||
3912 func_id == BPF_FUNC_map_pop_elem)) {
3913 verbose(env, "write into map forbidden\n");
3914 return -EACCES;
3915 }
3916
3917 if (!BPF_MAP_PTR(aux->map_state))
3918 bpf_map_ptr_store(aux, meta->map_ptr,
3919 meta->map_ptr->unpriv_array);
3920 else if (BPF_MAP_PTR(aux->map_state) != meta->map_ptr)
3921 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON,
3922 meta->map_ptr->unpriv_array);
3923 return 0;
3924}
3925
3926static int check_reference_leak(struct bpf_verifier_env *env)
3927{
3928 struct bpf_func_state *state = cur_func(env);
3929 int i;
3930
3931 for (i = 0; i < state->acquired_refs; i++) {
3932 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
3933 state->refs[i].id, state->refs[i].insn_idx);
3934 }
3935 return state->acquired_refs ? -EINVAL : 0;
3936}
3937
3938static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx)
3939{
3940 const struct bpf_func_proto *fn = NULL;
3941 struct bpf_reg_state *regs;
3942 struct bpf_call_arg_meta meta;
3943 bool changes_data;
3944 int i, err;
3945
3946
3947 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) {
3948 verbose(env, "invalid func %s#%d\n", func_id_name(func_id),
3949 func_id);
3950 return -EINVAL;
3951 }
3952
3953 if (env->ops->get_func_proto)
3954 fn = env->ops->get_func_proto(func_id, env->prog);
3955 if (!fn) {
3956 verbose(env, "unknown func %s#%d\n", func_id_name(func_id),
3957 func_id);
3958 return -EINVAL;
3959 }
3960
3961
3962 if (!env->prog->gpl_compatible && fn->gpl_only) {
3963 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
3964 return -EINVAL;
3965 }
3966
3967
3968 changes_data = bpf_helper_changes_pkt_data(fn->func);
3969 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
3970 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",
3971 func_id_name(func_id), func_id);
3972 return -EINVAL;
3973 }
3974
3975 memset(&meta, 0, sizeof(meta));
3976 meta.pkt_access = fn->pkt_access;
3977
3978 err = check_func_proto(fn, func_id);
3979 if (err) {
3980 verbose(env, "kernel subsystem misconfigured func %s#%d\n",
3981 func_id_name(func_id), func_id);
3982 return err;
3983 }
3984
3985 meta.func_id = func_id;
3986
3987 err = check_func_arg(env, BPF_REG_1, fn->arg1_type, &meta);
3988 if (err)
3989 return err;
3990 err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &meta);
3991 if (err)
3992 return err;
3993 err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &meta);
3994 if (err)
3995 return err;
3996 err = check_func_arg(env, BPF_REG_4, fn->arg4_type, &meta);
3997 if (err)
3998 return err;
3999 err = check_func_arg(env, BPF_REG_5, fn->arg5_type, &meta);
4000 if (err)
4001 return err;
4002
4003 err = record_func_map(env, &meta, func_id, insn_idx);
4004 if (err)
4005 return err;
4006
4007
4008
4009
4010 for (i = 0; i < meta.access_size; i++) {
4011 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,
4012 BPF_WRITE, -1, false);
4013 if (err)
4014 return err;
4015 }
4016
4017 if (func_id == BPF_FUNC_tail_call) {
4018 err = check_reference_leak(env);
4019 if (err) {
4020 verbose(env, "tail_call would lead to reference leak\n");
4021 return err;
4022 }
4023 } else if (is_release_function(func_id)) {
4024 err = release_reference(env, meta.ref_obj_id);
4025 if (err) {
4026 verbose(env, "func %s#%d reference has not been acquired before\n",
4027 func_id_name(func_id), func_id);
4028 return err;
4029 }
4030 }
4031
4032 regs = cur_regs(env);
4033
4034
4035
4036
4037 if (func_id == BPF_FUNC_get_local_storage &&
4038 !register_is_null(®s[BPF_REG_2])) {
4039 verbose(env, "get_local_storage() doesn't support non-zero flags\n");
4040 return -EINVAL;
4041 }
4042
4043
4044 for (i = 0; i < CALLER_SAVED_REGS; i++) {
4045 mark_reg_not_init(env, regs, caller_saved[i]);
4046 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
4047 }
4048
4049
4050 regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
4051
4052
4053 if (fn->ret_type == RET_INTEGER) {
4054
4055 mark_reg_unknown(env, regs, BPF_REG_0);
4056 } else if (fn->ret_type == RET_VOID) {
4057 regs[BPF_REG_0].type = NOT_INIT;
4058 } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL ||
4059 fn->ret_type == RET_PTR_TO_MAP_VALUE) {
4060
4061 mark_reg_known_zero(env, regs, BPF_REG_0);
4062
4063
4064
4065
4066 if (meta.map_ptr == NULL) {
4067 verbose(env,
4068 "kernel subsystem misconfigured verifier\n");
4069 return -EINVAL;
4070 }
4071 regs[BPF_REG_0].map_ptr = meta.map_ptr;
4072 if (fn->ret_type == RET_PTR_TO_MAP_VALUE) {
4073 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE;
4074 if (map_value_has_spin_lock(meta.map_ptr))
4075 regs[BPF_REG_0].id = ++env->id_gen;
4076 } else {
4077 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
4078 regs[BPF_REG_0].id = ++env->id_gen;
4079 }
4080 } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) {
4081 mark_reg_known_zero(env, regs, BPF_REG_0);
4082 regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL;
4083 regs[BPF_REG_0].id = ++env->id_gen;
4084 } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) {
4085 mark_reg_known_zero(env, regs, BPF_REG_0);
4086 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL;
4087 regs[BPF_REG_0].id = ++env->id_gen;
4088 } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) {
4089 mark_reg_known_zero(env, regs, BPF_REG_0);
4090 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL;
4091 regs[BPF_REG_0].id = ++env->id_gen;
4092 } else {
4093 verbose(env, "unknown return type %d of func %s#%d\n",
4094 fn->ret_type, func_id_name(func_id), func_id);
4095 return -EINVAL;
4096 }
4097
4098 if (is_ptr_cast_function(func_id)) {
4099
4100 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
4101 } else if (is_acquire_function(func_id)) {
4102 int id = acquire_reference_state(env, insn_idx);
4103
4104 if (id < 0)
4105 return id;
4106
4107 regs[BPF_REG_0].id = id;
4108
4109 regs[BPF_REG_0].ref_obj_id = id;
4110 }
4111
4112 do_refine_retval_range(regs, fn->ret_type, func_id, &meta);
4113
4114 err = check_map_func_compatibility(env, meta.map_ptr, func_id);
4115 if (err)
4116 return err;
4117
4118 if (func_id == BPF_FUNC_get_stack && !env->prog->has_callchain_buf) {
4119 const char *err_str;
4120
4121#ifdef CONFIG_PERF_EVENTS
4122 err = get_callchain_buffers(sysctl_perf_event_max_stack);
4123 err_str = "cannot get callchain buffer for func %s#%d\n";
4124#else
4125 err = -ENOTSUPP;
4126 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";
4127#endif
4128 if (err) {
4129 verbose(env, err_str, func_id_name(func_id), func_id);
4130 return err;
4131 }
4132
4133 env->prog->has_callchain_buf = true;
4134 }
4135
4136 if (changes_data)
4137 clear_all_pkt_pointers(env);
4138 return 0;
4139}
4140
4141static bool signed_add_overflows(s64 a, s64 b)
4142{
4143
4144 s64 res = (s64)((u64)a + (u64)b);
4145
4146 if (b < 0)
4147 return res > a;
4148 return res < a;
4149}
4150
4151static bool signed_sub_overflows(s64 a, s64 b)
4152{
4153
4154 s64 res = (s64)((u64)a - (u64)b);
4155
4156 if (b < 0)
4157 return res < a;
4158 return res > a;
4159}
4160
4161static bool check_reg_sane_offset(struct bpf_verifier_env *env,
4162 const struct bpf_reg_state *reg,
4163 enum bpf_reg_type type)
4164{
4165 bool known = tnum_is_const(reg->var_off);
4166 s64 val = reg->var_off.value;
4167 s64 smin = reg->smin_value;
4168
4169 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
4170 verbose(env, "math between %s pointer and %lld is not allowed\n",
4171 reg_type_str[type], val);
4172 return false;
4173 }
4174
4175 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
4176 verbose(env, "%s pointer offset %d is not allowed\n",
4177 reg_type_str[type], reg->off);
4178 return false;
4179 }
4180
4181 if (smin == S64_MIN) {
4182 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
4183 reg_type_str[type]);
4184 return false;
4185 }
4186
4187 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
4188 verbose(env, "value %lld makes %s pointer be out of bounds\n",
4189 smin, reg_type_str[type]);
4190 return false;
4191 }
4192
4193 return true;
4194}
4195
4196static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)
4197{
4198 return &env->insn_aux_data[env->insn_idx];
4199}
4200
4201static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
4202 u32 *ptr_limit, u8 opcode, bool off_is_neg)
4203{
4204 bool mask_to_left = (opcode == BPF_ADD && off_is_neg) ||
4205 (opcode == BPF_SUB && !off_is_neg);
4206 u32 off;
4207
4208 switch (ptr_reg->type) {
4209 case PTR_TO_STACK:
4210
4211
4212
4213 off = ptr_reg->off + ptr_reg->var_off.value;
4214 if (mask_to_left)
4215 *ptr_limit = MAX_BPF_STACK + off;
4216 else
4217 *ptr_limit = -off;
4218 return 0;
4219 case PTR_TO_MAP_VALUE:
4220 if (mask_to_left) {
4221 *ptr_limit = ptr_reg->umax_value + ptr_reg->off;
4222 } else {
4223 off = ptr_reg->smin_value + ptr_reg->off;
4224 *ptr_limit = ptr_reg->map_ptr->value_size - off;
4225 }
4226 return 0;
4227 default:
4228 return -EINVAL;
4229 }
4230}
4231
4232static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
4233 const struct bpf_insn *insn)
4234{
4235 return env->allow_ptr_leaks || BPF_SRC(insn->code) == BPF_K;
4236}
4237
4238static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
4239 u32 alu_state, u32 alu_limit)
4240{
4241
4242
4243
4244 if (aux->alu_state &&
4245 (aux->alu_state != alu_state ||
4246 aux->alu_limit != alu_limit))
4247 return -EACCES;
4248
4249
4250 aux->alu_state = alu_state;
4251 aux->alu_limit = alu_limit;
4252 return 0;
4253}
4254
4255static int sanitize_val_alu(struct bpf_verifier_env *env,
4256 struct bpf_insn *insn)
4257{
4258 struct bpf_insn_aux_data *aux = cur_aux(env);
4259
4260 if (can_skip_alu_sanitation(env, insn))
4261 return 0;
4262
4263 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
4264}
4265
4266static int sanitize_ptr_alu(struct bpf_verifier_env *env,
4267 struct bpf_insn *insn,
4268 const struct bpf_reg_state *ptr_reg,
4269 struct bpf_reg_state *dst_reg,
4270 bool off_is_neg)
4271{
4272 struct bpf_verifier_state *vstate = env->cur_state;
4273 struct bpf_insn_aux_data *aux = cur_aux(env);
4274 bool ptr_is_dst_reg = ptr_reg == dst_reg;
4275 u8 opcode = BPF_OP(insn->code);
4276 u32 alu_state, alu_limit;
4277 struct bpf_reg_state tmp;
4278 bool ret;
4279
4280 if (can_skip_alu_sanitation(env, insn))
4281 return 0;
4282
4283
4284
4285
4286
4287 if (vstate->speculative)
4288 goto do_sim;
4289
4290 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
4291 alu_state |= ptr_is_dst_reg ?
4292 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
4293
4294 if (retrieve_ptr_limit(ptr_reg, &alu_limit, opcode, off_is_neg))
4295 return 0;
4296 if (update_alu_sanitation_state(aux, alu_state, alu_limit))
4297 return -EACCES;
4298do_sim:
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308 if (!ptr_is_dst_reg) {
4309 tmp = *dst_reg;
4310 *dst_reg = *ptr_reg;
4311 }
4312 ret = push_stack(env, env->insn_idx + 1, env->insn_idx, true);
4313 if (!ptr_is_dst_reg && ret)
4314 *dst_reg = tmp;
4315 return !ret ? -EFAULT : 0;
4316}
4317
4318
4319
4320
4321
4322
4323static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
4324 struct bpf_insn *insn,
4325 const struct bpf_reg_state *ptr_reg,
4326 const struct bpf_reg_state *off_reg)
4327{
4328 struct bpf_verifier_state *vstate = env->cur_state;
4329 struct bpf_func_state *state = vstate->frame[vstate->curframe];
4330 struct bpf_reg_state *regs = state->regs, *dst_reg;
4331 bool known = tnum_is_const(off_reg->var_off);
4332 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value,
4333 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value;
4334 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value,
4335 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value;
4336 u32 dst = insn->dst_reg, src = insn->src_reg;
4337 u8 opcode = BPF_OP(insn->code);
4338 int ret;
4339
4340 dst_reg = ®s[dst];
4341
4342 if ((known && (smin_val != smax_val || umin_val != umax_val)) ||
4343 smin_val > smax_val || umin_val > umax_val) {
4344
4345
4346
4347 __mark_reg_unknown(dst_reg);
4348 return 0;
4349 }
4350
4351 if (BPF_CLASS(insn->code) != BPF_ALU64) {
4352
4353 verbose(env,
4354 "R%d 32-bit pointer arithmetic prohibited\n",
4355 dst);
4356 return -EACCES;
4357 }
4358
4359 switch (ptr_reg->type) {
4360 case PTR_TO_MAP_VALUE_OR_NULL:
4361 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
4362 dst, reg_type_str[ptr_reg->type]);
4363 return -EACCES;
4364 case CONST_PTR_TO_MAP:
4365 case PTR_TO_PACKET_END:
4366 case PTR_TO_SOCKET:
4367 case PTR_TO_SOCKET_OR_NULL:
4368 case PTR_TO_SOCK_COMMON:
4369 case PTR_TO_SOCK_COMMON_OR_NULL:
4370 case PTR_TO_TCP_SOCK:
4371 case PTR_TO_TCP_SOCK_OR_NULL:
4372 case PTR_TO_XDP_SOCK:
4373 verbose(env, "R%d pointer arithmetic on %s prohibited\n",
4374 dst, reg_type_str[ptr_reg->type]);
4375 return -EACCES;
4376 case PTR_TO_MAP_VALUE:
4377 if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) {
4378 verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n",
4379 off_reg == dst_reg ? dst : src);
4380 return -EACCES;
4381 }
4382
4383 default:
4384 break;
4385 }
4386
4387
4388
4389
4390 dst_reg->type = ptr_reg->type;
4391 dst_reg->id = ptr_reg->id;
4392
4393 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||
4394 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))
4395 return -EINVAL;
4396
4397 switch (opcode) {
4398 case BPF_ADD:
4399 ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0);
4400 if (ret < 0) {
4401 verbose(env, "R%d tried to add from different maps or paths\n", dst);
4402 return ret;
4403 }
4404
4405
4406
4407 if (known && (ptr_reg->off + smin_val ==
4408 (s64)(s32)(ptr_reg->off + smin_val))) {
4409
4410 dst_reg->smin_value = smin_ptr;
4411 dst_reg->smax_value = smax_ptr;
4412 dst_reg->umin_value = umin_ptr;
4413 dst_reg->umax_value = umax_ptr;
4414 dst_reg->var_off = ptr_reg->var_off;
4415 dst_reg->off = ptr_reg->off + smin_val;
4416 dst_reg->raw = ptr_reg->raw;
4417 break;
4418 }
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428 if (signed_add_overflows(smin_ptr, smin_val) ||
4429 signed_add_overflows(smax_ptr, smax_val)) {
4430 dst_reg->smin_value = S64_MIN;
4431 dst_reg->smax_value = S64_MAX;
4432 } else {
4433 dst_reg->smin_value = smin_ptr + smin_val;
4434 dst_reg->smax_value = smax_ptr + smax_val;
4435 }
4436 if (umin_ptr + umin_val < umin_ptr ||
4437 umax_ptr + umax_val < umax_ptr) {
4438 dst_reg->umin_value = 0;
4439 dst_reg->umax_value = U64_MAX;
4440 } else {
4441 dst_reg->umin_value = umin_ptr + umin_val;
4442 dst_reg->umax_value = umax_ptr + umax_val;
4443 }
4444 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
4445 dst_reg->off = ptr_reg->off;
4446 dst_reg->raw = ptr_reg->raw;
4447 if (reg_is_pkt_pointer(ptr_reg)) {
4448 dst_reg->id = ++env->id_gen;
4449
4450 dst_reg->raw = 0;
4451 }
4452 break;
4453 case BPF_SUB:
4454 ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0);
4455 if (ret < 0) {
4456 verbose(env, "R%d tried to sub from different maps or paths\n", dst);
4457 return ret;
4458 }
4459 if (dst_reg == off_reg) {
4460
4461 verbose(env, "R%d tried to subtract pointer from scalar\n",
4462 dst);
4463 return -EACCES;
4464 }
4465
4466
4467
4468
4469 if (ptr_reg->type == PTR_TO_STACK) {
4470 verbose(env, "R%d subtraction from stack pointer prohibited\n",
4471 dst);
4472 return -EACCES;
4473 }
4474 if (known && (ptr_reg->off - smin_val ==
4475 (s64)(s32)(ptr_reg->off - smin_val))) {
4476
4477 dst_reg->smin_value = smin_ptr;
4478 dst_reg->smax_value = smax_ptr;
4479 dst_reg->umin_value = umin_ptr;
4480 dst_reg->umax_value = umax_ptr;
4481 dst_reg->var_off = ptr_reg->var_off;
4482 dst_reg->id = ptr_reg->id;
4483 dst_reg->off = ptr_reg->off - smin_val;
4484 dst_reg->raw = ptr_reg->raw;
4485 break;
4486 }
4487
4488
4489
4490 if (signed_sub_overflows(smin_ptr, smax_val) ||
4491 signed_sub_overflows(smax_ptr, smin_val)) {
4492
4493 dst_reg->smin_value = S64_MIN;
4494 dst_reg->smax_value = S64_MAX;
4495 } else {
4496 dst_reg->smin_value = smin_ptr - smax_val;
4497 dst_reg->smax_value = smax_ptr - smin_val;
4498 }
4499 if (umin_ptr < umax_val) {
4500
4501 dst_reg->umin_value = 0;
4502 dst_reg->umax_value = U64_MAX;
4503 } else {
4504
4505 dst_reg->umin_value = umin_ptr - umax_val;
4506 dst_reg->umax_value = umax_ptr - umin_val;
4507 }
4508 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
4509 dst_reg->off = ptr_reg->off;
4510 dst_reg->raw = ptr_reg->raw;
4511 if (reg_is_pkt_pointer(ptr_reg)) {
4512 dst_reg->id = ++env->id_gen;
4513
4514 if (smin_val < 0)
4515 dst_reg->raw = 0;
4516 }
4517 break;
4518 case BPF_AND:
4519 case BPF_OR:
4520 case BPF_XOR:
4521
4522 verbose(env, "R%d bitwise operator %s on pointer prohibited\n",
4523 dst, bpf_alu_string[opcode >> 4]);
4524 return -EACCES;
4525 default:
4526
4527 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",
4528 dst, bpf_alu_string[opcode >> 4]);
4529 return -EACCES;
4530 }
4531
4532 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))
4533 return -EINVAL;
4534
4535 __update_reg_bounds(dst_reg);
4536 __reg_deduce_bounds(dst_reg);
4537 __reg_bound_offset(dst_reg);
4538
4539
4540
4541
4542 if (!env->allow_ptr_leaks) {
4543 if (dst_reg->type == PTR_TO_MAP_VALUE &&
4544 check_map_access(env, dst, dst_reg->off, 1, false)) {
4545 verbose(env, "R%d pointer arithmetic of map value goes out of range, "
4546 "prohibited for !root\n", dst);
4547 return -EACCES;
4548 } else if (dst_reg->type == PTR_TO_STACK &&
4549 check_stack_access(env, dst_reg, dst_reg->off +
4550 dst_reg->var_off.value, 1)) {
4551 verbose(env, "R%d stack pointer arithmetic goes out of range, "
4552 "prohibited for !root\n", dst);
4553 return -EACCES;
4554 }
4555 }
4556
4557 return 0;
4558}
4559
4560
4561
4562
4563
4564static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
4565 struct bpf_insn *insn,
4566 struct bpf_reg_state *dst_reg,
4567 struct bpf_reg_state src_reg)
4568{
4569 struct bpf_reg_state *regs = cur_regs(env);
4570 u8 opcode = BPF_OP(insn->code);
4571 bool src_known, dst_known;
4572 s64 smin_val, smax_val;
4573 u64 umin_val, umax_val;
4574 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
4575 u32 dst = insn->dst_reg;
4576 int ret;
4577
4578 if (insn_bitness == 32) {
4579
4580
4581
4582
4583 coerce_reg_to_size(dst_reg, 4);
4584 coerce_reg_to_size(&src_reg, 4);
4585 }
4586
4587 smin_val = src_reg.smin_value;
4588 smax_val = src_reg.smax_value;
4589 umin_val = src_reg.umin_value;
4590 umax_val = src_reg.umax_value;
4591 src_known = tnum_is_const(src_reg.var_off);
4592 dst_known = tnum_is_const(dst_reg->var_off);
4593
4594 if ((src_known && (smin_val != smax_val || umin_val != umax_val)) ||
4595 smin_val > smax_val || umin_val > umax_val) {
4596
4597
4598
4599 __mark_reg_unknown(dst_reg);
4600 return 0;
4601 }
4602
4603 if (!src_known &&
4604 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) {
4605 __mark_reg_unknown(dst_reg);
4606 return 0;
4607 }
4608
4609 switch (opcode) {
4610 case BPF_ADD:
4611 ret = sanitize_val_alu(env, insn);
4612 if (ret < 0) {
4613 verbose(env, "R%d tried to add from different pointers or scalars\n", dst);
4614 return ret;
4615 }
4616 if (signed_add_overflows(dst_reg->smin_value, smin_val) ||
4617 signed_add_overflows(dst_reg->smax_value, smax_val)) {
4618 dst_reg->smin_value = S64_MIN;
4619 dst_reg->smax_value = S64_MAX;
4620 } else {
4621 dst_reg->smin_value += smin_val;
4622 dst_reg->smax_value += smax_val;
4623 }
4624 if (dst_reg->umin_value + umin_val < umin_val ||
4625 dst_reg->umax_value + umax_val < umax_val) {
4626 dst_reg->umin_value = 0;
4627 dst_reg->umax_value = U64_MAX;
4628 } else {
4629 dst_reg->umin_value += umin_val;
4630 dst_reg->umax_value += umax_val;
4631 }
4632 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
4633 break;
4634 case BPF_SUB:
4635 ret = sanitize_val_alu(env, insn);
4636 if (ret < 0) {
4637 verbose(env, "R%d tried to sub from different pointers or scalars\n", dst);
4638 return ret;
4639 }
4640 if (signed_sub_overflows(dst_reg->smin_value, smax_val) ||
4641 signed_sub_overflows(dst_reg->smax_value, smin_val)) {
4642
4643 dst_reg->smin_value = S64_MIN;
4644 dst_reg->smax_value = S64_MAX;
4645 } else {
4646 dst_reg->smin_value -= smax_val;
4647 dst_reg->smax_value -= smin_val;
4648 }
4649 if (dst_reg->umin_value < umax_val) {
4650
4651 dst_reg->umin_value = 0;
4652 dst_reg->umax_value = U64_MAX;
4653 } else {
4654
4655 dst_reg->umin_value -= umax_val;
4656 dst_reg->umax_value -= umin_val;
4657 }
4658 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off);
4659 break;
4660 case BPF_MUL:
4661 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
4662 if (smin_val < 0 || dst_reg->smin_value < 0) {
4663
4664 __mark_reg_unbounded(dst_reg);
4665 __update_reg_bounds(dst_reg);
4666 break;
4667 }
4668
4669
4670
4671 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) {
4672
4673 __mark_reg_unbounded(dst_reg);
4674
4675 __update_reg_bounds(dst_reg);
4676 break;
4677 }
4678 dst_reg->umin_value *= umin_val;
4679 dst_reg->umax_value *= umax_val;
4680 if (dst_reg->umax_value > S64_MAX) {
4681
4682 dst_reg->smin_value = S64_MIN;
4683 dst_reg->smax_value = S64_MAX;
4684 } else {
4685 dst_reg->smin_value = dst_reg->umin_value;
4686 dst_reg->smax_value = dst_reg->umax_value;
4687 }
4688 break;
4689 case BPF_AND:
4690 if (src_known && dst_known) {
4691 __mark_reg_known(dst_reg, dst_reg->var_off.value &
4692 src_reg.var_off.value);
4693 break;
4694 }
4695
4696
4697
4698 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off);
4699 dst_reg->umin_value = dst_reg->var_off.value;
4700 dst_reg->umax_value = min(dst_reg->umax_value, umax_val);
4701 if (dst_reg->smin_value < 0 || smin_val < 0) {
4702
4703
4704
4705 dst_reg->smin_value = S64_MIN;
4706 dst_reg->smax_value = S64_MAX;
4707 } else {
4708
4709
4710
4711 dst_reg->smin_value = dst_reg->umin_value;
4712 dst_reg->smax_value = dst_reg->umax_value;
4713 }
4714
4715 __update_reg_bounds(dst_reg);
4716 break;
4717 case BPF_OR:
4718 if (src_known && dst_known) {
4719 __mark_reg_known(dst_reg, dst_reg->var_off.value |
4720 src_reg.var_off.value);
4721 break;
4722 }
4723
4724
4725
4726 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off);
4727 dst_reg->umin_value = max(dst_reg->umin_value, umin_val);
4728 dst_reg->umax_value = dst_reg->var_off.value |
4729 dst_reg->var_off.mask;
4730 if (dst_reg->smin_value < 0 || smin_val < 0) {
4731
4732
4733
4734 dst_reg->smin_value = S64_MIN;
4735 dst_reg->smax_value = S64_MAX;
4736 } else {
4737
4738
4739
4740 dst_reg->smin_value = dst_reg->umin_value;
4741 dst_reg->smax_value = dst_reg->umax_value;
4742 }
4743
4744 __update_reg_bounds(dst_reg);
4745 break;
4746 case BPF_LSH:
4747 if (umax_val >= insn_bitness) {
4748
4749
4750
4751 mark_reg_unknown(env, regs, insn->dst_reg);
4752 break;
4753 }
4754
4755
4756
4757 dst_reg->smin_value = S64_MIN;
4758 dst_reg->smax_value = S64_MAX;
4759
4760 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
4761 dst_reg->umin_value = 0;
4762 dst_reg->umax_value = U64_MAX;
4763 } else {
4764 dst_reg->umin_value <<= umin_val;
4765 dst_reg->umax_value <<= umax_val;
4766 }
4767 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val);
4768
4769 __update_reg_bounds(dst_reg);
4770 break;
4771 case BPF_RSH:
4772 if (umax_val >= insn_bitness) {
4773
4774
4775
4776 mark_reg_unknown(env, regs, insn->dst_reg);
4777 break;
4778 }
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793 dst_reg->smin_value = S64_MIN;
4794 dst_reg->smax_value = S64_MAX;
4795 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
4796 dst_reg->umin_value >>= umax_val;
4797 dst_reg->umax_value >>= umin_val;
4798
4799 __update_reg_bounds(dst_reg);
4800 break;
4801 case BPF_ARSH:
4802 if (umax_val >= insn_bitness) {
4803
4804
4805
4806 mark_reg_unknown(env, regs, insn->dst_reg);
4807 break;
4808 }
4809
4810
4811
4812
4813 dst_reg->smin_value >>= umin_val;
4814 dst_reg->smax_value >>= umin_val;
4815 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val);
4816
4817
4818
4819
4820 dst_reg->umin_value = 0;
4821 dst_reg->umax_value = U64_MAX;
4822 __update_reg_bounds(dst_reg);
4823 break;
4824 default:
4825 mark_reg_unknown(env, regs, insn->dst_reg);
4826 break;
4827 }
4828
4829 if (BPF_CLASS(insn->code) != BPF_ALU64) {
4830
4831 coerce_reg_to_size(dst_reg, 4);
4832 }
4833
4834 __reg_deduce_bounds(dst_reg);
4835 __reg_bound_offset(dst_reg);
4836 return 0;
4837}
4838
4839
4840
4841
4842static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
4843 struct bpf_insn *insn)
4844{
4845 struct bpf_verifier_state *vstate = env->cur_state;
4846 struct bpf_func_state *state = vstate->frame[vstate->curframe];
4847 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
4848 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
4849 u8 opcode = BPF_OP(insn->code);
4850 int err;
4851
4852 dst_reg = ®s[insn->dst_reg];
4853 src_reg = NULL;
4854 if (dst_reg->type != SCALAR_VALUE)
4855 ptr_reg = dst_reg;
4856 if (BPF_SRC(insn->code) == BPF_X) {
4857 src_reg = ®s[insn->src_reg];
4858 if (src_reg->type != SCALAR_VALUE) {
4859 if (dst_reg->type != SCALAR_VALUE) {
4860
4861
4862
4863
4864 if (opcode == BPF_SUB && env->allow_ptr_leaks) {
4865 mark_reg_unknown(env, regs, insn->dst_reg);
4866 return 0;
4867 }
4868 verbose(env, "R%d pointer %s pointer prohibited\n",
4869 insn->dst_reg,
4870 bpf_alu_string[opcode >> 4]);
4871 return -EACCES;
4872 } else {
4873
4874
4875
4876
4877 err = mark_chain_precision(env, insn->dst_reg);
4878 if (err)
4879 return err;
4880 return adjust_ptr_min_max_vals(env, insn,
4881 src_reg, dst_reg);
4882 }
4883 } else if (ptr_reg) {
4884
4885 err = mark_chain_precision(env, insn->src_reg);
4886 if (err)
4887 return err;
4888 return adjust_ptr_min_max_vals(env, insn,
4889 dst_reg, src_reg);
4890 }
4891 } else {
4892
4893
4894
4895 off_reg.type = SCALAR_VALUE;
4896 __mark_reg_known(&off_reg, insn->imm);
4897 src_reg = &off_reg;
4898 if (ptr_reg)
4899 return adjust_ptr_min_max_vals(env, insn,
4900 ptr_reg, src_reg);
4901 }
4902
4903
4904 if (WARN_ON_ONCE(ptr_reg)) {
4905 print_verifier_state(env, state);
4906 verbose(env, "verifier internal error: unexpected ptr_reg\n");
4907 return -EINVAL;
4908 }
4909 if (WARN_ON(!src_reg)) {
4910 print_verifier_state(env, state);
4911 verbose(env, "verifier internal error: no src_reg\n");
4912 return -EINVAL;
4913 }
4914 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
4915}
4916
4917
4918static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
4919{
4920 struct bpf_reg_state *regs = cur_regs(env);
4921 u8 opcode = BPF_OP(insn->code);
4922 int err;
4923
4924 if (opcode == BPF_END || opcode == BPF_NEG) {
4925 if (opcode == BPF_NEG) {
4926 if (BPF_SRC(insn->code) != 0 ||
4927 insn->src_reg != BPF_REG_0 ||
4928 insn->off != 0 || insn->imm != 0) {
4929 verbose(env, "BPF_NEG uses reserved fields\n");
4930 return -EINVAL;
4931 }
4932 } else {
4933 if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||
4934 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||
4935 BPF_CLASS(insn->code) == BPF_ALU64) {
4936 verbose(env, "BPF_END uses reserved fields\n");
4937 return -EINVAL;
4938 }
4939 }
4940
4941
4942 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
4943 if (err)
4944 return err;
4945
4946 if (is_pointer_value(env, insn->dst_reg)) {
4947 verbose(env, "R%d pointer arithmetic prohibited\n",
4948 insn->dst_reg);
4949 return -EACCES;
4950 }
4951
4952
4953 err = check_reg_arg(env, insn->dst_reg, DST_OP);
4954 if (err)
4955 return err;
4956
4957 } else if (opcode == BPF_MOV) {
4958
4959 if (BPF_SRC(insn->code) == BPF_X) {
4960 if (insn->imm != 0 || insn->off != 0) {
4961 verbose(env, "BPF_MOV uses reserved fields\n");
4962 return -EINVAL;
4963 }
4964
4965
4966 err = check_reg_arg(env, insn->src_reg, SRC_OP);
4967 if (err)
4968 return err;
4969 } else {
4970 if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
4971 verbose(env, "BPF_MOV uses reserved fields\n");
4972 return -EINVAL;
4973 }
4974 }
4975
4976
4977 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
4978 if (err)
4979 return err;
4980
4981 if (BPF_SRC(insn->code) == BPF_X) {
4982 struct bpf_reg_state *src_reg = regs + insn->src_reg;
4983 struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
4984
4985 if (BPF_CLASS(insn->code) == BPF_ALU64) {
4986
4987
4988
4989 *dst_reg = *src_reg;
4990 dst_reg->live |= REG_LIVE_WRITTEN;
4991 dst_reg->subreg_def = DEF_NOT_SUBREG;
4992 } else {
4993
4994 if (is_pointer_value(env, insn->src_reg)) {
4995 verbose(env,
4996 "R%d partial copy of pointer\n",
4997 insn->src_reg);
4998 return -EACCES;
4999 } else if (src_reg->type == SCALAR_VALUE) {
5000 *dst_reg = *src_reg;
5001 dst_reg->live |= REG_LIVE_WRITTEN;
5002 dst_reg->subreg_def = env->insn_idx + 1;
5003 } else {
5004 mark_reg_unknown(env, regs,
5005 insn->dst_reg);
5006 }
5007 coerce_reg_to_size(dst_reg, 4);
5008 }
5009 } else {
5010
5011
5012
5013
5014 mark_reg_unknown(env, regs, insn->dst_reg);
5015 regs[insn->dst_reg].type = SCALAR_VALUE;
5016 if (BPF_CLASS(insn->code) == BPF_ALU64) {
5017 __mark_reg_known(regs + insn->dst_reg,
5018 insn->imm);
5019 } else {
5020 __mark_reg_known(regs + insn->dst_reg,
5021 (u32)insn->imm);
5022 }
5023 }
5024
5025 } else if (opcode > BPF_END) {
5026 verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
5027 return -EINVAL;
5028
5029 } else {
5030
5031 if (BPF_SRC(insn->code) == BPF_X) {
5032 if (insn->imm != 0 || insn->off != 0) {
5033 verbose(env, "BPF_ALU uses reserved fields\n");
5034 return -EINVAL;
5035 }
5036
5037 err = check_reg_arg(env, insn->src_reg, SRC_OP);
5038 if (err)
5039 return err;
5040 } else {
5041 if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
5042 verbose(env, "BPF_ALU uses reserved fields\n");
5043 return -EINVAL;
5044 }
5045 }
5046
5047
5048 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
5049 if (err)
5050 return err;
5051
5052 if ((opcode == BPF_MOD || opcode == BPF_DIV) &&
5053 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {
5054 verbose(env, "div by zero\n");
5055 return -EINVAL;
5056 }
5057
5058 if ((opcode == BPF_LSH || opcode == BPF_RSH ||
5059 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
5060 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
5061
5062 if (insn->imm < 0 || insn->imm >= size) {
5063 verbose(env, "invalid shift %d\n", insn->imm);
5064 return -EINVAL;
5065 }
5066 }
5067
5068
5069 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
5070 if (err)
5071 return err;
5072
5073 return adjust_reg_min_max_vals(env, insn);
5074 }
5075
5076 return 0;
5077}
5078
5079static void __find_good_pkt_pointers(struct bpf_func_state *state,
5080 struct bpf_reg_state *dst_reg,
5081 enum bpf_reg_type type, u16 new_range)
5082{
5083 struct bpf_reg_state *reg;
5084 int i;
5085
5086 for (i = 0; i < MAX_BPF_REG; i++) {
5087 reg = &state->regs[i];
5088 if (reg->type == type && reg->id == dst_reg->id)
5089
5090 reg->range = max(reg->range, new_range);
5091 }
5092
5093 bpf_for_each_spilled_reg(i, state, reg) {
5094 if (!reg)
5095 continue;
5096 if (reg->type == type && reg->id == dst_reg->id)
5097 reg->range = max(reg->range, new_range);
5098 }
5099}
5100
5101static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
5102 struct bpf_reg_state *dst_reg,
5103 enum bpf_reg_type type,
5104 bool range_right_open)
5105{
5106 u16 new_range;
5107 int i;
5108
5109 if (dst_reg->off < 0 ||
5110 (dst_reg->off == 0 && range_right_open))
5111
5112 return;
5113
5114 if (dst_reg->umax_value > MAX_PACKET_OFF ||
5115 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF)
5116
5117
5118
5119 return;
5120
5121 new_range = dst_reg->off;
5122 if (range_right_open)
5123 new_range--;
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172 for (i = 0; i <= vstate->curframe; i++)
5173 __find_good_pkt_pointers(vstate->frame[i], dst_reg, type,
5174 new_range);
5175}
5176
5177
5178
5179
5180
5181
5182
5183static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,
5184 bool is_jmp32)
5185{
5186 struct bpf_reg_state reg_lo;
5187 s64 sval;
5188
5189 if (__is_pointer_value(false, reg))
5190 return -1;
5191
5192 if (is_jmp32) {
5193 reg_lo = *reg;
5194 reg = ®_lo;
5195
5196
5197
5198
5199 coerce_reg_to_size(reg, 4);
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213 if ((reg->umax_value ^ reg->umin_value) &
5214 (1ULL << 31)) {
5215 reg->smin_value = S32_MIN;
5216 reg->smax_value = S32_MAX;
5217 }
5218 reg->smin_value = (s64)(s32)reg->smin_value;
5219 reg->smax_value = (s64)(s32)reg->smax_value;
5220
5221 val = (u32)val;
5222 sval = (s64)(s32)val;
5223 } else {
5224 sval = (s64)val;
5225 }
5226
5227 switch (opcode) {
5228 case BPF_JEQ:
5229 if (tnum_is_const(reg->var_off))
5230 return !!tnum_equals_const(reg->var_off, val);
5231 break;
5232 case BPF_JNE:
5233 if (tnum_is_const(reg->var_off))
5234 return !tnum_equals_const(reg->var_off, val);
5235 break;
5236 case BPF_JSET:
5237 if ((~reg->var_off.mask & reg->var_off.value) & val)
5238 return 1;
5239 if (!((reg->var_off.mask | reg->var_off.value) & val))
5240 return 0;
5241 break;
5242 case BPF_JGT:
5243 if (reg->umin_value > val)
5244 return 1;
5245 else if (reg->umax_value <= val)
5246 return 0;
5247 break;
5248 case BPF_JSGT:
5249 if (reg->smin_value > sval)
5250 return 1;
5251 else if (reg->smax_value < sval)
5252 return 0;
5253 break;
5254 case BPF_JLT:
5255 if (reg->umax_value < val)
5256 return 1;
5257 else if (reg->umin_value >= val)
5258 return 0;
5259 break;
5260 case BPF_JSLT:
5261 if (reg->smax_value < sval)
5262 return 1;
5263 else if (reg->smin_value >= sval)
5264 return 0;
5265 break;
5266 case BPF_JGE:
5267 if (reg->umin_value >= val)
5268 return 1;
5269 else if (reg->umax_value < val)
5270 return 0;
5271 break;
5272 case BPF_JSGE:
5273 if (reg->smin_value >= sval)
5274 return 1;
5275 else if (reg->smax_value < sval)
5276 return 0;
5277 break;
5278 case BPF_JLE:
5279 if (reg->umax_value <= val)
5280 return 1;
5281 else if (reg->umin_value > val)
5282 return 0;
5283 break;
5284 case BPF_JSLE:
5285 if (reg->smax_value <= sval)
5286 return 1;
5287 else if (reg->smin_value > sval)
5288 return 0;
5289 break;
5290 }
5291
5292 return -1;
5293}
5294
5295
5296static u64 gen_hi_min(struct tnum var)
5297{
5298 return var.value & ~0xffffffffULL;
5299}
5300
5301
5302static u64 gen_hi_max(struct tnum var)
5303{
5304 return (var.value | var.mask) & ~0xffffffffULL;
5305}
5306
5307
5308
5309
5310static bool cmp_val_with_extended_s64(s64 sval, struct bpf_reg_state *reg)
5311{
5312 return ((s32)sval >= 0 &&
5313 reg->smin_value >= 0 && reg->smax_value <= S32_MAX) ||
5314 ((s32)sval < 0 &&
5315 reg->smax_value <= 0 && reg->smin_value >= S32_MIN);
5316}
5317
5318
5319
5320
5321
5322
5323static void reg_set_min_max(struct bpf_reg_state *true_reg,
5324 struct bpf_reg_state *false_reg, u64 val,
5325 u8 opcode, bool is_jmp32)
5326{
5327 s64 sval;
5328
5329
5330
5331
5332
5333
5334
5335 if (__is_pointer_value(false, false_reg))
5336 return;
5337
5338 val = is_jmp32 ? (u32)val : val;
5339 sval = is_jmp32 ? (s64)(s32)val : (s64)val;
5340
5341 switch (opcode) {
5342 case BPF_JEQ:
5343 case BPF_JNE:
5344 {
5345 struct bpf_reg_state *reg =
5346 opcode == BPF_JEQ ? true_reg : false_reg;
5347
5348
5349
5350
5351
5352 if (is_jmp32) {
5353 u64 old_v = reg->var_off.value;
5354 u64 hi_mask = ~0xffffffffULL;
5355
5356 reg->var_off.value = (old_v & hi_mask) | val;
5357 reg->var_off.mask &= hi_mask;
5358 } else {
5359 __mark_reg_known(reg, val);
5360 }
5361 break;
5362 }
5363 case BPF_JSET:
5364 false_reg->var_off = tnum_and(false_reg->var_off,
5365 tnum_const(~val));
5366 if (is_power_of_2(val))
5367 true_reg->var_off = tnum_or(true_reg->var_off,
5368 tnum_const(val));
5369 break;
5370 case BPF_JGE:
5371 case BPF_JGT:
5372 {
5373 u64 false_umax = opcode == BPF_JGT ? val : val - 1;
5374 u64 true_umin = opcode == BPF_JGT ? val + 1 : val;
5375
5376 if (is_jmp32) {
5377 false_umax += gen_hi_max(false_reg->var_off);
5378 true_umin += gen_hi_min(true_reg->var_off);
5379 }
5380 false_reg->umax_value = min(false_reg->umax_value, false_umax);
5381 true_reg->umin_value = max(true_reg->umin_value, true_umin);
5382 break;
5383 }
5384 case BPF_JSGE:
5385 case BPF_JSGT:
5386 {
5387 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1;
5388 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval;
5389
5390
5391
5392
5393 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
5394 break;
5395 false_reg->smax_value = min(false_reg->smax_value, false_smax);
5396 true_reg->smin_value = max(true_reg->smin_value, true_smin);
5397 break;
5398 }
5399 case BPF_JLE:
5400 case BPF_JLT:
5401 {
5402 u64 false_umin = opcode == BPF_JLT ? val : val + 1;
5403 u64 true_umax = opcode == BPF_JLT ? val - 1 : val;
5404
5405 if (is_jmp32) {
5406 false_umin += gen_hi_min(false_reg->var_off);
5407 true_umax += gen_hi_max(true_reg->var_off);
5408 }
5409 false_reg->umin_value = max(false_reg->umin_value, false_umin);
5410 true_reg->umax_value = min(true_reg->umax_value, true_umax);
5411 break;
5412 }
5413 case BPF_JSLE:
5414 case BPF_JSLT:
5415 {
5416 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1;
5417 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval;
5418
5419 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
5420 break;
5421 false_reg->smin_value = max(false_reg->smin_value, false_smin);
5422 true_reg->smax_value = min(true_reg->smax_value, true_smax);
5423 break;
5424 }
5425 default:
5426 break;
5427 }
5428
5429 __reg_deduce_bounds(false_reg);
5430 __reg_deduce_bounds(true_reg);
5431
5432 __reg_bound_offset(false_reg);
5433 __reg_bound_offset(true_reg);
5434
5435
5436
5437
5438 __update_reg_bounds(false_reg);
5439 __update_reg_bounds(true_reg);
5440}
5441
5442
5443
5444
5445static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,
5446 struct bpf_reg_state *false_reg, u64 val,
5447 u8 opcode, bool is_jmp32)
5448{
5449 s64 sval;
5450
5451 if (__is_pointer_value(false, false_reg))
5452 return;
5453
5454 val = is_jmp32 ? (u32)val : val;
5455 sval = is_jmp32 ? (s64)(s32)val : (s64)val;
5456
5457 switch (opcode) {
5458 case BPF_JEQ:
5459 case BPF_JNE:
5460 {
5461 struct bpf_reg_state *reg =
5462 opcode == BPF_JEQ ? true_reg : false_reg;
5463
5464 if (is_jmp32) {
5465 u64 old_v = reg->var_off.value;
5466 u64 hi_mask = ~0xffffffffULL;
5467
5468 reg->var_off.value = (old_v & hi_mask) | val;
5469 reg->var_off.mask &= hi_mask;
5470 } else {
5471 __mark_reg_known(reg, val);
5472 }
5473 break;
5474 }
5475 case BPF_JSET:
5476 false_reg->var_off = tnum_and(false_reg->var_off,
5477 tnum_const(~val));
5478 if (is_power_of_2(val))
5479 true_reg->var_off = tnum_or(true_reg->var_off,
5480 tnum_const(val));
5481 break;
5482 case BPF_JGE:
5483 case BPF_JGT:
5484 {
5485 u64 false_umin = opcode == BPF_JGT ? val : val + 1;
5486 u64 true_umax = opcode == BPF_JGT ? val - 1 : val;
5487
5488 if (is_jmp32) {
5489 false_umin += gen_hi_min(false_reg->var_off);
5490 true_umax += gen_hi_max(true_reg->var_off);
5491 }
5492 false_reg->umin_value = max(false_reg->umin_value, false_umin);
5493 true_reg->umax_value = min(true_reg->umax_value, true_umax);
5494 break;
5495 }
5496 case BPF_JSGE:
5497 case BPF_JSGT:
5498 {
5499 s64 false_smin = opcode == BPF_JSGT ? sval : sval + 1;
5500 s64 true_smax = opcode == BPF_JSGT ? sval - 1 : sval;
5501
5502 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
5503 break;
5504 false_reg->smin_value = max(false_reg->smin_value, false_smin);
5505 true_reg->smax_value = min(true_reg->smax_value, true_smax);
5506 break;
5507 }
5508 case BPF_JLE:
5509 case BPF_JLT:
5510 {
5511 u64 false_umax = opcode == BPF_JLT ? val : val - 1;
5512 u64 true_umin = opcode == BPF_JLT ? val + 1 : val;
5513
5514 if (is_jmp32) {
5515 false_umax += gen_hi_max(false_reg->var_off);
5516 true_umin += gen_hi_min(true_reg->var_off);
5517 }
5518 false_reg->umax_value = min(false_reg->umax_value, false_umax);
5519 true_reg->umin_value = max(true_reg->umin_value, true_umin);
5520 break;
5521 }
5522 case BPF_JSLE:
5523 case BPF_JSLT:
5524 {
5525 s64 false_smax = opcode == BPF_JSLT ? sval : sval - 1;
5526 s64 true_smin = opcode == BPF_JSLT ? sval + 1 : sval;
5527
5528 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
5529 break;
5530 false_reg->smax_value = min(false_reg->smax_value, false_smax);
5531 true_reg->smin_value = max(true_reg->smin_value, true_smin);
5532 break;
5533 }
5534 default:
5535 break;
5536 }
5537
5538 __reg_deduce_bounds(false_reg);
5539 __reg_deduce_bounds(true_reg);
5540
5541 __reg_bound_offset(false_reg);
5542 __reg_bound_offset(true_reg);
5543
5544
5545
5546
5547 __update_reg_bounds(false_reg);
5548 __update_reg_bounds(true_reg);
5549}
5550
5551
5552static void __reg_combine_min_max(struct bpf_reg_state *src_reg,
5553 struct bpf_reg_state *dst_reg)
5554{
5555 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value,
5556 dst_reg->umin_value);
5557 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value,
5558 dst_reg->umax_value);
5559 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value,
5560 dst_reg->smin_value);
5561 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value,
5562 dst_reg->smax_value);
5563 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off,
5564 dst_reg->var_off);
5565
5566 __update_reg_bounds(src_reg);
5567 __update_reg_bounds(dst_reg);
5568
5569 __reg_deduce_bounds(src_reg);
5570 __reg_deduce_bounds(dst_reg);
5571
5572 __reg_bound_offset(src_reg);
5573 __reg_bound_offset(dst_reg);
5574
5575
5576
5577
5578 __update_reg_bounds(src_reg);
5579 __update_reg_bounds(dst_reg);
5580}
5581
5582static void reg_combine_min_max(struct bpf_reg_state *true_src,
5583 struct bpf_reg_state *true_dst,
5584 struct bpf_reg_state *false_src,
5585 struct bpf_reg_state *false_dst,
5586 u8 opcode)
5587{
5588 switch (opcode) {
5589 case BPF_JEQ:
5590 __reg_combine_min_max(true_src, true_dst);
5591 break;
5592 case BPF_JNE:
5593 __reg_combine_min_max(false_src, false_dst);
5594 break;
5595 }
5596}
5597
5598static void mark_ptr_or_null_reg(struct bpf_func_state *state,
5599 struct bpf_reg_state *reg, u32 id,
5600 bool is_null)
5601{
5602 if (reg_type_may_be_null(reg->type) && reg->id == id) {
5603
5604
5605
5606
5607 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value ||
5608 !tnum_equals_const(reg->var_off, 0) ||
5609 reg->off)) {
5610 __mark_reg_known_zero(reg);
5611 reg->off = 0;
5612 }
5613 if (is_null) {
5614 reg->type = SCALAR_VALUE;
5615 } else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) {
5616 if (reg->map_ptr->inner_map_meta) {
5617 reg->type = CONST_PTR_TO_MAP;
5618 reg->map_ptr = reg->map_ptr->inner_map_meta;
5619 } else if (reg->map_ptr->map_type ==
5620 BPF_MAP_TYPE_XSKMAP) {
5621 reg->type = PTR_TO_XDP_SOCK;
5622 } else {
5623 reg->type = PTR_TO_MAP_VALUE;
5624 }
5625 } else if (reg->type == PTR_TO_SOCKET_OR_NULL) {
5626 reg->type = PTR_TO_SOCKET;
5627 } else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) {
5628 reg->type = PTR_TO_SOCK_COMMON;
5629 } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) {
5630 reg->type = PTR_TO_TCP_SOCK;
5631 }
5632 if (is_null) {
5633
5634
5635
5636
5637 reg->id = 0;
5638 reg->ref_obj_id = 0;
5639 } else if (!reg_may_point_to_spin_lock(reg)) {
5640
5641
5642
5643
5644
5645
5646 reg->id = 0;
5647 }
5648 }
5649}
5650
5651static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id,
5652 bool is_null)
5653{
5654 struct bpf_reg_state *reg;
5655 int i;
5656
5657 for (i = 0; i < MAX_BPF_REG; i++)
5658 mark_ptr_or_null_reg(state, &state->regs[i], id, is_null);
5659
5660 bpf_for_each_spilled_reg(i, state, reg) {
5661 if (!reg)
5662 continue;
5663 mark_ptr_or_null_reg(state, reg, id, is_null);
5664 }
5665}
5666
5667
5668
5669
5670static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
5671 bool is_null)
5672{
5673 struct bpf_func_state *state = vstate->frame[vstate->curframe];
5674 struct bpf_reg_state *regs = state->regs;
5675 u32 ref_obj_id = regs[regno].ref_obj_id;
5676 u32 id = regs[regno].id;
5677 int i;
5678
5679 if (ref_obj_id && ref_obj_id == id && is_null)
5680
5681
5682
5683
5684 WARN_ON_ONCE(release_reference_state(state, id));
5685
5686 for (i = 0; i <= vstate->curframe; i++)
5687 __mark_ptr_or_null_regs(vstate->frame[i], id, is_null);
5688}
5689
5690static bool try_match_pkt_pointers(const struct bpf_insn *insn,
5691 struct bpf_reg_state *dst_reg,
5692 struct bpf_reg_state *src_reg,
5693 struct bpf_verifier_state *this_branch,
5694 struct bpf_verifier_state *other_branch)
5695{
5696 if (BPF_SRC(insn->code) != BPF_X)
5697 return false;
5698
5699
5700 if (BPF_CLASS(insn->code) == BPF_JMP32)
5701 return false;
5702
5703 switch (BPF_OP(insn->code)) {
5704 case BPF_JGT:
5705 if ((dst_reg->type == PTR_TO_PACKET &&
5706 src_reg->type == PTR_TO_PACKET_END) ||
5707 (dst_reg->type == PTR_TO_PACKET_META &&
5708 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
5709
5710 find_good_pkt_pointers(this_branch, dst_reg,
5711 dst_reg->type, false);
5712 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
5713 src_reg->type == PTR_TO_PACKET) ||
5714 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
5715 src_reg->type == PTR_TO_PACKET_META)) {
5716
5717 find_good_pkt_pointers(other_branch, src_reg,
5718 src_reg->type, true);
5719 } else {
5720 return false;
5721 }
5722 break;
5723 case BPF_JLT:
5724 if ((dst_reg->type == PTR_TO_PACKET &&
5725 src_reg->type == PTR_TO_PACKET_END) ||
5726 (dst_reg->type == PTR_TO_PACKET_META &&
5727 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
5728
5729 find_good_pkt_pointers(other_branch, dst_reg,
5730 dst_reg->type, true);
5731 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
5732 src_reg->type == PTR_TO_PACKET) ||
5733 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
5734 src_reg->type == PTR_TO_PACKET_META)) {
5735
5736 find_good_pkt_pointers(this_branch, src_reg,
5737 src_reg->type, false);
5738 } else {
5739 return false;
5740 }
5741 break;
5742 case BPF_JGE:
5743 if ((dst_reg->type == PTR_TO_PACKET &&
5744 src_reg->type == PTR_TO_PACKET_END) ||
5745 (dst_reg->type == PTR_TO_PACKET_META &&
5746 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
5747
5748 find_good_pkt_pointers(this_branch, dst_reg,
5749 dst_reg->type, true);
5750 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
5751 src_reg->type == PTR_TO_PACKET) ||
5752 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
5753 src_reg->type == PTR_TO_PACKET_META)) {
5754
5755 find_good_pkt_pointers(other_branch, src_reg,
5756 src_reg->type, false);
5757 } else {
5758 return false;
5759 }
5760 break;
5761 case BPF_JLE:
5762 if ((dst_reg->type == PTR_TO_PACKET &&
5763 src_reg->type == PTR_TO_PACKET_END) ||
5764 (dst_reg->type == PTR_TO_PACKET_META &&
5765 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
5766
5767 find_good_pkt_pointers(other_branch, dst_reg,
5768 dst_reg->type, false);
5769 } else if ((dst_reg->type == PTR_TO_PACKET_END &&
5770 src_reg->type == PTR_TO_PACKET) ||
5771 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
5772 src_reg->type == PTR_TO_PACKET_META)) {
5773
5774 find_good_pkt_pointers(this_branch, src_reg,
5775 src_reg->type, true);
5776 } else {
5777 return false;
5778 }
5779 break;
5780 default:
5781 return false;
5782 }
5783
5784 return true;
5785}
5786
5787static int check_cond_jmp_op(struct bpf_verifier_env *env,
5788 struct bpf_insn *insn, int *insn_idx)
5789{
5790 struct bpf_verifier_state *this_branch = env->cur_state;
5791 struct bpf_verifier_state *other_branch;
5792 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
5793 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;
5794 u8 opcode = BPF_OP(insn->code);
5795 bool is_jmp32;
5796 int pred = -1;
5797 int err;
5798
5799
5800 if (opcode == BPF_JA || opcode > BPF_JSLE) {
5801 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
5802 return -EINVAL;
5803 }
5804
5805 if (BPF_SRC(insn->code) == BPF_X) {
5806 if (insn->imm != 0) {
5807 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
5808 return -EINVAL;
5809 }
5810
5811
5812 err = check_reg_arg(env, insn->src_reg, SRC_OP);
5813 if (err)
5814 return err;
5815
5816 if (is_pointer_value(env, insn->src_reg)) {
5817 verbose(env, "R%d pointer comparison prohibited\n",
5818 insn->src_reg);
5819 return -EACCES;
5820 }
5821 src_reg = ®s[insn->src_reg];
5822 } else {
5823 if (insn->src_reg != BPF_REG_0) {
5824 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
5825 return -EINVAL;
5826 }
5827 }
5828
5829
5830 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
5831 if (err)
5832 return err;
5833
5834 dst_reg = ®s[insn->dst_reg];
5835 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
5836
5837 if (BPF_SRC(insn->code) == BPF_K)
5838 pred = is_branch_taken(dst_reg, insn->imm,
5839 opcode, is_jmp32);
5840 else if (src_reg->type == SCALAR_VALUE &&
5841 tnum_is_const(src_reg->var_off))
5842 pred = is_branch_taken(dst_reg, src_reg->var_off.value,
5843 opcode, is_jmp32);
5844 if (pred >= 0) {
5845 err = mark_chain_precision(env, insn->dst_reg);
5846 if (BPF_SRC(insn->code) == BPF_X && !err)
5847 err = mark_chain_precision(env, insn->src_reg);
5848 if (err)
5849 return err;
5850 }
5851 if (pred == 1) {
5852
5853 *insn_idx += insn->off;
5854 return 0;
5855 } else if (pred == 0) {
5856
5857
5858
5859 return 0;
5860 }
5861
5862 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,
5863 false);
5864 if (!other_branch)
5865 return -EFAULT;
5866 other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
5867
5868
5869
5870
5871
5872
5873
5874
5875 if (BPF_SRC(insn->code) == BPF_X) {
5876 struct bpf_reg_state *src_reg = ®s[insn->src_reg];
5877 struct bpf_reg_state lo_reg0 = *dst_reg;
5878 struct bpf_reg_state lo_reg1 = *src_reg;
5879 struct bpf_reg_state *src_lo, *dst_lo;
5880
5881 dst_lo = &lo_reg0;
5882 src_lo = &lo_reg1;
5883 coerce_reg_to_size(dst_lo, 4);
5884 coerce_reg_to_size(src_lo, 4);
5885
5886 if (dst_reg->type == SCALAR_VALUE &&
5887 src_reg->type == SCALAR_VALUE) {
5888 if (tnum_is_const(src_reg->var_off) ||
5889 (is_jmp32 && tnum_is_const(src_lo->var_off)))
5890 reg_set_min_max(&other_branch_regs[insn->dst_reg],
5891 dst_reg,
5892 is_jmp32
5893 ? src_lo->var_off.value
5894 : src_reg->var_off.value,
5895 opcode, is_jmp32);
5896 else if (tnum_is_const(dst_reg->var_off) ||
5897 (is_jmp32 && tnum_is_const(dst_lo->var_off)))
5898 reg_set_min_max_inv(&other_branch_regs[insn->src_reg],
5899 src_reg,
5900 is_jmp32
5901 ? dst_lo->var_off.value
5902 : dst_reg->var_off.value,
5903 opcode, is_jmp32);
5904 else if (!is_jmp32 &&
5905 (opcode == BPF_JEQ || opcode == BPF_JNE))
5906
5907 reg_combine_min_max(&other_branch_regs[insn->src_reg],
5908 &other_branch_regs[insn->dst_reg],
5909 src_reg, dst_reg, opcode);
5910 }
5911 } else if (dst_reg->type == SCALAR_VALUE) {
5912 reg_set_min_max(&other_branch_regs[insn->dst_reg],
5913 dst_reg, insn->imm, opcode, is_jmp32);
5914 }
5915
5916
5917
5918
5919
5920 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
5921 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
5922 reg_type_may_be_null(dst_reg->type)) {
5923
5924
5925
5926 mark_ptr_or_null_regs(this_branch, insn->dst_reg,
5927 opcode == BPF_JNE);
5928 mark_ptr_or_null_regs(other_branch, insn->dst_reg,
5929 opcode == BPF_JEQ);
5930 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg],
5931 this_branch, other_branch) &&
5932 is_pointer_value(env, insn->dst_reg)) {
5933 verbose(env, "R%d pointer comparison prohibited\n",
5934 insn->dst_reg);
5935 return -EACCES;
5936 }
5937 if (env->log.level & BPF_LOG_LEVEL)
5938 print_verifier_state(env, this_branch->frame[this_branch->curframe]);
5939 return 0;
5940}
5941
5942
5943static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
5944{
5945 struct bpf_insn_aux_data *aux = cur_aux(env);
5946 struct bpf_reg_state *regs = cur_regs(env);
5947 struct bpf_map *map;
5948 int err;
5949
5950 if (BPF_SIZE(insn->code) != BPF_DW) {
5951 verbose(env, "invalid BPF_LD_IMM insn\n");
5952 return -EINVAL;
5953 }
5954 if (insn->off != 0) {
5955 verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
5956 return -EINVAL;
5957 }
5958
5959 err = check_reg_arg(env, insn->dst_reg, DST_OP);
5960 if (err)
5961 return err;
5962
5963 if (insn->src_reg == 0) {
5964 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
5965
5966 regs[insn->dst_reg].type = SCALAR_VALUE;
5967 __mark_reg_known(®s[insn->dst_reg], imm);
5968 return 0;
5969 }
5970
5971 map = env->used_maps[aux->map_index];
5972 mark_reg_known_zero(env, regs, insn->dst_reg);
5973 regs[insn->dst_reg].map_ptr = map;
5974
5975 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) {
5976 regs[insn->dst_reg].type = PTR_TO_MAP_VALUE;
5977 regs[insn->dst_reg].off = aux->map_off;
5978 if (map_value_has_spin_lock(map))
5979 regs[insn->dst_reg].id = ++env->id_gen;
5980 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD) {
5981 regs[insn->dst_reg].type = CONST_PTR_TO_MAP;
5982 } else {
5983 verbose(env, "bpf verifier is misconfigured\n");
5984 return -EINVAL;
5985 }
5986
5987 return 0;
5988}
5989
5990static bool may_access_skb(enum bpf_prog_type type)
5991{
5992 switch (type) {
5993 case BPF_PROG_TYPE_SOCKET_FILTER:
5994 case BPF_PROG_TYPE_SCHED_CLS:
5995 case BPF_PROG_TYPE_SCHED_ACT:
5996 return true;
5997 default:
5998 return false;
5999 }
6000}
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
6018{
6019 struct bpf_reg_state *regs = cur_regs(env);
6020 u8 mode = BPF_MODE(insn->code);
6021 int i, err;
6022
6023 if (!may_access_skb(env->prog->type)) {
6024 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
6025 return -EINVAL;
6026 }
6027
6028 if (!env->ops->gen_ld_abs) {
6029 verbose(env, "bpf verifier is misconfigured\n");
6030 return -EINVAL;
6031 }
6032
6033 if (env->subprog_cnt > 1) {
6034
6035
6036
6037
6038
6039
6040
6041 verbose(env, "BPF_LD_[ABS|IND] instructions cannot be mixed with bpf-to-bpf calls\n");
6042 return -EINVAL;
6043 }
6044
6045 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||
6046 BPF_SIZE(insn->code) == BPF_DW ||
6047 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {
6048 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
6049 return -EINVAL;
6050 }
6051
6052
6053 err = check_reg_arg(env, BPF_REG_6, SRC_OP);
6054 if (err)
6055 return err;
6056
6057
6058
6059
6060
6061 err = check_reference_leak(env);
6062 if (err) {
6063 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");
6064 return err;
6065 }
6066
6067 if (env->cur_state->active_spin_lock) {
6068 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");
6069 return -EINVAL;
6070 }
6071
6072 if (regs[BPF_REG_6].type != PTR_TO_CTX) {
6073 verbose(env,
6074 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
6075 return -EINVAL;
6076 }
6077
6078 if (mode == BPF_IND) {
6079
6080 err = check_reg_arg(env, insn->src_reg, SRC_OP);
6081 if (err)
6082 return err;
6083 }
6084
6085
6086 for (i = 0; i < CALLER_SAVED_REGS; i++) {
6087 mark_reg_not_init(env, regs, caller_saved[i]);
6088 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
6089 }
6090
6091
6092
6093
6094
6095 mark_reg_unknown(env, regs, BPF_REG_0);
6096
6097 regs[BPF_REG_0].subreg_def = env->insn_idx + 1;
6098 return 0;
6099}
6100
6101static int check_return_code(struct bpf_verifier_env *env)
6102{
6103 struct tnum enforce_attach_type_range = tnum_unknown;
6104 struct bpf_reg_state *reg;
6105 struct tnum range = tnum_range(0, 1);
6106
6107 switch (env->prog->type) {
6108 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
6109 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||
6110 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG)
6111 range = tnum_range(1, 1);
6112 break;
6113 case BPF_PROG_TYPE_CGROUP_SKB:
6114 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
6115 range = tnum_range(0, 3);
6116 enforce_attach_type_range = tnum_range(2, 3);
6117 }
6118 break;
6119 case BPF_PROG_TYPE_CGROUP_SOCK:
6120 case BPF_PROG_TYPE_SOCK_OPS:
6121 case BPF_PROG_TYPE_CGROUP_DEVICE:
6122 case BPF_PROG_TYPE_CGROUP_SYSCTL:
6123 case BPF_PROG_TYPE_CGROUP_SOCKOPT:
6124 break;
6125 default:
6126 return 0;
6127 }
6128
6129 reg = cur_regs(env) + BPF_REG_0;
6130 if (reg->type != SCALAR_VALUE) {
6131 verbose(env, "At program exit the register R0 is not a known value (%s)\n",
6132 reg_type_str[reg->type]);
6133 return -EINVAL;
6134 }
6135
6136 if (!tnum_in(range, reg->var_off)) {
6137 char tn_buf[48];
6138
6139 verbose(env, "At program exit the register R0 ");
6140 if (!tnum_is_unknown(reg->var_off)) {
6141 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
6142 verbose(env, "has value %s", tn_buf);
6143 } else {
6144 verbose(env, "has unknown scalar value");
6145 }
6146 tnum_strn(tn_buf, sizeof(tn_buf), range);
6147 verbose(env, " should have been in %s\n", tn_buf);
6148 return -EINVAL;
6149 }
6150
6151 if (!tnum_is_unknown(enforce_attach_type_range) &&
6152 tnum_in(enforce_attach_type_range, reg->var_off))
6153 env->prog->enforce_expected_attach_type = 1;
6154 return 0;
6155}
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190enum {
6191 DISCOVERED = 0x10,
6192 EXPLORED = 0x20,
6193 FALLTHROUGH = 1,
6194 BRANCH = 2,
6195};
6196
6197static u32 state_htab_size(struct bpf_verifier_env *env)
6198{
6199 return env->prog->len;
6200}
6201
6202static struct bpf_verifier_state_list **explored_state(
6203 struct bpf_verifier_env *env,
6204 int idx)
6205{
6206 struct bpf_verifier_state *cur = env->cur_state;
6207 struct bpf_func_state *state = cur->frame[cur->curframe];
6208
6209 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];
6210}
6211
6212static void init_explored_state(struct bpf_verifier_env *env, int idx)
6213{
6214 env->insn_aux_data[idx].prune_point = true;
6215}
6216
6217
6218
6219
6220
6221
6222static int push_insn(int t, int w, int e, struct bpf_verifier_env *env,
6223 bool loop_ok)
6224{
6225 int *insn_stack = env->cfg.insn_stack;
6226 int *insn_state = env->cfg.insn_state;
6227
6228 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
6229 return 0;
6230
6231 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
6232 return 0;
6233
6234 if (w < 0 || w >= env->prog->len) {
6235 verbose_linfo(env, t, "%d: ", t);
6236 verbose(env, "jump out of range from insn %d to %d\n", t, w);
6237 return -EINVAL;
6238 }
6239
6240 if (e == BRANCH)
6241
6242 init_explored_state(env, w);
6243
6244 if (insn_state[w] == 0) {
6245
6246 insn_state[t] = DISCOVERED | e;
6247 insn_state[w] = DISCOVERED;
6248 if (env->cfg.cur_stack >= env->prog->len)
6249 return -E2BIG;
6250 insn_stack[env->cfg.cur_stack++] = w;
6251 return 1;
6252 } else if ((insn_state[w] & 0xF0) == DISCOVERED) {
6253 if (loop_ok && env->allow_ptr_leaks)
6254 return 0;
6255 verbose_linfo(env, t, "%d: ", t);
6256 verbose_linfo(env, w, "%d: ", w);
6257 verbose(env, "back-edge from insn %d to %d\n", t, w);
6258 return -EINVAL;
6259 } else if (insn_state[w] == EXPLORED) {
6260
6261 insn_state[t] = DISCOVERED | e;
6262 } else {
6263 verbose(env, "insn state internal bug\n");
6264 return -EFAULT;
6265 }
6266 return 0;
6267}
6268
6269
6270
6271
6272static int check_cfg(struct bpf_verifier_env *env)
6273{
6274 struct bpf_insn *insns = env->prog->insnsi;
6275 int insn_cnt = env->prog->len;
6276 int *insn_stack, *insn_state;
6277 int ret = 0;
6278 int i, t;
6279
6280 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
6281 if (!insn_state)
6282 return -ENOMEM;
6283
6284 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
6285 if (!insn_stack) {
6286 kvfree(insn_state);
6287 return -ENOMEM;
6288 }
6289
6290 insn_state[0] = DISCOVERED;
6291 insn_stack[0] = 0;
6292 env->cfg.cur_stack = 1;
6293
6294peek_stack:
6295 if (env->cfg.cur_stack == 0)
6296 goto check_state;
6297 t = insn_stack[env->cfg.cur_stack - 1];
6298
6299 if (BPF_CLASS(insns[t].code) == BPF_JMP ||
6300 BPF_CLASS(insns[t].code) == BPF_JMP32) {
6301 u8 opcode = BPF_OP(insns[t].code);
6302
6303 if (opcode == BPF_EXIT) {
6304 goto mark_explored;
6305 } else if (opcode == BPF_CALL) {
6306 ret = push_insn(t, t + 1, FALLTHROUGH, env, false);
6307 if (ret == 1)
6308 goto peek_stack;
6309 else if (ret < 0)
6310 goto err_free;
6311 if (t + 1 < insn_cnt)
6312 init_explored_state(env, t + 1);
6313 if (insns[t].src_reg == BPF_PSEUDO_CALL) {
6314 init_explored_state(env, t);
6315 ret = push_insn(t, t + insns[t].imm + 1, BRANCH,
6316 env, false);
6317 if (ret == 1)
6318 goto peek_stack;
6319 else if (ret < 0)
6320 goto err_free;
6321 }
6322 } else if (opcode == BPF_JA) {
6323 if (BPF_SRC(insns[t].code) != BPF_K) {
6324 ret = -EINVAL;
6325 goto err_free;
6326 }
6327
6328 ret = push_insn(t, t + insns[t].off + 1,
6329 FALLTHROUGH, env, true);
6330 if (ret == 1)
6331 goto peek_stack;
6332 else if (ret < 0)
6333 goto err_free;
6334
6335
6336
6337
6338 init_explored_state(env, t + insns[t].off + 1);
6339
6340
6341
6342 if (t + 1 < insn_cnt)
6343 init_explored_state(env, t + 1);
6344 } else {
6345
6346 init_explored_state(env, t);
6347 ret = push_insn(t, t + 1, FALLTHROUGH, env, true);
6348 if (ret == 1)
6349 goto peek_stack;
6350 else if (ret < 0)
6351 goto err_free;
6352
6353 ret = push_insn(t, t + insns[t].off + 1, BRANCH, env, true);
6354 if (ret == 1)
6355 goto peek_stack;
6356 else if (ret < 0)
6357 goto err_free;
6358 }
6359 } else {
6360
6361
6362
6363 ret = push_insn(t, t + 1, FALLTHROUGH, env, false);
6364 if (ret == 1)
6365 goto peek_stack;
6366 else if (ret < 0)
6367 goto err_free;
6368 }
6369
6370mark_explored:
6371 insn_state[t] = EXPLORED;
6372 if (env->cfg.cur_stack-- <= 0) {
6373 verbose(env, "pop stack internal bug\n");
6374 ret = -EFAULT;
6375 goto err_free;
6376 }
6377 goto peek_stack;
6378
6379check_state:
6380 for (i = 0; i < insn_cnt; i++) {
6381 if (insn_state[i] != EXPLORED) {
6382 verbose(env, "unreachable insn %d\n", i);
6383 ret = -EINVAL;
6384 goto err_free;
6385 }
6386 }
6387 ret = 0;
6388
6389err_free:
6390 kvfree(insn_state);
6391 kvfree(insn_stack);
6392 env->cfg.insn_state = env->cfg.insn_stack = NULL;
6393 return ret;
6394}
6395
6396
6397#define MIN_BPF_FUNCINFO_SIZE 8
6398#define MAX_FUNCINFO_REC_SIZE 252
6399
6400static int check_btf_func(struct bpf_verifier_env *env,
6401 const union bpf_attr *attr,
6402 union bpf_attr __user *uattr)
6403{
6404 u32 i, nfuncs, urec_size, min_size;
6405 u32 krec_size = sizeof(struct bpf_func_info);
6406 struct bpf_func_info *krecord;
6407 const struct btf_type *type;
6408 struct bpf_prog *prog;
6409 const struct btf *btf;
6410 void __user *urecord;
6411 u32 prev_offset = 0;
6412 int ret = 0;
6413
6414 nfuncs = attr->func_info_cnt;
6415 if (!nfuncs)
6416 return 0;
6417
6418 if (nfuncs != env->subprog_cnt) {
6419 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
6420 return -EINVAL;
6421 }
6422
6423 urec_size = attr->func_info_rec_size;
6424 if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
6425 urec_size > MAX_FUNCINFO_REC_SIZE ||
6426 urec_size % sizeof(u32)) {
6427 verbose(env, "invalid func info rec size %u\n", urec_size);
6428 return -EINVAL;
6429 }
6430
6431 prog = env->prog;
6432 btf = prog->aux->btf;
6433
6434 urecord = u64_to_user_ptr(attr->func_info);
6435 min_size = min_t(u32, krec_size, urec_size);
6436
6437 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
6438 if (!krecord)
6439 return -ENOMEM;
6440
6441 for (i = 0; i < nfuncs; i++) {
6442 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
6443 if (ret) {
6444 if (ret == -E2BIG) {
6445 verbose(env, "nonzero tailing record in func info");
6446
6447
6448
6449 if (put_user(min_size, &uattr->func_info_rec_size))
6450 ret = -EFAULT;
6451 }
6452 goto err_free;
6453 }
6454
6455 if (copy_from_user(&krecord[i], urecord, min_size)) {
6456 ret = -EFAULT;
6457 goto err_free;
6458 }
6459
6460
6461 if (i == 0) {
6462 if (krecord[i].insn_off) {
6463 verbose(env,
6464 "nonzero insn_off %u for the first func info record",
6465 krecord[i].insn_off);
6466 ret = -EINVAL;
6467 goto err_free;
6468 }
6469 } else if (krecord[i].insn_off <= prev_offset) {
6470 verbose(env,
6471 "same or smaller insn offset (%u) than previous func info record (%u)",
6472 krecord[i].insn_off, prev_offset);
6473 ret = -EINVAL;
6474 goto err_free;
6475 }
6476
6477 if (env->subprog_info[i].start != krecord[i].insn_off) {
6478 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
6479 ret = -EINVAL;
6480 goto err_free;
6481 }
6482
6483
6484 type = btf_type_by_id(btf, krecord[i].type_id);
6485 if (!type || BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) {
6486 verbose(env, "invalid type id %d in func info",
6487 krecord[i].type_id);
6488 ret = -EINVAL;
6489 goto err_free;
6490 }
6491
6492 prev_offset = krecord[i].insn_off;
6493 urecord += urec_size;
6494 }
6495
6496 prog->aux->func_info = krecord;
6497 prog->aux->func_info_cnt = nfuncs;
6498 return 0;
6499
6500err_free:
6501 kvfree(krecord);
6502 return ret;
6503}
6504
6505static void adjust_btf_func(struct bpf_verifier_env *env)
6506{
6507 int i;
6508
6509 if (!env->prog->aux->func_info)
6510 return;
6511
6512 for (i = 0; i < env->subprog_cnt; i++)
6513 env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start;
6514}
6515
6516#define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \
6517 sizeof(((struct bpf_line_info *)(0))->line_col))
6518#define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE
6519
6520static int check_btf_line(struct bpf_verifier_env *env,
6521 const union bpf_attr *attr,
6522 union bpf_attr __user *uattr)
6523{
6524 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
6525 struct bpf_subprog_info *sub;
6526 struct bpf_line_info *linfo;
6527 struct bpf_prog *prog;
6528 const struct btf *btf;
6529 void __user *ulinfo;
6530 int err;
6531
6532 nr_linfo = attr->line_info_cnt;
6533 if (!nr_linfo)
6534 return 0;
6535
6536 rec_size = attr->line_info_rec_size;
6537 if (rec_size < MIN_BPF_LINEINFO_SIZE ||
6538 rec_size > MAX_LINEINFO_REC_SIZE ||
6539 rec_size & (sizeof(u32) - 1))
6540 return -EINVAL;
6541
6542
6543
6544
6545 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
6546 GFP_KERNEL | __GFP_NOWARN);
6547 if (!linfo)
6548 return -ENOMEM;
6549
6550 prog = env->prog;
6551 btf = prog->aux->btf;
6552
6553 s = 0;
6554 sub = env->subprog_info;
6555 ulinfo = u64_to_user_ptr(attr->line_info);
6556 expected_size = sizeof(struct bpf_line_info);
6557 ncopy = min_t(u32, expected_size, rec_size);
6558 for (i = 0; i < nr_linfo; i++) {
6559 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
6560 if (err) {
6561 if (err == -E2BIG) {
6562 verbose(env, "nonzero tailing record in line_info");
6563 if (put_user(expected_size,
6564 &uattr->line_info_rec_size))
6565 err = -EFAULT;
6566 }
6567 goto err_free;
6568 }
6569
6570 if (copy_from_user(&linfo[i], ulinfo, ncopy)) {
6571 err = -EFAULT;
6572 goto err_free;
6573 }
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586 if ((i && linfo[i].insn_off <= prev_offset) ||
6587 linfo[i].insn_off >= prog->len) {
6588 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
6589 i, linfo[i].insn_off, prev_offset,
6590 prog->len);
6591 err = -EINVAL;
6592 goto err_free;
6593 }
6594
6595 if (!prog->insnsi[linfo[i].insn_off].code) {
6596 verbose(env,
6597 "Invalid insn code at line_info[%u].insn_off\n",
6598 i);
6599 err = -EINVAL;
6600 goto err_free;
6601 }
6602
6603 if (!btf_name_by_offset(btf, linfo[i].line_off) ||
6604 !btf_name_by_offset(btf, linfo[i].file_name_off)) {
6605 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
6606 err = -EINVAL;
6607 goto err_free;
6608 }
6609
6610 if (s != env->subprog_cnt) {
6611 if (linfo[i].insn_off == sub[s].start) {
6612 sub[s].linfo_idx = i;
6613 s++;
6614 } else if (sub[s].start < linfo[i].insn_off) {
6615 verbose(env, "missing bpf_line_info for func#%u\n", s);
6616 err = -EINVAL;
6617 goto err_free;
6618 }
6619 }
6620
6621 prev_offset = linfo[i].insn_off;
6622 ulinfo += rec_size;
6623 }
6624
6625 if (s != env->subprog_cnt) {
6626 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
6627 env->subprog_cnt - s, s);
6628 err = -EINVAL;
6629 goto err_free;
6630 }
6631
6632 prog->aux->linfo = linfo;
6633 prog->aux->nr_linfo = nr_linfo;
6634
6635 return 0;
6636
6637err_free:
6638 kvfree(linfo);
6639 return err;
6640}
6641
6642static int check_btf_info(struct bpf_verifier_env *env,
6643 const union bpf_attr *attr,
6644 union bpf_attr __user *uattr)
6645{
6646 struct btf *btf;
6647 int err;
6648
6649 if (!attr->func_info_cnt && !attr->line_info_cnt)
6650 return 0;
6651
6652 btf = btf_get_by_fd(attr->prog_btf_fd);
6653 if (IS_ERR(btf))
6654 return PTR_ERR(btf);
6655 env->prog->aux->btf = btf;
6656
6657 err = check_btf_func(env, attr, uattr);
6658 if (err)
6659 return err;
6660
6661 err = check_btf_line(env, attr, uattr);
6662 if (err)
6663 return err;
6664
6665 return 0;
6666}
6667
6668
6669static bool range_within(struct bpf_reg_state *old,
6670 struct bpf_reg_state *cur)
6671{
6672 return old->umin_value <= cur->umin_value &&
6673 old->umax_value >= cur->umax_value &&
6674 old->smin_value <= cur->smin_value &&
6675 old->smax_value >= cur->smax_value;
6676}
6677
6678
6679#define ID_MAP_SIZE (MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE)
6680struct idpair {
6681 u32 old;
6682 u32 cur;
6683};
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695static bool check_ids(u32 old_id, u32 cur_id, struct idpair *idmap)
6696{
6697 unsigned int i;
6698
6699 for (i = 0; i < ID_MAP_SIZE; i++) {
6700 if (!idmap[i].old) {
6701
6702 idmap[i].old = old_id;
6703 idmap[i].cur = cur_id;
6704 return true;
6705 }
6706 if (idmap[i].old == old_id)
6707 return idmap[i].cur == cur_id;
6708 }
6709
6710 WARN_ON_ONCE(1);
6711 return false;
6712}
6713
6714static void clean_func_state(struct bpf_verifier_env *env,
6715 struct bpf_func_state *st)
6716{
6717 enum bpf_reg_liveness live;
6718 int i, j;
6719
6720 for (i = 0; i < BPF_REG_FP; i++) {
6721 live = st->regs[i].live;
6722
6723 st->regs[i].live |= REG_LIVE_DONE;
6724 if (!(live & REG_LIVE_READ))
6725
6726
6727
6728 __mark_reg_not_init(&st->regs[i]);
6729 }
6730
6731 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) {
6732 live = st->stack[i].spilled_ptr.live;
6733
6734 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE;
6735 if (!(live & REG_LIVE_READ)) {
6736 __mark_reg_not_init(&st->stack[i].spilled_ptr);
6737 for (j = 0; j < BPF_REG_SIZE; j++)
6738 st->stack[i].slot_type[j] = STACK_INVALID;
6739 }
6740 }
6741}
6742
6743static void clean_verifier_state(struct bpf_verifier_env *env,
6744 struct bpf_verifier_state *st)
6745{
6746 int i;
6747
6748 if (st->frame[0]->regs[0].live & REG_LIVE_DONE)
6749
6750 return;
6751
6752 for (i = 0; i <= st->curframe; i++)
6753 clean_func_state(env, st->frame[i]);
6754}
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788static void clean_live_states(struct bpf_verifier_env *env, int insn,
6789 struct bpf_verifier_state *cur)
6790{
6791 struct bpf_verifier_state_list *sl;
6792 int i;
6793
6794 sl = *explored_state(env, insn);
6795 while (sl) {
6796 if (sl->state.branches)
6797 goto next;
6798 if (sl->state.insn_idx != insn ||
6799 sl->state.curframe != cur->curframe)
6800 goto next;
6801 for (i = 0; i <= cur->curframe; i++)
6802 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite)
6803 goto next;
6804 clean_verifier_state(env, &sl->state);
6805next:
6806 sl = sl->next;
6807 }
6808}
6809
6810
6811static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur,
6812 struct idpair *idmap)
6813{
6814 bool equal;
6815
6816 if (!(rold->live & REG_LIVE_READ))
6817
6818 return true;
6819
6820 equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0;
6821
6822 if (rold->type == PTR_TO_STACK)
6823
6824
6825
6826 return equal && rold->frameno == rcur->frameno;
6827
6828 if (equal)
6829 return true;
6830
6831 if (rold->type == NOT_INIT)
6832
6833 return true;
6834 if (rcur->type == NOT_INIT)
6835 return false;
6836 switch (rold->type) {
6837 case SCALAR_VALUE:
6838 if (rcur->type == SCALAR_VALUE) {
6839 if (!rold->precise && !rcur->precise)
6840 return true;
6841
6842 return range_within(rold, rcur) &&
6843 tnum_in(rold->var_off, rcur->var_off);
6844 } else {
6845
6846
6847
6848
6849
6850
6851
6852 return false;
6853 }
6854 case PTR_TO_MAP_VALUE:
6855
6856
6857
6858
6859
6860
6861
6862
6863 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
6864 range_within(rold, rcur) &&
6865 tnum_in(rold->var_off, rcur->var_off);
6866 case PTR_TO_MAP_VALUE_OR_NULL:
6867
6868
6869
6870
6871
6872
6873
6874 if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL)
6875 return false;
6876 if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
6877 return false;
6878
6879 return check_ids(rold->id, rcur->id, idmap);
6880 case PTR_TO_PACKET_META:
6881 case PTR_TO_PACKET:
6882 if (rcur->type != rold->type)
6883 return false;
6884
6885
6886
6887
6888
6889
6890 if (rold->range > rcur->range)
6891 return false;
6892
6893
6894
6895 if (rold->off != rcur->off)
6896 return false;
6897
6898 if (rold->id && !check_ids(rold->id, rcur->id, idmap))
6899 return false;
6900
6901 return range_within(rold, rcur) &&
6902 tnum_in(rold->var_off, rcur->var_off);
6903 case PTR_TO_CTX:
6904 case CONST_PTR_TO_MAP:
6905 case PTR_TO_PACKET_END:
6906 case PTR_TO_FLOW_KEYS:
6907 case PTR_TO_SOCKET:
6908 case PTR_TO_SOCKET_OR_NULL:
6909 case PTR_TO_SOCK_COMMON:
6910 case PTR_TO_SOCK_COMMON_OR_NULL:
6911 case PTR_TO_TCP_SOCK:
6912 case PTR_TO_TCP_SOCK_OR_NULL:
6913 case PTR_TO_XDP_SOCK:
6914
6915
6916
6917 default:
6918
6919 return false;
6920 }
6921
6922
6923 WARN_ON_ONCE(1);
6924 return false;
6925}
6926
6927static bool stacksafe(struct bpf_func_state *old,
6928 struct bpf_func_state *cur,
6929 struct idpair *idmap)
6930{
6931 int i, spi;
6932
6933
6934
6935
6936
6937 for (i = 0; i < old->allocated_stack; i++) {
6938 spi = i / BPF_REG_SIZE;
6939
6940 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) {
6941 i += BPF_REG_SIZE - 1;
6942
6943 continue;
6944 }
6945
6946 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID)
6947 continue;
6948
6949
6950
6951
6952 if (i >= cur->allocated_stack)
6953 return false;
6954
6955
6956
6957
6958
6959 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC &&
6960 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO)
6961 continue;
6962 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
6963 cur->stack[spi].slot_type[i % BPF_REG_SIZE])
6964
6965
6966
6967
6968
6969 return false;
6970 if (i % BPF_REG_SIZE)
6971 continue;
6972 if (old->stack[spi].slot_type[0] != STACK_SPILL)
6973 continue;
6974 if (!regsafe(&old->stack[spi].spilled_ptr,
6975 &cur->stack[spi].spilled_ptr,
6976 idmap))
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987 return false;
6988 }
6989 return true;
6990}
6991
6992static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur)
6993{
6994 if (old->acquired_refs != cur->acquired_refs)
6995 return false;
6996 return !memcmp(old->refs, cur->refs,
6997 sizeof(*old->refs) * old->acquired_refs);
6998}
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026static bool func_states_equal(struct bpf_func_state *old,
7027 struct bpf_func_state *cur)
7028{
7029 struct idpair *idmap;
7030 bool ret = false;
7031 int i;
7032
7033 idmap = kcalloc(ID_MAP_SIZE, sizeof(struct idpair), GFP_KERNEL);
7034
7035 if (!idmap)
7036 return false;
7037
7038 for (i = 0; i < MAX_BPF_REG; i++) {
7039 if (!regsafe(&old->regs[i], &cur->regs[i], idmap))
7040 goto out_free;
7041 }
7042
7043 if (!stacksafe(old, cur, idmap))
7044 goto out_free;
7045
7046 if (!refsafe(old, cur))
7047 goto out_free;
7048 ret = true;
7049out_free:
7050 kfree(idmap);
7051 return ret;
7052}
7053
7054static bool states_equal(struct bpf_verifier_env *env,
7055 struct bpf_verifier_state *old,
7056 struct bpf_verifier_state *cur)
7057{
7058 int i;
7059
7060 if (old->curframe != cur->curframe)
7061 return false;
7062
7063
7064
7065
7066 if (old->speculative && !cur->speculative)
7067 return false;
7068
7069 if (old->active_spin_lock != cur->active_spin_lock)
7070 return false;
7071
7072
7073
7074
7075 for (i = 0; i <= old->curframe; i++) {
7076 if (old->frame[i]->callsite != cur->frame[i]->callsite)
7077 return false;
7078 if (!func_states_equal(old->frame[i], cur->frame[i]))
7079 return false;
7080 }
7081 return true;
7082}
7083
7084
7085
7086
7087static int propagate_liveness_reg(struct bpf_verifier_env *env,
7088 struct bpf_reg_state *reg,
7089 struct bpf_reg_state *parent_reg)
7090{
7091 u8 parent_flag = parent_reg->live & REG_LIVE_READ;
7092 u8 flag = reg->live & REG_LIVE_READ;
7093 int err;
7094
7095
7096
7097
7098
7099 if (parent_flag == REG_LIVE_READ64 ||
7100
7101 !flag ||
7102
7103 parent_flag == flag)
7104 return 0;
7105
7106 err = mark_reg_read(env, reg, parent_reg, flag);
7107 if (err)
7108 return err;
7109
7110 return flag;
7111}
7112
7113
7114
7115
7116
7117
7118
7119
7120static int propagate_liveness(struct bpf_verifier_env *env,
7121 const struct bpf_verifier_state *vstate,
7122 struct bpf_verifier_state *vparent)
7123{
7124 struct bpf_reg_state *state_reg, *parent_reg;
7125 struct bpf_func_state *state, *parent;
7126 int i, frame, err = 0;
7127
7128 if (vparent->curframe != vstate->curframe) {
7129 WARN(1, "propagate_live: parent frame %d current frame %d\n",
7130 vparent->curframe, vstate->curframe);
7131 return -EFAULT;
7132 }
7133
7134 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
7135 for (frame = 0; frame <= vstate->curframe; frame++) {
7136 parent = vparent->frame[frame];
7137 state = vstate->frame[frame];
7138 parent_reg = parent->regs;
7139 state_reg = state->regs;
7140
7141 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) {
7142 err = propagate_liveness_reg(env, &state_reg[i],
7143 &parent_reg[i]);
7144 if (err < 0)
7145 return err;
7146 if (err == REG_LIVE_READ64)
7147 mark_insn_zext(env, &parent_reg[i]);
7148 }
7149
7150
7151 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE &&
7152 i < parent->allocated_stack / BPF_REG_SIZE; i++) {
7153 parent_reg = &parent->stack[i].spilled_ptr;
7154 state_reg = &state->stack[i].spilled_ptr;
7155 err = propagate_liveness_reg(env, state_reg,
7156 parent_reg);
7157 if (err < 0)
7158 return err;
7159 }
7160 }
7161 return 0;
7162}
7163
7164
7165
7166
7167static int propagate_precision(struct bpf_verifier_env *env,
7168 const struct bpf_verifier_state *old)
7169{
7170 struct bpf_reg_state *state_reg;
7171 struct bpf_func_state *state;
7172 int i, err = 0;
7173
7174 state = old->frame[old->curframe];
7175 state_reg = state->regs;
7176 for (i = 0; i < BPF_REG_FP; i++, state_reg++) {
7177 if (state_reg->type != SCALAR_VALUE ||
7178 !state_reg->precise)
7179 continue;
7180 if (env->log.level & BPF_LOG_LEVEL2)
7181 verbose(env, "propagating r%d\n", i);
7182 err = mark_chain_precision(env, i);
7183 if (err < 0)
7184 return err;
7185 }
7186
7187 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
7188 if (state->stack[i].slot_type[0] != STACK_SPILL)
7189 continue;
7190 state_reg = &state->stack[i].spilled_ptr;
7191 if (state_reg->type != SCALAR_VALUE ||
7192 !state_reg->precise)
7193 continue;
7194 if (env->log.level & BPF_LOG_LEVEL2)
7195 verbose(env, "propagating fp%d\n",
7196 (-i - 1) * BPF_REG_SIZE);
7197 err = mark_chain_precision_stack(env, i);
7198 if (err < 0)
7199 return err;
7200 }
7201 return 0;
7202}
7203
7204static bool states_maybe_looping(struct bpf_verifier_state *old,
7205 struct bpf_verifier_state *cur)
7206{
7207 struct bpf_func_state *fold, *fcur;
7208 int i, fr = cur->curframe;
7209
7210 if (old->curframe != fr)
7211 return false;
7212
7213 fold = old->frame[fr];
7214 fcur = cur->frame[fr];
7215 for (i = 0; i < MAX_BPF_REG; i++)
7216 if (memcmp(&fold->regs[i], &fcur->regs[i],
7217 offsetof(struct bpf_reg_state, parent)))
7218 return false;
7219 return true;
7220}
7221
7222
7223static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
7224{
7225 struct bpf_verifier_state_list *new_sl;
7226 struct bpf_verifier_state_list *sl, **pprev;
7227 struct bpf_verifier_state *cur = env->cur_state, *new;
7228 int i, j, err, states_cnt = 0;
7229 bool add_new_state = false;
7230
7231 cur->last_insn_idx = env->prev_insn_idx;
7232 if (!env->insn_aux_data[insn_idx].prune_point)
7233
7234
7235
7236 return 0;
7237
7238
7239
7240
7241
7242
7243
7244
7245
7246 if (env->jmps_processed - env->prev_jmps_processed >= 2 &&
7247 env->insn_processed - env->prev_insn_processed >= 8)
7248 add_new_state = true;
7249
7250 pprev = explored_state(env, insn_idx);
7251 sl = *pprev;
7252
7253 clean_live_states(env, insn_idx, cur);
7254
7255 while (sl) {
7256 states_cnt++;
7257 if (sl->state.insn_idx != insn_idx)
7258 goto next;
7259 if (sl->state.branches) {
7260 if (states_maybe_looping(&sl->state, cur) &&
7261 states_equal(env, &sl->state, cur)) {
7262 verbose_linfo(env, insn_idx, "; ");
7263 verbose(env, "infinite loop detected at insn %d\n", insn_idx);
7264 return -EINVAL;
7265 }
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278 if (env->jmps_processed - env->prev_jmps_processed < 20 &&
7279 env->insn_processed - env->prev_insn_processed < 100)
7280 add_new_state = false;
7281 goto miss;
7282 }
7283 if (states_equal(env, &sl->state, cur)) {
7284 sl->hit_cnt++;
7285
7286
7287
7288
7289
7290
7291
7292
7293
7294
7295 err = propagate_liveness(env, &sl->state, cur);
7296
7297
7298
7299
7300
7301
7302 err = err ? : push_jmp_history(env, cur);
7303 err = err ? : propagate_precision(env, &sl->state);
7304 if (err)
7305 return err;
7306 return 1;
7307 }
7308miss:
7309
7310
7311
7312
7313
7314
7315 if (add_new_state)
7316 sl->miss_cnt++;
7317
7318
7319
7320
7321
7322 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) {
7323
7324
7325
7326 *pprev = sl->next;
7327 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) {
7328 u32 br = sl->state.branches;
7329
7330 WARN_ONCE(br,
7331 "BUG live_done but branches_to_explore %d\n",
7332 br);
7333 free_verifier_state(&sl->state, false);
7334 kfree(sl);
7335 env->peak_states--;
7336 } else {
7337
7338
7339
7340
7341 sl->next = env->free_list;
7342 env->free_list = sl;
7343 }
7344 sl = *pprev;
7345 continue;
7346 }
7347next:
7348 pprev = &sl->next;
7349 sl = *pprev;
7350 }
7351
7352 if (env->max_states_per_insn < states_cnt)
7353 env->max_states_per_insn = states_cnt;
7354
7355 if (!env->allow_ptr_leaks && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)
7356 return push_jmp_history(env, cur);
7357
7358 if (!add_new_state)
7359 return push_jmp_history(env, cur);
7360
7361
7362
7363
7364
7365
7366
7367
7368
7369
7370 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL);
7371 if (!new_sl)
7372 return -ENOMEM;
7373 env->total_states++;
7374 env->peak_states++;
7375 env->prev_jmps_processed = env->jmps_processed;
7376 env->prev_insn_processed = env->insn_processed;
7377
7378
7379 new = &new_sl->state;
7380 err = copy_verifier_state(new, cur);
7381 if (err) {
7382 free_verifier_state(new, false);
7383 kfree(new_sl);
7384 return err;
7385 }
7386 new->insn_idx = insn_idx;
7387 WARN_ONCE(new->branches != 1,
7388 "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx);
7389
7390 cur->parent = new;
7391 cur->first_insn_idx = insn_idx;
7392 clear_jmp_history(cur);
7393 new_sl->next = *explored_state(env, insn_idx);
7394 *explored_state(env, insn_idx) = new_sl;
7395
7396
7397
7398
7399
7400
7401
7402
7403
7404
7405
7406
7407
7408 for (j = 0; j <= cur->curframe; j++) {
7409 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++)
7410 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i];
7411 for (i = 0; i < BPF_REG_FP; i++)
7412 cur->frame[j]->regs[i].live = REG_LIVE_NONE;
7413 }
7414
7415
7416 for (j = 0; j <= cur->curframe; j++) {
7417 struct bpf_func_state *frame = cur->frame[j];
7418 struct bpf_func_state *newframe = new->frame[j];
7419
7420 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) {
7421 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE;
7422 frame->stack[i].spilled_ptr.parent =
7423 &newframe->stack[i].spilled_ptr;
7424 }
7425 }
7426 return 0;
7427}
7428
7429
7430static bool reg_type_mismatch_ok(enum bpf_reg_type type)
7431{
7432 switch (type) {
7433 case PTR_TO_CTX:
7434 case PTR_TO_SOCKET:
7435 case PTR_TO_SOCKET_OR_NULL:
7436 case PTR_TO_SOCK_COMMON:
7437 case PTR_TO_SOCK_COMMON_OR_NULL:
7438 case PTR_TO_TCP_SOCK:
7439 case PTR_TO_TCP_SOCK_OR_NULL:
7440 case PTR_TO_XDP_SOCK:
7441 return false;
7442 default:
7443 return true;
7444 }
7445}
7446
7447
7448
7449
7450
7451
7452
7453
7454
7455
7456
7457
7458
7459static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev)
7460{
7461 return src != prev && (!reg_type_mismatch_ok(src) ||
7462 !reg_type_mismatch_ok(prev));
7463}
7464
7465static int do_check(struct bpf_verifier_env *env)
7466{
7467 struct bpf_verifier_state *state;
7468 struct bpf_insn *insns = env->prog->insnsi;
7469 struct bpf_reg_state *regs;
7470 int insn_cnt = env->prog->len;
7471 bool do_print_state = false;
7472 int prev_insn_idx = -1;
7473
7474 env->prev_linfo = NULL;
7475
7476 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL);
7477 if (!state)
7478 return -ENOMEM;
7479 state->curframe = 0;
7480 state->speculative = false;
7481 state->branches = 1;
7482 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL);
7483 if (!state->frame[0]) {
7484 kfree(state);
7485 return -ENOMEM;
7486 }
7487 env->cur_state = state;
7488 init_func_state(env, state->frame[0],
7489 BPF_MAIN_FUNC ,
7490 0 ,
7491 0 );
7492
7493 for (;;) {
7494 struct bpf_insn *insn;
7495 u8 class;
7496 int err;
7497
7498 env->prev_insn_idx = prev_insn_idx;
7499 if (env->insn_idx >= insn_cnt) {
7500 verbose(env, "invalid insn idx %d insn_cnt %d\n",
7501 env->insn_idx, insn_cnt);
7502 return -EFAULT;
7503 }
7504
7505 insn = &insns[env->insn_idx];
7506 class = BPF_CLASS(insn->code);
7507
7508 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
7509 verbose(env,
7510 "BPF program is too large. Processed %d insn\n",
7511 env->insn_processed);
7512 return -E2BIG;
7513 }
7514
7515 err = is_state_visited(env, env->insn_idx);
7516 if (err < 0)
7517 return err;
7518 if (err == 1) {
7519
7520 if (env->log.level & BPF_LOG_LEVEL) {
7521 if (do_print_state)
7522 verbose(env, "\nfrom %d to %d%s: safe\n",
7523 env->prev_insn_idx, env->insn_idx,
7524 env->cur_state->speculative ?
7525 " (speculative execution)" : "");
7526 else
7527 verbose(env, "%d: safe\n", env->insn_idx);
7528 }
7529 goto process_bpf_exit;
7530 }
7531
7532 if (signal_pending(current))
7533 return -EAGAIN;
7534
7535 if (need_resched())
7536 cond_resched();
7537
7538 if (env->log.level & BPF_LOG_LEVEL2 ||
7539 (env->log.level & BPF_LOG_LEVEL && do_print_state)) {
7540 if (env->log.level & BPF_LOG_LEVEL2)
7541 verbose(env, "%d:", env->insn_idx);
7542 else
7543 verbose(env, "\nfrom %d to %d%s:",
7544 env->prev_insn_idx, env->insn_idx,
7545 env->cur_state->speculative ?
7546 " (speculative execution)" : "");
7547 print_verifier_state(env, state->frame[state->curframe]);
7548 do_print_state = false;
7549 }
7550
7551 if (env->log.level & BPF_LOG_LEVEL) {
7552 const struct bpf_insn_cbs cbs = {
7553 .cb_print = verbose,
7554 .private_data = env,
7555 };
7556
7557 verbose_linfo(env, env->insn_idx, "; ");
7558 verbose(env, "%d: ", env->insn_idx);
7559 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
7560 }
7561
7562 if (bpf_prog_is_dev_bound(env->prog->aux)) {
7563 err = bpf_prog_offload_verify_insn(env, env->insn_idx,
7564 env->prev_insn_idx);
7565 if (err)
7566 return err;
7567 }
7568
7569 regs = cur_regs(env);
7570 env->insn_aux_data[env->insn_idx].seen = true;
7571 prev_insn_idx = env->insn_idx;
7572
7573 if (class == BPF_ALU || class == BPF_ALU64) {
7574 err = check_alu_op(env, insn);
7575 if (err)
7576 return err;
7577
7578 } else if (class == BPF_LDX) {
7579 enum bpf_reg_type *prev_src_type, src_reg_type;
7580
7581
7582
7583
7584 err = check_reg_arg(env, insn->src_reg, SRC_OP);
7585 if (err)
7586 return err;
7587
7588 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
7589 if (err)
7590 return err;
7591
7592 src_reg_type = regs[insn->src_reg].type;
7593
7594
7595
7596
7597 err = check_mem_access(env, env->insn_idx, insn->src_reg,
7598 insn->off, BPF_SIZE(insn->code),
7599 BPF_READ, insn->dst_reg, false);
7600 if (err)
7601 return err;
7602
7603 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type;
7604
7605 if (*prev_src_type == NOT_INIT) {
7606
7607
7608
7609
7610 *prev_src_type = src_reg_type;
7611
7612 } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) {
7613
7614
7615
7616
7617
7618
7619
7620 verbose(env, "same insn cannot be used with different pointers\n");
7621 return -EINVAL;
7622 }
7623
7624 } else if (class == BPF_STX) {
7625 enum bpf_reg_type *prev_dst_type, dst_reg_type;
7626
7627 if (BPF_MODE(insn->code) == BPF_XADD) {
7628 err = check_xadd(env, env->insn_idx, insn);
7629 if (err)
7630 return err;
7631 env->insn_idx++;
7632 continue;
7633 }
7634
7635
7636 err = check_reg_arg(env, insn->src_reg, SRC_OP);
7637 if (err)
7638 return err;
7639
7640 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
7641 if (err)
7642 return err;
7643
7644 dst_reg_type = regs[insn->dst_reg].type;
7645
7646
7647 err = check_mem_access(env, env->insn_idx, insn->dst_reg,
7648 insn->off, BPF_SIZE(insn->code),
7649 BPF_WRITE, insn->src_reg, false);
7650 if (err)
7651 return err;
7652
7653 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type;
7654
7655 if (*prev_dst_type == NOT_INIT) {
7656 *prev_dst_type = dst_reg_type;
7657 } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) {
7658 verbose(env, "same insn cannot be used with different pointers\n");
7659 return -EINVAL;
7660 }
7661
7662 } else if (class == BPF_ST) {
7663 if (BPF_MODE(insn->code) != BPF_MEM ||
7664 insn->src_reg != BPF_REG_0) {
7665 verbose(env, "BPF_ST uses reserved fields\n");
7666 return -EINVAL;
7667 }
7668
7669 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
7670 if (err)
7671 return err;
7672
7673 if (is_ctx_reg(env, insn->dst_reg)) {
7674 verbose(env, "BPF_ST stores into R%d %s is not allowed\n",
7675 insn->dst_reg,
7676 reg_type_str[reg_state(env, insn->dst_reg)->type]);
7677 return -EACCES;
7678 }
7679
7680
7681 err = check_mem_access(env, env->insn_idx, insn->dst_reg,
7682 insn->off, BPF_SIZE(insn->code),
7683 BPF_WRITE, -1, false);
7684 if (err)
7685 return err;
7686
7687 } else if (class == BPF_JMP || class == BPF_JMP32) {
7688 u8 opcode = BPF_OP(insn->code);
7689
7690 env->jmps_processed++;
7691 if (opcode == BPF_CALL) {
7692 if (BPF_SRC(insn->code) != BPF_K ||
7693 insn->off != 0 ||
7694 (insn->src_reg != BPF_REG_0 &&
7695 insn->src_reg != BPF_PSEUDO_CALL) ||
7696 insn->dst_reg != BPF_REG_0 ||
7697 class == BPF_JMP32) {
7698 verbose(env, "BPF_CALL uses reserved fields\n");
7699 return -EINVAL;
7700 }
7701
7702 if (env->cur_state->active_spin_lock &&
7703 (insn->src_reg == BPF_PSEUDO_CALL ||
7704 insn->imm != BPF_FUNC_spin_unlock)) {
7705 verbose(env, "function calls are not allowed while holding a lock\n");
7706 return -EINVAL;
7707 }
7708 if (insn->src_reg == BPF_PSEUDO_CALL)
7709 err = check_func_call(env, insn, &env->insn_idx);
7710 else
7711 err = check_helper_call(env, insn->imm, env->insn_idx);
7712 if (err)
7713 return err;
7714
7715 } else if (opcode == BPF_JA) {
7716 if (BPF_SRC(insn->code) != BPF_K ||
7717 insn->imm != 0 ||
7718 insn->src_reg != BPF_REG_0 ||
7719 insn->dst_reg != BPF_REG_0 ||
7720 class == BPF_JMP32) {
7721 verbose(env, "BPF_JA uses reserved fields\n");
7722 return -EINVAL;
7723 }
7724
7725 env->insn_idx += insn->off + 1;
7726 continue;
7727
7728 } else if (opcode == BPF_EXIT) {
7729 if (BPF_SRC(insn->code) != BPF_K ||
7730 insn->imm != 0 ||
7731 insn->src_reg != BPF_REG_0 ||
7732 insn->dst_reg != BPF_REG_0 ||
7733 class == BPF_JMP32) {
7734 verbose(env, "BPF_EXIT uses reserved fields\n");
7735 return -EINVAL;
7736 }
7737
7738 if (env->cur_state->active_spin_lock) {
7739 verbose(env, "bpf_spin_unlock is missing\n");
7740 return -EINVAL;
7741 }
7742
7743 if (state->curframe) {
7744
7745 err = prepare_func_exit(env, &env->insn_idx);
7746 if (err)
7747 return err;
7748 do_print_state = true;
7749 continue;
7750 }
7751
7752 err = check_reference_leak(env);
7753 if (err)
7754 return err;
7755
7756
7757
7758
7759
7760
7761
7762 err = check_reg_arg(env, BPF_REG_0, SRC_OP);
7763 if (err)
7764 return err;
7765
7766 if (is_pointer_value(env, BPF_REG_0)) {
7767 verbose(env, "R0 leaks addr as return value\n");
7768 return -EACCES;
7769 }
7770
7771 err = check_return_code(env);
7772 if (err)
7773 return err;
7774process_bpf_exit:
7775 update_branch_counts(env, env->cur_state);
7776 err = pop_stack(env, &prev_insn_idx,
7777 &env->insn_idx);
7778 if (err < 0) {
7779 if (err != -ENOENT)
7780 return err;
7781 break;
7782 } else {
7783 do_print_state = true;
7784 continue;
7785 }
7786 } else {
7787 err = check_cond_jmp_op(env, insn, &env->insn_idx);
7788 if (err)
7789 return err;
7790 }
7791 } else if (class == BPF_LD) {
7792 u8 mode = BPF_MODE(insn->code);
7793
7794 if (mode == BPF_ABS || mode == BPF_IND) {
7795 err = check_ld_abs(env, insn);
7796 if (err)
7797 return err;
7798
7799 } else if (mode == BPF_IMM) {
7800 err = check_ld_imm(env, insn);
7801 if (err)
7802 return err;
7803
7804 env->insn_idx++;
7805 env->insn_aux_data[env->insn_idx].seen = true;
7806 } else {
7807 verbose(env, "invalid BPF_LD mode\n");
7808 return -EINVAL;
7809 }
7810 } else {
7811 verbose(env, "unknown insn class %d\n", class);
7812 return -EINVAL;
7813 }
7814
7815 env->insn_idx++;
7816 }
7817
7818 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
7819 return 0;
7820}
7821
7822static int check_map_prealloc(struct bpf_map *map)
7823{
7824 return (map->map_type != BPF_MAP_TYPE_HASH &&
7825 map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
7826 map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) ||
7827 !(map->map_flags & BPF_F_NO_PREALLOC);
7828}
7829
7830static bool is_tracing_prog_type(enum bpf_prog_type type)
7831{
7832 switch (type) {
7833 case BPF_PROG_TYPE_KPROBE:
7834 case BPF_PROG_TYPE_TRACEPOINT:
7835 case BPF_PROG_TYPE_PERF_EVENT:
7836 case BPF_PROG_TYPE_RAW_TRACEPOINT:
7837 return true;
7838 default:
7839 return false;
7840 }
7841}
7842
7843static int check_map_prog_compatibility(struct bpf_verifier_env *env,
7844 struct bpf_map *map,
7845 struct bpf_prog *prog)
7846
7847{
7848
7849
7850
7851
7852
7853 if (prog->type == BPF_PROG_TYPE_PERF_EVENT) {
7854 if (!check_map_prealloc(map)) {
7855 verbose(env, "perf_event programs can only use preallocated hash map\n");
7856 return -EINVAL;
7857 }
7858 if (map->inner_map_meta &&
7859 !check_map_prealloc(map->inner_map_meta)) {
7860 verbose(env, "perf_event programs can only use preallocated inner hash map\n");
7861 return -EINVAL;
7862 }
7863 }
7864
7865 if ((is_tracing_prog_type(prog->type) ||
7866 prog->type == BPF_PROG_TYPE_SOCKET_FILTER) &&
7867 map_value_has_spin_lock(map)) {
7868 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
7869 return -EINVAL;
7870 }
7871
7872 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) &&
7873 !bpf_offload_prog_map_match(prog, map)) {
7874 verbose(env, "offload device mismatch between prog and map\n");
7875 return -EINVAL;
7876 }
7877
7878 return 0;
7879}
7880
7881static bool bpf_map_is_cgroup_storage(struct bpf_map *map)
7882{
7883 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
7884 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
7885}
7886
7887
7888
7889
7890static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env)
7891{
7892 struct bpf_insn *insn = env->prog->insnsi;
7893 int insn_cnt = env->prog->len;
7894 int i, j, err;
7895
7896 err = bpf_prog_calc_tag(env->prog);
7897 if (err)
7898 return err;
7899
7900 for (i = 0; i < insn_cnt; i++, insn++) {
7901 if (BPF_CLASS(insn->code) == BPF_LDX &&
7902 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) {
7903 verbose(env, "BPF_LDX uses reserved fields\n");
7904 return -EINVAL;
7905 }
7906
7907 if (BPF_CLASS(insn->code) == BPF_STX &&
7908 ((BPF_MODE(insn->code) != BPF_MEM &&
7909 BPF_MODE(insn->code) != BPF_XADD) || insn->imm != 0)) {
7910 verbose(env, "BPF_STX uses reserved fields\n");
7911 return -EINVAL;
7912 }
7913
7914 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
7915 struct bpf_insn_aux_data *aux;
7916 struct bpf_map *map;
7917 struct fd f;
7918 u64 addr;
7919
7920 if (i == insn_cnt - 1 || insn[1].code != 0 ||
7921 insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
7922 insn[1].off != 0) {
7923 verbose(env, "invalid bpf_ld_imm64 insn\n");
7924 return -EINVAL;
7925 }
7926
7927 if (insn[0].src_reg == 0)
7928
7929 goto next_insn;
7930
7931
7932
7933
7934 if ((insn[0].src_reg != BPF_PSEUDO_MAP_FD &&
7935 insn[0].src_reg != BPF_PSEUDO_MAP_VALUE) ||
7936 (insn[0].src_reg == BPF_PSEUDO_MAP_FD &&
7937 insn[1].imm != 0)) {
7938 verbose(env,
7939 "unrecognized bpf_ld_imm64 insn\n");
7940 return -EINVAL;
7941 }
7942
7943 f = fdget(insn[0].imm);
7944 map = __bpf_map_get(f);
7945 if (IS_ERR(map)) {
7946 verbose(env, "fd %d is not pointing to valid bpf_map\n",
7947 insn[0].imm);
7948 return PTR_ERR(map);
7949 }
7950
7951 err = check_map_prog_compatibility(env, map, env->prog);
7952 if (err) {
7953 fdput(f);
7954 return err;
7955 }
7956
7957 aux = &env->insn_aux_data[i];
7958 if (insn->src_reg == BPF_PSEUDO_MAP_FD) {
7959 addr = (unsigned long)map;
7960 } else {
7961 u32 off = insn[1].imm;
7962
7963 if (off >= BPF_MAX_VAR_OFF) {
7964 verbose(env, "direct value offset of %u is not allowed\n", off);
7965 fdput(f);
7966 return -EINVAL;
7967 }
7968
7969 if (!map->ops->map_direct_value_addr) {
7970 verbose(env, "no direct value access support for this map type\n");
7971 fdput(f);
7972 return -EINVAL;
7973 }
7974
7975 err = map->ops->map_direct_value_addr(map, &addr, off);
7976 if (err) {
7977 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",
7978 map->value_size, off);
7979 fdput(f);
7980 return err;
7981 }
7982
7983 aux->map_off = off;
7984 addr += off;
7985 }
7986
7987 insn[0].imm = (u32)addr;
7988 insn[1].imm = addr >> 32;
7989
7990
7991 for (j = 0; j < env->used_map_cnt; j++) {
7992 if (env->used_maps[j] == map) {
7993 aux->map_index = j;
7994 fdput(f);
7995 goto next_insn;
7996 }
7997 }
7998
7999 if (env->used_map_cnt >= MAX_USED_MAPS) {
8000 fdput(f);
8001 return -E2BIG;
8002 }
8003
8004
8005
8006
8007
8008
8009 map = bpf_map_inc(map, false);
8010 if (IS_ERR(map)) {
8011 fdput(f);
8012 return PTR_ERR(map);
8013 }
8014
8015 aux->map_index = env->used_map_cnt;
8016 env->used_maps[env->used_map_cnt++] = map;
8017
8018 if (bpf_map_is_cgroup_storage(map) &&
8019 bpf_cgroup_storage_assign(env->prog, map)) {
8020 verbose(env, "only one cgroup storage of each type is allowed\n");
8021 fdput(f);
8022 return -EBUSY;
8023 }
8024
8025 fdput(f);
8026next_insn:
8027 insn++;
8028 i++;
8029 continue;
8030 }
8031
8032
8033 if (!bpf_opcode_in_insntable(insn->code)) {
8034 verbose(env, "unknown opcode %02x\n", insn->code);
8035 return -EINVAL;
8036 }
8037 }
8038
8039
8040
8041
8042
8043 return 0;
8044}
8045
8046
8047static void release_maps(struct bpf_verifier_env *env)
8048{
8049 enum bpf_cgroup_storage_type stype;
8050 int i;
8051
8052 for_each_cgroup_storage_type(stype) {
8053 if (!env->prog->aux->cgroup_storage[stype])
8054 continue;
8055 bpf_cgroup_storage_release(env->prog,
8056 env->prog->aux->cgroup_storage[stype]);
8057 }
8058
8059 for (i = 0; i < env->used_map_cnt; i++)
8060 bpf_map_put(env->used_maps[i]);
8061}
8062
8063
8064static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
8065{
8066 struct bpf_insn *insn = env->prog->insnsi;
8067 int insn_cnt = env->prog->len;
8068 int i;
8069
8070 for (i = 0; i < insn_cnt; i++, insn++)
8071 if (insn->code == (BPF_LD | BPF_IMM | BPF_DW))
8072 insn->src_reg = 0;
8073}
8074
8075
8076
8077
8078
8079static int adjust_insn_aux_data(struct bpf_verifier_env *env,
8080 struct bpf_prog *new_prog, u32 off, u32 cnt)
8081{
8082 struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data;
8083 struct bpf_insn *insn = new_prog->insnsi;
8084 u32 prog_len;
8085 int i;
8086
8087
8088
8089
8090
8091 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1);
8092
8093 if (cnt == 1)
8094 return 0;
8095 prog_len = new_prog->len;
8096 new_data = vzalloc(array_size(prog_len,
8097 sizeof(struct bpf_insn_aux_data)));
8098 if (!new_data)
8099 return -ENOMEM;
8100 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
8101 memcpy(new_data + off + cnt - 1, old_data + off,
8102 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
8103 for (i = off; i < off + cnt - 1; i++) {
8104 new_data[i].seen = true;
8105 new_data[i].zext_dst = insn_has_def32(env, insn + i);
8106 }
8107 env->insn_aux_data = new_data;
8108 vfree(old_data);
8109 return 0;
8110}
8111
8112static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
8113{
8114 int i;
8115
8116 if (len == 1)
8117 return;
8118
8119 for (i = 0; i <= env->subprog_cnt; i++) {
8120 if (env->subprog_info[i].start <= off)
8121 continue;
8122 env->subprog_info[i].start += len - 1;
8123 }
8124}
8125
8126static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
8127 const struct bpf_insn *patch, u32 len)
8128{
8129 struct bpf_prog *new_prog;
8130
8131 new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
8132 if (IS_ERR(new_prog)) {
8133 if (PTR_ERR(new_prog) == -ERANGE)
8134 verbose(env,
8135 "insn %d cannot be patched due to 16-bit range\n",
8136 env->insn_aux_data[off].orig_idx);
8137 return NULL;
8138 }
8139 if (adjust_insn_aux_data(env, new_prog, off, len))
8140 return NULL;
8141 adjust_subprog_starts(env, off, len);
8142 return new_prog;
8143}
8144
8145static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
8146 u32 off, u32 cnt)
8147{
8148 int i, j;
8149
8150
8151 for (i = 0; i < env->subprog_cnt; i++)
8152 if (env->subprog_info[i].start >= off)
8153 break;
8154
8155 for (j = i; j < env->subprog_cnt; j++)
8156 if (env->subprog_info[j].start >= off + cnt)
8157 break;
8158
8159
8160
8161 if (env->subprog_info[j].start != off + cnt)
8162 j--;
8163
8164 if (j > i) {
8165 struct bpf_prog_aux *aux = env->prog->aux;
8166 int move;
8167
8168
8169 move = env->subprog_cnt + 1 - j;
8170
8171 memmove(env->subprog_info + i,
8172 env->subprog_info + j,
8173 sizeof(*env->subprog_info) * move);
8174 env->subprog_cnt -= j - i;
8175
8176
8177 if (aux->func_info) {
8178 move = aux->func_info_cnt - j;
8179
8180 memmove(aux->func_info + i,
8181 aux->func_info + j,
8182 sizeof(*aux->func_info) * move);
8183 aux->func_info_cnt -= j - i;
8184
8185
8186
8187 }
8188 } else {
8189
8190 if (env->subprog_info[i].start == off)
8191 i++;
8192 }
8193
8194
8195 for (; i <= env->subprog_cnt; i++)
8196 env->subprog_info[i].start -= cnt;
8197
8198 return 0;
8199}
8200
8201static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
8202 u32 cnt)
8203{
8204 struct bpf_prog *prog = env->prog;
8205 u32 i, l_off, l_cnt, nr_linfo;
8206 struct bpf_line_info *linfo;
8207
8208 nr_linfo = prog->aux->nr_linfo;
8209 if (!nr_linfo)
8210 return 0;
8211
8212 linfo = prog->aux->linfo;
8213
8214
8215 for (i = 0; i < nr_linfo; i++)
8216 if (linfo[i].insn_off >= off)
8217 break;
8218
8219 l_off = i;
8220 l_cnt = 0;
8221 for (; i < nr_linfo; i++)
8222 if (linfo[i].insn_off < off + cnt)
8223 l_cnt++;
8224 else
8225 break;
8226
8227
8228
8229
8230
8231 if (prog->len != off && l_cnt &&
8232 (i == nr_linfo || linfo[i].insn_off != off + cnt)) {
8233 l_cnt--;
8234 linfo[--i].insn_off = off + cnt;
8235 }
8236
8237
8238 if (l_cnt) {
8239 memmove(linfo + l_off, linfo + i,
8240 sizeof(*linfo) * (nr_linfo - i));
8241
8242 prog->aux->nr_linfo -= l_cnt;
8243 nr_linfo = prog->aux->nr_linfo;
8244 }
8245
8246
8247 for (i = l_off; i < nr_linfo; i++)
8248 linfo[i].insn_off -= cnt;
8249
8250
8251 for (i = 0; i <= env->subprog_cnt; i++)
8252 if (env->subprog_info[i].linfo_idx > l_off) {
8253
8254
8255
8256 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)
8257 env->subprog_info[i].linfo_idx -= l_cnt;
8258 else
8259 env->subprog_info[i].linfo_idx = l_off;
8260 }
8261
8262 return 0;
8263}
8264
8265static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
8266{
8267 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
8268 unsigned int orig_prog_len = env->prog->len;
8269 int err;
8270
8271 if (bpf_prog_is_dev_bound(env->prog->aux))
8272 bpf_prog_offload_remove_insns(env, off, cnt);
8273
8274 err = bpf_remove_insns(env->prog, off, cnt);
8275 if (err)
8276 return err;
8277
8278 err = adjust_subprog_starts_after_remove(env, off, cnt);
8279 if (err)
8280 return err;
8281
8282 err = bpf_adj_linfo_after_remove(env, off, cnt);
8283 if (err)
8284 return err;
8285
8286 memmove(aux_data + off, aux_data + off + cnt,
8287 sizeof(*aux_data) * (orig_prog_len - off - cnt));
8288
8289 return 0;
8290}
8291
8292
8293
8294
8295
8296
8297
8298
8299
8300
8301
8302
8303static void sanitize_dead_code(struct bpf_verifier_env *env)
8304{
8305 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
8306 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
8307 struct bpf_insn *insn = env->prog->insnsi;
8308 const int insn_cnt = env->prog->len;
8309 int i;
8310
8311 for (i = 0; i < insn_cnt; i++) {
8312 if (aux_data[i].seen)
8313 continue;
8314 memcpy(insn + i, &trap, sizeof(trap));
8315 }
8316}
8317
8318static bool insn_is_cond_jump(u8 code)
8319{
8320 u8 op;
8321
8322 if (BPF_CLASS(code) == BPF_JMP32)
8323 return true;
8324
8325 if (BPF_CLASS(code) != BPF_JMP)
8326 return false;
8327
8328 op = BPF_OP(code);
8329 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
8330}
8331
8332static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
8333{
8334 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
8335 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
8336 struct bpf_insn *insn = env->prog->insnsi;
8337 const int insn_cnt = env->prog->len;
8338 int i;
8339
8340 for (i = 0; i < insn_cnt; i++, insn++) {
8341 if (!insn_is_cond_jump(insn->code))
8342 continue;
8343
8344 if (!aux_data[i + 1].seen)
8345 ja.off = insn->off;
8346 else if (!aux_data[i + 1 + insn->off].seen)
8347 ja.off = 0;
8348 else
8349 continue;
8350
8351 if (bpf_prog_is_dev_bound(env->prog->aux))
8352 bpf_prog_offload_replace_insn(env, i, &ja);
8353
8354 memcpy(insn, &ja, sizeof(ja));
8355 }
8356}
8357
8358static int opt_remove_dead_code(struct bpf_verifier_env *env)
8359{
8360 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
8361 int insn_cnt = env->prog->len;
8362 int i, err;
8363
8364 for (i = 0; i < insn_cnt; i++) {
8365 int j;
8366
8367 j = 0;
8368 while (i + j < insn_cnt && !aux_data[i + j].seen)
8369 j++;
8370 if (!j)
8371 continue;
8372
8373 err = verifier_remove_insns(env, i, j);
8374 if (err)
8375 return err;
8376 insn_cnt = env->prog->len;
8377 }
8378
8379 return 0;
8380}
8381
8382static int opt_remove_nops(struct bpf_verifier_env *env)
8383{
8384 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
8385 struct bpf_insn *insn = env->prog->insnsi;
8386 int insn_cnt = env->prog->len;
8387 int i, err;
8388
8389 for (i = 0; i < insn_cnt; i++) {
8390 if (memcmp(&insn[i], &ja, sizeof(ja)))
8391 continue;
8392
8393 err = verifier_remove_insns(env, i, 1);
8394 if (err)
8395 return err;
8396 insn_cnt--;
8397 i--;
8398 }
8399
8400 return 0;
8401}
8402
8403static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,
8404 const union bpf_attr *attr)
8405{
8406 struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4];
8407 struct bpf_insn_aux_data *aux = env->insn_aux_data;
8408 int i, patch_len, delta = 0, len = env->prog->len;
8409 struct bpf_insn *insns = env->prog->insnsi;
8410 struct bpf_prog *new_prog;
8411 bool rnd_hi32;
8412
8413 rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32;
8414 zext_patch[1] = BPF_ZEXT_REG(0);
8415 rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0);
8416 rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32);
8417 rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX);
8418 for (i = 0; i < len; i++) {
8419 int adj_idx = i + delta;
8420 struct bpf_insn insn;
8421
8422 insn = insns[adj_idx];
8423 if (!aux[adj_idx].zext_dst) {
8424 u8 code, class;
8425 u32 imm_rnd;
8426
8427 if (!rnd_hi32)
8428 continue;
8429
8430 code = insn.code;
8431 class = BPF_CLASS(code);
8432 if (insn_no_def(&insn))
8433 continue;
8434
8435
8436
8437
8438
8439 if (is_reg64(env, &insn, insn.dst_reg, NULL, DST_OP)) {
8440 if (class == BPF_LD &&
8441 BPF_MODE(code) == BPF_IMM)
8442 i++;
8443 continue;
8444 }
8445
8446
8447 if (class == BPF_LDX &&
8448 aux[adj_idx].ptr_type == PTR_TO_CTX)
8449 continue;
8450
8451 imm_rnd = get_random_int();
8452 rnd_hi32_patch[0] = insn;
8453 rnd_hi32_patch[1].imm = imm_rnd;
8454 rnd_hi32_patch[3].dst_reg = insn.dst_reg;
8455 patch = rnd_hi32_patch;
8456 patch_len = 4;
8457 goto apply_patch_buffer;
8458 }
8459
8460 if (!bpf_jit_needs_zext())
8461 continue;
8462
8463 zext_patch[0] = insn;
8464 zext_patch[1].dst_reg = insn.dst_reg;
8465 zext_patch[1].src_reg = insn.dst_reg;
8466 patch = zext_patch;
8467 patch_len = 2;
8468apply_patch_buffer:
8469 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);
8470 if (!new_prog)
8471 return -ENOMEM;
8472 env->prog = new_prog;
8473 insns = new_prog->insnsi;
8474 aux = env->insn_aux_data;
8475 delta += patch_len - 1;
8476 }
8477
8478 return 0;
8479}
8480
8481
8482
8483
8484
8485
8486static int convert_ctx_accesses(struct bpf_verifier_env *env)
8487{
8488 const struct bpf_verifier_ops *ops = env->ops;
8489 int i, cnt, size, ctx_field_size, delta = 0;
8490 const int insn_cnt = env->prog->len;
8491 struct bpf_insn insn_buf[16], *insn;
8492 u32 target_size, size_default, off;
8493 struct bpf_prog *new_prog;
8494 enum bpf_access_type type;
8495 bool is_narrower_load;
8496
8497 if (ops->gen_prologue || env->seen_direct_write) {
8498 if (!ops->gen_prologue) {
8499 verbose(env, "bpf verifier is misconfigured\n");
8500 return -EINVAL;
8501 }
8502 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,
8503 env->prog);
8504 if (cnt >= ARRAY_SIZE(insn_buf)) {
8505 verbose(env, "bpf verifier is misconfigured\n");
8506 return -EINVAL;
8507 } else if (cnt) {
8508 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
8509 if (!new_prog)
8510 return -ENOMEM;
8511
8512 env->prog = new_prog;
8513 delta += cnt - 1;
8514 }
8515 }
8516
8517 if (bpf_prog_is_dev_bound(env->prog->aux))
8518 return 0;
8519
8520 insn = env->prog->insnsi + delta;
8521
8522 for (i = 0; i < insn_cnt; i++, insn++) {
8523 bpf_convert_ctx_access_t convert_ctx_access;
8524
8525 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
8526 insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
8527 insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
8528 insn->code == (BPF_LDX | BPF_MEM | BPF_DW))
8529 type = BPF_READ;
8530 else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
8531 insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
8532 insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
8533 insn->code == (BPF_STX | BPF_MEM | BPF_DW))
8534 type = BPF_WRITE;
8535 else
8536 continue;
8537
8538 if (type == BPF_WRITE &&
8539 env->insn_aux_data[i + delta].sanitize_stack_off) {
8540 struct bpf_insn patch[] = {
8541
8542
8543
8544
8545
8546 BPF_ST_MEM(BPF_DW, BPF_REG_FP,
8547 env->insn_aux_data[i + delta].sanitize_stack_off,
8548 0),
8549
8550
8551
8552 *insn,
8553 };
8554
8555 cnt = ARRAY_SIZE(patch);
8556 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
8557 if (!new_prog)
8558 return -ENOMEM;
8559
8560 delta += cnt - 1;
8561 env->prog = new_prog;
8562 insn = new_prog->insnsi + i + delta;
8563 continue;
8564 }
8565
8566 switch (env->insn_aux_data[i + delta].ptr_type) {
8567 case PTR_TO_CTX:
8568 if (!ops->convert_ctx_access)
8569 continue;
8570 convert_ctx_access = ops->convert_ctx_access;
8571 break;
8572 case PTR_TO_SOCKET:
8573 case PTR_TO_SOCK_COMMON:
8574 convert_ctx_access = bpf_sock_convert_ctx_access;
8575 break;
8576 case PTR_TO_TCP_SOCK:
8577 convert_ctx_access = bpf_tcp_sock_convert_ctx_access;
8578 break;
8579 case PTR_TO_XDP_SOCK:
8580 convert_ctx_access = bpf_xdp_sock_convert_ctx_access;
8581 break;
8582 default:
8583 continue;
8584 }
8585
8586 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
8587 size = BPF_LDST_BYTES(insn);
8588
8589
8590
8591
8592
8593
8594 is_narrower_load = size < ctx_field_size;
8595 size_default = bpf_ctx_off_adjust_machine(ctx_field_size);
8596 off = insn->off;
8597 if (is_narrower_load) {
8598 u8 size_code;
8599
8600 if (type == BPF_WRITE) {
8601 verbose(env, "bpf verifier narrow ctx access misconfigured\n");
8602 return -EINVAL;
8603 }
8604
8605 size_code = BPF_H;
8606 if (ctx_field_size == 4)
8607 size_code = BPF_W;
8608 else if (ctx_field_size == 8)
8609 size_code = BPF_DW;
8610
8611 insn->off = off & ~(size_default - 1);
8612 insn->code = BPF_LDX | BPF_MEM | size_code;
8613 }
8614
8615 target_size = 0;
8616 cnt = convert_ctx_access(type, insn, insn_buf, env->prog,
8617 &target_size);
8618 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) ||
8619 (ctx_field_size && !target_size)) {
8620 verbose(env, "bpf verifier is misconfigured\n");
8621 return -EINVAL;
8622 }
8623
8624 if (is_narrower_load && size < target_size) {
8625 u8 shift = bpf_ctx_narrow_load_shift(off, size,
8626 size_default);
8627 if (ctx_field_size <= 4) {
8628 if (shift)
8629 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH,
8630 insn->dst_reg,
8631 shift);
8632 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
8633 (1 << size * 8) - 1);
8634 } else {
8635 if (shift)
8636 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH,
8637 insn->dst_reg,
8638 shift);
8639 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg,
8640 (1ULL << size * 8) - 1);
8641 }
8642 }
8643
8644 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
8645 if (!new_prog)
8646 return -ENOMEM;
8647
8648 delta += cnt - 1;
8649
8650
8651 env->prog = new_prog;
8652 insn = new_prog->insnsi + i + delta;
8653 }
8654
8655 return 0;
8656}
8657
8658static int jit_subprogs(struct bpf_verifier_env *env)
8659{
8660 struct bpf_prog *prog = env->prog, **func, *tmp;
8661 int i, j, subprog_start, subprog_end = 0, len, subprog;
8662 struct bpf_insn *insn;
8663 void *old_bpf_func;
8664 int err;
8665
8666 if (env->subprog_cnt <= 1)
8667 return 0;
8668
8669 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
8670 if (insn->code != (BPF_JMP | BPF_CALL) ||
8671 insn->src_reg != BPF_PSEUDO_CALL)
8672 continue;
8673
8674
8675
8676
8677 subprog = find_subprog(env, i + insn->imm + 1);
8678 if (subprog < 0) {
8679 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
8680 i + insn->imm + 1);
8681 return -EFAULT;
8682 }
8683
8684
8685
8686 insn->off = subprog;
8687
8688
8689
8690 env->insn_aux_data[i].call_imm = insn->imm;
8691
8692 insn->imm = 1;
8693 }
8694
8695 err = bpf_prog_alloc_jited_linfo(prog);
8696 if (err)
8697 goto out_undo_insn;
8698
8699 err = -ENOMEM;
8700 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
8701 if (!func)
8702 goto out_undo_insn;
8703
8704 for (i = 0; i < env->subprog_cnt; i++) {
8705 subprog_start = subprog_end;
8706 subprog_end = env->subprog_info[i + 1].start;
8707
8708 len = subprog_end - subprog_start;
8709
8710
8711
8712
8713
8714 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
8715 if (!func[i])
8716 goto out_free;
8717 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
8718 len * sizeof(struct bpf_insn));
8719 func[i]->type = prog->type;
8720 func[i]->len = len;
8721 if (bpf_prog_calc_tag(func[i]))
8722 goto out_free;
8723 func[i]->is_func = 1;
8724 func[i]->aux->func_idx = i;
8725
8726 func[i]->aux->btf = prog->aux->btf;
8727 func[i]->aux->func_info = prog->aux->func_info;
8728
8729
8730
8731
8732 func[i]->aux->name[0] = 'F';
8733 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
8734 func[i]->jit_requested = 1;
8735 func[i]->aux->linfo = prog->aux->linfo;
8736 func[i]->aux->nr_linfo = prog->aux->nr_linfo;
8737 func[i]->aux->jited_linfo = prog->aux->jited_linfo;
8738 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
8739 func[i] = bpf_int_jit_compile(func[i]);
8740 if (!func[i]->jited) {
8741 err = -ENOTSUPP;
8742 goto out_free;
8743 }
8744 cond_resched();
8745 }
8746
8747
8748
8749
8750 for (i = 0; i < env->subprog_cnt; i++) {
8751 insn = func[i]->insnsi;
8752 for (j = 0; j < func[i]->len; j++, insn++) {
8753 if (insn->code != (BPF_JMP | BPF_CALL) ||
8754 insn->src_reg != BPF_PSEUDO_CALL)
8755 continue;
8756 subprog = insn->off;
8757 insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) -
8758 __bpf_call_base;
8759 }
8760
8761
8762
8763
8764
8765
8766
8767
8768
8769
8770
8771
8772 func[i]->aux->func = func;
8773 func[i]->aux->func_cnt = env->subprog_cnt;
8774 }
8775 for (i = 0; i < env->subprog_cnt; i++) {
8776 old_bpf_func = func[i]->bpf_func;
8777 tmp = bpf_int_jit_compile(func[i]);
8778 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
8779 verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
8780 err = -ENOTSUPP;
8781 goto out_free;
8782 }
8783 cond_resched();
8784 }
8785
8786
8787
8788
8789 for (i = 0; i < env->subprog_cnt; i++) {
8790 bpf_prog_lock_ro(func[i]);
8791 bpf_prog_kallsyms_add(func[i]);
8792 }
8793
8794
8795
8796
8797
8798 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
8799 if (insn->code != (BPF_JMP | BPF_CALL) ||
8800 insn->src_reg != BPF_PSEUDO_CALL)
8801 continue;
8802 insn->off = env->insn_aux_data[i].call_imm;
8803 subprog = find_subprog(env, i + insn->off + 1);
8804 insn->imm = subprog;
8805 }
8806
8807 prog->jited = 1;
8808 prog->bpf_func = func[0]->bpf_func;
8809 prog->aux->func = func;
8810 prog->aux->func_cnt = env->subprog_cnt;
8811 bpf_prog_free_unused_jited_linfo(prog);
8812 return 0;
8813out_free:
8814 for (i = 0; i < env->subprog_cnt; i++)
8815 if (func[i])
8816 bpf_jit_free(func[i]);
8817 kfree(func);
8818out_undo_insn:
8819
8820 prog->jit_requested = 0;
8821 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
8822 if (insn->code != (BPF_JMP | BPF_CALL) ||
8823 insn->src_reg != BPF_PSEUDO_CALL)
8824 continue;
8825 insn->off = 0;
8826 insn->imm = env->insn_aux_data[i].call_imm;
8827 }
8828 bpf_prog_free_jited_linfo(prog);
8829 return err;
8830}
8831
8832static int fixup_call_args(struct bpf_verifier_env *env)
8833{
8834#ifndef CONFIG_BPF_JIT_ALWAYS_ON
8835 struct bpf_prog *prog = env->prog;
8836 struct bpf_insn *insn = prog->insnsi;
8837 int i, depth;
8838#endif
8839 int err = 0;
8840
8841 if (env->prog->jit_requested &&
8842 !bpf_prog_is_dev_bound(env->prog->aux)) {
8843 err = jit_subprogs(env);
8844 if (err == 0)
8845 return 0;
8846 if (err == -EFAULT)
8847 return err;
8848 }
8849#ifndef CONFIG_BPF_JIT_ALWAYS_ON
8850 for (i = 0; i < prog->len; i++, insn++) {
8851 if (insn->code != (BPF_JMP | BPF_CALL) ||
8852 insn->src_reg != BPF_PSEUDO_CALL)
8853 continue;
8854 depth = get_callee_stack_depth(env, insn, i);
8855 if (depth < 0)
8856 return depth;
8857 bpf_patch_call_args(insn, depth);
8858 }
8859 err = 0;
8860#endif
8861 return err;
8862}
8863
8864
8865
8866
8867
8868
8869static int fixup_bpf_calls(struct bpf_verifier_env *env)
8870{
8871 struct bpf_prog *prog = env->prog;
8872 struct bpf_insn *insn = prog->insnsi;
8873 const struct bpf_func_proto *fn;
8874 const int insn_cnt = prog->len;
8875 const struct bpf_map_ops *ops;
8876 struct bpf_insn_aux_data *aux;
8877 struct bpf_insn insn_buf[16];
8878 struct bpf_prog *new_prog;
8879 struct bpf_map *map_ptr;
8880 int i, cnt, delta = 0;
8881
8882 for (i = 0; i < insn_cnt; i++, insn++) {
8883 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
8884 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
8885 insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
8886 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
8887 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
8888 struct bpf_insn mask_and_div[] = {
8889 BPF_MOV32_REG(insn->src_reg, insn->src_reg),
8890
8891 BPF_JMP_IMM(BPF_JNE, insn->src_reg, 0, 2),
8892 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),
8893 BPF_JMP_IMM(BPF_JA, 0, 0, 1),
8894 *insn,
8895 };
8896 struct bpf_insn mask_and_mod[] = {
8897 BPF_MOV32_REG(insn->src_reg, insn->src_reg),
8898
8899 BPF_JMP_IMM(BPF_JEQ, insn->src_reg, 0, 1),
8900 *insn,
8901 };
8902 struct bpf_insn *patchlet;
8903
8904 if (insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
8905 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
8906 patchlet = mask_and_div + (is64 ? 1 : 0);
8907 cnt = ARRAY_SIZE(mask_and_div) - (is64 ? 1 : 0);
8908 } else {
8909 patchlet = mask_and_mod + (is64 ? 1 : 0);
8910 cnt = ARRAY_SIZE(mask_and_mod) - (is64 ? 1 : 0);
8911 }
8912
8913 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
8914 if (!new_prog)
8915 return -ENOMEM;
8916
8917 delta += cnt - 1;
8918 env->prog = prog = new_prog;
8919 insn = new_prog->insnsi + i + delta;
8920 continue;
8921 }
8922
8923 if (BPF_CLASS(insn->code) == BPF_LD &&
8924 (BPF_MODE(insn->code) == BPF_ABS ||
8925 BPF_MODE(insn->code) == BPF_IND)) {
8926 cnt = env->ops->gen_ld_abs(insn, insn_buf);
8927 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
8928 verbose(env, "bpf verifier is misconfigured\n");
8929 return -EINVAL;
8930 }
8931
8932 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
8933 if (!new_prog)
8934 return -ENOMEM;
8935
8936 delta += cnt - 1;
8937 env->prog = prog = new_prog;
8938 insn = new_prog->insnsi + i + delta;
8939 continue;
8940 }
8941
8942 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
8943 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
8944 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
8945 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
8946 struct bpf_insn insn_buf[16];
8947 struct bpf_insn *patch = &insn_buf[0];
8948 bool issrc, isneg;
8949 u32 off_reg;
8950
8951 aux = &env->insn_aux_data[i + delta];
8952 if (!aux->alu_state ||
8953 aux->alu_state == BPF_ALU_NON_POINTER)
8954 continue;
8955
8956 isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
8957 issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
8958 BPF_ALU_SANITIZE_SRC;
8959
8960 off_reg = issrc ? insn->src_reg : insn->dst_reg;
8961 if (isneg)
8962 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
8963 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit - 1);
8964 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
8965 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
8966 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
8967 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
8968 if (issrc) {
8969 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX,
8970 off_reg);
8971 insn->src_reg = BPF_REG_AX;
8972 } else {
8973 *patch++ = BPF_ALU64_REG(BPF_AND, off_reg,
8974 BPF_REG_AX);
8975 }
8976 if (isneg)
8977 insn->code = insn->code == code_add ?
8978 code_sub : code_add;
8979 *patch++ = *insn;
8980 if (issrc && isneg)
8981 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
8982 cnt = patch - insn_buf;
8983
8984 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
8985 if (!new_prog)
8986 return -ENOMEM;
8987
8988 delta += cnt - 1;
8989 env->prog = prog = new_prog;
8990 insn = new_prog->insnsi + i + delta;
8991 continue;
8992 }
8993
8994 if (insn->code != (BPF_JMP | BPF_CALL))
8995 continue;
8996 if (insn->src_reg == BPF_PSEUDO_CALL)
8997 continue;
8998
8999 if (insn->imm == BPF_FUNC_get_route_realm)
9000 prog->dst_needed = 1;
9001 if (insn->imm == BPF_FUNC_get_prandom_u32)
9002 bpf_user_rnd_init_once();
9003 if (insn->imm == BPF_FUNC_override_return)
9004 prog->kprobe_override = 1;
9005 if (insn->imm == BPF_FUNC_tail_call) {
9006
9007
9008
9009
9010
9011 prog->cb_access = 1;
9012 env->prog->aux->stack_depth = MAX_BPF_STACK;
9013 env->prog->aux->max_pkt_offset = MAX_PACKET_OFF;
9014
9015
9016
9017
9018
9019
9020 insn->imm = 0;
9021 insn->code = BPF_JMP | BPF_TAIL_CALL;
9022
9023 aux = &env->insn_aux_data[i + delta];
9024 if (!bpf_map_ptr_unpriv(aux))
9025 continue;
9026
9027
9028
9029
9030
9031
9032
9033 if (bpf_map_ptr_poisoned(aux)) {
9034 verbose(env, "tail_call abusing map_ptr\n");
9035 return -EINVAL;
9036 }
9037
9038 map_ptr = BPF_MAP_PTR(aux->map_state);
9039 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
9040 map_ptr->max_entries, 2);
9041 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
9042 container_of(map_ptr,
9043 struct bpf_array,
9044 map)->index_mask);
9045 insn_buf[2] = *insn;
9046 cnt = 3;
9047 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
9048 if (!new_prog)
9049 return -ENOMEM;
9050
9051 delta += cnt - 1;
9052 env->prog = prog = new_prog;
9053 insn = new_prog->insnsi + i + delta;
9054 continue;
9055 }
9056
9057
9058
9059
9060
9061 if (prog->jit_requested && BITS_PER_LONG == 64 &&
9062 (insn->imm == BPF_FUNC_map_lookup_elem ||
9063 insn->imm == BPF_FUNC_map_update_elem ||
9064 insn->imm == BPF_FUNC_map_delete_elem ||
9065 insn->imm == BPF_FUNC_map_push_elem ||
9066 insn->imm == BPF_FUNC_map_pop_elem ||
9067 insn->imm == BPF_FUNC_map_peek_elem)) {
9068 aux = &env->insn_aux_data[i + delta];
9069 if (bpf_map_ptr_poisoned(aux))
9070 goto patch_call_imm;
9071
9072 map_ptr = BPF_MAP_PTR(aux->map_state);
9073 ops = map_ptr->ops;
9074 if (insn->imm == BPF_FUNC_map_lookup_elem &&
9075 ops->map_gen_lookup) {
9076 cnt = ops->map_gen_lookup(map_ptr, insn_buf);
9077 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
9078 verbose(env, "bpf verifier is misconfigured\n");
9079 return -EINVAL;
9080 }
9081
9082 new_prog = bpf_patch_insn_data(env, i + delta,
9083 insn_buf, cnt);
9084 if (!new_prog)
9085 return -ENOMEM;
9086
9087 delta += cnt - 1;
9088 env->prog = prog = new_prog;
9089 insn = new_prog->insnsi + i + delta;
9090 continue;
9091 }
9092
9093 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
9094 (void *(*)(struct bpf_map *map, void *key))NULL));
9095 BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
9096 (int (*)(struct bpf_map *map, void *key))NULL));
9097 BUILD_BUG_ON(!__same_type(ops->map_update_elem,
9098 (int (*)(struct bpf_map *map, void *key, void *value,
9099 u64 flags))NULL));
9100 BUILD_BUG_ON(!__same_type(ops->map_push_elem,
9101 (int (*)(struct bpf_map *map, void *value,
9102 u64 flags))NULL));
9103 BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
9104 (int (*)(struct bpf_map *map, void *value))NULL));
9105 BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
9106 (int (*)(struct bpf_map *map, void *value))NULL));
9107
9108 switch (insn->imm) {
9109 case BPF_FUNC_map_lookup_elem:
9110 insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -
9111 __bpf_call_base;
9112 continue;
9113 case BPF_FUNC_map_update_elem:
9114 insn->imm = BPF_CAST_CALL(ops->map_update_elem) -
9115 __bpf_call_base;
9116 continue;
9117 case BPF_FUNC_map_delete_elem:
9118 insn->imm = BPF_CAST_CALL(ops->map_delete_elem) -
9119 __bpf_call_base;
9120 continue;
9121 case BPF_FUNC_map_push_elem:
9122 insn->imm = BPF_CAST_CALL(ops->map_push_elem) -
9123 __bpf_call_base;
9124 continue;
9125 case BPF_FUNC_map_pop_elem:
9126 insn->imm = BPF_CAST_CALL(ops->map_pop_elem) -
9127 __bpf_call_base;
9128 continue;
9129 case BPF_FUNC_map_peek_elem:
9130 insn->imm = BPF_CAST_CALL(ops->map_peek_elem) -
9131 __bpf_call_base;
9132 continue;
9133 }
9134
9135 goto patch_call_imm;
9136 }
9137
9138patch_call_imm:
9139 fn = env->ops->get_func_proto(insn->imm, env->prog);
9140
9141
9142
9143 if (!fn->func) {
9144 verbose(env,
9145 "kernel subsystem misconfigured func %s#%d\n",
9146 func_id_name(insn->imm), insn->imm);
9147 return -EFAULT;
9148 }
9149 insn->imm = fn->func - __bpf_call_base;
9150 }
9151
9152 return 0;
9153}
9154
9155static void free_states(struct bpf_verifier_env *env)
9156{
9157 struct bpf_verifier_state_list *sl, *sln;
9158 int i;
9159
9160 sl = env->free_list;
9161 while (sl) {
9162 sln = sl->next;
9163 free_verifier_state(&sl->state, false);
9164 kfree(sl);
9165 sl = sln;
9166 }
9167
9168 if (!env->explored_states)
9169 return;
9170
9171 for (i = 0; i < state_htab_size(env); i++) {
9172 sl = env->explored_states[i];
9173
9174 while (sl) {
9175 sln = sl->next;
9176 free_verifier_state(&sl->state, false);
9177 kfree(sl);
9178 sl = sln;
9179 }
9180 }
9181
9182 kvfree(env->explored_states);
9183}
9184
9185static void print_verification_stats(struct bpf_verifier_env *env)
9186{
9187 int i;
9188
9189 if (env->log.level & BPF_LOG_STATS) {
9190 verbose(env, "verification time %lld usec\n",
9191 div_u64(env->verification_time, 1000));
9192 verbose(env, "stack depth ");
9193 for (i = 0; i < env->subprog_cnt; i++) {
9194 u32 depth = env->subprog_info[i].stack_depth;
9195
9196 verbose(env, "%d", depth);
9197 if (i + 1 < env->subprog_cnt)
9198 verbose(env, "+");
9199 }
9200 verbose(env, "\n");
9201 }
9202 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "
9203 "total_states %d peak_states %d mark_read %d\n",
9204 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,
9205 env->max_states_per_insn, env->total_states,
9206 env->peak_states, env->longest_mark_read_walk);
9207}
9208
9209int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
9210 union bpf_attr __user *uattr)
9211{
9212 u64 start_time = ktime_get_ns();
9213 struct bpf_verifier_env *env;
9214 struct bpf_verifier_log *log;
9215 int i, len, ret = -EINVAL;
9216 bool is_priv;
9217
9218
9219 if (ARRAY_SIZE(bpf_verifier_ops) == 0)
9220 return -EINVAL;
9221
9222
9223
9224
9225 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);
9226 if (!env)
9227 return -ENOMEM;
9228 log = &env->log;
9229
9230 len = (*prog)->len;
9231 env->insn_aux_data =
9232 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
9233 ret = -ENOMEM;
9234 if (!env->insn_aux_data)
9235 goto err_free_env;
9236 for (i = 0; i < len; i++)
9237 env->insn_aux_data[i].orig_idx = i;
9238 env->prog = *prog;
9239 env->ops = bpf_verifier_ops[env->prog->type];
9240 is_priv = capable(CAP_SYS_ADMIN);
9241
9242
9243 if (!is_priv)
9244 mutex_lock(&bpf_verifier_lock);
9245
9246 if (attr->log_level || attr->log_buf || attr->log_size) {
9247
9248
9249
9250 log->level = attr->log_level;
9251 log->ubuf = (char __user *) (unsigned long) attr->log_buf;
9252 log->len_total = attr->log_size;
9253
9254 ret = -EINVAL;
9255
9256 if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 ||
9257 !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK)
9258 goto err_unlock;
9259 }
9260
9261 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
9262 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
9263 env->strict_alignment = true;
9264 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
9265 env->strict_alignment = false;
9266
9267 env->allow_ptr_leaks = is_priv;
9268
9269 ret = replace_map_fd_with_map_ptr(env);
9270 if (ret < 0)
9271 goto skip_full_check;
9272
9273 if (bpf_prog_is_dev_bound(env->prog->aux)) {
9274 ret = bpf_prog_offload_verifier_prep(env->prog);
9275 if (ret)
9276 goto skip_full_check;
9277 }
9278
9279 env->explored_states = kvcalloc(state_htab_size(env),
9280 sizeof(struct bpf_verifier_state_list *),
9281 GFP_USER);
9282 ret = -ENOMEM;
9283 if (!env->explored_states)
9284 goto skip_full_check;
9285
9286 ret = check_subprogs(env);
9287 if (ret < 0)
9288 goto skip_full_check;
9289
9290 ret = check_btf_info(env, attr, uattr);
9291 if (ret < 0)
9292 goto skip_full_check;
9293
9294 ret = check_cfg(env);
9295 if (ret < 0)
9296 goto skip_full_check;
9297
9298 ret = do_check(env);
9299 if (env->cur_state) {
9300 free_verifier_state(env->cur_state, true);
9301 env->cur_state = NULL;
9302 }
9303
9304 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux))
9305 ret = bpf_prog_offload_finalize(env);
9306
9307skip_full_check:
9308 while (!pop_stack(env, NULL, NULL));
9309 free_states(env);
9310
9311 if (ret == 0)
9312 ret = check_max_stack_depth(env);
9313
9314
9315 if (is_priv) {
9316 if (ret == 0)
9317 opt_hard_wire_dead_code_branches(env);
9318 if (ret == 0)
9319 ret = opt_remove_dead_code(env);
9320 if (ret == 0)
9321 ret = opt_remove_nops(env);
9322 } else {
9323 if (ret == 0)
9324 sanitize_dead_code(env);
9325 }
9326
9327 if (ret == 0)
9328
9329 ret = convert_ctx_accesses(env);
9330
9331 if (ret == 0)
9332 ret = fixup_bpf_calls(env);
9333
9334
9335
9336
9337 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) {
9338 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr);
9339 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret
9340 : false;
9341 }
9342
9343 if (ret == 0)
9344 ret = fixup_call_args(env);
9345
9346 env->verification_time = ktime_get_ns() - start_time;
9347 print_verification_stats(env);
9348
9349 if (log->level && bpf_verifier_log_full(log))
9350 ret = -ENOSPC;
9351 if (log->level && !log->ubuf) {
9352 ret = -EFAULT;
9353 goto err_release_maps;
9354 }
9355
9356 if (ret == 0 && env->used_map_cnt) {
9357
9358 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
9359 sizeof(env->used_maps[0]),
9360 GFP_KERNEL);
9361
9362 if (!env->prog->aux->used_maps) {
9363 ret = -ENOMEM;
9364 goto err_release_maps;
9365 }
9366
9367 memcpy(env->prog->aux->used_maps, env->used_maps,
9368 sizeof(env->used_maps[0]) * env->used_map_cnt);
9369 env->prog->aux->used_map_cnt = env->used_map_cnt;
9370
9371
9372
9373
9374 convert_pseudo_ld_imm64(env);
9375 }
9376
9377 if (ret == 0)
9378 adjust_btf_func(env);
9379
9380err_release_maps:
9381 if (!env->prog->aux->used_maps)
9382
9383
9384
9385 release_maps(env);
9386 *prog = env->prog;
9387err_unlock:
9388 if (!is_priv)
9389 mutex_unlock(&bpf_verifier_lock);
9390 vfree(env->insn_aux_data);
9391err_free_env:
9392 kfree(env);
9393 return ret;
9394}
9395