1
2
3
4#include <uapi/linux/btf.h>
5#include <uapi/linux/types.h>
6#include <linux/seq_file.h>
7#include <linux/compiler.h>
8#include <linux/ctype.h>
9#include <linux/errno.h>
10#include <linux/slab.h>
11#include <linux/anon_inodes.h>
12#include <linux/file.h>
13#include <linux/uaccess.h>
14#include <linux/kernel.h>
15#include <linux/idr.h>
16#include <linux/sort.h>
17#include <linux/bpf_verifier.h>
18#include <linux/btf.h>
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160#define BITS_PER_U128 (sizeof(u64) * BITS_PER_BYTE * 2)
161#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1)
162#define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK)
163#define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3)
164#define BITS_ROUNDUP_BYTES(bits) \
165 (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits))
166
167#define BTF_INFO_MASK 0x8f00ffff
168#define BTF_INT_MASK 0x0fffffff
169#define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE)
170#define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET)
171
172
173
174
175
176#define BTF_MAX_SIZE (16 * 1024 * 1024)
177
178#define for_each_member(i, struct_type, member) \
179 for (i = 0, member = btf_type_member(struct_type); \
180 i < btf_type_vlen(struct_type); \
181 i++, member++)
182
183#define for_each_member_from(i, from, struct_type, member) \
184 for (i = from, member = btf_type_member(struct_type) + from; \
185 i < btf_type_vlen(struct_type); \
186 i++, member++)
187
188#define for_each_vsi(i, struct_type, member) \
189 for (i = 0, member = btf_type_var_secinfo(struct_type); \
190 i < btf_type_vlen(struct_type); \
191 i++, member++)
192
193#define for_each_vsi_from(i, from, struct_type, member) \
194 for (i = from, member = btf_type_var_secinfo(struct_type) + from; \
195 i < btf_type_vlen(struct_type); \
196 i++, member++)
197
198static DEFINE_IDR(btf_idr);
199static DEFINE_SPINLOCK(btf_idr_lock);
200
201struct btf {
202 void *data;
203 struct btf_type **types;
204 u32 *resolved_ids;
205 u32 *resolved_sizes;
206 const char *strings;
207 void *nohdr_data;
208 struct btf_header hdr;
209 u32 nr_types;
210 u32 types_size;
211 u32 data_size;
212 refcount_t refcnt;
213 u32 id;
214 struct rcu_head rcu;
215};
216
217enum verifier_phase {
218 CHECK_META,
219 CHECK_TYPE,
220};
221
222struct resolve_vertex {
223 const struct btf_type *t;
224 u32 type_id;
225 u16 next_member;
226};
227
228enum visit_state {
229 NOT_VISITED,
230 VISITED,
231 RESOLVED,
232};
233
234enum resolve_mode {
235 RESOLVE_TBD,
236 RESOLVE_PTR,
237 RESOLVE_STRUCT_OR_ARRAY,
238
239
240};
241
242#define MAX_RESOLVE_DEPTH 32
243
244struct btf_sec_info {
245 u32 off;
246 u32 len;
247};
248
249struct btf_verifier_env {
250 struct btf *btf;
251 u8 *visit_states;
252 struct resolve_vertex stack[MAX_RESOLVE_DEPTH];
253 struct bpf_verifier_log log;
254 u32 log_type_id;
255 u32 top_stack;
256 enum verifier_phase phase;
257 enum resolve_mode resolve_mode;
258};
259
260static const char * const btf_kind_str[NR_BTF_KINDS] = {
261 [BTF_KIND_UNKN] = "UNKNOWN",
262 [BTF_KIND_INT] = "INT",
263 [BTF_KIND_PTR] = "PTR",
264 [BTF_KIND_ARRAY] = "ARRAY",
265 [BTF_KIND_STRUCT] = "STRUCT",
266 [BTF_KIND_UNION] = "UNION",
267 [BTF_KIND_ENUM] = "ENUM",
268 [BTF_KIND_FWD] = "FWD",
269 [BTF_KIND_TYPEDEF] = "TYPEDEF",
270 [BTF_KIND_VOLATILE] = "VOLATILE",
271 [BTF_KIND_CONST] = "CONST",
272 [BTF_KIND_RESTRICT] = "RESTRICT",
273 [BTF_KIND_FUNC] = "FUNC",
274 [BTF_KIND_FUNC_PROTO] = "FUNC_PROTO",
275 [BTF_KIND_VAR] = "VAR",
276 [BTF_KIND_DATASEC] = "DATASEC",
277};
278
279struct btf_kind_operations {
280 s32 (*check_meta)(struct btf_verifier_env *env,
281 const struct btf_type *t,
282 u32 meta_left);
283 int (*resolve)(struct btf_verifier_env *env,
284 const struct resolve_vertex *v);
285 int (*check_member)(struct btf_verifier_env *env,
286 const struct btf_type *struct_type,
287 const struct btf_member *member,
288 const struct btf_type *member_type);
289 int (*check_kflag_member)(struct btf_verifier_env *env,
290 const struct btf_type *struct_type,
291 const struct btf_member *member,
292 const struct btf_type *member_type);
293 void (*log_details)(struct btf_verifier_env *env,
294 const struct btf_type *t);
295 void (*seq_show)(const struct btf *btf, const struct btf_type *t,
296 u32 type_id, void *data, u8 bits_offsets,
297 struct seq_file *m);
298};
299
300static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS];
301static struct btf_type btf_void;
302
303static int btf_resolve(struct btf_verifier_env *env,
304 const struct btf_type *t, u32 type_id);
305
306static bool btf_type_is_modifier(const struct btf_type *t)
307{
308
309
310
311
312
313
314
315
316
317
318 switch (BTF_INFO_KIND(t->info)) {
319 case BTF_KIND_TYPEDEF:
320 case BTF_KIND_VOLATILE:
321 case BTF_KIND_CONST:
322 case BTF_KIND_RESTRICT:
323 return true;
324 }
325
326 return false;
327}
328
329bool btf_type_is_void(const struct btf_type *t)
330{
331 return t == &btf_void;
332}
333
334static bool btf_type_is_fwd(const struct btf_type *t)
335{
336 return BTF_INFO_KIND(t->info) == BTF_KIND_FWD;
337}
338
339static bool btf_type_is_func(const struct btf_type *t)
340{
341 return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC;
342}
343
344static bool btf_type_is_func_proto(const struct btf_type *t)
345{
346 return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO;
347}
348
349static bool btf_type_nosize(const struct btf_type *t)
350{
351 return btf_type_is_void(t) || btf_type_is_fwd(t) ||
352 btf_type_is_func(t) || btf_type_is_func_proto(t);
353}
354
355static bool btf_type_nosize_or_null(const struct btf_type *t)
356{
357 return !t || btf_type_nosize(t);
358}
359
360
361
362
363static bool btf_type_is_struct(const struct btf_type *t)
364{
365 u8 kind = BTF_INFO_KIND(t->info);
366
367 return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION;
368}
369
370static bool __btf_type_is_struct(const struct btf_type *t)
371{
372 return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT;
373}
374
375static bool btf_type_is_array(const struct btf_type *t)
376{
377 return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
378}
379
380static bool btf_type_is_ptr(const struct btf_type *t)
381{
382 return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;
383}
384
385static bool btf_type_is_int(const struct btf_type *t)
386{
387 return BTF_INFO_KIND(t->info) == BTF_KIND_INT;
388}
389
390static bool btf_type_is_var(const struct btf_type *t)
391{
392 return BTF_INFO_KIND(t->info) == BTF_KIND_VAR;
393}
394
395static bool btf_type_is_datasec(const struct btf_type *t)
396{
397 return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
398}
399
400
401
402
403static bool btf_type_is_resolve_source_only(const struct btf_type *t)
404{
405 return btf_type_is_var(t) ||
406 btf_type_is_datasec(t);
407}
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425static bool btf_type_needs_resolve(const struct btf_type *t)
426{
427 return btf_type_is_modifier(t) ||
428 btf_type_is_ptr(t) ||
429 btf_type_is_struct(t) ||
430 btf_type_is_array(t) ||
431 btf_type_is_var(t) ||
432 btf_type_is_datasec(t);
433}
434
435
436static bool btf_type_has_size(const struct btf_type *t)
437{
438 switch (BTF_INFO_KIND(t->info)) {
439 case BTF_KIND_INT:
440 case BTF_KIND_STRUCT:
441 case BTF_KIND_UNION:
442 case BTF_KIND_ENUM:
443 case BTF_KIND_DATASEC:
444 return true;
445 }
446
447 return false;
448}
449
450static const char *btf_int_encoding_str(u8 encoding)
451{
452 if (encoding == 0)
453 return "(none)";
454 else if (encoding == BTF_INT_SIGNED)
455 return "SIGNED";
456 else if (encoding == BTF_INT_CHAR)
457 return "CHAR";
458 else if (encoding == BTF_INT_BOOL)
459 return "BOOL";
460 else
461 return "UNKN";
462}
463
464static u16 btf_type_vlen(const struct btf_type *t)
465{
466 return BTF_INFO_VLEN(t->info);
467}
468
469static bool btf_type_kflag(const struct btf_type *t)
470{
471 return BTF_INFO_KFLAG(t->info);
472}
473
474static u32 btf_member_bit_offset(const struct btf_type *struct_type,
475 const struct btf_member *member)
476{
477 return btf_type_kflag(struct_type) ? BTF_MEMBER_BIT_OFFSET(member->offset)
478 : member->offset;
479}
480
481static u32 btf_member_bitfield_size(const struct btf_type *struct_type,
482 const struct btf_member *member)
483{
484 return btf_type_kflag(struct_type) ? BTF_MEMBER_BITFIELD_SIZE(member->offset)
485 : 0;
486}
487
488static u32 btf_type_int(const struct btf_type *t)
489{
490 return *(u32 *)(t + 1);
491}
492
493static const struct btf_array *btf_type_array(const struct btf_type *t)
494{
495 return (const struct btf_array *)(t + 1);
496}
497
498static const struct btf_member *btf_type_member(const struct btf_type *t)
499{
500 return (const struct btf_member *)(t + 1);
501}
502
503static const struct btf_enum *btf_type_enum(const struct btf_type *t)
504{
505 return (const struct btf_enum *)(t + 1);
506}
507
508static const struct btf_var *btf_type_var(const struct btf_type *t)
509{
510 return (const struct btf_var *)(t + 1);
511}
512
513static const struct btf_var_secinfo *btf_type_var_secinfo(const struct btf_type *t)
514{
515 return (const struct btf_var_secinfo *)(t + 1);
516}
517
518static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
519{
520 return kind_ops[BTF_INFO_KIND(t->info)];
521}
522
523static bool btf_name_offset_valid(const struct btf *btf, u32 offset)
524{
525 return BTF_STR_OFFSET_VALID(offset) &&
526 offset < btf->hdr.str_len;
527}
528
529static bool __btf_name_char_ok(char c, bool first, bool dot_ok)
530{
531 if ((first ? !isalpha(c) :
532 !isalnum(c)) &&
533 c != '_' &&
534 ((c == '.' && !dot_ok) ||
535 c != '.'))
536 return false;
537 return true;
538}
539
540static bool __btf_name_valid(const struct btf *btf, u32 offset, bool dot_ok)
541{
542
543 const char *src = &btf->strings[offset];
544 const char *src_limit;
545
546 if (!__btf_name_char_ok(*src, true, dot_ok))
547 return false;
548
549
550 src_limit = src + KSYM_NAME_LEN;
551 src++;
552 while (*src && src < src_limit) {
553 if (!__btf_name_char_ok(*src, false, dot_ok))
554 return false;
555 src++;
556 }
557
558 return !*src;
559}
560
561
562
563
564static bool btf_name_valid_identifier(const struct btf *btf, u32 offset)
565{
566 return __btf_name_valid(btf, offset, false);
567}
568
569static bool btf_name_valid_section(const struct btf *btf, u32 offset)
570{
571 return __btf_name_valid(btf, offset, true);
572}
573
574static const char *__btf_name_by_offset(const struct btf *btf, u32 offset)
575{
576 if (!offset)
577 return "(anon)";
578 else if (offset < btf->hdr.str_len)
579 return &btf->strings[offset];
580 else
581 return "(invalid-name-offset)";
582}
583
584const char *btf_name_by_offset(const struct btf *btf, u32 offset)
585{
586 if (offset < btf->hdr.str_len)
587 return &btf->strings[offset];
588
589 return NULL;
590}
591
592const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id)
593{
594 if (type_id > btf->nr_types)
595 return NULL;
596
597 return btf->types[type_id];
598}
599
600
601
602
603
604static bool btf_type_int_is_regular(const struct btf_type *t)
605{
606 u8 nr_bits, nr_bytes;
607 u32 int_data;
608
609 int_data = btf_type_int(t);
610 nr_bits = BTF_INT_BITS(int_data);
611 nr_bytes = BITS_ROUNDUP_BYTES(nr_bits);
612 if (BITS_PER_BYTE_MASKED(nr_bits) ||
613 BTF_INT_OFFSET(int_data) ||
614 (nr_bytes != sizeof(u8) && nr_bytes != sizeof(u16) &&
615 nr_bytes != sizeof(u32) && nr_bytes != sizeof(u64) &&
616 nr_bytes != (2 * sizeof(u64)))) {
617 return false;
618 }
619
620 return true;
621}
622
623
624
625
626
627bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s,
628 const struct btf_member *m,
629 u32 expected_offset, u32 expected_size)
630{
631 const struct btf_type *t;
632 u32 id, int_data;
633 u8 nr_bits;
634
635 id = m->type;
636 t = btf_type_id_size(btf, &id, NULL);
637 if (!t || !btf_type_is_int(t))
638 return false;
639
640 int_data = btf_type_int(t);
641 nr_bits = BTF_INT_BITS(int_data);
642 if (btf_type_kflag(s)) {
643 u32 bitfield_size = BTF_MEMBER_BITFIELD_SIZE(m->offset);
644 u32 bit_offset = BTF_MEMBER_BIT_OFFSET(m->offset);
645
646
647
648
649 return !bitfield_size &&
650 BITS_ROUNDUP_BYTES(bit_offset) == expected_offset &&
651 BITS_ROUNDUP_BYTES(nr_bits) == expected_size;
652 }
653
654 if (BTF_INT_OFFSET(int_data) ||
655 BITS_PER_BYTE_MASKED(m->offset) ||
656 BITS_ROUNDUP_BYTES(m->offset) != expected_offset ||
657 BITS_PER_BYTE_MASKED(nr_bits) ||
658 BITS_ROUNDUP_BYTES(nr_bits) != expected_size)
659 return false;
660
661 return true;
662}
663
664__printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log,
665 const char *fmt, ...)
666{
667 va_list args;
668
669 va_start(args, fmt);
670 bpf_verifier_vlog(log, fmt, args);
671 va_end(args);
672}
673
674__printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env,
675 const char *fmt, ...)
676{
677 struct bpf_verifier_log *log = &env->log;
678 va_list args;
679
680 if (!bpf_verifier_log_needed(log))
681 return;
682
683 va_start(args, fmt);
684 bpf_verifier_vlog(log, fmt, args);
685 va_end(args);
686}
687
688__printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env,
689 const struct btf_type *t,
690 bool log_details,
691 const char *fmt, ...)
692{
693 struct bpf_verifier_log *log = &env->log;
694 u8 kind = BTF_INFO_KIND(t->info);
695 struct btf *btf = env->btf;
696 va_list args;
697
698 if (!bpf_verifier_log_needed(log))
699 return;
700
701 __btf_verifier_log(log, "[%u] %s %s%s",
702 env->log_type_id,
703 btf_kind_str[kind],
704 __btf_name_by_offset(btf, t->name_off),
705 log_details ? " " : "");
706
707 if (log_details)
708 btf_type_ops(t)->log_details(env, t);
709
710 if (fmt && *fmt) {
711 __btf_verifier_log(log, " ");
712 va_start(args, fmt);
713 bpf_verifier_vlog(log, fmt, args);
714 va_end(args);
715 }
716
717 __btf_verifier_log(log, "\n");
718}
719
720#define btf_verifier_log_type(env, t, ...) \
721 __btf_verifier_log_type((env), (t), true, __VA_ARGS__)
722#define btf_verifier_log_basic(env, t, ...) \
723 __btf_verifier_log_type((env), (t), false, __VA_ARGS__)
724
725__printf(4, 5)
726static void btf_verifier_log_member(struct btf_verifier_env *env,
727 const struct btf_type *struct_type,
728 const struct btf_member *member,
729 const char *fmt, ...)
730{
731 struct bpf_verifier_log *log = &env->log;
732 struct btf *btf = env->btf;
733 va_list args;
734
735 if (!bpf_verifier_log_needed(log))
736 return;
737
738
739
740
741
742
743
744 if (env->phase != CHECK_META)
745 btf_verifier_log_type(env, struct_type, NULL);
746
747 if (btf_type_kflag(struct_type))
748 __btf_verifier_log(log,
749 "\t%s type_id=%u bitfield_size=%u bits_offset=%u",
750 __btf_name_by_offset(btf, member->name_off),
751 member->type,
752 BTF_MEMBER_BITFIELD_SIZE(member->offset),
753 BTF_MEMBER_BIT_OFFSET(member->offset));
754 else
755 __btf_verifier_log(log, "\t%s type_id=%u bits_offset=%u",
756 __btf_name_by_offset(btf, member->name_off),
757 member->type, member->offset);
758
759 if (fmt && *fmt) {
760 __btf_verifier_log(log, " ");
761 va_start(args, fmt);
762 bpf_verifier_vlog(log, fmt, args);
763 va_end(args);
764 }
765
766 __btf_verifier_log(log, "\n");
767}
768
769__printf(4, 5)
770static void btf_verifier_log_vsi(struct btf_verifier_env *env,
771 const struct btf_type *datasec_type,
772 const struct btf_var_secinfo *vsi,
773 const char *fmt, ...)
774{
775 struct bpf_verifier_log *log = &env->log;
776 va_list args;
777
778 if (!bpf_verifier_log_needed(log))
779 return;
780 if (env->phase != CHECK_META)
781 btf_verifier_log_type(env, datasec_type, NULL);
782
783 __btf_verifier_log(log, "\t type_id=%u offset=%u size=%u",
784 vsi->type, vsi->offset, vsi->size);
785 if (fmt && *fmt) {
786 __btf_verifier_log(log, " ");
787 va_start(args, fmt);
788 bpf_verifier_vlog(log, fmt, args);
789 va_end(args);
790 }
791
792 __btf_verifier_log(log, "\n");
793}
794
795static void btf_verifier_log_hdr(struct btf_verifier_env *env,
796 u32 btf_data_size)
797{
798 struct bpf_verifier_log *log = &env->log;
799 const struct btf *btf = env->btf;
800 const struct btf_header *hdr;
801
802 if (!bpf_verifier_log_needed(log))
803 return;
804
805 hdr = &btf->hdr;
806 __btf_verifier_log(log, "magic: 0x%x\n", hdr->magic);
807 __btf_verifier_log(log, "version: %u\n", hdr->version);
808 __btf_verifier_log(log, "flags: 0x%x\n", hdr->flags);
809 __btf_verifier_log(log, "hdr_len: %u\n", hdr->hdr_len);
810 __btf_verifier_log(log, "type_off: %u\n", hdr->type_off);
811 __btf_verifier_log(log, "type_len: %u\n", hdr->type_len);
812 __btf_verifier_log(log, "str_off: %u\n", hdr->str_off);
813 __btf_verifier_log(log, "str_len: %u\n", hdr->str_len);
814 __btf_verifier_log(log, "btf_total_size: %u\n", btf_data_size);
815}
816
817static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t)
818{
819 struct btf *btf = env->btf;
820
821
822
823
824
825 if (btf->types_size - btf->nr_types < 2) {
826
827
828 struct btf_type **new_types;
829 u32 expand_by, new_size;
830
831 if (btf->types_size == BTF_MAX_TYPE) {
832 btf_verifier_log(env, "Exceeded max num of types");
833 return -E2BIG;
834 }
835
836 expand_by = max_t(u32, btf->types_size >> 2, 16);
837 new_size = min_t(u32, BTF_MAX_TYPE,
838 btf->types_size + expand_by);
839
840 new_types = kvcalloc(new_size, sizeof(*new_types),
841 GFP_KERNEL | __GFP_NOWARN);
842 if (!new_types)
843 return -ENOMEM;
844
845 if (btf->nr_types == 0)
846 new_types[0] = &btf_void;
847 else
848 memcpy(new_types, btf->types,
849 sizeof(*btf->types) * (btf->nr_types + 1));
850
851 kvfree(btf->types);
852 btf->types = new_types;
853 btf->types_size = new_size;
854 }
855
856 btf->types[++(btf->nr_types)] = t;
857
858 return 0;
859}
860
861static int btf_alloc_id(struct btf *btf)
862{
863 int id;
864
865 idr_preload(GFP_KERNEL);
866 spin_lock_bh(&btf_idr_lock);
867 id = idr_alloc_cyclic(&btf_idr, btf, 1, INT_MAX, GFP_ATOMIC);
868 if (id > 0)
869 btf->id = id;
870 spin_unlock_bh(&btf_idr_lock);
871 idr_preload_end();
872
873 if (WARN_ON_ONCE(!id))
874 return -ENOSPC;
875
876 return id > 0 ? 0 : id;
877}
878
879static void btf_free_id(struct btf *btf)
880{
881 unsigned long flags;
882
883
884
885
886
887
888
889
890
891
892 spin_lock_irqsave(&btf_idr_lock, flags);
893 idr_remove(&btf_idr, btf->id);
894 spin_unlock_irqrestore(&btf_idr_lock, flags);
895}
896
897static void btf_free(struct btf *btf)
898{
899 kvfree(btf->types);
900 kvfree(btf->resolved_sizes);
901 kvfree(btf->resolved_ids);
902 kvfree(btf->data);
903 kfree(btf);
904}
905
906static void btf_free_rcu(struct rcu_head *rcu)
907{
908 struct btf *btf = container_of(rcu, struct btf, rcu);
909
910 btf_free(btf);
911}
912
913void btf_put(struct btf *btf)
914{
915 if (btf && refcount_dec_and_test(&btf->refcnt)) {
916 btf_free_id(btf);
917 call_rcu(&btf->rcu, btf_free_rcu);
918 }
919}
920
921static int env_resolve_init(struct btf_verifier_env *env)
922{
923 struct btf *btf = env->btf;
924 u32 nr_types = btf->nr_types;
925 u32 *resolved_sizes = NULL;
926 u32 *resolved_ids = NULL;
927 u8 *visit_states = NULL;
928
929
930 resolved_sizes = kvcalloc(nr_types + 1, sizeof(*resolved_sizes),
931 GFP_KERNEL | __GFP_NOWARN);
932 if (!resolved_sizes)
933 goto nomem;
934
935 resolved_ids = kvcalloc(nr_types + 1, sizeof(*resolved_ids),
936 GFP_KERNEL | __GFP_NOWARN);
937 if (!resolved_ids)
938 goto nomem;
939
940 visit_states = kvcalloc(nr_types + 1, sizeof(*visit_states),
941 GFP_KERNEL | __GFP_NOWARN);
942 if (!visit_states)
943 goto nomem;
944
945 btf->resolved_sizes = resolved_sizes;
946 btf->resolved_ids = resolved_ids;
947 env->visit_states = visit_states;
948
949 return 0;
950
951nomem:
952 kvfree(resolved_sizes);
953 kvfree(resolved_ids);
954 kvfree(visit_states);
955 return -ENOMEM;
956}
957
958static void btf_verifier_env_free(struct btf_verifier_env *env)
959{
960 kvfree(env->visit_states);
961 kfree(env);
962}
963
964static bool env_type_is_resolve_sink(const struct btf_verifier_env *env,
965 const struct btf_type *next_type)
966{
967 switch (env->resolve_mode) {
968 case RESOLVE_TBD:
969
970 return !btf_type_needs_resolve(next_type);
971 case RESOLVE_PTR:
972
973
974
975 return !btf_type_is_modifier(next_type) &&
976 !btf_type_is_ptr(next_type);
977 case RESOLVE_STRUCT_OR_ARRAY:
978
979
980
981 return !btf_type_is_modifier(next_type) &&
982 !btf_type_is_array(next_type) &&
983 !btf_type_is_struct(next_type);
984 default:
985 BUG();
986 }
987}
988
989static bool env_type_is_resolved(const struct btf_verifier_env *env,
990 u32 type_id)
991{
992 return env->visit_states[type_id] == RESOLVED;
993}
994
995static int env_stack_push(struct btf_verifier_env *env,
996 const struct btf_type *t, u32 type_id)
997{
998 struct resolve_vertex *v;
999
1000 if (env->top_stack == MAX_RESOLVE_DEPTH)
1001 return -E2BIG;
1002
1003 if (env->visit_states[type_id] != NOT_VISITED)
1004 return -EEXIST;
1005
1006 env->visit_states[type_id] = VISITED;
1007
1008 v = &env->stack[env->top_stack++];
1009 v->t = t;
1010 v->type_id = type_id;
1011 v->next_member = 0;
1012
1013 if (env->resolve_mode == RESOLVE_TBD) {
1014 if (btf_type_is_ptr(t))
1015 env->resolve_mode = RESOLVE_PTR;
1016 else if (btf_type_is_struct(t) || btf_type_is_array(t))
1017 env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY;
1018 }
1019
1020 return 0;
1021}
1022
1023static void env_stack_set_next_member(struct btf_verifier_env *env,
1024 u16 next_member)
1025{
1026 env->stack[env->top_stack - 1].next_member = next_member;
1027}
1028
1029static void env_stack_pop_resolved(struct btf_verifier_env *env,
1030 u32 resolved_type_id,
1031 u32 resolved_size)
1032{
1033 u32 type_id = env->stack[--(env->top_stack)].type_id;
1034 struct btf *btf = env->btf;
1035
1036 btf->resolved_sizes[type_id] = resolved_size;
1037 btf->resolved_ids[type_id] = resolved_type_id;
1038 env->visit_states[type_id] = RESOLVED;
1039}
1040
1041static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env)
1042{
1043 return env->top_stack ? &env->stack[env->top_stack - 1] : NULL;
1044}
1045
1046
1047static const struct btf_type *btf_type_id_resolve(const struct btf *btf,
1048 u32 *type_id)
1049{
1050 *type_id = btf->resolved_ids[*type_id];
1051 return btf_type_by_id(btf, *type_id);
1052}
1053
1054const struct btf_type *btf_type_id_size(const struct btf *btf,
1055 u32 *type_id, u32 *ret_size)
1056{
1057 const struct btf_type *size_type;
1058 u32 size_type_id = *type_id;
1059 u32 size = 0;
1060
1061 size_type = btf_type_by_id(btf, size_type_id);
1062 if (btf_type_nosize_or_null(size_type))
1063 return NULL;
1064
1065 if (btf_type_has_size(size_type)) {
1066 size = size_type->size;
1067 } else if (btf_type_is_array(size_type)) {
1068 size = btf->resolved_sizes[size_type_id];
1069 } else if (btf_type_is_ptr(size_type)) {
1070 size = sizeof(void *);
1071 } else {
1072 if (WARN_ON_ONCE(!btf_type_is_modifier(size_type) &&
1073 !btf_type_is_var(size_type)))
1074 return NULL;
1075
1076 size_type_id = btf->resolved_ids[size_type_id];
1077 size_type = btf_type_by_id(btf, size_type_id);
1078 if (btf_type_nosize_or_null(size_type))
1079 return NULL;
1080 else if (btf_type_has_size(size_type))
1081 size = size_type->size;
1082 else if (btf_type_is_array(size_type))
1083 size = btf->resolved_sizes[size_type_id];
1084 else if (btf_type_is_ptr(size_type))
1085 size = sizeof(void *);
1086 else
1087 return NULL;
1088 }
1089
1090 *type_id = size_type_id;
1091 if (ret_size)
1092 *ret_size = size;
1093
1094 return size_type;
1095}
1096
1097static int btf_df_check_member(struct btf_verifier_env *env,
1098 const struct btf_type *struct_type,
1099 const struct btf_member *member,
1100 const struct btf_type *member_type)
1101{
1102 btf_verifier_log_basic(env, struct_type,
1103 "Unsupported check_member");
1104 return -EINVAL;
1105}
1106
1107static int btf_df_check_kflag_member(struct btf_verifier_env *env,
1108 const struct btf_type *struct_type,
1109 const struct btf_member *member,
1110 const struct btf_type *member_type)
1111{
1112 btf_verifier_log_basic(env, struct_type,
1113 "Unsupported check_kflag_member");
1114 return -EINVAL;
1115}
1116
1117
1118
1119
1120static int btf_generic_check_kflag_member(struct btf_verifier_env *env,
1121 const struct btf_type *struct_type,
1122 const struct btf_member *member,
1123 const struct btf_type *member_type)
1124{
1125 if (BTF_MEMBER_BITFIELD_SIZE(member->offset)) {
1126 btf_verifier_log_member(env, struct_type, member,
1127 "Invalid member bitfield_size");
1128 return -EINVAL;
1129 }
1130
1131
1132
1133
1134 return btf_type_ops(member_type)->check_member(env, struct_type,
1135 member,
1136 member_type);
1137}
1138
1139static int btf_df_resolve(struct btf_verifier_env *env,
1140 const struct resolve_vertex *v)
1141{
1142 btf_verifier_log_basic(env, v->t, "Unsupported resolve");
1143 return -EINVAL;
1144}
1145
1146static void btf_df_seq_show(const struct btf *btf, const struct btf_type *t,
1147 u32 type_id, void *data, u8 bits_offsets,
1148 struct seq_file *m)
1149{
1150 seq_printf(m, "<unsupported kind:%u>", BTF_INFO_KIND(t->info));
1151}
1152
1153static int btf_int_check_member(struct btf_verifier_env *env,
1154 const struct btf_type *struct_type,
1155 const struct btf_member *member,
1156 const struct btf_type *member_type)
1157{
1158 u32 int_data = btf_type_int(member_type);
1159 u32 struct_bits_off = member->offset;
1160 u32 struct_size = struct_type->size;
1161 u32 nr_copy_bits;
1162 u32 bytes_offset;
1163
1164 if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) {
1165 btf_verifier_log_member(env, struct_type, member,
1166 "bits_offset exceeds U32_MAX");
1167 return -EINVAL;
1168 }
1169
1170 struct_bits_off += BTF_INT_OFFSET(int_data);
1171 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1172 nr_copy_bits = BTF_INT_BITS(int_data) +
1173 BITS_PER_BYTE_MASKED(struct_bits_off);
1174
1175 if (nr_copy_bits > BITS_PER_U128) {
1176 btf_verifier_log_member(env, struct_type, member,
1177 "nr_copy_bits exceeds 128");
1178 return -EINVAL;
1179 }
1180
1181 if (struct_size < bytes_offset ||
1182 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
1183 btf_verifier_log_member(env, struct_type, member,
1184 "Member exceeds struct_size");
1185 return -EINVAL;
1186 }
1187
1188 return 0;
1189}
1190
1191static int btf_int_check_kflag_member(struct btf_verifier_env *env,
1192 const struct btf_type *struct_type,
1193 const struct btf_member *member,
1194 const struct btf_type *member_type)
1195{
1196 u32 struct_bits_off, nr_bits, nr_int_data_bits, bytes_offset;
1197 u32 int_data = btf_type_int(member_type);
1198 u32 struct_size = struct_type->size;
1199 u32 nr_copy_bits;
1200
1201
1202 if (!btf_type_int_is_regular(member_type)) {
1203 btf_verifier_log_member(env, struct_type, member,
1204 "Invalid member base type");
1205 return -EINVAL;
1206 }
1207
1208
1209 nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset);
1210 struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset);
1211 nr_int_data_bits = BTF_INT_BITS(int_data);
1212 if (!nr_bits) {
1213
1214
1215
1216 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1217 btf_verifier_log_member(env, struct_type, member,
1218 "Invalid member offset");
1219 return -EINVAL;
1220 }
1221
1222 nr_bits = nr_int_data_bits;
1223 } else if (nr_bits > nr_int_data_bits) {
1224 btf_verifier_log_member(env, struct_type, member,
1225 "Invalid member bitfield_size");
1226 return -EINVAL;
1227 }
1228
1229 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1230 nr_copy_bits = nr_bits + BITS_PER_BYTE_MASKED(struct_bits_off);
1231 if (nr_copy_bits > BITS_PER_U128) {
1232 btf_verifier_log_member(env, struct_type, member,
1233 "nr_copy_bits exceeds 128");
1234 return -EINVAL;
1235 }
1236
1237 if (struct_size < bytes_offset ||
1238 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
1239 btf_verifier_log_member(env, struct_type, member,
1240 "Member exceeds struct_size");
1241 return -EINVAL;
1242 }
1243
1244 return 0;
1245}
1246
1247static s32 btf_int_check_meta(struct btf_verifier_env *env,
1248 const struct btf_type *t,
1249 u32 meta_left)
1250{
1251 u32 int_data, nr_bits, meta_needed = sizeof(int_data);
1252 u16 encoding;
1253
1254 if (meta_left < meta_needed) {
1255 btf_verifier_log_basic(env, t,
1256 "meta_left:%u meta_needed:%u",
1257 meta_left, meta_needed);
1258 return -EINVAL;
1259 }
1260
1261 if (btf_type_vlen(t)) {
1262 btf_verifier_log_type(env, t, "vlen != 0");
1263 return -EINVAL;
1264 }
1265
1266 if (btf_type_kflag(t)) {
1267 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
1268 return -EINVAL;
1269 }
1270
1271 int_data = btf_type_int(t);
1272 if (int_data & ~BTF_INT_MASK) {
1273 btf_verifier_log_basic(env, t, "Invalid int_data:%x",
1274 int_data);
1275 return -EINVAL;
1276 }
1277
1278 nr_bits = BTF_INT_BITS(int_data) + BTF_INT_OFFSET(int_data);
1279
1280 if (nr_bits > BITS_PER_U128) {
1281 btf_verifier_log_type(env, t, "nr_bits exceeds %zu",
1282 BITS_PER_U128);
1283 return -EINVAL;
1284 }
1285
1286 if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) {
1287 btf_verifier_log_type(env, t, "nr_bits exceeds type_size");
1288 return -EINVAL;
1289 }
1290
1291
1292
1293
1294
1295
1296
1297 encoding = BTF_INT_ENCODING(int_data);
1298 if (encoding &&
1299 encoding != BTF_INT_SIGNED &&
1300 encoding != BTF_INT_CHAR &&
1301 encoding != BTF_INT_BOOL) {
1302 btf_verifier_log_type(env, t, "Unsupported encoding");
1303 return -ENOTSUPP;
1304 }
1305
1306 btf_verifier_log_type(env, t, NULL);
1307
1308 return meta_needed;
1309}
1310
1311static void btf_int_log(struct btf_verifier_env *env,
1312 const struct btf_type *t)
1313{
1314 int int_data = btf_type_int(t);
1315
1316 btf_verifier_log(env,
1317 "size=%u bits_offset=%u nr_bits=%u encoding=%s",
1318 t->size, BTF_INT_OFFSET(int_data),
1319 BTF_INT_BITS(int_data),
1320 btf_int_encoding_str(BTF_INT_ENCODING(int_data)));
1321}
1322
1323static void btf_int128_print(struct seq_file *m, void *data)
1324{
1325
1326
1327
1328
1329
1330
1331
1332 u64 upper_num, lower_num;
1333
1334#ifdef __BIG_ENDIAN_BITFIELD
1335 upper_num = *(u64 *)data;
1336 lower_num = *(u64 *)(data + 8);
1337#else
1338 upper_num = *(u64 *)(data + 8);
1339 lower_num = *(u64 *)data;
1340#endif
1341 if (upper_num == 0)
1342 seq_printf(m, "0x%llx", lower_num);
1343 else
1344 seq_printf(m, "0x%llx%016llx", upper_num, lower_num);
1345}
1346
1347static void btf_int128_shift(u64 *print_num, u16 left_shift_bits,
1348 u16 right_shift_bits)
1349{
1350 u64 upper_num, lower_num;
1351
1352#ifdef __BIG_ENDIAN_BITFIELD
1353 upper_num = print_num[0];
1354 lower_num = print_num[1];
1355#else
1356 upper_num = print_num[1];
1357 lower_num = print_num[0];
1358#endif
1359
1360
1361 if (left_shift_bits >= 64) {
1362 upper_num = lower_num << (left_shift_bits - 64);
1363 lower_num = 0;
1364 } else {
1365 upper_num = (upper_num << left_shift_bits) |
1366 (lower_num >> (64 - left_shift_bits));
1367 lower_num = lower_num << left_shift_bits;
1368 }
1369
1370 if (right_shift_bits >= 64) {
1371 lower_num = upper_num >> (right_shift_bits - 64);
1372 upper_num = 0;
1373 } else {
1374 lower_num = (lower_num >> right_shift_bits) |
1375 (upper_num << (64 - right_shift_bits));
1376 upper_num = upper_num >> right_shift_bits;
1377 }
1378
1379#ifdef __BIG_ENDIAN_BITFIELD
1380 print_num[0] = upper_num;
1381 print_num[1] = lower_num;
1382#else
1383 print_num[0] = lower_num;
1384 print_num[1] = upper_num;
1385#endif
1386}
1387
1388static void btf_bitfield_seq_show(void *data, u8 bits_offset,
1389 u8 nr_bits, struct seq_file *m)
1390{
1391 u16 left_shift_bits, right_shift_bits;
1392 u8 nr_copy_bytes;
1393 u8 nr_copy_bits;
1394 u64 print_num[2] = {};
1395
1396 nr_copy_bits = nr_bits + bits_offset;
1397 nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits);
1398
1399 memcpy(print_num, data, nr_copy_bytes);
1400
1401#ifdef __BIG_ENDIAN_BITFIELD
1402 left_shift_bits = bits_offset;
1403#else
1404 left_shift_bits = BITS_PER_U128 - nr_copy_bits;
1405#endif
1406 right_shift_bits = BITS_PER_U128 - nr_bits;
1407
1408 btf_int128_shift(print_num, left_shift_bits, right_shift_bits);
1409 btf_int128_print(m, print_num);
1410}
1411
1412
1413static void btf_int_bits_seq_show(const struct btf *btf,
1414 const struct btf_type *t,
1415 void *data, u8 bits_offset,
1416 struct seq_file *m)
1417{
1418 u32 int_data = btf_type_int(t);
1419 u8 nr_bits = BTF_INT_BITS(int_data);
1420 u8 total_bits_offset;
1421
1422
1423
1424
1425
1426 total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data);
1427 data += BITS_ROUNDDOWN_BYTES(total_bits_offset);
1428 bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset);
1429 btf_bitfield_seq_show(data, bits_offset, nr_bits, m);
1430}
1431
1432static void btf_int_seq_show(const struct btf *btf, const struct btf_type *t,
1433 u32 type_id, void *data, u8 bits_offset,
1434 struct seq_file *m)
1435{
1436 u32 int_data = btf_type_int(t);
1437 u8 encoding = BTF_INT_ENCODING(int_data);
1438 bool sign = encoding & BTF_INT_SIGNED;
1439 u8 nr_bits = BTF_INT_BITS(int_data);
1440
1441 if (bits_offset || BTF_INT_OFFSET(int_data) ||
1442 BITS_PER_BYTE_MASKED(nr_bits)) {
1443 btf_int_bits_seq_show(btf, t, data, bits_offset, m);
1444 return;
1445 }
1446
1447 switch (nr_bits) {
1448 case 128:
1449 btf_int128_print(m, data);
1450 break;
1451 case 64:
1452 if (sign)
1453 seq_printf(m, "%lld", *(s64 *)data);
1454 else
1455 seq_printf(m, "%llu", *(u64 *)data);
1456 break;
1457 case 32:
1458 if (sign)
1459 seq_printf(m, "%d", *(s32 *)data);
1460 else
1461 seq_printf(m, "%u", *(u32 *)data);
1462 break;
1463 case 16:
1464 if (sign)
1465 seq_printf(m, "%d", *(s16 *)data);
1466 else
1467 seq_printf(m, "%u", *(u16 *)data);
1468 break;
1469 case 8:
1470 if (sign)
1471 seq_printf(m, "%d", *(s8 *)data);
1472 else
1473 seq_printf(m, "%u", *(u8 *)data);
1474 break;
1475 default:
1476 btf_int_bits_seq_show(btf, t, data, bits_offset, m);
1477 }
1478}
1479
1480static const struct btf_kind_operations int_ops = {
1481 .check_meta = btf_int_check_meta,
1482 .resolve = btf_df_resolve,
1483 .check_member = btf_int_check_member,
1484 .check_kflag_member = btf_int_check_kflag_member,
1485 .log_details = btf_int_log,
1486 .seq_show = btf_int_seq_show,
1487};
1488
1489static int btf_modifier_check_member(struct btf_verifier_env *env,
1490 const struct btf_type *struct_type,
1491 const struct btf_member *member,
1492 const struct btf_type *member_type)
1493{
1494 const struct btf_type *resolved_type;
1495 u32 resolved_type_id = member->type;
1496 struct btf_member resolved_member;
1497 struct btf *btf = env->btf;
1498
1499 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
1500 if (!resolved_type) {
1501 btf_verifier_log_member(env, struct_type, member,
1502 "Invalid member");
1503 return -EINVAL;
1504 }
1505
1506 resolved_member = *member;
1507 resolved_member.type = resolved_type_id;
1508
1509 return btf_type_ops(resolved_type)->check_member(env, struct_type,
1510 &resolved_member,
1511 resolved_type);
1512}
1513
1514static int btf_modifier_check_kflag_member(struct btf_verifier_env *env,
1515 const struct btf_type *struct_type,
1516 const struct btf_member *member,
1517 const struct btf_type *member_type)
1518{
1519 const struct btf_type *resolved_type;
1520 u32 resolved_type_id = member->type;
1521 struct btf_member resolved_member;
1522 struct btf *btf = env->btf;
1523
1524 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
1525 if (!resolved_type) {
1526 btf_verifier_log_member(env, struct_type, member,
1527 "Invalid member");
1528 return -EINVAL;
1529 }
1530
1531 resolved_member = *member;
1532 resolved_member.type = resolved_type_id;
1533
1534 return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type,
1535 &resolved_member,
1536 resolved_type);
1537}
1538
1539static int btf_ptr_check_member(struct btf_verifier_env *env,
1540 const struct btf_type *struct_type,
1541 const struct btf_member *member,
1542 const struct btf_type *member_type)
1543{
1544 u32 struct_size, struct_bits_off, bytes_offset;
1545
1546 struct_size = struct_type->size;
1547 struct_bits_off = member->offset;
1548 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1549
1550 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1551 btf_verifier_log_member(env, struct_type, member,
1552 "Member is not byte aligned");
1553 return -EINVAL;
1554 }
1555
1556 if (struct_size - bytes_offset < sizeof(void *)) {
1557 btf_verifier_log_member(env, struct_type, member,
1558 "Member exceeds struct_size");
1559 return -EINVAL;
1560 }
1561
1562 return 0;
1563}
1564
1565static int btf_ref_type_check_meta(struct btf_verifier_env *env,
1566 const struct btf_type *t,
1567 u32 meta_left)
1568{
1569 if (btf_type_vlen(t)) {
1570 btf_verifier_log_type(env, t, "vlen != 0");
1571 return -EINVAL;
1572 }
1573
1574 if (btf_type_kflag(t)) {
1575 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
1576 return -EINVAL;
1577 }
1578
1579 if (!BTF_TYPE_ID_VALID(t->type)) {
1580 btf_verifier_log_type(env, t, "Invalid type_id");
1581 return -EINVAL;
1582 }
1583
1584
1585
1586
1587 if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) {
1588 if (!t->name_off ||
1589 !btf_name_valid_identifier(env->btf, t->name_off)) {
1590 btf_verifier_log_type(env, t, "Invalid name");
1591 return -EINVAL;
1592 }
1593 } else {
1594 if (t->name_off) {
1595 btf_verifier_log_type(env, t, "Invalid name");
1596 return -EINVAL;
1597 }
1598 }
1599
1600 btf_verifier_log_type(env, t, NULL);
1601
1602 return 0;
1603}
1604
1605static int btf_modifier_resolve(struct btf_verifier_env *env,
1606 const struct resolve_vertex *v)
1607{
1608 const struct btf_type *t = v->t;
1609 const struct btf_type *next_type;
1610 u32 next_type_id = t->type;
1611 struct btf *btf = env->btf;
1612
1613 next_type = btf_type_by_id(btf, next_type_id);
1614 if (!next_type || btf_type_is_resolve_source_only(next_type)) {
1615 btf_verifier_log_type(env, v->t, "Invalid type_id");
1616 return -EINVAL;
1617 }
1618
1619 if (!env_type_is_resolve_sink(env, next_type) &&
1620 !env_type_is_resolved(env, next_type_id))
1621 return env_stack_push(env, next_type, next_type_id);
1622
1623
1624
1625
1626
1627
1628
1629 if (!btf_type_id_size(btf, &next_type_id, NULL)) {
1630 if (env_type_is_resolved(env, next_type_id))
1631 next_type = btf_type_id_resolve(btf, &next_type_id);
1632
1633
1634 if (!btf_type_is_void(next_type) &&
1635 !btf_type_is_fwd(next_type) &&
1636 !btf_type_is_func_proto(next_type)) {
1637 btf_verifier_log_type(env, v->t, "Invalid type_id");
1638 return -EINVAL;
1639 }
1640 }
1641
1642 env_stack_pop_resolved(env, next_type_id, 0);
1643
1644 return 0;
1645}
1646
1647static int btf_var_resolve(struct btf_verifier_env *env,
1648 const struct resolve_vertex *v)
1649{
1650 const struct btf_type *next_type;
1651 const struct btf_type *t = v->t;
1652 u32 next_type_id = t->type;
1653 struct btf *btf = env->btf;
1654
1655 next_type = btf_type_by_id(btf, next_type_id);
1656 if (!next_type || btf_type_is_resolve_source_only(next_type)) {
1657 btf_verifier_log_type(env, v->t, "Invalid type_id");
1658 return -EINVAL;
1659 }
1660
1661 if (!env_type_is_resolve_sink(env, next_type) &&
1662 !env_type_is_resolved(env, next_type_id))
1663 return env_stack_push(env, next_type, next_type_id);
1664
1665 if (btf_type_is_modifier(next_type)) {
1666 const struct btf_type *resolved_type;
1667 u32 resolved_type_id;
1668
1669 resolved_type_id = next_type_id;
1670 resolved_type = btf_type_id_resolve(btf, &resolved_type_id);
1671
1672 if (btf_type_is_ptr(resolved_type) &&
1673 !env_type_is_resolve_sink(env, resolved_type) &&
1674 !env_type_is_resolved(env, resolved_type_id))
1675 return env_stack_push(env, resolved_type,
1676 resolved_type_id);
1677 }
1678
1679
1680
1681
1682
1683 if (!btf_type_id_size(btf, &next_type_id, NULL)) {
1684 btf_verifier_log_type(env, v->t, "Invalid type_id");
1685 return -EINVAL;
1686 }
1687
1688 env_stack_pop_resolved(env, next_type_id, 0);
1689
1690 return 0;
1691}
1692
1693static int btf_ptr_resolve(struct btf_verifier_env *env,
1694 const struct resolve_vertex *v)
1695{
1696 const struct btf_type *next_type;
1697 const struct btf_type *t = v->t;
1698 u32 next_type_id = t->type;
1699 struct btf *btf = env->btf;
1700
1701 next_type = btf_type_by_id(btf, next_type_id);
1702 if (!next_type || btf_type_is_resolve_source_only(next_type)) {
1703 btf_verifier_log_type(env, v->t, "Invalid type_id");
1704 return -EINVAL;
1705 }
1706
1707 if (!env_type_is_resolve_sink(env, next_type) &&
1708 !env_type_is_resolved(env, next_type_id))
1709 return env_stack_push(env, next_type, next_type_id);
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719 if (btf_type_is_modifier(next_type)) {
1720 const struct btf_type *resolved_type;
1721 u32 resolved_type_id;
1722
1723 resolved_type_id = next_type_id;
1724 resolved_type = btf_type_id_resolve(btf, &resolved_type_id);
1725
1726 if (btf_type_is_ptr(resolved_type) &&
1727 !env_type_is_resolve_sink(env, resolved_type) &&
1728 !env_type_is_resolved(env, resolved_type_id))
1729 return env_stack_push(env, resolved_type,
1730 resolved_type_id);
1731 }
1732
1733 if (!btf_type_id_size(btf, &next_type_id, NULL)) {
1734 if (env_type_is_resolved(env, next_type_id))
1735 next_type = btf_type_id_resolve(btf, &next_type_id);
1736
1737 if (!btf_type_is_void(next_type) &&
1738 !btf_type_is_fwd(next_type) &&
1739 !btf_type_is_func_proto(next_type)) {
1740 btf_verifier_log_type(env, v->t, "Invalid type_id");
1741 return -EINVAL;
1742 }
1743 }
1744
1745 env_stack_pop_resolved(env, next_type_id, 0);
1746
1747 return 0;
1748}
1749
1750static void btf_modifier_seq_show(const struct btf *btf,
1751 const struct btf_type *t,
1752 u32 type_id, void *data,
1753 u8 bits_offset, struct seq_file *m)
1754{
1755 t = btf_type_id_resolve(btf, &type_id);
1756
1757 btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m);
1758}
1759
1760static void btf_var_seq_show(const struct btf *btf, const struct btf_type *t,
1761 u32 type_id, void *data, u8 bits_offset,
1762 struct seq_file *m)
1763{
1764 t = btf_type_id_resolve(btf, &type_id);
1765
1766 btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m);
1767}
1768
1769static void btf_ptr_seq_show(const struct btf *btf, const struct btf_type *t,
1770 u32 type_id, void *data, u8 bits_offset,
1771 struct seq_file *m)
1772{
1773
1774 seq_printf(m, "%p", *(void **)data);
1775}
1776
1777static void btf_ref_type_log(struct btf_verifier_env *env,
1778 const struct btf_type *t)
1779{
1780 btf_verifier_log(env, "type_id=%u", t->type);
1781}
1782
1783static struct btf_kind_operations modifier_ops = {
1784 .check_meta = btf_ref_type_check_meta,
1785 .resolve = btf_modifier_resolve,
1786 .check_member = btf_modifier_check_member,
1787 .check_kflag_member = btf_modifier_check_kflag_member,
1788 .log_details = btf_ref_type_log,
1789 .seq_show = btf_modifier_seq_show,
1790};
1791
1792static struct btf_kind_operations ptr_ops = {
1793 .check_meta = btf_ref_type_check_meta,
1794 .resolve = btf_ptr_resolve,
1795 .check_member = btf_ptr_check_member,
1796 .check_kflag_member = btf_generic_check_kflag_member,
1797 .log_details = btf_ref_type_log,
1798 .seq_show = btf_ptr_seq_show,
1799};
1800
1801static s32 btf_fwd_check_meta(struct btf_verifier_env *env,
1802 const struct btf_type *t,
1803 u32 meta_left)
1804{
1805 if (btf_type_vlen(t)) {
1806 btf_verifier_log_type(env, t, "vlen != 0");
1807 return -EINVAL;
1808 }
1809
1810 if (t->type) {
1811 btf_verifier_log_type(env, t, "type != 0");
1812 return -EINVAL;
1813 }
1814
1815
1816 if (!t->name_off ||
1817 !btf_name_valid_identifier(env->btf, t->name_off)) {
1818 btf_verifier_log_type(env, t, "Invalid name");
1819 return -EINVAL;
1820 }
1821
1822 btf_verifier_log_type(env, t, NULL);
1823
1824 return 0;
1825}
1826
1827static void btf_fwd_type_log(struct btf_verifier_env *env,
1828 const struct btf_type *t)
1829{
1830 btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct");
1831}
1832
1833static struct btf_kind_operations fwd_ops = {
1834 .check_meta = btf_fwd_check_meta,
1835 .resolve = btf_df_resolve,
1836 .check_member = btf_df_check_member,
1837 .check_kflag_member = btf_df_check_kflag_member,
1838 .log_details = btf_fwd_type_log,
1839 .seq_show = btf_df_seq_show,
1840};
1841
1842static int btf_array_check_member(struct btf_verifier_env *env,
1843 const struct btf_type *struct_type,
1844 const struct btf_member *member,
1845 const struct btf_type *member_type)
1846{
1847 u32 struct_bits_off = member->offset;
1848 u32 struct_size, bytes_offset;
1849 u32 array_type_id, array_size;
1850 struct btf *btf = env->btf;
1851
1852 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1853 btf_verifier_log_member(env, struct_type, member,
1854 "Member is not byte aligned");
1855 return -EINVAL;
1856 }
1857
1858 array_type_id = member->type;
1859 btf_type_id_size(btf, &array_type_id, &array_size);
1860 struct_size = struct_type->size;
1861 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1862 if (struct_size - bytes_offset < array_size) {
1863 btf_verifier_log_member(env, struct_type, member,
1864 "Member exceeds struct_size");
1865 return -EINVAL;
1866 }
1867
1868 return 0;
1869}
1870
1871static s32 btf_array_check_meta(struct btf_verifier_env *env,
1872 const struct btf_type *t,
1873 u32 meta_left)
1874{
1875 const struct btf_array *array = btf_type_array(t);
1876 u32 meta_needed = sizeof(*array);
1877
1878 if (meta_left < meta_needed) {
1879 btf_verifier_log_basic(env, t,
1880 "meta_left:%u meta_needed:%u",
1881 meta_left, meta_needed);
1882 return -EINVAL;
1883 }
1884
1885
1886 if (t->name_off) {
1887 btf_verifier_log_type(env, t, "Invalid name");
1888 return -EINVAL;
1889 }
1890
1891 if (btf_type_vlen(t)) {
1892 btf_verifier_log_type(env, t, "vlen != 0");
1893 return -EINVAL;
1894 }
1895
1896 if (btf_type_kflag(t)) {
1897 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
1898 return -EINVAL;
1899 }
1900
1901 if (t->size) {
1902 btf_verifier_log_type(env, t, "size != 0");
1903 return -EINVAL;
1904 }
1905
1906
1907
1908
1909 if (!array->type || !BTF_TYPE_ID_VALID(array->type)) {
1910 btf_verifier_log_type(env, t, "Invalid elem");
1911 return -EINVAL;
1912 }
1913
1914 if (!array->index_type || !BTF_TYPE_ID_VALID(array->index_type)) {
1915 btf_verifier_log_type(env, t, "Invalid index");
1916 return -EINVAL;
1917 }
1918
1919 btf_verifier_log_type(env, t, NULL);
1920
1921 return meta_needed;
1922}
1923
1924static int btf_array_resolve(struct btf_verifier_env *env,
1925 const struct resolve_vertex *v)
1926{
1927 const struct btf_array *array = btf_type_array(v->t);
1928 const struct btf_type *elem_type, *index_type;
1929 u32 elem_type_id, index_type_id;
1930 struct btf *btf = env->btf;
1931 u32 elem_size;
1932
1933
1934 index_type_id = array->index_type;
1935 index_type = btf_type_by_id(btf, index_type_id);
1936 if (btf_type_nosize_or_null(index_type) ||
1937 btf_type_is_resolve_source_only(index_type)) {
1938 btf_verifier_log_type(env, v->t, "Invalid index");
1939 return -EINVAL;
1940 }
1941
1942 if (!env_type_is_resolve_sink(env, index_type) &&
1943 !env_type_is_resolved(env, index_type_id))
1944 return env_stack_push(env, index_type, index_type_id);
1945
1946 index_type = btf_type_id_size(btf, &index_type_id, NULL);
1947 if (!index_type || !btf_type_is_int(index_type) ||
1948 !btf_type_int_is_regular(index_type)) {
1949 btf_verifier_log_type(env, v->t, "Invalid index");
1950 return -EINVAL;
1951 }
1952
1953
1954 elem_type_id = array->type;
1955 elem_type = btf_type_by_id(btf, elem_type_id);
1956 if (btf_type_nosize_or_null(elem_type) ||
1957 btf_type_is_resolve_source_only(elem_type)) {
1958 btf_verifier_log_type(env, v->t,
1959 "Invalid elem");
1960 return -EINVAL;
1961 }
1962
1963 if (!env_type_is_resolve_sink(env, elem_type) &&
1964 !env_type_is_resolved(env, elem_type_id))
1965 return env_stack_push(env, elem_type, elem_type_id);
1966
1967 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
1968 if (!elem_type) {
1969 btf_verifier_log_type(env, v->t, "Invalid elem");
1970 return -EINVAL;
1971 }
1972
1973 if (btf_type_is_int(elem_type) && !btf_type_int_is_regular(elem_type)) {
1974 btf_verifier_log_type(env, v->t, "Invalid array of int");
1975 return -EINVAL;
1976 }
1977
1978 if (array->nelems && elem_size > U32_MAX / array->nelems) {
1979 btf_verifier_log_type(env, v->t,
1980 "Array size overflows U32_MAX");
1981 return -EINVAL;
1982 }
1983
1984 env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems);
1985
1986 return 0;
1987}
1988
1989static void btf_array_log(struct btf_verifier_env *env,
1990 const struct btf_type *t)
1991{
1992 const struct btf_array *array = btf_type_array(t);
1993
1994 btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u",
1995 array->type, array->index_type, array->nelems);
1996}
1997
1998static void btf_array_seq_show(const struct btf *btf, const struct btf_type *t,
1999 u32 type_id, void *data, u8 bits_offset,
2000 struct seq_file *m)
2001{
2002 const struct btf_array *array = btf_type_array(t);
2003 const struct btf_kind_operations *elem_ops;
2004 const struct btf_type *elem_type;
2005 u32 i, elem_size, elem_type_id;
2006
2007 elem_type_id = array->type;
2008 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
2009 elem_ops = btf_type_ops(elem_type);
2010 seq_puts(m, "[");
2011 for (i = 0; i < array->nelems; i++) {
2012 if (i)
2013 seq_puts(m, ",");
2014
2015 elem_ops->seq_show(btf, elem_type, elem_type_id, data,
2016 bits_offset, m);
2017 data += elem_size;
2018 }
2019 seq_puts(m, "]");
2020}
2021
2022static struct btf_kind_operations array_ops = {
2023 .check_meta = btf_array_check_meta,
2024 .resolve = btf_array_resolve,
2025 .check_member = btf_array_check_member,
2026 .check_kflag_member = btf_generic_check_kflag_member,
2027 .log_details = btf_array_log,
2028 .seq_show = btf_array_seq_show,
2029};
2030
2031static int btf_struct_check_member(struct btf_verifier_env *env,
2032 const struct btf_type *struct_type,
2033 const struct btf_member *member,
2034 const struct btf_type *member_type)
2035{
2036 u32 struct_bits_off = member->offset;
2037 u32 struct_size, bytes_offset;
2038
2039 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2040 btf_verifier_log_member(env, struct_type, member,
2041 "Member is not byte aligned");
2042 return -EINVAL;
2043 }
2044
2045 struct_size = struct_type->size;
2046 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
2047 if (struct_size - bytes_offset < member_type->size) {
2048 btf_verifier_log_member(env, struct_type, member,
2049 "Member exceeds struct_size");
2050 return -EINVAL;
2051 }
2052
2053 return 0;
2054}
2055
2056static s32 btf_struct_check_meta(struct btf_verifier_env *env,
2057 const struct btf_type *t,
2058 u32 meta_left)
2059{
2060 bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION;
2061 const struct btf_member *member;
2062 u32 meta_needed, last_offset;
2063 struct btf *btf = env->btf;
2064 u32 struct_size = t->size;
2065 u32 offset;
2066 u16 i;
2067
2068 meta_needed = btf_type_vlen(t) * sizeof(*member);
2069 if (meta_left < meta_needed) {
2070 btf_verifier_log_basic(env, t,
2071 "meta_left:%u meta_needed:%u",
2072 meta_left, meta_needed);
2073 return -EINVAL;
2074 }
2075
2076
2077 if (t->name_off &&
2078 !btf_name_valid_identifier(env->btf, t->name_off)) {
2079 btf_verifier_log_type(env, t, "Invalid name");
2080 return -EINVAL;
2081 }
2082
2083 btf_verifier_log_type(env, t, NULL);
2084
2085 last_offset = 0;
2086 for_each_member(i, t, member) {
2087 if (!btf_name_offset_valid(btf, member->name_off)) {
2088 btf_verifier_log_member(env, t, member,
2089 "Invalid member name_offset:%u",
2090 member->name_off);
2091 return -EINVAL;
2092 }
2093
2094
2095 if (member->name_off &&
2096 !btf_name_valid_identifier(btf, member->name_off)) {
2097 btf_verifier_log_member(env, t, member, "Invalid name");
2098 return -EINVAL;
2099 }
2100
2101 if (!member->type || !BTF_TYPE_ID_VALID(member->type)) {
2102 btf_verifier_log_member(env, t, member,
2103 "Invalid type_id");
2104 return -EINVAL;
2105 }
2106
2107 offset = btf_member_bit_offset(t, member);
2108 if (is_union && offset) {
2109 btf_verifier_log_member(env, t, member,
2110 "Invalid member bits_offset");
2111 return -EINVAL;
2112 }
2113
2114
2115
2116
2117
2118 if (last_offset > offset) {
2119 btf_verifier_log_member(env, t, member,
2120 "Invalid member bits_offset");
2121 return -EINVAL;
2122 }
2123
2124 if (BITS_ROUNDUP_BYTES(offset) > struct_size) {
2125 btf_verifier_log_member(env, t, member,
2126 "Member bits_offset exceeds its struct size");
2127 return -EINVAL;
2128 }
2129
2130 btf_verifier_log_member(env, t, member, NULL);
2131 last_offset = offset;
2132 }
2133
2134 return meta_needed;
2135}
2136
2137static int btf_struct_resolve(struct btf_verifier_env *env,
2138 const struct resolve_vertex *v)
2139{
2140 const struct btf_member *member;
2141 int err;
2142 u16 i;
2143
2144
2145
2146
2147
2148 if (v->next_member) {
2149 const struct btf_type *last_member_type;
2150 const struct btf_member *last_member;
2151 u16 last_member_type_id;
2152
2153 last_member = btf_type_member(v->t) + v->next_member - 1;
2154 last_member_type_id = last_member->type;
2155 if (WARN_ON_ONCE(!env_type_is_resolved(env,
2156 last_member_type_id)))
2157 return -EINVAL;
2158
2159 last_member_type = btf_type_by_id(env->btf,
2160 last_member_type_id);
2161 if (btf_type_kflag(v->t))
2162 err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t,
2163 last_member,
2164 last_member_type);
2165 else
2166 err = btf_type_ops(last_member_type)->check_member(env, v->t,
2167 last_member,
2168 last_member_type);
2169 if (err)
2170 return err;
2171 }
2172
2173 for_each_member_from(i, v->next_member, v->t, member) {
2174 u32 member_type_id = member->type;
2175 const struct btf_type *member_type = btf_type_by_id(env->btf,
2176 member_type_id);
2177
2178 if (btf_type_nosize_or_null(member_type) ||
2179 btf_type_is_resolve_source_only(member_type)) {
2180 btf_verifier_log_member(env, v->t, member,
2181 "Invalid member");
2182 return -EINVAL;
2183 }
2184
2185 if (!env_type_is_resolve_sink(env, member_type) &&
2186 !env_type_is_resolved(env, member_type_id)) {
2187 env_stack_set_next_member(env, i + 1);
2188 return env_stack_push(env, member_type, member_type_id);
2189 }
2190
2191 if (btf_type_kflag(v->t))
2192 err = btf_type_ops(member_type)->check_kflag_member(env, v->t,
2193 member,
2194 member_type);
2195 else
2196 err = btf_type_ops(member_type)->check_member(env, v->t,
2197 member,
2198 member_type);
2199 if (err)
2200 return err;
2201 }
2202
2203 env_stack_pop_resolved(env, 0, 0);
2204
2205 return 0;
2206}
2207
2208static void btf_struct_log(struct btf_verifier_env *env,
2209 const struct btf_type *t)
2210{
2211 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
2212}
2213
2214
2215
2216
2217
2218int btf_find_spin_lock(const struct btf *btf, const struct btf_type *t)
2219{
2220 const struct btf_member *member;
2221 u32 i, off = -ENOENT;
2222
2223 if (!__btf_type_is_struct(t))
2224 return -EINVAL;
2225
2226 for_each_member(i, t, member) {
2227 const struct btf_type *member_type = btf_type_by_id(btf,
2228 member->type);
2229 if (!__btf_type_is_struct(member_type))
2230 continue;
2231 if (member_type->size != sizeof(struct bpf_spin_lock))
2232 continue;
2233 if (strcmp(__btf_name_by_offset(btf, member_type->name_off),
2234 "bpf_spin_lock"))
2235 continue;
2236 if (off != -ENOENT)
2237
2238 return -E2BIG;
2239 off = btf_member_bit_offset(t, member);
2240 if (off % 8)
2241
2242 return -EINVAL;
2243 off /= 8;
2244 if (off % __alignof__(struct bpf_spin_lock))
2245
2246 return -EINVAL;
2247 }
2248 return off;
2249}
2250
2251static void btf_struct_seq_show(const struct btf *btf, const struct btf_type *t,
2252 u32 type_id, void *data, u8 bits_offset,
2253 struct seq_file *m)
2254{
2255 const char *seq = BTF_INFO_KIND(t->info) == BTF_KIND_UNION ? "|" : ",";
2256 const struct btf_member *member;
2257 u32 i;
2258
2259 seq_puts(m, "{");
2260 for_each_member(i, t, member) {
2261 const struct btf_type *member_type = btf_type_by_id(btf,
2262 member->type);
2263 const struct btf_kind_operations *ops;
2264 u32 member_offset, bitfield_size;
2265 u32 bytes_offset;
2266 u8 bits8_offset;
2267
2268 if (i)
2269 seq_puts(m, seq);
2270
2271 member_offset = btf_member_bit_offset(t, member);
2272 bitfield_size = btf_member_bitfield_size(t, member);
2273 bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset);
2274 bits8_offset = BITS_PER_BYTE_MASKED(member_offset);
2275 if (bitfield_size) {
2276 btf_bitfield_seq_show(data + bytes_offset, bits8_offset,
2277 bitfield_size, m);
2278 } else {
2279 ops = btf_type_ops(member_type);
2280 ops->seq_show(btf, member_type, member->type,
2281 data + bytes_offset, bits8_offset, m);
2282 }
2283 }
2284 seq_puts(m, "}");
2285}
2286
2287static struct btf_kind_operations struct_ops = {
2288 .check_meta = btf_struct_check_meta,
2289 .resolve = btf_struct_resolve,
2290 .check_member = btf_struct_check_member,
2291 .check_kflag_member = btf_generic_check_kflag_member,
2292 .log_details = btf_struct_log,
2293 .seq_show = btf_struct_seq_show,
2294};
2295
2296static int btf_enum_check_member(struct btf_verifier_env *env,
2297 const struct btf_type *struct_type,
2298 const struct btf_member *member,
2299 const struct btf_type *member_type)
2300{
2301 u32 struct_bits_off = member->offset;
2302 u32 struct_size, bytes_offset;
2303
2304 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2305 btf_verifier_log_member(env, struct_type, member,
2306 "Member is not byte aligned");
2307 return -EINVAL;
2308 }
2309
2310 struct_size = struct_type->size;
2311 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
2312 if (struct_size - bytes_offset < sizeof(int)) {
2313 btf_verifier_log_member(env, struct_type, member,
2314 "Member exceeds struct_size");
2315 return -EINVAL;
2316 }
2317
2318 return 0;
2319}
2320
2321static int btf_enum_check_kflag_member(struct btf_verifier_env *env,
2322 const struct btf_type *struct_type,
2323 const struct btf_member *member,
2324 const struct btf_type *member_type)
2325{
2326 u32 struct_bits_off, nr_bits, bytes_end, struct_size;
2327 u32 int_bitsize = sizeof(int) * BITS_PER_BYTE;
2328
2329 struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset);
2330 nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset);
2331 if (!nr_bits) {
2332 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2333 btf_verifier_log_member(env, struct_type, member,
2334 "Member is not byte aligned");
2335 return -EINVAL;
2336 }
2337
2338 nr_bits = int_bitsize;
2339 } else if (nr_bits > int_bitsize) {
2340 btf_verifier_log_member(env, struct_type, member,
2341 "Invalid member bitfield_size");
2342 return -EINVAL;
2343 }
2344
2345 struct_size = struct_type->size;
2346 bytes_end = BITS_ROUNDUP_BYTES(struct_bits_off + nr_bits);
2347 if (struct_size < bytes_end) {
2348 btf_verifier_log_member(env, struct_type, member,
2349 "Member exceeds struct_size");
2350 return -EINVAL;
2351 }
2352
2353 return 0;
2354}
2355
2356static s32 btf_enum_check_meta(struct btf_verifier_env *env,
2357 const struct btf_type *t,
2358 u32 meta_left)
2359{
2360 const struct btf_enum *enums = btf_type_enum(t);
2361 struct btf *btf = env->btf;
2362 u16 i, nr_enums;
2363 u32 meta_needed;
2364
2365 nr_enums = btf_type_vlen(t);
2366 meta_needed = nr_enums * sizeof(*enums);
2367
2368 if (meta_left < meta_needed) {
2369 btf_verifier_log_basic(env, t,
2370 "meta_left:%u meta_needed:%u",
2371 meta_left, meta_needed);
2372 return -EINVAL;
2373 }
2374
2375 if (btf_type_kflag(t)) {
2376 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2377 return -EINVAL;
2378 }
2379
2380 if (t->size != sizeof(int)) {
2381 btf_verifier_log_type(env, t, "Expected size:%zu",
2382 sizeof(int));
2383 return -EINVAL;
2384 }
2385
2386
2387 if (t->name_off &&
2388 !btf_name_valid_identifier(env->btf, t->name_off)) {
2389 btf_verifier_log_type(env, t, "Invalid name");
2390 return -EINVAL;
2391 }
2392
2393 btf_verifier_log_type(env, t, NULL);
2394
2395 for (i = 0; i < nr_enums; i++) {
2396 if (!btf_name_offset_valid(btf, enums[i].name_off)) {
2397 btf_verifier_log(env, "\tInvalid name_offset:%u",
2398 enums[i].name_off);
2399 return -EINVAL;
2400 }
2401
2402
2403 if (!enums[i].name_off ||
2404 !btf_name_valid_identifier(btf, enums[i].name_off)) {
2405 btf_verifier_log_type(env, t, "Invalid name");
2406 return -EINVAL;
2407 }
2408
2409
2410 btf_verifier_log(env, "\t%s val=%d\n",
2411 __btf_name_by_offset(btf, enums[i].name_off),
2412 enums[i].val);
2413 }
2414
2415 return meta_needed;
2416}
2417
2418static void btf_enum_log(struct btf_verifier_env *env,
2419 const struct btf_type *t)
2420{
2421 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
2422}
2423
2424static void btf_enum_seq_show(const struct btf *btf, const struct btf_type *t,
2425 u32 type_id, void *data, u8 bits_offset,
2426 struct seq_file *m)
2427{
2428 const struct btf_enum *enums = btf_type_enum(t);
2429 u32 i, nr_enums = btf_type_vlen(t);
2430 int v = *(int *)data;
2431
2432 for (i = 0; i < nr_enums; i++) {
2433 if (v == enums[i].val) {
2434 seq_printf(m, "%s",
2435 __btf_name_by_offset(btf,
2436 enums[i].name_off));
2437 return;
2438 }
2439 }
2440
2441 seq_printf(m, "%d", v);
2442}
2443
2444static struct btf_kind_operations enum_ops = {
2445 .check_meta = btf_enum_check_meta,
2446 .resolve = btf_df_resolve,
2447 .check_member = btf_enum_check_member,
2448 .check_kflag_member = btf_enum_check_kflag_member,
2449 .log_details = btf_enum_log,
2450 .seq_show = btf_enum_seq_show,
2451};
2452
2453static s32 btf_func_proto_check_meta(struct btf_verifier_env *env,
2454 const struct btf_type *t,
2455 u32 meta_left)
2456{
2457 u32 meta_needed = btf_type_vlen(t) * sizeof(struct btf_param);
2458
2459 if (meta_left < meta_needed) {
2460 btf_verifier_log_basic(env, t,
2461 "meta_left:%u meta_needed:%u",
2462 meta_left, meta_needed);
2463 return -EINVAL;
2464 }
2465
2466 if (t->name_off) {
2467 btf_verifier_log_type(env, t, "Invalid name");
2468 return -EINVAL;
2469 }
2470
2471 if (btf_type_kflag(t)) {
2472 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2473 return -EINVAL;
2474 }
2475
2476 btf_verifier_log_type(env, t, NULL);
2477
2478 return meta_needed;
2479}
2480
2481static void btf_func_proto_log(struct btf_verifier_env *env,
2482 const struct btf_type *t)
2483{
2484 const struct btf_param *args = (const struct btf_param *)(t + 1);
2485 u16 nr_args = btf_type_vlen(t), i;
2486
2487 btf_verifier_log(env, "return=%u args=(", t->type);
2488 if (!nr_args) {
2489 btf_verifier_log(env, "void");
2490 goto done;
2491 }
2492
2493 if (nr_args == 1 && !args[0].type) {
2494
2495 btf_verifier_log(env, "vararg");
2496 goto done;
2497 }
2498
2499 btf_verifier_log(env, "%u %s", args[0].type,
2500 __btf_name_by_offset(env->btf,
2501 args[0].name_off));
2502 for (i = 1; i < nr_args - 1; i++)
2503 btf_verifier_log(env, ", %u %s", args[i].type,
2504 __btf_name_by_offset(env->btf,
2505 args[i].name_off));
2506
2507 if (nr_args > 1) {
2508 const struct btf_param *last_arg = &args[nr_args - 1];
2509
2510 if (last_arg->type)
2511 btf_verifier_log(env, ", %u %s", last_arg->type,
2512 __btf_name_by_offset(env->btf,
2513 last_arg->name_off));
2514 else
2515 btf_verifier_log(env, ", vararg");
2516 }
2517
2518done:
2519 btf_verifier_log(env, ")");
2520}
2521
2522static struct btf_kind_operations func_proto_ops = {
2523 .check_meta = btf_func_proto_check_meta,
2524 .resolve = btf_df_resolve,
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534 .check_member = btf_df_check_member,
2535 .check_kflag_member = btf_df_check_kflag_member,
2536 .log_details = btf_func_proto_log,
2537 .seq_show = btf_df_seq_show,
2538};
2539
2540static s32 btf_func_check_meta(struct btf_verifier_env *env,
2541 const struct btf_type *t,
2542 u32 meta_left)
2543{
2544 if (!t->name_off ||
2545 !btf_name_valid_identifier(env->btf, t->name_off)) {
2546 btf_verifier_log_type(env, t, "Invalid name");
2547 return -EINVAL;
2548 }
2549
2550 if (btf_type_vlen(t)) {
2551 btf_verifier_log_type(env, t, "vlen != 0");
2552 return -EINVAL;
2553 }
2554
2555 if (btf_type_kflag(t)) {
2556 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2557 return -EINVAL;
2558 }
2559
2560 btf_verifier_log_type(env, t, NULL);
2561
2562 return 0;
2563}
2564
2565static struct btf_kind_operations func_ops = {
2566 .check_meta = btf_func_check_meta,
2567 .resolve = btf_df_resolve,
2568 .check_member = btf_df_check_member,
2569 .check_kflag_member = btf_df_check_kflag_member,
2570 .log_details = btf_ref_type_log,
2571 .seq_show = btf_df_seq_show,
2572};
2573
2574static s32 btf_var_check_meta(struct btf_verifier_env *env,
2575 const struct btf_type *t,
2576 u32 meta_left)
2577{
2578 const struct btf_var *var;
2579 u32 meta_needed = sizeof(*var);
2580
2581 if (meta_left < meta_needed) {
2582 btf_verifier_log_basic(env, t,
2583 "meta_left:%u meta_needed:%u",
2584 meta_left, meta_needed);
2585 return -EINVAL;
2586 }
2587
2588 if (btf_type_vlen(t)) {
2589 btf_verifier_log_type(env, t, "vlen != 0");
2590 return -EINVAL;
2591 }
2592
2593 if (btf_type_kflag(t)) {
2594 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2595 return -EINVAL;
2596 }
2597
2598 if (!t->name_off ||
2599 !__btf_name_valid(env->btf, t->name_off, true)) {
2600 btf_verifier_log_type(env, t, "Invalid name");
2601 return -EINVAL;
2602 }
2603
2604
2605 if (!t->type || !BTF_TYPE_ID_VALID(t->type)) {
2606 btf_verifier_log_type(env, t, "Invalid type_id");
2607 return -EINVAL;
2608 }
2609
2610 var = btf_type_var(t);
2611 if (var->linkage != BTF_VAR_STATIC &&
2612 var->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2613 btf_verifier_log_type(env, t, "Linkage not supported");
2614 return -EINVAL;
2615 }
2616
2617 btf_verifier_log_type(env, t, NULL);
2618
2619 return meta_needed;
2620}
2621
2622static void btf_var_log(struct btf_verifier_env *env, const struct btf_type *t)
2623{
2624 const struct btf_var *var = btf_type_var(t);
2625
2626 btf_verifier_log(env, "type_id=%u linkage=%u", t->type, var->linkage);
2627}
2628
2629static const struct btf_kind_operations var_ops = {
2630 .check_meta = btf_var_check_meta,
2631 .resolve = btf_var_resolve,
2632 .check_member = btf_df_check_member,
2633 .check_kflag_member = btf_df_check_kflag_member,
2634 .log_details = btf_var_log,
2635 .seq_show = btf_var_seq_show,
2636};
2637
2638static s32 btf_datasec_check_meta(struct btf_verifier_env *env,
2639 const struct btf_type *t,
2640 u32 meta_left)
2641{
2642 const struct btf_var_secinfo *vsi;
2643 u64 last_vsi_end_off = 0, sum = 0;
2644 u32 i, meta_needed;
2645
2646 meta_needed = btf_type_vlen(t) * sizeof(*vsi);
2647 if (meta_left < meta_needed) {
2648 btf_verifier_log_basic(env, t,
2649 "meta_left:%u meta_needed:%u",
2650 meta_left, meta_needed);
2651 return -EINVAL;
2652 }
2653
2654 if (!btf_type_vlen(t)) {
2655 btf_verifier_log_type(env, t, "vlen == 0");
2656 return -EINVAL;
2657 }
2658
2659 if (!t->size) {
2660 btf_verifier_log_type(env, t, "size == 0");
2661 return -EINVAL;
2662 }
2663
2664 if (btf_type_kflag(t)) {
2665 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2666 return -EINVAL;
2667 }
2668
2669 if (!t->name_off ||
2670 !btf_name_valid_section(env->btf, t->name_off)) {
2671 btf_verifier_log_type(env, t, "Invalid name");
2672 return -EINVAL;
2673 }
2674
2675 btf_verifier_log_type(env, t, NULL);
2676
2677 for_each_vsi(i, t, vsi) {
2678
2679 if (!vsi->type || !BTF_TYPE_ID_VALID(vsi->type)) {
2680 btf_verifier_log_vsi(env, t, vsi,
2681 "Invalid type_id");
2682 return -EINVAL;
2683 }
2684
2685 if (vsi->offset < last_vsi_end_off || vsi->offset >= t->size) {
2686 btf_verifier_log_vsi(env, t, vsi,
2687 "Invalid offset");
2688 return -EINVAL;
2689 }
2690
2691 if (!vsi->size || vsi->size > t->size) {
2692 btf_verifier_log_vsi(env, t, vsi,
2693 "Invalid size");
2694 return -EINVAL;
2695 }
2696
2697 last_vsi_end_off = vsi->offset + vsi->size;
2698 if (last_vsi_end_off > t->size) {
2699 btf_verifier_log_vsi(env, t, vsi,
2700 "Invalid offset+size");
2701 return -EINVAL;
2702 }
2703
2704 btf_verifier_log_vsi(env, t, vsi, NULL);
2705 sum += vsi->size;
2706 }
2707
2708 if (t->size < sum) {
2709 btf_verifier_log_type(env, t, "Invalid btf_info size");
2710 return -EINVAL;
2711 }
2712
2713 return meta_needed;
2714}
2715
2716static int btf_datasec_resolve(struct btf_verifier_env *env,
2717 const struct resolve_vertex *v)
2718{
2719 const struct btf_var_secinfo *vsi;
2720 struct btf *btf = env->btf;
2721 u16 i;
2722
2723 for_each_vsi_from(i, v->next_member, v->t, vsi) {
2724 u32 var_type_id = vsi->type, type_id, type_size = 0;
2725 const struct btf_type *var_type = btf_type_by_id(env->btf,
2726 var_type_id);
2727 if (!var_type || !btf_type_is_var(var_type)) {
2728 btf_verifier_log_vsi(env, v->t, vsi,
2729 "Not a VAR kind member");
2730 return -EINVAL;
2731 }
2732
2733 if (!env_type_is_resolve_sink(env, var_type) &&
2734 !env_type_is_resolved(env, var_type_id)) {
2735 env_stack_set_next_member(env, i + 1);
2736 return env_stack_push(env, var_type, var_type_id);
2737 }
2738
2739 type_id = var_type->type;
2740 if (!btf_type_id_size(btf, &type_id, &type_size)) {
2741 btf_verifier_log_vsi(env, v->t, vsi, "Invalid type");
2742 return -EINVAL;
2743 }
2744
2745 if (vsi->size < type_size) {
2746 btf_verifier_log_vsi(env, v->t, vsi, "Invalid size");
2747 return -EINVAL;
2748 }
2749 }
2750
2751 env_stack_pop_resolved(env, 0, 0);
2752 return 0;
2753}
2754
2755static void btf_datasec_log(struct btf_verifier_env *env,
2756 const struct btf_type *t)
2757{
2758 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
2759}
2760
2761static void btf_datasec_seq_show(const struct btf *btf,
2762 const struct btf_type *t, u32 type_id,
2763 void *data, u8 bits_offset,
2764 struct seq_file *m)
2765{
2766 const struct btf_var_secinfo *vsi;
2767 const struct btf_type *var;
2768 u32 i;
2769
2770 seq_printf(m, "section (\"%s\") = {", __btf_name_by_offset(btf, t->name_off));
2771 for_each_vsi(i, t, vsi) {
2772 var = btf_type_by_id(btf, vsi->type);
2773 if (i)
2774 seq_puts(m, ",");
2775 btf_type_ops(var)->seq_show(btf, var, vsi->type,
2776 data + vsi->offset, bits_offset, m);
2777 }
2778 seq_puts(m, "}");
2779}
2780
2781static const struct btf_kind_operations datasec_ops = {
2782 .check_meta = btf_datasec_check_meta,
2783 .resolve = btf_datasec_resolve,
2784 .check_member = btf_df_check_member,
2785 .check_kflag_member = btf_df_check_kflag_member,
2786 .log_details = btf_datasec_log,
2787 .seq_show = btf_datasec_seq_show,
2788};
2789
2790static int btf_func_proto_check(struct btf_verifier_env *env,
2791 const struct btf_type *t)
2792{
2793 const struct btf_type *ret_type;
2794 const struct btf_param *args;
2795 const struct btf *btf;
2796 u16 nr_args, i;
2797 int err;
2798
2799 btf = env->btf;
2800 args = (const struct btf_param *)(t + 1);
2801 nr_args = btf_type_vlen(t);
2802
2803
2804 if (t->type) {
2805 u32 ret_type_id = t->type;
2806
2807 ret_type = btf_type_by_id(btf, ret_type_id);
2808 if (!ret_type) {
2809 btf_verifier_log_type(env, t, "Invalid return type");
2810 return -EINVAL;
2811 }
2812
2813 if (btf_type_needs_resolve(ret_type) &&
2814 !env_type_is_resolved(env, ret_type_id)) {
2815 err = btf_resolve(env, ret_type, ret_type_id);
2816 if (err)
2817 return err;
2818 }
2819
2820
2821 if (!btf_type_id_size(btf, &ret_type_id, NULL)) {
2822 btf_verifier_log_type(env, t, "Invalid return type");
2823 return -EINVAL;
2824 }
2825 }
2826
2827 if (!nr_args)
2828 return 0;
2829
2830
2831 if (!args[nr_args - 1].type) {
2832 if (args[nr_args - 1].name_off) {
2833 btf_verifier_log_type(env, t, "Invalid arg#%u",
2834 nr_args);
2835 return -EINVAL;
2836 }
2837 nr_args--;
2838 }
2839
2840 err = 0;
2841 for (i = 0; i < nr_args; i++) {
2842 const struct btf_type *arg_type;
2843 u32 arg_type_id;
2844
2845 arg_type_id = args[i].type;
2846 arg_type = btf_type_by_id(btf, arg_type_id);
2847 if (!arg_type) {
2848 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
2849 err = -EINVAL;
2850 break;
2851 }
2852
2853 if (args[i].name_off &&
2854 (!btf_name_offset_valid(btf, args[i].name_off) ||
2855 !btf_name_valid_identifier(btf, args[i].name_off))) {
2856 btf_verifier_log_type(env, t,
2857 "Invalid arg#%u", i + 1);
2858 err = -EINVAL;
2859 break;
2860 }
2861
2862 if (btf_type_needs_resolve(arg_type) &&
2863 !env_type_is_resolved(env, arg_type_id)) {
2864 err = btf_resolve(env, arg_type, arg_type_id);
2865 if (err)
2866 break;
2867 }
2868
2869 if (!btf_type_id_size(btf, &arg_type_id, NULL)) {
2870 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
2871 err = -EINVAL;
2872 break;
2873 }
2874 }
2875
2876 return err;
2877}
2878
2879static int btf_func_check(struct btf_verifier_env *env,
2880 const struct btf_type *t)
2881{
2882 const struct btf_type *proto_type;
2883 const struct btf_param *args;
2884 const struct btf *btf;
2885 u16 nr_args, i;
2886
2887 btf = env->btf;
2888 proto_type = btf_type_by_id(btf, t->type);
2889
2890 if (!proto_type || !btf_type_is_func_proto(proto_type)) {
2891 btf_verifier_log_type(env, t, "Invalid type_id");
2892 return -EINVAL;
2893 }
2894
2895 args = (const struct btf_param *)(proto_type + 1);
2896 nr_args = btf_type_vlen(proto_type);
2897 for (i = 0; i < nr_args; i++) {
2898 if (!args[i].name_off && args[i].type) {
2899 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
2900 return -EINVAL;
2901 }
2902 }
2903
2904 return 0;
2905}
2906
2907static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = {
2908 [BTF_KIND_INT] = &int_ops,
2909 [BTF_KIND_PTR] = &ptr_ops,
2910 [BTF_KIND_ARRAY] = &array_ops,
2911 [BTF_KIND_STRUCT] = &struct_ops,
2912 [BTF_KIND_UNION] = &struct_ops,
2913 [BTF_KIND_ENUM] = &enum_ops,
2914 [BTF_KIND_FWD] = &fwd_ops,
2915 [BTF_KIND_TYPEDEF] = &modifier_ops,
2916 [BTF_KIND_VOLATILE] = &modifier_ops,
2917 [BTF_KIND_CONST] = &modifier_ops,
2918 [BTF_KIND_RESTRICT] = &modifier_ops,
2919 [BTF_KIND_FUNC] = &func_ops,
2920 [BTF_KIND_FUNC_PROTO] = &func_proto_ops,
2921 [BTF_KIND_VAR] = &var_ops,
2922 [BTF_KIND_DATASEC] = &datasec_ops,
2923};
2924
2925static s32 btf_check_meta(struct btf_verifier_env *env,
2926 const struct btf_type *t,
2927 u32 meta_left)
2928{
2929 u32 saved_meta_left = meta_left;
2930 s32 var_meta_size;
2931
2932 if (meta_left < sizeof(*t)) {
2933 btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu",
2934 env->log_type_id, meta_left, sizeof(*t));
2935 return -EINVAL;
2936 }
2937 meta_left -= sizeof(*t);
2938
2939 if (t->info & ~BTF_INFO_MASK) {
2940 btf_verifier_log(env, "[%u] Invalid btf_info:%x",
2941 env->log_type_id, t->info);
2942 return -EINVAL;
2943 }
2944
2945 if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX ||
2946 BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) {
2947 btf_verifier_log(env, "[%u] Invalid kind:%u",
2948 env->log_type_id, BTF_INFO_KIND(t->info));
2949 return -EINVAL;
2950 }
2951
2952 if (!btf_name_offset_valid(env->btf, t->name_off)) {
2953 btf_verifier_log(env, "[%u] Invalid name_offset:%u",
2954 env->log_type_id, t->name_off);
2955 return -EINVAL;
2956 }
2957
2958 var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left);
2959 if (var_meta_size < 0)
2960 return var_meta_size;
2961
2962 meta_left -= var_meta_size;
2963
2964 return saved_meta_left - meta_left;
2965}
2966
2967static int btf_check_all_metas(struct btf_verifier_env *env)
2968{
2969 struct btf *btf = env->btf;
2970 struct btf_header *hdr;
2971 void *cur, *end;
2972
2973 hdr = &btf->hdr;
2974 cur = btf->nohdr_data + hdr->type_off;
2975 end = cur + hdr->type_len;
2976
2977 env->log_type_id = 1;
2978 while (cur < end) {
2979 struct btf_type *t = cur;
2980 s32 meta_size;
2981
2982 meta_size = btf_check_meta(env, t, end - cur);
2983 if (meta_size < 0)
2984 return meta_size;
2985
2986 btf_add_type(env, t);
2987 cur += meta_size;
2988 env->log_type_id++;
2989 }
2990
2991 return 0;
2992}
2993
2994static bool btf_resolve_valid(struct btf_verifier_env *env,
2995 const struct btf_type *t,
2996 u32 type_id)
2997{
2998 struct btf *btf = env->btf;
2999
3000 if (!env_type_is_resolved(env, type_id))
3001 return false;
3002
3003 if (btf_type_is_struct(t) || btf_type_is_datasec(t))
3004 return !btf->resolved_ids[type_id] &&
3005 !btf->resolved_sizes[type_id];
3006
3007 if (btf_type_is_modifier(t) || btf_type_is_ptr(t) ||
3008 btf_type_is_var(t)) {
3009 t = btf_type_id_resolve(btf, &type_id);
3010 return t &&
3011 !btf_type_is_modifier(t) &&
3012 !btf_type_is_var(t) &&
3013 !btf_type_is_datasec(t);
3014 }
3015
3016 if (btf_type_is_array(t)) {
3017 const struct btf_array *array = btf_type_array(t);
3018 const struct btf_type *elem_type;
3019 u32 elem_type_id = array->type;
3020 u32 elem_size;
3021
3022 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
3023 return elem_type && !btf_type_is_modifier(elem_type) &&
3024 (array->nelems * elem_size ==
3025 btf->resolved_sizes[type_id]);
3026 }
3027
3028 return false;
3029}
3030
3031static int btf_resolve(struct btf_verifier_env *env,
3032 const struct btf_type *t, u32 type_id)
3033{
3034 u32 save_log_type_id = env->log_type_id;
3035 const struct resolve_vertex *v;
3036 int err = 0;
3037
3038 env->resolve_mode = RESOLVE_TBD;
3039 env_stack_push(env, t, type_id);
3040 while (!err && (v = env_stack_peak(env))) {
3041 env->log_type_id = v->type_id;
3042 err = btf_type_ops(v->t)->resolve(env, v);
3043 }
3044
3045 env->log_type_id = type_id;
3046 if (err == -E2BIG) {
3047 btf_verifier_log_type(env, t,
3048 "Exceeded max resolving depth:%u",
3049 MAX_RESOLVE_DEPTH);
3050 } else if (err == -EEXIST) {
3051 btf_verifier_log_type(env, t, "Loop detected");
3052 }
3053
3054
3055 if (!err && !btf_resolve_valid(env, t, type_id)) {
3056 btf_verifier_log_type(env, t, "Invalid resolve state");
3057 err = -EINVAL;
3058 }
3059
3060 env->log_type_id = save_log_type_id;
3061 return err;
3062}
3063
3064static int btf_check_all_types(struct btf_verifier_env *env)
3065{
3066 struct btf *btf = env->btf;
3067 u32 type_id;
3068 int err;
3069
3070 err = env_resolve_init(env);
3071 if (err)
3072 return err;
3073
3074 env->phase++;
3075 for (type_id = 1; type_id <= btf->nr_types; type_id++) {
3076 const struct btf_type *t = btf_type_by_id(btf, type_id);
3077
3078 env->log_type_id = type_id;
3079 if (btf_type_needs_resolve(t) &&
3080 !env_type_is_resolved(env, type_id)) {
3081 err = btf_resolve(env, t, type_id);
3082 if (err)
3083 return err;
3084 }
3085
3086 if (btf_type_is_func_proto(t)) {
3087 err = btf_func_proto_check(env, t);
3088 if (err)
3089 return err;
3090 }
3091
3092 if (btf_type_is_func(t)) {
3093 err = btf_func_check(env, t);
3094 if (err)
3095 return err;
3096 }
3097 }
3098
3099 return 0;
3100}
3101
3102static int btf_parse_type_sec(struct btf_verifier_env *env)
3103{
3104 const struct btf_header *hdr = &env->btf->hdr;
3105 int err;
3106
3107
3108 if (hdr->type_off & (sizeof(u32) - 1)) {
3109 btf_verifier_log(env, "Unaligned type_off");
3110 return -EINVAL;
3111 }
3112
3113 if (!hdr->type_len) {
3114 btf_verifier_log(env, "No type found");
3115 return -EINVAL;
3116 }
3117
3118 err = btf_check_all_metas(env);
3119 if (err)
3120 return err;
3121
3122 return btf_check_all_types(env);
3123}
3124
3125static int btf_parse_str_sec(struct btf_verifier_env *env)
3126{
3127 const struct btf_header *hdr;
3128 struct btf *btf = env->btf;
3129 const char *start, *end;
3130
3131 hdr = &btf->hdr;
3132 start = btf->nohdr_data + hdr->str_off;
3133 end = start + hdr->str_len;
3134
3135 if (end != btf->data + btf->data_size) {
3136 btf_verifier_log(env, "String section is not at the end");
3137 return -EINVAL;
3138 }
3139
3140 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET ||
3141 start[0] || end[-1]) {
3142 btf_verifier_log(env, "Invalid string section");
3143 return -EINVAL;
3144 }
3145
3146 btf->strings = start;
3147
3148 return 0;
3149}
3150
3151static const size_t btf_sec_info_offset[] = {
3152 offsetof(struct btf_header, type_off),
3153 offsetof(struct btf_header, str_off),
3154};
3155
3156static int btf_sec_info_cmp(const void *a, const void *b)
3157{
3158 const struct btf_sec_info *x = a;
3159 const struct btf_sec_info *y = b;
3160
3161 return (int)(x->off - y->off) ? : (int)(x->len - y->len);
3162}
3163
3164static int btf_check_sec_info(struct btf_verifier_env *env,
3165 u32 btf_data_size)
3166{
3167 struct btf_sec_info secs[ARRAY_SIZE(btf_sec_info_offset)];
3168 u32 total, expected_total, i;
3169 const struct btf_header *hdr;
3170 const struct btf *btf;
3171
3172 btf = env->btf;
3173 hdr = &btf->hdr;
3174
3175
3176 for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++)
3177 secs[i] = *(struct btf_sec_info *)((void *)hdr +
3178 btf_sec_info_offset[i]);
3179
3180 sort(secs, ARRAY_SIZE(btf_sec_info_offset),
3181 sizeof(struct btf_sec_info), btf_sec_info_cmp, NULL);
3182
3183
3184 total = 0;
3185 expected_total = btf_data_size - hdr->hdr_len;
3186 for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++) {
3187 if (expected_total < secs[i].off) {
3188 btf_verifier_log(env, "Invalid section offset");
3189 return -EINVAL;
3190 }
3191 if (total < secs[i].off) {
3192
3193 btf_verifier_log(env, "Unsupported section found");
3194 return -EINVAL;
3195 }
3196 if (total > secs[i].off) {
3197 btf_verifier_log(env, "Section overlap found");
3198 return -EINVAL;
3199 }
3200 if (expected_total - total < secs[i].len) {
3201 btf_verifier_log(env,
3202 "Total section length too long");
3203 return -EINVAL;
3204 }
3205 total += secs[i].len;
3206 }
3207
3208
3209 if (expected_total != total) {
3210 btf_verifier_log(env, "Unsupported section found");
3211 return -EINVAL;
3212 }
3213
3214 return 0;
3215}
3216
3217static int btf_parse_hdr(struct btf_verifier_env *env)
3218{
3219 u32 hdr_len, hdr_copy, btf_data_size;
3220 const struct btf_header *hdr;
3221 struct btf *btf;
3222 int err;
3223
3224 btf = env->btf;
3225 btf_data_size = btf->data_size;
3226
3227 if (btf_data_size <
3228 offsetof(struct btf_header, hdr_len) + sizeof(hdr->hdr_len)) {
3229 btf_verifier_log(env, "hdr_len not found");
3230 return -EINVAL;
3231 }
3232
3233 hdr = btf->data;
3234 hdr_len = hdr->hdr_len;
3235 if (btf_data_size < hdr_len) {
3236 btf_verifier_log(env, "btf_header not found");
3237 return -EINVAL;
3238 }
3239
3240
3241 if (hdr_len > sizeof(btf->hdr)) {
3242 u8 *expected_zero = btf->data + sizeof(btf->hdr);
3243 u8 *end = btf->data + hdr_len;
3244
3245 for (; expected_zero < end; expected_zero++) {
3246 if (*expected_zero) {
3247 btf_verifier_log(env, "Unsupported btf_header");
3248 return -E2BIG;
3249 }
3250 }
3251 }
3252
3253 hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr));
3254 memcpy(&btf->hdr, btf->data, hdr_copy);
3255
3256 hdr = &btf->hdr;
3257
3258 btf_verifier_log_hdr(env, btf_data_size);
3259
3260 if (hdr->magic != BTF_MAGIC) {
3261 btf_verifier_log(env, "Invalid magic");
3262 return -EINVAL;
3263 }
3264
3265 if (hdr->version != BTF_VERSION) {
3266 btf_verifier_log(env, "Unsupported version");
3267 return -ENOTSUPP;
3268 }
3269
3270 if (hdr->flags) {
3271 btf_verifier_log(env, "Unsupported flags");
3272 return -ENOTSUPP;
3273 }
3274
3275 if (btf_data_size == hdr->hdr_len) {
3276 btf_verifier_log(env, "No data");
3277 return -EINVAL;
3278 }
3279
3280 err = btf_check_sec_info(env, btf_data_size);
3281 if (err)
3282 return err;
3283
3284 return 0;
3285}
3286
3287static struct btf *btf_parse(void __user *btf_data, u32 btf_data_size,
3288 u32 log_level, char __user *log_ubuf, u32 log_size)
3289{
3290 struct btf_verifier_env *env = NULL;
3291 struct bpf_verifier_log *log;
3292 struct btf *btf = NULL;
3293 u8 *data;
3294 int err;
3295
3296 if (btf_data_size > BTF_MAX_SIZE)
3297 return ERR_PTR(-E2BIG);
3298
3299 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
3300 if (!env)
3301 return ERR_PTR(-ENOMEM);
3302
3303 log = &env->log;
3304 if (log_level || log_ubuf || log_size) {
3305
3306
3307
3308 log->level = log_level;
3309 log->ubuf = log_ubuf;
3310 log->len_total = log_size;
3311
3312
3313 if (log->len_total < 128 || log->len_total > UINT_MAX >> 8 ||
3314 !log->level || !log->ubuf) {
3315 err = -EINVAL;
3316 goto errout;
3317 }
3318 }
3319
3320 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
3321 if (!btf) {
3322 err = -ENOMEM;
3323 goto errout;
3324 }
3325 env->btf = btf;
3326
3327 data = kvmalloc(btf_data_size, GFP_KERNEL | __GFP_NOWARN);
3328 if (!data) {
3329 err = -ENOMEM;
3330 goto errout;
3331 }
3332
3333 btf->data = data;
3334 btf->data_size = btf_data_size;
3335
3336 if (copy_from_user(data, btf_data, btf_data_size)) {
3337 err = -EFAULT;
3338 goto errout;
3339 }
3340
3341 err = btf_parse_hdr(env);
3342 if (err)
3343 goto errout;
3344
3345 btf->nohdr_data = btf->data + btf->hdr.hdr_len;
3346
3347 err = btf_parse_str_sec(env);
3348 if (err)
3349 goto errout;
3350
3351 err = btf_parse_type_sec(env);
3352 if (err)
3353 goto errout;
3354
3355 if (log->level && bpf_verifier_log_full(log)) {
3356 err = -ENOSPC;
3357 goto errout;
3358 }
3359
3360 btf_verifier_env_free(env);
3361 refcount_set(&btf->refcnt, 1);
3362 return btf;
3363
3364errout:
3365 btf_verifier_env_free(env);
3366 if (btf)
3367 btf_free(btf);
3368 return ERR_PTR(err);
3369}
3370
3371void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,
3372 struct seq_file *m)
3373{
3374 const struct btf_type *t = btf_type_by_id(btf, type_id);
3375
3376 btf_type_ops(t)->seq_show(btf, t, type_id, obj, 0, m);
3377}
3378
3379static int btf_release(struct inode *inode, struct file *filp)
3380{
3381 btf_put(filp->private_data);
3382 return 0;
3383}
3384
3385const struct file_operations btf_fops = {
3386 .release = btf_release,
3387};
3388
3389static int __btf_new_fd(struct btf *btf)
3390{
3391 return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC);
3392}
3393
3394int btf_new_fd(const union bpf_attr *attr)
3395{
3396 struct btf *btf;
3397 int ret;
3398
3399 btf = btf_parse(u64_to_user_ptr(attr->btf),
3400 attr->btf_size, attr->btf_log_level,
3401 u64_to_user_ptr(attr->btf_log_buf),
3402 attr->btf_log_size);
3403 if (IS_ERR(btf))
3404 return PTR_ERR(btf);
3405
3406 ret = btf_alloc_id(btf);
3407 if (ret) {
3408 btf_free(btf);
3409 return ret;
3410 }
3411
3412
3413
3414
3415
3416
3417
3418 ret = __btf_new_fd(btf);
3419 if (ret < 0)
3420 btf_put(btf);
3421
3422 return ret;
3423}
3424
3425struct btf *btf_get_by_fd(int fd)
3426{
3427 struct btf *btf;
3428 struct fd f;
3429
3430 f = fdget(fd);
3431
3432 if (!f.file)
3433 return ERR_PTR(-EBADF);
3434
3435 if (f.file->f_op != &btf_fops) {
3436 fdput(f);
3437 return ERR_PTR(-EINVAL);
3438 }
3439
3440 btf = f.file->private_data;
3441 refcount_inc(&btf->refcnt);
3442 fdput(f);
3443
3444 return btf;
3445}
3446
3447int btf_get_info_by_fd(const struct btf *btf,
3448 const union bpf_attr *attr,
3449 union bpf_attr __user *uattr)
3450{
3451 struct bpf_btf_info __user *uinfo;
3452 struct bpf_btf_info info = {};
3453 u32 info_copy, btf_copy;
3454 void __user *ubtf;
3455 u32 uinfo_len;
3456
3457 uinfo = u64_to_user_ptr(attr->info.info);
3458 uinfo_len = attr->info.info_len;
3459
3460 info_copy = min_t(u32, uinfo_len, sizeof(info));
3461 if (copy_from_user(&info, uinfo, info_copy))
3462 return -EFAULT;
3463
3464 info.id = btf->id;
3465 ubtf = u64_to_user_ptr(info.btf);
3466 btf_copy = min_t(u32, btf->data_size, info.btf_size);
3467 if (copy_to_user(ubtf, btf->data, btf_copy))
3468 return -EFAULT;
3469 info.btf_size = btf->data_size;
3470
3471 if (copy_to_user(uinfo, &info, info_copy) ||
3472 put_user(info_copy, &uattr->info.info_len))
3473 return -EFAULT;
3474
3475 return 0;
3476}
3477
3478int btf_get_fd_by_id(u32 id)
3479{
3480 struct btf *btf;
3481 int fd;
3482
3483 rcu_read_lock();
3484 btf = idr_find(&btf_idr, id);
3485 if (!btf || !refcount_inc_not_zero(&btf->refcnt))
3486 btf = ERR_PTR(-ENOENT);
3487 rcu_read_unlock();
3488
3489 if (IS_ERR(btf))
3490 return PTR_ERR(btf);
3491
3492 fd = __btf_new_fd(btf);
3493 if (fd < 0)
3494 btf_put(btf);
3495
3496 return fd;
3497}
3498
3499u32 btf_id(const struct btf *btf)
3500{
3501 return btf->id;
3502}
3503