1
2
3
4#include <uapi/linux/btf.h>
5#include <uapi/linux/bpf.h>
6#include <uapi/linux/bpf_perf_event.h>
7#include <uapi/linux/types.h>
8#include <linux/seq_file.h>
9#include <linux/compiler.h>
10#include <linux/ctype.h>
11#include <linux/errno.h>
12#include <linux/slab.h>
13#include <linux/anon_inodes.h>
14#include <linux/file.h>
15#include <linux/uaccess.h>
16#include <linux/kernel.h>
17#include <linux/idr.h>
18#include <linux/sort.h>
19#include <linux/bpf_verifier.h>
20#include <linux/btf.h>
21#include <linux/btf_ids.h>
22#include <linux/skmsg.h>
23#include <linux/perf_event.h>
24#include <linux/bsearch.h>
25#include <linux/kobject.h>
26#include <linux/sysfs.h>
27#include <net/sock.h>
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169#define BITS_PER_U128 (sizeof(u64) * BITS_PER_BYTE * 2)
170#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1)
171#define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK)
172#define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3)
173#define BITS_ROUNDUP_BYTES(bits) \
174 (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits))
175
176#define BTF_INFO_MASK 0x9f00ffff
177#define BTF_INT_MASK 0x0fffffff
178#define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE)
179#define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET)
180
181
182
183
184
185#define BTF_MAX_SIZE (16 * 1024 * 1024)
186
187#define for_each_member_from(i, from, struct_type, member) \
188 for (i = from, member = btf_type_member(struct_type) + from; \
189 i < btf_type_vlen(struct_type); \
190 i++, member++)
191
192#define for_each_vsi_from(i, from, struct_type, member) \
193 for (i = from, member = btf_type_var_secinfo(struct_type) + from; \
194 i < btf_type_vlen(struct_type); \
195 i++, member++)
196
197DEFINE_IDR(btf_idr);
198DEFINE_SPINLOCK(btf_idr_lock);
199
200struct btf {
201 void *data;
202 struct btf_type **types;
203 u32 *resolved_ids;
204 u32 *resolved_sizes;
205 const char *strings;
206 void *nohdr_data;
207 struct btf_header hdr;
208 u32 nr_types;
209 u32 types_size;
210 u32 data_size;
211 refcount_t refcnt;
212 u32 id;
213 struct rcu_head rcu;
214
215
216 struct btf *base_btf;
217 u32 start_id;
218 u32 start_str_off;
219 char name[MODULE_NAME_LEN];
220 bool kernel_btf;
221};
222
223enum verifier_phase {
224 CHECK_META,
225 CHECK_TYPE,
226};
227
228struct resolve_vertex {
229 const struct btf_type *t;
230 u32 type_id;
231 u16 next_member;
232};
233
234enum visit_state {
235 NOT_VISITED,
236 VISITED,
237 RESOLVED,
238};
239
240enum resolve_mode {
241 RESOLVE_TBD,
242 RESOLVE_PTR,
243 RESOLVE_STRUCT_OR_ARRAY,
244
245
246};
247
248#define MAX_RESOLVE_DEPTH 32
249
250struct btf_sec_info {
251 u32 off;
252 u32 len;
253};
254
255struct btf_verifier_env {
256 struct btf *btf;
257 u8 *visit_states;
258 struct resolve_vertex stack[MAX_RESOLVE_DEPTH];
259 struct bpf_verifier_log log;
260 u32 log_type_id;
261 u32 top_stack;
262 enum verifier_phase phase;
263 enum resolve_mode resolve_mode;
264};
265
266static const char * const btf_kind_str[NR_BTF_KINDS] = {
267 [BTF_KIND_UNKN] = "UNKNOWN",
268 [BTF_KIND_INT] = "INT",
269 [BTF_KIND_PTR] = "PTR",
270 [BTF_KIND_ARRAY] = "ARRAY",
271 [BTF_KIND_STRUCT] = "STRUCT",
272 [BTF_KIND_UNION] = "UNION",
273 [BTF_KIND_ENUM] = "ENUM",
274 [BTF_KIND_FWD] = "FWD",
275 [BTF_KIND_TYPEDEF] = "TYPEDEF",
276 [BTF_KIND_VOLATILE] = "VOLATILE",
277 [BTF_KIND_CONST] = "CONST",
278 [BTF_KIND_RESTRICT] = "RESTRICT",
279 [BTF_KIND_FUNC] = "FUNC",
280 [BTF_KIND_FUNC_PROTO] = "FUNC_PROTO",
281 [BTF_KIND_VAR] = "VAR",
282 [BTF_KIND_DATASEC] = "DATASEC",
283 [BTF_KIND_FLOAT] = "FLOAT",
284};
285
286const char *btf_type_str(const struct btf_type *t)
287{
288 return btf_kind_str[BTF_INFO_KIND(t->info)];
289}
290
291
292#define BTF_SHOW_OBJ_SAFE_SIZE 32
293
294
295
296
297
298
299
300
301#define BTF_SHOW_OBJ_BASE_TYPE_SIZE 16
302
303
304#define BTF_SHOW_NAME_SIZE 80
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349struct btf_show {
350 u64 flags;
351 void *target;
352 void (*showfn)(struct btf_show *show, const char *fmt, va_list args);
353 const struct btf *btf;
354
355 struct {
356 u8 depth;
357 u8 depth_to_show;
358 u8 depth_check;
359 u8 array_member:1,
360 array_terminated:1;
361 u16 array_encoding;
362 u32 type_id;
363 int status;
364 const struct btf_type *type;
365 const struct btf_member *member;
366 char name[BTF_SHOW_NAME_SIZE];
367 } state;
368 struct {
369 u32 size;
370 void *head;
371 void *data;
372 u8 safe[BTF_SHOW_OBJ_SAFE_SIZE];
373 } obj;
374};
375
376struct btf_kind_operations {
377 s32 (*check_meta)(struct btf_verifier_env *env,
378 const struct btf_type *t,
379 u32 meta_left);
380 int (*resolve)(struct btf_verifier_env *env,
381 const struct resolve_vertex *v);
382 int (*check_member)(struct btf_verifier_env *env,
383 const struct btf_type *struct_type,
384 const struct btf_member *member,
385 const struct btf_type *member_type);
386 int (*check_kflag_member)(struct btf_verifier_env *env,
387 const struct btf_type *struct_type,
388 const struct btf_member *member,
389 const struct btf_type *member_type);
390 void (*log_details)(struct btf_verifier_env *env,
391 const struct btf_type *t);
392 void (*show)(const struct btf *btf, const struct btf_type *t,
393 u32 type_id, void *data, u8 bits_offsets,
394 struct btf_show *show);
395};
396
397static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS];
398static struct btf_type btf_void;
399
400static int btf_resolve(struct btf_verifier_env *env,
401 const struct btf_type *t, u32 type_id);
402
403static bool btf_type_is_modifier(const struct btf_type *t)
404{
405
406
407
408
409
410
411
412
413
414
415 switch (BTF_INFO_KIND(t->info)) {
416 case BTF_KIND_TYPEDEF:
417 case BTF_KIND_VOLATILE:
418 case BTF_KIND_CONST:
419 case BTF_KIND_RESTRICT:
420 return true;
421 }
422
423 return false;
424}
425
426bool btf_type_is_void(const struct btf_type *t)
427{
428 return t == &btf_void;
429}
430
431static bool btf_type_is_fwd(const struct btf_type *t)
432{
433 return BTF_INFO_KIND(t->info) == BTF_KIND_FWD;
434}
435
436static bool btf_type_nosize(const struct btf_type *t)
437{
438 return btf_type_is_void(t) || btf_type_is_fwd(t) ||
439 btf_type_is_func(t) || btf_type_is_func_proto(t);
440}
441
442static bool btf_type_nosize_or_null(const struct btf_type *t)
443{
444 return !t || btf_type_nosize(t);
445}
446
447static bool __btf_type_is_struct(const struct btf_type *t)
448{
449 return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT;
450}
451
452static bool btf_type_is_array(const struct btf_type *t)
453{
454 return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
455}
456
457static bool btf_type_is_datasec(const struct btf_type *t)
458{
459 return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
460}
461
462u32 btf_nr_types(const struct btf *btf)
463{
464 u32 total = 0;
465
466 while (btf) {
467 total += btf->nr_types;
468 btf = btf->base_btf;
469 }
470
471 return total;
472}
473
474s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind)
475{
476 const struct btf_type *t;
477 const char *tname;
478 u32 i, total;
479
480 total = btf_nr_types(btf);
481 for (i = 1; i < total; i++) {
482 t = btf_type_by_id(btf, i);
483 if (BTF_INFO_KIND(t->info) != kind)
484 continue;
485
486 tname = btf_name_by_offset(btf, t->name_off);
487 if (!strcmp(tname, name))
488 return i;
489 }
490
491 return -ENOENT;
492}
493
494const struct btf_type *btf_type_skip_modifiers(const struct btf *btf,
495 u32 id, u32 *res_id)
496{
497 const struct btf_type *t = btf_type_by_id(btf, id);
498
499 while (btf_type_is_modifier(t)) {
500 id = t->type;
501 t = btf_type_by_id(btf, t->type);
502 }
503
504 if (res_id)
505 *res_id = id;
506
507 return t;
508}
509
510const struct btf_type *btf_type_resolve_ptr(const struct btf *btf,
511 u32 id, u32 *res_id)
512{
513 const struct btf_type *t;
514
515 t = btf_type_skip_modifiers(btf, id, NULL);
516 if (!btf_type_is_ptr(t))
517 return NULL;
518
519 return btf_type_skip_modifiers(btf, t->type, res_id);
520}
521
522const struct btf_type *btf_type_resolve_func_ptr(const struct btf *btf,
523 u32 id, u32 *res_id)
524{
525 const struct btf_type *ptype;
526
527 ptype = btf_type_resolve_ptr(btf, id, res_id);
528 if (ptype && btf_type_is_func_proto(ptype))
529 return ptype;
530
531 return NULL;
532}
533
534
535
536
537static bool btf_type_is_resolve_source_only(const struct btf_type *t)
538{
539 return btf_type_is_var(t) ||
540 btf_type_is_datasec(t);
541}
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559static bool btf_type_needs_resolve(const struct btf_type *t)
560{
561 return btf_type_is_modifier(t) ||
562 btf_type_is_ptr(t) ||
563 btf_type_is_struct(t) ||
564 btf_type_is_array(t) ||
565 btf_type_is_var(t) ||
566 btf_type_is_datasec(t);
567}
568
569
570static bool btf_type_has_size(const struct btf_type *t)
571{
572 switch (BTF_INFO_KIND(t->info)) {
573 case BTF_KIND_INT:
574 case BTF_KIND_STRUCT:
575 case BTF_KIND_UNION:
576 case BTF_KIND_ENUM:
577 case BTF_KIND_DATASEC:
578 case BTF_KIND_FLOAT:
579 return true;
580 }
581
582 return false;
583}
584
585static const char *btf_int_encoding_str(u8 encoding)
586{
587 if (encoding == 0)
588 return "(none)";
589 else if (encoding == BTF_INT_SIGNED)
590 return "SIGNED";
591 else if (encoding == BTF_INT_CHAR)
592 return "CHAR";
593 else if (encoding == BTF_INT_BOOL)
594 return "BOOL";
595 else
596 return "UNKN";
597}
598
599static u32 btf_type_int(const struct btf_type *t)
600{
601 return *(u32 *)(t + 1);
602}
603
604static const struct btf_array *btf_type_array(const struct btf_type *t)
605{
606 return (const struct btf_array *)(t + 1);
607}
608
609static const struct btf_enum *btf_type_enum(const struct btf_type *t)
610{
611 return (const struct btf_enum *)(t + 1);
612}
613
614static const struct btf_var *btf_type_var(const struct btf_type *t)
615{
616 return (const struct btf_var *)(t + 1);
617}
618
619static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
620{
621 return kind_ops[BTF_INFO_KIND(t->info)];
622}
623
624static bool btf_name_offset_valid(const struct btf *btf, u32 offset)
625{
626 if (!BTF_STR_OFFSET_VALID(offset))
627 return false;
628
629 while (offset < btf->start_str_off)
630 btf = btf->base_btf;
631
632 offset -= btf->start_str_off;
633 return offset < btf->hdr.str_len;
634}
635
636static bool __btf_name_char_ok(char c, bool first, bool dot_ok)
637{
638 if ((first ? !isalpha(c) :
639 !isalnum(c)) &&
640 c != '_' &&
641 ((c == '.' && !dot_ok) ||
642 c != '.'))
643 return false;
644 return true;
645}
646
647static const char *btf_str_by_offset(const struct btf *btf, u32 offset)
648{
649 while (offset < btf->start_str_off)
650 btf = btf->base_btf;
651
652 offset -= btf->start_str_off;
653 if (offset < btf->hdr.str_len)
654 return &btf->strings[offset];
655
656 return NULL;
657}
658
659static bool __btf_name_valid(const struct btf *btf, u32 offset, bool dot_ok)
660{
661
662 const char *src = btf_str_by_offset(btf, offset);
663 const char *src_limit;
664
665 if (!__btf_name_char_ok(*src, true, dot_ok))
666 return false;
667
668
669 src_limit = src + KSYM_NAME_LEN;
670 src++;
671 while (*src && src < src_limit) {
672 if (!__btf_name_char_ok(*src, false, dot_ok))
673 return false;
674 src++;
675 }
676
677 return !*src;
678}
679
680
681
682
683static bool btf_name_valid_identifier(const struct btf *btf, u32 offset)
684{
685 return __btf_name_valid(btf, offset, false);
686}
687
688static bool btf_name_valid_section(const struct btf *btf, u32 offset)
689{
690 return __btf_name_valid(btf, offset, true);
691}
692
693static const char *__btf_name_by_offset(const struct btf *btf, u32 offset)
694{
695 const char *name;
696
697 if (!offset)
698 return "(anon)";
699
700 name = btf_str_by_offset(btf, offset);
701 return name ?: "(invalid-name-offset)";
702}
703
704const char *btf_name_by_offset(const struct btf *btf, u32 offset)
705{
706 return btf_str_by_offset(btf, offset);
707}
708
709const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id)
710{
711 while (type_id < btf->start_id)
712 btf = btf->base_btf;
713
714 type_id -= btf->start_id;
715 if (type_id >= btf->nr_types)
716 return NULL;
717 return btf->types[type_id];
718}
719
720
721
722
723
724static bool btf_type_int_is_regular(const struct btf_type *t)
725{
726 u8 nr_bits, nr_bytes;
727 u32 int_data;
728
729 int_data = btf_type_int(t);
730 nr_bits = BTF_INT_BITS(int_data);
731 nr_bytes = BITS_ROUNDUP_BYTES(nr_bits);
732 if (BITS_PER_BYTE_MASKED(nr_bits) ||
733 BTF_INT_OFFSET(int_data) ||
734 (nr_bytes != sizeof(u8) && nr_bytes != sizeof(u16) &&
735 nr_bytes != sizeof(u32) && nr_bytes != sizeof(u64) &&
736 nr_bytes != (2 * sizeof(u64)))) {
737 return false;
738 }
739
740 return true;
741}
742
743
744
745
746
747bool btf_member_is_reg_int(const struct btf *btf, const struct btf_type *s,
748 const struct btf_member *m,
749 u32 expected_offset, u32 expected_size)
750{
751 const struct btf_type *t;
752 u32 id, int_data;
753 u8 nr_bits;
754
755 id = m->type;
756 t = btf_type_id_size(btf, &id, NULL);
757 if (!t || !btf_type_is_int(t))
758 return false;
759
760 int_data = btf_type_int(t);
761 nr_bits = BTF_INT_BITS(int_data);
762 if (btf_type_kflag(s)) {
763 u32 bitfield_size = BTF_MEMBER_BITFIELD_SIZE(m->offset);
764 u32 bit_offset = BTF_MEMBER_BIT_OFFSET(m->offset);
765
766
767
768
769 return !bitfield_size &&
770 BITS_ROUNDUP_BYTES(bit_offset) == expected_offset &&
771 BITS_ROUNDUP_BYTES(nr_bits) == expected_size;
772 }
773
774 if (BTF_INT_OFFSET(int_data) ||
775 BITS_PER_BYTE_MASKED(m->offset) ||
776 BITS_ROUNDUP_BYTES(m->offset) != expected_offset ||
777 BITS_PER_BYTE_MASKED(nr_bits) ||
778 BITS_ROUNDUP_BYTES(nr_bits) != expected_size)
779 return false;
780
781 return true;
782}
783
784
785static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf,
786 u32 id)
787{
788 const struct btf_type *t = btf_type_by_id(btf, id);
789
790 while (btf_type_is_modifier(t) &&
791 BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) {
792 t = btf_type_by_id(btf, t->type);
793 }
794
795 return t;
796}
797
798#define BTF_SHOW_MAX_ITER 10
799
800#define BTF_KIND_BIT(kind) (1ULL << kind)
801
802
803
804
805
806
807
808static const char *btf_show_name(struct btf_show *show)
809{
810
811 const char *array_suffixes = "[][][][][][][][][][]";
812 const char *array_suffix = &array_suffixes[strlen(array_suffixes)];
813
814 const char *ptr_suffixes = "**********";
815 const char *ptr_suffix = &ptr_suffixes[strlen(ptr_suffixes)];
816 const char *name = NULL, *prefix = "", *parens = "";
817 const struct btf_member *m = show->state.member;
818 const struct btf_type *t = show->state.type;
819 const struct btf_array *array;
820 u32 id = show->state.type_id;
821 const char *member = NULL;
822 bool show_member = false;
823 u64 kinds = 0;
824 int i;
825
826 show->state.name[0] = '\0';
827
828
829
830
831
832
833 if (show->state.array_member)
834 return "";
835
836
837 if (m) {
838 member = btf_name_by_offset(show->btf, m->name_off);
839 show_member = strlen(member) > 0;
840 id = m->type;
841 }
842
843
844
845
846
847
848
849
850 t = btf_type_by_id(show->btf, id);
851 if (!t)
852 return "";
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876 for (i = 0; i < BTF_SHOW_MAX_ITER; i++) {
877
878 switch (BTF_INFO_KIND(t->info)) {
879 case BTF_KIND_TYPEDEF:
880 if (!name)
881 name = btf_name_by_offset(show->btf,
882 t->name_off);
883 kinds |= BTF_KIND_BIT(BTF_KIND_TYPEDEF);
884 id = t->type;
885 break;
886 case BTF_KIND_ARRAY:
887 kinds |= BTF_KIND_BIT(BTF_KIND_ARRAY);
888 parens = "[";
889 if (!t)
890 return "";
891 array = btf_type_array(t);
892 if (array_suffix > array_suffixes)
893 array_suffix -= 2;
894 id = array->type;
895 break;
896 case BTF_KIND_PTR:
897 kinds |= BTF_KIND_BIT(BTF_KIND_PTR);
898 if (ptr_suffix > ptr_suffixes)
899 ptr_suffix -= 1;
900 id = t->type;
901 break;
902 default:
903 id = 0;
904 break;
905 }
906 if (!id)
907 break;
908 t = btf_type_skip_qualifiers(show->btf, id);
909 }
910
911 if (i == BTF_SHOW_MAX_ITER)
912 return "";
913
914 if (!name)
915 name = btf_name_by_offset(show->btf, t->name_off);
916
917 switch (BTF_INFO_KIND(t->info)) {
918 case BTF_KIND_STRUCT:
919 case BTF_KIND_UNION:
920 prefix = BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT ?
921 "struct" : "union";
922
923 if (!(kinds & (BTF_KIND_BIT(BTF_KIND_ARRAY))))
924 parens = "{";
925 break;
926 case BTF_KIND_ENUM:
927 prefix = "enum";
928 break;
929 default:
930 break;
931 }
932
933
934 if (kinds & BTF_KIND_BIT(BTF_KIND_PTR))
935 parens = "";
936
937 if (kinds & BTF_KIND_BIT(BTF_KIND_TYPEDEF))
938 prefix = "";
939
940 if (!name)
941 name = "";
942
943
944 if (show->flags & BTF_SHOW_NONAME)
945 snprintf(show->state.name, sizeof(show->state.name), "%s",
946 parens);
947 else
948 snprintf(show->state.name, sizeof(show->state.name),
949 "%s%s%s(%s%s%s%s%s%s)%s",
950
951 show_member ? "." : "",
952 show_member ? member : "",
953 show_member ? " = " : "",
954
955 prefix,
956 strlen(prefix) > 0 && strlen(name) > 0 ? " " : "",
957
958 name,
959
960 strlen(ptr_suffix) > 0 ? " " : "", ptr_suffix,
961 array_suffix, parens);
962
963 return show->state.name;
964}
965
966static const char *__btf_show_indent(struct btf_show *show)
967{
968 const char *indents = " ";
969 const char *indent = &indents[strlen(indents)];
970
971 if ((indent - show->state.depth) >= indents)
972 return indent - show->state.depth;
973 return indents;
974}
975
976static const char *btf_show_indent(struct btf_show *show)
977{
978 return show->flags & BTF_SHOW_COMPACT ? "" : __btf_show_indent(show);
979}
980
981static const char *btf_show_newline(struct btf_show *show)
982{
983 return show->flags & BTF_SHOW_COMPACT ? "" : "\n";
984}
985
986static const char *btf_show_delim(struct btf_show *show)
987{
988 if (show->state.depth == 0)
989 return "";
990
991 if ((show->flags & BTF_SHOW_COMPACT) && show->state.type &&
992 BTF_INFO_KIND(show->state.type->info) == BTF_KIND_UNION)
993 return "|";
994
995 return ",";
996}
997
998__printf(2, 3) static void btf_show(struct btf_show *show, const char *fmt, ...)
999{
1000 va_list args;
1001
1002 if (!show->state.depth_check) {
1003 va_start(args, fmt);
1004 show->showfn(show, fmt, args);
1005 va_end(args);
1006 }
1007}
1008
1009
1010
1011
1012
1013
1014#define btf_show_type_value(show, fmt, value) \
1015 do { \
1016 if ((value) != 0 || (show->flags & BTF_SHOW_ZERO) || \
1017 show->state.depth == 0) { \
1018 btf_show(show, "%s%s" fmt "%s%s", \
1019 btf_show_indent(show), \
1020 btf_show_name(show), \
1021 value, btf_show_delim(show), \
1022 btf_show_newline(show)); \
1023 if (show->state.depth > show->state.depth_to_show) \
1024 show->state.depth_to_show = show->state.depth; \
1025 } \
1026 } while (0)
1027
1028#define btf_show_type_values(show, fmt, ...) \
1029 do { \
1030 btf_show(show, "%s%s" fmt "%s%s", btf_show_indent(show), \
1031 btf_show_name(show), \
1032 __VA_ARGS__, btf_show_delim(show), \
1033 btf_show_newline(show)); \
1034 if (show->state.depth > show->state.depth_to_show) \
1035 show->state.depth_to_show = show->state.depth; \
1036 } while (0)
1037
1038
1039static int btf_show_obj_size_left(struct btf_show *show, void *data)
1040{
1041 return show->obj.head + show->obj.size - data;
1042}
1043
1044
1045static bool btf_show_obj_is_safe(struct btf_show *show, void *data, int size)
1046{
1047 return data >= show->obj.data &&
1048 (data + size) < (show->obj.data + BTF_SHOW_OBJ_SAFE_SIZE);
1049}
1050
1051
1052
1053
1054
1055
1056
1057static void *__btf_show_obj_safe(struct btf_show *show, void *data, int size)
1058{
1059 if (btf_show_obj_is_safe(show, data, size))
1060 return show->obj.safe + (data - show->obj.data);
1061 return NULL;
1062}
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092static void *btf_show_obj_safe(struct btf_show *show,
1093 const struct btf_type *t,
1094 void *data)
1095{
1096 const struct btf_type *rt;
1097 int size_left, size;
1098 void *safe = NULL;
1099
1100 if (show->flags & BTF_SHOW_UNSAFE)
1101 return data;
1102
1103 rt = btf_resolve_size(show->btf, t, &size);
1104 if (IS_ERR(rt)) {
1105 show->state.status = PTR_ERR(rt);
1106 return NULL;
1107 }
1108
1109
1110
1111
1112
1113
1114 if (show->state.depth == 0) {
1115 show->obj.size = size;
1116 show->obj.head = data;
1117 } else {
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139 safe = __btf_show_obj_safe(show, data,
1140 min(size,
1141 BTF_SHOW_OBJ_BASE_TYPE_SIZE));
1142 }
1143
1144
1145
1146
1147
1148
1149 if (!safe) {
1150 size_left = btf_show_obj_size_left(show, data);
1151 if (size_left > BTF_SHOW_OBJ_SAFE_SIZE)
1152 size_left = BTF_SHOW_OBJ_SAFE_SIZE;
1153 show->state.status = copy_from_kernel_nofault(show->obj.safe,
1154 data, size_left);
1155 if (!show->state.status) {
1156 show->obj.data = data;
1157 safe = show->obj.safe;
1158 }
1159 }
1160
1161 return safe;
1162}
1163
1164
1165
1166
1167
1168static void *btf_show_start_type(struct btf_show *show,
1169 const struct btf_type *t,
1170 u32 type_id, void *data)
1171{
1172 show->state.type = t;
1173 show->state.type_id = type_id;
1174 show->state.name[0] = '\0';
1175
1176 return btf_show_obj_safe(show, t, data);
1177}
1178
1179static void btf_show_end_type(struct btf_show *show)
1180{
1181 show->state.type = NULL;
1182 show->state.type_id = 0;
1183 show->state.name[0] = '\0';
1184}
1185
1186static void *btf_show_start_aggr_type(struct btf_show *show,
1187 const struct btf_type *t,
1188 u32 type_id, void *data)
1189{
1190 void *safe_data = btf_show_start_type(show, t, type_id, data);
1191
1192 if (!safe_data)
1193 return safe_data;
1194
1195 btf_show(show, "%s%s%s", btf_show_indent(show),
1196 btf_show_name(show),
1197 btf_show_newline(show));
1198 show->state.depth++;
1199 return safe_data;
1200}
1201
1202static void btf_show_end_aggr_type(struct btf_show *show,
1203 const char *suffix)
1204{
1205 show->state.depth--;
1206 btf_show(show, "%s%s%s%s", btf_show_indent(show), suffix,
1207 btf_show_delim(show), btf_show_newline(show));
1208 btf_show_end_type(show);
1209}
1210
1211static void btf_show_start_member(struct btf_show *show,
1212 const struct btf_member *m)
1213{
1214 show->state.member = m;
1215}
1216
1217static void btf_show_start_array_member(struct btf_show *show)
1218{
1219 show->state.array_member = 1;
1220 btf_show_start_member(show, NULL);
1221}
1222
1223static void btf_show_end_member(struct btf_show *show)
1224{
1225 show->state.member = NULL;
1226}
1227
1228static void btf_show_end_array_member(struct btf_show *show)
1229{
1230 show->state.array_member = 0;
1231 btf_show_end_member(show);
1232}
1233
1234static void *btf_show_start_array_type(struct btf_show *show,
1235 const struct btf_type *t,
1236 u32 type_id,
1237 u16 array_encoding,
1238 void *data)
1239{
1240 show->state.array_encoding = array_encoding;
1241 show->state.array_terminated = 0;
1242 return btf_show_start_aggr_type(show, t, type_id, data);
1243}
1244
1245static void btf_show_end_array_type(struct btf_show *show)
1246{
1247 show->state.array_encoding = 0;
1248 show->state.array_terminated = 0;
1249 btf_show_end_aggr_type(show, "]");
1250}
1251
1252static void *btf_show_start_struct_type(struct btf_show *show,
1253 const struct btf_type *t,
1254 u32 type_id,
1255 void *data)
1256{
1257 return btf_show_start_aggr_type(show, t, type_id, data);
1258}
1259
1260static void btf_show_end_struct_type(struct btf_show *show)
1261{
1262 btf_show_end_aggr_type(show, "}");
1263}
1264
1265__printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log,
1266 const char *fmt, ...)
1267{
1268 va_list args;
1269
1270 va_start(args, fmt);
1271 bpf_verifier_vlog(log, fmt, args);
1272 va_end(args);
1273}
1274
1275__printf(2, 3) static void btf_verifier_log(struct btf_verifier_env *env,
1276 const char *fmt, ...)
1277{
1278 struct bpf_verifier_log *log = &env->log;
1279 va_list args;
1280
1281 if (!bpf_verifier_log_needed(log))
1282 return;
1283
1284 va_start(args, fmt);
1285 bpf_verifier_vlog(log, fmt, args);
1286 va_end(args);
1287}
1288
1289__printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env,
1290 const struct btf_type *t,
1291 bool log_details,
1292 const char *fmt, ...)
1293{
1294 struct bpf_verifier_log *log = &env->log;
1295 u8 kind = BTF_INFO_KIND(t->info);
1296 struct btf *btf = env->btf;
1297 va_list args;
1298
1299 if (!bpf_verifier_log_needed(log))
1300 return;
1301
1302
1303
1304
1305
1306 if (log->level == BPF_LOG_KERNEL && !fmt)
1307 return;
1308
1309 __btf_verifier_log(log, "[%u] %s %s%s",
1310 env->log_type_id,
1311 btf_kind_str[kind],
1312 __btf_name_by_offset(btf, t->name_off),
1313 log_details ? " " : "");
1314
1315 if (log_details)
1316 btf_type_ops(t)->log_details(env, t);
1317
1318 if (fmt && *fmt) {
1319 __btf_verifier_log(log, " ");
1320 va_start(args, fmt);
1321 bpf_verifier_vlog(log, fmt, args);
1322 va_end(args);
1323 }
1324
1325 __btf_verifier_log(log, "\n");
1326}
1327
1328#define btf_verifier_log_type(env, t, ...) \
1329 __btf_verifier_log_type((env), (t), true, __VA_ARGS__)
1330#define btf_verifier_log_basic(env, t, ...) \
1331 __btf_verifier_log_type((env), (t), false, __VA_ARGS__)
1332
1333__printf(4, 5)
1334static void btf_verifier_log_member(struct btf_verifier_env *env,
1335 const struct btf_type *struct_type,
1336 const struct btf_member *member,
1337 const char *fmt, ...)
1338{
1339 struct bpf_verifier_log *log = &env->log;
1340 struct btf *btf = env->btf;
1341 va_list args;
1342
1343 if (!bpf_verifier_log_needed(log))
1344 return;
1345
1346 if (log->level == BPF_LOG_KERNEL && !fmt)
1347 return;
1348
1349
1350
1351
1352
1353
1354 if (env->phase != CHECK_META)
1355 btf_verifier_log_type(env, struct_type, NULL);
1356
1357 if (btf_type_kflag(struct_type))
1358 __btf_verifier_log(log,
1359 "\t%s type_id=%u bitfield_size=%u bits_offset=%u",
1360 __btf_name_by_offset(btf, member->name_off),
1361 member->type,
1362 BTF_MEMBER_BITFIELD_SIZE(member->offset),
1363 BTF_MEMBER_BIT_OFFSET(member->offset));
1364 else
1365 __btf_verifier_log(log, "\t%s type_id=%u bits_offset=%u",
1366 __btf_name_by_offset(btf, member->name_off),
1367 member->type, member->offset);
1368
1369 if (fmt && *fmt) {
1370 __btf_verifier_log(log, " ");
1371 va_start(args, fmt);
1372 bpf_verifier_vlog(log, fmt, args);
1373 va_end(args);
1374 }
1375
1376 __btf_verifier_log(log, "\n");
1377}
1378
1379__printf(4, 5)
1380static void btf_verifier_log_vsi(struct btf_verifier_env *env,
1381 const struct btf_type *datasec_type,
1382 const struct btf_var_secinfo *vsi,
1383 const char *fmt, ...)
1384{
1385 struct bpf_verifier_log *log = &env->log;
1386 va_list args;
1387
1388 if (!bpf_verifier_log_needed(log))
1389 return;
1390 if (log->level == BPF_LOG_KERNEL && !fmt)
1391 return;
1392 if (env->phase != CHECK_META)
1393 btf_verifier_log_type(env, datasec_type, NULL);
1394
1395 __btf_verifier_log(log, "\t type_id=%u offset=%u size=%u",
1396 vsi->type, vsi->offset, vsi->size);
1397 if (fmt && *fmt) {
1398 __btf_verifier_log(log, " ");
1399 va_start(args, fmt);
1400 bpf_verifier_vlog(log, fmt, args);
1401 va_end(args);
1402 }
1403
1404 __btf_verifier_log(log, "\n");
1405}
1406
1407static void btf_verifier_log_hdr(struct btf_verifier_env *env,
1408 u32 btf_data_size)
1409{
1410 struct bpf_verifier_log *log = &env->log;
1411 const struct btf *btf = env->btf;
1412 const struct btf_header *hdr;
1413
1414 if (!bpf_verifier_log_needed(log))
1415 return;
1416
1417 if (log->level == BPF_LOG_KERNEL)
1418 return;
1419 hdr = &btf->hdr;
1420 __btf_verifier_log(log, "magic: 0x%x\n", hdr->magic);
1421 __btf_verifier_log(log, "version: %u\n", hdr->version);
1422 __btf_verifier_log(log, "flags: 0x%x\n", hdr->flags);
1423 __btf_verifier_log(log, "hdr_len: %u\n", hdr->hdr_len);
1424 __btf_verifier_log(log, "type_off: %u\n", hdr->type_off);
1425 __btf_verifier_log(log, "type_len: %u\n", hdr->type_len);
1426 __btf_verifier_log(log, "str_off: %u\n", hdr->str_off);
1427 __btf_verifier_log(log, "str_len: %u\n", hdr->str_len);
1428 __btf_verifier_log(log, "btf_total_size: %u\n", btf_data_size);
1429}
1430
1431static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t)
1432{
1433 struct btf *btf = env->btf;
1434
1435 if (btf->types_size == btf->nr_types) {
1436
1437
1438 struct btf_type **new_types;
1439 u32 expand_by, new_size;
1440
1441 if (btf->start_id + btf->types_size == BTF_MAX_TYPE) {
1442 btf_verifier_log(env, "Exceeded max num of types");
1443 return -E2BIG;
1444 }
1445
1446 expand_by = max_t(u32, btf->types_size >> 2, 16);
1447 new_size = min_t(u32, BTF_MAX_TYPE,
1448 btf->types_size + expand_by);
1449
1450 new_types = kvcalloc(new_size, sizeof(*new_types),
1451 GFP_KERNEL | __GFP_NOWARN);
1452 if (!new_types)
1453 return -ENOMEM;
1454
1455 if (btf->nr_types == 0) {
1456 if (!btf->base_btf) {
1457
1458 new_types[0] = &btf_void;
1459 btf->nr_types++;
1460 }
1461 } else {
1462 memcpy(new_types, btf->types,
1463 sizeof(*btf->types) * btf->nr_types);
1464 }
1465
1466 kvfree(btf->types);
1467 btf->types = new_types;
1468 btf->types_size = new_size;
1469 }
1470
1471 btf->types[btf->nr_types++] = t;
1472
1473 return 0;
1474}
1475
1476static int btf_alloc_id(struct btf *btf)
1477{
1478 int id;
1479
1480 idr_preload(GFP_KERNEL);
1481 spin_lock_bh(&btf_idr_lock);
1482 id = idr_alloc_cyclic(&btf_idr, btf, 1, INT_MAX, GFP_ATOMIC);
1483 if (id > 0)
1484 btf->id = id;
1485 spin_unlock_bh(&btf_idr_lock);
1486 idr_preload_end();
1487
1488 if (WARN_ON_ONCE(!id))
1489 return -ENOSPC;
1490
1491 return id > 0 ? 0 : id;
1492}
1493
1494static void btf_free_id(struct btf *btf)
1495{
1496 unsigned long flags;
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507 spin_lock_irqsave(&btf_idr_lock, flags);
1508 idr_remove(&btf_idr, btf->id);
1509 spin_unlock_irqrestore(&btf_idr_lock, flags);
1510}
1511
1512static void btf_free(struct btf *btf)
1513{
1514 kvfree(btf->types);
1515 kvfree(btf->resolved_sizes);
1516 kvfree(btf->resolved_ids);
1517 kvfree(btf->data);
1518 kfree(btf);
1519}
1520
1521static void btf_free_rcu(struct rcu_head *rcu)
1522{
1523 struct btf *btf = container_of(rcu, struct btf, rcu);
1524
1525 btf_free(btf);
1526}
1527
1528void btf_get(struct btf *btf)
1529{
1530 refcount_inc(&btf->refcnt);
1531}
1532
1533void btf_put(struct btf *btf)
1534{
1535 if (btf && refcount_dec_and_test(&btf->refcnt)) {
1536 btf_free_id(btf);
1537 call_rcu(&btf->rcu, btf_free_rcu);
1538 }
1539}
1540
1541static int env_resolve_init(struct btf_verifier_env *env)
1542{
1543 struct btf *btf = env->btf;
1544 u32 nr_types = btf->nr_types;
1545 u32 *resolved_sizes = NULL;
1546 u32 *resolved_ids = NULL;
1547 u8 *visit_states = NULL;
1548
1549 resolved_sizes = kvcalloc(nr_types, sizeof(*resolved_sizes),
1550 GFP_KERNEL | __GFP_NOWARN);
1551 if (!resolved_sizes)
1552 goto nomem;
1553
1554 resolved_ids = kvcalloc(nr_types, sizeof(*resolved_ids),
1555 GFP_KERNEL | __GFP_NOWARN);
1556 if (!resolved_ids)
1557 goto nomem;
1558
1559 visit_states = kvcalloc(nr_types, sizeof(*visit_states),
1560 GFP_KERNEL | __GFP_NOWARN);
1561 if (!visit_states)
1562 goto nomem;
1563
1564 btf->resolved_sizes = resolved_sizes;
1565 btf->resolved_ids = resolved_ids;
1566 env->visit_states = visit_states;
1567
1568 return 0;
1569
1570nomem:
1571 kvfree(resolved_sizes);
1572 kvfree(resolved_ids);
1573 kvfree(visit_states);
1574 return -ENOMEM;
1575}
1576
1577static void btf_verifier_env_free(struct btf_verifier_env *env)
1578{
1579 kvfree(env->visit_states);
1580 kfree(env);
1581}
1582
1583static bool env_type_is_resolve_sink(const struct btf_verifier_env *env,
1584 const struct btf_type *next_type)
1585{
1586 switch (env->resolve_mode) {
1587 case RESOLVE_TBD:
1588
1589 return !btf_type_needs_resolve(next_type);
1590 case RESOLVE_PTR:
1591
1592
1593
1594 return !btf_type_is_modifier(next_type) &&
1595 !btf_type_is_ptr(next_type);
1596 case RESOLVE_STRUCT_OR_ARRAY:
1597
1598
1599
1600 return !btf_type_is_modifier(next_type) &&
1601 !btf_type_is_array(next_type) &&
1602 !btf_type_is_struct(next_type);
1603 default:
1604 BUG();
1605 }
1606}
1607
1608static bool env_type_is_resolved(const struct btf_verifier_env *env,
1609 u32 type_id)
1610{
1611
1612 if (type_id < env->btf->start_id)
1613 return true;
1614
1615 return env->visit_states[type_id - env->btf->start_id] == RESOLVED;
1616}
1617
1618static int env_stack_push(struct btf_verifier_env *env,
1619 const struct btf_type *t, u32 type_id)
1620{
1621 const struct btf *btf = env->btf;
1622 struct resolve_vertex *v;
1623
1624 if (env->top_stack == MAX_RESOLVE_DEPTH)
1625 return -E2BIG;
1626
1627 if (type_id < btf->start_id
1628 || env->visit_states[type_id - btf->start_id] != NOT_VISITED)
1629 return -EEXIST;
1630
1631 env->visit_states[type_id - btf->start_id] = VISITED;
1632
1633 v = &env->stack[env->top_stack++];
1634 v->t = t;
1635 v->type_id = type_id;
1636 v->next_member = 0;
1637
1638 if (env->resolve_mode == RESOLVE_TBD) {
1639 if (btf_type_is_ptr(t))
1640 env->resolve_mode = RESOLVE_PTR;
1641 else if (btf_type_is_struct(t) || btf_type_is_array(t))
1642 env->resolve_mode = RESOLVE_STRUCT_OR_ARRAY;
1643 }
1644
1645 return 0;
1646}
1647
1648static void env_stack_set_next_member(struct btf_verifier_env *env,
1649 u16 next_member)
1650{
1651 env->stack[env->top_stack - 1].next_member = next_member;
1652}
1653
1654static void env_stack_pop_resolved(struct btf_verifier_env *env,
1655 u32 resolved_type_id,
1656 u32 resolved_size)
1657{
1658 u32 type_id = env->stack[--(env->top_stack)].type_id;
1659 struct btf *btf = env->btf;
1660
1661 type_id -= btf->start_id;
1662 btf->resolved_sizes[type_id] = resolved_size;
1663 btf->resolved_ids[type_id] = resolved_type_id;
1664 env->visit_states[type_id] = RESOLVED;
1665}
1666
1667static const struct resolve_vertex *env_stack_peak(struct btf_verifier_env *env)
1668{
1669 return env->top_stack ? &env->stack[env->top_stack - 1] : NULL;
1670}
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692static const struct btf_type *
1693__btf_resolve_size(const struct btf *btf, const struct btf_type *type,
1694 u32 *type_size, const struct btf_type **elem_type,
1695 u32 *elem_id, u32 *total_nelems, u32 *type_id)
1696{
1697 const struct btf_type *array_type = NULL;
1698 const struct btf_array *array = NULL;
1699 u32 i, size, nelems = 1, id = 0;
1700
1701 for (i = 0; i < MAX_RESOLVE_DEPTH; i++) {
1702 switch (BTF_INFO_KIND(type->info)) {
1703
1704 case BTF_KIND_INT:
1705 case BTF_KIND_STRUCT:
1706 case BTF_KIND_UNION:
1707 case BTF_KIND_ENUM:
1708 case BTF_KIND_FLOAT:
1709 size = type->size;
1710 goto resolved;
1711
1712 case BTF_KIND_PTR:
1713 size = sizeof(void *);
1714 goto resolved;
1715
1716
1717 case BTF_KIND_TYPEDEF:
1718 case BTF_KIND_VOLATILE:
1719 case BTF_KIND_CONST:
1720 case BTF_KIND_RESTRICT:
1721 id = type->type;
1722 type = btf_type_by_id(btf, type->type);
1723 break;
1724
1725 case BTF_KIND_ARRAY:
1726 if (!array_type)
1727 array_type = type;
1728 array = btf_type_array(type);
1729 if (nelems && array->nelems > U32_MAX / nelems)
1730 return ERR_PTR(-EINVAL);
1731 nelems *= array->nelems;
1732 type = btf_type_by_id(btf, array->type);
1733 break;
1734
1735
1736 default:
1737 return ERR_PTR(-EINVAL);
1738 }
1739 }
1740
1741 return ERR_PTR(-EINVAL);
1742
1743resolved:
1744 if (nelems && size > U32_MAX / nelems)
1745 return ERR_PTR(-EINVAL);
1746
1747 *type_size = nelems * size;
1748 if (total_nelems)
1749 *total_nelems = nelems;
1750 if (elem_type)
1751 *elem_type = type;
1752 if (elem_id)
1753 *elem_id = array ? array->type : 0;
1754 if (type_id && id)
1755 *type_id = id;
1756
1757 return array_type ? : type;
1758}
1759
1760const struct btf_type *
1761btf_resolve_size(const struct btf *btf, const struct btf_type *type,
1762 u32 *type_size)
1763{
1764 return __btf_resolve_size(btf, type, type_size, NULL, NULL, NULL, NULL);
1765}
1766
1767static u32 btf_resolved_type_id(const struct btf *btf, u32 type_id)
1768{
1769 while (type_id < btf->start_id)
1770 btf = btf->base_btf;
1771
1772 return btf->resolved_ids[type_id - btf->start_id];
1773}
1774
1775
1776static const struct btf_type *btf_type_id_resolve(const struct btf *btf,
1777 u32 *type_id)
1778{
1779 *type_id = btf_resolved_type_id(btf, *type_id);
1780 return btf_type_by_id(btf, *type_id);
1781}
1782
1783static u32 btf_resolved_type_size(const struct btf *btf, u32 type_id)
1784{
1785 while (type_id < btf->start_id)
1786 btf = btf->base_btf;
1787
1788 return btf->resolved_sizes[type_id - btf->start_id];
1789}
1790
1791const struct btf_type *btf_type_id_size(const struct btf *btf,
1792 u32 *type_id, u32 *ret_size)
1793{
1794 const struct btf_type *size_type;
1795 u32 size_type_id = *type_id;
1796 u32 size = 0;
1797
1798 size_type = btf_type_by_id(btf, size_type_id);
1799 if (btf_type_nosize_or_null(size_type))
1800 return NULL;
1801
1802 if (btf_type_has_size(size_type)) {
1803 size = size_type->size;
1804 } else if (btf_type_is_array(size_type)) {
1805 size = btf_resolved_type_size(btf, size_type_id);
1806 } else if (btf_type_is_ptr(size_type)) {
1807 size = sizeof(void *);
1808 } else {
1809 if (WARN_ON_ONCE(!btf_type_is_modifier(size_type) &&
1810 !btf_type_is_var(size_type)))
1811 return NULL;
1812
1813 size_type_id = btf_resolved_type_id(btf, size_type_id);
1814 size_type = btf_type_by_id(btf, size_type_id);
1815 if (btf_type_nosize_or_null(size_type))
1816 return NULL;
1817 else if (btf_type_has_size(size_type))
1818 size = size_type->size;
1819 else if (btf_type_is_array(size_type))
1820 size = btf_resolved_type_size(btf, size_type_id);
1821 else if (btf_type_is_ptr(size_type))
1822 size = sizeof(void *);
1823 else
1824 return NULL;
1825 }
1826
1827 *type_id = size_type_id;
1828 if (ret_size)
1829 *ret_size = size;
1830
1831 return size_type;
1832}
1833
1834static int btf_df_check_member(struct btf_verifier_env *env,
1835 const struct btf_type *struct_type,
1836 const struct btf_member *member,
1837 const struct btf_type *member_type)
1838{
1839 btf_verifier_log_basic(env, struct_type,
1840 "Unsupported check_member");
1841 return -EINVAL;
1842}
1843
1844static int btf_df_check_kflag_member(struct btf_verifier_env *env,
1845 const struct btf_type *struct_type,
1846 const struct btf_member *member,
1847 const struct btf_type *member_type)
1848{
1849 btf_verifier_log_basic(env, struct_type,
1850 "Unsupported check_kflag_member");
1851 return -EINVAL;
1852}
1853
1854
1855
1856
1857static int btf_generic_check_kflag_member(struct btf_verifier_env *env,
1858 const struct btf_type *struct_type,
1859 const struct btf_member *member,
1860 const struct btf_type *member_type)
1861{
1862 if (BTF_MEMBER_BITFIELD_SIZE(member->offset)) {
1863 btf_verifier_log_member(env, struct_type, member,
1864 "Invalid member bitfield_size");
1865 return -EINVAL;
1866 }
1867
1868
1869
1870
1871 return btf_type_ops(member_type)->check_member(env, struct_type,
1872 member,
1873 member_type);
1874}
1875
1876static int btf_df_resolve(struct btf_verifier_env *env,
1877 const struct resolve_vertex *v)
1878{
1879 btf_verifier_log_basic(env, v->t, "Unsupported resolve");
1880 return -EINVAL;
1881}
1882
1883static void btf_df_show(const struct btf *btf, const struct btf_type *t,
1884 u32 type_id, void *data, u8 bits_offsets,
1885 struct btf_show *show)
1886{
1887 btf_show(show, "<unsupported kind:%u>", BTF_INFO_KIND(t->info));
1888}
1889
1890static int btf_int_check_member(struct btf_verifier_env *env,
1891 const struct btf_type *struct_type,
1892 const struct btf_member *member,
1893 const struct btf_type *member_type)
1894{
1895 u32 int_data = btf_type_int(member_type);
1896 u32 struct_bits_off = member->offset;
1897 u32 struct_size = struct_type->size;
1898 u32 nr_copy_bits;
1899 u32 bytes_offset;
1900
1901 if (U32_MAX - struct_bits_off < BTF_INT_OFFSET(int_data)) {
1902 btf_verifier_log_member(env, struct_type, member,
1903 "bits_offset exceeds U32_MAX");
1904 return -EINVAL;
1905 }
1906
1907 struct_bits_off += BTF_INT_OFFSET(int_data);
1908 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1909 nr_copy_bits = BTF_INT_BITS(int_data) +
1910 BITS_PER_BYTE_MASKED(struct_bits_off);
1911
1912 if (nr_copy_bits > BITS_PER_U128) {
1913 btf_verifier_log_member(env, struct_type, member,
1914 "nr_copy_bits exceeds 128");
1915 return -EINVAL;
1916 }
1917
1918 if (struct_size < bytes_offset ||
1919 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
1920 btf_verifier_log_member(env, struct_type, member,
1921 "Member exceeds struct_size");
1922 return -EINVAL;
1923 }
1924
1925 return 0;
1926}
1927
1928static int btf_int_check_kflag_member(struct btf_verifier_env *env,
1929 const struct btf_type *struct_type,
1930 const struct btf_member *member,
1931 const struct btf_type *member_type)
1932{
1933 u32 struct_bits_off, nr_bits, nr_int_data_bits, bytes_offset;
1934 u32 int_data = btf_type_int(member_type);
1935 u32 struct_size = struct_type->size;
1936 u32 nr_copy_bits;
1937
1938
1939 if (!btf_type_int_is_regular(member_type)) {
1940 btf_verifier_log_member(env, struct_type, member,
1941 "Invalid member base type");
1942 return -EINVAL;
1943 }
1944
1945
1946 nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset);
1947 struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset);
1948 nr_int_data_bits = BTF_INT_BITS(int_data);
1949 if (!nr_bits) {
1950
1951
1952
1953 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
1954 btf_verifier_log_member(env, struct_type, member,
1955 "Invalid member offset");
1956 return -EINVAL;
1957 }
1958
1959 nr_bits = nr_int_data_bits;
1960 } else if (nr_bits > nr_int_data_bits) {
1961 btf_verifier_log_member(env, struct_type, member,
1962 "Invalid member bitfield_size");
1963 return -EINVAL;
1964 }
1965
1966 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
1967 nr_copy_bits = nr_bits + BITS_PER_BYTE_MASKED(struct_bits_off);
1968 if (nr_copy_bits > BITS_PER_U128) {
1969 btf_verifier_log_member(env, struct_type, member,
1970 "nr_copy_bits exceeds 128");
1971 return -EINVAL;
1972 }
1973
1974 if (struct_size < bytes_offset ||
1975 struct_size - bytes_offset < BITS_ROUNDUP_BYTES(nr_copy_bits)) {
1976 btf_verifier_log_member(env, struct_type, member,
1977 "Member exceeds struct_size");
1978 return -EINVAL;
1979 }
1980
1981 return 0;
1982}
1983
1984static s32 btf_int_check_meta(struct btf_verifier_env *env,
1985 const struct btf_type *t,
1986 u32 meta_left)
1987{
1988 u32 int_data, nr_bits, meta_needed = sizeof(int_data);
1989 u16 encoding;
1990
1991 if (meta_left < meta_needed) {
1992 btf_verifier_log_basic(env, t,
1993 "meta_left:%u meta_needed:%u",
1994 meta_left, meta_needed);
1995 return -EINVAL;
1996 }
1997
1998 if (btf_type_vlen(t)) {
1999 btf_verifier_log_type(env, t, "vlen != 0");
2000 return -EINVAL;
2001 }
2002
2003 if (btf_type_kflag(t)) {
2004 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2005 return -EINVAL;
2006 }
2007
2008 int_data = btf_type_int(t);
2009 if (int_data & ~BTF_INT_MASK) {
2010 btf_verifier_log_basic(env, t, "Invalid int_data:%x",
2011 int_data);
2012 return -EINVAL;
2013 }
2014
2015 nr_bits = BTF_INT_BITS(int_data) + BTF_INT_OFFSET(int_data);
2016
2017 if (nr_bits > BITS_PER_U128) {
2018 btf_verifier_log_type(env, t, "nr_bits exceeds %zu",
2019 BITS_PER_U128);
2020 return -EINVAL;
2021 }
2022
2023 if (BITS_ROUNDUP_BYTES(nr_bits) > t->size) {
2024 btf_verifier_log_type(env, t, "nr_bits exceeds type_size");
2025 return -EINVAL;
2026 }
2027
2028
2029
2030
2031
2032
2033
2034 encoding = BTF_INT_ENCODING(int_data);
2035 if (encoding &&
2036 encoding != BTF_INT_SIGNED &&
2037 encoding != BTF_INT_CHAR &&
2038 encoding != BTF_INT_BOOL) {
2039 btf_verifier_log_type(env, t, "Unsupported encoding");
2040 return -ENOTSUPP;
2041 }
2042
2043 btf_verifier_log_type(env, t, NULL);
2044
2045 return meta_needed;
2046}
2047
2048static void btf_int_log(struct btf_verifier_env *env,
2049 const struct btf_type *t)
2050{
2051 int int_data = btf_type_int(t);
2052
2053 btf_verifier_log(env,
2054 "size=%u bits_offset=%u nr_bits=%u encoding=%s",
2055 t->size, BTF_INT_OFFSET(int_data),
2056 BTF_INT_BITS(int_data),
2057 btf_int_encoding_str(BTF_INT_ENCODING(int_data)));
2058}
2059
2060static void btf_int128_print(struct btf_show *show, void *data)
2061{
2062
2063
2064
2065
2066
2067
2068
2069 u64 upper_num, lower_num;
2070
2071#ifdef __BIG_ENDIAN_BITFIELD
2072 upper_num = *(u64 *)data;
2073 lower_num = *(u64 *)(data + 8);
2074#else
2075 upper_num = *(u64 *)(data + 8);
2076 lower_num = *(u64 *)data;
2077#endif
2078 if (upper_num == 0)
2079 btf_show_type_value(show, "0x%llx", lower_num);
2080 else
2081 btf_show_type_values(show, "0x%llx%016llx", upper_num,
2082 lower_num);
2083}
2084
2085static void btf_int128_shift(u64 *print_num, u16 left_shift_bits,
2086 u16 right_shift_bits)
2087{
2088 u64 upper_num, lower_num;
2089
2090#ifdef __BIG_ENDIAN_BITFIELD
2091 upper_num = print_num[0];
2092 lower_num = print_num[1];
2093#else
2094 upper_num = print_num[1];
2095 lower_num = print_num[0];
2096#endif
2097
2098
2099 if (left_shift_bits >= 64) {
2100 upper_num = lower_num << (left_shift_bits - 64);
2101 lower_num = 0;
2102 } else {
2103 upper_num = (upper_num << left_shift_bits) |
2104 (lower_num >> (64 - left_shift_bits));
2105 lower_num = lower_num << left_shift_bits;
2106 }
2107
2108 if (right_shift_bits >= 64) {
2109 lower_num = upper_num >> (right_shift_bits - 64);
2110 upper_num = 0;
2111 } else {
2112 lower_num = (lower_num >> right_shift_bits) |
2113 (upper_num << (64 - right_shift_bits));
2114 upper_num = upper_num >> right_shift_bits;
2115 }
2116
2117#ifdef __BIG_ENDIAN_BITFIELD
2118 print_num[0] = upper_num;
2119 print_num[1] = lower_num;
2120#else
2121 print_num[0] = lower_num;
2122 print_num[1] = upper_num;
2123#endif
2124}
2125
2126static void btf_bitfield_show(void *data, u8 bits_offset,
2127 u8 nr_bits, struct btf_show *show)
2128{
2129 u16 left_shift_bits, right_shift_bits;
2130 u8 nr_copy_bytes;
2131 u8 nr_copy_bits;
2132 u64 print_num[2] = {};
2133
2134 nr_copy_bits = nr_bits + bits_offset;
2135 nr_copy_bytes = BITS_ROUNDUP_BYTES(nr_copy_bits);
2136
2137 memcpy(print_num, data, nr_copy_bytes);
2138
2139#ifdef __BIG_ENDIAN_BITFIELD
2140 left_shift_bits = bits_offset;
2141#else
2142 left_shift_bits = BITS_PER_U128 - nr_copy_bits;
2143#endif
2144 right_shift_bits = BITS_PER_U128 - nr_bits;
2145
2146 btf_int128_shift(print_num, left_shift_bits, right_shift_bits);
2147 btf_int128_print(show, print_num);
2148}
2149
2150
2151static void btf_int_bits_show(const struct btf *btf,
2152 const struct btf_type *t,
2153 void *data, u8 bits_offset,
2154 struct btf_show *show)
2155{
2156 u32 int_data = btf_type_int(t);
2157 u8 nr_bits = BTF_INT_BITS(int_data);
2158 u8 total_bits_offset;
2159
2160
2161
2162
2163
2164 total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data);
2165 data += BITS_ROUNDDOWN_BYTES(total_bits_offset);
2166 bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset);
2167 btf_bitfield_show(data, bits_offset, nr_bits, show);
2168}
2169
2170static void btf_int_show(const struct btf *btf, const struct btf_type *t,
2171 u32 type_id, void *data, u8 bits_offset,
2172 struct btf_show *show)
2173{
2174 u32 int_data = btf_type_int(t);
2175 u8 encoding = BTF_INT_ENCODING(int_data);
2176 bool sign = encoding & BTF_INT_SIGNED;
2177 u8 nr_bits = BTF_INT_BITS(int_data);
2178 void *safe_data;
2179
2180 safe_data = btf_show_start_type(show, t, type_id, data);
2181 if (!safe_data)
2182 return;
2183
2184 if (bits_offset || BTF_INT_OFFSET(int_data) ||
2185 BITS_PER_BYTE_MASKED(nr_bits)) {
2186 btf_int_bits_show(btf, t, safe_data, bits_offset, show);
2187 goto out;
2188 }
2189
2190 switch (nr_bits) {
2191 case 128:
2192 btf_int128_print(show, safe_data);
2193 break;
2194 case 64:
2195 if (sign)
2196 btf_show_type_value(show, "%lld", *(s64 *)safe_data);
2197 else
2198 btf_show_type_value(show, "%llu", *(u64 *)safe_data);
2199 break;
2200 case 32:
2201 if (sign)
2202 btf_show_type_value(show, "%d", *(s32 *)safe_data);
2203 else
2204 btf_show_type_value(show, "%u", *(u32 *)safe_data);
2205 break;
2206 case 16:
2207 if (sign)
2208 btf_show_type_value(show, "%d", *(s16 *)safe_data);
2209 else
2210 btf_show_type_value(show, "%u", *(u16 *)safe_data);
2211 break;
2212 case 8:
2213 if (show->state.array_encoding == BTF_INT_CHAR) {
2214
2215 if (show->state.array_terminated)
2216 break;
2217 if (*(char *)data == '\0') {
2218 show->state.array_terminated = 1;
2219 break;
2220 }
2221 if (isprint(*(char *)data)) {
2222 btf_show_type_value(show, "'%c'",
2223 *(char *)safe_data);
2224 break;
2225 }
2226 }
2227 if (sign)
2228 btf_show_type_value(show, "%d", *(s8 *)safe_data);
2229 else
2230 btf_show_type_value(show, "%u", *(u8 *)safe_data);
2231 break;
2232 default:
2233 btf_int_bits_show(btf, t, safe_data, bits_offset, show);
2234 break;
2235 }
2236out:
2237 btf_show_end_type(show);
2238}
2239
2240static const struct btf_kind_operations int_ops = {
2241 .check_meta = btf_int_check_meta,
2242 .resolve = btf_df_resolve,
2243 .check_member = btf_int_check_member,
2244 .check_kflag_member = btf_int_check_kflag_member,
2245 .log_details = btf_int_log,
2246 .show = btf_int_show,
2247};
2248
2249static int btf_modifier_check_member(struct btf_verifier_env *env,
2250 const struct btf_type *struct_type,
2251 const struct btf_member *member,
2252 const struct btf_type *member_type)
2253{
2254 const struct btf_type *resolved_type;
2255 u32 resolved_type_id = member->type;
2256 struct btf_member resolved_member;
2257 struct btf *btf = env->btf;
2258
2259 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
2260 if (!resolved_type) {
2261 btf_verifier_log_member(env, struct_type, member,
2262 "Invalid member");
2263 return -EINVAL;
2264 }
2265
2266 resolved_member = *member;
2267 resolved_member.type = resolved_type_id;
2268
2269 return btf_type_ops(resolved_type)->check_member(env, struct_type,
2270 &resolved_member,
2271 resolved_type);
2272}
2273
2274static int btf_modifier_check_kflag_member(struct btf_verifier_env *env,
2275 const struct btf_type *struct_type,
2276 const struct btf_member *member,
2277 const struct btf_type *member_type)
2278{
2279 const struct btf_type *resolved_type;
2280 u32 resolved_type_id = member->type;
2281 struct btf_member resolved_member;
2282 struct btf *btf = env->btf;
2283
2284 resolved_type = btf_type_id_size(btf, &resolved_type_id, NULL);
2285 if (!resolved_type) {
2286 btf_verifier_log_member(env, struct_type, member,
2287 "Invalid member");
2288 return -EINVAL;
2289 }
2290
2291 resolved_member = *member;
2292 resolved_member.type = resolved_type_id;
2293
2294 return btf_type_ops(resolved_type)->check_kflag_member(env, struct_type,
2295 &resolved_member,
2296 resolved_type);
2297}
2298
2299static int btf_ptr_check_member(struct btf_verifier_env *env,
2300 const struct btf_type *struct_type,
2301 const struct btf_member *member,
2302 const struct btf_type *member_type)
2303{
2304 u32 struct_size, struct_bits_off, bytes_offset;
2305
2306 struct_size = struct_type->size;
2307 struct_bits_off = member->offset;
2308 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
2309
2310 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2311 btf_verifier_log_member(env, struct_type, member,
2312 "Member is not byte aligned");
2313 return -EINVAL;
2314 }
2315
2316 if (struct_size - bytes_offset < sizeof(void *)) {
2317 btf_verifier_log_member(env, struct_type, member,
2318 "Member exceeds struct_size");
2319 return -EINVAL;
2320 }
2321
2322 return 0;
2323}
2324
2325static int btf_ref_type_check_meta(struct btf_verifier_env *env,
2326 const struct btf_type *t,
2327 u32 meta_left)
2328{
2329 if (btf_type_vlen(t)) {
2330 btf_verifier_log_type(env, t, "vlen != 0");
2331 return -EINVAL;
2332 }
2333
2334 if (btf_type_kflag(t)) {
2335 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2336 return -EINVAL;
2337 }
2338
2339 if (!BTF_TYPE_ID_VALID(t->type)) {
2340 btf_verifier_log_type(env, t, "Invalid type_id");
2341 return -EINVAL;
2342 }
2343
2344
2345
2346
2347 if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) {
2348 if (!t->name_off ||
2349 !btf_name_valid_identifier(env->btf, t->name_off)) {
2350 btf_verifier_log_type(env, t, "Invalid name");
2351 return -EINVAL;
2352 }
2353 } else {
2354 if (t->name_off) {
2355 btf_verifier_log_type(env, t, "Invalid name");
2356 return -EINVAL;
2357 }
2358 }
2359
2360 btf_verifier_log_type(env, t, NULL);
2361
2362 return 0;
2363}
2364
2365static int btf_modifier_resolve(struct btf_verifier_env *env,
2366 const struct resolve_vertex *v)
2367{
2368 const struct btf_type *t = v->t;
2369 const struct btf_type *next_type;
2370 u32 next_type_id = t->type;
2371 struct btf *btf = env->btf;
2372
2373 next_type = btf_type_by_id(btf, next_type_id);
2374 if (!next_type || btf_type_is_resolve_source_only(next_type)) {
2375 btf_verifier_log_type(env, v->t, "Invalid type_id");
2376 return -EINVAL;
2377 }
2378
2379 if (!env_type_is_resolve_sink(env, next_type) &&
2380 !env_type_is_resolved(env, next_type_id))
2381 return env_stack_push(env, next_type, next_type_id);
2382
2383
2384
2385
2386
2387
2388
2389 if (!btf_type_id_size(btf, &next_type_id, NULL)) {
2390 if (env_type_is_resolved(env, next_type_id))
2391 next_type = btf_type_id_resolve(btf, &next_type_id);
2392
2393
2394 if (!btf_type_is_void(next_type) &&
2395 !btf_type_is_fwd(next_type) &&
2396 !btf_type_is_func_proto(next_type)) {
2397 btf_verifier_log_type(env, v->t, "Invalid type_id");
2398 return -EINVAL;
2399 }
2400 }
2401
2402 env_stack_pop_resolved(env, next_type_id, 0);
2403
2404 return 0;
2405}
2406
2407static int btf_var_resolve(struct btf_verifier_env *env,
2408 const struct resolve_vertex *v)
2409{
2410 const struct btf_type *next_type;
2411 const struct btf_type *t = v->t;
2412 u32 next_type_id = t->type;
2413 struct btf *btf = env->btf;
2414
2415 next_type = btf_type_by_id(btf, next_type_id);
2416 if (!next_type || btf_type_is_resolve_source_only(next_type)) {
2417 btf_verifier_log_type(env, v->t, "Invalid type_id");
2418 return -EINVAL;
2419 }
2420
2421 if (!env_type_is_resolve_sink(env, next_type) &&
2422 !env_type_is_resolved(env, next_type_id))
2423 return env_stack_push(env, next_type, next_type_id);
2424
2425 if (btf_type_is_modifier(next_type)) {
2426 const struct btf_type *resolved_type;
2427 u32 resolved_type_id;
2428
2429 resolved_type_id = next_type_id;
2430 resolved_type = btf_type_id_resolve(btf, &resolved_type_id);
2431
2432 if (btf_type_is_ptr(resolved_type) &&
2433 !env_type_is_resolve_sink(env, resolved_type) &&
2434 !env_type_is_resolved(env, resolved_type_id))
2435 return env_stack_push(env, resolved_type,
2436 resolved_type_id);
2437 }
2438
2439
2440
2441
2442
2443 if (!btf_type_id_size(btf, &next_type_id, NULL)) {
2444 btf_verifier_log_type(env, v->t, "Invalid type_id");
2445 return -EINVAL;
2446 }
2447
2448 env_stack_pop_resolved(env, next_type_id, 0);
2449
2450 return 0;
2451}
2452
2453static int btf_ptr_resolve(struct btf_verifier_env *env,
2454 const struct resolve_vertex *v)
2455{
2456 const struct btf_type *next_type;
2457 const struct btf_type *t = v->t;
2458 u32 next_type_id = t->type;
2459 struct btf *btf = env->btf;
2460
2461 next_type = btf_type_by_id(btf, next_type_id);
2462 if (!next_type || btf_type_is_resolve_source_only(next_type)) {
2463 btf_verifier_log_type(env, v->t, "Invalid type_id");
2464 return -EINVAL;
2465 }
2466
2467 if (!env_type_is_resolve_sink(env, next_type) &&
2468 !env_type_is_resolved(env, next_type_id))
2469 return env_stack_push(env, next_type, next_type_id);
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479 if (btf_type_is_modifier(next_type)) {
2480 const struct btf_type *resolved_type;
2481 u32 resolved_type_id;
2482
2483 resolved_type_id = next_type_id;
2484 resolved_type = btf_type_id_resolve(btf, &resolved_type_id);
2485
2486 if (btf_type_is_ptr(resolved_type) &&
2487 !env_type_is_resolve_sink(env, resolved_type) &&
2488 !env_type_is_resolved(env, resolved_type_id))
2489 return env_stack_push(env, resolved_type,
2490 resolved_type_id);
2491 }
2492
2493 if (!btf_type_id_size(btf, &next_type_id, NULL)) {
2494 if (env_type_is_resolved(env, next_type_id))
2495 next_type = btf_type_id_resolve(btf, &next_type_id);
2496
2497 if (!btf_type_is_void(next_type) &&
2498 !btf_type_is_fwd(next_type) &&
2499 !btf_type_is_func_proto(next_type)) {
2500 btf_verifier_log_type(env, v->t, "Invalid type_id");
2501 return -EINVAL;
2502 }
2503 }
2504
2505 env_stack_pop_resolved(env, next_type_id, 0);
2506
2507 return 0;
2508}
2509
2510static void btf_modifier_show(const struct btf *btf,
2511 const struct btf_type *t,
2512 u32 type_id, void *data,
2513 u8 bits_offset, struct btf_show *show)
2514{
2515 if (btf->resolved_ids)
2516 t = btf_type_id_resolve(btf, &type_id);
2517 else
2518 t = btf_type_skip_modifiers(btf, type_id, NULL);
2519
2520 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
2521}
2522
2523static void btf_var_show(const struct btf *btf, const struct btf_type *t,
2524 u32 type_id, void *data, u8 bits_offset,
2525 struct btf_show *show)
2526{
2527 t = btf_type_id_resolve(btf, &type_id);
2528
2529 btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
2530}
2531
2532static void btf_ptr_show(const struct btf *btf, const struct btf_type *t,
2533 u32 type_id, void *data, u8 bits_offset,
2534 struct btf_show *show)
2535{
2536 void *safe_data;
2537
2538 safe_data = btf_show_start_type(show, t, type_id, data);
2539 if (!safe_data)
2540 return;
2541
2542
2543 if (show->flags & BTF_SHOW_PTR_RAW)
2544 btf_show_type_value(show, "0x%px", *(void **)safe_data);
2545 else
2546 btf_show_type_value(show, "0x%p", *(void **)safe_data);
2547 btf_show_end_type(show);
2548}
2549
2550static void btf_ref_type_log(struct btf_verifier_env *env,
2551 const struct btf_type *t)
2552{
2553 btf_verifier_log(env, "type_id=%u", t->type);
2554}
2555
2556static struct btf_kind_operations modifier_ops = {
2557 .check_meta = btf_ref_type_check_meta,
2558 .resolve = btf_modifier_resolve,
2559 .check_member = btf_modifier_check_member,
2560 .check_kflag_member = btf_modifier_check_kflag_member,
2561 .log_details = btf_ref_type_log,
2562 .show = btf_modifier_show,
2563};
2564
2565static struct btf_kind_operations ptr_ops = {
2566 .check_meta = btf_ref_type_check_meta,
2567 .resolve = btf_ptr_resolve,
2568 .check_member = btf_ptr_check_member,
2569 .check_kflag_member = btf_generic_check_kflag_member,
2570 .log_details = btf_ref_type_log,
2571 .show = btf_ptr_show,
2572};
2573
2574static s32 btf_fwd_check_meta(struct btf_verifier_env *env,
2575 const struct btf_type *t,
2576 u32 meta_left)
2577{
2578 if (btf_type_vlen(t)) {
2579 btf_verifier_log_type(env, t, "vlen != 0");
2580 return -EINVAL;
2581 }
2582
2583 if (t->type) {
2584 btf_verifier_log_type(env, t, "type != 0");
2585 return -EINVAL;
2586 }
2587
2588
2589 if (!t->name_off ||
2590 !btf_name_valid_identifier(env->btf, t->name_off)) {
2591 btf_verifier_log_type(env, t, "Invalid name");
2592 return -EINVAL;
2593 }
2594
2595 btf_verifier_log_type(env, t, NULL);
2596
2597 return 0;
2598}
2599
2600static void btf_fwd_type_log(struct btf_verifier_env *env,
2601 const struct btf_type *t)
2602{
2603 btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct");
2604}
2605
2606static struct btf_kind_operations fwd_ops = {
2607 .check_meta = btf_fwd_check_meta,
2608 .resolve = btf_df_resolve,
2609 .check_member = btf_df_check_member,
2610 .check_kflag_member = btf_df_check_kflag_member,
2611 .log_details = btf_fwd_type_log,
2612 .show = btf_df_show,
2613};
2614
2615static int btf_array_check_member(struct btf_verifier_env *env,
2616 const struct btf_type *struct_type,
2617 const struct btf_member *member,
2618 const struct btf_type *member_type)
2619{
2620 u32 struct_bits_off = member->offset;
2621 u32 struct_size, bytes_offset;
2622 u32 array_type_id, array_size;
2623 struct btf *btf = env->btf;
2624
2625 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2626 btf_verifier_log_member(env, struct_type, member,
2627 "Member is not byte aligned");
2628 return -EINVAL;
2629 }
2630
2631 array_type_id = member->type;
2632 btf_type_id_size(btf, &array_type_id, &array_size);
2633 struct_size = struct_type->size;
2634 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
2635 if (struct_size - bytes_offset < array_size) {
2636 btf_verifier_log_member(env, struct_type, member,
2637 "Member exceeds struct_size");
2638 return -EINVAL;
2639 }
2640
2641 return 0;
2642}
2643
2644static s32 btf_array_check_meta(struct btf_verifier_env *env,
2645 const struct btf_type *t,
2646 u32 meta_left)
2647{
2648 const struct btf_array *array = btf_type_array(t);
2649 u32 meta_needed = sizeof(*array);
2650
2651 if (meta_left < meta_needed) {
2652 btf_verifier_log_basic(env, t,
2653 "meta_left:%u meta_needed:%u",
2654 meta_left, meta_needed);
2655 return -EINVAL;
2656 }
2657
2658
2659 if (t->name_off) {
2660 btf_verifier_log_type(env, t, "Invalid name");
2661 return -EINVAL;
2662 }
2663
2664 if (btf_type_vlen(t)) {
2665 btf_verifier_log_type(env, t, "vlen != 0");
2666 return -EINVAL;
2667 }
2668
2669 if (btf_type_kflag(t)) {
2670 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2671 return -EINVAL;
2672 }
2673
2674 if (t->size) {
2675 btf_verifier_log_type(env, t, "size != 0");
2676 return -EINVAL;
2677 }
2678
2679
2680
2681
2682 if (!array->type || !BTF_TYPE_ID_VALID(array->type)) {
2683 btf_verifier_log_type(env, t, "Invalid elem");
2684 return -EINVAL;
2685 }
2686
2687 if (!array->index_type || !BTF_TYPE_ID_VALID(array->index_type)) {
2688 btf_verifier_log_type(env, t, "Invalid index");
2689 return -EINVAL;
2690 }
2691
2692 btf_verifier_log_type(env, t, NULL);
2693
2694 return meta_needed;
2695}
2696
2697static int btf_array_resolve(struct btf_verifier_env *env,
2698 const struct resolve_vertex *v)
2699{
2700 const struct btf_array *array = btf_type_array(v->t);
2701 const struct btf_type *elem_type, *index_type;
2702 u32 elem_type_id, index_type_id;
2703 struct btf *btf = env->btf;
2704 u32 elem_size;
2705
2706
2707 index_type_id = array->index_type;
2708 index_type = btf_type_by_id(btf, index_type_id);
2709 if (btf_type_nosize_or_null(index_type) ||
2710 btf_type_is_resolve_source_only(index_type)) {
2711 btf_verifier_log_type(env, v->t, "Invalid index");
2712 return -EINVAL;
2713 }
2714
2715 if (!env_type_is_resolve_sink(env, index_type) &&
2716 !env_type_is_resolved(env, index_type_id))
2717 return env_stack_push(env, index_type, index_type_id);
2718
2719 index_type = btf_type_id_size(btf, &index_type_id, NULL);
2720 if (!index_type || !btf_type_is_int(index_type) ||
2721 !btf_type_int_is_regular(index_type)) {
2722 btf_verifier_log_type(env, v->t, "Invalid index");
2723 return -EINVAL;
2724 }
2725
2726
2727 elem_type_id = array->type;
2728 elem_type = btf_type_by_id(btf, elem_type_id);
2729 if (btf_type_nosize_or_null(elem_type) ||
2730 btf_type_is_resolve_source_only(elem_type)) {
2731 btf_verifier_log_type(env, v->t,
2732 "Invalid elem");
2733 return -EINVAL;
2734 }
2735
2736 if (!env_type_is_resolve_sink(env, elem_type) &&
2737 !env_type_is_resolved(env, elem_type_id))
2738 return env_stack_push(env, elem_type, elem_type_id);
2739
2740 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
2741 if (!elem_type) {
2742 btf_verifier_log_type(env, v->t, "Invalid elem");
2743 return -EINVAL;
2744 }
2745
2746 if (btf_type_is_int(elem_type) && !btf_type_int_is_regular(elem_type)) {
2747 btf_verifier_log_type(env, v->t, "Invalid array of int");
2748 return -EINVAL;
2749 }
2750
2751 if (array->nelems && elem_size > U32_MAX / array->nelems) {
2752 btf_verifier_log_type(env, v->t,
2753 "Array size overflows U32_MAX");
2754 return -EINVAL;
2755 }
2756
2757 env_stack_pop_resolved(env, elem_type_id, elem_size * array->nelems);
2758
2759 return 0;
2760}
2761
2762static void btf_array_log(struct btf_verifier_env *env,
2763 const struct btf_type *t)
2764{
2765 const struct btf_array *array = btf_type_array(t);
2766
2767 btf_verifier_log(env, "type_id=%u index_type_id=%u nr_elems=%u",
2768 array->type, array->index_type, array->nelems);
2769}
2770
2771static void __btf_array_show(const struct btf *btf, const struct btf_type *t,
2772 u32 type_id, void *data, u8 bits_offset,
2773 struct btf_show *show)
2774{
2775 const struct btf_array *array = btf_type_array(t);
2776 const struct btf_kind_operations *elem_ops;
2777 const struct btf_type *elem_type;
2778 u32 i, elem_size = 0, elem_type_id;
2779 u16 encoding = 0;
2780
2781 elem_type_id = array->type;
2782 elem_type = btf_type_skip_modifiers(btf, elem_type_id, NULL);
2783 if (elem_type && btf_type_has_size(elem_type))
2784 elem_size = elem_type->size;
2785
2786 if (elem_type && btf_type_is_int(elem_type)) {
2787 u32 int_type = btf_type_int(elem_type);
2788
2789 encoding = BTF_INT_ENCODING(int_type);
2790
2791
2792
2793
2794
2795
2796 if (elem_size == 1)
2797 encoding = BTF_INT_CHAR;
2798 }
2799
2800 if (!btf_show_start_array_type(show, t, type_id, encoding, data))
2801 return;
2802
2803 if (!elem_type)
2804 goto out;
2805 elem_ops = btf_type_ops(elem_type);
2806
2807 for (i = 0; i < array->nelems; i++) {
2808
2809 btf_show_start_array_member(show);
2810
2811 elem_ops->show(btf, elem_type, elem_type_id, data,
2812 bits_offset, show);
2813 data += elem_size;
2814
2815 btf_show_end_array_member(show);
2816
2817 if (show->state.array_terminated)
2818 break;
2819 }
2820out:
2821 btf_show_end_array_type(show);
2822}
2823
2824static void btf_array_show(const struct btf *btf, const struct btf_type *t,
2825 u32 type_id, void *data, u8 bits_offset,
2826 struct btf_show *show)
2827{
2828 const struct btf_member *m = show->state.member;
2829
2830
2831
2832
2833
2834
2835 if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) {
2836 if (!show->state.depth_check) {
2837 show->state.depth_check = show->state.depth + 1;
2838 show->state.depth_to_show = 0;
2839 }
2840 __btf_array_show(btf, t, type_id, data, bits_offset, show);
2841 show->state.member = m;
2842
2843 if (show->state.depth_check != show->state.depth + 1)
2844 return;
2845 show->state.depth_check = 0;
2846
2847 if (show->state.depth_to_show <= show->state.depth)
2848 return;
2849
2850
2851
2852
2853 }
2854 __btf_array_show(btf, t, type_id, data, bits_offset, show);
2855}
2856
2857static struct btf_kind_operations array_ops = {
2858 .check_meta = btf_array_check_meta,
2859 .resolve = btf_array_resolve,
2860 .check_member = btf_array_check_member,
2861 .check_kflag_member = btf_generic_check_kflag_member,
2862 .log_details = btf_array_log,
2863 .show = btf_array_show,
2864};
2865
2866static int btf_struct_check_member(struct btf_verifier_env *env,
2867 const struct btf_type *struct_type,
2868 const struct btf_member *member,
2869 const struct btf_type *member_type)
2870{
2871 u32 struct_bits_off = member->offset;
2872 u32 struct_size, bytes_offset;
2873
2874 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
2875 btf_verifier_log_member(env, struct_type, member,
2876 "Member is not byte aligned");
2877 return -EINVAL;
2878 }
2879
2880 struct_size = struct_type->size;
2881 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
2882 if (struct_size - bytes_offset < member_type->size) {
2883 btf_verifier_log_member(env, struct_type, member,
2884 "Member exceeds struct_size");
2885 return -EINVAL;
2886 }
2887
2888 return 0;
2889}
2890
2891static s32 btf_struct_check_meta(struct btf_verifier_env *env,
2892 const struct btf_type *t,
2893 u32 meta_left)
2894{
2895 bool is_union = BTF_INFO_KIND(t->info) == BTF_KIND_UNION;
2896 const struct btf_member *member;
2897 u32 meta_needed, last_offset;
2898 struct btf *btf = env->btf;
2899 u32 struct_size = t->size;
2900 u32 offset;
2901 u16 i;
2902
2903 meta_needed = btf_type_vlen(t) * sizeof(*member);
2904 if (meta_left < meta_needed) {
2905 btf_verifier_log_basic(env, t,
2906 "meta_left:%u meta_needed:%u",
2907 meta_left, meta_needed);
2908 return -EINVAL;
2909 }
2910
2911
2912 if (t->name_off &&
2913 !btf_name_valid_identifier(env->btf, t->name_off)) {
2914 btf_verifier_log_type(env, t, "Invalid name");
2915 return -EINVAL;
2916 }
2917
2918 btf_verifier_log_type(env, t, NULL);
2919
2920 last_offset = 0;
2921 for_each_member(i, t, member) {
2922 if (!btf_name_offset_valid(btf, member->name_off)) {
2923 btf_verifier_log_member(env, t, member,
2924 "Invalid member name_offset:%u",
2925 member->name_off);
2926 return -EINVAL;
2927 }
2928
2929
2930 if (member->name_off &&
2931 !btf_name_valid_identifier(btf, member->name_off)) {
2932 btf_verifier_log_member(env, t, member, "Invalid name");
2933 return -EINVAL;
2934 }
2935
2936 if (!member->type || !BTF_TYPE_ID_VALID(member->type)) {
2937 btf_verifier_log_member(env, t, member,
2938 "Invalid type_id");
2939 return -EINVAL;
2940 }
2941
2942 offset = btf_member_bit_offset(t, member);
2943 if (is_union && offset) {
2944 btf_verifier_log_member(env, t, member,
2945 "Invalid member bits_offset");
2946 return -EINVAL;
2947 }
2948
2949
2950
2951
2952
2953 if (last_offset > offset) {
2954 btf_verifier_log_member(env, t, member,
2955 "Invalid member bits_offset");
2956 return -EINVAL;
2957 }
2958
2959 if (BITS_ROUNDUP_BYTES(offset) > struct_size) {
2960 btf_verifier_log_member(env, t, member,
2961 "Member bits_offset exceeds its struct size");
2962 return -EINVAL;
2963 }
2964
2965 btf_verifier_log_member(env, t, member, NULL);
2966 last_offset = offset;
2967 }
2968
2969 return meta_needed;
2970}
2971
2972static int btf_struct_resolve(struct btf_verifier_env *env,
2973 const struct resolve_vertex *v)
2974{
2975 const struct btf_member *member;
2976 int err;
2977 u16 i;
2978
2979
2980
2981
2982
2983 if (v->next_member) {
2984 const struct btf_type *last_member_type;
2985 const struct btf_member *last_member;
2986 u16 last_member_type_id;
2987
2988 last_member = btf_type_member(v->t) + v->next_member - 1;
2989 last_member_type_id = last_member->type;
2990 if (WARN_ON_ONCE(!env_type_is_resolved(env,
2991 last_member_type_id)))
2992 return -EINVAL;
2993
2994 last_member_type = btf_type_by_id(env->btf,
2995 last_member_type_id);
2996 if (btf_type_kflag(v->t))
2997 err = btf_type_ops(last_member_type)->check_kflag_member(env, v->t,
2998 last_member,
2999 last_member_type);
3000 else
3001 err = btf_type_ops(last_member_type)->check_member(env, v->t,
3002 last_member,
3003 last_member_type);
3004 if (err)
3005 return err;
3006 }
3007
3008 for_each_member_from(i, v->next_member, v->t, member) {
3009 u32 member_type_id = member->type;
3010 const struct btf_type *member_type = btf_type_by_id(env->btf,
3011 member_type_id);
3012
3013 if (btf_type_nosize_or_null(member_type) ||
3014 btf_type_is_resolve_source_only(member_type)) {
3015 btf_verifier_log_member(env, v->t, member,
3016 "Invalid member");
3017 return -EINVAL;
3018 }
3019
3020 if (!env_type_is_resolve_sink(env, member_type) &&
3021 !env_type_is_resolved(env, member_type_id)) {
3022 env_stack_set_next_member(env, i + 1);
3023 return env_stack_push(env, member_type, member_type_id);
3024 }
3025
3026 if (btf_type_kflag(v->t))
3027 err = btf_type_ops(member_type)->check_kflag_member(env, v->t,
3028 member,
3029 member_type);
3030 else
3031 err = btf_type_ops(member_type)->check_member(env, v->t,
3032 member,
3033 member_type);
3034 if (err)
3035 return err;
3036 }
3037
3038 env_stack_pop_resolved(env, 0, 0);
3039
3040 return 0;
3041}
3042
3043static void btf_struct_log(struct btf_verifier_env *env,
3044 const struct btf_type *t)
3045{
3046 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
3047}
3048
3049static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t,
3050 const char *name, int sz, int align)
3051{
3052 const struct btf_member *member;
3053 u32 i, off = -ENOENT;
3054
3055 for_each_member(i, t, member) {
3056 const struct btf_type *member_type = btf_type_by_id(btf,
3057 member->type);
3058 if (!__btf_type_is_struct(member_type))
3059 continue;
3060 if (member_type->size != sz)
3061 continue;
3062 if (strcmp(__btf_name_by_offset(btf, member_type->name_off), name))
3063 continue;
3064 if (off != -ENOENT)
3065
3066 return -E2BIG;
3067 off = btf_member_bit_offset(t, member);
3068 if (off % 8)
3069
3070 return -EINVAL;
3071 off /= 8;
3072 if (off % align)
3073 return -EINVAL;
3074 }
3075 return off;
3076}
3077
3078static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t,
3079 const char *name, int sz, int align)
3080{
3081 const struct btf_var_secinfo *vsi;
3082 u32 i, off = -ENOENT;
3083
3084 for_each_vsi(i, t, vsi) {
3085 const struct btf_type *var = btf_type_by_id(btf, vsi->type);
3086 const struct btf_type *var_type = btf_type_by_id(btf, var->type);
3087
3088 if (!__btf_type_is_struct(var_type))
3089 continue;
3090 if (var_type->size != sz)
3091 continue;
3092 if (vsi->size != sz)
3093 continue;
3094 if (strcmp(__btf_name_by_offset(btf, var_type->name_off), name))
3095 continue;
3096 if (off != -ENOENT)
3097
3098 return -E2BIG;
3099 off = vsi->offset;
3100 if (off % align)
3101 return -EINVAL;
3102 }
3103 return off;
3104}
3105
3106static int btf_find_field(const struct btf *btf, const struct btf_type *t,
3107 const char *name, int sz, int align)
3108{
3109
3110 if (__btf_type_is_struct(t))
3111 return btf_find_struct_field(btf, t, name, sz, align);
3112 else if (btf_type_is_datasec(t))
3113 return btf_find_datasec_var(btf, t, name, sz, align);
3114 return -EINVAL;
3115}
3116
3117
3118
3119
3120
3121int btf_find_spin_lock(const struct btf *btf, const struct btf_type *t)
3122{
3123 return btf_find_field(btf, t, "bpf_spin_lock",
3124 sizeof(struct bpf_spin_lock),
3125 __alignof__(struct bpf_spin_lock));
3126}
3127
3128int btf_find_timer(const struct btf *btf, const struct btf_type *t)
3129{
3130 return btf_find_field(btf, t, "bpf_timer",
3131 sizeof(struct bpf_timer),
3132 __alignof__(struct bpf_timer));
3133}
3134
3135static void __btf_struct_show(const struct btf *btf, const struct btf_type *t,
3136 u32 type_id, void *data, u8 bits_offset,
3137 struct btf_show *show)
3138{
3139 const struct btf_member *member;
3140 void *safe_data;
3141 u32 i;
3142
3143 safe_data = btf_show_start_struct_type(show, t, type_id, data);
3144 if (!safe_data)
3145 return;
3146
3147 for_each_member(i, t, member) {
3148 const struct btf_type *member_type = btf_type_by_id(btf,
3149 member->type);
3150 const struct btf_kind_operations *ops;
3151 u32 member_offset, bitfield_size;
3152 u32 bytes_offset;
3153 u8 bits8_offset;
3154
3155 btf_show_start_member(show, member);
3156
3157 member_offset = btf_member_bit_offset(t, member);
3158 bitfield_size = btf_member_bitfield_size(t, member);
3159 bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset);
3160 bits8_offset = BITS_PER_BYTE_MASKED(member_offset);
3161 if (bitfield_size) {
3162 safe_data = btf_show_start_type(show, member_type,
3163 member->type,
3164 data + bytes_offset);
3165 if (safe_data)
3166 btf_bitfield_show(safe_data,
3167 bits8_offset,
3168 bitfield_size, show);
3169 btf_show_end_type(show);
3170 } else {
3171 ops = btf_type_ops(member_type);
3172 ops->show(btf, member_type, member->type,
3173 data + bytes_offset, bits8_offset, show);
3174 }
3175
3176 btf_show_end_member(show);
3177 }
3178
3179 btf_show_end_struct_type(show);
3180}
3181
3182static void btf_struct_show(const struct btf *btf, const struct btf_type *t,
3183 u32 type_id, void *data, u8 bits_offset,
3184 struct btf_show *show)
3185{
3186 const struct btf_member *m = show->state.member;
3187
3188
3189
3190
3191
3192
3193 if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) {
3194 if (!show->state.depth_check) {
3195 show->state.depth_check = show->state.depth + 1;
3196 show->state.depth_to_show = 0;
3197 }
3198 __btf_struct_show(btf, t, type_id, data, bits_offset, show);
3199
3200 show->state.member = m;
3201 if (show->state.depth_check != show->state.depth + 1)
3202 return;
3203 show->state.depth_check = 0;
3204
3205 if (show->state.depth_to_show <= show->state.depth)
3206 return;
3207
3208
3209
3210
3211 }
3212
3213 __btf_struct_show(btf, t, type_id, data, bits_offset, show);
3214}
3215
3216static struct btf_kind_operations struct_ops = {
3217 .check_meta = btf_struct_check_meta,
3218 .resolve = btf_struct_resolve,
3219 .check_member = btf_struct_check_member,
3220 .check_kflag_member = btf_generic_check_kflag_member,
3221 .log_details = btf_struct_log,
3222 .show = btf_struct_show,
3223};
3224
3225static int btf_enum_check_member(struct btf_verifier_env *env,
3226 const struct btf_type *struct_type,
3227 const struct btf_member *member,
3228 const struct btf_type *member_type)
3229{
3230 u32 struct_bits_off = member->offset;
3231 u32 struct_size, bytes_offset;
3232
3233 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
3234 btf_verifier_log_member(env, struct_type, member,
3235 "Member is not byte aligned");
3236 return -EINVAL;
3237 }
3238
3239 struct_size = struct_type->size;
3240 bytes_offset = BITS_ROUNDDOWN_BYTES(struct_bits_off);
3241 if (struct_size - bytes_offset < member_type->size) {
3242 btf_verifier_log_member(env, struct_type, member,
3243 "Member exceeds struct_size");
3244 return -EINVAL;
3245 }
3246
3247 return 0;
3248}
3249
3250static int btf_enum_check_kflag_member(struct btf_verifier_env *env,
3251 const struct btf_type *struct_type,
3252 const struct btf_member *member,
3253 const struct btf_type *member_type)
3254{
3255 u32 struct_bits_off, nr_bits, bytes_end, struct_size;
3256 u32 int_bitsize = sizeof(int) * BITS_PER_BYTE;
3257
3258 struct_bits_off = BTF_MEMBER_BIT_OFFSET(member->offset);
3259 nr_bits = BTF_MEMBER_BITFIELD_SIZE(member->offset);
3260 if (!nr_bits) {
3261 if (BITS_PER_BYTE_MASKED(struct_bits_off)) {
3262 btf_verifier_log_member(env, struct_type, member,
3263 "Member is not byte aligned");
3264 return -EINVAL;
3265 }
3266
3267 nr_bits = int_bitsize;
3268 } else if (nr_bits > int_bitsize) {
3269 btf_verifier_log_member(env, struct_type, member,
3270 "Invalid member bitfield_size");
3271 return -EINVAL;
3272 }
3273
3274 struct_size = struct_type->size;
3275 bytes_end = BITS_ROUNDUP_BYTES(struct_bits_off + nr_bits);
3276 if (struct_size < bytes_end) {
3277 btf_verifier_log_member(env, struct_type, member,
3278 "Member exceeds struct_size");
3279 return -EINVAL;
3280 }
3281
3282 return 0;
3283}
3284
3285static s32 btf_enum_check_meta(struct btf_verifier_env *env,
3286 const struct btf_type *t,
3287 u32 meta_left)
3288{
3289 const struct btf_enum *enums = btf_type_enum(t);
3290 struct btf *btf = env->btf;
3291 u16 i, nr_enums;
3292 u32 meta_needed;
3293
3294 nr_enums = btf_type_vlen(t);
3295 meta_needed = nr_enums * sizeof(*enums);
3296
3297 if (meta_left < meta_needed) {
3298 btf_verifier_log_basic(env, t,
3299 "meta_left:%u meta_needed:%u",
3300 meta_left, meta_needed);
3301 return -EINVAL;
3302 }
3303
3304 if (btf_type_kflag(t)) {
3305 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3306 return -EINVAL;
3307 }
3308
3309 if (t->size > 8 || !is_power_of_2(t->size)) {
3310 btf_verifier_log_type(env, t, "Unexpected size");
3311 return -EINVAL;
3312 }
3313
3314
3315 if (t->name_off &&
3316 !btf_name_valid_identifier(env->btf, t->name_off)) {
3317 btf_verifier_log_type(env, t, "Invalid name");
3318 return -EINVAL;
3319 }
3320
3321 btf_verifier_log_type(env, t, NULL);
3322
3323 for (i = 0; i < nr_enums; i++) {
3324 if (!btf_name_offset_valid(btf, enums[i].name_off)) {
3325 btf_verifier_log(env, "\tInvalid name_offset:%u",
3326 enums[i].name_off);
3327 return -EINVAL;
3328 }
3329
3330
3331 if (!enums[i].name_off ||
3332 !btf_name_valid_identifier(btf, enums[i].name_off)) {
3333 btf_verifier_log_type(env, t, "Invalid name");
3334 return -EINVAL;
3335 }
3336
3337 if (env->log.level == BPF_LOG_KERNEL)
3338 continue;
3339 btf_verifier_log(env, "\t%s val=%d\n",
3340 __btf_name_by_offset(btf, enums[i].name_off),
3341 enums[i].val);
3342 }
3343
3344 return meta_needed;
3345}
3346
3347static void btf_enum_log(struct btf_verifier_env *env,
3348 const struct btf_type *t)
3349{
3350 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
3351}
3352
3353static void btf_enum_show(const struct btf *btf, const struct btf_type *t,
3354 u32 type_id, void *data, u8 bits_offset,
3355 struct btf_show *show)
3356{
3357 const struct btf_enum *enums = btf_type_enum(t);
3358 u32 i, nr_enums = btf_type_vlen(t);
3359 void *safe_data;
3360 int v;
3361
3362 safe_data = btf_show_start_type(show, t, type_id, data);
3363 if (!safe_data)
3364 return;
3365
3366 v = *(int *)safe_data;
3367
3368 for (i = 0; i < nr_enums; i++) {
3369 if (v != enums[i].val)
3370 continue;
3371
3372 btf_show_type_value(show, "%s",
3373 __btf_name_by_offset(btf,
3374 enums[i].name_off));
3375
3376 btf_show_end_type(show);
3377 return;
3378 }
3379
3380 btf_show_type_value(show, "%d", v);
3381 btf_show_end_type(show);
3382}
3383
3384static struct btf_kind_operations enum_ops = {
3385 .check_meta = btf_enum_check_meta,
3386 .resolve = btf_df_resolve,
3387 .check_member = btf_enum_check_member,
3388 .check_kflag_member = btf_enum_check_kflag_member,
3389 .log_details = btf_enum_log,
3390 .show = btf_enum_show,
3391};
3392
3393static s32 btf_func_proto_check_meta(struct btf_verifier_env *env,
3394 const struct btf_type *t,
3395 u32 meta_left)
3396{
3397 u32 meta_needed = btf_type_vlen(t) * sizeof(struct btf_param);
3398
3399 if (meta_left < meta_needed) {
3400 btf_verifier_log_basic(env, t,
3401 "meta_left:%u meta_needed:%u",
3402 meta_left, meta_needed);
3403 return -EINVAL;
3404 }
3405
3406 if (t->name_off) {
3407 btf_verifier_log_type(env, t, "Invalid name");
3408 return -EINVAL;
3409 }
3410
3411 if (btf_type_kflag(t)) {
3412 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3413 return -EINVAL;
3414 }
3415
3416 btf_verifier_log_type(env, t, NULL);
3417
3418 return meta_needed;
3419}
3420
3421static void btf_func_proto_log(struct btf_verifier_env *env,
3422 const struct btf_type *t)
3423{
3424 const struct btf_param *args = (const struct btf_param *)(t + 1);
3425 u16 nr_args = btf_type_vlen(t), i;
3426
3427 btf_verifier_log(env, "return=%u args=(", t->type);
3428 if (!nr_args) {
3429 btf_verifier_log(env, "void");
3430 goto done;
3431 }
3432
3433 if (nr_args == 1 && !args[0].type) {
3434
3435 btf_verifier_log(env, "vararg");
3436 goto done;
3437 }
3438
3439 btf_verifier_log(env, "%u %s", args[0].type,
3440 __btf_name_by_offset(env->btf,
3441 args[0].name_off));
3442 for (i = 1; i < nr_args - 1; i++)
3443 btf_verifier_log(env, ", %u %s", args[i].type,
3444 __btf_name_by_offset(env->btf,
3445 args[i].name_off));
3446
3447 if (nr_args > 1) {
3448 const struct btf_param *last_arg = &args[nr_args - 1];
3449
3450 if (last_arg->type)
3451 btf_verifier_log(env, ", %u %s", last_arg->type,
3452 __btf_name_by_offset(env->btf,
3453 last_arg->name_off));
3454 else
3455 btf_verifier_log(env, ", vararg");
3456 }
3457
3458done:
3459 btf_verifier_log(env, ")");
3460}
3461
3462static struct btf_kind_operations func_proto_ops = {
3463 .check_meta = btf_func_proto_check_meta,
3464 .resolve = btf_df_resolve,
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474 .check_member = btf_df_check_member,
3475 .check_kflag_member = btf_df_check_kflag_member,
3476 .log_details = btf_func_proto_log,
3477 .show = btf_df_show,
3478};
3479
3480static s32 btf_func_check_meta(struct btf_verifier_env *env,
3481 const struct btf_type *t,
3482 u32 meta_left)
3483{
3484 if (!t->name_off ||
3485 !btf_name_valid_identifier(env->btf, t->name_off)) {
3486 btf_verifier_log_type(env, t, "Invalid name");
3487 return -EINVAL;
3488 }
3489
3490 if (btf_type_vlen(t) > BTF_FUNC_GLOBAL) {
3491 btf_verifier_log_type(env, t, "Invalid func linkage");
3492 return -EINVAL;
3493 }
3494
3495 if (btf_type_kflag(t)) {
3496 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3497 return -EINVAL;
3498 }
3499
3500 btf_verifier_log_type(env, t, NULL);
3501
3502 return 0;
3503}
3504
3505static struct btf_kind_operations func_ops = {
3506 .check_meta = btf_func_check_meta,
3507 .resolve = btf_df_resolve,
3508 .check_member = btf_df_check_member,
3509 .check_kflag_member = btf_df_check_kflag_member,
3510 .log_details = btf_ref_type_log,
3511 .show = btf_df_show,
3512};
3513
3514static s32 btf_var_check_meta(struct btf_verifier_env *env,
3515 const struct btf_type *t,
3516 u32 meta_left)
3517{
3518 const struct btf_var *var;
3519 u32 meta_needed = sizeof(*var);
3520
3521 if (meta_left < meta_needed) {
3522 btf_verifier_log_basic(env, t,
3523 "meta_left:%u meta_needed:%u",
3524 meta_left, meta_needed);
3525 return -EINVAL;
3526 }
3527
3528 if (btf_type_vlen(t)) {
3529 btf_verifier_log_type(env, t, "vlen != 0");
3530 return -EINVAL;
3531 }
3532
3533 if (btf_type_kflag(t)) {
3534 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3535 return -EINVAL;
3536 }
3537
3538 if (!t->name_off ||
3539 !__btf_name_valid(env->btf, t->name_off, true)) {
3540 btf_verifier_log_type(env, t, "Invalid name");
3541 return -EINVAL;
3542 }
3543
3544
3545 if (!t->type || !BTF_TYPE_ID_VALID(t->type)) {
3546 btf_verifier_log_type(env, t, "Invalid type_id");
3547 return -EINVAL;
3548 }
3549
3550 var = btf_type_var(t);
3551 if (var->linkage != BTF_VAR_STATIC &&
3552 var->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
3553 btf_verifier_log_type(env, t, "Linkage not supported");
3554 return -EINVAL;
3555 }
3556
3557 btf_verifier_log_type(env, t, NULL);
3558
3559 return meta_needed;
3560}
3561
3562static void btf_var_log(struct btf_verifier_env *env, const struct btf_type *t)
3563{
3564 const struct btf_var *var = btf_type_var(t);
3565
3566 btf_verifier_log(env, "type_id=%u linkage=%u", t->type, var->linkage);
3567}
3568
3569static const struct btf_kind_operations var_ops = {
3570 .check_meta = btf_var_check_meta,
3571 .resolve = btf_var_resolve,
3572 .check_member = btf_df_check_member,
3573 .check_kflag_member = btf_df_check_kflag_member,
3574 .log_details = btf_var_log,
3575 .show = btf_var_show,
3576};
3577
3578static s32 btf_datasec_check_meta(struct btf_verifier_env *env,
3579 const struct btf_type *t,
3580 u32 meta_left)
3581{
3582 const struct btf_var_secinfo *vsi;
3583 u64 last_vsi_end_off = 0, sum = 0;
3584 u32 i, meta_needed;
3585
3586 meta_needed = btf_type_vlen(t) * sizeof(*vsi);
3587 if (meta_left < meta_needed) {
3588 btf_verifier_log_basic(env, t,
3589 "meta_left:%u meta_needed:%u",
3590 meta_left, meta_needed);
3591 return -EINVAL;
3592 }
3593
3594 if (!t->size) {
3595 btf_verifier_log_type(env, t, "size == 0");
3596 return -EINVAL;
3597 }
3598
3599 if (btf_type_kflag(t)) {
3600 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3601 return -EINVAL;
3602 }
3603
3604 if (!t->name_off ||
3605 !btf_name_valid_section(env->btf, t->name_off)) {
3606 btf_verifier_log_type(env, t, "Invalid name");
3607 return -EINVAL;
3608 }
3609
3610 btf_verifier_log_type(env, t, NULL);
3611
3612 for_each_vsi(i, t, vsi) {
3613
3614 if (!vsi->type || !BTF_TYPE_ID_VALID(vsi->type)) {
3615 btf_verifier_log_vsi(env, t, vsi,
3616 "Invalid type_id");
3617 return -EINVAL;
3618 }
3619
3620 if (vsi->offset < last_vsi_end_off || vsi->offset >= t->size) {
3621 btf_verifier_log_vsi(env, t, vsi,
3622 "Invalid offset");
3623 return -EINVAL;
3624 }
3625
3626 if (!vsi->size || vsi->size > t->size) {
3627 btf_verifier_log_vsi(env, t, vsi,
3628 "Invalid size");
3629 return -EINVAL;
3630 }
3631
3632 last_vsi_end_off = vsi->offset + vsi->size;
3633 if (last_vsi_end_off > t->size) {
3634 btf_verifier_log_vsi(env, t, vsi,
3635 "Invalid offset+size");
3636 return -EINVAL;
3637 }
3638
3639 btf_verifier_log_vsi(env, t, vsi, NULL);
3640 sum += vsi->size;
3641 }
3642
3643 if (t->size < sum) {
3644 btf_verifier_log_type(env, t, "Invalid btf_info size");
3645 return -EINVAL;
3646 }
3647
3648 return meta_needed;
3649}
3650
3651static int btf_datasec_resolve(struct btf_verifier_env *env,
3652 const struct resolve_vertex *v)
3653{
3654 const struct btf_var_secinfo *vsi;
3655 struct btf *btf = env->btf;
3656 u16 i;
3657
3658 for_each_vsi_from(i, v->next_member, v->t, vsi) {
3659 u32 var_type_id = vsi->type, type_id, type_size = 0;
3660 const struct btf_type *var_type = btf_type_by_id(env->btf,
3661 var_type_id);
3662 if (!var_type || !btf_type_is_var(var_type)) {
3663 btf_verifier_log_vsi(env, v->t, vsi,
3664 "Not a VAR kind member");
3665 return -EINVAL;
3666 }
3667
3668 if (!env_type_is_resolve_sink(env, var_type) &&
3669 !env_type_is_resolved(env, var_type_id)) {
3670 env_stack_set_next_member(env, i + 1);
3671 return env_stack_push(env, var_type, var_type_id);
3672 }
3673
3674 type_id = var_type->type;
3675 if (!btf_type_id_size(btf, &type_id, &type_size)) {
3676 btf_verifier_log_vsi(env, v->t, vsi, "Invalid type");
3677 return -EINVAL;
3678 }
3679
3680 if (vsi->size < type_size) {
3681 btf_verifier_log_vsi(env, v->t, vsi, "Invalid size");
3682 return -EINVAL;
3683 }
3684 }
3685
3686 env_stack_pop_resolved(env, 0, 0);
3687 return 0;
3688}
3689
3690static void btf_datasec_log(struct btf_verifier_env *env,
3691 const struct btf_type *t)
3692{
3693 btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
3694}
3695
3696static void btf_datasec_show(const struct btf *btf,
3697 const struct btf_type *t, u32 type_id,
3698 void *data, u8 bits_offset,
3699 struct btf_show *show)
3700{
3701 const struct btf_var_secinfo *vsi;
3702 const struct btf_type *var;
3703 u32 i;
3704
3705 if (!btf_show_start_type(show, t, type_id, data))
3706 return;
3707
3708 btf_show_type_value(show, "section (\"%s\") = {",
3709 __btf_name_by_offset(btf, t->name_off));
3710 for_each_vsi(i, t, vsi) {
3711 var = btf_type_by_id(btf, vsi->type);
3712 if (i)
3713 btf_show(show, ",");
3714 btf_type_ops(var)->show(btf, var, vsi->type,
3715 data + vsi->offset, bits_offset, show);
3716 }
3717 btf_show_end_type(show);
3718}
3719
3720static const struct btf_kind_operations datasec_ops = {
3721 .check_meta = btf_datasec_check_meta,
3722 .resolve = btf_datasec_resolve,
3723 .check_member = btf_df_check_member,
3724 .check_kflag_member = btf_df_check_kflag_member,
3725 .log_details = btf_datasec_log,
3726 .show = btf_datasec_show,
3727};
3728
3729static s32 btf_float_check_meta(struct btf_verifier_env *env,
3730 const struct btf_type *t,
3731 u32 meta_left)
3732{
3733 if (btf_type_vlen(t)) {
3734 btf_verifier_log_type(env, t, "vlen != 0");
3735 return -EINVAL;
3736 }
3737
3738 if (btf_type_kflag(t)) {
3739 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3740 return -EINVAL;
3741 }
3742
3743 if (t->size != 2 && t->size != 4 && t->size != 8 && t->size != 12 &&
3744 t->size != 16) {
3745 btf_verifier_log_type(env, t, "Invalid type_size");
3746 return -EINVAL;
3747 }
3748
3749 btf_verifier_log_type(env, t, NULL);
3750
3751 return 0;
3752}
3753
3754static int btf_float_check_member(struct btf_verifier_env *env,
3755 const struct btf_type *struct_type,
3756 const struct btf_member *member,
3757 const struct btf_type *member_type)
3758{
3759 u64 start_offset_bytes;
3760 u64 end_offset_bytes;
3761 u64 misalign_bits;
3762 u64 align_bytes;
3763 u64 align_bits;
3764
3765
3766
3767
3768
3769 align_bytes = min_t(u64, sizeof(void *), member_type->size);
3770 align_bits = align_bytes * BITS_PER_BYTE;
3771 div64_u64_rem(member->offset, align_bits, &misalign_bits);
3772 if (misalign_bits) {
3773 btf_verifier_log_member(env, struct_type, member,
3774 "Member is not properly aligned");
3775 return -EINVAL;
3776 }
3777
3778 start_offset_bytes = member->offset / BITS_PER_BYTE;
3779 end_offset_bytes = start_offset_bytes + member_type->size;
3780 if (end_offset_bytes > struct_type->size) {
3781 btf_verifier_log_member(env, struct_type, member,
3782 "Member exceeds struct_size");
3783 return -EINVAL;
3784 }
3785
3786 return 0;
3787}
3788
3789static void btf_float_log(struct btf_verifier_env *env,
3790 const struct btf_type *t)
3791{
3792 btf_verifier_log(env, "size=%u", t->size);
3793}
3794
3795static const struct btf_kind_operations float_ops = {
3796 .check_meta = btf_float_check_meta,
3797 .resolve = btf_df_resolve,
3798 .check_member = btf_float_check_member,
3799 .check_kflag_member = btf_generic_check_kflag_member,
3800 .log_details = btf_float_log,
3801 .show = btf_df_show,
3802};
3803
3804static int btf_func_proto_check(struct btf_verifier_env *env,
3805 const struct btf_type *t)
3806{
3807 const struct btf_type *ret_type;
3808 const struct btf_param *args;
3809 const struct btf *btf;
3810 u16 nr_args, i;
3811 int err;
3812
3813 btf = env->btf;
3814 args = (const struct btf_param *)(t + 1);
3815 nr_args = btf_type_vlen(t);
3816
3817
3818 if (t->type) {
3819 u32 ret_type_id = t->type;
3820
3821 ret_type = btf_type_by_id(btf, ret_type_id);
3822 if (!ret_type) {
3823 btf_verifier_log_type(env, t, "Invalid return type");
3824 return -EINVAL;
3825 }
3826
3827 if (btf_type_needs_resolve(ret_type) &&
3828 !env_type_is_resolved(env, ret_type_id)) {
3829 err = btf_resolve(env, ret_type, ret_type_id);
3830 if (err)
3831 return err;
3832 }
3833
3834
3835 if (!btf_type_id_size(btf, &ret_type_id, NULL)) {
3836 btf_verifier_log_type(env, t, "Invalid return type");
3837 return -EINVAL;
3838 }
3839 }
3840
3841 if (!nr_args)
3842 return 0;
3843
3844
3845 if (!args[nr_args - 1].type) {
3846 if (args[nr_args - 1].name_off) {
3847 btf_verifier_log_type(env, t, "Invalid arg#%u",
3848 nr_args);
3849 return -EINVAL;
3850 }
3851 nr_args--;
3852 }
3853
3854 err = 0;
3855 for (i = 0; i < nr_args; i++) {
3856 const struct btf_type *arg_type;
3857 u32 arg_type_id;
3858
3859 arg_type_id = args[i].type;
3860 arg_type = btf_type_by_id(btf, arg_type_id);
3861 if (!arg_type) {
3862 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
3863 err = -EINVAL;
3864 break;
3865 }
3866
3867 if (args[i].name_off &&
3868 (!btf_name_offset_valid(btf, args[i].name_off) ||
3869 !btf_name_valid_identifier(btf, args[i].name_off))) {
3870 btf_verifier_log_type(env, t,
3871 "Invalid arg#%u", i + 1);
3872 err = -EINVAL;
3873 break;
3874 }
3875
3876 if (btf_type_needs_resolve(arg_type) &&
3877 !env_type_is_resolved(env, arg_type_id)) {
3878 err = btf_resolve(env, arg_type, arg_type_id);
3879 if (err)
3880 break;
3881 }
3882
3883 if (!btf_type_id_size(btf, &arg_type_id, NULL)) {
3884 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
3885 err = -EINVAL;
3886 break;
3887 }
3888 }
3889
3890 return err;
3891}
3892
3893static int btf_func_check(struct btf_verifier_env *env,
3894 const struct btf_type *t)
3895{
3896 const struct btf_type *proto_type;
3897 const struct btf_param *args;
3898 const struct btf *btf;
3899 u16 nr_args, i;
3900
3901 btf = env->btf;
3902 proto_type = btf_type_by_id(btf, t->type);
3903
3904 if (!proto_type || !btf_type_is_func_proto(proto_type)) {
3905 btf_verifier_log_type(env, t, "Invalid type_id");
3906 return -EINVAL;
3907 }
3908
3909 args = (const struct btf_param *)(proto_type + 1);
3910 nr_args = btf_type_vlen(proto_type);
3911 for (i = 0; i < nr_args; i++) {
3912 if (!args[i].name_off && args[i].type) {
3913 btf_verifier_log_type(env, t, "Invalid arg#%u", i + 1);
3914 return -EINVAL;
3915 }
3916 }
3917
3918 return 0;
3919}
3920
3921static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = {
3922 [BTF_KIND_INT] = &int_ops,
3923 [BTF_KIND_PTR] = &ptr_ops,
3924 [BTF_KIND_ARRAY] = &array_ops,
3925 [BTF_KIND_STRUCT] = &struct_ops,
3926 [BTF_KIND_UNION] = &struct_ops,
3927 [BTF_KIND_ENUM] = &enum_ops,
3928 [BTF_KIND_FWD] = &fwd_ops,
3929 [BTF_KIND_TYPEDEF] = &modifier_ops,
3930 [BTF_KIND_VOLATILE] = &modifier_ops,
3931 [BTF_KIND_CONST] = &modifier_ops,
3932 [BTF_KIND_RESTRICT] = &modifier_ops,
3933 [BTF_KIND_FUNC] = &func_ops,
3934 [BTF_KIND_FUNC_PROTO] = &func_proto_ops,
3935 [BTF_KIND_VAR] = &var_ops,
3936 [BTF_KIND_DATASEC] = &datasec_ops,
3937 [BTF_KIND_FLOAT] = &float_ops,
3938};
3939
3940static s32 btf_check_meta(struct btf_verifier_env *env,
3941 const struct btf_type *t,
3942 u32 meta_left)
3943{
3944 u32 saved_meta_left = meta_left;
3945 s32 var_meta_size;
3946
3947 if (meta_left < sizeof(*t)) {
3948 btf_verifier_log(env, "[%u] meta_left:%u meta_needed:%zu",
3949 env->log_type_id, meta_left, sizeof(*t));
3950 return -EINVAL;
3951 }
3952 meta_left -= sizeof(*t);
3953
3954 if (t->info & ~BTF_INFO_MASK) {
3955 btf_verifier_log(env, "[%u] Invalid btf_info:%x",
3956 env->log_type_id, t->info);
3957 return -EINVAL;
3958 }
3959
3960 if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX ||
3961 BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) {
3962 btf_verifier_log(env, "[%u] Invalid kind:%u",
3963 env->log_type_id, BTF_INFO_KIND(t->info));
3964 return -EINVAL;
3965 }
3966
3967 if (!btf_name_offset_valid(env->btf, t->name_off)) {
3968 btf_verifier_log(env, "[%u] Invalid name_offset:%u",
3969 env->log_type_id, t->name_off);
3970 return -EINVAL;
3971 }
3972
3973 var_meta_size = btf_type_ops(t)->check_meta(env, t, meta_left);
3974 if (var_meta_size < 0)
3975 return var_meta_size;
3976
3977 meta_left -= var_meta_size;
3978
3979 return saved_meta_left - meta_left;
3980}
3981
3982static int btf_check_all_metas(struct btf_verifier_env *env)
3983{
3984 struct btf *btf = env->btf;
3985 struct btf_header *hdr;
3986 void *cur, *end;
3987
3988 hdr = &btf->hdr;
3989 cur = btf->nohdr_data + hdr->type_off;
3990 end = cur + hdr->type_len;
3991
3992 env->log_type_id = btf->base_btf ? btf->start_id : 1;
3993 while (cur < end) {
3994 struct btf_type *t = cur;
3995 s32 meta_size;
3996
3997 meta_size = btf_check_meta(env, t, end - cur);
3998 if (meta_size < 0)
3999 return meta_size;
4000
4001 btf_add_type(env, t);
4002 cur += meta_size;
4003 env->log_type_id++;
4004 }
4005
4006 return 0;
4007}
4008
4009static bool btf_resolve_valid(struct btf_verifier_env *env,
4010 const struct btf_type *t,
4011 u32 type_id)
4012{
4013 struct btf *btf = env->btf;
4014
4015 if (!env_type_is_resolved(env, type_id))
4016 return false;
4017
4018 if (btf_type_is_struct(t) || btf_type_is_datasec(t))
4019 return !btf_resolved_type_id(btf, type_id) &&
4020 !btf_resolved_type_size(btf, type_id);
4021
4022 if (btf_type_is_modifier(t) || btf_type_is_ptr(t) ||
4023 btf_type_is_var(t)) {
4024 t = btf_type_id_resolve(btf, &type_id);
4025 return t &&
4026 !btf_type_is_modifier(t) &&
4027 !btf_type_is_var(t) &&
4028 !btf_type_is_datasec(t);
4029 }
4030
4031 if (btf_type_is_array(t)) {
4032 const struct btf_array *array = btf_type_array(t);
4033 const struct btf_type *elem_type;
4034 u32 elem_type_id = array->type;
4035 u32 elem_size;
4036
4037 elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
4038 return elem_type && !btf_type_is_modifier(elem_type) &&
4039 (array->nelems * elem_size ==
4040 btf_resolved_type_size(btf, type_id));
4041 }
4042
4043 return false;
4044}
4045
4046static int btf_resolve(struct btf_verifier_env *env,
4047 const struct btf_type *t, u32 type_id)
4048{
4049 u32 save_log_type_id = env->log_type_id;
4050 const struct resolve_vertex *v;
4051 int err = 0;
4052
4053 env->resolve_mode = RESOLVE_TBD;
4054 env_stack_push(env, t, type_id);
4055 while (!err && (v = env_stack_peak(env))) {
4056 env->log_type_id = v->type_id;
4057 err = btf_type_ops(v->t)->resolve(env, v);
4058 }
4059
4060 env->log_type_id = type_id;
4061 if (err == -E2BIG) {
4062 btf_verifier_log_type(env, t,
4063 "Exceeded max resolving depth:%u",
4064 MAX_RESOLVE_DEPTH);
4065 } else if (err == -EEXIST) {
4066 btf_verifier_log_type(env, t, "Loop detected");
4067 }
4068
4069
4070 if (!err && !btf_resolve_valid(env, t, type_id)) {
4071 btf_verifier_log_type(env, t, "Invalid resolve state");
4072 err = -EINVAL;
4073 }
4074
4075 env->log_type_id = save_log_type_id;
4076 return err;
4077}
4078
4079static int btf_check_all_types(struct btf_verifier_env *env)
4080{
4081 struct btf *btf = env->btf;
4082 const struct btf_type *t;
4083 u32 type_id, i;
4084 int err;
4085
4086 err = env_resolve_init(env);
4087 if (err)
4088 return err;
4089
4090 env->phase++;
4091 for (i = btf->base_btf ? 0 : 1; i < btf->nr_types; i++) {
4092 type_id = btf->start_id + i;
4093 t = btf_type_by_id(btf, type_id);
4094
4095 env->log_type_id = type_id;
4096 if (btf_type_needs_resolve(t) &&
4097 !env_type_is_resolved(env, type_id)) {
4098 err = btf_resolve(env, t, type_id);
4099 if (err)
4100 return err;
4101 }
4102
4103 if (btf_type_is_func_proto(t)) {
4104 err = btf_func_proto_check(env, t);
4105 if (err)
4106 return err;
4107 }
4108
4109 if (btf_type_is_func(t)) {
4110 err = btf_func_check(env, t);
4111 if (err)
4112 return err;
4113 }
4114 }
4115
4116 return 0;
4117}
4118
4119static int btf_parse_type_sec(struct btf_verifier_env *env)
4120{
4121 const struct btf_header *hdr = &env->btf->hdr;
4122 int err;
4123
4124
4125 if (hdr->type_off & (sizeof(u32) - 1)) {
4126 btf_verifier_log(env, "Unaligned type_off");
4127 return -EINVAL;
4128 }
4129
4130 if (!env->btf->base_btf && !hdr->type_len) {
4131 btf_verifier_log(env, "No type found");
4132 return -EINVAL;
4133 }
4134
4135 err = btf_check_all_metas(env);
4136 if (err)
4137 return err;
4138
4139 return btf_check_all_types(env);
4140}
4141
4142static int btf_parse_str_sec(struct btf_verifier_env *env)
4143{
4144 const struct btf_header *hdr;
4145 struct btf *btf = env->btf;
4146 const char *start, *end;
4147
4148 hdr = &btf->hdr;
4149 start = btf->nohdr_data + hdr->str_off;
4150 end = start + hdr->str_len;
4151
4152 if (end != btf->data + btf->data_size) {
4153 btf_verifier_log(env, "String section is not at the end");
4154 return -EINVAL;
4155 }
4156
4157 btf->strings = start;
4158
4159 if (btf->base_btf && !hdr->str_len)
4160 return 0;
4161 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET || end[-1]) {
4162 btf_verifier_log(env, "Invalid string section");
4163 return -EINVAL;
4164 }
4165 if (!btf->base_btf && start[0]) {
4166 btf_verifier_log(env, "Invalid string section");
4167 return -EINVAL;
4168 }
4169
4170 return 0;
4171}
4172
4173static const size_t btf_sec_info_offset[] = {
4174 offsetof(struct btf_header, type_off),
4175 offsetof(struct btf_header, str_off),
4176};
4177
4178static int btf_sec_info_cmp(const void *a, const void *b)
4179{
4180 const struct btf_sec_info *x = a;
4181 const struct btf_sec_info *y = b;
4182
4183 return (int)(x->off - y->off) ? : (int)(x->len - y->len);
4184}
4185
4186static int btf_check_sec_info(struct btf_verifier_env *env,
4187 u32 btf_data_size)
4188{
4189 struct btf_sec_info secs[ARRAY_SIZE(btf_sec_info_offset)];
4190 u32 total, expected_total, i;
4191 const struct btf_header *hdr;
4192 const struct btf *btf;
4193
4194 btf = env->btf;
4195 hdr = &btf->hdr;
4196
4197
4198 for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++)
4199 secs[i] = *(struct btf_sec_info *)((void *)hdr +
4200 btf_sec_info_offset[i]);
4201
4202 sort(secs, ARRAY_SIZE(btf_sec_info_offset),
4203 sizeof(struct btf_sec_info), btf_sec_info_cmp, NULL);
4204
4205
4206 total = 0;
4207 expected_total = btf_data_size - hdr->hdr_len;
4208 for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++) {
4209 if (expected_total < secs[i].off) {
4210 btf_verifier_log(env, "Invalid section offset");
4211 return -EINVAL;
4212 }
4213 if (total < secs[i].off) {
4214
4215 btf_verifier_log(env, "Unsupported section found");
4216 return -EINVAL;
4217 }
4218 if (total > secs[i].off) {
4219 btf_verifier_log(env, "Section overlap found");
4220 return -EINVAL;
4221 }
4222 if (expected_total - total < secs[i].len) {
4223 btf_verifier_log(env,
4224 "Total section length too long");
4225 return -EINVAL;
4226 }
4227 total += secs[i].len;
4228 }
4229
4230
4231 if (expected_total != total) {
4232 btf_verifier_log(env, "Unsupported section found");
4233 return -EINVAL;
4234 }
4235
4236 return 0;
4237}
4238
4239static int btf_parse_hdr(struct btf_verifier_env *env)
4240{
4241 u32 hdr_len, hdr_copy, btf_data_size;
4242 const struct btf_header *hdr;
4243 struct btf *btf;
4244 int err;
4245
4246 btf = env->btf;
4247 btf_data_size = btf->data_size;
4248
4249 if (btf_data_size <
4250 offsetof(struct btf_header, hdr_len) + sizeof(hdr->hdr_len)) {
4251 btf_verifier_log(env, "hdr_len not found");
4252 return -EINVAL;
4253 }
4254
4255 hdr = btf->data;
4256 hdr_len = hdr->hdr_len;
4257 if (btf_data_size < hdr_len) {
4258 btf_verifier_log(env, "btf_header not found");
4259 return -EINVAL;
4260 }
4261
4262
4263 if (hdr_len > sizeof(btf->hdr)) {
4264 u8 *expected_zero = btf->data + sizeof(btf->hdr);
4265 u8 *end = btf->data + hdr_len;
4266
4267 for (; expected_zero < end; expected_zero++) {
4268 if (*expected_zero) {
4269 btf_verifier_log(env, "Unsupported btf_header");
4270 return -E2BIG;
4271 }
4272 }
4273 }
4274
4275 hdr_copy = min_t(u32, hdr_len, sizeof(btf->hdr));
4276 memcpy(&btf->hdr, btf->data, hdr_copy);
4277
4278 hdr = &btf->hdr;
4279
4280 btf_verifier_log_hdr(env, btf_data_size);
4281
4282 if (hdr->magic != BTF_MAGIC) {
4283 btf_verifier_log(env, "Invalid magic");
4284 return -EINVAL;
4285 }
4286
4287 if (hdr->version != BTF_VERSION) {
4288 btf_verifier_log(env, "Unsupported version");
4289 return -ENOTSUPP;
4290 }
4291
4292 if (hdr->flags) {
4293 btf_verifier_log(env, "Unsupported flags");
4294 return -ENOTSUPP;
4295 }
4296
4297 if (!btf->base_btf && btf_data_size == hdr->hdr_len) {
4298 btf_verifier_log(env, "No data");
4299 return -EINVAL;
4300 }
4301
4302 err = btf_check_sec_info(env, btf_data_size);
4303 if (err)
4304 return err;
4305
4306 return 0;
4307}
4308
4309static struct btf *btf_parse(bpfptr_t btf_data, u32 btf_data_size,
4310 u32 log_level, char __user *log_ubuf, u32 log_size)
4311{
4312 struct btf_verifier_env *env = NULL;
4313 struct bpf_verifier_log *log;
4314 struct btf *btf = NULL;
4315 u8 *data;
4316 int err;
4317
4318 if (btf_data_size > BTF_MAX_SIZE)
4319 return ERR_PTR(-E2BIG);
4320
4321 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
4322 if (!env)
4323 return ERR_PTR(-ENOMEM);
4324
4325 log = &env->log;
4326 if (log_level || log_ubuf || log_size) {
4327
4328
4329
4330 log->level = log_level;
4331 log->ubuf = log_ubuf;
4332 log->len_total = log_size;
4333
4334
4335 if (log->len_total < 128 || log->len_total > UINT_MAX >> 8 ||
4336 !log->level || !log->ubuf) {
4337 err = -EINVAL;
4338 goto errout;
4339 }
4340 }
4341
4342 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
4343 if (!btf) {
4344 err = -ENOMEM;
4345 goto errout;
4346 }
4347 env->btf = btf;
4348
4349 data = kvmalloc(btf_data_size, GFP_KERNEL | __GFP_NOWARN);
4350 if (!data) {
4351 err = -ENOMEM;
4352 goto errout;
4353 }
4354
4355 btf->data = data;
4356 btf->data_size = btf_data_size;
4357
4358 if (copy_from_bpfptr(data, btf_data, btf_data_size)) {
4359 err = -EFAULT;
4360 goto errout;
4361 }
4362
4363 err = btf_parse_hdr(env);
4364 if (err)
4365 goto errout;
4366
4367 btf->nohdr_data = btf->data + btf->hdr.hdr_len;
4368
4369 err = btf_parse_str_sec(env);
4370 if (err)
4371 goto errout;
4372
4373 err = btf_parse_type_sec(env);
4374 if (err)
4375 goto errout;
4376
4377 if (log->level && bpf_verifier_log_full(log)) {
4378 err = -ENOSPC;
4379 goto errout;
4380 }
4381
4382 btf_verifier_env_free(env);
4383 refcount_set(&btf->refcnt, 1);
4384 return btf;
4385
4386errout:
4387 btf_verifier_env_free(env);
4388 if (btf)
4389 btf_free(btf);
4390 return ERR_PTR(err);
4391}
4392
4393extern char __weak __start_BTF[];
4394extern char __weak __stop_BTF[];
4395extern struct btf *btf_vmlinux;
4396
4397#define BPF_MAP_TYPE(_id, _ops)
4398#define BPF_LINK_TYPE(_id, _name)
4399static union {
4400 struct bpf_ctx_convert {
4401#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
4402 prog_ctx_type _id##_prog; \
4403 kern_ctx_type _id##_kern;
4404#include <linux/bpf_types.h>
4405#undef BPF_PROG_TYPE
4406 } *__t;
4407
4408 const struct btf_type *t;
4409} bpf_ctx_convert;
4410enum {
4411#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
4412 __ctx_convert##_id,
4413#include <linux/bpf_types.h>
4414#undef BPF_PROG_TYPE
4415 __ctx_convert_unused,
4416};
4417static u8 bpf_ctx_convert_map[] = {
4418#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
4419 [_id] = __ctx_convert##_id,
4420#include <linux/bpf_types.h>
4421#undef BPF_PROG_TYPE
4422 0,
4423};
4424#undef BPF_MAP_TYPE
4425#undef BPF_LINK_TYPE
4426
4427static const struct btf_member *
4428btf_get_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf,
4429 const struct btf_type *t, enum bpf_prog_type prog_type,
4430 int arg)
4431{
4432 const struct btf_type *conv_struct;
4433 const struct btf_type *ctx_struct;
4434 const struct btf_member *ctx_type;
4435 const char *tname, *ctx_tname;
4436
4437 conv_struct = bpf_ctx_convert.t;
4438 if (!conv_struct) {
4439 bpf_log(log, "btf_vmlinux is malformed\n");
4440 return NULL;
4441 }
4442 t = btf_type_by_id(btf, t->type);
4443 while (btf_type_is_modifier(t))
4444 t = btf_type_by_id(btf, t->type);
4445 if (!btf_type_is_struct(t)) {
4446
4447
4448
4449
4450
4451 return NULL;
4452 }
4453 tname = btf_name_by_offset(btf, t->name_off);
4454 if (!tname) {
4455 bpf_log(log, "arg#%d struct doesn't have a name\n", arg);
4456 return NULL;
4457 }
4458
4459 ctx_type = btf_type_member(conv_struct) + bpf_ctx_convert_map[prog_type] * 2;
4460
4461
4462
4463 ctx_struct = btf_type_by_id(btf_vmlinux, ctx_type->type);
4464 if (!ctx_struct)
4465
4466 return NULL;
4467 ctx_tname = btf_name_by_offset(btf_vmlinux, ctx_struct->name_off);
4468 if (!ctx_tname) {
4469
4470 bpf_log(log, "Please fix kernel include/linux/bpf_types.h\n");
4471 return NULL;
4472 }
4473
4474
4475
4476
4477
4478
4479
4480 if (strcmp(ctx_tname, tname))
4481 return NULL;
4482 return ctx_type;
4483}
4484
4485static const struct bpf_map_ops * const btf_vmlinux_map_ops[] = {
4486#define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
4487#define BPF_LINK_TYPE(_id, _name)
4488#define BPF_MAP_TYPE(_id, _ops) \
4489 [_id] = &_ops,
4490#include <linux/bpf_types.h>
4491#undef BPF_PROG_TYPE
4492#undef BPF_LINK_TYPE
4493#undef BPF_MAP_TYPE
4494};
4495
4496static int btf_vmlinux_map_ids_init(const struct btf *btf,
4497 struct bpf_verifier_log *log)
4498{
4499 const struct bpf_map_ops *ops;
4500 int i, btf_id;
4501
4502 for (i = 0; i < ARRAY_SIZE(btf_vmlinux_map_ops); ++i) {
4503 ops = btf_vmlinux_map_ops[i];
4504 if (!ops || (!ops->map_btf_name && !ops->map_btf_id))
4505 continue;
4506 if (!ops->map_btf_name || !ops->map_btf_id) {
4507 bpf_log(log, "map type %d is misconfigured\n", i);
4508 return -EINVAL;
4509 }
4510 btf_id = btf_find_by_name_kind(btf, ops->map_btf_name,
4511 BTF_KIND_STRUCT);
4512 if (btf_id < 0)
4513 return btf_id;
4514 *ops->map_btf_id = btf_id;
4515 }
4516
4517 return 0;
4518}
4519
4520static int btf_translate_to_vmlinux(struct bpf_verifier_log *log,
4521 struct btf *btf,
4522 const struct btf_type *t,
4523 enum bpf_prog_type prog_type,
4524 int arg)
4525{
4526 const struct btf_member *prog_ctx_type, *kern_ctx_type;
4527
4528 prog_ctx_type = btf_get_prog_ctx_type(log, btf, t, prog_type, arg);
4529 if (!prog_ctx_type)
4530 return -ENOENT;
4531 kern_ctx_type = prog_ctx_type + 1;
4532 return kern_ctx_type->type;
4533}
4534
4535BTF_ID_LIST(bpf_ctx_convert_btf_id)
4536BTF_ID(struct, bpf_ctx_convert)
4537
4538struct btf *btf_parse_vmlinux(void)
4539{
4540 struct btf_verifier_env *env = NULL;
4541 struct bpf_verifier_log *log;
4542 struct btf *btf = NULL;
4543 int err;
4544
4545 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
4546 if (!env)
4547 return ERR_PTR(-ENOMEM);
4548
4549 log = &env->log;
4550 log->level = BPF_LOG_KERNEL;
4551
4552 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
4553 if (!btf) {
4554 err = -ENOMEM;
4555 goto errout;
4556 }
4557 env->btf = btf;
4558
4559 btf->data = __start_BTF;
4560 btf->data_size = __stop_BTF - __start_BTF;
4561 btf->kernel_btf = true;
4562 snprintf(btf->name, sizeof(btf->name), "vmlinux");
4563
4564 err = btf_parse_hdr(env);
4565 if (err)
4566 goto errout;
4567
4568 btf->nohdr_data = btf->data + btf->hdr.hdr_len;
4569
4570 err = btf_parse_str_sec(env);
4571 if (err)
4572 goto errout;
4573
4574 err = btf_check_all_metas(env);
4575 if (err)
4576 goto errout;
4577
4578
4579 bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]);
4580
4581
4582 err = btf_vmlinux_map_ids_init(btf, log);
4583 if (err < 0)
4584 goto errout;
4585
4586 bpf_struct_ops_init(btf, log);
4587
4588 refcount_set(&btf->refcnt, 1);
4589
4590 err = btf_alloc_id(btf);
4591 if (err)
4592 goto errout;
4593
4594 btf_verifier_env_free(env);
4595 return btf;
4596
4597errout:
4598 btf_verifier_env_free(env);
4599 if (btf) {
4600 kvfree(btf->types);
4601 kfree(btf);
4602 }
4603 return ERR_PTR(err);
4604}
4605
4606#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
4607
4608static struct btf *btf_parse_module(const char *module_name, const void *data, unsigned int data_size)
4609{
4610 struct btf_verifier_env *env = NULL;
4611 struct bpf_verifier_log *log;
4612 struct btf *btf = NULL, *base_btf;
4613 int err;
4614
4615 base_btf = bpf_get_btf_vmlinux();
4616 if (IS_ERR(base_btf))
4617 return base_btf;
4618 if (!base_btf)
4619 return ERR_PTR(-EINVAL);
4620
4621 env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
4622 if (!env)
4623 return ERR_PTR(-ENOMEM);
4624
4625 log = &env->log;
4626 log->level = BPF_LOG_KERNEL;
4627
4628 btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
4629 if (!btf) {
4630 err = -ENOMEM;
4631 goto errout;
4632 }
4633 env->btf = btf;
4634
4635 btf->base_btf = base_btf;
4636 btf->start_id = base_btf->nr_types;
4637 btf->start_str_off = base_btf->hdr.str_len;
4638 btf->kernel_btf = true;
4639 snprintf(btf->name, sizeof(btf->name), "%s", module_name);
4640
4641 btf->data = kvmalloc(data_size, GFP_KERNEL | __GFP_NOWARN);
4642 if (!btf->data) {
4643 err = -ENOMEM;
4644 goto errout;
4645 }
4646 memcpy(btf->data, data, data_size);
4647 btf->data_size = data_size;
4648
4649 err = btf_parse_hdr(env);
4650 if (err)
4651 goto errout;
4652
4653 btf->nohdr_data = btf->data + btf->hdr.hdr_len;
4654
4655 err = btf_parse_str_sec(env);
4656 if (err)
4657 goto errout;
4658
4659 err = btf_check_all_metas(env);
4660 if (err)
4661 goto errout;
4662
4663 btf_verifier_env_free(env);
4664 refcount_set(&btf->refcnt, 1);
4665 return btf;
4666
4667errout:
4668 btf_verifier_env_free(env);
4669 if (btf) {
4670 kvfree(btf->data);
4671 kvfree(btf->types);
4672 kfree(btf);
4673 }
4674 return ERR_PTR(err);
4675}
4676
4677#endif
4678
4679struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog)
4680{
4681 struct bpf_prog *tgt_prog = prog->aux->dst_prog;
4682
4683 if (tgt_prog)
4684 return tgt_prog->aux->btf;
4685 else
4686 return prog->aux->attach_btf;
4687}
4688
4689static bool is_string_ptr(struct btf *btf, const struct btf_type *t)
4690{
4691
4692 t = btf_type_by_id(btf, t->type);
4693
4694
4695 if (BTF_INFO_KIND(t->info) == BTF_KIND_CONST)
4696 t = btf_type_by_id(btf, t->type);
4697
4698
4699 return btf_type_is_int(t) && t->size == 1;
4700}
4701
4702bool btf_ctx_access(int off, int size, enum bpf_access_type type,
4703 const struct bpf_prog *prog,
4704 struct bpf_insn_access_aux *info)
4705{
4706 const struct btf_type *t = prog->aux->attach_func_proto;
4707 struct bpf_prog *tgt_prog = prog->aux->dst_prog;
4708 struct btf *btf = bpf_prog_get_target_btf(prog);
4709 const char *tname = prog->aux->attach_func_name;
4710 struct bpf_verifier_log *log = info->log;
4711 const struct btf_param *args;
4712 u32 nr_args, arg;
4713 int i, ret;
4714
4715 if (off % 8) {
4716 bpf_log(log, "func '%s' offset %d is not multiple of 8\n",
4717 tname, off);
4718 return false;
4719 }
4720 arg = off / 8;
4721 args = (const struct btf_param *)(t + 1);
4722
4723
4724
4725 nr_args = t ? btf_type_vlen(t) : MAX_BPF_FUNC_REG_ARGS;
4726 if (prog->aux->attach_btf_trace) {
4727
4728 args++;
4729 nr_args--;
4730 }
4731
4732 if (arg > nr_args) {
4733 bpf_log(log, "func '%s' doesn't have %d-th argument\n",
4734 tname, arg + 1);
4735 return false;
4736 }
4737
4738 if (arg == nr_args) {
4739 switch (prog->expected_attach_type) {
4740 case BPF_LSM_MAC:
4741 case BPF_TRACE_FEXIT:
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755 if (!t)
4756 return true;
4757 t = btf_type_by_id(btf, t->type);
4758 break;
4759 case BPF_MODIFY_RETURN:
4760
4761
4762
4763 if (!t)
4764 return false;
4765
4766 t = btf_type_skip_modifiers(btf, t->type, NULL);
4767 if (!btf_type_is_small_int(t)) {
4768 bpf_log(log,
4769 "ret type %s not allowed for fmod_ret\n",
4770 btf_kind_str[BTF_INFO_KIND(t->info)]);
4771 return false;
4772 }
4773 break;
4774 default:
4775 bpf_log(log, "func '%s' doesn't have %d-th argument\n",
4776 tname, arg + 1);
4777 return false;
4778 }
4779 } else {
4780 if (!t)
4781
4782 return true;
4783 t = btf_type_by_id(btf, args[arg].type);
4784 }
4785
4786
4787 while (btf_type_is_modifier(t))
4788 t = btf_type_by_id(btf, t->type);
4789 if (btf_type_is_small_int(t) || btf_type_is_enum(t))
4790
4791 return true;
4792 if (!btf_type_is_ptr(t)) {
4793 bpf_log(log,
4794 "func '%s' arg%d '%s' has type %s. Only pointer access is allowed\n",
4795 tname, arg,
4796 __btf_name_by_offset(btf, t->name_off),
4797 btf_kind_str[BTF_INFO_KIND(t->info)]);
4798 return false;
4799 }
4800
4801
4802 for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
4803 const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
4804
4805 if (ctx_arg_info->offset == off &&
4806 (ctx_arg_info->reg_type == PTR_TO_RDONLY_BUF_OR_NULL ||
4807 ctx_arg_info->reg_type == PTR_TO_RDWR_BUF_OR_NULL)) {
4808 info->reg_type = ctx_arg_info->reg_type;
4809 return true;
4810 }
4811 }
4812
4813 if (t->type == 0)
4814
4815
4816
4817
4818 return true;
4819
4820 if (is_string_ptr(btf, t))
4821 return true;
4822
4823
4824 for (i = 0; i < prog->aux->ctx_arg_info_size; i++) {
4825 const struct bpf_ctx_arg_aux *ctx_arg_info = &prog->aux->ctx_arg_info[i];
4826
4827 if (ctx_arg_info->offset == off) {
4828 if (!ctx_arg_info->btf_id) {
4829 bpf_log(log,"invalid btf_id for context argument offset %u\n", off);
4830 return false;
4831 }
4832
4833 info->reg_type = ctx_arg_info->reg_type;
4834 info->btf = btf_vmlinux;
4835 info->btf_id = ctx_arg_info->btf_id;
4836 return true;
4837 }
4838 }
4839
4840 info->reg_type = PTR_TO_BTF_ID;
4841 if (tgt_prog) {
4842 enum bpf_prog_type tgt_type;
4843
4844 if (tgt_prog->type == BPF_PROG_TYPE_EXT)
4845 tgt_type = tgt_prog->aux->saved_dst_prog_type;
4846 else
4847 tgt_type = tgt_prog->type;
4848
4849 ret = btf_translate_to_vmlinux(log, btf, t, tgt_type, arg);
4850 if (ret > 0) {
4851 info->btf = btf_vmlinux;
4852 info->btf_id = ret;
4853 return true;
4854 } else {
4855 return false;
4856 }
4857 }
4858
4859 info->btf = btf;
4860 info->btf_id = t->type;
4861 t = btf_type_by_id(btf, t->type);
4862
4863 while (btf_type_is_modifier(t)) {
4864 info->btf_id = t->type;
4865 t = btf_type_by_id(btf, t->type);
4866 }
4867 if (!btf_type_is_struct(t)) {
4868 bpf_log(log,
4869 "func '%s' arg%d type %s is not a struct\n",
4870 tname, arg, btf_kind_str[BTF_INFO_KIND(t->info)]);
4871 return false;
4872 }
4873 bpf_log(log, "func '%s' arg%d has btf_id %d type %s '%s'\n",
4874 tname, arg, info->btf_id, btf_kind_str[BTF_INFO_KIND(t->info)],
4875 __btf_name_by_offset(btf, t->name_off));
4876 return true;
4877}
4878
4879enum bpf_struct_walk_result {
4880
4881 WALK_SCALAR = 0,
4882 WALK_PTR,
4883 WALK_STRUCT,
4884};
4885
4886static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf,
4887 const struct btf_type *t, int off, int size,
4888 u32 *next_btf_id)
4889{
4890 u32 i, moff, mtrue_end, msize = 0, total_nelems = 0;
4891 const struct btf_type *mtype, *elem_type = NULL;
4892 const struct btf_member *member;
4893 const char *tname, *mname;
4894 u32 vlen, elem_id, mid;
4895
4896again:
4897 tname = __btf_name_by_offset(btf, t->name_off);
4898 if (!btf_type_is_struct(t)) {
4899 bpf_log(log, "Type '%s' is not a struct\n", tname);
4900 return -EINVAL;
4901 }
4902
4903 vlen = btf_type_vlen(t);
4904 if (off + size > t->size) {
4905
4906
4907
4908 struct btf_array *array_elem;
4909
4910 if (vlen == 0)
4911 goto error;
4912
4913 member = btf_type_member(t) + vlen - 1;
4914 mtype = btf_type_skip_modifiers(btf, member->type,
4915 NULL);
4916 if (!btf_type_is_array(mtype))
4917 goto error;
4918
4919 array_elem = (struct btf_array *)(mtype + 1);
4920 if (array_elem->nelems != 0)
4921 goto error;
4922
4923 moff = btf_member_bit_offset(t, member) / 8;
4924 if (off < moff)
4925 goto error;
4926
4927
4928
4929
4930 t = btf_type_skip_modifiers(btf, array_elem->type,
4931 NULL);
4932 if (!btf_type_is_struct(t))
4933 goto error;
4934
4935 off = (off - moff) % t->size;
4936 goto again;
4937
4938error:
4939 bpf_log(log, "access beyond struct %s at off %u size %u\n",
4940 tname, off, size);
4941 return -EACCES;
4942 }
4943
4944 for_each_member(i, t, member) {
4945
4946 moff = btf_member_bit_offset(t, member) / 8;
4947 if (off + size <= moff)
4948
4949 break;
4950
4951 if (btf_member_bitfield_size(t, member)) {
4952 u32 end_bit = btf_member_bit_offset(t, member) +
4953 btf_member_bitfield_size(t, member);
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963 if (off <= moff &&
4964 BITS_ROUNDUP_BYTES(end_bit) <= off + size)
4965 return WALK_SCALAR;
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979 continue;
4980 }
4981
4982
4983 if (off < moff)
4984 break;
4985
4986
4987 mid = member->type;
4988 mtype = btf_type_by_id(btf, member->type);
4989 mname = __btf_name_by_offset(btf, member->name_off);
4990
4991 mtype = __btf_resolve_size(btf, mtype, &msize,
4992 &elem_type, &elem_id, &total_nelems,
4993 &mid);
4994 if (IS_ERR(mtype)) {
4995 bpf_log(log, "field %s doesn't have size\n", mname);
4996 return -EFAULT;
4997 }
4998
4999 mtrue_end = moff + msize;
5000 if (off >= mtrue_end)
5001
5002 continue;
5003
5004 if (btf_type_is_array(mtype)) {
5005 u32 elem_idx;
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048 if (moff == mtrue_end)
5049 continue;
5050
5051 msize /= total_nelems;
5052 elem_idx = (off - moff) / msize;
5053 moff += elem_idx * msize;
5054 mtype = elem_type;
5055 mid = elem_id;
5056 }
5057
5058
5059
5060
5061 if (btf_type_is_struct(mtype)) {
5062
5063 t = mtype;
5064
5065
5066 if (off == moff) {
5067 *next_btf_id = mid;
5068 return WALK_STRUCT;
5069 }
5070
5071
5072 off -= moff;
5073 goto again;
5074 }
5075
5076 if (btf_type_is_ptr(mtype)) {
5077 const struct btf_type *stype;
5078 u32 id;
5079
5080 if (msize != size || off != moff) {
5081 bpf_log(log,
5082 "cannot access ptr member %s with moff %u in struct %s with off %u size %u\n",
5083 mname, moff, tname, off, size);
5084 return -EACCES;
5085 }
5086 stype = btf_type_skip_modifiers(btf, mtype->type, &id);
5087 if (btf_type_is_struct(stype)) {
5088 *next_btf_id = id;
5089 return WALK_PTR;
5090 }
5091 }
5092
5093
5094
5095
5096
5097
5098
5099 if (off + size > mtrue_end) {
5100 bpf_log(log,
5101 "access beyond the end of member %s (mend:%u) in struct %s with off %u size %u\n",
5102 mname, mtrue_end, tname, off, size);
5103 return -EACCES;
5104 }
5105
5106 return WALK_SCALAR;
5107 }
5108 bpf_log(log, "struct %s doesn't have field at offset %d\n", tname, off);
5109 return -EINVAL;
5110}
5111
5112int btf_struct_access(struct bpf_verifier_log *log, const struct btf *btf,
5113 const struct btf_type *t, int off, int size,
5114 enum bpf_access_type atype __maybe_unused,
5115 u32 *next_btf_id)
5116{
5117 int err;
5118 u32 id;
5119
5120 do {
5121 err = btf_struct_walk(log, btf, t, off, size, &id);
5122
5123 switch (err) {
5124 case WALK_PTR:
5125
5126
5127
5128 *next_btf_id = id;
5129 return PTR_TO_BTF_ID;
5130 case WALK_SCALAR:
5131 return SCALAR_VALUE;
5132 case WALK_STRUCT:
5133
5134
5135
5136
5137 t = btf_type_by_id(btf, id);
5138 off = 0;
5139 break;
5140 default:
5141
5142
5143
5144 if (WARN_ONCE(err > 0, "unknown btf_struct_walk return value"))
5145 return -EINVAL;
5146 return err;
5147 }
5148 } while (t);
5149
5150 return -EINVAL;
5151}
5152
5153
5154
5155
5156
5157
5158static bool btf_types_are_same(const struct btf *btf1, u32 id1,
5159 const struct btf *btf2, u32 id2)
5160{
5161 if (id1 != id2)
5162 return false;
5163 if (btf1 == btf2)
5164 return true;
5165 return btf_type_by_id(btf1, id1) == btf_type_by_id(btf2, id2);
5166}
5167
5168bool btf_struct_ids_match(struct bpf_verifier_log *log,
5169 const struct btf *btf, u32 id, int off,
5170 const struct btf *need_btf, u32 need_type_id)
5171{
5172 const struct btf_type *type;
5173 int err;
5174
5175
5176 if (off == 0 && btf_types_are_same(btf, id, need_btf, need_type_id))
5177 return true;
5178
5179again:
5180 type = btf_type_by_id(btf, id);
5181 if (!type)
5182 return false;
5183 err = btf_struct_walk(log, btf, type, off, 1, &id);
5184 if (err != WALK_STRUCT)
5185 return false;
5186
5187
5188
5189
5190
5191
5192 if (!btf_types_are_same(btf, id, need_btf, need_type_id)) {
5193 off = 0;
5194 goto again;
5195 }
5196
5197 return true;
5198}
5199
5200static int __get_type_size(struct btf *btf, u32 btf_id,
5201 const struct btf_type **bad_type)
5202{
5203 const struct btf_type *t;
5204
5205 if (!btf_id)
5206
5207 return 0;
5208 t = btf_type_by_id(btf, btf_id);
5209 while (t && btf_type_is_modifier(t))
5210 t = btf_type_by_id(btf, t->type);
5211 if (!t) {
5212 *bad_type = btf_type_by_id(btf, 0);
5213 return -EINVAL;
5214 }
5215 if (btf_type_is_ptr(t))
5216
5217 return sizeof(void *);
5218 if (btf_type_is_int(t) || btf_type_is_enum(t))
5219 return t->size;
5220 *bad_type = t;
5221 return -EINVAL;
5222}
5223
5224int btf_distill_func_proto(struct bpf_verifier_log *log,
5225 struct btf *btf,
5226 const struct btf_type *func,
5227 const char *tname,
5228 struct btf_func_model *m)
5229{
5230 const struct btf_param *args;
5231 const struct btf_type *t;
5232 u32 i, nargs;
5233 int ret;
5234
5235 if (!func) {
5236
5237
5238
5239 for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++)
5240 m->arg_size[i] = 8;
5241 m->ret_size = 8;
5242 m->nr_args = MAX_BPF_FUNC_REG_ARGS;
5243 return 0;
5244 }
5245 args = (const struct btf_param *)(func + 1);
5246 nargs = btf_type_vlen(func);
5247 if (nargs >= MAX_BPF_FUNC_ARGS) {
5248 bpf_log(log,
5249 "The function %s has %d arguments. Too many.\n",
5250 tname, nargs);
5251 return -EINVAL;
5252 }
5253 ret = __get_type_size(btf, func->type, &t);
5254 if (ret < 0) {
5255 bpf_log(log,
5256 "The function %s return type %s is unsupported.\n",
5257 tname, btf_kind_str[BTF_INFO_KIND(t->info)]);
5258 return -EINVAL;
5259 }
5260 m->ret_size = ret;
5261
5262 for (i = 0; i < nargs; i++) {
5263 if (i == nargs - 1 && args[i].type == 0) {
5264 bpf_log(log,
5265 "The function %s with variable args is unsupported.\n",
5266 tname);
5267 return -EINVAL;
5268 }
5269 ret = __get_type_size(btf, args[i].type, &t);
5270 if (ret < 0) {
5271 bpf_log(log,
5272 "The function %s arg%d type %s is unsupported.\n",
5273 tname, i, btf_kind_str[BTF_INFO_KIND(t->info)]);
5274 return -EINVAL;
5275 }
5276 if (ret == 0) {
5277 bpf_log(log,
5278 "The function %s has malformed void argument.\n",
5279 tname);
5280 return -EINVAL;
5281 }
5282 m->arg_size[i] = ret;
5283 }
5284 m->nr_args = nargs;
5285 return 0;
5286}
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296static int btf_check_func_type_match(struct bpf_verifier_log *log,
5297 struct btf *btf1, const struct btf_type *t1,
5298 struct btf *btf2, const struct btf_type *t2)
5299{
5300 const struct btf_param *args1, *args2;
5301 const char *fn1, *fn2, *s1, *s2;
5302 u32 nargs1, nargs2, i;
5303
5304 fn1 = btf_name_by_offset(btf1, t1->name_off);
5305 fn2 = btf_name_by_offset(btf2, t2->name_off);
5306
5307 if (btf_func_linkage(t1) != BTF_FUNC_GLOBAL) {
5308 bpf_log(log, "%s() is not a global function\n", fn1);
5309 return -EINVAL;
5310 }
5311 if (btf_func_linkage(t2) != BTF_FUNC_GLOBAL) {
5312 bpf_log(log, "%s() is not a global function\n", fn2);
5313 return -EINVAL;
5314 }
5315
5316 t1 = btf_type_by_id(btf1, t1->type);
5317 if (!t1 || !btf_type_is_func_proto(t1))
5318 return -EFAULT;
5319 t2 = btf_type_by_id(btf2, t2->type);
5320 if (!t2 || !btf_type_is_func_proto(t2))
5321 return -EFAULT;
5322
5323 args1 = (const struct btf_param *)(t1 + 1);
5324 nargs1 = btf_type_vlen(t1);
5325 args2 = (const struct btf_param *)(t2 + 1);
5326 nargs2 = btf_type_vlen(t2);
5327
5328 if (nargs1 != nargs2) {
5329 bpf_log(log, "%s() has %d args while %s() has %d args\n",
5330 fn1, nargs1, fn2, nargs2);
5331 return -EINVAL;
5332 }
5333
5334 t1 = btf_type_skip_modifiers(btf1, t1->type, NULL);
5335 t2 = btf_type_skip_modifiers(btf2, t2->type, NULL);
5336 if (t1->info != t2->info) {
5337 bpf_log(log,
5338 "Return type %s of %s() doesn't match type %s of %s()\n",
5339 btf_type_str(t1), fn1,
5340 btf_type_str(t2), fn2);
5341 return -EINVAL;
5342 }
5343
5344 for (i = 0; i < nargs1; i++) {
5345 t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL);
5346 t2 = btf_type_skip_modifiers(btf2, args2[i].type, NULL);
5347
5348 if (t1->info != t2->info) {
5349 bpf_log(log, "arg%d in %s() is %s while %s() has %s\n",
5350 i, fn1, btf_type_str(t1),
5351 fn2, btf_type_str(t2));
5352 return -EINVAL;
5353 }
5354 if (btf_type_has_size(t1) && t1->size != t2->size) {
5355 bpf_log(log,
5356 "arg%d in %s() has size %d while %s() has %d\n",
5357 i, fn1, t1->size,
5358 fn2, t2->size);
5359 return -EINVAL;
5360 }
5361
5362
5363
5364
5365
5366 if (btf_type_is_int(t1) || btf_type_is_enum(t1))
5367 continue;
5368 if (!btf_type_is_ptr(t1)) {
5369 bpf_log(log,
5370 "arg%d in %s() has unrecognized type\n",
5371 i, fn1);
5372 return -EINVAL;
5373 }
5374 t1 = btf_type_skip_modifiers(btf1, t1->type, NULL);
5375 t2 = btf_type_skip_modifiers(btf2, t2->type, NULL);
5376 if (!btf_type_is_struct(t1)) {
5377 bpf_log(log,
5378 "arg%d in %s() is not a pointer to context\n",
5379 i, fn1);
5380 return -EINVAL;
5381 }
5382 if (!btf_type_is_struct(t2)) {
5383 bpf_log(log,
5384 "arg%d in %s() is not a pointer to context\n",
5385 i, fn2);
5386 return -EINVAL;
5387 }
5388
5389
5390
5391
5392
5393
5394 s1 = btf_name_by_offset(btf1, t1->name_off);
5395 s2 = btf_name_by_offset(btf2, t2->name_off);
5396 if (strcmp(s1, s2)) {
5397 bpf_log(log,
5398 "arg%d %s(struct %s *) doesn't match %s(struct %s *)\n",
5399 i, fn1, s1, fn2, s2);
5400 return -EINVAL;
5401 }
5402 }
5403 return 0;
5404}
5405
5406
5407int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
5408 struct btf *btf2, const struct btf_type *t2)
5409{
5410 struct btf *btf1 = prog->aux->btf;
5411 const struct btf_type *t1;
5412 u32 btf_id = 0;
5413
5414 if (!prog->aux->func_info) {
5415 bpf_log(log, "Program extension requires BTF\n");
5416 return -EINVAL;
5417 }
5418
5419 btf_id = prog->aux->func_info[0].type_id;
5420 if (!btf_id)
5421 return -EFAULT;
5422
5423 t1 = btf_type_by_id(btf1, btf_id);
5424 if (!t1 || !btf_type_is_func(t1))
5425 return -EFAULT;
5426
5427 return btf_check_func_type_match(log, btf1, t1, btf2, t2);
5428}
5429
5430static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
5431#ifdef CONFIG_NET
5432 [PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
5433 [PTR_TO_SOCK_COMMON] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
5434 [PTR_TO_TCP_SOCK] = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
5435#endif
5436};
5437
5438static int btf_check_func_arg_match(struct bpf_verifier_env *env,
5439 const struct btf *btf, u32 func_id,
5440 struct bpf_reg_state *regs,
5441 bool ptr_to_mem_ok)
5442{
5443 struct bpf_verifier_log *log = &env->log;
5444 const char *func_name, *ref_tname;
5445 const struct btf_type *t, *ref_t;
5446 const struct btf_param *args;
5447 u32 i, nargs, ref_id;
5448
5449 t = btf_type_by_id(btf, func_id);
5450 if (!t || !btf_type_is_func(t)) {
5451
5452
5453
5454 bpf_log(log, "BTF of func_id %u doesn't point to KIND_FUNC\n",
5455 func_id);
5456 return -EFAULT;
5457 }
5458 func_name = btf_name_by_offset(btf, t->name_off);
5459
5460 t = btf_type_by_id(btf, t->type);
5461 if (!t || !btf_type_is_func_proto(t)) {
5462 bpf_log(log, "Invalid BTF of func %s\n", func_name);
5463 return -EFAULT;
5464 }
5465 args = (const struct btf_param *)(t + 1);
5466 nargs = btf_type_vlen(t);
5467 if (nargs > MAX_BPF_FUNC_REG_ARGS) {
5468 bpf_log(log, "Function %s has %d > %d args\n", func_name, nargs,
5469 MAX_BPF_FUNC_REG_ARGS);
5470 return -EINVAL;
5471 }
5472
5473
5474
5475
5476 for (i = 0; i < nargs; i++) {
5477 u32 regno = i + 1;
5478 struct bpf_reg_state *reg = ®s[regno];
5479
5480 t = btf_type_skip_modifiers(btf, args[i].type, NULL);
5481 if (btf_type_is_scalar(t)) {
5482 if (reg->type == SCALAR_VALUE)
5483 continue;
5484 bpf_log(log, "R%d is not a scalar\n", regno);
5485 return -EINVAL;
5486 }
5487
5488 if (!btf_type_is_ptr(t)) {
5489 bpf_log(log, "Unrecognized arg#%d type %s\n",
5490 i, btf_type_str(t));
5491 return -EINVAL;
5492 }
5493
5494 ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
5495 ref_tname = btf_name_by_offset(btf, ref_t->name_off);
5496 if (btf_is_kernel(btf)) {
5497 const struct btf_type *reg_ref_t;
5498 const struct btf *reg_btf;
5499 const char *reg_ref_tname;
5500 u32 reg_ref_id;
5501
5502 if (!btf_type_is_struct(ref_t)) {
5503 bpf_log(log, "kernel function %s args#%d pointer type %s %s is not supported\n",
5504 func_name, i, btf_type_str(ref_t),
5505 ref_tname);
5506 return -EINVAL;
5507 }
5508
5509 if (reg->type == PTR_TO_BTF_ID) {
5510 reg_btf = reg->btf;
5511 reg_ref_id = reg->btf_id;
5512 } else if (reg2btf_ids[reg->type]) {
5513 reg_btf = btf_vmlinux;
5514 reg_ref_id = *reg2btf_ids[reg->type];
5515 } else {
5516 bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d is not a pointer to btf_id\n",
5517 func_name, i,
5518 btf_type_str(ref_t), ref_tname, regno);
5519 return -EINVAL;
5520 }
5521
5522 reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id,
5523 ®_ref_id);
5524 reg_ref_tname = btf_name_by_offset(reg_btf,
5525 reg_ref_t->name_off);
5526 if (!btf_struct_ids_match(log, reg_btf, reg_ref_id,
5527 reg->off, btf, ref_id)) {
5528 bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
5529 func_name, i,
5530 btf_type_str(ref_t), ref_tname,
5531 regno, btf_type_str(reg_ref_t),
5532 reg_ref_tname);
5533 return -EINVAL;
5534 }
5535 } else if (btf_get_prog_ctx_type(log, btf, t,
5536 env->prog->type, i)) {
5537
5538
5539
5540 if (reg->type != PTR_TO_CTX) {
5541 bpf_log(log,
5542 "arg#%d expected pointer to ctx, but got %s\n",
5543 i, btf_type_str(t));
5544 return -EINVAL;
5545 }
5546 if (check_ctx_reg(env, reg, regno))
5547 return -EINVAL;
5548 } else if (ptr_to_mem_ok) {
5549 const struct btf_type *resolve_ret;
5550 u32 type_size;
5551
5552 resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
5553 if (IS_ERR(resolve_ret)) {
5554 bpf_log(log,
5555 "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5556 i, btf_type_str(ref_t), ref_tname,
5557 PTR_ERR(resolve_ret));
5558 return -EINVAL;
5559 }
5560
5561 if (check_mem_reg(env, reg, regno, type_size))
5562 return -EINVAL;
5563 } else {
5564 return -EINVAL;
5565 }
5566 }
5567
5568 return 0;
5569}
5570
5571
5572
5573
5574
5575
5576
5577
5578int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
5579 struct bpf_reg_state *regs)
5580{
5581 struct bpf_prog *prog = env->prog;
5582 struct btf *btf = prog->aux->btf;
5583 bool is_global;
5584 u32 btf_id;
5585 int err;
5586
5587 if (!prog->aux->func_info)
5588 return -EINVAL;
5589
5590 btf_id = prog->aux->func_info[subprog].type_id;
5591 if (!btf_id)
5592 return -EFAULT;
5593
5594 if (prog->aux->func_info_aux[subprog].unreliable)
5595 return -EINVAL;
5596
5597 is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
5598 err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global);
5599
5600
5601
5602
5603
5604 if (err)
5605 prog->aux->func_info_aux[subprog].unreliable = true;
5606 return err;
5607}
5608
5609int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
5610 const struct btf *btf, u32 func_id,
5611 struct bpf_reg_state *regs)
5612{
5613 return btf_check_func_arg_match(env, btf, func_id, regs, false);
5614}
5615
5616
5617
5618
5619
5620
5621
5622
5623int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
5624 struct bpf_reg_state *regs)
5625{
5626 struct bpf_verifier_log *log = &env->log;
5627 struct bpf_prog *prog = env->prog;
5628 enum bpf_prog_type prog_type = prog->type;
5629 struct btf *btf = prog->aux->btf;
5630 const struct btf_param *args;
5631 const struct btf_type *t, *ref_t;
5632 u32 i, nargs, btf_id;
5633 const char *tname;
5634
5635 if (!prog->aux->func_info ||
5636 prog->aux->func_info_aux[subprog].linkage != BTF_FUNC_GLOBAL) {
5637 bpf_log(log, "Verifier bug\n");
5638 return -EFAULT;
5639 }
5640
5641 btf_id = prog->aux->func_info[subprog].type_id;
5642 if (!btf_id) {
5643 bpf_log(log, "Global functions need valid BTF\n");
5644 return -EFAULT;
5645 }
5646
5647 t = btf_type_by_id(btf, btf_id);
5648 if (!t || !btf_type_is_func(t)) {
5649
5650
5651
5652 bpf_log(log, "BTF of func#%d doesn't point to KIND_FUNC\n",
5653 subprog);
5654 return -EFAULT;
5655 }
5656 tname = btf_name_by_offset(btf, t->name_off);
5657
5658 if (log->level & BPF_LOG_LEVEL)
5659 bpf_log(log, "Validating %s() func#%d...\n",
5660 tname, subprog);
5661
5662 if (prog->aux->func_info_aux[subprog].unreliable) {
5663 bpf_log(log, "Verifier bug in function %s()\n", tname);
5664 return -EFAULT;
5665 }
5666 if (prog_type == BPF_PROG_TYPE_EXT)
5667 prog_type = prog->aux->dst_prog->type;
5668
5669 t = btf_type_by_id(btf, t->type);
5670 if (!t || !btf_type_is_func_proto(t)) {
5671 bpf_log(log, "Invalid type of function %s()\n", tname);
5672 return -EFAULT;
5673 }
5674 args = (const struct btf_param *)(t + 1);
5675 nargs = btf_type_vlen(t);
5676 if (nargs > MAX_BPF_FUNC_REG_ARGS) {
5677 bpf_log(log, "Global function %s() with %d > %d args. Buggy compiler.\n",
5678 tname, nargs, MAX_BPF_FUNC_REG_ARGS);
5679 return -EINVAL;
5680 }
5681
5682 t = btf_type_by_id(btf, t->type);
5683 while (btf_type_is_modifier(t))
5684 t = btf_type_by_id(btf, t->type);
5685 if (!btf_type_is_int(t) && !btf_type_is_enum(t)) {
5686 bpf_log(log,
5687 "Global function %s() doesn't return scalar. Only those are supported.\n",
5688 tname);
5689 return -EINVAL;
5690 }
5691
5692
5693
5694 for (i = 0; i < nargs; i++) {
5695 struct bpf_reg_state *reg = ®s[i + 1];
5696
5697 t = btf_type_by_id(btf, args[i].type);
5698 while (btf_type_is_modifier(t))
5699 t = btf_type_by_id(btf, t->type);
5700 if (btf_type_is_int(t) || btf_type_is_enum(t)) {
5701 reg->type = SCALAR_VALUE;
5702 continue;
5703 }
5704 if (btf_type_is_ptr(t)) {
5705 if (btf_get_prog_ctx_type(log, btf, t, prog_type, i)) {
5706 reg->type = PTR_TO_CTX;
5707 continue;
5708 }
5709
5710 t = btf_type_skip_modifiers(btf, t->type, NULL);
5711
5712 ref_t = btf_resolve_size(btf, t, ®->mem_size);
5713 if (IS_ERR(ref_t)) {
5714 bpf_log(log,
5715 "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
5716 i, btf_type_str(t), btf_name_by_offset(btf, t->name_off),
5717 PTR_ERR(ref_t));
5718 return -EINVAL;
5719 }
5720
5721 reg->type = PTR_TO_MEM_OR_NULL;
5722 reg->id = ++env->id_gen;
5723
5724 continue;
5725 }
5726 bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n",
5727 i, btf_kind_str[BTF_INFO_KIND(t->info)], tname);
5728 return -EINVAL;
5729 }
5730 return 0;
5731}
5732
5733static void btf_type_show(const struct btf *btf, u32 type_id, void *obj,
5734 struct btf_show *show)
5735{
5736 const struct btf_type *t = btf_type_by_id(btf, type_id);
5737
5738 show->btf = btf;
5739 memset(&show->state, 0, sizeof(show->state));
5740 memset(&show->obj, 0, sizeof(show->obj));
5741
5742 btf_type_ops(t)->show(btf, t, type_id, obj, 0, show);
5743}
5744
5745static void btf_seq_show(struct btf_show *show, const char *fmt,
5746 va_list args)
5747{
5748 seq_vprintf((struct seq_file *)show->target, fmt, args);
5749}
5750
5751int btf_type_seq_show_flags(const struct btf *btf, u32 type_id,
5752 void *obj, struct seq_file *m, u64 flags)
5753{
5754 struct btf_show sseq;
5755
5756 sseq.target = m;
5757 sseq.showfn = btf_seq_show;
5758 sseq.flags = flags;
5759
5760 btf_type_show(btf, type_id, obj, &sseq);
5761
5762 return sseq.state.status;
5763}
5764
5765void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj,
5766 struct seq_file *m)
5767{
5768 (void) btf_type_seq_show_flags(btf, type_id, obj, m,
5769 BTF_SHOW_NONAME | BTF_SHOW_COMPACT |
5770 BTF_SHOW_ZERO | BTF_SHOW_UNSAFE);
5771}
5772
5773struct btf_show_snprintf {
5774 struct btf_show show;
5775 int len_left;
5776 int len;
5777};
5778
5779static void btf_snprintf_show(struct btf_show *show, const char *fmt,
5780 va_list args)
5781{
5782 struct btf_show_snprintf *ssnprintf = (struct btf_show_snprintf *)show;
5783 int len;
5784
5785 len = vsnprintf(show->target, ssnprintf->len_left, fmt, args);
5786
5787 if (len < 0) {
5788 ssnprintf->len_left = 0;
5789 ssnprintf->len = len;
5790 } else if (len > ssnprintf->len_left) {
5791
5792 ssnprintf->len_left = 0;
5793 ssnprintf->len += len;
5794 } else {
5795 ssnprintf->len_left -= len;
5796 ssnprintf->len += len;
5797 show->target += len;
5798 }
5799}
5800
5801int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj,
5802 char *buf, int len, u64 flags)
5803{
5804 struct btf_show_snprintf ssnprintf;
5805
5806 ssnprintf.show.target = buf;
5807 ssnprintf.show.flags = flags;
5808 ssnprintf.show.showfn = btf_snprintf_show;
5809 ssnprintf.len_left = len;
5810 ssnprintf.len = 0;
5811
5812 btf_type_show(btf, type_id, obj, (struct btf_show *)&ssnprintf);
5813
5814
5815 if (ssnprintf.show.state.status)
5816 return ssnprintf.show.state.status;
5817
5818
5819 return ssnprintf.len;
5820}
5821
5822#ifdef CONFIG_PROC_FS
5823static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp)
5824{
5825 const struct btf *btf = filp->private_data;
5826
5827 seq_printf(m, "btf_id:\t%u\n", btf->id);
5828}
5829#endif
5830
5831static int btf_release(struct inode *inode, struct file *filp)
5832{
5833 btf_put(filp->private_data);
5834 return 0;
5835}
5836
5837const struct file_operations btf_fops = {
5838#ifdef CONFIG_PROC_FS
5839 .show_fdinfo = bpf_btf_show_fdinfo,
5840#endif
5841 .release = btf_release,
5842};
5843
5844static int __btf_new_fd(struct btf *btf)
5845{
5846 return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC);
5847}
5848
5849int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr)
5850{
5851 struct btf *btf;
5852 int ret;
5853
5854 btf = btf_parse(make_bpfptr(attr->btf, uattr.is_kernel),
5855 attr->btf_size, attr->btf_log_level,
5856 u64_to_user_ptr(attr->btf_log_buf),
5857 attr->btf_log_size);
5858 if (IS_ERR(btf))
5859 return PTR_ERR(btf);
5860
5861 ret = btf_alloc_id(btf);
5862 if (ret) {
5863 btf_free(btf);
5864 return ret;
5865 }
5866
5867
5868
5869
5870
5871
5872
5873 ret = __btf_new_fd(btf);
5874 if (ret < 0)
5875 btf_put(btf);
5876
5877 return ret;
5878}
5879
5880struct btf *btf_get_by_fd(int fd)
5881{
5882 struct btf *btf;
5883 struct fd f;
5884
5885 f = fdget(fd);
5886
5887 if (!f.file)
5888 return ERR_PTR(-EBADF);
5889
5890 if (f.file->f_op != &btf_fops) {
5891 fdput(f);
5892 return ERR_PTR(-EINVAL);
5893 }
5894
5895 btf = f.file->private_data;
5896 refcount_inc(&btf->refcnt);
5897 fdput(f);
5898
5899 return btf;
5900}
5901
5902int btf_get_info_by_fd(const struct btf *btf,
5903 const union bpf_attr *attr,
5904 union bpf_attr __user *uattr)
5905{
5906 struct bpf_btf_info __user *uinfo;
5907 struct bpf_btf_info info;
5908 u32 info_copy, btf_copy;
5909 void __user *ubtf;
5910 char __user *uname;
5911 u32 uinfo_len, uname_len, name_len;
5912 int ret = 0;
5913
5914 uinfo = u64_to_user_ptr(attr->info.info);
5915 uinfo_len = attr->info.info_len;
5916
5917 info_copy = min_t(u32, uinfo_len, sizeof(info));
5918 memset(&info, 0, sizeof(info));
5919 if (copy_from_user(&info, uinfo, info_copy))
5920 return -EFAULT;
5921
5922 info.id = btf->id;
5923 ubtf = u64_to_user_ptr(info.btf);
5924 btf_copy = min_t(u32, btf->data_size, info.btf_size);
5925 if (copy_to_user(ubtf, btf->data, btf_copy))
5926 return -EFAULT;
5927 info.btf_size = btf->data_size;
5928
5929 info.kernel_btf = btf->kernel_btf;
5930
5931 uname = u64_to_user_ptr(info.name);
5932 uname_len = info.name_len;
5933 if (!uname ^ !uname_len)
5934 return -EINVAL;
5935
5936 name_len = strlen(btf->name);
5937 info.name_len = name_len;
5938
5939 if (uname) {
5940 if (uname_len >= name_len + 1) {
5941 if (copy_to_user(uname, btf->name, name_len + 1))
5942 return -EFAULT;
5943 } else {
5944 char zero = '\0';
5945
5946 if (copy_to_user(uname, btf->name, uname_len - 1))
5947 return -EFAULT;
5948 if (put_user(zero, uname + uname_len - 1))
5949 return -EFAULT;
5950
5951 ret = -ENOSPC;
5952 }
5953 }
5954
5955 if (copy_to_user(uinfo, &info, info_copy) ||
5956 put_user(info_copy, &uattr->info.info_len))
5957 return -EFAULT;
5958
5959 return ret;
5960}
5961
5962int btf_get_fd_by_id(u32 id)
5963{
5964 struct btf *btf;
5965 int fd;
5966
5967 rcu_read_lock();
5968 btf = idr_find(&btf_idr, id);
5969 if (!btf || !refcount_inc_not_zero(&btf->refcnt))
5970 btf = ERR_PTR(-ENOENT);
5971 rcu_read_unlock();
5972
5973 if (IS_ERR(btf))
5974 return PTR_ERR(btf);
5975
5976 fd = __btf_new_fd(btf);
5977 if (fd < 0)
5978 btf_put(btf);
5979
5980 return fd;
5981}
5982
5983u32 btf_obj_id(const struct btf *btf)
5984{
5985 return btf->id;
5986}
5987
5988bool btf_is_kernel(const struct btf *btf)
5989{
5990 return btf->kernel_btf;
5991}
5992
5993bool btf_is_module(const struct btf *btf)
5994{
5995 return btf->kernel_btf && strcmp(btf->name, "vmlinux") != 0;
5996}
5997
5998static int btf_id_cmp_func(const void *a, const void *b)
5999{
6000 const int *pa = a, *pb = b;
6001
6002 return *pa - *pb;
6003}
6004
6005bool btf_id_set_contains(const struct btf_id_set *set, u32 id)
6006{
6007 return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL;
6008}
6009
6010#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
6011struct btf_module {
6012 struct list_head list;
6013 struct module *module;
6014 struct btf *btf;
6015 struct bin_attribute *sysfs_attr;
6016};
6017
6018static LIST_HEAD(btf_modules);
6019static DEFINE_MUTEX(btf_module_mutex);
6020
6021static ssize_t
6022btf_module_read(struct file *file, struct kobject *kobj,
6023 struct bin_attribute *bin_attr,
6024 char *buf, loff_t off, size_t len)
6025{
6026 const struct btf *btf = bin_attr->private;
6027
6028 memcpy(buf, btf->data + off, len);
6029 return len;
6030}
6031
6032static int btf_module_notify(struct notifier_block *nb, unsigned long op,
6033 void *module)
6034{
6035 struct btf_module *btf_mod, *tmp;
6036 struct module *mod = module;
6037 struct btf *btf;
6038 int err = 0;
6039
6040 if (mod->btf_data_size == 0 ||
6041 (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING))
6042 goto out;
6043
6044 switch (op) {
6045 case MODULE_STATE_COMING:
6046 btf_mod = kzalloc(sizeof(*btf_mod), GFP_KERNEL);
6047 if (!btf_mod) {
6048 err = -ENOMEM;
6049 goto out;
6050 }
6051 btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size);
6052 if (IS_ERR(btf)) {
6053 pr_warn("failed to validate module [%s] BTF: %ld\n",
6054 mod->name, PTR_ERR(btf));
6055 kfree(btf_mod);
6056 err = PTR_ERR(btf);
6057 goto out;
6058 }
6059 err = btf_alloc_id(btf);
6060 if (err) {
6061 btf_free(btf);
6062 kfree(btf_mod);
6063 goto out;
6064 }
6065
6066 mutex_lock(&btf_module_mutex);
6067 btf_mod->module = module;
6068 btf_mod->btf = btf;
6069 list_add(&btf_mod->list, &btf_modules);
6070 mutex_unlock(&btf_module_mutex);
6071
6072 if (IS_ENABLED(CONFIG_SYSFS)) {
6073 struct bin_attribute *attr;
6074
6075 attr = kzalloc(sizeof(*attr), GFP_KERNEL);
6076 if (!attr)
6077 goto out;
6078
6079 sysfs_bin_attr_init(attr);
6080 attr->attr.name = btf->name;
6081 attr->attr.mode = 0444;
6082 attr->size = btf->data_size;
6083 attr->private = btf;
6084 attr->read = btf_module_read;
6085
6086 err = sysfs_create_bin_file(btf_kobj, attr);
6087 if (err) {
6088 pr_warn("failed to register module [%s] BTF in sysfs: %d\n",
6089 mod->name, err);
6090 kfree(attr);
6091 err = 0;
6092 goto out;
6093 }
6094
6095 btf_mod->sysfs_attr = attr;
6096 }
6097
6098 break;
6099 case MODULE_STATE_GOING:
6100 mutex_lock(&btf_module_mutex);
6101 list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
6102 if (btf_mod->module != module)
6103 continue;
6104
6105 list_del(&btf_mod->list);
6106 if (btf_mod->sysfs_attr)
6107 sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr);
6108 btf_put(btf_mod->btf);
6109 kfree(btf_mod->sysfs_attr);
6110 kfree(btf_mod);
6111 break;
6112 }
6113 mutex_unlock(&btf_module_mutex);
6114 break;
6115 }
6116out:
6117 return notifier_from_errno(err);
6118}
6119
6120static struct notifier_block btf_module_nb = {
6121 .notifier_call = btf_module_notify,
6122};
6123
6124static int __init btf_module_init(void)
6125{
6126 register_module_notifier(&btf_module_nb);
6127 return 0;
6128}
6129
6130fs_initcall(btf_module_init);
6131#endif
6132
6133struct module *btf_try_get_module(const struct btf *btf)
6134{
6135 struct module *res = NULL;
6136#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
6137 struct btf_module *btf_mod, *tmp;
6138
6139 mutex_lock(&btf_module_mutex);
6140 list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
6141 if (btf_mod->btf != btf)
6142 continue;
6143
6144 if (try_module_get(btf_mod->module))
6145 res = btf_mod->module;
6146
6147 break;
6148 }
6149 mutex_unlock(&btf_module_mutex);
6150#endif
6151
6152 return res;
6153}
6154
6155BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int, flags)
6156{
6157 struct btf *btf;
6158 long ret;
6159
6160 if (flags)
6161 return -EINVAL;
6162
6163 if (name_sz <= 1 || name[name_sz - 1])
6164 return -EINVAL;
6165
6166 btf = bpf_get_btf_vmlinux();
6167 if (IS_ERR(btf))
6168 return PTR_ERR(btf);
6169
6170 ret = btf_find_by_name_kind(btf, name, kind);
6171
6172
6173
6174 if (ret < 0) {
6175 struct btf *mod_btf;
6176 int id;
6177
6178
6179 spin_lock_bh(&btf_idr_lock);
6180 idr_for_each_entry(&btf_idr, mod_btf, id) {
6181 if (!btf_is_module(mod_btf))
6182 continue;
6183
6184
6185
6186 btf_get(mod_btf);
6187 spin_unlock_bh(&btf_idr_lock);
6188 ret = btf_find_by_name_kind(mod_btf, name, kind);
6189 if (ret > 0) {
6190 int btf_obj_fd;
6191
6192 btf_obj_fd = __btf_new_fd(mod_btf);
6193 if (btf_obj_fd < 0) {
6194 btf_put(mod_btf);
6195 return btf_obj_fd;
6196 }
6197 return ret | (((u64)btf_obj_fd) << 32);
6198 }
6199 spin_lock_bh(&btf_idr_lock);
6200 btf_put(mod_btf);
6201 }
6202 spin_unlock_bh(&btf_idr_lock);
6203 }
6204 return ret;
6205}
6206
6207const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = {
6208 .func = bpf_btf_find_by_name_kind,
6209 .gpl_only = false,
6210 .ret_type = RET_INTEGER,
6211 .arg1_type = ARG_PTR_TO_MEM,
6212 .arg2_type = ARG_CONST_SIZE,
6213 .arg3_type = ARG_ANYTHING,
6214 .arg4_type = ARG_ANYTHING,
6215};
6216
6217BTF_ID_LIST_GLOBAL_SINGLE(btf_task_struct_ids, struct, task_struct)
6218