1
2
3
4
5
6
7#include <stdbool.h>
8#include <stddef.h>
9#include <stdio.h>
10#include <stdlib.h>
11#include <string.h>
12#include <unistd.h>
13#include <errno.h>
14#include <linux/err.h>
15#include <linux/btf.h>
16#include <elf.h>
17#include <libelf.h>
18#include <gelf.h>
19#include <fcntl.h>
20#include "libbpf.h"
21#include "btf.h"
22#include "libbpf_internal.h"
23#include "strset.h"
24
25#define BTF_EXTERN_SEC ".extern"
26
27struct src_sec {
28 const char *sec_name;
29
30 int id;
31
32 int dst_id;
33
34 int dst_off;
35
36 bool skipped;
37
38 bool ephemeral;
39
40
41 size_t sec_idx;
42 Elf_Scn *scn;
43 Elf64_Shdr *shdr;
44 Elf_Data *data;
45
46
47 int sec_type_id;
48};
49
50struct src_obj {
51 const char *filename;
52 int fd;
53 Elf *elf;
54
55 size_t shstrs_sec_idx;
56
57 size_t symtab_sec_idx;
58
59 struct btf *btf;
60 struct btf_ext *btf_ext;
61
62
63 struct src_sec *secs;
64 int sec_cnt;
65
66
67 int *sym_map;
68
69 int *btf_type_map;
70};
71
72
73struct btf_ext_sec_data {
74 size_t rec_cnt;
75 __u32 rec_sz;
76 void *recs;
77};
78
79struct glob_sym {
80
81 int sym_idx;
82
83 int sec_id;
84
85 int name_off;
86
87 int btf_id;
88
89
90
91
92 int underlying_btf_id;
93
94 int var_idx;
95
96
97 bool is_extern;
98
99 bool is_weak;
100};
101
102struct dst_sec {
103 char *sec_name;
104
105 int id;
106
107 bool ephemeral;
108
109
110 size_t sec_idx;
111 Elf_Scn *scn;
112 Elf64_Shdr *shdr;
113 Elf_Data *data;
114
115
116 int sec_sz;
117
118 void *raw_data;
119
120
121 int sec_sym_idx;
122
123
124 bool has_btf;
125 int sec_var_cnt;
126 struct btf_var_secinfo *sec_vars;
127
128
129 struct btf_ext_sec_data func_info;
130 struct btf_ext_sec_data line_info;
131 struct btf_ext_sec_data core_relo_info;
132};
133
134struct bpf_linker {
135 char *filename;
136 int fd;
137 Elf *elf;
138 Elf64_Ehdr *elf_hdr;
139
140
141 struct dst_sec *secs;
142 int sec_cnt;
143
144 struct strset *strtab_strs;
145 size_t strtab_sec_idx;
146 size_t symtab_sec_idx;
147
148 struct btf *btf;
149 struct btf_ext *btf_ext;
150
151
152 int glob_sym_cnt;
153 struct glob_sym *glob_syms;
154};
155
156#define pr_warn_elf(fmt, ...) \
157 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1))
158
159static int init_output_elf(struct bpf_linker *linker, const char *file);
160
161static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
162 const struct bpf_linker_file_opts *opts,
163 struct src_obj *obj);
164static int linker_sanity_check_elf(struct src_obj *obj);
165static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec);
166static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec);
167static int linker_sanity_check_btf(struct src_obj *obj);
168static int linker_sanity_check_btf_ext(struct src_obj *obj);
169static int linker_fixup_btf(struct src_obj *obj);
170static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj);
171static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj);
172static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
173 Elf64_Sym *sym, const char *sym_name, int src_sym_idx);
174static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj);
175static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj);
176static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj);
177
178static int finalize_btf(struct bpf_linker *linker);
179static int finalize_btf_ext(struct bpf_linker *linker);
180
181void bpf_linker__free(struct bpf_linker *linker)
182{
183 int i;
184
185 if (!linker)
186 return;
187
188 free(linker->filename);
189
190 if (linker->elf)
191 elf_end(linker->elf);
192
193 if (linker->fd >= 0)
194 close(linker->fd);
195
196 strset__free(linker->strtab_strs);
197
198 btf__free(linker->btf);
199 btf_ext__free(linker->btf_ext);
200
201 for (i = 1; i < linker->sec_cnt; i++) {
202 struct dst_sec *sec = &linker->secs[i];
203
204 free(sec->sec_name);
205 free(sec->raw_data);
206 free(sec->sec_vars);
207
208 free(sec->func_info.recs);
209 free(sec->line_info.recs);
210 free(sec->core_relo_info.recs);
211 }
212 free(linker->secs);
213
214 free(linker);
215}
216
217struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts)
218{
219 struct bpf_linker *linker;
220 int err;
221
222 if (!OPTS_VALID(opts, bpf_linker_opts))
223 return errno = EINVAL, NULL;
224
225 if (elf_version(EV_CURRENT) == EV_NONE) {
226 pr_warn_elf("libelf initialization failed");
227 return errno = EINVAL, NULL;
228 }
229
230 linker = calloc(1, sizeof(*linker));
231 if (!linker)
232 return errno = ENOMEM, NULL;
233
234 linker->fd = -1;
235
236 err = init_output_elf(linker, filename);
237 if (err)
238 goto err_out;
239
240 return linker;
241
242err_out:
243 bpf_linker__free(linker);
244 return errno = -err, NULL;
245}
246
247static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name)
248{
249 struct dst_sec *secs = linker->secs, *sec;
250 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2;
251
252 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
253 if (!secs)
254 return NULL;
255
256
257 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs));
258
259 linker->secs = secs;
260 linker->sec_cnt = new_cnt;
261
262 sec = &linker->secs[new_cnt - 1];
263 sec->id = new_cnt - 1;
264 sec->sec_name = strdup(sec_name);
265 if (!sec->sec_name)
266 return NULL;
267
268 return sec;
269}
270
271static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx)
272{
273 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
274 Elf64_Sym *syms, *sym;
275 size_t sym_cnt = symtab->sec_sz / sizeof(*sym);
276
277 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym));
278 if (!syms)
279 return NULL;
280
281 sym = &syms[sym_cnt];
282 memset(sym, 0, sizeof(*sym));
283
284 symtab->raw_data = syms;
285 symtab->sec_sz += sizeof(*sym);
286 symtab->shdr->sh_size += sizeof(*sym);
287 symtab->data->d_size += sizeof(*sym);
288
289 if (sym_idx)
290 *sym_idx = sym_cnt;
291
292 return sym;
293}
294
295static int init_output_elf(struct bpf_linker *linker, const char *file)
296{
297 int err, str_off;
298 Elf64_Sym *init_sym;
299 struct dst_sec *sec;
300
301 linker->filename = strdup(file);
302 if (!linker->filename)
303 return -ENOMEM;
304
305 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0644);
306 if (linker->fd < 0) {
307 err = -errno;
308 pr_warn("failed to create '%s': %d\n", file, err);
309 return err;
310 }
311
312 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL);
313 if (!linker->elf) {
314 pr_warn_elf("failed to create ELF object");
315 return -EINVAL;
316 }
317
318
319 linker->elf_hdr = elf64_newehdr(linker->elf);
320 if (!linker->elf_hdr) {
321 pr_warn_elf("failed to create ELF header");
322 return -EINVAL;
323 }
324
325 linker->elf_hdr->e_machine = EM_BPF;
326 linker->elf_hdr->e_type = ET_REL;
327#if __BYTE_ORDER == __LITTLE_ENDIAN
328 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;
329#elif __BYTE_ORDER == __BIG_ENDIAN
330 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
331#else
332#error "Unknown __BYTE_ORDER"
333#endif
334
335
336
337 linker->strtab_strs = strset__new(INT_MAX, "", sizeof(""));
338 if (libbpf_get_error(linker->strtab_strs))
339 return libbpf_get_error(linker->strtab_strs);
340
341 sec = add_dst_sec(linker, ".strtab");
342 if (!sec)
343 return -ENOMEM;
344
345 sec->scn = elf_newscn(linker->elf);
346 if (!sec->scn) {
347 pr_warn_elf("failed to create STRTAB section");
348 return -EINVAL;
349 }
350
351 sec->shdr = elf64_getshdr(sec->scn);
352 if (!sec->shdr)
353 return -EINVAL;
354
355 sec->data = elf_newdata(sec->scn);
356 if (!sec->data) {
357 pr_warn_elf("failed to create STRTAB data");
358 return -EINVAL;
359 }
360
361 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
362 if (str_off < 0)
363 return str_off;
364
365 sec->sec_idx = elf_ndxscn(sec->scn);
366 linker->elf_hdr->e_shstrndx = sec->sec_idx;
367 linker->strtab_sec_idx = sec->sec_idx;
368
369 sec->shdr->sh_name = str_off;
370 sec->shdr->sh_type = SHT_STRTAB;
371 sec->shdr->sh_flags = SHF_STRINGS;
372 sec->shdr->sh_offset = 0;
373 sec->shdr->sh_link = 0;
374 sec->shdr->sh_info = 0;
375 sec->shdr->sh_addralign = 1;
376 sec->shdr->sh_size = sec->sec_sz = 0;
377 sec->shdr->sh_entsize = 0;
378
379
380 sec = add_dst_sec(linker, ".symtab");
381 if (!sec)
382 return -ENOMEM;
383
384 sec->scn = elf_newscn(linker->elf);
385 if (!sec->scn) {
386 pr_warn_elf("failed to create SYMTAB section");
387 return -EINVAL;
388 }
389
390 sec->shdr = elf64_getshdr(sec->scn);
391 if (!sec->shdr)
392 return -EINVAL;
393
394 sec->data = elf_newdata(sec->scn);
395 if (!sec->data) {
396 pr_warn_elf("failed to create SYMTAB data");
397 return -EINVAL;
398 }
399
400 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
401 if (str_off < 0)
402 return str_off;
403
404 sec->sec_idx = elf_ndxscn(sec->scn);
405 linker->symtab_sec_idx = sec->sec_idx;
406
407 sec->shdr->sh_name = str_off;
408 sec->shdr->sh_type = SHT_SYMTAB;
409 sec->shdr->sh_flags = 0;
410 sec->shdr->sh_offset = 0;
411 sec->shdr->sh_link = linker->strtab_sec_idx;
412
413
414
415 sec->shdr->sh_info = 0;
416 sec->shdr->sh_addralign = 8;
417 sec->shdr->sh_entsize = sizeof(Elf64_Sym);
418
419
420 linker->btf = btf__new_empty();
421 err = libbpf_get_error(linker->btf);
422 if (err)
423 return err;
424
425
426 init_sym = add_new_sym(linker, NULL);
427 if (!init_sym)
428 return -EINVAL;
429
430 init_sym->st_name = 0;
431 init_sym->st_info = 0;
432 init_sym->st_other = 0;
433 init_sym->st_shndx = SHN_UNDEF;
434 init_sym->st_value = 0;
435 init_sym->st_size = 0;
436
437 return 0;
438}
439
440int bpf_linker__add_file(struct bpf_linker *linker, const char *filename,
441 const struct bpf_linker_file_opts *opts)
442{
443 struct src_obj obj = {};
444 int err = 0;
445
446 if (!OPTS_VALID(opts, bpf_linker_file_opts))
447 return libbpf_err(-EINVAL);
448
449 if (!linker->elf)
450 return libbpf_err(-EINVAL);
451
452 err = err ?: linker_load_obj_file(linker, filename, opts, &obj);
453 err = err ?: linker_append_sec_data(linker, &obj);
454 err = err ?: linker_append_elf_syms(linker, &obj);
455 err = err ?: linker_append_elf_relos(linker, &obj);
456 err = err ?: linker_append_btf(linker, &obj);
457 err = err ?: linker_append_btf_ext(linker, &obj);
458
459
460 free(obj.btf_type_map);
461 btf__free(obj.btf);
462 btf_ext__free(obj.btf_ext);
463 free(obj.secs);
464 free(obj.sym_map);
465 if (obj.elf)
466 elf_end(obj.elf);
467 if (obj.fd >= 0)
468 close(obj.fd);
469
470 return libbpf_err(err);
471}
472
473static bool is_dwarf_sec_name(const char *name)
474{
475
476 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0;
477}
478
479static bool is_ignored_sec(struct src_sec *sec)
480{
481 Elf64_Shdr *shdr = sec->shdr;
482 const char *name = sec->sec_name;
483
484
485 if (shdr->sh_type == SHT_STRTAB)
486 return true;
487
488
489 if (shdr->sh_type == SHT_LLVM_ADDRSIG)
490 return true;
491
492
493 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 &&
494 strcmp(sec->sec_name, ".text") == 0)
495 return true;
496
497
498 if (is_dwarf_sec_name(sec->sec_name))
499 return true;
500
501 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) {
502 name += sizeof(".rel") - 1;
503
504 if (is_dwarf_sec_name(name))
505 return true;
506
507
508 if (strcmp(name, BTF_ELF_SEC) == 0 ||
509 strcmp(name, BTF_EXT_ELF_SEC) == 0)
510 return true;
511 }
512
513 return false;
514}
515
516static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name)
517{
518 struct src_sec *secs = obj->secs, *sec;
519 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2;
520
521 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs));
522 if (!secs)
523 return NULL;
524
525
526 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs));
527
528 obj->secs = secs;
529 obj->sec_cnt = new_cnt;
530
531 sec = &obj->secs[new_cnt - 1];
532 sec->id = new_cnt - 1;
533 sec->sec_name = sec_name;
534
535 return sec;
536}
537
538static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
539 const struct bpf_linker_file_opts *opts,
540 struct src_obj *obj)
541{
542#if __BYTE_ORDER == __LITTLE_ENDIAN
543 const int host_endianness = ELFDATA2LSB;
544#elif __BYTE_ORDER == __BIG_ENDIAN
545 const int host_endianness = ELFDATA2MSB;
546#else
547#error "Unknown __BYTE_ORDER"
548#endif
549 int err = 0;
550 Elf_Scn *scn;
551 Elf_Data *data;
552 Elf64_Ehdr *ehdr;
553 Elf64_Shdr *shdr;
554 struct src_sec *sec;
555
556 pr_debug("linker: adding object file '%s'...\n", filename);
557
558 obj->filename = filename;
559
560 obj->fd = open(filename, O_RDONLY);
561 if (obj->fd < 0) {
562 err = -errno;
563 pr_warn("failed to open file '%s': %d\n", filename, err);
564 return err;
565 }
566 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
567 if (!obj->elf) {
568 err = -errno;
569 pr_warn_elf("failed to parse ELF file '%s'", filename);
570 return err;
571 }
572
573
574 ehdr = elf64_getehdr(obj->elf);
575 if (!ehdr) {
576 err = -errno;
577 pr_warn_elf("failed to get ELF header for %s", filename);
578 return err;
579 }
580 if (ehdr->e_ident[EI_DATA] != host_endianness) {
581 err = -EOPNOTSUPP;
582 pr_warn_elf("unsupported byte order of ELF file %s", filename);
583 return err;
584 }
585 if (ehdr->e_type != ET_REL
586 || ehdr->e_machine != EM_BPF
587 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
588 err = -EOPNOTSUPP;
589 pr_warn_elf("unsupported kind of ELF file %s", filename);
590 return err;
591 }
592
593 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) {
594 err = -errno;
595 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename);
596 return err;
597 }
598
599 scn = NULL;
600 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) {
601 size_t sec_idx = elf_ndxscn(scn);
602 const char *sec_name;
603
604 shdr = elf64_getshdr(scn);
605 if (!shdr) {
606 err = -errno;
607 pr_warn_elf("failed to get section #%zu header for %s",
608 sec_idx, filename);
609 return err;
610 }
611
612 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name);
613 if (!sec_name) {
614 err = -errno;
615 pr_warn_elf("failed to get section #%zu name for %s",
616 sec_idx, filename);
617 return err;
618 }
619
620 data = elf_getdata(scn, 0);
621 if (!data) {
622 err = -errno;
623 pr_warn_elf("failed to get section #%zu (%s) data from %s",
624 sec_idx, sec_name, filename);
625 return err;
626 }
627
628 sec = add_src_sec(obj, sec_name);
629 if (!sec)
630 return -ENOMEM;
631
632 sec->scn = scn;
633 sec->shdr = shdr;
634 sec->data = data;
635 sec->sec_idx = elf_ndxscn(scn);
636
637 if (is_ignored_sec(sec)) {
638 sec->skipped = true;
639 continue;
640 }
641
642 switch (shdr->sh_type) {
643 case SHT_SYMTAB:
644 if (obj->symtab_sec_idx) {
645 err = -EOPNOTSUPP;
646 pr_warn("multiple SYMTAB sections found, not supported\n");
647 return err;
648 }
649 obj->symtab_sec_idx = sec_idx;
650 break;
651 case SHT_STRTAB:
652
653 break;
654 case SHT_PROGBITS:
655 if (strcmp(sec_name, BTF_ELF_SEC) == 0) {
656 obj->btf = btf__new(data->d_buf, shdr->sh_size);
657 err = libbpf_get_error(obj->btf);
658 if (err) {
659 pr_warn("failed to parse .BTF from %s: %d\n", filename, err);
660 return err;
661 }
662 sec->skipped = true;
663 continue;
664 }
665 if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) {
666 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
667 err = libbpf_get_error(obj->btf_ext);
668 if (err) {
669 pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);
670 return err;
671 }
672 sec->skipped = true;
673 continue;
674 }
675
676
677 break;
678 case SHT_NOBITS:
679
680 break;
681 case SHT_REL:
682
683 break;
684 default:
685 pr_warn("unrecognized section #%zu (%s) in %s\n",
686 sec_idx, sec_name, filename);
687 err = -EINVAL;
688 return err;
689 }
690 }
691
692 err = err ?: linker_sanity_check_elf(obj);
693 err = err ?: linker_sanity_check_btf(obj);
694 err = err ?: linker_sanity_check_btf_ext(obj);
695 err = err ?: linker_fixup_btf(obj);
696
697 return err;
698}
699
700static bool is_pow_of_2(size_t x)
701{
702 return x && (x & (x - 1)) == 0;
703}
704
705static int linker_sanity_check_elf(struct src_obj *obj)
706{
707 struct src_sec *sec;
708 int i, err;
709
710 if (!obj->symtab_sec_idx) {
711 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename);
712 return -EINVAL;
713 }
714 if (!obj->shstrs_sec_idx) {
715 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename);
716 return -EINVAL;
717 }
718
719 for (i = 1; i < obj->sec_cnt; i++) {
720 sec = &obj->secs[i];
721
722 if (sec->sec_name[0] == '\0') {
723 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename);
724 return -EINVAL;
725 }
726
727 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign))
728 return -EINVAL;
729 if (sec->shdr->sh_addralign != sec->data->d_align)
730 return -EINVAL;
731
732 if (sec->shdr->sh_size != sec->data->d_size)
733 return -EINVAL;
734
735 switch (sec->shdr->sh_type) {
736 case SHT_SYMTAB:
737 err = linker_sanity_check_elf_symtab(obj, sec);
738 if (err)
739 return err;
740 break;
741 case SHT_STRTAB:
742 break;
743 case SHT_PROGBITS:
744 if (sec->shdr->sh_flags & SHF_EXECINSTR) {
745 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0)
746 return -EINVAL;
747 }
748 break;
749 case SHT_NOBITS:
750 break;
751 case SHT_REL:
752 err = linker_sanity_check_elf_relos(obj, sec);
753 if (err)
754 return err;
755 break;
756 case SHT_LLVM_ADDRSIG:
757 break;
758 default:
759 pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n",
760 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename);
761 return -EINVAL;
762 }
763 }
764
765 return 0;
766}
767
768static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec)
769{
770 struct src_sec *link_sec;
771 Elf64_Sym *sym;
772 int i, n;
773
774 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym))
775 return -EINVAL;
776 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
777 return -EINVAL;
778
779 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) {
780 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n",
781 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
782 return -EINVAL;
783 }
784 link_sec = &obj->secs[sec->shdr->sh_link];
785 if (link_sec->shdr->sh_type != SHT_STRTAB) {
786 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n",
787 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
788 return -EINVAL;
789 }
790
791 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
792 sym = sec->data->d_buf;
793 for (i = 0; i < n; i++, sym++) {
794 int sym_type = ELF64_ST_TYPE(sym->st_info);
795 int sym_bind = ELF64_ST_BIND(sym->st_info);
796 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
797
798 if (i == 0) {
799 if (sym->st_name != 0 || sym->st_info != 0
800 || sym->st_other != 0 || sym->st_shndx != 0
801 || sym->st_value != 0 || sym->st_size != 0) {
802 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename);
803 return -EINVAL;
804 }
805 continue;
806 }
807 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) {
808 pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n",
809 i, sec->sec_idx, sym_bind);
810 return -EINVAL;
811 }
812 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) {
813 pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n",
814 i, sec->sec_idx, sym_vis);
815 return -EINVAL;
816 }
817 if (sym->st_shndx == 0) {
818 if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL
819 || sym->st_value != 0 || sym->st_size != 0) {
820 pr_warn("ELF sym #%d is invalid extern symbol in %s\n",
821 i, obj->filename);
822
823 return -EINVAL;
824 }
825 continue;
826 }
827 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) {
828 pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n",
829 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename);
830 return -EINVAL;
831 }
832 if (sym_type == STT_SECTION) {
833 if (sym->st_value != 0)
834 return -EINVAL;
835 continue;
836 }
837 }
838
839 return 0;
840}
841
842static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec)
843{
844 struct src_sec *link_sec, *sym_sec;
845 Elf64_Rel *relo;
846 int i, n;
847
848 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel))
849 return -EINVAL;
850 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0)
851 return -EINVAL;
852
853
854 if (sec->shdr->sh_link != obj->symtab_sec_idx) {
855 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n",
856 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename);
857 return -EINVAL;
858 }
859
860
861 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) {
862 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n",
863 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
864 return -EINVAL;
865 }
866 link_sec = &obj->secs[sec->shdr->sh_info];
867
868
869 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0
870 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) {
871 pr_warn("ELF relo section #%zu name has invalid name in %s\n",
872 sec->sec_idx, obj->filename);
873 return -EINVAL;
874 }
875
876
877 if (link_sec->skipped)
878 return 0;
879
880
881 if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) {
882 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n",
883 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename);
884 return -EINVAL;
885 }
886
887
888 n = sec->shdr->sh_size / sec->shdr->sh_entsize;
889 relo = sec->data->d_buf;
890 sym_sec = &obj->secs[obj->symtab_sec_idx];
891 for (i = 0; i < n; i++, relo++) {
892 size_t sym_idx = ELF64_R_SYM(relo->r_info);
893 size_t sym_type = ELF64_R_TYPE(relo->r_info);
894
895 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 &&
896 sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) {
897 pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n",
898 i, sec->sec_idx, sym_type, obj->filename);
899 return -EINVAL;
900 }
901
902 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) {
903 pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n",
904 i, sec->sec_idx, sym_idx, obj->filename);
905 return -EINVAL;
906 }
907
908 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) {
909 if (relo->r_offset % sizeof(struct bpf_insn) != 0) {
910 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n",
911 i, sec->sec_idx, sym_idx, obj->filename);
912 return -EINVAL;
913 }
914 }
915 }
916
917 return 0;
918}
919
920static int check_btf_type_id(__u32 *type_id, void *ctx)
921{
922 struct btf *btf = ctx;
923
924 if (*type_id > btf__get_nr_types(btf))
925 return -EINVAL;
926
927 return 0;
928}
929
930static int check_btf_str_off(__u32 *str_off, void *ctx)
931{
932 struct btf *btf = ctx;
933 const char *s;
934
935 s = btf__str_by_offset(btf, *str_off);
936
937 if (!s)
938 return -EINVAL;
939
940 return 0;
941}
942
943static int linker_sanity_check_btf(struct src_obj *obj)
944{
945 struct btf_type *t;
946 int i, n, err = 0;
947
948 if (!obj->btf)
949 return 0;
950
951 n = btf__get_nr_types(obj->btf);
952 for (i = 1; i <= n; i++) {
953 t = btf_type_by_id(obj->btf, i);
954
955 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf);
956 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf);
957 if (err)
958 return err;
959 }
960
961 return 0;
962}
963
964static int linker_sanity_check_btf_ext(struct src_obj *obj)
965{
966 int err = 0;
967
968 if (!obj->btf_ext)
969 return 0;
970
971
972 if (!obj->btf)
973 return -EINVAL;
974
975 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf);
976 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf);
977 if (err)
978 return err;
979
980 return 0;
981}
982
983static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec)
984{
985 Elf_Scn *scn;
986 Elf_Data *data;
987 Elf64_Shdr *shdr;
988 int name_off;
989
990 dst_sec->sec_sz = 0;
991 dst_sec->sec_idx = 0;
992 dst_sec->ephemeral = src_sec->ephemeral;
993
994
995 if (src_sec->ephemeral)
996 return 0;
997
998 scn = elf_newscn(linker->elf);
999 if (!scn)
1000 return -ENOMEM;
1001 data = elf_newdata(scn);
1002 if (!data)
1003 return -ENOMEM;
1004 shdr = elf64_getshdr(scn);
1005 if (!shdr)
1006 return -ENOMEM;
1007
1008 dst_sec->scn = scn;
1009 dst_sec->shdr = shdr;
1010 dst_sec->data = data;
1011 dst_sec->sec_idx = elf_ndxscn(scn);
1012
1013 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name);
1014 if (name_off < 0)
1015 return name_off;
1016
1017 shdr->sh_name = name_off;
1018 shdr->sh_type = src_sec->shdr->sh_type;
1019 shdr->sh_flags = src_sec->shdr->sh_flags;
1020 shdr->sh_size = 0;
1021
1022
1023
1024
1025 shdr->sh_link = 0;
1026 shdr->sh_info = 0;
1027 shdr->sh_addralign = src_sec->shdr->sh_addralign;
1028 shdr->sh_entsize = src_sec->shdr->sh_entsize;
1029
1030 data->d_type = src_sec->data->d_type;
1031 data->d_size = 0;
1032 data->d_buf = NULL;
1033 data->d_align = src_sec->data->d_align;
1034 data->d_off = 0;
1035
1036 return 0;
1037}
1038
1039static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name)
1040{
1041 struct dst_sec *sec;
1042 int i;
1043
1044 for (i = 1; i < linker->sec_cnt; i++) {
1045 sec = &linker->secs[i];
1046
1047 if (strcmp(sec->sec_name, sec_name) == 0)
1048 return sec;
1049 }
1050
1051 return NULL;
1052}
1053
1054static bool secs_match(struct dst_sec *dst, struct src_sec *src)
1055{
1056 if (dst->ephemeral || src->ephemeral)
1057 return true;
1058
1059 if (dst->shdr->sh_type != src->shdr->sh_type) {
1060 pr_warn("sec %s types mismatch\n", dst->sec_name);
1061 return false;
1062 }
1063 if (dst->shdr->sh_flags != src->shdr->sh_flags) {
1064 pr_warn("sec %s flags mismatch\n", dst->sec_name);
1065 return false;
1066 }
1067 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) {
1068 pr_warn("sec %s entsize mismatch\n", dst->sec_name);
1069 return false;
1070 }
1071
1072 return true;
1073}
1074
1075static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec)
1076{
1077 if (dst_sec->sec_sz != src_sec->shdr->sh_size)
1078 return false;
1079 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0)
1080 return false;
1081 return true;
1082}
1083
1084static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1085{
1086 void *tmp;
1087 size_t dst_align, src_align;
1088 size_t dst_align_sz, dst_final_sz;
1089 int err;
1090
1091
1092
1093
1094 if (src->ephemeral)
1095 return 0;
1096
1097
1098
1099
1100
1101
1102
1103 if (dst->ephemeral) {
1104 err = init_sec(linker, dst, src);
1105 if (err)
1106 return err;
1107 }
1108
1109 dst_align = dst->shdr->sh_addralign;
1110 src_align = src->shdr->sh_addralign;
1111 if (dst_align == 0)
1112 dst_align = 1;
1113 if (dst_align < src_align)
1114 dst_align = src_align;
1115
1116 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align;
1117
1118
1119 dst_final_sz = dst_align_sz + src->shdr->sh_size;
1120
1121 if (src->shdr->sh_type != SHT_NOBITS) {
1122 tmp = realloc(dst->raw_data, dst_final_sz);
1123 if (!tmp)
1124 return -ENOMEM;
1125 dst->raw_data = tmp;
1126
1127
1128 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1129
1130 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1131 }
1132
1133 dst->sec_sz = dst_final_sz;
1134 dst->shdr->sh_size = dst_final_sz;
1135 dst->data->d_size = dst_final_sz;
1136
1137 dst->shdr->sh_addralign = dst_align;
1138 dst->data->d_align = dst_align;
1139
1140 src->dst_off = dst_align_sz;
1141
1142 return 0;
1143}
1144
1145static bool is_data_sec(struct src_sec *sec)
1146{
1147 if (!sec || sec->skipped)
1148 return false;
1149
1150 if (sec->ephemeral)
1151 return true;
1152 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS;
1153}
1154
1155static bool is_relo_sec(struct src_sec *sec)
1156{
1157 if (!sec || sec->skipped || sec->ephemeral)
1158 return false;
1159 return sec->shdr->sh_type == SHT_REL;
1160}
1161
1162static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj)
1163{
1164 int i, err;
1165
1166 for (i = 1; i < obj->sec_cnt; i++) {
1167 struct src_sec *src_sec;
1168 struct dst_sec *dst_sec;
1169
1170 src_sec = &obj->secs[i];
1171 if (!is_data_sec(src_sec))
1172 continue;
1173
1174 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
1175 if (!dst_sec) {
1176 dst_sec = add_dst_sec(linker, src_sec->sec_name);
1177 if (!dst_sec)
1178 return -ENOMEM;
1179 err = init_sec(linker, dst_sec, src_sec);
1180 if (err) {
1181 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
1182 return err;
1183 }
1184 } else {
1185 if (!secs_match(dst_sec, src_sec)) {
1186 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name);
1187 return -1;
1188 }
1189
1190
1191 if (strcmp(src_sec->sec_name, "license") == 0
1192 || strcmp(src_sec->sec_name, "version") == 0) {
1193 if (!sec_content_is_same(dst_sec, src_sec)) {
1194 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name);
1195 return -EINVAL;
1196 }
1197 src_sec->skipped = true;
1198 src_sec->dst_id = dst_sec->id;
1199 continue;
1200 }
1201 }
1202
1203
1204 src_sec->dst_id = dst_sec->id;
1205
1206 err = extend_sec(linker, dst_sec, src_sec);
1207 if (err)
1208 return err;
1209 }
1210
1211 return 0;
1212}
1213
1214static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj)
1215{
1216 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
1217 Elf64_Sym *sym = symtab->data->d_buf;
1218 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err;
1219 int str_sec_idx = symtab->shdr->sh_link;
1220 const char *sym_name;
1221
1222 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map));
1223 if (!obj->sym_map)
1224 return -ENOMEM;
1225
1226 for (i = 0; i < n; i++, sym++) {
1227
1228
1229
1230 if (i == 0)
1231 continue;
1232
1233 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
1234 if (!sym_name) {
1235 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename);
1236 return -EINVAL;
1237 }
1238
1239 err = linker_append_elf_sym(linker, obj, sym, sym_name, i);
1240 if (err)
1241 return err;
1242 }
1243
1244 return 0;
1245}
1246
1247static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx)
1248{
1249 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx];
1250 Elf64_Sym *syms = symtab->raw_data;
1251
1252 return &syms[sym_idx];
1253}
1254
1255static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name)
1256{
1257 struct glob_sym *glob_sym;
1258 const char *name;
1259 int i;
1260
1261 for (i = 0; i < linker->glob_sym_cnt; i++) {
1262 glob_sym = &linker->glob_syms[i];
1263 name = strset__data(linker->strtab_strs) + glob_sym->name_off;
1264
1265 if (strcmp(name, sym_name) == 0)
1266 return glob_sym;
1267 }
1268
1269 return NULL;
1270}
1271
1272static struct glob_sym *add_glob_sym(struct bpf_linker *linker)
1273{
1274 struct glob_sym *syms, *sym;
1275
1276 syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1,
1277 sizeof(*linker->glob_syms));
1278 if (!syms)
1279 return NULL;
1280
1281 sym = &syms[linker->glob_sym_cnt];
1282 memset(sym, 0, sizeof(*sym));
1283 sym->var_idx = -1;
1284
1285 linker->glob_syms = syms;
1286 linker->glob_sym_cnt++;
1287
1288 return sym;
1289}
1290
1291static bool glob_sym_btf_matches(const char *sym_name, bool exact,
1292 const struct btf *btf1, __u32 id1,
1293 const struct btf *btf2, __u32 id2)
1294{
1295 const struct btf_type *t1, *t2;
1296 bool is_static1, is_static2;
1297 const char *n1, *n2;
1298 int i, n;
1299
1300recur:
1301 n1 = n2 = NULL;
1302 t1 = skip_mods_and_typedefs(btf1, id1, &id1);
1303 t2 = skip_mods_and_typedefs(btf2, id2, &id2);
1304
1305
1306 if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) {
1307 n1 = btf__str_by_offset(btf1, t1->name_off);
1308 n2 = btf__str_by_offset(btf2, t2->name_off);
1309 if (strcmp(n1, n2) != 0) {
1310 pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n",
1311 sym_name, n1, n2);
1312 return false;
1313 }
1314
1315 if (btf_is_fwd(t1)) {
1316 if (btf_kflag(t1) && btf_is_union(t2))
1317 return true;
1318 if (!btf_kflag(t1) && btf_is_struct(t2))
1319 return true;
1320 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1321 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2));
1322 } else {
1323 if (btf_kflag(t2) && btf_is_union(t1))
1324 return true;
1325 if (!btf_kflag(t2) && btf_is_struct(t1))
1326 return true;
1327 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n",
1328 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1));
1329 }
1330 return false;
1331 }
1332
1333 if (btf_kind(t1) != btf_kind(t2)) {
1334 pr_warn("global '%s': incompatible BTF kinds %s and %s\n",
1335 sym_name, btf_kind_str(t1), btf_kind_str(t2));
1336 return false;
1337 }
1338
1339 switch (btf_kind(t1)) {
1340 case BTF_KIND_STRUCT:
1341 case BTF_KIND_UNION:
1342 case BTF_KIND_ENUM:
1343 case BTF_KIND_FWD:
1344 case BTF_KIND_FUNC:
1345 case BTF_KIND_VAR:
1346 n1 = btf__str_by_offset(btf1, t1->name_off);
1347 n2 = btf__str_by_offset(btf2, t2->name_off);
1348 if (strcmp(n1, n2) != 0) {
1349 pr_warn("global '%s': incompatible %s names '%s' and '%s'\n",
1350 sym_name, btf_kind_str(t1), n1, n2);
1351 return false;
1352 }
1353 break;
1354 default:
1355 break;
1356 }
1357
1358 switch (btf_kind(t1)) {
1359 case BTF_KIND_UNKN:
1360 case BTF_KIND_FWD:
1361 return true;
1362 case BTF_KIND_INT:
1363 case BTF_KIND_FLOAT:
1364 case BTF_KIND_ENUM:
1365
1366 if (t1->size != t2->size) {
1367 pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
1368 sym_name, btf_kind_str(t1), n1, t1->size, t2->size);
1369 return false;
1370 }
1371 return true;
1372 case BTF_KIND_PTR:
1373
1374
1375
1376
1377 exact = false;
1378 id1 = t1->type;
1379 id2 = t2->type;
1380 goto recur;
1381 case BTF_KIND_ARRAY:
1382
1383 id1 = btf_array(t1)->type;
1384 id2 = btf_array(t2)->type;
1385 goto recur;
1386 case BTF_KIND_FUNC:
1387
1388 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC;
1389 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC;
1390 if (is_static1 != is_static2) {
1391 pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1);
1392 return false;
1393 }
1394
1395 id1 = t1->type;
1396 id2 = t2->type;
1397 goto recur;
1398 case BTF_KIND_VAR:
1399
1400 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC;
1401 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC;
1402 if (is_static1 != is_static2) {
1403 pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1);
1404 return false;
1405 }
1406
1407 id1 = t1->type;
1408 id2 = t2->type;
1409 goto recur;
1410 case BTF_KIND_STRUCT:
1411 case BTF_KIND_UNION: {
1412 const struct btf_member *m1, *m2;
1413
1414 if (!exact)
1415 return true;
1416
1417 if (btf_vlen(t1) != btf_vlen(t2)) {
1418 pr_warn("global '%s': incompatible number of %s fields %u and %u\n",
1419 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1420 return false;
1421 }
1422
1423 n = btf_vlen(t1);
1424 m1 = btf_members(t1);
1425 m2 = btf_members(t2);
1426 for (i = 0; i < n; i++, m1++, m2++) {
1427 n1 = btf__str_by_offset(btf1, m1->name_off);
1428 n2 = btf__str_by_offset(btf2, m2->name_off);
1429 if (strcmp(n1, n2) != 0) {
1430 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n",
1431 sym_name, i, n1, n2);
1432 return false;
1433 }
1434 if (m1->offset != m2->offset) {
1435 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n",
1436 sym_name, i, n1);
1437 return false;
1438 }
1439 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1440 return false;
1441 }
1442
1443 return true;
1444 }
1445 case BTF_KIND_FUNC_PROTO: {
1446 const struct btf_param *m1, *m2;
1447
1448 if (btf_vlen(t1) != btf_vlen(t2)) {
1449 pr_warn("global '%s': incompatible number of %s params %u and %u\n",
1450 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2));
1451 return false;
1452 }
1453
1454 n = btf_vlen(t1);
1455 m1 = btf_params(t1);
1456 m2 = btf_params(t2);
1457 for (i = 0; i < n; i++, m1++, m2++) {
1458
1459 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type))
1460 return false;
1461 }
1462
1463
1464 id1 = t1->type;
1465 id2 = t2->type;
1466 goto recur;
1467 }
1468
1469
1470 case BTF_KIND_TYPEDEF:
1471 case BTF_KIND_VOLATILE:
1472 case BTF_KIND_CONST:
1473 case BTF_KIND_RESTRICT:
1474
1475 case BTF_KIND_DATASEC:
1476 default:
1477 pr_warn("global '%s': unsupported BTF kind %s\n",
1478 sym_name, btf_kind_str(t1));
1479 return false;
1480 }
1481}
1482
1483static bool map_defs_match(const char *sym_name,
1484 const struct btf *main_btf,
1485 const struct btf_map_def *main_def,
1486 const struct btf_map_def *main_inner_def,
1487 const struct btf *extra_btf,
1488 const struct btf_map_def *extra_def,
1489 const struct btf_map_def *extra_inner_def)
1490{
1491 const char *reason;
1492
1493 if (main_def->map_type != extra_def->map_type) {
1494 reason = "type";
1495 goto mismatch;
1496 }
1497
1498
1499 if (main_def->key_size != extra_def->key_size) {
1500 reason = "key_size";
1501 goto mismatch;
1502 }
1503 if (!!main_def->key_type_id != !!extra_def->key_type_id) {
1504 reason = "key type";
1505 goto mismatch;
1506 }
1507 if ((main_def->parts & MAP_DEF_KEY_TYPE)
1508 && !glob_sym_btf_matches(sym_name, true ,
1509 main_btf, main_def->key_type_id,
1510 extra_btf, extra_def->key_type_id)) {
1511 reason = "key type";
1512 goto mismatch;
1513 }
1514
1515
1516 if (main_def->value_size != extra_def->value_size) {
1517 reason = "value_size";
1518 goto mismatch;
1519 }
1520 if (!!main_def->value_type_id != !!extra_def->value_type_id) {
1521 reason = "value type";
1522 goto mismatch;
1523 }
1524 if ((main_def->parts & MAP_DEF_VALUE_TYPE)
1525 && !glob_sym_btf_matches(sym_name, true ,
1526 main_btf, main_def->value_type_id,
1527 extra_btf, extra_def->value_type_id)) {
1528 reason = "key type";
1529 goto mismatch;
1530 }
1531
1532 if (main_def->max_entries != extra_def->max_entries) {
1533 reason = "max_entries";
1534 goto mismatch;
1535 }
1536 if (main_def->map_flags != extra_def->map_flags) {
1537 reason = "map_flags";
1538 goto mismatch;
1539 }
1540 if (main_def->numa_node != extra_def->numa_node) {
1541 reason = "numa_node";
1542 goto mismatch;
1543 }
1544 if (main_def->pinning != extra_def->pinning) {
1545 reason = "pinning";
1546 goto mismatch;
1547 }
1548
1549 if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) {
1550 reason = "inner map";
1551 goto mismatch;
1552 }
1553
1554 if (main_def->parts & MAP_DEF_INNER_MAP) {
1555 char inner_map_name[128];
1556
1557 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name);
1558
1559 return map_defs_match(inner_map_name,
1560 main_btf, main_inner_def, NULL,
1561 extra_btf, extra_inner_def, NULL);
1562 }
1563
1564 return true;
1565
1566mismatch:
1567 pr_warn("global '%s': map %s mismatch\n", sym_name, reason);
1568 return false;
1569}
1570
1571static bool glob_map_defs_match(const char *sym_name,
1572 struct bpf_linker *linker, struct glob_sym *glob_sym,
1573 struct src_obj *obj, Elf64_Sym *sym, int btf_id)
1574{
1575 struct btf_map_def dst_def = {}, dst_inner_def = {};
1576 struct btf_map_def src_def = {}, src_inner_def = {};
1577 const struct btf_type *t;
1578 int err;
1579
1580 t = btf__type_by_id(obj->btf, btf_id);
1581 if (!btf_is_var(t)) {
1582 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id);
1583 return false;
1584 }
1585 t = skip_mods_and_typedefs(obj->btf, t->type, NULL);
1586
1587 err = parse_btf_map_def(sym_name, obj->btf, t, true , &src_def, &src_inner_def);
1588 if (err) {
1589 pr_warn("global '%s': invalid map definition\n", sym_name);
1590 return false;
1591 }
1592
1593
1594 t = btf__type_by_id(linker->btf, glob_sym->btf_id);
1595 t = skip_mods_and_typedefs(linker->btf, t->type, NULL);
1596 err = parse_btf_map_def(sym_name, linker->btf, t, true , &dst_def, &dst_inner_def);
1597 if (err) {
1598
1599 pr_warn("global '%s': invalid dst map definition\n", sym_name);
1600 return false;
1601 }
1602
1603
1604
1605
1606
1607 return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def,
1608 obj->btf, &src_def, &src_inner_def);
1609}
1610
1611static bool glob_syms_match(const char *sym_name,
1612 struct bpf_linker *linker, struct glob_sym *glob_sym,
1613 struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id)
1614{
1615 const struct btf_type *src_t;
1616
1617
1618
1619
1620 if (!glob_sym->btf_id || !btf_id) {
1621 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name);
1622 return false;
1623 }
1624
1625 src_t = btf__type_by_id(obj->btf, btf_id);
1626 if (!btf_is_var(src_t) && !btf_is_func(src_t)) {
1627 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n",
1628 btf_kind_str(src_t), sym_name);
1629 return false;
1630 }
1631
1632
1633 if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0)
1634 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id);
1635
1636 if (!glob_sym_btf_matches(sym_name, true ,
1637 linker->btf, glob_sym->btf_id, obj->btf, btf_id))
1638 return false;
1639
1640 return true;
1641}
1642
1643static bool btf_is_non_static(const struct btf_type *t)
1644{
1645 return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC)
1646 || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC);
1647}
1648
1649static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name,
1650 int *out_btf_sec_id, int *out_btf_id)
1651{
1652 int i, j, n = btf__get_nr_types(obj->btf), m, btf_id = 0;
1653 const struct btf_type *t;
1654 const struct btf_var_secinfo *vi;
1655 const char *name;
1656
1657 for (i = 1; i <= n; i++) {
1658 t = btf__type_by_id(obj->btf, i);
1659
1660
1661
1662
1663 if (btf_is_non_static(t)) {
1664 name = btf__str_by_offset(obj->btf, t->name_off);
1665 if (strcmp(name, sym_name) != 0)
1666 continue;
1667
1668
1669 btf_id = i;
1670 continue;
1671 }
1672
1673 if (!btf_is_datasec(t))
1674 continue;
1675
1676 vi = btf_var_secinfos(t);
1677 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
1678 t = btf__type_by_id(obj->btf, vi->type);
1679 name = btf__str_by_offset(obj->btf, t->name_off);
1680
1681 if (strcmp(name, sym_name) != 0)
1682 continue;
1683 if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC)
1684 continue;
1685 if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC)
1686 continue;
1687
1688 if (btf_id && btf_id != vi->type) {
1689 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n",
1690 sym_name, btf_id, vi->type);
1691 return -EINVAL;
1692 }
1693
1694 *out_btf_sec_id = i;
1695 *out_btf_id = vi->type;
1696
1697 return 0;
1698 }
1699 }
1700
1701
1702 if (btf_id) {
1703 *out_btf_sec_id = 0;
1704 *out_btf_id = btf_id;
1705 return 0;
1706 }
1707
1708 pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name);
1709 return -ENOENT;
1710}
1711
1712static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name)
1713{
1714 struct src_sec *sec;
1715 int i;
1716
1717 for (i = 1; i < obj->sec_cnt; i++) {
1718 sec = &obj->secs[i];
1719
1720 if (strcmp(sec->sec_name, sec_name) == 0)
1721 return sec;
1722 }
1723
1724 return NULL;
1725}
1726
1727static int complete_extern_btf_info(struct btf *dst_btf, int dst_id,
1728 struct btf *src_btf, int src_id)
1729{
1730 struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id);
1731 struct btf_type *src_t = btf_type_by_id(src_btf, src_id);
1732 struct btf_param *src_p, *dst_p;
1733 const char *s;
1734 int i, n, off;
1735
1736
1737
1738
1739 if (btf_is_var(dst_t)) {
1740 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
1741 return 0;
1742 }
1743
1744 dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0);
1745
1746
1747 src_t = btf_type_by_id(src_btf, src_t->type);
1748 dst_t = btf_type_by_id(dst_btf, dst_t->type);
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761 src_p = btf_params(src_t);
1762 dst_p = btf_params(dst_t);
1763 for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) {
1764 if (!src_p->name_off)
1765 continue;
1766
1767
1768 s = btf__str_by_offset(src_btf, src_p->name_off);
1769 off = btf__add_str(dst_btf, s);
1770 if (off < 0)
1771 return off;
1772 dst_p->name_off = off;
1773 }
1774 return 0;
1775}
1776
1777static void sym_update_bind(Elf64_Sym *sym, int sym_bind)
1778{
1779 sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info));
1780}
1781
1782static void sym_update_type(Elf64_Sym *sym, int sym_type)
1783{
1784 sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type);
1785}
1786
1787static void sym_update_visibility(Elf64_Sym *sym, int sym_vis)
1788{
1789
1790
1791
1792 sym->st_other &= ~0x03;
1793 sym->st_other |= sym_vis;
1794}
1795
1796static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj,
1797 Elf64_Sym *sym, const char *sym_name, int src_sym_idx)
1798{
1799 struct src_sec *src_sec = NULL;
1800 struct dst_sec *dst_sec = NULL;
1801 struct glob_sym *glob_sym = NULL;
1802 int name_off, sym_type, sym_bind, sym_vis, err;
1803 int btf_sec_id = 0, btf_id = 0;
1804 size_t dst_sym_idx;
1805 Elf64_Sym *dst_sym;
1806 bool sym_is_extern;
1807
1808 sym_type = ELF64_ST_TYPE(sym->st_info);
1809 sym_bind = ELF64_ST_BIND(sym->st_info);
1810 sym_vis = ELF64_ST_VISIBILITY(sym->st_other);
1811 sym_is_extern = sym->st_shndx == SHN_UNDEF;
1812
1813 if (sym_is_extern) {
1814 if (!obj->btf) {
1815 pr_warn("externs without BTF info are not supported\n");
1816 return -ENOTSUP;
1817 }
1818 } else if (sym->st_shndx < SHN_LORESERVE) {
1819 src_sec = &obj->secs[sym->st_shndx];
1820 if (src_sec->skipped)
1821 return 0;
1822 dst_sec = &linker->secs[src_sec->dst_id];
1823
1824
1825 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) {
1826 obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx;
1827 return 0;
1828 }
1829 }
1830
1831 if (sym_bind == STB_LOCAL)
1832 goto add_sym;
1833
1834
1835 err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id);
1836 if (err)
1837 return err;
1838
1839 if (sym_is_extern && btf_sec_id) {
1840 const char *sec_name = NULL;
1841 const struct btf_type *t;
1842
1843 t = btf__type_by_id(obj->btf, btf_sec_id);
1844 sec_name = btf__str_by_offset(obj->btf, t->name_off);
1845
1846
1847
1848
1849
1850
1851
1852 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) {
1853 src_sec = find_src_sec_by_name(obj, sec_name);
1854 if (!src_sec) {
1855 pr_warn("failed to find matching ELF sec '%s'\n", sec_name);
1856 return -ENOENT;
1857 }
1858 dst_sec = &linker->secs[src_sec->dst_id];
1859 }
1860 }
1861
1862 glob_sym = find_glob_sym(linker, sym_name);
1863 if (glob_sym) {
1864
1865
1866
1867
1868 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1869
1870
1871
1872
1873
1874 if (!sym_is_extern && !glob_sym->is_extern
1875 && !glob_sym->is_weak && sym_bind != STB_WEAK) {
1876 pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n",
1877 src_sym_idx, sym_name, obj->filename);
1878 return -EINVAL;
1879 }
1880
1881 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id))
1882 return -EINVAL;
1883
1884 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx);
1885
1886
1887
1888
1889
1890 if (sym_bind == STB_GLOBAL) {
1891
1892
1893
1894
1895 sym_update_bind(dst_sym, STB_GLOBAL);
1896 glob_sym->is_weak = false;
1897 }
1898
1899
1900
1901
1902
1903
1904
1905 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other))
1906 sym_update_visibility(dst_sym, sym_vis);
1907
1908
1909
1910
1911
1912 if (sym_is_extern)
1913 return 0;
1914
1915
1916
1917
1918
1919
1920
1921 if (!glob_sym->is_extern && sym_bind == STB_WEAK)
1922 return 0;
1923
1924
1925
1926
1927
1928 sym_update_type(dst_sym, sym_type);
1929 dst_sym->st_shndx = dst_sec->sec_idx;
1930 dst_sym->st_value = src_sec->dst_off + sym->st_value;
1931 dst_sym->st_size = sym->st_size;
1932
1933
1934 glob_sym->sec_id = dst_sec->id;
1935 glob_sym->is_extern = false;
1936
1937 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id,
1938 obj->btf, btf_id))
1939 return -EINVAL;
1940
1941
1942 glob_sym->underlying_btf_id = 0;
1943
1944 obj->sym_map[src_sym_idx] = glob_sym->sym_idx;
1945 return 0;
1946 }
1947
1948add_sym:
1949 name_off = strset__add_str(linker->strtab_strs, sym_name);
1950 if (name_off < 0)
1951 return name_off;
1952
1953 dst_sym = add_new_sym(linker, &dst_sym_idx);
1954 if (!dst_sym)
1955 return -ENOMEM;
1956
1957 dst_sym->st_name = name_off;
1958 dst_sym->st_info = sym->st_info;
1959 dst_sym->st_other = sym->st_other;
1960 dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx;
1961 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value;
1962 dst_sym->st_size = sym->st_size;
1963
1964 obj->sym_map[src_sym_idx] = dst_sym_idx;
1965
1966 if (sym_type == STT_SECTION && dst_sym) {
1967 dst_sec->sec_sym_idx = dst_sym_idx;
1968 dst_sym->st_value = 0;
1969 }
1970
1971 if (sym_bind != STB_LOCAL) {
1972 glob_sym = add_glob_sym(linker);
1973 if (!glob_sym)
1974 return -ENOMEM;
1975
1976 glob_sym->sym_idx = dst_sym_idx;
1977
1978
1979
1980
1981
1982
1983 glob_sym->sec_id = dst_sec ? dst_sec->id : 0;
1984 glob_sym->name_off = name_off;
1985
1986 glob_sym->btf_id = 0;
1987 glob_sym->is_extern = sym_is_extern;
1988 glob_sym->is_weak = sym_bind == STB_WEAK;
1989 }
1990
1991 return 0;
1992}
1993
1994static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj)
1995{
1996 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx];
1997 struct dst_sec *dst_symtab = &linker->secs[linker->symtab_sec_idx];
1998 int i, err;
1999
2000 for (i = 1; i < obj->sec_cnt; i++) {
2001 struct src_sec *src_sec, *src_linked_sec;
2002 struct dst_sec *dst_sec, *dst_linked_sec;
2003 Elf64_Rel *src_rel, *dst_rel;
2004 int j, n;
2005
2006 src_sec = &obj->secs[i];
2007 if (!is_relo_sec(src_sec))
2008 continue;
2009
2010
2011 src_linked_sec = &obj->secs[src_sec->shdr->sh_info];
2012 if (src_linked_sec->skipped)
2013 continue;
2014
2015 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name);
2016 if (!dst_sec) {
2017 dst_sec = add_dst_sec(linker, src_sec->sec_name);
2018 if (!dst_sec)
2019 return -ENOMEM;
2020 err = init_sec(linker, dst_sec, src_sec);
2021 if (err) {
2022 pr_warn("failed to init section '%s'\n", src_sec->sec_name);
2023 return err;
2024 }
2025 } else if (!secs_match(dst_sec, src_sec)) {
2026 pr_warn("sections %s are not compatible\n", src_sec->sec_name);
2027 return -1;
2028 }
2029
2030
2031 dst_sec->shdr->sh_link = linker->symtab_sec_idx;
2032
2033
2034 dst_linked_sec = &linker->secs[src_linked_sec->dst_id];
2035 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx;
2036
2037 src_sec->dst_id = dst_sec->id;
2038 err = extend_sec(linker, dst_sec, src_sec);
2039 if (err)
2040 return err;
2041
2042 src_rel = src_sec->data->d_buf;
2043 dst_rel = dst_sec->raw_data + src_sec->dst_off;
2044 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize;
2045 for (j = 0; j < n; j++, src_rel++, dst_rel++) {
2046 size_t src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2047 size_t sym_type = ELF64_R_TYPE(src_rel->r_info);
2048 Elf64_Sym *src_sym, *dst_sym;
2049 size_t dst_sym_idx;
2050
2051 src_sym_idx = ELF64_R_SYM(src_rel->r_info);
2052 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx;
2053
2054 dst_sym_idx = obj->sym_map[src_sym_idx];
2055 dst_sym = dst_symtab->raw_data + sizeof(*dst_sym) * dst_sym_idx;
2056 dst_rel->r_offset += src_linked_sec->dst_off;
2057 sym_type = ELF64_R_TYPE(src_rel->r_info);
2058 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type);
2059
2060 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) {
2061 struct src_sec *sec = &obj->secs[src_sym->st_shndx];
2062 struct bpf_insn *insn;
2063
2064 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) {
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074 insn = dst_linked_sec->raw_data + dst_rel->r_offset;
2075 if (insn->code == (BPF_JMP | BPF_CALL))
2076 insn->imm += sec->dst_off / sizeof(struct bpf_insn);
2077 else
2078 insn->imm += sec->dst_off;
2079 } else {
2080 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n");
2081 return -EINVAL;
2082 }
2083 }
2084
2085 }
2086 }
2087
2088 return 0;
2089}
2090
2091static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx,
2092 int sym_type, const char *sym_name)
2093{
2094 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx];
2095 Elf64_Sym *sym = symtab->data->d_buf;
2096 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize;
2097 int str_sec_idx = symtab->shdr->sh_link;
2098 const char *name;
2099
2100 for (i = 0; i < n; i++, sym++) {
2101 if (sym->st_shndx != sec_idx)
2102 continue;
2103 if (ELF64_ST_TYPE(sym->st_info) != sym_type)
2104 continue;
2105
2106 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name);
2107 if (!name)
2108 return NULL;
2109
2110 if (strcmp(sym_name, name) != 0)
2111 continue;
2112
2113 return sym;
2114 }
2115
2116 return NULL;
2117}
2118
2119static int linker_fixup_btf(struct src_obj *obj)
2120{
2121 const char *sec_name;
2122 struct src_sec *sec;
2123 int i, j, n, m;
2124
2125 if (!obj->btf)
2126 return 0;
2127
2128 n = btf__get_nr_types(obj->btf);
2129 for (i = 1; i <= n; i++) {
2130 struct btf_var_secinfo *vi;
2131 struct btf_type *t;
2132
2133 t = btf_type_by_id(obj->btf, i);
2134 if (btf_kind(t) != BTF_KIND_DATASEC)
2135 continue;
2136
2137 sec_name = btf__str_by_offset(obj->btf, t->name_off);
2138 sec = find_src_sec_by_name(obj, sec_name);
2139 if (sec) {
2140
2141 if (sec->shdr)
2142 t->size = sec->shdr->sh_size;
2143 } else {
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168 if (strcmp(sec_name, BTF_EXTERN_SEC) == 0)
2169 continue;
2170
2171 sec = add_src_sec(obj, sec_name);
2172 if (!sec)
2173 return -ENOMEM;
2174
2175 sec->ephemeral = true;
2176 sec->sec_idx = 0;
2177 }
2178
2179
2180 sec->sec_type_id = i;
2181
2182
2183 vi = btf_var_secinfos(t);
2184 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) {
2185 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type);
2186 const char *var_name = btf__str_by_offset(obj->btf, vt->name_off);
2187 int var_linkage = btf_var(vt)->linkage;
2188 Elf64_Sym *sym;
2189
2190
2191 if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED)
2192 continue;
2193
2194 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name);
2195 if (!sym) {
2196 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name);
2197 return -ENOENT;
2198 }
2199
2200 vi->offset = sym->st_value;
2201 }
2202 }
2203
2204 return 0;
2205}
2206
2207static int remap_type_id(__u32 *type_id, void *ctx)
2208{
2209 int *id_map = ctx;
2210 int new_id = id_map[*type_id];
2211
2212
2213 if (new_id == 0 && *type_id != 0) {
2214 pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id);
2215 return -EINVAL;
2216 }
2217
2218 *type_id = id_map[*type_id];
2219
2220 return 0;
2221}
2222
2223static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj)
2224{
2225 const struct btf_type *t;
2226 int i, j, n, start_id, id;
2227 const char *name;
2228
2229 if (!obj->btf)
2230 return 0;
2231
2232 start_id = btf__get_nr_types(linker->btf) + 1;
2233 n = btf__get_nr_types(obj->btf);
2234
2235 obj->btf_type_map = calloc(n + 1, sizeof(int));
2236 if (!obj->btf_type_map)
2237 return -ENOMEM;
2238
2239 for (i = 1; i <= n; i++) {
2240 struct glob_sym *glob_sym = NULL;
2241
2242 t = btf__type_by_id(obj->btf, i);
2243
2244
2245 if (btf_kind(t) == BTF_KIND_DATASEC)
2246 continue;
2247
2248 if (btf_is_non_static(t)) {
2249
2250 name = btf__str_by_offset(obj->btf, t->name_off);
2251 glob_sym = find_glob_sym(linker, name);
2252
2253
2254
2255
2256
2257 if (!glob_sym)
2258 continue;
2259
2260
2261
2262
2263
2264 if (glob_sym->underlying_btf_id == 0)
2265 glob_sym->underlying_btf_id = -t->type;
2266
2267
2268
2269
2270
2271 if (glob_sym->btf_id) {
2272
2273 obj->btf_type_map[i] = glob_sym->btf_id;
2274 continue;
2275 }
2276 }
2277
2278 id = btf__add_type(linker->btf, obj->btf, t);
2279 if (id < 0) {
2280 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename);
2281 return id;
2282 }
2283
2284 obj->btf_type_map[i] = id;
2285
2286
2287 if (glob_sym) {
2288 glob_sym->btf_id = id;
2289 glob_sym->underlying_btf_id = -t->type;
2290 }
2291 }
2292
2293
2294 n = btf__get_nr_types(linker->btf);
2295 for (i = start_id; i <= n; i++) {
2296 struct btf_type *dst_t = btf_type_by_id(linker->btf, i);
2297
2298 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map))
2299 return -EINVAL;
2300 }
2301
2302
2303
2304
2305 for (i = 0; i < linker->glob_sym_cnt; i++) {
2306 struct glob_sym *glob_sym = &linker->glob_syms[i];
2307 struct btf_type *glob_t;
2308
2309 if (glob_sym->underlying_btf_id >= 0)
2310 continue;
2311
2312 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id];
2313
2314 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id);
2315 glob_t->type = glob_sym->underlying_btf_id;
2316 }
2317
2318
2319 for (i = 1; i < obj->sec_cnt; i++) {
2320 struct src_sec *src_sec;
2321 struct dst_sec *dst_sec;
2322 const struct btf_var_secinfo *src_var;
2323 struct btf_var_secinfo *dst_var;
2324
2325 src_sec = &obj->secs[i];
2326 if (!src_sec->sec_type_id || src_sec->skipped)
2327 continue;
2328 dst_sec = &linker->secs[src_sec->dst_id];
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338 dst_sec->has_btf = true;
2339
2340 t = btf__type_by_id(obj->btf, src_sec->sec_type_id);
2341 src_var = btf_var_secinfos(t);
2342 n = btf_vlen(t);
2343 for (j = 0; j < n; j++, src_var++) {
2344 void *sec_vars = dst_sec->sec_vars;
2345 int new_id = obj->btf_type_map[src_var->type];
2346 struct glob_sym *glob_sym = NULL;
2347
2348 t = btf_type_by_id(linker->btf, new_id);
2349 if (btf_is_non_static(t)) {
2350 name = btf__str_by_offset(linker->btf, t->name_off);
2351 glob_sym = find_glob_sym(linker, name);
2352 if (glob_sym->sec_id != dst_sec->id) {
2353 pr_warn("global '%s': section mismatch %d vs %d\n",
2354 name, glob_sym->sec_id, dst_sec->id);
2355 return -EINVAL;
2356 }
2357 }
2358
2359
2360
2361
2362
2363
2364 if (glob_sym && glob_sym->var_idx >= 0) {
2365 __s64 sz;
2366
2367 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2368
2369
2370
2371
2372 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id);
2373 if (sz < 0) {
2374 pr_warn("global '%s': failed to resolve size of underlying type: %d\n",
2375 name, (int)sz);
2376 return -EINVAL;
2377 }
2378 dst_var->size = sz;
2379 continue;
2380 }
2381
2382 sec_vars = libbpf_reallocarray(sec_vars,
2383 dst_sec->sec_var_cnt + 1,
2384 sizeof(*dst_sec->sec_vars));
2385 if (!sec_vars)
2386 return -ENOMEM;
2387
2388 dst_sec->sec_vars = sec_vars;
2389 dst_sec->sec_var_cnt++;
2390
2391 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1];
2392 dst_var->type = obj->btf_type_map[src_var->type];
2393 dst_var->size = src_var->size;
2394 dst_var->offset = src_sec->dst_off + src_var->offset;
2395
2396 if (glob_sym)
2397 glob_sym->var_idx = dst_sec->sec_var_cnt - 1;
2398 }
2399 }
2400
2401 return 0;
2402}
2403
2404static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec)
2405{
2406 void *tmp;
2407
2408 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz);
2409 if (!tmp)
2410 return NULL;
2411 ext_data->recs = tmp;
2412
2413 tmp += ext_data->rec_cnt * ext_data->rec_sz;
2414 memcpy(tmp, src_rec, ext_data->rec_sz);
2415
2416 ext_data->rec_cnt++;
2417
2418 return tmp;
2419}
2420
2421static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj)
2422{
2423 const struct btf_ext_info_sec *ext_sec;
2424 const char *sec_name, *s;
2425 struct src_sec *src_sec;
2426 struct dst_sec *dst_sec;
2427 int rec_sz, str_off, i;
2428
2429 if (!obj->btf_ext)
2430 return 0;
2431
2432 rec_sz = obj->btf_ext->func_info.rec_size;
2433 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) {
2434 struct bpf_func_info_min *src_rec, *dst_rec;
2435
2436 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2437 src_sec = find_src_sec_by_name(obj, sec_name);
2438 if (!src_sec) {
2439 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2440 return -EINVAL;
2441 }
2442 dst_sec = &linker->secs[src_sec->dst_id];
2443
2444 if (dst_sec->func_info.rec_sz == 0)
2445 dst_sec->func_info.rec_sz = rec_sz;
2446 if (dst_sec->func_info.rec_sz != rec_sz) {
2447 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2448 return -EINVAL;
2449 }
2450
2451 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) {
2452 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec);
2453 if (!dst_rec)
2454 return -ENOMEM;
2455
2456 dst_rec->insn_off += src_sec->dst_off;
2457 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2458 }
2459 }
2460
2461 rec_sz = obj->btf_ext->line_info.rec_size;
2462 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) {
2463 struct bpf_line_info_min *src_rec, *dst_rec;
2464
2465 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2466 src_sec = find_src_sec_by_name(obj, sec_name);
2467 if (!src_sec) {
2468 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2469 return -EINVAL;
2470 }
2471 dst_sec = &linker->secs[src_sec->dst_id];
2472
2473 if (dst_sec->line_info.rec_sz == 0)
2474 dst_sec->line_info.rec_sz = rec_sz;
2475 if (dst_sec->line_info.rec_sz != rec_sz) {
2476 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2477 return -EINVAL;
2478 }
2479
2480 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) {
2481 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec);
2482 if (!dst_rec)
2483 return -ENOMEM;
2484
2485 dst_rec->insn_off += src_sec->dst_off;
2486
2487 s = btf__str_by_offset(obj->btf, src_rec->file_name_off);
2488 str_off = btf__add_str(linker->btf, s);
2489 if (str_off < 0)
2490 return -ENOMEM;
2491 dst_rec->file_name_off = str_off;
2492
2493 s = btf__str_by_offset(obj->btf, src_rec->line_off);
2494 str_off = btf__add_str(linker->btf, s);
2495 if (str_off < 0)
2496 return -ENOMEM;
2497 dst_rec->line_off = str_off;
2498
2499
2500 }
2501 }
2502
2503 rec_sz = obj->btf_ext->core_relo_info.rec_size;
2504 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) {
2505 struct bpf_core_relo *src_rec, *dst_rec;
2506
2507 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off);
2508 src_sec = find_src_sec_by_name(obj, sec_name);
2509 if (!src_sec) {
2510 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name);
2511 return -EINVAL;
2512 }
2513 dst_sec = &linker->secs[src_sec->dst_id];
2514
2515 if (dst_sec->core_relo_info.rec_sz == 0)
2516 dst_sec->core_relo_info.rec_sz = rec_sz;
2517 if (dst_sec->core_relo_info.rec_sz != rec_sz) {
2518 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name);
2519 return -EINVAL;
2520 }
2521
2522 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) {
2523 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec);
2524 if (!dst_rec)
2525 return -ENOMEM;
2526
2527 dst_rec->insn_off += src_sec->dst_off;
2528 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id];
2529
2530 s = btf__str_by_offset(obj->btf, src_rec->access_str_off);
2531 str_off = btf__add_str(linker->btf, s);
2532 if (str_off < 0)
2533 return -ENOMEM;
2534 dst_rec->access_str_off = str_off;
2535
2536
2537 }
2538 }
2539
2540 return 0;
2541}
2542
2543int bpf_linker__finalize(struct bpf_linker *linker)
2544{
2545 struct dst_sec *sec;
2546 size_t strs_sz;
2547 const void *strs;
2548 int err, i;
2549
2550 if (!linker->elf)
2551 return libbpf_err(-EINVAL);
2552
2553 err = finalize_btf(linker);
2554 if (err)
2555 return libbpf_err(err);
2556
2557
2558 strs_sz = strset__data_size(linker->strtab_strs);
2559 strs = strset__data(linker->strtab_strs);
2560
2561 sec = &linker->secs[linker->strtab_sec_idx];
2562 sec->data->d_align = 1;
2563 sec->data->d_off = 0LL;
2564 sec->data->d_buf = (void *)strs;
2565 sec->data->d_type = ELF_T_BYTE;
2566 sec->data->d_size = strs_sz;
2567 sec->shdr->sh_size = strs_sz;
2568
2569 for (i = 1; i < linker->sec_cnt; i++) {
2570 sec = &linker->secs[i];
2571
2572
2573 if (sec->sec_idx == linker->strtab_sec_idx)
2574 continue;
2575
2576
2577 if (!sec->scn)
2578 continue;
2579
2580 sec->data->d_buf = sec->raw_data;
2581 }
2582
2583
2584 if (elf_update(linker->elf, ELF_C_NULL) < 0) {
2585 err = -errno;
2586 pr_warn_elf("failed to finalize ELF layout");
2587 return libbpf_err(err);
2588 }
2589
2590
2591 if (elf_update(linker->elf, ELF_C_WRITE) < 0) {
2592 err = -errno;
2593 pr_warn_elf("failed to write ELF contents");
2594 return libbpf_err(err);
2595 }
2596
2597 elf_end(linker->elf);
2598 close(linker->fd);
2599
2600 linker->elf = NULL;
2601 linker->fd = -1;
2602
2603 return 0;
2604}
2605
2606static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name,
2607 size_t align, const void *raw_data, size_t raw_sz)
2608{
2609 Elf_Scn *scn;
2610 Elf_Data *data;
2611 Elf64_Shdr *shdr;
2612 int name_off;
2613
2614 name_off = strset__add_str(linker->strtab_strs, sec_name);
2615 if (name_off < 0)
2616 return name_off;
2617
2618 scn = elf_newscn(linker->elf);
2619 if (!scn)
2620 return -ENOMEM;
2621 data = elf_newdata(scn);
2622 if (!data)
2623 return -ENOMEM;
2624 shdr = elf64_getshdr(scn);
2625 if (!shdr)
2626 return -EINVAL;
2627
2628 shdr->sh_name = name_off;
2629 shdr->sh_type = SHT_PROGBITS;
2630 shdr->sh_flags = 0;
2631 shdr->sh_size = raw_sz;
2632 shdr->sh_link = 0;
2633 shdr->sh_info = 0;
2634 shdr->sh_addralign = align;
2635 shdr->sh_entsize = 0;
2636
2637 data->d_type = ELF_T_BYTE;
2638 data->d_size = raw_sz;
2639 data->d_buf = (void *)raw_data;
2640 data->d_align = align;
2641 data->d_off = 0;
2642
2643 return 0;
2644}
2645
2646static int finalize_btf(struct bpf_linker *linker)
2647{
2648 struct btf *btf = linker->btf;
2649 const void *raw_data;
2650 int i, j, id, err;
2651 __u32 raw_sz;
2652
2653
2654 if (btf__get_nr_types(linker->btf) == 0)
2655 return 0;
2656
2657 for (i = 1; i < linker->sec_cnt; i++) {
2658 struct dst_sec *sec = &linker->secs[i];
2659
2660 if (!sec->has_btf)
2661 continue;
2662
2663 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz);
2664 if (id < 0) {
2665 pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n",
2666 sec->sec_name, id);
2667 return id;
2668 }
2669
2670 for (j = 0; j < sec->sec_var_cnt; j++) {
2671 struct btf_var_secinfo *vi = &sec->sec_vars[j];
2672
2673 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size))
2674 return -EINVAL;
2675 }
2676 }
2677
2678 err = finalize_btf_ext(linker);
2679 if (err) {
2680 pr_warn(".BTF.ext generation failed: %d\n", err);
2681 return err;
2682 }
2683
2684 err = btf__dedup(linker->btf, linker->btf_ext, NULL);
2685 if (err) {
2686 pr_warn("BTF dedup failed: %d\n", err);
2687 return err;
2688 }
2689
2690
2691 raw_data = btf__get_raw_data(linker->btf, &raw_sz);
2692 if (!raw_data)
2693 return -ENOMEM;
2694
2695 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2696 if (err) {
2697 pr_warn("failed to write out .BTF ELF section: %d\n", err);
2698 return err;
2699 }
2700
2701
2702 if (linker->btf_ext) {
2703 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz);
2704 if (!raw_data)
2705 return -ENOMEM;
2706
2707 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2708 if (err) {
2709 pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);
2710 return err;
2711 }
2712 }
2713
2714 return 0;
2715}
2716
2717static int emit_btf_ext_data(struct bpf_linker *linker, void *output,
2718 const char *sec_name, struct btf_ext_sec_data *sec_data)
2719{
2720 struct btf_ext_info_sec *sec_info;
2721 void *cur = output;
2722 int str_off;
2723 size_t sz;
2724
2725 if (!sec_data->rec_cnt)
2726 return 0;
2727
2728 str_off = btf__add_str(linker->btf, sec_name);
2729 if (str_off < 0)
2730 return -ENOMEM;
2731
2732 sec_info = cur;
2733 sec_info->sec_name_off = str_off;
2734 sec_info->num_info = sec_data->rec_cnt;
2735 cur += sizeof(struct btf_ext_info_sec);
2736
2737 sz = sec_data->rec_cnt * sec_data->rec_sz;
2738 memcpy(cur, sec_data->recs, sz);
2739 cur += sz;
2740
2741 return cur - output;
2742}
2743
2744static int finalize_btf_ext(struct bpf_linker *linker)
2745{
2746 size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0;
2747 size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0;
2748 struct btf_ext_header *hdr;
2749 void *data, *cur;
2750 int i, err, sz;
2751
2752
2753
2754
2755
2756 for (i = 1; i < linker->sec_cnt; i++) {
2757 struct dst_sec *sec = &linker->secs[i];
2758
2759 if (sec->func_info.rec_cnt) {
2760 if (func_rec_sz == 0)
2761 func_rec_sz = sec->func_info.rec_sz;
2762 if (func_rec_sz != sec->func_info.rec_sz) {
2763 pr_warn("mismatch in func_info record size %zu != %u\n",
2764 func_rec_sz, sec->func_info.rec_sz);
2765 return -EINVAL;
2766 }
2767
2768 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt;
2769 }
2770 if (sec->line_info.rec_cnt) {
2771 if (line_rec_sz == 0)
2772 line_rec_sz = sec->line_info.rec_sz;
2773 if (line_rec_sz != sec->line_info.rec_sz) {
2774 pr_warn("mismatch in line_info record size %zu != %u\n",
2775 line_rec_sz, sec->line_info.rec_sz);
2776 return -EINVAL;
2777 }
2778
2779 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt;
2780 }
2781 if (sec->core_relo_info.rec_cnt) {
2782 if (core_relo_rec_sz == 0)
2783 core_relo_rec_sz = sec->core_relo_info.rec_sz;
2784 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) {
2785 pr_warn("mismatch in core_relo_info record size %zu != %u\n",
2786 core_relo_rec_sz, sec->core_relo_info.rec_sz);
2787 return -EINVAL;
2788 }
2789
2790 core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt;
2791 }
2792 }
2793
2794 if (!funcs_sz && !lines_sz && !core_relos_sz)
2795 return 0;
2796
2797 total_sz += sizeof(struct btf_ext_header);
2798 if (funcs_sz) {
2799 funcs_sz += sizeof(__u32);
2800 total_sz += funcs_sz;
2801 }
2802 if (lines_sz) {
2803 lines_sz += sizeof(__u32);
2804 total_sz += lines_sz;
2805 }
2806 if (core_relos_sz) {
2807 core_relos_sz += sizeof(__u32);
2808 total_sz += core_relos_sz;
2809 }
2810
2811 cur = data = calloc(1, total_sz);
2812 if (!data)
2813 return -ENOMEM;
2814
2815 hdr = cur;
2816 hdr->magic = BTF_MAGIC;
2817 hdr->version = BTF_VERSION;
2818 hdr->flags = 0;
2819 hdr->hdr_len = sizeof(struct btf_ext_header);
2820 cur += sizeof(struct btf_ext_header);
2821
2822
2823 hdr->func_info_off = 0;
2824 hdr->func_info_len = funcs_sz;
2825 hdr->line_info_off = funcs_sz;
2826 hdr->line_info_len = lines_sz;
2827 hdr->core_relo_off = funcs_sz + lines_sz;
2828 hdr->core_relo_len = core_relos_sz;
2829
2830 if (funcs_sz) {
2831 *(__u32 *)cur = func_rec_sz;
2832 cur += sizeof(__u32);
2833
2834 for (i = 1; i < linker->sec_cnt; i++) {
2835 struct dst_sec *sec = &linker->secs[i];
2836
2837 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info);
2838 if (sz < 0) {
2839 err = sz;
2840 goto out;
2841 }
2842
2843 cur += sz;
2844 }
2845 }
2846
2847 if (lines_sz) {
2848 *(__u32 *)cur = line_rec_sz;
2849 cur += sizeof(__u32);
2850
2851 for (i = 1; i < linker->sec_cnt; i++) {
2852 struct dst_sec *sec = &linker->secs[i];
2853
2854 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info);
2855 if (sz < 0) {
2856 err = sz;
2857 goto out;
2858 }
2859
2860 cur += sz;
2861 }
2862 }
2863
2864 if (core_relos_sz) {
2865 *(__u32 *)cur = core_relo_rec_sz;
2866 cur += sizeof(__u32);
2867
2868 for (i = 1; i < linker->sec_cnt; i++) {
2869 struct dst_sec *sec = &linker->secs[i];
2870
2871 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info);
2872 if (sz < 0) {
2873 err = sz;
2874 goto out;
2875 }
2876
2877 cur += sz;
2878 }
2879 }
2880
2881 linker->btf_ext = btf_ext__new(data, total_sz);
2882 err = libbpf_get_error(linker->btf_ext);
2883 if (err) {
2884 linker->btf_ext = NULL;
2885 pr_warn("failed to parse final .BTF.ext data: %d\n", err);
2886 goto out;
2887 }
2888
2889out:
2890 free(data);
2891 return err;
2892}
2893