1#include <sys/sysmacros.h>
2#include <sys/types.h>
3#include <errno.h>
4#include <stdio.h>
5#include <stdlib.h>
6#include <string.h>
7#include <fcntl.h>
8#include <unistd.h>
9#include <inttypes.h>
10#include <byteswap.h>
11#include <sys/stat.h>
12#include <sys/mman.h>
13#include <linux/stringify.h>
14
15#include "util.h"
16#include "event.h"
17#include "debug.h"
18#include "evlist.h"
19#include "symbol.h"
20#include <elf.h>
21
22#include "tsc.h"
23#include "session.h"
24#include "jit.h"
25#include "jitdump.h"
26#include "genelf.h"
27#include "../builtin.h"
28
29#include "sane_ctype.h"
30
31struct jit_buf_desc {
32 struct perf_data *output;
33 struct perf_session *session;
34 struct machine *machine;
35 union jr_entry *entry;
36 void *buf;
37 uint64_t sample_type;
38 size_t bufsize;
39 FILE *in;
40 bool needs_bswap;
41 bool use_arch_timestamp;
42 void *debug_data;
43 void *unwinding_data;
44 uint64_t unwinding_size;
45 uint64_t unwinding_mapped_size;
46 uint64_t eh_frame_hdr_size;
47 size_t nr_debug_entries;
48 uint32_t code_load_count;
49 u64 bytes_written;
50 struct rb_root code_root;
51 char dir[PATH_MAX];
52};
53
54struct debug_line_info {
55 unsigned long vma;
56 unsigned int lineno;
57
58 char const filename[0];
59};
60
61struct jit_tool {
62 struct perf_tool tool;
63 struct perf_data output;
64 struct perf_data input;
65 u64 bytes_written;
66};
67
68#define hmax(a, b) ((a) > (b) ? (a) : (b))
69#define get_jit_tool(t) (container_of(tool, struct jit_tool, tool))
70
71static int
72jit_emit_elf(char *filename,
73 const char *sym,
74 uint64_t code_addr,
75 const void *code,
76 int csize,
77 void *debug,
78 int nr_debug_entries,
79 void *unwinding,
80 uint32_t unwinding_header_size,
81 uint32_t unwinding_size)
82{
83 int ret, fd;
84
85 if (verbose > 0)
86 fprintf(stderr, "write ELF image %s\n", filename);
87
88 fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644);
89 if (fd == -1) {
90 pr_warning("cannot create jit ELF %s: %s\n", filename, strerror(errno));
91 return -1;
92 }
93
94 ret = jit_write_elf(fd, code_addr, sym, (const void *)code, csize, debug, nr_debug_entries,
95 unwinding, unwinding_header_size, unwinding_size);
96
97 close(fd);
98
99 if (ret)
100 unlink(filename);
101
102 return ret;
103}
104
105static void
106jit_close(struct jit_buf_desc *jd)
107{
108 if (!(jd && jd->in))
109 return;
110 funlockfile(jd->in);
111 fclose(jd->in);
112 jd->in = NULL;
113}
114
115static int
116jit_validate_events(struct perf_session *session __maybe_unused)
117{
118
119
120
121
122
123 return 0;
124
125#if 0
126 struct perf_evsel *evsel;
127
128
129
130
131 evlist__for_each_entry(session->evlist, evsel) {
132 if (evsel->attr.use_clockid == 0 || evsel->attr.clockid != CLOCK_MONOTONIC)
133 return -1;
134 }
135 return 0;
136#endif
137}
138
139static int
140jit_open(struct jit_buf_desc *jd, const char *name)
141{
142 struct jitheader header;
143 struct jr_prefix *prefix;
144 ssize_t bs, bsz = 0;
145 void *n, *buf = NULL;
146 int ret, retval = -1;
147
148 jd->in = fopen(name, "r");
149 if (!jd->in)
150 return -1;
151
152 bsz = hmax(sizeof(header), sizeof(*prefix));
153
154 buf = malloc(bsz);
155 if (!buf)
156 goto error;
157
158
159
160
161 flockfile(jd->in);
162
163 ret = fread(buf, sizeof(header), 1, jd->in);
164 if (ret != 1)
165 goto error;
166
167 memcpy(&header, buf, sizeof(header));
168
169 if (header.magic != JITHEADER_MAGIC) {
170 if (header.magic != JITHEADER_MAGIC_SW)
171 goto error;
172 jd->needs_bswap = true;
173 }
174
175 if (jd->needs_bswap) {
176 header.version = bswap_32(header.version);
177 header.total_size = bswap_32(header.total_size);
178 header.pid = bswap_32(header.pid);
179 header.elf_mach = bswap_32(header.elf_mach);
180 header.timestamp = bswap_64(header.timestamp);
181 header.flags = bswap_64(header.flags);
182 }
183
184 jd->use_arch_timestamp = header.flags & JITDUMP_FLAGS_ARCH_TIMESTAMP;
185
186 if (verbose > 2)
187 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n",
188 header.version,
189 header.total_size,
190 (unsigned long long)header.timestamp,
191 header.pid,
192 header.elf_mach,
193 jd->use_arch_timestamp);
194
195 if (header.version > JITHEADER_VERSION) {
196 pr_err("wrong jitdump version %u, expected " __stringify(JITHEADER_VERSION),
197 header.version);
198 goto error;
199 }
200
201 if (header.flags & JITDUMP_FLAGS_RESERVED) {
202 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n",
203 (unsigned long long)header.flags & JITDUMP_FLAGS_RESERVED);
204 goto error;
205 }
206
207 if (jd->use_arch_timestamp && !jd->session->time_conv.time_mult) {
208 pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n");
209 goto error;
210 }
211
212
213
214
215 if (!jd->use_arch_timestamp && jit_validate_events(jd->session)) {
216 pr_err("error, jitted code must be sampled with perf record -k 1\n");
217 goto error;
218 }
219
220 bs = header.total_size - sizeof(header);
221
222 if (bs > bsz) {
223 n = realloc(buf, bs);
224 if (!n)
225 goto error;
226 bsz = bs;
227 buf = n;
228
229 ret = fread(buf, bs - bsz, 1, jd->in);
230 if (ret != 1)
231 goto error;
232 }
233
234
235
236 strcpy(jd->dir, name);
237 dirname(jd->dir);
238
239 return 0;
240error:
241 funlockfile(jd->in);
242 fclose(jd->in);
243 return retval;
244}
245
246static union jr_entry *
247jit_get_next_entry(struct jit_buf_desc *jd)
248{
249 struct jr_prefix *prefix;
250 union jr_entry *jr;
251 void *addr;
252 size_t bs, size;
253 int id, ret;
254
255 if (!(jd && jd->in))
256 return NULL;
257
258 if (jd->buf == NULL) {
259 size_t sz = getpagesize();
260 if (sz < sizeof(*prefix))
261 sz = sizeof(*prefix);
262
263 jd->buf = malloc(sz);
264 if (jd->buf == NULL)
265 return NULL;
266
267 jd->bufsize = sz;
268 }
269
270 prefix = jd->buf;
271
272
273
274
275 ret = fread(prefix, sizeof(*prefix), 1, jd->in);
276 if (ret != 1)
277 return NULL;
278
279 if (jd->needs_bswap) {
280 prefix->id = bswap_32(prefix->id);
281 prefix->total_size = bswap_32(prefix->total_size);
282 prefix->timestamp = bswap_64(prefix->timestamp);
283 }
284 id = prefix->id;
285 size = prefix->total_size;
286
287 bs = (size_t)size;
288 if (bs < sizeof(*prefix))
289 return NULL;
290
291 if (id >= JIT_CODE_MAX) {
292 pr_warning("next_entry: unknown record type %d, skipping\n", id);
293 }
294 if (bs > jd->bufsize) {
295 void *n;
296 n = realloc(jd->buf, bs);
297 if (!n)
298 return NULL;
299 jd->buf = n;
300 jd->bufsize = bs;
301 }
302
303 addr = ((void *)jd->buf) + sizeof(*prefix);
304
305 ret = fread(addr, bs - sizeof(*prefix), 1, jd->in);
306 if (ret != 1)
307 return NULL;
308
309 jr = (union jr_entry *)jd->buf;
310
311 switch(id) {
312 case JIT_CODE_DEBUG_INFO:
313 if (jd->needs_bswap) {
314 uint64_t n;
315 jr->info.code_addr = bswap_64(jr->info.code_addr);
316 jr->info.nr_entry = bswap_64(jr->info.nr_entry);
317 for (n = 0 ; n < jr->info.nr_entry; n++) {
318 jr->info.entries[n].addr = bswap_64(jr->info.entries[n].addr);
319 jr->info.entries[n].lineno = bswap_32(jr->info.entries[n].lineno);
320 jr->info.entries[n].discrim = bswap_32(jr->info.entries[n].discrim);
321 }
322 }
323 break;
324 case JIT_CODE_UNWINDING_INFO:
325 if (jd->needs_bswap) {
326 jr->unwinding.unwinding_size = bswap_64(jr->unwinding.unwinding_size);
327 jr->unwinding.eh_frame_hdr_size = bswap_64(jr->unwinding.eh_frame_hdr_size);
328 jr->unwinding.mapped_size = bswap_64(jr->unwinding.mapped_size);
329 }
330 break;
331 case JIT_CODE_CLOSE:
332 break;
333 case JIT_CODE_LOAD:
334 if (jd->needs_bswap) {
335 jr->load.pid = bswap_32(jr->load.pid);
336 jr->load.tid = bswap_32(jr->load.tid);
337 jr->load.vma = bswap_64(jr->load.vma);
338 jr->load.code_addr = bswap_64(jr->load.code_addr);
339 jr->load.code_size = bswap_64(jr->load.code_size);
340 jr->load.code_index= bswap_64(jr->load.code_index);
341 }
342 jd->code_load_count++;
343 break;
344 case JIT_CODE_MOVE:
345 if (jd->needs_bswap) {
346 jr->move.pid = bswap_32(jr->move.pid);
347 jr->move.tid = bswap_32(jr->move.tid);
348 jr->move.vma = bswap_64(jr->move.vma);
349 jr->move.old_code_addr = bswap_64(jr->move.old_code_addr);
350 jr->move.new_code_addr = bswap_64(jr->move.new_code_addr);
351 jr->move.code_size = bswap_64(jr->move.code_size);
352 jr->move.code_index = bswap_64(jr->move.code_index);
353 }
354 break;
355 case JIT_CODE_MAX:
356 default:
357
358 break;
359 }
360 return jr;
361}
362
363static int
364jit_inject_event(struct jit_buf_desc *jd, union perf_event *event)
365{
366 ssize_t size;
367
368 size = perf_data__write(jd->output, event, event->header.size);
369 if (size < 0)
370 return -1;
371
372 jd->bytes_written += size;
373 return 0;
374}
375
376static uint64_t convert_timestamp(struct jit_buf_desc *jd, uint64_t timestamp)
377{
378 struct perf_tsc_conversion tc;
379
380 if (!jd->use_arch_timestamp)
381 return timestamp;
382
383 tc.time_shift = jd->session->time_conv.time_shift;
384 tc.time_mult = jd->session->time_conv.time_mult;
385 tc.time_zero = jd->session->time_conv.time_zero;
386
387 if (!tc.time_mult)
388 return 0;
389
390 return tsc_to_perf_time(timestamp, &tc);
391}
392
393static int jit_repipe_code_load(struct jit_buf_desc *jd, union jr_entry *jr)
394{
395 struct perf_sample sample;
396 union perf_event *event;
397 struct perf_tool *tool = jd->session->tool;
398 uint64_t code, addr;
399 uintptr_t uaddr;
400 char *filename;
401 struct stat st;
402 size_t size;
403 u16 idr_size;
404 const char *sym;
405 uint32_t count;
406 int ret, csize, usize;
407 pid_t pid, tid;
408 struct {
409 u32 pid, tid;
410 u64 time;
411 } *id;
412
413 pid = jr->load.pid;
414 tid = jr->load.tid;
415 csize = jr->load.code_size;
416 usize = jd->unwinding_mapped_size;
417 addr = jr->load.code_addr;
418 sym = (void *)((unsigned long)jr + sizeof(jr->load));
419 code = (unsigned long)jr + jr->load.p.total_size - csize;
420 count = jr->load.code_index;
421 idr_size = jd->machine->id_hdr_size;
422
423 event = calloc(1, sizeof(*event) + idr_size);
424 if (!event)
425 return -1;
426
427 filename = event->mmap2.filename;
428 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%u.so",
429 jd->dir,
430 pid,
431 count);
432
433 size++;
434
435 size = PERF_ALIGN(size, sizeof(u64));
436 uaddr = (uintptr_t)code;
437 ret = jit_emit_elf(filename, sym, addr, (const void *)uaddr, csize, jd->debug_data, jd->nr_debug_entries,
438 jd->unwinding_data, jd->eh_frame_hdr_size, jd->unwinding_size);
439
440 if (jd->debug_data && jd->nr_debug_entries) {
441 free(jd->debug_data);
442 jd->debug_data = NULL;
443 jd->nr_debug_entries = 0;
444 }
445
446 if (jd->unwinding_data && jd->eh_frame_hdr_size) {
447 free(jd->unwinding_data);
448 jd->unwinding_data = NULL;
449 jd->eh_frame_hdr_size = 0;
450 jd->unwinding_mapped_size = 0;
451 jd->unwinding_size = 0;
452 }
453
454 if (ret) {
455 free(event);
456 return -1;
457 }
458 if (stat(filename, &st))
459 memset(&st, 0, sizeof(st));
460
461 event->mmap2.header.type = PERF_RECORD_MMAP2;
462 event->mmap2.header.misc = PERF_RECORD_MISC_USER;
463 event->mmap2.header.size = (sizeof(event->mmap2) -
464 (sizeof(event->mmap2.filename) - size) + idr_size);
465
466 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET;
467 event->mmap2.start = addr;
468 event->mmap2.len = usize ? ALIGN_8(csize) + usize : csize;
469 event->mmap2.pid = pid;
470 event->mmap2.tid = tid;
471 event->mmap2.ino = st.st_ino;
472 event->mmap2.maj = major(st.st_dev);
473 event->mmap2.min = minor(st.st_dev);
474 event->mmap2.prot = st.st_mode;
475 event->mmap2.flags = MAP_SHARED;
476 event->mmap2.ino_generation = 1;
477
478 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size);
479 if (jd->sample_type & PERF_SAMPLE_TID) {
480 id->pid = pid;
481 id->tid = tid;
482 }
483 if (jd->sample_type & PERF_SAMPLE_TIME)
484 id->time = convert_timestamp(jd, jr->load.p.timestamp);
485
486
487
488
489
490 memset(&sample, 0, sizeof(sample));
491 sample.cpumode = PERF_RECORD_MISC_USER;
492 sample.pid = pid;
493 sample.tid = tid;
494 sample.time = id->time;
495 sample.ip = addr;
496
497 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine);
498 if (ret)
499 return ret;
500
501 ret = jit_inject_event(jd, event);
502
503
504
505 if (!ret)
506 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine);
507
508 return ret;
509}
510
511static int jit_repipe_code_move(struct jit_buf_desc *jd, union jr_entry *jr)
512{
513 struct perf_sample sample;
514 union perf_event *event;
515 struct perf_tool *tool = jd->session->tool;
516 char *filename;
517 size_t size;
518 struct stat st;
519 int usize;
520 u16 idr_size;
521 int ret;
522 pid_t pid, tid;
523 struct {
524 u32 pid, tid;
525 u64 time;
526 } *id;
527
528 pid = jr->move.pid;
529 tid = jr->move.tid;
530 usize = jd->unwinding_mapped_size;
531 idr_size = jd->machine->id_hdr_size;
532
533
534
535
536 event = calloc(1, sizeof(*event) + 16);
537 if (!event)
538 return -1;
539
540 filename = event->mmap2.filename;
541 size = snprintf(filename, PATH_MAX, "%s/jitted-%d-%"PRIu64,
542 jd->dir,
543 pid,
544 jr->move.code_index);
545
546 size++;
547
548 if (stat(filename, &st))
549 memset(&st, 0, sizeof(st));
550
551 size = PERF_ALIGN(size, sizeof(u64));
552
553 event->mmap2.header.type = PERF_RECORD_MMAP2;
554 event->mmap2.header.misc = PERF_RECORD_MISC_USER;
555 event->mmap2.header.size = (sizeof(event->mmap2) -
556 (sizeof(event->mmap2.filename) - size) + idr_size);
557 event->mmap2.pgoff = GEN_ELF_TEXT_OFFSET;
558 event->mmap2.start = jr->move.new_code_addr;
559 event->mmap2.len = usize ? ALIGN_8(jr->move.code_size) + usize
560 : jr->move.code_size;
561 event->mmap2.pid = pid;
562 event->mmap2.tid = tid;
563 event->mmap2.ino = st.st_ino;
564 event->mmap2.maj = major(st.st_dev);
565 event->mmap2.min = minor(st.st_dev);
566 event->mmap2.prot = st.st_mode;
567 event->mmap2.flags = MAP_SHARED;
568 event->mmap2.ino_generation = 1;
569
570 id = (void *)((unsigned long)event + event->mmap.header.size - idr_size);
571 if (jd->sample_type & PERF_SAMPLE_TID) {
572 id->pid = pid;
573 id->tid = tid;
574 }
575 if (jd->sample_type & PERF_SAMPLE_TIME)
576 id->time = convert_timestamp(jd, jr->load.p.timestamp);
577
578
579
580
581
582 memset(&sample, 0, sizeof(sample));
583 sample.cpumode = PERF_RECORD_MISC_USER;
584 sample.pid = pid;
585 sample.tid = tid;
586 sample.time = id->time;
587 sample.ip = jr->move.new_code_addr;
588
589 ret = perf_event__process_mmap2(tool, event, &sample, jd->machine);
590 if (ret)
591 return ret;
592
593 ret = jit_inject_event(jd, event);
594 if (!ret)
595 build_id__mark_dso_hit(tool, event, &sample, NULL, jd->machine);
596
597 return ret;
598}
599
600static int jit_repipe_debug_info(struct jit_buf_desc *jd, union jr_entry *jr)
601{
602 void *data;
603 size_t sz;
604
605 if (!(jd && jr))
606 return -1;
607
608 sz = jr->prefix.total_size - sizeof(jr->info);
609 data = malloc(sz);
610 if (!data)
611 return -1;
612
613 memcpy(data, &jr->info.entries, sz);
614
615 jd->debug_data = data;
616
617
618
619
620
621 jd->nr_debug_entries = jr->info.nr_entry;
622
623 return 0;
624}
625
626static int
627jit_repipe_unwinding_info(struct jit_buf_desc *jd, union jr_entry *jr)
628{
629 void *unwinding_data;
630 uint32_t unwinding_data_size;
631
632 if (!(jd && jr))
633 return -1;
634
635 unwinding_data_size = jr->prefix.total_size - sizeof(jr->unwinding);
636 unwinding_data = malloc(unwinding_data_size);
637 if (!unwinding_data)
638 return -1;
639
640 memcpy(unwinding_data, &jr->unwinding.unwinding_data,
641 unwinding_data_size);
642
643 jd->eh_frame_hdr_size = jr->unwinding.eh_frame_hdr_size;
644 jd->unwinding_size = jr->unwinding.unwinding_size;
645 jd->unwinding_mapped_size = jr->unwinding.mapped_size;
646 jd->unwinding_data = unwinding_data;
647
648 return 0;
649}
650
651static int
652jit_process_dump(struct jit_buf_desc *jd)
653{
654 union jr_entry *jr;
655 int ret = 0;
656
657 while ((jr = jit_get_next_entry(jd))) {
658 switch(jr->prefix.id) {
659 case JIT_CODE_LOAD:
660 ret = jit_repipe_code_load(jd, jr);
661 break;
662 case JIT_CODE_MOVE:
663 ret = jit_repipe_code_move(jd, jr);
664 break;
665 case JIT_CODE_DEBUG_INFO:
666 ret = jit_repipe_debug_info(jd, jr);
667 break;
668 case JIT_CODE_UNWINDING_INFO:
669 ret = jit_repipe_unwinding_info(jd, jr);
670 break;
671 default:
672 ret = 0;
673 continue;
674 }
675 }
676 return ret;
677}
678
679static int
680jit_inject(struct jit_buf_desc *jd, char *path)
681{
682 int ret;
683
684 if (verbose > 0)
685 fprintf(stderr, "injecting: %s\n", path);
686
687 ret = jit_open(jd, path);
688 if (ret)
689 return -1;
690
691 ret = jit_process_dump(jd);
692
693 jit_close(jd);
694
695 if (verbose > 0)
696 fprintf(stderr, "injected: %s (%d)\n", path, ret);
697
698 return 0;
699}
700
701
702
703
704
705
706static int
707jit_detect(char *mmap_name, pid_t pid)
708 {
709 char *p;
710 char *end = NULL;
711 pid_t pid2;
712
713 if (verbose > 2)
714 fprintf(stderr, "jit marker trying : %s\n", mmap_name);
715
716
717
718 p = strrchr(mmap_name, '/');
719 if (!p)
720 return -1;
721
722
723
724
725 if (strncmp(p, "/jit-", 5))
726 return -1;
727
728
729
730
731 p += 5;
732
733
734
735
736 if (!isdigit(*p))
737 return -1;
738
739 pid2 = (int)strtol(p, &end, 10);
740 if (!end)
741 return -1;
742
743
744
745
746
747 if (pid && pid2 != pid)
748 return -1;
749
750
751
752 if (strcmp(end, ".dump"))
753 return -1;
754
755 if (verbose > 0)
756 fprintf(stderr, "jit marker found: %s\n", mmap_name);
757
758 return 0;
759}
760
761int
762jit_process(struct perf_session *session,
763 struct perf_data *output,
764 struct machine *machine,
765 char *filename,
766 pid_t pid,
767 u64 *nbytes)
768{
769 struct perf_evsel *first;
770 struct jit_buf_desc jd;
771 int ret;
772
773
774
775
776 if (jit_detect(filename, pid))
777 return 0;
778
779 memset(&jd, 0, sizeof(jd));
780
781 jd.session = session;
782 jd.output = output;
783 jd.machine = machine;
784
785
786
787
788
789 first = perf_evlist__first(session->evlist);
790 jd.sample_type = first->attr.sample_type;
791
792 *nbytes = 0;
793
794 ret = jit_inject(&jd, filename);
795 if (!ret) {
796 *nbytes = jd.bytes_written;
797 ret = 1;
798 }
799
800 return ret;
801}
802
