1
2
3
4
5
6
7#ifndef _GNU_SOURCE
8#define _GNU_SOURCE
9#endif
10#include <stdlib.h>
11#include <stdbool.h>
12#include <string.h>
13#include <errno.h>
14#include <stdint.h>
15#include <inttypes.h>
16#include <linux/compiler.h>
17#include <linux/string.h>
18#include <linux/zalloc.h>
19
20#include "../auxtrace.h"
21
22#include "intel-pt-insn-decoder.h"
23#include "intel-pt-pkt-decoder.h"
24#include "intel-pt-decoder.h"
25#include "intel-pt-log.h"
26
27#define BITULL(x) (1ULL << (x))
28
29
30#define INTEL_PT_CYC_ENABLE BITULL(1)
31#define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19))
32#define INTEL_PT_CYC_THRESHOLD_SHIFT 19
33
34#define INTEL_PT_BLK_SIZE 1024
35
36#define BIT63 (((uint64_t)1 << 63))
37
38#define SEVEN_BYTES 0xffffffffffffffULL
39
40#define NO_VMCS 0xffffffffffULL
41
42#define INTEL_PT_RETURN 1
43
44
45
46
47
48#define INTEL_PT_MAX_LOOPS 100000
49
50struct intel_pt_blk {
51 struct intel_pt_blk *prev;
52 uint64_t ip[INTEL_PT_BLK_SIZE];
53};
54
55struct intel_pt_stack {
56 struct intel_pt_blk *blk;
57 struct intel_pt_blk *spare;
58 int pos;
59};
60
61enum intel_pt_p_once {
62 INTEL_PT_PRT_ONCE_UNK_VMCS,
63 INTEL_PT_PRT_ONCE_ERANGE,
64};
65
66enum intel_pt_pkt_state {
67 INTEL_PT_STATE_NO_PSB,
68 INTEL_PT_STATE_NO_IP,
69 INTEL_PT_STATE_ERR_RESYNC,
70 INTEL_PT_STATE_IN_SYNC,
71 INTEL_PT_STATE_TNT_CONT,
72 INTEL_PT_STATE_TNT,
73 INTEL_PT_STATE_TIP,
74 INTEL_PT_STATE_TIP_PGD,
75 INTEL_PT_STATE_FUP,
76 INTEL_PT_STATE_FUP_NO_TIP,
77 INTEL_PT_STATE_FUP_IN_PSB,
78 INTEL_PT_STATE_RESAMPLE,
79 INTEL_PT_STATE_VM_TIME_CORRELATION,
80};
81
82static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
83{
84 switch (pkt_state) {
85 case INTEL_PT_STATE_NO_PSB:
86 case INTEL_PT_STATE_NO_IP:
87 case INTEL_PT_STATE_ERR_RESYNC:
88 case INTEL_PT_STATE_IN_SYNC:
89 case INTEL_PT_STATE_TNT_CONT:
90 case INTEL_PT_STATE_RESAMPLE:
91 case INTEL_PT_STATE_VM_TIME_CORRELATION:
92 return true;
93 case INTEL_PT_STATE_TNT:
94 case INTEL_PT_STATE_TIP:
95 case INTEL_PT_STATE_TIP_PGD:
96 case INTEL_PT_STATE_FUP:
97 case INTEL_PT_STATE_FUP_NO_TIP:
98 case INTEL_PT_STATE_FUP_IN_PSB:
99 return false;
100 default:
101 return true;
102 };
103}
104
105#ifdef INTEL_PT_STRICT
106#define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
107#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
108#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
109#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
110#else
111#define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
112#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
113#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
114#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
115#endif
116
117struct intel_pt_decoder {
118 int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
119 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
120 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
121 uint64_t max_insn_cnt, void *data);
122 bool (*pgd_ip)(uint64_t ip, void *data);
123 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data);
124 struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs);
125 void *data;
126 struct intel_pt_state state;
127 const unsigned char *buf;
128 size_t len;
129 bool return_compression;
130 bool branch_enable;
131 bool mtc_insn;
132 bool pge;
133 bool have_tma;
134 bool have_cyc;
135 bool fixup_last_mtc;
136 bool have_last_ip;
137 bool in_psb;
138 bool hop;
139 bool leap;
140 bool vm_time_correlation;
141 bool vm_tm_corr_dry_run;
142 bool vm_tm_corr_reliable;
143 bool vm_tm_corr_same_buf;
144 bool vm_tm_corr_continuous;
145 bool nr;
146 bool next_nr;
147 enum intel_pt_param_flags flags;
148 uint64_t pos;
149 uint64_t last_ip;
150 uint64_t ip;
151 uint64_t pip_payload;
152 uint64_t timestamp;
153 uint64_t tsc_timestamp;
154 uint64_t ref_timestamp;
155 uint64_t buf_timestamp;
156 uint64_t sample_timestamp;
157 uint64_t ret_addr;
158 uint64_t ctc_timestamp;
159 uint64_t ctc_delta;
160 uint64_t cycle_cnt;
161 uint64_t cyc_ref_timestamp;
162 uint64_t first_timestamp;
163 uint64_t last_reliable_timestamp;
164 uint64_t vmcs;
165 uint64_t print_once;
166 uint64_t last_ctc;
167 uint32_t last_mtc;
168 uint32_t tsc_ctc_ratio_n;
169 uint32_t tsc_ctc_ratio_d;
170 uint32_t tsc_ctc_mult;
171 uint32_t tsc_slip;
172 uint32_t ctc_rem_mask;
173 int mtc_shift;
174 struct intel_pt_stack stack;
175 enum intel_pt_pkt_state pkt_state;
176 enum intel_pt_pkt_ctx pkt_ctx;
177 enum intel_pt_pkt_ctx prev_pkt_ctx;
178 enum intel_pt_blk_type blk_type;
179 int blk_type_pos;
180 struct intel_pt_pkt packet;
181 struct intel_pt_pkt tnt;
182 int pkt_step;
183 int pkt_len;
184 int last_packet_type;
185 unsigned int cbr;
186 unsigned int cbr_seen;
187 unsigned int max_non_turbo_ratio;
188 double max_non_turbo_ratio_fp;
189 double cbr_cyc_to_tsc;
190 double calc_cyc_to_tsc;
191 bool have_calc_cyc_to_tsc;
192 int exec_mode;
193 unsigned int insn_bytes;
194 uint64_t period;
195 enum intel_pt_period_type period_type;
196 uint64_t tot_insn_cnt;
197 uint64_t period_insn_cnt;
198 uint64_t period_mask;
199 uint64_t period_ticks;
200 uint64_t last_masked_timestamp;
201 uint64_t tot_cyc_cnt;
202 uint64_t sample_tot_cyc_cnt;
203 uint64_t base_cyc_cnt;
204 uint64_t cyc_cnt_timestamp;
205 uint64_t ctl;
206 uint64_t cyc_threshold;
207 double tsc_to_cyc;
208 bool continuous_period;
209 bool overflow;
210 bool set_fup_tx_flags;
211 bool set_fup_ptw;
212 bool set_fup_mwait;
213 bool set_fup_pwre;
214 bool set_fup_exstop;
215 bool set_fup_bep;
216 bool sample_cyc;
217 unsigned int fup_tx_flags;
218 unsigned int tx_flags;
219 uint64_t fup_ptw_payload;
220 uint64_t fup_mwait_payload;
221 uint64_t fup_pwre_payload;
222 uint64_t cbr_payload;
223 uint64_t timestamp_insn_cnt;
224 uint64_t sample_insn_cnt;
225 uint64_t stuck_ip;
226 int max_loops;
227 int no_progress;
228 int stuck_ip_prd;
229 int stuck_ip_cnt;
230 uint64_t psb_ip;
231 const unsigned char *next_buf;
232 size_t next_len;
233 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
234};
235
236static uint64_t intel_pt_lower_power_of_2(uint64_t x)
237{
238 int i;
239
240 for (i = 0; x != 1; i++)
241 x >>= 1;
242
243 return x << i;
244}
245
246__printf(1, 2)
247static void p_log(const char *fmt, ...)
248{
249 char buf[512];
250 va_list args;
251
252 va_start(args, fmt);
253 vsnprintf(buf, sizeof(buf), fmt, args);
254 va_end(args);
255
256 fprintf(stderr, "%s\n", buf);
257 intel_pt_log("%s\n", buf);
258}
259
260static bool intel_pt_print_once(struct intel_pt_decoder *decoder,
261 enum intel_pt_p_once id)
262{
263 uint64_t bit = 1ULL << id;
264
265 if (decoder->print_once & bit)
266 return false;
267 decoder->print_once |= bit;
268 return true;
269}
270
271static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
272{
273 if (!(ctl & INTEL_PT_CYC_ENABLE))
274 return 0;
275
276 return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
277}
278
279static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
280{
281 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
282 uint64_t period;
283
284 period = intel_pt_lower_power_of_2(decoder->period);
285 decoder->period_mask = ~(period - 1);
286 decoder->period_ticks = period;
287 }
288}
289
290static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
291{
292 if (!d)
293 return 0;
294 return (t / d) * n + ((t % d) * n) / d;
295}
296
297struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
298{
299 struct intel_pt_decoder *decoder;
300
301 if (!params->get_trace || !params->walk_insn)
302 return NULL;
303
304 decoder = zalloc(sizeof(struct intel_pt_decoder));
305 if (!decoder)
306 return NULL;
307
308 decoder->get_trace = params->get_trace;
309 decoder->walk_insn = params->walk_insn;
310 decoder->pgd_ip = params->pgd_ip;
311 decoder->lookahead = params->lookahead;
312 decoder->findnew_vmcs_info = params->findnew_vmcs_info;
313 decoder->data = params->data;
314 decoder->return_compression = params->return_compression;
315 decoder->branch_enable = params->branch_enable;
316 decoder->hop = params->quick >= 1;
317 decoder->leap = params->quick >= 2;
318 decoder->vm_time_correlation = params->vm_time_correlation;
319 decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;
320 decoder->first_timestamp = params->first_timestamp;
321 decoder->last_reliable_timestamp = params->first_timestamp;
322 decoder->max_loops = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;
323
324 decoder->flags = params->flags;
325
326 decoder->ctl = params->ctl;
327 decoder->period = params->period;
328 decoder->period_type = params->period_type;
329
330 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
331 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
332
333 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
334
335 intel_pt_setup_period(decoder);
336
337 decoder->mtc_shift = params->mtc_period;
338 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
339
340 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
341 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
342
343 if (!decoder->tsc_ctc_ratio_n)
344 decoder->tsc_ctc_ratio_d = 0;
345
346 if (decoder->tsc_ctc_ratio_d) {
347 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
348 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
349 decoder->tsc_ctc_ratio_d;
350 }
351
352
353
354
355
356
357
358 decoder->tsc_slip = 0x10000;
359
360 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
361 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
362 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
363 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
364 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
365
366 if (decoder->hop)
367 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
368
369 return decoder;
370}
371
372void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,
373 uint64_t first_timestamp)
374{
375 decoder->first_timestamp = first_timestamp;
376}
377
378static void intel_pt_pop_blk(struct intel_pt_stack *stack)
379{
380 struct intel_pt_blk *blk = stack->blk;
381
382 stack->blk = blk->prev;
383 if (!stack->spare)
384 stack->spare = blk;
385 else
386 free(blk);
387}
388
389static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
390{
391 if (!stack->pos) {
392 if (!stack->blk)
393 return 0;
394 intel_pt_pop_blk(stack);
395 if (!stack->blk)
396 return 0;
397 stack->pos = INTEL_PT_BLK_SIZE;
398 }
399 return stack->blk->ip[--stack->pos];
400}
401
402static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
403{
404 struct intel_pt_blk *blk;
405
406 if (stack->spare) {
407 blk = stack->spare;
408 stack->spare = NULL;
409 } else {
410 blk = malloc(sizeof(struct intel_pt_blk));
411 if (!blk)
412 return -ENOMEM;
413 }
414
415 blk->prev = stack->blk;
416 stack->blk = blk;
417 stack->pos = 0;
418 return 0;
419}
420
421static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
422{
423 int err;
424
425 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
426 err = intel_pt_alloc_blk(stack);
427 if (err)
428 return err;
429 }
430
431 stack->blk->ip[stack->pos++] = ip;
432 return 0;
433}
434
435static void intel_pt_clear_stack(struct intel_pt_stack *stack)
436{
437 while (stack->blk)
438 intel_pt_pop_blk(stack);
439 stack->pos = 0;
440}
441
442static void intel_pt_free_stack(struct intel_pt_stack *stack)
443{
444 intel_pt_clear_stack(stack);
445 zfree(&stack->blk);
446 zfree(&stack->spare);
447}
448
449void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
450{
451 intel_pt_free_stack(&decoder->stack);
452 free(decoder);
453}
454
455static int intel_pt_ext_err(int code)
456{
457 switch (code) {
458 case -ENOMEM:
459 return INTEL_PT_ERR_NOMEM;
460 case -ENOSYS:
461 return INTEL_PT_ERR_INTERN;
462 case -EBADMSG:
463 return INTEL_PT_ERR_BADPKT;
464 case -ENODATA:
465 return INTEL_PT_ERR_NODATA;
466 case -EILSEQ:
467 return INTEL_PT_ERR_NOINSN;
468 case -ENOENT:
469 return INTEL_PT_ERR_MISMAT;
470 case -EOVERFLOW:
471 return INTEL_PT_ERR_OVR;
472 case -ENOSPC:
473 return INTEL_PT_ERR_LOST;
474 case -ELOOP:
475 return INTEL_PT_ERR_NELOOP;
476 default:
477 return INTEL_PT_ERR_UNK;
478 }
479}
480
481static const char *intel_pt_err_msgs[] = {
482 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
483 [INTEL_PT_ERR_INTERN] = "Internal error",
484 [INTEL_PT_ERR_BADPKT] = "Bad packet",
485 [INTEL_PT_ERR_NODATA] = "No more data",
486 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
487 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
488 [INTEL_PT_ERR_OVR] = "Overflow packet",
489 [INTEL_PT_ERR_LOST] = "Lost trace data",
490 [INTEL_PT_ERR_UNK] = "Unknown error!",
491 [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)",
492};
493
494int intel_pt__strerror(int code, char *buf, size_t buflen)
495{
496 if (code < 1 || code >= INTEL_PT_ERR_MAX)
497 code = INTEL_PT_ERR_UNK;
498 strlcpy(buf, intel_pt_err_msgs[code], buflen);
499 return 0;
500}
501
502static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
503 uint64_t last_ip)
504{
505 uint64_t ip;
506
507 switch (packet->count) {
508 case 1:
509 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
510 packet->payload;
511 break;
512 case 2:
513 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
514 packet->payload;
515 break;
516 case 3:
517 ip = packet->payload;
518
519 if (ip & (uint64_t)0x800000000000ULL)
520 ip |= (uint64_t)0xffff000000000000ULL;
521 break;
522 case 4:
523 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
524 packet->payload;
525 break;
526 case 6:
527 ip = packet->payload;
528 break;
529 default:
530 return 0;
531 }
532
533 return ip;
534}
535
536static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
537{
538 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
539 decoder->have_last_ip = true;
540}
541
542static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
543{
544 intel_pt_set_last_ip(decoder);
545 decoder->ip = decoder->last_ip;
546}
547
548static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
549{
550 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
551 decoder->buf);
552}
553
554static int intel_pt_bug(struct intel_pt_decoder *decoder)
555{
556 intel_pt_log("ERROR: Internal error\n");
557 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
558 return -ENOSYS;
559}
560
561static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
562{
563 decoder->tx_flags = 0;
564}
565
566static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
567{
568 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
569}
570
571static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)
572{
573 decoder->pip_payload = decoder->packet.payload;
574}
575
576static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)
577{
578 decoder->next_nr = decoder->pip_payload & 1;
579}
580
581static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)
582{
583 decoder->nr = decoder->pip_payload & 1;
584 decoder->next_nr = decoder->nr;
585}
586
587static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)
588{
589 intel_pt_update_pip(decoder);
590 intel_pt_set_nr(decoder);
591}
592
593static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
594{
595 intel_pt_clear_tx_flags(decoder);
596 decoder->have_tma = false;
597 decoder->pkt_len = 1;
598 decoder->pkt_step = 1;
599 intel_pt_decoder_log_packet(decoder);
600 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
601 intel_pt_log("ERROR: Bad packet\n");
602 decoder->pkt_state = INTEL_PT_STATE_ERR1;
603 }
604 return -EBADMSG;
605}
606
607static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
608{
609 decoder->sample_timestamp = decoder->timestamp;
610 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
611}
612
613static void intel_pt_reposition(struct intel_pt_decoder *decoder)
614{
615 decoder->ip = 0;
616 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
617 decoder->timestamp = 0;
618 decoder->have_tma = false;
619}
620
621static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
622{
623 struct intel_pt_buffer buffer = { .buf = 0, };
624 int ret;
625
626 decoder->pkt_step = 0;
627
628 intel_pt_log("Getting more data\n");
629 ret = decoder->get_trace(&buffer, decoder->data);
630 if (ret)
631 return ret;
632 decoder->buf = buffer.buf;
633 decoder->len = buffer.len;
634 if (!decoder->len) {
635 intel_pt_log("No more data\n");
636 return -ENODATA;
637 }
638 decoder->buf_timestamp = buffer.ref_timestamp;
639 if (!buffer.consecutive || reposition) {
640 intel_pt_reposition(decoder);
641 decoder->ref_timestamp = buffer.ref_timestamp;
642 decoder->state.trace_nr = buffer.trace_nr;
643 decoder->vm_tm_corr_same_buf = false;
644 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
645 decoder->ref_timestamp);
646 return -ENOLINK;
647 }
648
649 return 0;
650}
651
652static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
653 bool reposition)
654{
655 if (!decoder->next_buf)
656 return intel_pt_get_data(decoder, reposition);
657
658 decoder->buf = decoder->next_buf;
659 decoder->len = decoder->next_len;
660 decoder->next_buf = 0;
661 decoder->next_len = 0;
662 return 0;
663}
664
665static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
666{
667 unsigned char *buf = decoder->temp_buf;
668 size_t old_len, len, n;
669 int ret;
670
671 old_len = decoder->len;
672 len = decoder->len;
673 memcpy(buf, decoder->buf, len);
674
675 ret = intel_pt_get_data(decoder, false);
676 if (ret) {
677 decoder->pos += old_len;
678 return ret < 0 ? ret : -EINVAL;
679 }
680
681 n = INTEL_PT_PKT_MAX_SZ - len;
682 if (n > decoder->len)
683 n = decoder->len;
684 memcpy(buf + len, decoder->buf, n);
685 len += n;
686
687 decoder->prev_pkt_ctx = decoder->pkt_ctx;
688 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
689 if (ret < (int)old_len) {
690 decoder->next_buf = decoder->buf;
691 decoder->next_len = decoder->len;
692 decoder->buf = buf;
693 decoder->len = old_len;
694 return intel_pt_bad_packet(decoder);
695 }
696
697 decoder->next_buf = decoder->buf + (ret - old_len);
698 decoder->next_len = decoder->len - (ret - old_len);
699
700 decoder->buf = buf;
701 decoder->len = ret;
702
703 return ret;
704}
705
706struct intel_pt_pkt_info {
707 struct intel_pt_decoder *decoder;
708 struct intel_pt_pkt packet;
709 uint64_t pos;
710 int pkt_len;
711 int last_packet_type;
712 void *data;
713};
714
715typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
716
717
718static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
719 intel_pt_pkt_cb_t cb, void *data)
720{
721 struct intel_pt_pkt_info pkt_info;
722 const unsigned char *buf = decoder->buf;
723 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
724 size_t len = decoder->len;
725 int ret;
726
727 pkt_info.decoder = decoder;
728 pkt_info.pos = decoder->pos;
729 pkt_info.pkt_len = decoder->pkt_step;
730 pkt_info.last_packet_type = decoder->last_packet_type;
731 pkt_info.data = data;
732
733 while (1) {
734 do {
735 pkt_info.pos += pkt_info.pkt_len;
736 buf += pkt_info.pkt_len;
737 len -= pkt_info.pkt_len;
738
739 if (!len)
740 return INTEL_PT_NEED_MORE_BYTES;
741
742 ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
743 &pkt_ctx);
744 if (!ret)
745 return INTEL_PT_NEED_MORE_BYTES;
746 if (ret < 0)
747 return ret;
748
749 pkt_info.pkt_len = ret;
750 } while (pkt_info.packet.type == INTEL_PT_PAD);
751
752 ret = cb(&pkt_info);
753 if (ret)
754 return 0;
755
756 pkt_info.last_packet_type = pkt_info.packet.type;
757 }
758}
759
760struct intel_pt_calc_cyc_to_tsc_info {
761 uint64_t cycle_cnt;
762 unsigned int cbr;
763 uint32_t last_mtc;
764 uint64_t ctc_timestamp;
765 uint64_t ctc_delta;
766 uint64_t tsc_timestamp;
767 uint64_t timestamp;
768 bool have_tma;
769 bool fixup_last_mtc;
770 bool from_mtc;
771 double cbr_cyc_to_tsc;
772};
773
774
775
776
777
778
779
780
781static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
782 uint32_t *last_mtc)
783{
784 uint32_t first_missing_bit = 1U << (16 - mtc_shift);
785 uint32_t mask = ~(first_missing_bit - 1);
786
787 *last_mtc |= mtc & mask;
788 if (*last_mtc >= mtc) {
789 *last_mtc -= first_missing_bit;
790 *last_mtc &= 0xff;
791 }
792}
793
794static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
795{
796 struct intel_pt_decoder *decoder = pkt_info->decoder;
797 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
798 uint64_t timestamp;
799 double cyc_to_tsc;
800 unsigned int cbr;
801 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
802
803 switch (pkt_info->packet.type) {
804 case INTEL_PT_TNT:
805 case INTEL_PT_TIP_PGE:
806 case INTEL_PT_TIP:
807 case INTEL_PT_FUP:
808 case INTEL_PT_PSB:
809 case INTEL_PT_PIP:
810 case INTEL_PT_MODE_EXEC:
811 case INTEL_PT_MODE_TSX:
812 case INTEL_PT_PSBEND:
813 case INTEL_PT_PAD:
814 case INTEL_PT_VMCS:
815 case INTEL_PT_MNT:
816 case INTEL_PT_PTWRITE:
817 case INTEL_PT_PTWRITE_IP:
818 case INTEL_PT_BBP:
819 case INTEL_PT_BIP:
820 case INTEL_PT_BEP:
821 case INTEL_PT_BEP_IP:
822 return 0;
823
824 case INTEL_PT_MTC:
825 if (!data->have_tma)
826 return 0;
827
828 mtc = pkt_info->packet.payload;
829 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
830 data->fixup_last_mtc = false;
831 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
832 &data->last_mtc);
833 }
834 if (mtc > data->last_mtc)
835 mtc_delta = mtc - data->last_mtc;
836 else
837 mtc_delta = mtc + 256 - data->last_mtc;
838 data->ctc_delta += mtc_delta << decoder->mtc_shift;
839 data->last_mtc = mtc;
840
841 if (decoder->tsc_ctc_mult) {
842 timestamp = data->ctc_timestamp +
843 data->ctc_delta * decoder->tsc_ctc_mult;
844 } else {
845 timestamp = data->ctc_timestamp +
846 multdiv(data->ctc_delta,
847 decoder->tsc_ctc_ratio_n,
848 decoder->tsc_ctc_ratio_d);
849 }
850
851 if (timestamp < data->timestamp)
852 return 1;
853
854 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
855 data->timestamp = timestamp;
856 return 0;
857 }
858
859 break;
860
861 case INTEL_PT_TSC:
862
863
864
865
866 if (data->from_mtc)
867 return 1;
868 timestamp = pkt_info->packet.payload |
869 (data->timestamp & (0xffULL << 56));
870 if (data->from_mtc && timestamp < data->timestamp &&
871 data->timestamp - timestamp < decoder->tsc_slip)
872 return 1;
873 if (timestamp < data->timestamp)
874 timestamp += (1ULL << 56);
875 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
876 if (data->from_mtc)
877 return 1;
878 data->tsc_timestamp = timestamp;
879 data->timestamp = timestamp;
880 return 0;
881 }
882 break;
883
884 case INTEL_PT_TMA:
885 if (data->from_mtc)
886 return 1;
887
888 if (!decoder->tsc_ctc_ratio_d)
889 return 0;
890
891 ctc = pkt_info->packet.payload;
892 fc = pkt_info->packet.count;
893 ctc_rem = ctc & decoder->ctc_rem_mask;
894
895 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
896
897 data->ctc_timestamp = data->tsc_timestamp - fc;
898 if (decoder->tsc_ctc_mult) {
899 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
900 } else {
901 data->ctc_timestamp -=
902 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
903 decoder->tsc_ctc_ratio_d);
904 }
905
906 data->ctc_delta = 0;
907 data->have_tma = true;
908 data->fixup_last_mtc = true;
909
910 return 0;
911
912 case INTEL_PT_CYC:
913 data->cycle_cnt += pkt_info->packet.payload;
914 return 0;
915
916 case INTEL_PT_CBR:
917 cbr = pkt_info->packet.payload;
918 if (data->cbr && data->cbr != cbr)
919 return 1;
920 data->cbr = cbr;
921 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
922 return 0;
923
924 case INTEL_PT_TIP_PGD:
925 case INTEL_PT_TRACESTOP:
926 case INTEL_PT_EXSTOP:
927 case INTEL_PT_EXSTOP_IP:
928 case INTEL_PT_MWAIT:
929 case INTEL_PT_PWRE:
930 case INTEL_PT_PWRX:
931 case INTEL_PT_OVF:
932 case INTEL_PT_BAD:
933 default:
934 return 1;
935 }
936
937 if (!data->cbr && decoder->cbr) {
938 data->cbr = decoder->cbr;
939 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
940 }
941
942 if (!data->cycle_cnt)
943 return 1;
944
945 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
946
947 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
948 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
949 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
950 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
951 return 1;
952 }
953
954 decoder->calc_cyc_to_tsc = cyc_to_tsc;
955 decoder->have_calc_cyc_to_tsc = true;
956
957 if (data->cbr) {
958 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
959 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
960 } else {
961 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
962 cyc_to_tsc, pkt_info->pos);
963 }
964
965 return 1;
966}
967
968static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
969 bool from_mtc)
970{
971 struct intel_pt_calc_cyc_to_tsc_info data = {
972 .cycle_cnt = 0,
973 .cbr = 0,
974 .last_mtc = decoder->last_mtc,
975 .ctc_timestamp = decoder->ctc_timestamp,
976 .ctc_delta = decoder->ctc_delta,
977 .tsc_timestamp = decoder->tsc_timestamp,
978 .timestamp = decoder->timestamp,
979 .have_tma = decoder->have_tma,
980 .fixup_last_mtc = decoder->fixup_last_mtc,
981 .from_mtc = from_mtc,
982 .cbr_cyc_to_tsc = 0,
983 };
984
985
986
987
988
989
990 if (!from_mtc)
991 return;
992
993 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
994}
995
996static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
997{
998 int ret;
999
1000 decoder->last_packet_type = decoder->packet.type;
1001
1002 do {
1003 decoder->pos += decoder->pkt_step;
1004 decoder->buf += decoder->pkt_step;
1005 decoder->len -= decoder->pkt_step;
1006
1007 if (!decoder->len) {
1008 ret = intel_pt_get_next_data(decoder, false);
1009 if (ret)
1010 return ret;
1011 }
1012
1013 decoder->prev_pkt_ctx = decoder->pkt_ctx;
1014 ret = intel_pt_get_packet(decoder->buf, decoder->len,
1015 &decoder->packet, &decoder->pkt_ctx);
1016 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
1017 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
1018 ret = intel_pt_get_split_packet(decoder);
1019 if (ret < 0)
1020 return ret;
1021 }
1022 if (ret <= 0)
1023 return intel_pt_bad_packet(decoder);
1024
1025 decoder->pkt_len = ret;
1026 decoder->pkt_step = ret;
1027 intel_pt_decoder_log_packet(decoder);
1028 } while (decoder->packet.type == INTEL_PT_PAD);
1029
1030 return 0;
1031}
1032
1033static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
1034{
1035 uint64_t timestamp, masked_timestamp;
1036
1037 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1038 masked_timestamp = timestamp & decoder->period_mask;
1039 if (decoder->continuous_period) {
1040 if (masked_timestamp > decoder->last_masked_timestamp)
1041 return 1;
1042 } else {
1043 timestamp += 1;
1044 masked_timestamp = timestamp & decoder->period_mask;
1045 if (masked_timestamp > decoder->last_masked_timestamp) {
1046 decoder->last_masked_timestamp = masked_timestamp;
1047 decoder->continuous_period = true;
1048 }
1049 }
1050
1051 if (masked_timestamp < decoder->last_masked_timestamp)
1052 return decoder->period_ticks;
1053
1054 return decoder->period_ticks - (timestamp - masked_timestamp);
1055}
1056
1057static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
1058{
1059 switch (decoder->period_type) {
1060 case INTEL_PT_PERIOD_INSTRUCTIONS:
1061 return decoder->period - decoder->period_insn_cnt;
1062 case INTEL_PT_PERIOD_TICKS:
1063 return intel_pt_next_period(decoder);
1064 case INTEL_PT_PERIOD_NONE:
1065 case INTEL_PT_PERIOD_MTC:
1066 default:
1067 return 0;
1068 }
1069}
1070
1071static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
1072{
1073 uint64_t timestamp, masked_timestamp;
1074
1075 switch (decoder->period_type) {
1076 case INTEL_PT_PERIOD_INSTRUCTIONS:
1077 decoder->period_insn_cnt = 0;
1078 break;
1079 case INTEL_PT_PERIOD_TICKS:
1080 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1081 masked_timestamp = timestamp & decoder->period_mask;
1082 if (masked_timestamp > decoder->last_masked_timestamp)
1083 decoder->last_masked_timestamp = masked_timestamp;
1084 else
1085 decoder->last_masked_timestamp += decoder->period_ticks;
1086 break;
1087 case INTEL_PT_PERIOD_NONE:
1088 case INTEL_PT_PERIOD_MTC:
1089 default:
1090 break;
1091 }
1092
1093 decoder->state.type |= INTEL_PT_INSTRUCTION;
1094}
1095
1096static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
1097 struct intel_pt_insn *intel_pt_insn, uint64_t ip)
1098{
1099 uint64_t max_insn_cnt, insn_cnt = 0;
1100 int err;
1101
1102 if (!decoder->mtc_insn)
1103 decoder->mtc_insn = true;
1104
1105 max_insn_cnt = intel_pt_next_sample(decoder);
1106
1107 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
1108 max_insn_cnt, decoder->data);
1109
1110 decoder->tot_insn_cnt += insn_cnt;
1111 decoder->timestamp_insn_cnt += insn_cnt;
1112 decoder->sample_insn_cnt += insn_cnt;
1113 decoder->period_insn_cnt += insn_cnt;
1114
1115 if (err) {
1116 decoder->no_progress = 0;
1117 decoder->pkt_state = INTEL_PT_STATE_ERR2;
1118 intel_pt_log_at("ERROR: Failed to get instruction",
1119 decoder->ip);
1120 if (err == -ENOENT)
1121 return -ENOLINK;
1122 return -EILSEQ;
1123 }
1124
1125 if (ip && decoder->ip == ip) {
1126 err = -EAGAIN;
1127 goto out;
1128 }
1129
1130 if (max_insn_cnt && insn_cnt >= max_insn_cnt)
1131 intel_pt_sample_insn(decoder);
1132
1133 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1134 decoder->state.type = INTEL_PT_INSTRUCTION;
1135 decoder->state.from_ip = decoder->ip;
1136 decoder->state.to_ip = 0;
1137 decoder->ip += intel_pt_insn->length;
1138 err = INTEL_PT_RETURN;
1139 goto out;
1140 }
1141
1142 if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1143
1144 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
1145 intel_pt_insn->rel) {
1146 err = intel_pt_push(&decoder->stack, decoder->ip +
1147 intel_pt_insn->length);
1148 if (err)
1149 goto out;
1150 }
1151 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1152 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1153 }
1154
1155 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1156 int cnt = decoder->no_progress++;
1157
1158 decoder->state.from_ip = decoder->ip;
1159 decoder->ip += intel_pt_insn->length +
1160 intel_pt_insn->rel;
1161 decoder->state.to_ip = decoder->ip;
1162 err = INTEL_PT_RETURN;
1163
1164
1165
1166
1167
1168
1169
1170
1171 if (cnt) {
1172 if (cnt == 1) {
1173 decoder->stuck_ip = decoder->state.to_ip;
1174 decoder->stuck_ip_prd = 1;
1175 decoder->stuck_ip_cnt = 1;
1176 } else if (cnt > decoder->max_loops ||
1177 decoder->state.to_ip == decoder->stuck_ip) {
1178 intel_pt_log_at("ERROR: Never-ending loop",
1179 decoder->state.to_ip);
1180 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1181 err = -ELOOP;
1182 goto out;
1183 } else if (!--decoder->stuck_ip_cnt) {
1184 decoder->stuck_ip_prd += 1;
1185 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1186 decoder->stuck_ip = decoder->state.to_ip;
1187 }
1188 }
1189 goto out_no_progress;
1190 }
1191out:
1192 decoder->no_progress = 0;
1193out_no_progress:
1194 decoder->state.insn_op = intel_pt_insn->op;
1195 decoder->state.insn_len = intel_pt_insn->length;
1196 memcpy(decoder->state.insn, intel_pt_insn->buf,
1197 INTEL_PT_INSN_BUF_SZ);
1198
1199 if (decoder->tx_flags & INTEL_PT_IN_TX)
1200 decoder->state.flags |= INTEL_PT_IN_TX;
1201
1202 return err;
1203}
1204
1205static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1206{
1207 bool ret = false;
1208
1209 if (decoder->set_fup_tx_flags) {
1210 decoder->set_fup_tx_flags = false;
1211 decoder->tx_flags = decoder->fup_tx_flags;
1212 decoder->state.type = INTEL_PT_TRANSACTION;
1213 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1214 decoder->state.type |= INTEL_PT_BRANCH;
1215 decoder->state.from_ip = decoder->ip;
1216 decoder->state.to_ip = 0;
1217 decoder->state.flags = decoder->fup_tx_flags;
1218 return true;
1219 }
1220 if (decoder->set_fup_ptw) {
1221 decoder->set_fup_ptw = false;
1222 decoder->state.type = INTEL_PT_PTW;
1223 decoder->state.flags |= INTEL_PT_FUP_IP;
1224 decoder->state.from_ip = decoder->ip;
1225 decoder->state.to_ip = 0;
1226 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1227 return true;
1228 }
1229 if (decoder->set_fup_mwait) {
1230 decoder->set_fup_mwait = false;
1231 decoder->state.type = INTEL_PT_MWAIT_OP;
1232 decoder->state.from_ip = decoder->ip;
1233 decoder->state.to_ip = 0;
1234 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1235 ret = true;
1236 }
1237 if (decoder->set_fup_pwre) {
1238 decoder->set_fup_pwre = false;
1239 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1240 decoder->state.type &= ~INTEL_PT_BRANCH;
1241 decoder->state.from_ip = decoder->ip;
1242 decoder->state.to_ip = 0;
1243 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1244 ret = true;
1245 }
1246 if (decoder->set_fup_exstop) {
1247 decoder->set_fup_exstop = false;
1248 decoder->state.type |= INTEL_PT_EX_STOP;
1249 decoder->state.type &= ~INTEL_PT_BRANCH;
1250 decoder->state.flags |= INTEL_PT_FUP_IP;
1251 decoder->state.from_ip = decoder->ip;
1252 decoder->state.to_ip = 0;
1253 ret = true;
1254 }
1255 if (decoder->set_fup_bep) {
1256 decoder->set_fup_bep = false;
1257 decoder->state.type |= INTEL_PT_BLK_ITEMS;
1258 decoder->state.type &= ~INTEL_PT_BRANCH;
1259 decoder->state.from_ip = decoder->ip;
1260 decoder->state.to_ip = 0;
1261 ret = true;
1262 }
1263 return ret;
1264}
1265
1266static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1267 struct intel_pt_insn *intel_pt_insn,
1268 uint64_t ip, int err)
1269{
1270 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1271 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1272 ip == decoder->ip + intel_pt_insn->length;
1273}
1274
1275static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1276{
1277 struct intel_pt_insn intel_pt_insn;
1278 uint64_t ip;
1279 int err;
1280
1281 ip = decoder->last_ip;
1282
1283 while (1) {
1284 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1285 if (err == INTEL_PT_RETURN)
1286 return 0;
1287 if (err == -EAGAIN ||
1288 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1289 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1290
1291 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1292 if (intel_pt_fup_event(decoder) && no_tip)
1293 return 0;
1294 return -EAGAIN;
1295 }
1296 decoder->set_fup_tx_flags = false;
1297 if (err)
1298 return err;
1299
1300 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1301 intel_pt_log_at("ERROR: Unexpected indirect branch",
1302 decoder->ip);
1303 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1304 return -ENOENT;
1305 }
1306
1307 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1308 intel_pt_log_at("ERROR: Unexpected conditional branch",
1309 decoder->ip);
1310 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1311 return -ENOENT;
1312 }
1313
1314 intel_pt_bug(decoder);
1315 }
1316}
1317
1318static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1319{
1320 struct intel_pt_insn intel_pt_insn;
1321 int err;
1322
1323 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1324 if (err == INTEL_PT_RETURN &&
1325 decoder->pgd_ip &&
1326 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1327 (decoder->state.type & INTEL_PT_BRANCH) &&
1328 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1329
1330 decoder->no_progress = 0;
1331 decoder->pge = false;
1332 decoder->continuous_period = false;
1333 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1334 decoder->state.type |= INTEL_PT_TRACE_END;
1335 intel_pt_update_nr(decoder);
1336 return 0;
1337 }
1338 if (err == INTEL_PT_RETURN)
1339 return 0;
1340 if (err)
1341 return err;
1342
1343 intel_pt_update_nr(decoder);
1344
1345 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1346 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1347 decoder->pge = false;
1348 decoder->continuous_period = false;
1349 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1350 decoder->state.from_ip = decoder->ip;
1351 if (decoder->packet.count == 0) {
1352 decoder->state.to_ip = 0;
1353 } else {
1354 decoder->state.to_ip = decoder->last_ip;
1355 decoder->ip = decoder->last_ip;
1356 }
1357 decoder->state.type |= INTEL_PT_TRACE_END;
1358 } else {
1359 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1360 decoder->state.from_ip = decoder->ip;
1361 if (decoder->packet.count == 0) {
1362 decoder->state.to_ip = 0;
1363 } else {
1364 decoder->state.to_ip = decoder->last_ip;
1365 decoder->ip = decoder->last_ip;
1366 }
1367 }
1368 return 0;
1369 }
1370
1371 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1372 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1373 intel_pt_insn.rel;
1374
1375 if (decoder->pgd_ip &&
1376 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1377 decoder->pgd_ip(to_ip, decoder->data)) {
1378
1379 decoder->pge = false;
1380 decoder->continuous_period = false;
1381 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1382 decoder->ip = to_ip;
1383 decoder->state.from_ip = decoder->ip;
1384 decoder->state.to_ip = to_ip;
1385 decoder->state.type |= INTEL_PT_TRACE_END;
1386 return 0;
1387 }
1388 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1389 decoder->ip);
1390 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1391 return -ENOENT;
1392 }
1393
1394 return intel_pt_bug(decoder);
1395}
1396
1397static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1398{
1399 struct intel_pt_insn intel_pt_insn;
1400 int err;
1401
1402 while (1) {
1403 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1404 if (err == INTEL_PT_RETURN)
1405 return 0;
1406 if (err)
1407 return err;
1408
1409 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1410 if (!decoder->return_compression) {
1411 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1412 decoder->ip);
1413 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1414 return -ENOENT;
1415 }
1416 if (!decoder->ret_addr) {
1417 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1418 decoder->ip);
1419 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1420 return -ENOENT;
1421 }
1422 if (!(decoder->tnt.payload & BIT63)) {
1423 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1424 decoder->ip);
1425 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1426 return -ENOENT;
1427 }
1428 decoder->tnt.count -= 1;
1429 if (decoder->tnt.count)
1430 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1431 else
1432 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1433 decoder->tnt.payload <<= 1;
1434 decoder->state.from_ip = decoder->ip;
1435 decoder->ip = decoder->ret_addr;
1436 decoder->state.to_ip = decoder->ip;
1437 return 0;
1438 }
1439
1440 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1441
1442 err = intel_pt_get_next_packet(decoder);
1443 if (err)
1444 return err;
1445 if (decoder->packet.type != INTEL_PT_TIP ||
1446 decoder->packet.count == 0) {
1447 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1448 decoder->ip);
1449 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1450 decoder->pkt_step = 0;
1451 return -ENOENT;
1452 }
1453 intel_pt_set_last_ip(decoder);
1454 decoder->state.from_ip = decoder->ip;
1455 decoder->state.to_ip = decoder->last_ip;
1456 decoder->ip = decoder->last_ip;
1457 intel_pt_update_nr(decoder);
1458 return 0;
1459 }
1460
1461 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1462 decoder->tnt.count -= 1;
1463 if (decoder->tnt.count)
1464 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1465 else
1466 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1467 if (decoder->tnt.payload & BIT63) {
1468 decoder->tnt.payload <<= 1;
1469 decoder->state.from_ip = decoder->ip;
1470 decoder->ip += intel_pt_insn.length +
1471 intel_pt_insn.rel;
1472 decoder->state.to_ip = decoder->ip;
1473 return 0;
1474 }
1475
1476 if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1477 decoder->tnt.payload <<= 1;
1478 decoder->state.type = INTEL_PT_INSTRUCTION;
1479 decoder->state.from_ip = decoder->ip;
1480 decoder->state.to_ip = 0;
1481 decoder->ip += intel_pt_insn.length;
1482 return 0;
1483 }
1484 decoder->sample_cyc = false;
1485 decoder->ip += intel_pt_insn.length;
1486 if (!decoder->tnt.count) {
1487 intel_pt_update_sample_time(decoder);
1488 return -EAGAIN;
1489 }
1490 decoder->tnt.payload <<= 1;
1491 continue;
1492 }
1493
1494 return intel_pt_bug(decoder);
1495 }
1496}
1497
1498static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1499{
1500 unsigned int fup_tx_flags;
1501 int err;
1502
1503 fup_tx_flags = decoder->packet.payload &
1504 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1505 err = intel_pt_get_next_packet(decoder);
1506 if (err)
1507 return err;
1508 if (decoder->packet.type == INTEL_PT_FUP) {
1509 decoder->fup_tx_flags = fup_tx_flags;
1510 decoder->set_fup_tx_flags = true;
1511 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1512 *no_tip = true;
1513 } else {
1514 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1515 decoder->pos);
1516 intel_pt_update_in_tx(decoder);
1517 }
1518 return 0;
1519}
1520
1521static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
1522{
1523 timestamp |= (ref_timestamp & (0xffULL << 56));
1524
1525 if (timestamp < ref_timestamp) {
1526 if (ref_timestamp - timestamp > (1ULL << 55))
1527 timestamp += (1ULL << 56);
1528 } else {
1529 if (timestamp - ref_timestamp > (1ULL << 55))
1530 timestamp -= (1ULL << 56);
1531 }
1532
1533 return timestamp;
1534}
1535
1536
1537static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,
1538 uint64_t timestamp)
1539{
1540 uint64_t max_timestamp = decoder->buf_timestamp;
1541
1542 if (!max_timestamp) {
1543 max_timestamp = decoder->last_reliable_timestamp +
1544 0x400000000ULL;
1545 }
1546 return timestamp >= decoder->last_reliable_timestamp &&
1547 timestamp < decoder->buf_timestamp;
1548}
1549
1550static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1551{
1552 uint64_t timestamp;
1553 bool bad = false;
1554
1555 decoder->have_tma = false;
1556
1557 if (decoder->ref_timestamp) {
1558 timestamp = intel_pt_8b_tsc(decoder->packet.payload,
1559 decoder->ref_timestamp);
1560 decoder->tsc_timestamp = timestamp;
1561 decoder->timestamp = timestamp;
1562 decoder->ref_timestamp = 0;
1563 decoder->timestamp_insn_cnt = 0;
1564 } else if (decoder->timestamp) {
1565 timestamp = decoder->packet.payload |
1566 (decoder->timestamp & (0xffULL << 56));
1567 decoder->tsc_timestamp = timestamp;
1568 if (timestamp < decoder->timestamp &&
1569 decoder->timestamp - timestamp < decoder->tsc_slip) {
1570 intel_pt_log_to("Suppressing backwards timestamp",
1571 timestamp);
1572 timestamp = decoder->timestamp;
1573 }
1574 if (timestamp < decoder->timestamp) {
1575 if (!decoder->buf_timestamp ||
1576 (timestamp + (1ULL << 56) < decoder->buf_timestamp)) {
1577 intel_pt_log_to("Wraparound timestamp", timestamp);
1578 timestamp += (1ULL << 56);
1579 decoder->tsc_timestamp = timestamp;
1580 } else {
1581 intel_pt_log_to("Suppressing bad timestamp", timestamp);
1582 timestamp = decoder->timestamp;
1583 bad = true;
1584 }
1585 }
1586 if (decoder->vm_time_correlation &&
1587 (bad || !intel_pt_time_in_range(decoder, timestamp)) &&
1588 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
1589 p_log("Timestamp out of range");
1590 decoder->timestamp = timestamp;
1591 decoder->timestamp_insn_cnt = 0;
1592 }
1593
1594 if (decoder->last_packet_type == INTEL_PT_CYC) {
1595 decoder->cyc_ref_timestamp = decoder->timestamp;
1596 decoder->cycle_cnt = 0;
1597 decoder->have_calc_cyc_to_tsc = false;
1598 intel_pt_calc_cyc_to_tsc(decoder, false);
1599 }
1600
1601 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1602}
1603
1604static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1605{
1606 intel_pt_log("ERROR: Buffer overflow\n");
1607 intel_pt_clear_tx_flags(decoder);
1608 intel_pt_set_nr(decoder);
1609 decoder->timestamp_insn_cnt = 0;
1610 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1611 decoder->overflow = true;
1612 return -EOVERFLOW;
1613}
1614
1615static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1616{
1617 if (decoder->have_cyc)
1618 return;
1619
1620 decoder->cyc_cnt_timestamp = decoder->timestamp;
1621 decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1622}
1623
1624static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1625{
1626 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1627
1628 if (decoder->pge)
1629 intel_pt_mtc_cyc_cnt_pge(decoder);
1630}
1631
1632static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1633{
1634 uint64_t tot_cyc_cnt, tsc_delta;
1635
1636 if (decoder->have_cyc)
1637 return;
1638
1639 decoder->sample_cyc = true;
1640
1641 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)
1642 return;
1643
1644 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1645 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1646
1647 if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1648 decoder->tot_cyc_cnt = tot_cyc_cnt;
1649}
1650
1651static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1652{
1653 uint32_t ctc = decoder->packet.payload;
1654 uint32_t fc = decoder->packet.count;
1655 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1656
1657 if (!decoder->tsc_ctc_ratio_d)
1658 return;
1659
1660 if (decoder->pge && !decoder->in_psb)
1661 intel_pt_mtc_cyc_cnt_pge(decoder);
1662 else
1663 intel_pt_mtc_cyc_cnt_upd(decoder);
1664
1665 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1666 decoder->last_ctc = ctc - ctc_rem;
1667 decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1668 if (decoder->tsc_ctc_mult) {
1669 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1670 } else {
1671 decoder->ctc_timestamp -= multdiv(ctc_rem,
1672 decoder->tsc_ctc_ratio_n,
1673 decoder->tsc_ctc_ratio_d);
1674 }
1675 decoder->ctc_delta = 0;
1676 decoder->have_tma = true;
1677 decoder->fixup_last_mtc = true;
1678 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n",
1679 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1680}
1681
1682static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1683{
1684 uint64_t timestamp;
1685 uint32_t mtc, mtc_delta;
1686
1687 if (!decoder->have_tma)
1688 return;
1689
1690 mtc = decoder->packet.payload;
1691
1692 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1693 decoder->fixup_last_mtc = false;
1694 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1695 &decoder->last_mtc);
1696 }
1697
1698 if (mtc > decoder->last_mtc)
1699 mtc_delta = mtc - decoder->last_mtc;
1700 else
1701 mtc_delta = mtc + 256 - decoder->last_mtc;
1702
1703 decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1704
1705 if (decoder->tsc_ctc_mult) {
1706 timestamp = decoder->ctc_timestamp +
1707 decoder->ctc_delta * decoder->tsc_ctc_mult;
1708 } else {
1709 timestamp = decoder->ctc_timestamp +
1710 multdiv(decoder->ctc_delta,
1711 decoder->tsc_ctc_ratio_n,
1712 decoder->tsc_ctc_ratio_d);
1713 }
1714
1715 if (timestamp < decoder->timestamp)
1716 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1717 timestamp, decoder->timestamp);
1718 else
1719 decoder->timestamp = timestamp;
1720
1721 intel_pt_mtc_cyc_cnt_upd(decoder);
1722
1723 decoder->timestamp_insn_cnt = 0;
1724 decoder->last_mtc = mtc;
1725
1726 if (decoder->last_packet_type == INTEL_PT_CYC) {
1727 decoder->cyc_ref_timestamp = decoder->timestamp;
1728 decoder->cycle_cnt = 0;
1729 decoder->have_calc_cyc_to_tsc = false;
1730 intel_pt_calc_cyc_to_tsc(decoder, true);
1731 }
1732
1733 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1734}
1735
1736static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1737{
1738 unsigned int cbr = decoder->packet.payload & 0xff;
1739
1740 decoder->cbr_payload = decoder->packet.payload;
1741
1742 if (decoder->cbr == cbr)
1743 return;
1744
1745 decoder->cbr = cbr;
1746 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1747
1748 intel_pt_mtc_cyc_cnt_cbr(decoder);
1749}
1750
1751static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1752{
1753 uint64_t timestamp = decoder->cyc_ref_timestamp;
1754
1755 decoder->have_cyc = true;
1756
1757 decoder->cycle_cnt += decoder->packet.payload;
1758 if (decoder->pge)
1759 decoder->tot_cyc_cnt += decoder->packet.payload;
1760 decoder->sample_cyc = true;
1761
1762 if (!decoder->cyc_ref_timestamp)
1763 return;
1764
1765 if (decoder->have_calc_cyc_to_tsc)
1766 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1767 else if (decoder->cbr)
1768 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1769 else
1770 return;
1771
1772 if (timestamp < decoder->timestamp)
1773 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1774 timestamp, decoder->timestamp);
1775 else
1776 decoder->timestamp = timestamp;
1777
1778 decoder->timestamp_insn_cnt = 0;
1779
1780 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1781}
1782
1783static void intel_pt_bbp(struct intel_pt_decoder *decoder)
1784{
1785 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
1786 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
1787 decoder->state.items.is_32_bit = false;
1788 }
1789 decoder->blk_type = decoder->packet.payload;
1790 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
1791 if (decoder->blk_type == INTEL_PT_GP_REGS)
1792 decoder->state.items.is_32_bit = decoder->packet.count;
1793 if (decoder->blk_type_pos < 0) {
1794 intel_pt_log("WARNING: Unknown block type %u\n",
1795 decoder->blk_type);
1796 } else if (decoder->state.items.mask[decoder->blk_type_pos]) {
1797 intel_pt_log("WARNING: Duplicate block type %u\n",
1798 decoder->blk_type);
1799 }
1800}
1801
1802static void intel_pt_bip(struct intel_pt_decoder *decoder)
1803{
1804 uint32_t id = decoder->packet.count;
1805 uint32_t bit = 1 << id;
1806 int pos = decoder->blk_type_pos;
1807
1808 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) {
1809 intel_pt_log("WARNING: Unknown block item %u type %d\n",
1810 id, decoder->blk_type);
1811 return;
1812 }
1813
1814 if (decoder->state.items.mask[pos] & bit) {
1815 intel_pt_log("WARNING: Duplicate block item %u type %d\n",
1816 id, decoder->blk_type);
1817 }
1818
1819 decoder->state.items.mask[pos] |= bit;
1820 decoder->state.items.val[pos][id] = decoder->packet.payload;
1821}
1822
1823
1824static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1825{
1826 int err;
1827
1828 decoder->in_psb = true;
1829
1830 while (1) {
1831 err = intel_pt_get_next_packet(decoder);
1832 if (err)
1833 goto out;
1834
1835 switch (decoder->packet.type) {
1836 case INTEL_PT_PSBEND:
1837 err = 0;
1838 goto out;
1839
1840 case INTEL_PT_TIP_PGD:
1841 case INTEL_PT_TIP_PGE:
1842 case INTEL_PT_TIP:
1843 case INTEL_PT_TNT:
1844 case INTEL_PT_TRACESTOP:
1845 case INTEL_PT_BAD:
1846 case INTEL_PT_PSB:
1847 case INTEL_PT_PTWRITE:
1848 case INTEL_PT_PTWRITE_IP:
1849 case INTEL_PT_EXSTOP:
1850 case INTEL_PT_EXSTOP_IP:
1851 case INTEL_PT_MWAIT:
1852 case INTEL_PT_PWRE:
1853 case INTEL_PT_PWRX:
1854 case INTEL_PT_BBP:
1855 case INTEL_PT_BIP:
1856 case INTEL_PT_BEP:
1857 case INTEL_PT_BEP_IP:
1858 decoder->have_tma = false;
1859 intel_pt_log("ERROR: Unexpected packet\n");
1860 err = -EAGAIN;
1861 goto out;
1862
1863 case INTEL_PT_OVF:
1864 err = intel_pt_overflow(decoder);
1865 goto out;
1866
1867 case INTEL_PT_TSC:
1868 intel_pt_calc_tsc_timestamp(decoder);
1869 break;
1870
1871 case INTEL_PT_TMA:
1872 intel_pt_calc_tma(decoder);
1873 break;
1874
1875 case INTEL_PT_CBR:
1876 intel_pt_calc_cbr(decoder);
1877 break;
1878
1879 case INTEL_PT_MODE_EXEC:
1880 decoder->exec_mode = decoder->packet.payload;
1881 break;
1882
1883 case INTEL_PT_PIP:
1884 intel_pt_set_pip(decoder);
1885 break;
1886
1887 case INTEL_PT_FUP:
1888 decoder->pge = true;
1889 if (decoder->packet.count) {
1890 intel_pt_set_last_ip(decoder);
1891 decoder->psb_ip = decoder->last_ip;
1892 }
1893 break;
1894
1895 case INTEL_PT_MODE_TSX:
1896 intel_pt_update_in_tx(decoder);
1897 break;
1898
1899 case INTEL_PT_MTC:
1900 intel_pt_calc_mtc_timestamp(decoder);
1901 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1902 decoder->state.type |= INTEL_PT_INSTRUCTION;
1903 break;
1904
1905 case INTEL_PT_CYC:
1906 intel_pt_calc_cyc_timestamp(decoder);
1907 break;
1908
1909 case INTEL_PT_VMCS:
1910 case INTEL_PT_MNT:
1911 case INTEL_PT_PAD:
1912 default:
1913 break;
1914 }
1915 }
1916out:
1917 decoder->in_psb = false;
1918
1919 return err;
1920}
1921
1922static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1923{
1924 int err;
1925
1926 if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1927 decoder->tx_flags = 0;
1928 decoder->state.flags &= ~INTEL_PT_IN_TX;
1929 decoder->state.flags |= INTEL_PT_ABORT_TX;
1930 } else {
1931 decoder->state.flags |= INTEL_PT_ASYNC;
1932 }
1933
1934 while (1) {
1935 err = intel_pt_get_next_packet(decoder);
1936 if (err)
1937 return err;
1938
1939 switch (decoder->packet.type) {
1940 case INTEL_PT_TNT:
1941 case INTEL_PT_FUP:
1942 case INTEL_PT_TRACESTOP:
1943 case INTEL_PT_PSB:
1944 case INTEL_PT_TSC:
1945 case INTEL_PT_TMA:
1946 case INTEL_PT_MODE_TSX:
1947 case INTEL_PT_BAD:
1948 case INTEL_PT_PSBEND:
1949 case INTEL_PT_PTWRITE:
1950 case INTEL_PT_PTWRITE_IP:
1951 case INTEL_PT_EXSTOP:
1952 case INTEL_PT_EXSTOP_IP:
1953 case INTEL_PT_MWAIT:
1954 case INTEL_PT_PWRE:
1955 case INTEL_PT_PWRX:
1956 case INTEL_PT_BBP:
1957 case INTEL_PT_BIP:
1958 case INTEL_PT_BEP:
1959 case INTEL_PT_BEP_IP:
1960 intel_pt_log("ERROR: Missing TIP after FUP\n");
1961 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1962 decoder->pkt_step = 0;
1963 return -ENOENT;
1964
1965 case INTEL_PT_CBR:
1966 intel_pt_calc_cbr(decoder);
1967 break;
1968
1969 case INTEL_PT_OVF:
1970 return intel_pt_overflow(decoder);
1971
1972 case INTEL_PT_TIP_PGD:
1973 decoder->state.from_ip = decoder->ip;
1974 if (decoder->packet.count == 0) {
1975 decoder->state.to_ip = 0;
1976 } else {
1977 intel_pt_set_ip(decoder);
1978 decoder->state.to_ip = decoder->ip;
1979 }
1980 decoder->pge = false;
1981 decoder->continuous_period = false;
1982 decoder->state.type |= INTEL_PT_TRACE_END;
1983 intel_pt_update_nr(decoder);
1984 return 0;
1985
1986 case INTEL_PT_TIP_PGE:
1987 decoder->pge = true;
1988 intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1989 decoder->ip);
1990 decoder->state.from_ip = 0;
1991 if (decoder->packet.count == 0) {
1992 decoder->state.to_ip = 0;
1993 } else {
1994 intel_pt_set_ip(decoder);
1995 decoder->state.to_ip = decoder->ip;
1996 }
1997 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1998 intel_pt_mtc_cyc_cnt_pge(decoder);
1999 intel_pt_set_nr(decoder);
2000 return 0;
2001
2002 case INTEL_PT_TIP:
2003 decoder->state.from_ip = decoder->ip;
2004 if (decoder->packet.count == 0) {
2005 decoder->state.to_ip = 0;
2006 } else {
2007 intel_pt_set_ip(decoder);
2008 decoder->state.to_ip = decoder->ip;
2009 }
2010 intel_pt_update_nr(decoder);
2011 return 0;
2012
2013 case INTEL_PT_PIP:
2014 intel_pt_update_pip(decoder);
2015 break;
2016
2017 case INTEL_PT_MTC:
2018 intel_pt_calc_mtc_timestamp(decoder);
2019 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
2020 decoder->state.type |= INTEL_PT_INSTRUCTION;
2021 break;
2022
2023 case INTEL_PT_CYC:
2024 intel_pt_calc_cyc_timestamp(decoder);
2025 break;
2026
2027 case INTEL_PT_MODE_EXEC:
2028 decoder->exec_mode = decoder->packet.payload;
2029 break;
2030
2031 case INTEL_PT_VMCS:
2032 case INTEL_PT_MNT:
2033 case INTEL_PT_PAD:
2034 break;
2035
2036 default:
2037 return intel_pt_bug(decoder);
2038 }
2039 }
2040}
2041
2042static int intel_pt_resample(struct intel_pt_decoder *decoder)
2043{
2044 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2045 decoder->state.type = INTEL_PT_INSTRUCTION;
2046 decoder->state.from_ip = decoder->ip;
2047 decoder->state.to_ip = 0;
2048 return 0;
2049}
2050
2051struct intel_pt_vm_tsc_info {
2052 struct intel_pt_pkt pip_packet;
2053 struct intel_pt_pkt vmcs_packet;
2054 struct intel_pt_pkt tma_packet;
2055 bool tsc, pip, vmcs, tma, psbend;
2056 uint64_t ctc_delta;
2057 uint64_t last_ctc;
2058 int max_lookahead;
2059};
2060
2061
2062static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2063{
2064 struct intel_pt_vm_tsc_info *data = pkt_info->data;
2065
2066 switch (pkt_info->packet.type) {
2067 case INTEL_PT_PAD:
2068 case INTEL_PT_MNT:
2069 case INTEL_PT_MODE_EXEC:
2070 case INTEL_PT_MODE_TSX:
2071 case INTEL_PT_MTC:
2072 case INTEL_PT_FUP:
2073 case INTEL_PT_CYC:
2074 case INTEL_PT_CBR:
2075 break;
2076
2077 case INTEL_PT_TSC:
2078 data->tsc = true;
2079 break;
2080
2081 case INTEL_PT_TMA:
2082 data->tma_packet = pkt_info->packet;
2083 data->tma = true;
2084 break;
2085
2086 case INTEL_PT_PIP:
2087 data->pip_packet = pkt_info->packet;
2088 data->pip = true;
2089 break;
2090
2091 case INTEL_PT_VMCS:
2092 data->vmcs_packet = pkt_info->packet;
2093 data->vmcs = true;
2094 break;
2095
2096 case INTEL_PT_PSBEND:
2097 data->psbend = true;
2098 return 1;
2099
2100 case INTEL_PT_TIP_PGE:
2101 case INTEL_PT_PTWRITE:
2102 case INTEL_PT_PTWRITE_IP:
2103 case INTEL_PT_EXSTOP:
2104 case INTEL_PT_EXSTOP_IP:
2105 case INTEL_PT_MWAIT:
2106 case INTEL_PT_PWRE:
2107 case INTEL_PT_PWRX:
2108 case INTEL_PT_BBP:
2109 case INTEL_PT_BIP:
2110 case INTEL_PT_BEP:
2111 case INTEL_PT_BEP_IP:
2112 case INTEL_PT_OVF:
2113 case INTEL_PT_BAD:
2114 case INTEL_PT_TNT:
2115 case INTEL_PT_TIP_PGD:
2116 case INTEL_PT_TIP:
2117 case INTEL_PT_PSB:
2118 case INTEL_PT_TRACESTOP:
2119 default:
2120 return 1;
2121 }
2122
2123 return 0;
2124}
2125
2126struct intel_pt_ovf_fup_info {
2127 int max_lookahead;
2128 bool found;
2129};
2130
2131
2132static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2133{
2134 struct intel_pt_ovf_fup_info *data = pkt_info->data;
2135
2136 if (pkt_info->packet.type == INTEL_PT_CYC ||
2137 pkt_info->packet.type == INTEL_PT_MTC ||
2138 pkt_info->packet.type == INTEL_PT_TSC)
2139 return !--(data->max_lookahead);
2140 data->found = pkt_info->packet.type == INTEL_PT_FUP;
2141 return 1;
2142}
2143
2144static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)
2145{
2146 struct intel_pt_ovf_fup_info data = {
2147 .max_lookahead = 16,
2148 .found = false,
2149 };
2150
2151 intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);
2152 return data.found;
2153}
2154
2155
2156static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2157{
2158 struct intel_pt_vm_tsc_info *data = pkt_info->data;
2159
2160 if (pkt_info->packet.type == INTEL_PT_CYC ||
2161 pkt_info->packet.type == INTEL_PT_MTC)
2162 return !--(data->max_lookahead);
2163
2164 if (pkt_info->packet.type == INTEL_PT_TMA) {
2165 data->tma_packet = pkt_info->packet;
2166 data->tma = true;
2167 }
2168 return 1;
2169}
2170
2171static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)
2172{
2173 if (decoder->tsc_ctc_mult)
2174 return ctc * decoder->tsc_ctc_mult;
2175 else
2176 return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);
2177}
2178
2179static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,
2180 uint32_t ctc,
2181 uint32_t fc,
2182 uint64_t last_ctc_timestamp,
2183 uint64_t ctc_delta,
2184 uint32_t last_ctc)
2185{
2186
2187 uint64_t last_mtc_ctc = last_ctc + ctc_delta;
2188
2189
2190
2191
2192
2193 uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc);
2194
2195 uint64_t new_ctc_delta = ctc_delta + delta;
2196 uint64_t expected_tsc;
2197
2198
2199
2200
2201
2202 expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;
2203
2204 if (intel_pt_enable_logging) {
2205 intel_pt_log_x64(last_mtc_ctc);
2206 intel_pt_log_x32(last_ctc);
2207 intel_pt_log_x64(ctc_delta);
2208 intel_pt_log_x64(delta);
2209 intel_pt_log_x32(ctc);
2210 intel_pt_log_x64(new_ctc_delta);
2211 intel_pt_log_x64(last_ctc_timestamp);
2212 intel_pt_log_x32(fc);
2213 intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));
2214 intel_pt_log_x64(expected_tsc);
2215 }
2216
2217 return expected_tsc;
2218}
2219
2220static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,
2221 struct intel_pt_vm_tsc_info *data)
2222{
2223 uint32_t ctc = data->tma_packet.payload;
2224 uint32_t fc = data->tma_packet.count;
2225
2226 return intel_pt_calc_expected_tsc(decoder, ctc, fc,
2227 decoder->ctc_timestamp,
2228 data->ctc_delta, data->last_ctc);
2229}
2230
2231static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,
2232 struct intel_pt_vmcs_info *vmcs_info)
2233{
2234 uint64_t payload = decoder->packet.payload;
2235
2236
2237 decoder->packet.payload -= vmcs_info->tsc_offset;
2238
2239 decoder->packet.payload &= SEVEN_BYTES;
2240
2241
2242
2243
2244
2245 if (!decoder->vm_tm_corr_dry_run)
2246 memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);
2247
2248 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2249 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n",
2250 payload, decoder->packet.payload, vmcs_info->vmcs,
2251 vmcs_info->tsc_offset);
2252}
2253
2254static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,
2255 uint64_t tsc_offset)
2256{
2257 struct intel_pt_vmcs_info vmcs_info = {
2258 .vmcs = NO_VMCS,
2259 .tsc_offset = tsc_offset
2260 };
2261
2262 intel_pt_translate_vm_tsc(decoder, &vmcs_info);
2263}
2264
2265static inline bool in_vm(uint64_t pip_payload)
2266{
2267 return pip_payload & 1;
2268}
2269
2270static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet)
2271{
2272 return pip_packet->payload & 1;
2273}
2274
2275static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info)
2276{
2277 p_log("VMCS: %#" PRIx64 " TSC Offset %#" PRIx64,
2278 vmcs_info->vmcs, vmcs_info->tsc_offset);
2279}
2280
2281static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,
2282 struct intel_pt_vm_tsc_info *data)
2283{
2284 memset(data, 0, sizeof(*data));
2285 data->ctc_delta = decoder->ctc_delta;
2286 data->last_ctc = decoder->last_ctc;
2287 intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);
2288 if (data->tsc && !data->psbend)
2289 p_log("ERROR: PSB without PSBEND");
2290 decoder->in_psb = data->psbend;
2291}
2292
2293static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,
2294 struct intel_pt_vm_tsc_info *data,
2295 struct intel_pt_vmcs_info *vmcs_info,
2296 uint64_t host_tsc)
2297{
2298 if (!decoder->in_psb) {
2299
2300 p_log("ERROR: First TSC is not in PSB+");
2301 }
2302
2303 if (data->pip) {
2304 if (pip_in_vm(&data->pip_packet)) {
2305 if (vmcs_info && vmcs_info->tsc_offset) {
2306 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2307 decoder->vm_tm_corr_reliable = true;
2308 } else {
2309 p_log("ERROR: First TSC, unknown TSC Offset");
2310 }
2311 } else {
2312 decoder->vm_tm_corr_reliable = true;
2313 }
2314 } else {
2315 decoder->vm_tm_corr_reliable = false;
2316 if (intel_pt_time_in_range(decoder, host_tsc)) {
2317
2318 } else {
2319
2320 if (vmcs_info && vmcs_info->tsc_offset)
2321 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2322 else
2323 p_log("ERROR: First TSC, no PIP, unknown TSC Offset");
2324 }
2325 }
2326}
2327
2328static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,
2329 struct intel_pt_vm_tsc_info *data)
2330{
2331 struct intel_pt_vmcs_info *vmcs_info;
2332 uint64_t tsc_offset = 0;
2333 uint64_t vmcs;
2334 bool reliable = true;
2335 uint64_t expected_tsc;
2336 uint64_t host_tsc;
2337 uint64_t ref_timestamp;
2338
2339 bool assign = false;
2340 bool assign_reliable = false;
2341
2342
2343 if (!decoder->in_psb) {
2344 memset(data, 0, sizeof(*data));
2345 data->ctc_delta = decoder->ctc_delta;
2346 data->last_ctc = decoder->last_ctc;
2347 data->max_lookahead = 16;
2348 intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);
2349 if (decoder->pge) {
2350 data->pip = true;
2351 data->pip_packet.payload = decoder->pip_payload;
2352 }
2353 }
2354
2355
2356 if (!data->tma) {
2357 p_log("ERROR: TSC without TMA");
2358 return;
2359 }
2360
2361 vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;
2362 if (vmcs == NO_VMCS)
2363 vmcs = 0;
2364
2365 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2366
2367 ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;
2368 host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);
2369
2370 if (!decoder->ctc_timestamp) {
2371 intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);
2372 return;
2373 }
2374
2375 expected_tsc = intel_pt_expected_tsc(decoder, data);
2376
2377 tsc_offset = host_tsc - expected_tsc;
2378
2379
2380 if (data->pip) {
2381 if (pip_in_vm(&data->pip_packet)) {
2382 if (!vmcs_info) {
2383
2384 p_log("ERROR: Missing VMCS");
2385 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2386 decoder->vm_tm_corr_reliable = false;
2387 return;
2388 }
2389 } else {
2390 decoder->last_reliable_timestamp = host_tsc;
2391 decoder->vm_tm_corr_reliable = true;
2392 return;
2393 }
2394 } else {
2395 reliable = false;
2396 if (decoder->in_psb) {
2397 if (!tsc_offset)
2398 return;
2399
2400
2401
2402
2403
2404
2405
2406 if (vmcs_info &&
2407 (tsc_offset & SEVEN_BYTES) ==
2408 (vmcs_info->tsc_offset & SEVEN_BYTES)) {
2409
2410 goto guest;
2411 }
2412 }
2413
2414
2415
2416
2417
2418
2419 if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {
2420
2421 decoder->vm_tm_corr_reliable = false;
2422 return;
2423 }
2424 }
2425
2426guest:
2427
2428
2429 if (vmcs_info && vmcs_info->vmcs) {
2430 if (vmcs_info->tsc_offset && vmcs_info->reliable) {
2431 assign = false;
2432 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&
2433 decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {
2434
2435 assign = true;
2436 assign_reliable = true;
2437 } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {
2438
2439
2440
2441
2442 assign = true;
2443 assign_reliable = false;
2444 }
2445 }
2446
2447
2448 if (assign && (vmcs_info->tsc_offset != tsc_offset ||
2449 vmcs_info->reliable != assign_reliable)) {
2450 bool print = vmcs_info->tsc_offset != tsc_offset;
2451
2452 vmcs_info->tsc_offset = tsc_offset;
2453 vmcs_info->reliable = assign_reliable;
2454 if (print)
2455 intel_pt_print_vmcs_info(vmcs_info);
2456 }
2457
2458
2459 if (vmcs_info && vmcs_info->tsc_offset) {
2460 if (!vmcs_info->reliable)
2461 reliable = false;
2462 intel_pt_translate_vm_tsc(decoder, vmcs_info);
2463 } else {
2464 reliable = false;
2465 if (vmcs_info) {
2466 if (!vmcs_info->error_printed) {
2467 p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64,
2468 vmcs_info->vmcs);
2469 vmcs_info->error_printed = true;
2470 }
2471 } else {
2472 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2473 p_log("ERROR: Unknown VMCS");
2474 }
2475 intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);
2476 }
2477
2478 decoder->vm_tm_corr_reliable = reliable;
2479}
2480
2481static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)
2482{
2483 uint64_t host_tsc = decoder->packet.payload;
2484 uint64_t guest_tsc = decoder->packet.payload;
2485 struct intel_pt_vmcs_info *vmcs_info;
2486 uint64_t vmcs;
2487
2488 vmcs = decoder->vmcs;
2489 if (vmcs == NO_VMCS)
2490 vmcs = 0;
2491
2492 vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);
2493
2494 if (decoder->pge) {
2495 if (in_vm(decoder->pip_payload)) {
2496 if (!vmcs_info) {
2497
2498 p_log("ERROR: Missing VMCS");
2499 }
2500 } else {
2501 return;
2502 }
2503 } else {
2504 if (intel_pt_time_in_range(decoder, host_tsc)) {
2505
2506 return;
2507 }
2508 }
2509
2510 if (vmcs_info) {
2511
2512 host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES;
2513 host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);
2514 intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64
2515 " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n",
2516 guest_tsc, host_tsc, vmcs_info->vmcs,
2517 vmcs_info->tsc_offset);
2518 if (!intel_pt_time_in_range(decoder, host_tsc) &&
2519 intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))
2520 p_log("Timestamp out of range");
2521 } else {
2522 if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))
2523 p_log("ERROR: Unknown VMCS");
2524 host_tsc = decoder->timestamp;
2525 }
2526
2527 decoder->packet.payload = host_tsc;
2528
2529 if (!decoder->vm_tm_corr_dry_run)
2530 memcpy((void *)decoder->buf + 1, &host_tsc, 8);
2531}
2532
2533static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)
2534{
2535 struct intel_pt_vm_tsc_info data = { .psbend = false };
2536 bool pge;
2537 int err;
2538
2539 if (decoder->in_psb)
2540 intel_pt_vm_tm_corr_psb(decoder, &data);
2541
2542 while (1) {
2543 err = intel_pt_get_next_packet(decoder);
2544 if (err == -ENOLINK)
2545 continue;
2546 if (err)
2547 break;
2548
2549 switch (decoder->packet.type) {
2550 case INTEL_PT_TIP_PGD:
2551 decoder->pge = false;
2552 decoder->vm_tm_corr_continuous = false;
2553 break;
2554
2555 case INTEL_PT_TNT:
2556 case INTEL_PT_TIP:
2557 case INTEL_PT_TIP_PGE:
2558 decoder->pge = true;
2559 break;
2560
2561 case INTEL_PT_OVF:
2562 decoder->in_psb = false;
2563 pge = decoder->pge;
2564 decoder->pge = intel_pt_ovf_fup_lookahead(decoder);
2565 if (pge != decoder->pge)
2566 intel_pt_log("Surprising PGE change in OVF!");
2567 if (!decoder->pge)
2568 decoder->vm_tm_corr_continuous = false;
2569 break;
2570
2571 case INTEL_PT_FUP:
2572 if (decoder->in_psb)
2573 decoder->pge = true;
2574 break;
2575
2576 case INTEL_PT_TRACESTOP:
2577 decoder->pge = false;
2578 decoder->vm_tm_corr_continuous = false;
2579 decoder->have_tma = false;
2580 break;
2581
2582 case INTEL_PT_PSB:
2583 intel_pt_vm_tm_corr_psb(decoder, &data);
2584 break;
2585
2586 case INTEL_PT_PIP:
2587 decoder->pip_payload = decoder->packet.payload;
2588 break;
2589
2590 case INTEL_PT_MTC:
2591 intel_pt_calc_mtc_timestamp(decoder);
2592 break;
2593
2594 case INTEL_PT_TSC:
2595 intel_pt_vm_tm_corr_tsc(decoder, &data);
2596 intel_pt_calc_tsc_timestamp(decoder);
2597 decoder->vm_tm_corr_same_buf = true;
2598 decoder->vm_tm_corr_continuous = decoder->pge;
2599 break;
2600
2601 case INTEL_PT_TMA:
2602 intel_pt_calc_tma(decoder);
2603 break;
2604
2605 case INTEL_PT_CYC:
2606 intel_pt_calc_cyc_timestamp(decoder);
2607 break;
2608
2609 case INTEL_PT_CBR:
2610 intel_pt_calc_cbr(decoder);
2611 break;
2612
2613 case INTEL_PT_PSBEND:
2614 decoder->in_psb = false;
2615 data.psbend = false;
2616 break;
2617
2618 case INTEL_PT_VMCS:
2619 if (decoder->packet.payload != NO_VMCS)
2620 decoder->vmcs = decoder->packet.payload;
2621 break;
2622
2623 case INTEL_PT_BBP:
2624 decoder->blk_type = decoder->packet.payload;
2625 break;
2626
2627 case INTEL_PT_BIP:
2628 if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&
2629 decoder->packet.count == 2)
2630 intel_pt_vm_tm_corr_pebs_tsc(decoder);
2631 break;
2632
2633 case INTEL_PT_BEP:
2634 case INTEL_PT_BEP_IP:
2635 decoder->blk_type = 0;
2636 break;
2637
2638 case INTEL_PT_MODE_EXEC:
2639 case INTEL_PT_MODE_TSX:
2640 case INTEL_PT_MNT:
2641 case INTEL_PT_PAD:
2642 case INTEL_PT_PTWRITE_IP:
2643 case INTEL_PT_PTWRITE:
2644 case INTEL_PT_MWAIT:
2645 case INTEL_PT_PWRE:
2646 case INTEL_PT_EXSTOP_IP:
2647 case INTEL_PT_EXSTOP:
2648 case INTEL_PT_PWRX:
2649 case INTEL_PT_BAD:
2650 default:
2651 break;
2652 }
2653 }
2654
2655 return err;
2656}
2657
2658#define HOP_PROCESS 0
2659#define HOP_IGNORE 1
2660#define HOP_RETURN 2
2661#define HOP_AGAIN 3
2662
2663static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2664
2665
2666static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)
2667{
2668
2669 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
2670 *err = intel_pt_scan_for_psb(decoder);
2671 if (*err)
2672 return HOP_RETURN;
2673 }
2674
2675 switch (decoder->packet.type) {
2676 case INTEL_PT_TNT:
2677 return HOP_IGNORE;
2678
2679 case INTEL_PT_TIP_PGD:
2680 if (!decoder->packet.count) {
2681 intel_pt_set_nr(decoder);
2682 return HOP_IGNORE;
2683 }
2684 intel_pt_set_ip(decoder);
2685 decoder->state.type |= INTEL_PT_TRACE_END;
2686 decoder->state.from_ip = 0;
2687 decoder->state.to_ip = decoder->ip;
2688 intel_pt_update_nr(decoder);
2689 return HOP_RETURN;
2690
2691 case INTEL_PT_TIP:
2692 if (!decoder->packet.count) {
2693 intel_pt_set_nr(decoder);
2694 return HOP_IGNORE;
2695 }
2696 intel_pt_set_ip(decoder);
2697 decoder->state.type = INTEL_PT_INSTRUCTION;
2698 decoder->state.from_ip = decoder->ip;
2699 decoder->state.to_ip = 0;
2700 intel_pt_update_nr(decoder);
2701 return HOP_RETURN;
2702
2703 case INTEL_PT_FUP:
2704 if (!decoder->packet.count)
2705 return HOP_IGNORE;
2706 intel_pt_set_ip(decoder);
2707 if (intel_pt_fup_event(decoder))
2708 return HOP_RETURN;
2709 if (!decoder->branch_enable)
2710 *no_tip = true;
2711 if (*no_tip) {
2712 decoder->state.type = INTEL_PT_INSTRUCTION;
2713 decoder->state.from_ip = decoder->ip;
2714 decoder->state.to_ip = 0;
2715 return HOP_RETURN;
2716 }
2717 *err = intel_pt_walk_fup_tip(decoder);
2718 if (!*err)
2719 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2720 return HOP_RETURN;
2721
2722 case INTEL_PT_PSB:
2723 decoder->state.psb_offset = decoder->pos;
2724 decoder->psb_ip = 0;
2725 decoder->last_ip = 0;
2726 decoder->have_last_ip = true;
2727 *err = intel_pt_walk_psbend(decoder);
2728 if (*err == -EAGAIN)
2729 return HOP_AGAIN;
2730 if (*err)
2731 return HOP_RETURN;
2732 decoder->state.type = INTEL_PT_PSB_EVT;
2733 if (decoder->psb_ip) {
2734 decoder->state.type |= INTEL_PT_INSTRUCTION;
2735 decoder->ip = decoder->psb_ip;
2736 }
2737 decoder->state.from_ip = decoder->psb_ip;
2738 decoder->state.to_ip = 0;
2739 return HOP_RETURN;
2740
2741 case INTEL_PT_BAD:
2742 case INTEL_PT_PAD:
2743 case INTEL_PT_TIP_PGE:
2744 case INTEL_PT_TSC:
2745 case INTEL_PT_TMA:
2746 case INTEL_PT_MODE_EXEC:
2747 case INTEL_PT_MODE_TSX:
2748 case INTEL_PT_MTC:
2749 case INTEL_PT_CYC:
2750 case INTEL_PT_VMCS:
2751 case INTEL_PT_PSBEND:
2752 case INTEL_PT_CBR:
2753 case INTEL_PT_TRACESTOP:
2754 case INTEL_PT_PIP:
2755 case INTEL_PT_OVF:
2756 case INTEL_PT_MNT:
2757 case INTEL_PT_PTWRITE:
2758 case INTEL_PT_PTWRITE_IP:
2759 case INTEL_PT_EXSTOP:
2760 case INTEL_PT_EXSTOP_IP:
2761 case INTEL_PT_MWAIT:
2762 case INTEL_PT_PWRE:
2763 case INTEL_PT_PWRX:
2764 case INTEL_PT_BBP:
2765 case INTEL_PT_BIP:
2766 case INTEL_PT_BEP:
2767 case INTEL_PT_BEP_IP:
2768 default:
2769 return HOP_PROCESS;
2770 }
2771}
2772
2773struct intel_pt_psb_info {
2774 struct intel_pt_pkt fup_packet;
2775 bool fup;
2776 int after_psbend;
2777};
2778
2779
2780static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2781{
2782 struct intel_pt_psb_info *data = pkt_info->data;
2783
2784 switch (pkt_info->packet.type) {
2785 case INTEL_PT_PAD:
2786 case INTEL_PT_MNT:
2787 case INTEL_PT_TSC:
2788 case INTEL_PT_TMA:
2789 case INTEL_PT_MODE_EXEC:
2790 case INTEL_PT_MODE_TSX:
2791 case INTEL_PT_MTC:
2792 case INTEL_PT_CYC:
2793 case INTEL_PT_VMCS:
2794 case INTEL_PT_CBR:
2795 case INTEL_PT_PIP:
2796 if (data->after_psbend) {
2797 data->after_psbend -= 1;
2798 if (!data->after_psbend)
2799 return 1;
2800 }
2801 break;
2802
2803 case INTEL_PT_FUP:
2804 if (data->after_psbend)
2805 return 1;
2806 if (data->fup || pkt_info->packet.count == 0)
2807 return 1;
2808 data->fup_packet = pkt_info->packet;
2809 data->fup = true;
2810 break;
2811
2812 case INTEL_PT_PSBEND:
2813 if (!data->fup)
2814 return 1;
2815
2816 data->after_psbend = 6;
2817 break;
2818
2819 case INTEL_PT_TIP_PGE:
2820
2821 if (data->after_psbend)
2822 data->fup = false;
2823 return 1;
2824
2825 case INTEL_PT_PTWRITE:
2826 case INTEL_PT_PTWRITE_IP:
2827 case INTEL_PT_EXSTOP:
2828 case INTEL_PT_EXSTOP_IP:
2829 case INTEL_PT_MWAIT:
2830 case INTEL_PT_PWRE:
2831 case INTEL_PT_PWRX:
2832 case INTEL_PT_BBP:
2833 case INTEL_PT_BIP:
2834 case INTEL_PT_BEP:
2835 case INTEL_PT_BEP_IP:
2836 if (data->after_psbend) {
2837 data->after_psbend -= 1;
2838 if (!data->after_psbend)
2839 return 1;
2840 break;
2841 }
2842 return 1;
2843
2844 case INTEL_PT_OVF:
2845 case INTEL_PT_BAD:
2846 case INTEL_PT_TNT:
2847 case INTEL_PT_TIP_PGD:
2848 case INTEL_PT_TIP:
2849 case INTEL_PT_PSB:
2850 case INTEL_PT_TRACESTOP:
2851 default:
2852 return 1;
2853 }
2854
2855 return 0;
2856}
2857
2858static int intel_pt_psb(struct intel_pt_decoder *decoder)
2859{
2860 int err;
2861
2862 decoder->last_ip = 0;
2863 decoder->psb_ip = 0;
2864 decoder->have_last_ip = true;
2865 intel_pt_clear_stack(&decoder->stack);
2866 err = intel_pt_walk_psbend(decoder);
2867 if (err)
2868 return err;
2869 decoder->state.type = INTEL_PT_PSB_EVT;
2870 decoder->state.from_ip = decoder->psb_ip;
2871 decoder->state.to_ip = 0;
2872 return 0;
2873}
2874
2875static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)
2876{
2877 int err;
2878
2879 if (decoder->ip != decoder->last_ip) {
2880 err = intel_pt_walk_fup(decoder);
2881 if (!err || err != -EAGAIN)
2882 return err;
2883 }
2884
2885 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2886 err = intel_pt_psb(decoder);
2887 if (err) {
2888 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2889 return -ENOENT;
2890 }
2891
2892 return 0;
2893}
2894
2895static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)
2896{
2897 struct intel_pt_psb_info data = { .fup = false };
2898
2899 if (!decoder->branch_enable || !decoder->pge)
2900 return false;
2901
2902 intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);
2903 if (!data.fup)
2904 return false;
2905
2906 decoder->packet = data.fup_packet;
2907 intel_pt_set_last_ip(decoder);
2908 decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;
2909
2910 *err = intel_pt_fup_in_psb(decoder);
2911
2912 return true;
2913}
2914
2915static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
2916{
2917 int last_packet_type = INTEL_PT_PAD;
2918 bool no_tip = false;
2919 int err;
2920
2921 while (1) {
2922 err = intel_pt_get_next_packet(decoder);
2923 if (err)
2924 return err;
2925next:
2926 if (decoder->cyc_threshold) {
2927 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
2928 decoder->sample_cyc = false;
2929 last_packet_type = decoder->packet.type;
2930 }
2931
2932 if (decoder->hop) {
2933 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
2934 case HOP_IGNORE:
2935 continue;
2936 case HOP_RETURN:
2937 return err;
2938 case HOP_AGAIN:
2939 goto next;
2940 default:
2941 break;
2942 }
2943 }
2944
2945 switch (decoder->packet.type) {
2946 case INTEL_PT_TNT:
2947 if (!decoder->packet.count)
2948 break;
2949 decoder->tnt = decoder->packet;
2950 decoder->pkt_state = INTEL_PT_STATE_TNT;
2951 err = intel_pt_walk_tnt(decoder);
2952 if (err == -EAGAIN)
2953 break;
2954 return err;
2955
2956 case INTEL_PT_TIP_PGD:
2957 if (decoder->packet.count != 0)
2958 intel_pt_set_last_ip(decoder);
2959 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
2960 return intel_pt_walk_tip(decoder);
2961
2962 case INTEL_PT_TIP_PGE: {
2963 decoder->pge = true;
2964 intel_pt_mtc_cyc_cnt_pge(decoder);
2965 intel_pt_set_nr(decoder);
2966 if (decoder->packet.count == 0) {
2967 intel_pt_log_at("Skipping zero TIP.PGE",
2968 decoder->pos);
2969 break;
2970 }
2971 intel_pt_set_ip(decoder);
2972 decoder->state.from_ip = 0;
2973 decoder->state.to_ip = decoder->ip;
2974 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2975
2976
2977
2978
2979 if (decoder->hop)
2980 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2981 return 0;
2982 }
2983
2984 case INTEL_PT_OVF:
2985 return intel_pt_overflow(decoder);
2986
2987 case INTEL_PT_TIP:
2988 if (decoder->packet.count != 0)
2989 intel_pt_set_last_ip(decoder);
2990 decoder->pkt_state = INTEL_PT_STATE_TIP;
2991 return intel_pt_walk_tip(decoder);
2992
2993 case INTEL_PT_FUP:
2994 if (decoder->packet.count == 0) {
2995 intel_pt_log_at("Skipping zero FUP",
2996 decoder->pos);
2997 no_tip = false;
2998 break;
2999 }
3000 intel_pt_set_last_ip(decoder);
3001 if (!decoder->branch_enable) {
3002 decoder->ip = decoder->last_ip;
3003 if (intel_pt_fup_event(decoder))
3004 return 0;
3005 no_tip = false;
3006 break;
3007 }
3008 if (decoder->set_fup_mwait)
3009 no_tip = true;
3010 if (no_tip)
3011 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
3012 else
3013 decoder->pkt_state = INTEL_PT_STATE_FUP;
3014 err = intel_pt_walk_fup(decoder);
3015 if (err != -EAGAIN)
3016 return err;
3017 if (no_tip) {
3018 no_tip = false;
3019 break;
3020 }
3021 return intel_pt_walk_fup_tip(decoder);
3022
3023 case INTEL_PT_TRACESTOP:
3024 decoder->pge = false;
3025 decoder->continuous_period = false;
3026 intel_pt_clear_tx_flags(decoder);
3027 decoder->have_tma = false;
3028 break;
3029
3030 case INTEL_PT_PSB:
3031 decoder->state.psb_offset = decoder->pos;
3032 decoder->psb_ip = 0;
3033 if (intel_pt_psb_with_fup(decoder, &err))
3034 return err;
3035 err = intel_pt_psb(decoder);
3036 if (err == -EAGAIN)
3037 goto next;
3038 return err;
3039
3040 case INTEL_PT_PIP:
3041 intel_pt_update_pip(decoder);
3042 break;
3043
3044 case INTEL_PT_MTC:
3045 intel_pt_calc_mtc_timestamp(decoder);
3046 if (decoder->period_type != INTEL_PT_PERIOD_MTC)
3047 break;
3048
3049
3050
3051
3052 if (!decoder->mtc_insn)
3053 break;
3054 decoder->mtc_insn = false;
3055
3056 if (!decoder->timestamp)
3057 break;
3058 decoder->state.type = INTEL_PT_INSTRUCTION;
3059 decoder->state.from_ip = decoder->ip;
3060 decoder->state.to_ip = 0;
3061 decoder->mtc_insn = false;
3062 return 0;
3063
3064 case INTEL_PT_TSC:
3065 intel_pt_calc_tsc_timestamp(decoder);
3066 break;
3067
3068 case INTEL_PT_TMA:
3069 intel_pt_calc_tma(decoder);
3070 break;
3071
3072 case INTEL_PT_CYC:
3073 intel_pt_calc_cyc_timestamp(decoder);
3074 break;
3075
3076 case INTEL_PT_CBR:
3077 intel_pt_calc_cbr(decoder);
3078 if (decoder->cbr != decoder->cbr_seen) {
3079 decoder->state.type = 0;
3080 return 0;
3081 }
3082 break;
3083
3084 case INTEL_PT_MODE_EXEC:
3085 decoder->exec_mode = decoder->packet.payload;
3086 break;
3087
3088 case INTEL_PT_MODE_TSX:
3089
3090 if (!decoder->pge || decoder->in_psb) {
3091 intel_pt_update_in_tx(decoder);
3092 break;
3093 }
3094 err = intel_pt_mode_tsx(decoder, &no_tip);
3095 if (err)
3096 return err;
3097 goto next;
3098
3099 case INTEL_PT_BAD:
3100 return intel_pt_bug(decoder);
3101
3102 case INTEL_PT_PSBEND:
3103 case INTEL_PT_VMCS:
3104 case INTEL_PT_MNT:
3105 case INTEL_PT_PAD:
3106 break;
3107
3108 case INTEL_PT_PTWRITE_IP:
3109 decoder->fup_ptw_payload = decoder->packet.payload;
3110 err = intel_pt_get_next_packet(decoder);
3111 if (err)
3112 return err;
3113 if (decoder->packet.type == INTEL_PT_FUP) {
3114 decoder->set_fup_ptw = true;
3115 no_tip = true;
3116 } else {
3117 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
3118 decoder->pos);
3119 }
3120 goto next;
3121
3122 case INTEL_PT_PTWRITE:
3123 decoder->state.type = INTEL_PT_PTW;
3124 decoder->state.from_ip = decoder->ip;
3125 decoder->state.to_ip = 0;
3126 decoder->state.ptw_payload = decoder->packet.payload;
3127 return 0;
3128
3129 case INTEL_PT_MWAIT:
3130 decoder->fup_mwait_payload = decoder->packet.payload;
3131 decoder->set_fup_mwait = true;
3132 break;
3133
3134 case INTEL_PT_PWRE:
3135 if (decoder->set_fup_mwait) {
3136 decoder->fup_pwre_payload =
3137 decoder->packet.payload;
3138 decoder->set_fup_pwre = true;
3139 break;
3140 }
3141 decoder->state.type = INTEL_PT_PWR_ENTRY;
3142 decoder->state.from_ip = decoder->ip;
3143 decoder->state.to_ip = 0;
3144 decoder->state.pwrx_payload = decoder->packet.payload;
3145 return 0;
3146
3147 case INTEL_PT_EXSTOP_IP:
3148 err = intel_pt_get_next_packet(decoder);
3149 if (err)
3150 return err;
3151 if (decoder->packet.type == INTEL_PT_FUP) {
3152 decoder->set_fup_exstop = true;
3153 no_tip = true;
3154 } else {
3155 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
3156 decoder->pos);
3157 }
3158 goto next;
3159
3160 case INTEL_PT_EXSTOP:
3161 decoder->state.type = INTEL_PT_EX_STOP;
3162 decoder->state.from_ip = decoder->ip;
3163 decoder->state.to_ip = 0;
3164 return 0;
3165
3166 case INTEL_PT_PWRX:
3167 decoder->state.type = INTEL_PT_PWR_EXIT;
3168 decoder->state.from_ip = decoder->ip;
3169 decoder->state.to_ip = 0;
3170 decoder->state.pwrx_payload = decoder->packet.payload;
3171 return 0;
3172
3173 case INTEL_PT_BBP:
3174 intel_pt_bbp(decoder);
3175 break;
3176
3177 case INTEL_PT_BIP:
3178 intel_pt_bip(decoder);
3179 break;
3180
3181 case INTEL_PT_BEP:
3182 decoder->state.type = INTEL_PT_BLK_ITEMS;
3183 decoder->state.from_ip = decoder->ip;
3184 decoder->state.to_ip = 0;
3185 return 0;
3186
3187 case INTEL_PT_BEP_IP:
3188 err = intel_pt_get_next_packet(decoder);
3189 if (err)
3190 return err;
3191 if (decoder->packet.type == INTEL_PT_FUP) {
3192 decoder->set_fup_bep = true;
3193 no_tip = true;
3194 } else {
3195 intel_pt_log_at("ERROR: Missing FUP after BEP",
3196 decoder->pos);
3197 }
3198 goto next;
3199
3200 default:
3201 return intel_pt_bug(decoder);
3202 }
3203 }
3204}
3205
3206static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
3207{
3208 return decoder->packet.count &&
3209 (decoder->have_last_ip || decoder->packet.count == 3 ||
3210 decoder->packet.count == 6);
3211}
3212
3213
3214static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
3215{
3216 int err;
3217
3218 decoder->in_psb = true;
3219
3220 while (1) {
3221 err = intel_pt_get_next_packet(decoder);
3222 if (err)
3223 goto out;
3224
3225 switch (decoder->packet.type) {
3226 case INTEL_PT_TIP_PGD:
3227 decoder->continuous_period = false;
3228 __fallthrough;
3229 case INTEL_PT_TIP_PGE:
3230 case INTEL_PT_TIP:
3231 case INTEL_PT_PTWRITE:
3232 case INTEL_PT_PTWRITE_IP:
3233 case INTEL_PT_EXSTOP:
3234 case INTEL_PT_EXSTOP_IP:
3235 case INTEL_PT_MWAIT:
3236 case INTEL_PT_PWRE:
3237 case INTEL_PT_PWRX:
3238 case INTEL_PT_BBP:
3239 case INTEL_PT_BIP:
3240 case INTEL_PT_BEP:
3241 case INTEL_PT_BEP_IP:
3242 intel_pt_log("ERROR: Unexpected packet\n");
3243 err = -ENOENT;
3244 goto out;
3245
3246 case INTEL_PT_FUP:
3247 decoder->pge = true;
3248 if (intel_pt_have_ip(decoder)) {
3249 uint64_t current_ip = decoder->ip;
3250
3251 intel_pt_set_ip(decoder);
3252 decoder->psb_ip = decoder->ip;
3253 if (current_ip)
3254 intel_pt_log_to("Setting IP",
3255 decoder->ip);
3256 }
3257 break;
3258
3259 case INTEL_PT_MTC:
3260 intel_pt_calc_mtc_timestamp(decoder);
3261 break;
3262
3263 case INTEL_PT_TSC:
3264 intel_pt_calc_tsc_timestamp(decoder);
3265 break;
3266
3267 case INTEL_PT_TMA:
3268 intel_pt_calc_tma(decoder);
3269 break;
3270
3271 case INTEL_PT_CYC:
3272 intel_pt_calc_cyc_timestamp(decoder);
3273 break;
3274
3275 case INTEL_PT_CBR:
3276 intel_pt_calc_cbr(decoder);
3277 break;
3278
3279 case INTEL_PT_PIP:
3280 intel_pt_set_pip(decoder);
3281 break;
3282
3283 case INTEL_PT_MODE_EXEC:
3284 decoder->exec_mode = decoder->packet.payload;
3285 break;
3286
3287 case INTEL_PT_MODE_TSX:
3288 intel_pt_update_in_tx(decoder);
3289 break;
3290
3291 case INTEL_PT_TRACESTOP:
3292 decoder->pge = false;
3293 decoder->continuous_period = false;
3294 intel_pt_clear_tx_flags(decoder);
3295 __fallthrough;
3296
3297 case INTEL_PT_TNT:
3298 decoder->have_tma = false;
3299 intel_pt_log("ERROR: Unexpected packet\n");
3300 if (decoder->ip)
3301 decoder->pkt_state = INTEL_PT_STATE_ERR4;
3302 else
3303 decoder->pkt_state = INTEL_PT_STATE_ERR3;
3304 err = -ENOENT;
3305 goto out;
3306
3307 case INTEL_PT_BAD:
3308 err = intel_pt_bug(decoder);
3309 goto out;
3310
3311 case INTEL_PT_OVF:
3312 err = intel_pt_overflow(decoder);
3313 goto out;
3314
3315 case INTEL_PT_PSBEND:
3316 err = 0;
3317 goto out;
3318
3319 case INTEL_PT_PSB:
3320 case INTEL_PT_VMCS:
3321 case INTEL_PT_MNT:
3322 case INTEL_PT_PAD:
3323 default:
3324 break;
3325 }
3326 }
3327out:
3328 decoder->in_psb = false;
3329
3330 return err;
3331}
3332
3333static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
3334{
3335 int err;
3336
3337 while (1) {
3338 err = intel_pt_get_next_packet(decoder);
3339 if (err)
3340 return err;
3341
3342 switch (decoder->packet.type) {
3343 case INTEL_PT_TIP_PGD:
3344 decoder->continuous_period = false;
3345 decoder->pge = false;
3346 if (intel_pt_have_ip(decoder))
3347 intel_pt_set_ip(decoder);
3348 if (!decoder->ip)
3349 break;
3350 decoder->state.type |= INTEL_PT_TRACE_END;
3351 return 0;
3352
3353 case INTEL_PT_TIP_PGE:
3354 decoder->pge = true;
3355 intel_pt_mtc_cyc_cnt_pge(decoder);
3356 if (intel_pt_have_ip(decoder))
3357 intel_pt_set_ip(decoder);
3358 if (!decoder->ip)
3359 break;
3360 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
3361 return 0;
3362
3363 case INTEL_PT_TIP:
3364 decoder->pge = true;
3365 if (intel_pt_have_ip(decoder))
3366 intel_pt_set_ip(decoder);
3367 if (!decoder->ip)
3368 break;
3369 return 0;
3370
3371 case INTEL_PT_FUP:
3372 if (intel_pt_have_ip(decoder))
3373 intel_pt_set_ip(decoder);
3374 if (decoder->ip)
3375 return 0;
3376 break;
3377
3378 case INTEL_PT_MTC:
3379 intel_pt_calc_mtc_timestamp(decoder);
3380 break;
3381
3382 case INTEL_PT_TSC:
3383 intel_pt_calc_tsc_timestamp(decoder);
3384 break;
3385
3386 case INTEL_PT_TMA:
3387 intel_pt_calc_tma(decoder);
3388 break;
3389
3390 case INTEL_PT_CYC:
3391 intel_pt_calc_cyc_timestamp(decoder);
3392 break;
3393
3394 case INTEL_PT_CBR:
3395 intel_pt_calc_cbr(decoder);
3396 break;
3397
3398 case INTEL_PT_PIP:
3399 intel_pt_set_pip(decoder);
3400 break;
3401
3402 case INTEL_PT_MODE_EXEC:
3403 decoder->exec_mode = decoder->packet.payload;
3404 break;
3405
3406 case INTEL_PT_MODE_TSX:
3407 intel_pt_update_in_tx(decoder);
3408 break;
3409
3410 case INTEL_PT_OVF:
3411 return intel_pt_overflow(decoder);
3412
3413 case INTEL_PT_BAD:
3414 return intel_pt_bug(decoder);
3415
3416 case INTEL_PT_TRACESTOP:
3417 decoder->pge = false;
3418 decoder->continuous_period = false;
3419 intel_pt_clear_tx_flags(decoder);
3420 decoder->have_tma = false;
3421 break;
3422
3423 case INTEL_PT_PSB:
3424 decoder->state.psb_offset = decoder->pos;
3425 decoder->psb_ip = 0;
3426 decoder->last_ip = 0;
3427 decoder->have_last_ip = true;
3428 intel_pt_clear_stack(&decoder->stack);
3429 err = intel_pt_walk_psb(decoder);
3430 if (err)
3431 return err;
3432 decoder->state.type = INTEL_PT_PSB_EVT;
3433 decoder->state.from_ip = decoder->psb_ip;
3434 decoder->state.to_ip = 0;
3435 return 0;
3436
3437 case INTEL_PT_TNT:
3438 case INTEL_PT_PSBEND:
3439 case INTEL_PT_VMCS:
3440 case INTEL_PT_MNT:
3441 case INTEL_PT_PAD:
3442 case INTEL_PT_PTWRITE:
3443 case INTEL_PT_PTWRITE_IP:
3444 case INTEL_PT_EXSTOP:
3445 case INTEL_PT_EXSTOP_IP:
3446 case INTEL_PT_MWAIT:
3447 case INTEL_PT_PWRE:
3448 case INTEL_PT_PWRX:
3449 case INTEL_PT_BBP:
3450 case INTEL_PT_BIP:
3451 case INTEL_PT_BEP:
3452 case INTEL_PT_BEP_IP:
3453 default:
3454 break;
3455 }
3456 }
3457}
3458
3459static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
3460{
3461 int err;
3462
3463 decoder->set_fup_tx_flags = false;
3464 decoder->set_fup_ptw = false;
3465 decoder->set_fup_mwait = false;
3466 decoder->set_fup_pwre = false;
3467 decoder->set_fup_exstop = false;
3468 decoder->set_fup_bep = false;
3469
3470 if (!decoder->branch_enable) {
3471 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3472 decoder->overflow = false;
3473 decoder->state.type = 0;
3474 return 0;
3475 }
3476
3477 intel_pt_log("Scanning for full IP\n");
3478 err = intel_pt_walk_to_ip(decoder);
3479 if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))
3480 return err;
3481
3482
3483 if (decoder->hop)
3484 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3485 else
3486 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3487 decoder->overflow = false;
3488
3489 decoder->state.from_ip = 0;
3490 decoder->state.to_ip = decoder->ip;
3491 intel_pt_log_to("Setting IP", decoder->ip);
3492
3493 return 0;
3494}
3495
3496static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
3497{
3498 const unsigned char *end = decoder->buf + decoder->len;
3499 size_t i;
3500
3501 for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
3502 if (i > decoder->len)
3503 continue;
3504 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
3505 return i;
3506 }
3507 return 0;
3508}
3509
3510static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
3511{
3512 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
3513 const char *psb = INTEL_PT_PSB_STR;
3514
3515 if (rest_psb > decoder->len ||
3516 memcmp(decoder->buf, psb + part_psb, rest_psb))
3517 return 0;
3518
3519 return rest_psb;
3520}
3521
3522static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
3523 int part_psb)
3524{
3525 int rest_psb, ret;
3526
3527 decoder->pos += decoder->len;
3528 decoder->len = 0;
3529
3530 ret = intel_pt_get_next_data(decoder, false);
3531 if (ret)
3532 return ret;
3533
3534 rest_psb = intel_pt_rest_psb(decoder, part_psb);
3535 if (!rest_psb)
3536 return 0;
3537
3538 decoder->pos -= part_psb;
3539 decoder->next_buf = decoder->buf + rest_psb;
3540 decoder->next_len = decoder->len - rest_psb;
3541 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3542 decoder->buf = decoder->temp_buf;
3543 decoder->len = INTEL_PT_PSB_LEN;
3544
3545 return 0;
3546}
3547
3548static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
3549{
3550 unsigned char *next;
3551 int ret;
3552
3553 intel_pt_log("Scanning for PSB\n");
3554 while (1) {
3555 if (!decoder->len) {
3556 ret = intel_pt_get_next_data(decoder, false);
3557 if (ret)
3558 return ret;
3559 }
3560
3561 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
3562 INTEL_PT_PSB_LEN);
3563 if (!next) {
3564 int part_psb;
3565
3566 part_psb = intel_pt_part_psb(decoder);
3567 if (part_psb) {
3568 ret = intel_pt_get_split_psb(decoder, part_psb);
3569 if (ret)
3570 return ret;
3571 } else {
3572 decoder->pos += decoder->len;
3573 decoder->len = 0;
3574 }
3575 continue;
3576 }
3577
3578 decoder->pkt_step = next - decoder->buf;
3579 return intel_pt_get_next_packet(decoder);
3580 }
3581}
3582
3583static int intel_pt_sync(struct intel_pt_decoder *decoder)
3584{
3585 int err;
3586
3587 decoder->pge = false;
3588 decoder->continuous_period = false;
3589 decoder->have_last_ip = false;
3590 decoder->last_ip = 0;
3591 decoder->psb_ip = 0;
3592 decoder->ip = 0;
3593 intel_pt_clear_stack(&decoder->stack);
3594
3595 err = intel_pt_scan_for_psb(decoder);
3596 if (err)
3597 return err;
3598
3599 if (decoder->vm_time_correlation) {
3600 decoder->in_psb = true;
3601 if (!decoder->timestamp)
3602 decoder->timestamp = 1;
3603 decoder->state.type = 0;
3604 decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;
3605 return 0;
3606 }
3607
3608 decoder->have_last_ip = true;
3609 decoder->pkt_state = INTEL_PT_STATE_NO_IP;
3610
3611 err = intel_pt_walk_psb(decoder);
3612 if (err)
3613 return err;
3614
3615 decoder->state.type = INTEL_PT_PSB_EVT;
3616 decoder->state.from_ip = decoder->psb_ip;
3617 decoder->state.to_ip = 0;
3618
3619 if (decoder->ip) {
3620
3621
3622
3623
3624 if (decoder->hop)
3625 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
3626 else
3627 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
3628 }
3629
3630 return 0;
3631}
3632
3633static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
3634{
3635 uint64_t est = decoder->sample_insn_cnt << 1;
3636
3637 if (!decoder->cbr || !decoder->max_non_turbo_ratio)
3638 goto out;
3639
3640 est *= decoder->max_non_turbo_ratio;
3641 est /= decoder->cbr;
3642out:
3643 return decoder->sample_timestamp + est;
3644}
3645
3646const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
3647{
3648 int err;
3649
3650 do {
3651 decoder->state.type = INTEL_PT_BRANCH;
3652 decoder->state.flags = 0;
3653
3654 switch (decoder->pkt_state) {
3655 case INTEL_PT_STATE_NO_PSB:
3656 err = intel_pt_sync(decoder);
3657 break;
3658 case INTEL_PT_STATE_NO_IP:
3659 decoder->have_last_ip = false;
3660 decoder->last_ip = 0;
3661 decoder->ip = 0;
3662 __fallthrough;
3663 case INTEL_PT_STATE_ERR_RESYNC:
3664 err = intel_pt_sync_ip(decoder);
3665 break;
3666 case INTEL_PT_STATE_IN_SYNC:
3667 err = intel_pt_walk_trace(decoder);
3668 break;
3669 case INTEL_PT_STATE_TNT:
3670 case INTEL_PT_STATE_TNT_CONT:
3671 err = intel_pt_walk_tnt(decoder);
3672 if (err == -EAGAIN)
3673 err = intel_pt_walk_trace(decoder);
3674 break;
3675 case INTEL_PT_STATE_TIP:
3676 case INTEL_PT_STATE_TIP_PGD:
3677 err = intel_pt_walk_tip(decoder);
3678 break;
3679 case INTEL_PT_STATE_FUP:
3680 err = intel_pt_walk_fup(decoder);
3681 if (err == -EAGAIN)
3682 err = intel_pt_walk_fup_tip(decoder);
3683 break;
3684 case INTEL_PT_STATE_FUP_NO_TIP:
3685 err = intel_pt_walk_fup(decoder);
3686 if (err == -EAGAIN)
3687 err = intel_pt_walk_trace(decoder);
3688 break;
3689 case INTEL_PT_STATE_FUP_IN_PSB:
3690 err = intel_pt_fup_in_psb(decoder);
3691 break;
3692 case INTEL_PT_STATE_RESAMPLE:
3693 err = intel_pt_resample(decoder);
3694 break;
3695 case INTEL_PT_STATE_VM_TIME_CORRELATION:
3696 err = intel_pt_vm_time_correlation(decoder);
3697 break;
3698 default:
3699 err = intel_pt_bug(decoder);
3700 break;
3701 }
3702 } while (err == -ENOLINK);
3703
3704 if (err) {
3705 decoder->state.err = intel_pt_ext_err(err);
3706 decoder->state.from_ip = decoder->ip;
3707 intel_pt_update_sample_time(decoder);
3708 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3709 intel_pt_set_nr(decoder);
3710 } else {
3711 decoder->state.err = 0;
3712 if (decoder->cbr != decoder->cbr_seen) {
3713 decoder->cbr_seen = decoder->cbr;
3714 if (!decoder->state.type) {
3715 decoder->state.from_ip = decoder->ip;
3716 decoder->state.to_ip = 0;
3717 }
3718 decoder->state.type |= INTEL_PT_CBR_CHG;
3719 decoder->state.cbr_payload = decoder->cbr_payload;
3720 decoder->state.cbr = decoder->cbr;
3721 }
3722 if (intel_pt_sample_time(decoder->pkt_state)) {
3723 intel_pt_update_sample_time(decoder);
3724 if (decoder->sample_cyc) {
3725 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3726 decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
3727 decoder->sample_cyc = false;
3728 }
3729 }
3730
3731
3732
3733
3734 if (!decoder->have_cyc)
3735 decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
3736 }
3737
3738
3739 if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)
3740 decoder->sample_timestamp = decoder->tsc_timestamp;
3741
3742 decoder->state.from_nr = decoder->nr;
3743 decoder->state.to_nr = decoder->next_nr;
3744 decoder->nr = decoder->next_nr;
3745
3746 decoder->state.timestamp = decoder->sample_timestamp;
3747 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
3748 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
3749 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
3750
3751 return &decoder->state;
3752}
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
3766{
3767 unsigned char *next;
3768
3769 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3770 if (next) {
3771 *len -= next - *buf;
3772 *buf = next;
3773 return true;
3774 }
3775 return false;
3776}
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
3791{
3792 unsigned char *next;
3793
3794 if (!*len)
3795 return false;
3796
3797 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3798 if (next) {
3799 *len -= next - *buf;
3800 *buf = next;
3801 return true;
3802 }
3803 return false;
3804}
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
3816{
3817 const char *n = INTEL_PT_PSB_STR;
3818 unsigned char *p;
3819 size_t k;
3820
3821 if (len < INTEL_PT_PSB_LEN)
3822 return NULL;
3823
3824 k = len - INTEL_PT_PSB_LEN + 1;
3825 while (1) {
3826 p = memrchr(buf, n[0], k);
3827 if (!p)
3828 return NULL;
3829 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
3830 return p;
3831 k = p - buf;
3832 if (!k)
3833 return NULL;
3834 }
3835}
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
3851 size_t *rem)
3852{
3853 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
3854 struct intel_pt_pkt packet;
3855 int ret;
3856
3857 while (len) {
3858 ret = intel_pt_get_packet(buf, len, &packet, &ctx);
3859 if (ret <= 0)
3860 return false;
3861 if (packet.type == INTEL_PT_TSC) {
3862 *tsc = packet.payload;
3863 *rem = len;
3864 return true;
3865 }
3866 if (packet.type == INTEL_PT_PSBEND)
3867 return false;
3868 buf += ret;
3869 len -= ret;
3870 }
3871 return false;
3872}
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
3888{
3889 const uint64_t halfway = (1ULL << 55);
3890
3891 if (tsc1 == tsc2)
3892 return 0;
3893
3894 if (tsc1 < tsc2) {
3895 if (tsc2 - tsc1 < halfway)
3896 return -1;
3897 else
3898 return 1;
3899 } else {
3900 if (tsc1 - tsc2 < halfway)
3901 return 1;
3902 else
3903 return -1;
3904 }
3905}
3906
3907#define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920static unsigned char *adj_for_padding(unsigned char *buf_b,
3921 unsigned char *buf_a, size_t len_a)
3922{
3923 unsigned char *p = buf_b - MAX_PADDING;
3924 unsigned char *q = buf_a + len_a - MAX_PADDING;
3925 int i;
3926
3927 for (i = MAX_PADDING; i; i--, p++, q++) {
3928 if (*p != *q)
3929 break;
3930 }
3931
3932 return p;
3933}
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
3955 size_t len_a,
3956 unsigned char *buf_b,
3957 size_t len_b, bool *consecutive,
3958 bool ooo_tsc)
3959{
3960 uint64_t tsc_a, tsc_b;
3961 unsigned char *p;
3962 size_t len, rem_a, rem_b;
3963
3964 p = intel_pt_last_psb(buf_a, len_a);
3965 if (!p)
3966 return buf_b;
3967
3968 len = len_a - (p - buf_a);
3969 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
3970
3971 len_a -= len;
3972 p = intel_pt_last_psb(buf_a, len_a);
3973 if (!p)
3974 return buf_b;
3975 len = len_a - (p - buf_a);
3976 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
3977 return buf_b;
3978 }
3979
3980 while (1) {
3981
3982 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
3983 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
3984
3985
3986 if (!cmp && rem_b >= rem_a) {
3987 unsigned char *start;
3988
3989 *consecutive = true;
3990 start = buf_b + len_b - (rem_b - rem_a);
3991 return adj_for_padding(start, buf_a, len_a);
3992 }
3993 if (cmp < 0 && !ooo_tsc)
3994 return buf_b;
3995 }
3996
3997 if (!intel_pt_step_psb(&buf_b, &len_b))
3998 return buf_b + len_b;
3999 }
4000}
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
4021 unsigned char *buf_b, size_t len_b,
4022 bool have_tsc, bool *consecutive,
4023 bool ooo_tsc)
4024{
4025 unsigned char *found;
4026
4027
4028 if (!intel_pt_next_psb(&buf_b, &len_b))
4029 return buf_b + len_b;
4030
4031 if (!intel_pt_next_psb(&buf_a, &len_a))
4032 return buf_b;
4033
4034 if (have_tsc) {
4035 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
4036 consecutive, ooo_tsc);
4037 if (found)
4038 return found;
4039 }
4040
4041
4042
4043
4044
4045 while (len_b < len_a) {
4046 if (!intel_pt_step_psb(&buf_a, &len_a))
4047 return buf_b;
4048 }
4049
4050
4051 while (1) {
4052
4053 found = memmem(buf_a, len_a, buf_b, len_a);
4054 if (found) {
4055 *consecutive = true;
4056 return adj_for_padding(buf_b + len_a, buf_a, len_a);
4057 }
4058
4059
4060 if (!intel_pt_step_psb(&buf_a, &len_a))
4061 return buf_b;
4062 }
4063}
4064
4065
4066
4067
4068
4069
4070
4071struct fast_forward_data {
4072 uint64_t timestamp;
4073 uint64_t buf_timestamp;
4074};
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data)
4087{
4088 struct fast_forward_data *d = data;
4089 unsigned char *buf;
4090 uint64_t tsc;
4091 size_t rem;
4092 size_t len;
4093
4094 buf = (unsigned char *)buffer->buf;
4095 len = buffer->len;
4096
4097 if (!intel_pt_next_psb(&buf, &len) ||
4098 !intel_pt_next_tsc(buf, len, &tsc, &rem))
4099 return 0;
4100
4101 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
4102
4103 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
4104 tsc, buffer->ref_timestamp);
4105
4106
4107
4108
4109
4110 if (tsc < d->timestamp)
4111 d->buf_timestamp = buffer->ref_timestamp;
4112 else
4113 return 1;
4114
4115 return 0;
4116}
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
4128{
4129 struct fast_forward_data d = { .timestamp = timestamp };
4130 unsigned char *buf;
4131 size_t len;
4132 int err;
4133
4134 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
4135
4136
4137 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
4138 if (err < 0)
4139 return err;
4140
4141
4142 if (d.buf_timestamp) {
4143 do {
4144 decoder->pos += decoder->len;
4145 decoder->len = 0;
4146 err = intel_pt_get_next_data(decoder, true);
4147
4148 if (err && err != -ENOLINK)
4149 return err;
4150 } while (decoder->buf_timestamp != d.buf_timestamp);
4151 }
4152
4153 if (!decoder->buf)
4154 return 0;
4155
4156 buf = (unsigned char *)decoder->buf;
4157 len = decoder->len;
4158
4159 if (!intel_pt_next_psb(&buf, &len))
4160 return 0;
4161
4162
4163
4164
4165
4166 do {
4167 uint64_t tsc;
4168 size_t rem;
4169
4170 if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
4171 break;
4172 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
4173
4174
4175
4176
4177
4178
4179 if (tsc < timestamp) {
4180 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
4181 decoder->pos += decoder->len - len;
4182 decoder->buf = buf;
4183 decoder->len = len;
4184 intel_pt_reposition(decoder);
4185 } else {
4186 break;
4187 }
4188 } while (intel_pt_step_psb(&buf, &len));
4189
4190 return 0;
4191}
4192