1
2
3
4
5
6
7#ifndef _GNU_SOURCE
8#define _GNU_SOURCE
9#endif
10#include <stdlib.h>
11#include <stdbool.h>
12#include <string.h>
13#include <errno.h>
14#include <stdint.h>
15#include <inttypes.h>
16#include <linux/compiler.h>
17#include <linux/string.h>
18#include <linux/zalloc.h>
19
20#include "../auxtrace.h"
21
22#include "intel-pt-insn-decoder.h"
23#include "intel-pt-pkt-decoder.h"
24#include "intel-pt-decoder.h"
25#include "intel-pt-log.h"
26
27#define BITULL(x) (1ULL << (x))
28
29
30#define INTEL_PT_CYC_ENABLE BITULL(1)
31#define INTEL_PT_CYC_THRESHOLD (BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19))
32#define INTEL_PT_CYC_THRESHOLD_SHIFT 19
33
34#define INTEL_PT_BLK_SIZE 1024
35
36#define BIT63 (((uint64_t)1 << 63))
37
38#define INTEL_PT_RETURN 1
39
40
41#define INTEL_PT_MAX_LOOPS 10000
42
43struct intel_pt_blk {
44 struct intel_pt_blk *prev;
45 uint64_t ip[INTEL_PT_BLK_SIZE];
46};
47
48struct intel_pt_stack {
49 struct intel_pt_blk *blk;
50 struct intel_pt_blk *spare;
51 int pos;
52};
53
54enum intel_pt_pkt_state {
55 INTEL_PT_STATE_NO_PSB,
56 INTEL_PT_STATE_NO_IP,
57 INTEL_PT_STATE_ERR_RESYNC,
58 INTEL_PT_STATE_IN_SYNC,
59 INTEL_PT_STATE_TNT_CONT,
60 INTEL_PT_STATE_TNT,
61 INTEL_PT_STATE_TIP,
62 INTEL_PT_STATE_TIP_PGD,
63 INTEL_PT_STATE_FUP,
64 INTEL_PT_STATE_FUP_NO_TIP,
65 INTEL_PT_STATE_FUP_IN_PSB,
66 INTEL_PT_STATE_RESAMPLE,
67};
68
69static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
70{
71 switch (pkt_state) {
72 case INTEL_PT_STATE_NO_PSB:
73 case INTEL_PT_STATE_NO_IP:
74 case INTEL_PT_STATE_ERR_RESYNC:
75 case INTEL_PT_STATE_IN_SYNC:
76 case INTEL_PT_STATE_TNT_CONT:
77 case INTEL_PT_STATE_RESAMPLE:
78 return true;
79 case INTEL_PT_STATE_TNT:
80 case INTEL_PT_STATE_TIP:
81 case INTEL_PT_STATE_TIP_PGD:
82 case INTEL_PT_STATE_FUP:
83 case INTEL_PT_STATE_FUP_NO_TIP:
84 case INTEL_PT_STATE_FUP_IN_PSB:
85 return false;
86 default:
87 return true;
88 };
89}
90
91#ifdef INTEL_PT_STRICT
92#define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
93#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
94#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
95#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
96#else
97#define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
98#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
99#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
100#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
101#endif
102
103struct intel_pt_decoder {
104 int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
105 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
106 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
107 uint64_t max_insn_cnt, void *data);
108 bool (*pgd_ip)(uint64_t ip, void *data);
109 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data);
110 void *data;
111 struct intel_pt_state state;
112 const unsigned char *buf;
113 size_t len;
114 bool return_compression;
115 bool branch_enable;
116 bool mtc_insn;
117 bool pge;
118 bool have_tma;
119 bool have_cyc;
120 bool fixup_last_mtc;
121 bool have_last_ip;
122 bool in_psb;
123 bool hop;
124 bool leap;
125 bool nr;
126 bool next_nr;
127 enum intel_pt_param_flags flags;
128 uint64_t pos;
129 uint64_t last_ip;
130 uint64_t ip;
131 uint64_t pip_payload;
132 uint64_t timestamp;
133 uint64_t tsc_timestamp;
134 uint64_t ref_timestamp;
135 uint64_t buf_timestamp;
136 uint64_t sample_timestamp;
137 uint64_t ret_addr;
138 uint64_t ctc_timestamp;
139 uint64_t ctc_delta;
140 uint64_t cycle_cnt;
141 uint64_t cyc_ref_timestamp;
142 uint32_t last_mtc;
143 uint32_t tsc_ctc_ratio_n;
144 uint32_t tsc_ctc_ratio_d;
145 uint32_t tsc_ctc_mult;
146 uint32_t tsc_slip;
147 uint32_t ctc_rem_mask;
148 int mtc_shift;
149 struct intel_pt_stack stack;
150 enum intel_pt_pkt_state pkt_state;
151 enum intel_pt_pkt_ctx pkt_ctx;
152 enum intel_pt_pkt_ctx prev_pkt_ctx;
153 enum intel_pt_blk_type blk_type;
154 int blk_type_pos;
155 struct intel_pt_pkt packet;
156 struct intel_pt_pkt tnt;
157 int pkt_step;
158 int pkt_len;
159 int last_packet_type;
160 unsigned int cbr;
161 unsigned int cbr_seen;
162 unsigned int max_non_turbo_ratio;
163 double max_non_turbo_ratio_fp;
164 double cbr_cyc_to_tsc;
165 double calc_cyc_to_tsc;
166 bool have_calc_cyc_to_tsc;
167 int exec_mode;
168 unsigned int insn_bytes;
169 uint64_t period;
170 enum intel_pt_period_type period_type;
171 uint64_t tot_insn_cnt;
172 uint64_t period_insn_cnt;
173 uint64_t period_mask;
174 uint64_t period_ticks;
175 uint64_t last_masked_timestamp;
176 uint64_t tot_cyc_cnt;
177 uint64_t sample_tot_cyc_cnt;
178 uint64_t base_cyc_cnt;
179 uint64_t cyc_cnt_timestamp;
180 uint64_t ctl;
181 uint64_t cyc_threshold;
182 double tsc_to_cyc;
183 bool continuous_period;
184 bool overflow;
185 bool set_fup_tx_flags;
186 bool set_fup_ptw;
187 bool set_fup_mwait;
188 bool set_fup_pwre;
189 bool set_fup_exstop;
190 bool set_fup_bep;
191 bool sample_cyc;
192 unsigned int fup_tx_flags;
193 unsigned int tx_flags;
194 uint64_t fup_ptw_payload;
195 uint64_t fup_mwait_payload;
196 uint64_t fup_pwre_payload;
197 uint64_t cbr_payload;
198 uint64_t timestamp_insn_cnt;
199 uint64_t sample_insn_cnt;
200 uint64_t stuck_ip;
201 int no_progress;
202 int stuck_ip_prd;
203 int stuck_ip_cnt;
204 uint64_t psb_ip;
205 const unsigned char *next_buf;
206 size_t next_len;
207 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
208};
209
210static uint64_t intel_pt_lower_power_of_2(uint64_t x)
211{
212 int i;
213
214 for (i = 0; x != 1; i++)
215 x >>= 1;
216
217 return x << i;
218}
219
220static uint64_t intel_pt_cyc_threshold(uint64_t ctl)
221{
222 if (!(ctl & INTEL_PT_CYC_ENABLE))
223 return 0;
224
225 return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;
226}
227
228static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
229{
230 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
231 uint64_t period;
232
233 period = intel_pt_lower_power_of_2(decoder->period);
234 decoder->period_mask = ~(period - 1);
235 decoder->period_ticks = period;
236 }
237}
238
239static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
240{
241 if (!d)
242 return 0;
243 return (t / d) * n + ((t % d) * n) / d;
244}
245
246struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
247{
248 struct intel_pt_decoder *decoder;
249
250 if (!params->get_trace || !params->walk_insn)
251 return NULL;
252
253 decoder = zalloc(sizeof(struct intel_pt_decoder));
254 if (!decoder)
255 return NULL;
256
257 decoder->get_trace = params->get_trace;
258 decoder->walk_insn = params->walk_insn;
259 decoder->pgd_ip = params->pgd_ip;
260 decoder->lookahead = params->lookahead;
261 decoder->data = params->data;
262 decoder->return_compression = params->return_compression;
263 decoder->branch_enable = params->branch_enable;
264 decoder->hop = params->quick >= 1;
265 decoder->leap = params->quick >= 2;
266
267 decoder->flags = params->flags;
268
269 decoder->ctl = params->ctl;
270 decoder->period = params->period;
271 decoder->period_type = params->period_type;
272
273 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
274 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
275
276 decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);
277
278 intel_pt_setup_period(decoder);
279
280 decoder->mtc_shift = params->mtc_period;
281 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
282
283 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
284 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
285
286 if (!decoder->tsc_ctc_ratio_n)
287 decoder->tsc_ctc_ratio_d = 0;
288
289 if (decoder->tsc_ctc_ratio_d) {
290 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
291 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
292 decoder->tsc_ctc_ratio_d;
293 }
294
295
296
297
298
299
300
301 decoder->tsc_slip = 0x10000;
302
303 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
304 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
305 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
306 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
307 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
308
309 if (decoder->hop)
310 intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");
311
312 return decoder;
313}
314
315static void intel_pt_pop_blk(struct intel_pt_stack *stack)
316{
317 struct intel_pt_blk *blk = stack->blk;
318
319 stack->blk = blk->prev;
320 if (!stack->spare)
321 stack->spare = blk;
322 else
323 free(blk);
324}
325
326static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
327{
328 if (!stack->pos) {
329 if (!stack->blk)
330 return 0;
331 intel_pt_pop_blk(stack);
332 if (!stack->blk)
333 return 0;
334 stack->pos = INTEL_PT_BLK_SIZE;
335 }
336 return stack->blk->ip[--stack->pos];
337}
338
339static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
340{
341 struct intel_pt_blk *blk;
342
343 if (stack->spare) {
344 blk = stack->spare;
345 stack->spare = NULL;
346 } else {
347 blk = malloc(sizeof(struct intel_pt_blk));
348 if (!blk)
349 return -ENOMEM;
350 }
351
352 blk->prev = stack->blk;
353 stack->blk = blk;
354 stack->pos = 0;
355 return 0;
356}
357
358static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
359{
360 int err;
361
362 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
363 err = intel_pt_alloc_blk(stack);
364 if (err)
365 return err;
366 }
367
368 stack->blk->ip[stack->pos++] = ip;
369 return 0;
370}
371
372static void intel_pt_clear_stack(struct intel_pt_stack *stack)
373{
374 while (stack->blk)
375 intel_pt_pop_blk(stack);
376 stack->pos = 0;
377}
378
379static void intel_pt_free_stack(struct intel_pt_stack *stack)
380{
381 intel_pt_clear_stack(stack);
382 zfree(&stack->blk);
383 zfree(&stack->spare);
384}
385
386void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
387{
388 intel_pt_free_stack(&decoder->stack);
389 free(decoder);
390}
391
392static int intel_pt_ext_err(int code)
393{
394 switch (code) {
395 case -ENOMEM:
396 return INTEL_PT_ERR_NOMEM;
397 case -ENOSYS:
398 return INTEL_PT_ERR_INTERN;
399 case -EBADMSG:
400 return INTEL_PT_ERR_BADPKT;
401 case -ENODATA:
402 return INTEL_PT_ERR_NODATA;
403 case -EILSEQ:
404 return INTEL_PT_ERR_NOINSN;
405 case -ENOENT:
406 return INTEL_PT_ERR_MISMAT;
407 case -EOVERFLOW:
408 return INTEL_PT_ERR_OVR;
409 case -ENOSPC:
410 return INTEL_PT_ERR_LOST;
411 case -ELOOP:
412 return INTEL_PT_ERR_NELOOP;
413 default:
414 return INTEL_PT_ERR_UNK;
415 }
416}
417
418static const char *intel_pt_err_msgs[] = {
419 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
420 [INTEL_PT_ERR_INTERN] = "Internal error",
421 [INTEL_PT_ERR_BADPKT] = "Bad packet",
422 [INTEL_PT_ERR_NODATA] = "No more data",
423 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
424 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
425 [INTEL_PT_ERR_OVR] = "Overflow packet",
426 [INTEL_PT_ERR_LOST] = "Lost trace data",
427 [INTEL_PT_ERR_UNK] = "Unknown error!",
428 [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
429};
430
431int intel_pt__strerror(int code, char *buf, size_t buflen)
432{
433 if (code < 1 || code >= INTEL_PT_ERR_MAX)
434 code = INTEL_PT_ERR_UNK;
435 strlcpy(buf, intel_pt_err_msgs[code], buflen);
436 return 0;
437}
438
439static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
440 uint64_t last_ip)
441{
442 uint64_t ip;
443
444 switch (packet->count) {
445 case 1:
446 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
447 packet->payload;
448 break;
449 case 2:
450 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
451 packet->payload;
452 break;
453 case 3:
454 ip = packet->payload;
455
456 if (ip & (uint64_t)0x800000000000ULL)
457 ip |= (uint64_t)0xffff000000000000ULL;
458 break;
459 case 4:
460 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
461 packet->payload;
462 break;
463 case 6:
464 ip = packet->payload;
465 break;
466 default:
467 return 0;
468 }
469
470 return ip;
471}
472
473static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
474{
475 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
476 decoder->have_last_ip = true;
477}
478
479static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
480{
481 intel_pt_set_last_ip(decoder);
482 decoder->ip = decoder->last_ip;
483}
484
485static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
486{
487 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
488 decoder->buf);
489}
490
491static int intel_pt_bug(struct intel_pt_decoder *decoder)
492{
493 intel_pt_log("ERROR: Internal error\n");
494 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
495 return -ENOSYS;
496}
497
498static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
499{
500 decoder->tx_flags = 0;
501}
502
503static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
504{
505 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
506}
507
508static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)
509{
510 decoder->pip_payload = decoder->packet.payload;
511}
512
513static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)
514{
515 decoder->next_nr = decoder->pip_payload & 1;
516}
517
518static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)
519{
520 decoder->nr = decoder->pip_payload & 1;
521 decoder->next_nr = decoder->nr;
522}
523
524static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)
525{
526 intel_pt_update_pip(decoder);
527 intel_pt_set_nr(decoder);
528}
529
530static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
531{
532 intel_pt_clear_tx_flags(decoder);
533 decoder->have_tma = false;
534 decoder->pkt_len = 1;
535 decoder->pkt_step = 1;
536 intel_pt_decoder_log_packet(decoder);
537 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
538 intel_pt_log("ERROR: Bad packet\n");
539 decoder->pkt_state = INTEL_PT_STATE_ERR1;
540 }
541 return -EBADMSG;
542}
543
544static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
545{
546 decoder->sample_timestamp = decoder->timestamp;
547 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
548}
549
550static void intel_pt_reposition(struct intel_pt_decoder *decoder)
551{
552 decoder->ip = 0;
553 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
554 decoder->timestamp = 0;
555 decoder->have_tma = false;
556}
557
558static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)
559{
560 struct intel_pt_buffer buffer = { .buf = 0, };
561 int ret;
562
563 decoder->pkt_step = 0;
564
565 intel_pt_log("Getting more data\n");
566 ret = decoder->get_trace(&buffer, decoder->data);
567 if (ret)
568 return ret;
569 decoder->buf = buffer.buf;
570 decoder->len = buffer.len;
571 if (!decoder->len) {
572 intel_pt_log("No more data\n");
573 return -ENODATA;
574 }
575 decoder->buf_timestamp = buffer.ref_timestamp;
576 if (!buffer.consecutive || reposition) {
577 intel_pt_reposition(decoder);
578 decoder->ref_timestamp = buffer.ref_timestamp;
579 decoder->state.trace_nr = buffer.trace_nr;
580 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
581 decoder->ref_timestamp);
582 return -ENOLINK;
583 }
584
585 return 0;
586}
587
588static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,
589 bool reposition)
590{
591 if (!decoder->next_buf)
592 return intel_pt_get_data(decoder, reposition);
593
594 decoder->buf = decoder->next_buf;
595 decoder->len = decoder->next_len;
596 decoder->next_buf = 0;
597 decoder->next_len = 0;
598 return 0;
599}
600
601static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
602{
603 unsigned char *buf = decoder->temp_buf;
604 size_t old_len, len, n;
605 int ret;
606
607 old_len = decoder->len;
608 len = decoder->len;
609 memcpy(buf, decoder->buf, len);
610
611 ret = intel_pt_get_data(decoder, false);
612 if (ret) {
613 decoder->pos += old_len;
614 return ret < 0 ? ret : -EINVAL;
615 }
616
617 n = INTEL_PT_PKT_MAX_SZ - len;
618 if (n > decoder->len)
619 n = decoder->len;
620 memcpy(buf + len, decoder->buf, n);
621 len += n;
622
623 decoder->prev_pkt_ctx = decoder->pkt_ctx;
624 ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);
625 if (ret < (int)old_len) {
626 decoder->next_buf = decoder->buf;
627 decoder->next_len = decoder->len;
628 decoder->buf = buf;
629 decoder->len = old_len;
630 return intel_pt_bad_packet(decoder);
631 }
632
633 decoder->next_buf = decoder->buf + (ret - old_len);
634 decoder->next_len = decoder->len - (ret - old_len);
635
636 decoder->buf = buf;
637 decoder->len = ret;
638
639 return ret;
640}
641
642struct intel_pt_pkt_info {
643 struct intel_pt_decoder *decoder;
644 struct intel_pt_pkt packet;
645 uint64_t pos;
646 int pkt_len;
647 int last_packet_type;
648 void *data;
649};
650
651typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
652
653
654static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
655 intel_pt_pkt_cb_t cb, void *data)
656{
657 struct intel_pt_pkt_info pkt_info;
658 const unsigned char *buf = decoder->buf;
659 enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;
660 size_t len = decoder->len;
661 int ret;
662
663 pkt_info.decoder = decoder;
664 pkt_info.pos = decoder->pos;
665 pkt_info.pkt_len = decoder->pkt_step;
666 pkt_info.last_packet_type = decoder->last_packet_type;
667 pkt_info.data = data;
668
669 while (1) {
670 do {
671 pkt_info.pos += pkt_info.pkt_len;
672 buf += pkt_info.pkt_len;
673 len -= pkt_info.pkt_len;
674
675 if (!len)
676 return INTEL_PT_NEED_MORE_BYTES;
677
678 ret = intel_pt_get_packet(buf, len, &pkt_info.packet,
679 &pkt_ctx);
680 if (!ret)
681 return INTEL_PT_NEED_MORE_BYTES;
682 if (ret < 0)
683 return ret;
684
685 pkt_info.pkt_len = ret;
686 } while (pkt_info.packet.type == INTEL_PT_PAD);
687
688 ret = cb(&pkt_info);
689 if (ret)
690 return 0;
691
692 pkt_info.last_packet_type = pkt_info.packet.type;
693 }
694}
695
696struct intel_pt_calc_cyc_to_tsc_info {
697 uint64_t cycle_cnt;
698 unsigned int cbr;
699 uint32_t last_mtc;
700 uint64_t ctc_timestamp;
701 uint64_t ctc_delta;
702 uint64_t tsc_timestamp;
703 uint64_t timestamp;
704 bool have_tma;
705 bool fixup_last_mtc;
706 bool from_mtc;
707 double cbr_cyc_to_tsc;
708};
709
710
711
712
713
714
715
716
717static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
718 uint32_t *last_mtc)
719{
720 uint32_t first_missing_bit = 1U << (16 - mtc_shift);
721 uint32_t mask = ~(first_missing_bit - 1);
722
723 *last_mtc |= mtc & mask;
724 if (*last_mtc >= mtc) {
725 *last_mtc -= first_missing_bit;
726 *last_mtc &= 0xff;
727 }
728}
729
730static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
731{
732 struct intel_pt_decoder *decoder = pkt_info->decoder;
733 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
734 uint64_t timestamp;
735 double cyc_to_tsc;
736 unsigned int cbr;
737 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
738
739 switch (pkt_info->packet.type) {
740 case INTEL_PT_TNT:
741 case INTEL_PT_TIP_PGE:
742 case INTEL_PT_TIP:
743 case INTEL_PT_FUP:
744 case INTEL_PT_PSB:
745 case INTEL_PT_PIP:
746 case INTEL_PT_MODE_EXEC:
747 case INTEL_PT_MODE_TSX:
748 case INTEL_PT_PSBEND:
749 case INTEL_PT_PAD:
750 case INTEL_PT_VMCS:
751 case INTEL_PT_MNT:
752 case INTEL_PT_PTWRITE:
753 case INTEL_PT_PTWRITE_IP:
754 case INTEL_PT_BBP:
755 case INTEL_PT_BIP:
756 case INTEL_PT_BEP:
757 case INTEL_PT_BEP_IP:
758 return 0;
759
760 case INTEL_PT_MTC:
761 if (!data->have_tma)
762 return 0;
763
764 mtc = pkt_info->packet.payload;
765 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
766 data->fixup_last_mtc = false;
767 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
768 &data->last_mtc);
769 }
770 if (mtc > data->last_mtc)
771 mtc_delta = mtc - data->last_mtc;
772 else
773 mtc_delta = mtc + 256 - data->last_mtc;
774 data->ctc_delta += mtc_delta << decoder->mtc_shift;
775 data->last_mtc = mtc;
776
777 if (decoder->tsc_ctc_mult) {
778 timestamp = data->ctc_timestamp +
779 data->ctc_delta * decoder->tsc_ctc_mult;
780 } else {
781 timestamp = data->ctc_timestamp +
782 multdiv(data->ctc_delta,
783 decoder->tsc_ctc_ratio_n,
784 decoder->tsc_ctc_ratio_d);
785 }
786
787 if (timestamp < data->timestamp)
788 return 1;
789
790 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
791 data->timestamp = timestamp;
792 return 0;
793 }
794
795 break;
796
797 case INTEL_PT_TSC:
798
799
800
801
802 if (data->from_mtc)
803 return 1;
804 timestamp = pkt_info->packet.payload |
805 (data->timestamp & (0xffULL << 56));
806 if (data->from_mtc && timestamp < data->timestamp &&
807 data->timestamp - timestamp < decoder->tsc_slip)
808 return 1;
809 if (timestamp < data->timestamp)
810 timestamp += (1ULL << 56);
811 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
812 if (data->from_mtc)
813 return 1;
814 data->tsc_timestamp = timestamp;
815 data->timestamp = timestamp;
816 return 0;
817 }
818 break;
819
820 case INTEL_PT_TMA:
821 if (data->from_mtc)
822 return 1;
823
824 if (!decoder->tsc_ctc_ratio_d)
825 return 0;
826
827 ctc = pkt_info->packet.payload;
828 fc = pkt_info->packet.count;
829 ctc_rem = ctc & decoder->ctc_rem_mask;
830
831 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
832
833 data->ctc_timestamp = data->tsc_timestamp - fc;
834 if (decoder->tsc_ctc_mult) {
835 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
836 } else {
837 data->ctc_timestamp -=
838 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
839 decoder->tsc_ctc_ratio_d);
840 }
841
842 data->ctc_delta = 0;
843 data->have_tma = true;
844 data->fixup_last_mtc = true;
845
846 return 0;
847
848 case INTEL_PT_CYC:
849 data->cycle_cnt += pkt_info->packet.payload;
850 return 0;
851
852 case INTEL_PT_CBR:
853 cbr = pkt_info->packet.payload;
854 if (data->cbr && data->cbr != cbr)
855 return 1;
856 data->cbr = cbr;
857 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
858 return 0;
859
860 case INTEL_PT_TIP_PGD:
861 case INTEL_PT_TRACESTOP:
862 case INTEL_PT_EXSTOP:
863 case INTEL_PT_EXSTOP_IP:
864 case INTEL_PT_MWAIT:
865 case INTEL_PT_PWRE:
866 case INTEL_PT_PWRX:
867 case INTEL_PT_OVF:
868 case INTEL_PT_BAD:
869 default:
870 return 1;
871 }
872
873 if (!data->cbr && decoder->cbr) {
874 data->cbr = decoder->cbr;
875 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
876 }
877
878 if (!data->cycle_cnt)
879 return 1;
880
881 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
882
883 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
884 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
885 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
886 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
887 return 1;
888 }
889
890 decoder->calc_cyc_to_tsc = cyc_to_tsc;
891 decoder->have_calc_cyc_to_tsc = true;
892
893 if (data->cbr) {
894 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
895 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
896 } else {
897 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
898 cyc_to_tsc, pkt_info->pos);
899 }
900
901 return 1;
902}
903
904static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
905 bool from_mtc)
906{
907 struct intel_pt_calc_cyc_to_tsc_info data = {
908 .cycle_cnt = 0,
909 .cbr = 0,
910 .last_mtc = decoder->last_mtc,
911 .ctc_timestamp = decoder->ctc_timestamp,
912 .ctc_delta = decoder->ctc_delta,
913 .tsc_timestamp = decoder->tsc_timestamp,
914 .timestamp = decoder->timestamp,
915 .have_tma = decoder->have_tma,
916 .fixup_last_mtc = decoder->fixup_last_mtc,
917 .from_mtc = from_mtc,
918 .cbr_cyc_to_tsc = 0,
919 };
920
921
922
923
924
925
926 if (!from_mtc)
927 return;
928
929 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
930}
931
932static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
933{
934 int ret;
935
936 decoder->last_packet_type = decoder->packet.type;
937
938 do {
939 decoder->pos += decoder->pkt_step;
940 decoder->buf += decoder->pkt_step;
941 decoder->len -= decoder->pkt_step;
942
943 if (!decoder->len) {
944 ret = intel_pt_get_next_data(decoder, false);
945 if (ret)
946 return ret;
947 }
948
949 decoder->prev_pkt_ctx = decoder->pkt_ctx;
950 ret = intel_pt_get_packet(decoder->buf, decoder->len,
951 &decoder->packet, &decoder->pkt_ctx);
952 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
953 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
954 ret = intel_pt_get_split_packet(decoder);
955 if (ret < 0)
956 return ret;
957 }
958 if (ret <= 0)
959 return intel_pt_bad_packet(decoder);
960
961 decoder->pkt_len = ret;
962 decoder->pkt_step = ret;
963 intel_pt_decoder_log_packet(decoder);
964 } while (decoder->packet.type == INTEL_PT_PAD);
965
966 return 0;
967}
968
969static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
970{
971 uint64_t timestamp, masked_timestamp;
972
973 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
974 masked_timestamp = timestamp & decoder->period_mask;
975 if (decoder->continuous_period) {
976 if (masked_timestamp > decoder->last_masked_timestamp)
977 return 1;
978 } else {
979 timestamp += 1;
980 masked_timestamp = timestamp & decoder->period_mask;
981 if (masked_timestamp > decoder->last_masked_timestamp) {
982 decoder->last_masked_timestamp = masked_timestamp;
983 decoder->continuous_period = true;
984 }
985 }
986
987 if (masked_timestamp < decoder->last_masked_timestamp)
988 return decoder->period_ticks;
989
990 return decoder->period_ticks - (timestamp - masked_timestamp);
991}
992
993static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
994{
995 switch (decoder->period_type) {
996 case INTEL_PT_PERIOD_INSTRUCTIONS:
997 return decoder->period - decoder->period_insn_cnt;
998 case INTEL_PT_PERIOD_TICKS:
999 return intel_pt_next_period(decoder);
1000 case INTEL_PT_PERIOD_NONE:
1001 case INTEL_PT_PERIOD_MTC:
1002 default:
1003 return 0;
1004 }
1005}
1006
1007static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
1008{
1009 uint64_t timestamp, masked_timestamp;
1010
1011 switch (decoder->period_type) {
1012 case INTEL_PT_PERIOD_INSTRUCTIONS:
1013 decoder->period_insn_cnt = 0;
1014 break;
1015 case INTEL_PT_PERIOD_TICKS:
1016 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
1017 masked_timestamp = timestamp & decoder->period_mask;
1018 if (masked_timestamp > decoder->last_masked_timestamp)
1019 decoder->last_masked_timestamp = masked_timestamp;
1020 else
1021 decoder->last_masked_timestamp += decoder->period_ticks;
1022 break;
1023 case INTEL_PT_PERIOD_NONE:
1024 case INTEL_PT_PERIOD_MTC:
1025 default:
1026 break;
1027 }
1028
1029 decoder->state.type |= INTEL_PT_INSTRUCTION;
1030}
1031
1032static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
1033 struct intel_pt_insn *intel_pt_insn, uint64_t ip)
1034{
1035 uint64_t max_insn_cnt, insn_cnt = 0;
1036 int err;
1037
1038 if (!decoder->mtc_insn)
1039 decoder->mtc_insn = true;
1040
1041 max_insn_cnt = intel_pt_next_sample(decoder);
1042
1043 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
1044 max_insn_cnt, decoder->data);
1045
1046 decoder->tot_insn_cnt += insn_cnt;
1047 decoder->timestamp_insn_cnt += insn_cnt;
1048 decoder->sample_insn_cnt += insn_cnt;
1049 decoder->period_insn_cnt += insn_cnt;
1050
1051 if (err) {
1052 decoder->no_progress = 0;
1053 decoder->pkt_state = INTEL_PT_STATE_ERR2;
1054 intel_pt_log_at("ERROR: Failed to get instruction",
1055 decoder->ip);
1056 if (err == -ENOENT)
1057 return -ENOLINK;
1058 return -EILSEQ;
1059 }
1060
1061 if (ip && decoder->ip == ip) {
1062 err = -EAGAIN;
1063 goto out;
1064 }
1065
1066 if (max_insn_cnt && insn_cnt >= max_insn_cnt)
1067 intel_pt_sample_insn(decoder);
1068
1069 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1070 decoder->state.type = INTEL_PT_INSTRUCTION;
1071 decoder->state.from_ip = decoder->ip;
1072 decoder->state.to_ip = 0;
1073 decoder->ip += intel_pt_insn->length;
1074 err = INTEL_PT_RETURN;
1075 goto out;
1076 }
1077
1078 if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1079
1080 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
1081 intel_pt_insn->rel) {
1082 err = intel_pt_push(&decoder->stack, decoder->ip +
1083 intel_pt_insn->length);
1084 if (err)
1085 goto out;
1086 }
1087 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1088 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1089 }
1090
1091 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1092 int cnt = decoder->no_progress++;
1093
1094 decoder->state.from_ip = decoder->ip;
1095 decoder->ip += intel_pt_insn->length +
1096 intel_pt_insn->rel;
1097 decoder->state.to_ip = decoder->ip;
1098 err = INTEL_PT_RETURN;
1099
1100
1101
1102
1103
1104
1105
1106
1107 if (cnt) {
1108 if (cnt == 1) {
1109 decoder->stuck_ip = decoder->state.to_ip;
1110 decoder->stuck_ip_prd = 1;
1111 decoder->stuck_ip_cnt = 1;
1112 } else if (cnt > INTEL_PT_MAX_LOOPS ||
1113 decoder->state.to_ip == decoder->stuck_ip) {
1114 intel_pt_log_at("ERROR: Never-ending loop",
1115 decoder->state.to_ip);
1116 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1117 err = -ELOOP;
1118 goto out;
1119 } else if (!--decoder->stuck_ip_cnt) {
1120 decoder->stuck_ip_prd += 1;
1121 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1122 decoder->stuck_ip = decoder->state.to_ip;
1123 }
1124 }
1125 goto out_no_progress;
1126 }
1127out:
1128 decoder->no_progress = 0;
1129out_no_progress:
1130 decoder->state.insn_op = intel_pt_insn->op;
1131 decoder->state.insn_len = intel_pt_insn->length;
1132 memcpy(decoder->state.insn, intel_pt_insn->buf,
1133 INTEL_PT_INSN_BUF_SZ);
1134
1135 if (decoder->tx_flags & INTEL_PT_IN_TX)
1136 decoder->state.flags |= INTEL_PT_IN_TX;
1137
1138 return err;
1139}
1140
1141static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1142{
1143 bool ret = false;
1144
1145 if (decoder->set_fup_tx_flags) {
1146 decoder->set_fup_tx_flags = false;
1147 decoder->tx_flags = decoder->fup_tx_flags;
1148 decoder->state.type = INTEL_PT_TRANSACTION;
1149 if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)
1150 decoder->state.type |= INTEL_PT_BRANCH;
1151 decoder->state.from_ip = decoder->ip;
1152 decoder->state.to_ip = 0;
1153 decoder->state.flags = decoder->fup_tx_flags;
1154 return true;
1155 }
1156 if (decoder->set_fup_ptw) {
1157 decoder->set_fup_ptw = false;
1158 decoder->state.type = INTEL_PT_PTW;
1159 decoder->state.flags |= INTEL_PT_FUP_IP;
1160 decoder->state.from_ip = decoder->ip;
1161 decoder->state.to_ip = 0;
1162 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1163 return true;
1164 }
1165 if (decoder->set_fup_mwait) {
1166 decoder->set_fup_mwait = false;
1167 decoder->state.type = INTEL_PT_MWAIT_OP;
1168 decoder->state.from_ip = decoder->ip;
1169 decoder->state.to_ip = 0;
1170 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1171 ret = true;
1172 }
1173 if (decoder->set_fup_pwre) {
1174 decoder->set_fup_pwre = false;
1175 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1176 decoder->state.type &= ~INTEL_PT_BRANCH;
1177 decoder->state.from_ip = decoder->ip;
1178 decoder->state.to_ip = 0;
1179 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1180 ret = true;
1181 }
1182 if (decoder->set_fup_exstop) {
1183 decoder->set_fup_exstop = false;
1184 decoder->state.type |= INTEL_PT_EX_STOP;
1185 decoder->state.type &= ~INTEL_PT_BRANCH;
1186 decoder->state.flags |= INTEL_PT_FUP_IP;
1187 decoder->state.from_ip = decoder->ip;
1188 decoder->state.to_ip = 0;
1189 ret = true;
1190 }
1191 if (decoder->set_fup_bep) {
1192 decoder->set_fup_bep = false;
1193 decoder->state.type |= INTEL_PT_BLK_ITEMS;
1194 decoder->state.type &= ~INTEL_PT_BRANCH;
1195 decoder->state.from_ip = decoder->ip;
1196 decoder->state.to_ip = 0;
1197 ret = true;
1198 }
1199 return ret;
1200}
1201
1202static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1203 struct intel_pt_insn *intel_pt_insn,
1204 uint64_t ip, int err)
1205{
1206 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1207 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1208 ip == decoder->ip + intel_pt_insn->length;
1209}
1210
1211static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1212{
1213 struct intel_pt_insn intel_pt_insn;
1214 uint64_t ip;
1215 int err;
1216
1217 ip = decoder->last_ip;
1218
1219 while (1) {
1220 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1221 if (err == INTEL_PT_RETURN)
1222 return 0;
1223 if (err == -EAGAIN ||
1224 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1225 bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;
1226
1227 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1228 if (intel_pt_fup_event(decoder) && no_tip)
1229 return 0;
1230 return -EAGAIN;
1231 }
1232 decoder->set_fup_tx_flags = false;
1233 if (err)
1234 return err;
1235
1236 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1237 intel_pt_log_at("ERROR: Unexpected indirect branch",
1238 decoder->ip);
1239 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1240 return -ENOENT;
1241 }
1242
1243 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1244 intel_pt_log_at("ERROR: Unexpected conditional branch",
1245 decoder->ip);
1246 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1247 return -ENOENT;
1248 }
1249
1250 intel_pt_bug(decoder);
1251 }
1252}
1253
1254static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1255{
1256 struct intel_pt_insn intel_pt_insn;
1257 int err;
1258
1259 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1260 if (err == INTEL_PT_RETURN &&
1261 decoder->pgd_ip &&
1262 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1263 (decoder->state.type & INTEL_PT_BRANCH) &&
1264 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1265
1266 decoder->no_progress = 0;
1267 decoder->pge = false;
1268 decoder->continuous_period = false;
1269 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1270 decoder->state.type |= INTEL_PT_TRACE_END;
1271 intel_pt_update_nr(decoder);
1272 return 0;
1273 }
1274 if (err == INTEL_PT_RETURN)
1275 return 0;
1276 if (err)
1277 return err;
1278
1279 intel_pt_update_nr(decoder);
1280
1281 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1282 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1283 decoder->pge = false;
1284 decoder->continuous_period = false;
1285 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1286 decoder->state.from_ip = decoder->ip;
1287 if (decoder->packet.count == 0) {
1288 decoder->state.to_ip = 0;
1289 } else {
1290 decoder->state.to_ip = decoder->last_ip;
1291 decoder->ip = decoder->last_ip;
1292 }
1293 decoder->state.type |= INTEL_PT_TRACE_END;
1294 } else {
1295 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1296 decoder->state.from_ip = decoder->ip;
1297 if (decoder->packet.count == 0) {
1298 decoder->state.to_ip = 0;
1299 } else {
1300 decoder->state.to_ip = decoder->last_ip;
1301 decoder->ip = decoder->last_ip;
1302 }
1303 }
1304 return 0;
1305 }
1306
1307 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1308 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1309 intel_pt_insn.rel;
1310
1311 if (decoder->pgd_ip &&
1312 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1313 decoder->pgd_ip(to_ip, decoder->data)) {
1314
1315 decoder->pge = false;
1316 decoder->continuous_period = false;
1317 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1318 decoder->ip = to_ip;
1319 decoder->state.from_ip = decoder->ip;
1320 decoder->state.to_ip = to_ip;
1321 decoder->state.type |= INTEL_PT_TRACE_END;
1322 return 0;
1323 }
1324 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1325 decoder->ip);
1326 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1327 return -ENOENT;
1328 }
1329
1330 return intel_pt_bug(decoder);
1331}
1332
1333static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1334{
1335 struct intel_pt_insn intel_pt_insn;
1336 int err;
1337
1338 while (1) {
1339 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1340 if (err == INTEL_PT_RETURN)
1341 return 0;
1342 if (err)
1343 return err;
1344
1345 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1346 if (!decoder->return_compression) {
1347 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1348 decoder->ip);
1349 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1350 return -ENOENT;
1351 }
1352 if (!decoder->ret_addr) {
1353 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1354 decoder->ip);
1355 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1356 return -ENOENT;
1357 }
1358 if (!(decoder->tnt.payload & BIT63)) {
1359 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1360 decoder->ip);
1361 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1362 return -ENOENT;
1363 }
1364 decoder->tnt.count -= 1;
1365 if (decoder->tnt.count)
1366 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1367 else
1368 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1369 decoder->tnt.payload <<= 1;
1370 decoder->state.from_ip = decoder->ip;
1371 decoder->ip = decoder->ret_addr;
1372 decoder->state.to_ip = decoder->ip;
1373 return 0;
1374 }
1375
1376 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1377
1378 err = intel_pt_get_next_packet(decoder);
1379 if (err)
1380 return err;
1381 if (decoder->packet.type != INTEL_PT_TIP ||
1382 decoder->packet.count == 0) {
1383 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1384 decoder->ip);
1385 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1386 decoder->pkt_step = 0;
1387 return -ENOENT;
1388 }
1389 intel_pt_set_last_ip(decoder);
1390 decoder->state.from_ip = decoder->ip;
1391 decoder->state.to_ip = decoder->last_ip;
1392 decoder->ip = decoder->last_ip;
1393 intel_pt_update_nr(decoder);
1394 return 0;
1395 }
1396
1397 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1398 decoder->tnt.count -= 1;
1399 if (decoder->tnt.count)
1400 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1401 else
1402 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1403 if (decoder->tnt.payload & BIT63) {
1404 decoder->tnt.payload <<= 1;
1405 decoder->state.from_ip = decoder->ip;
1406 decoder->ip += intel_pt_insn.length +
1407 intel_pt_insn.rel;
1408 decoder->state.to_ip = decoder->ip;
1409 return 0;
1410 }
1411
1412 if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1413 decoder->tnt.payload <<= 1;
1414 decoder->state.type = INTEL_PT_INSTRUCTION;
1415 decoder->state.from_ip = decoder->ip;
1416 decoder->state.to_ip = 0;
1417 decoder->ip += intel_pt_insn.length;
1418 return 0;
1419 }
1420 decoder->sample_cyc = false;
1421 decoder->ip += intel_pt_insn.length;
1422 if (!decoder->tnt.count) {
1423 intel_pt_update_sample_time(decoder);
1424 return -EAGAIN;
1425 }
1426 decoder->tnt.payload <<= 1;
1427 continue;
1428 }
1429
1430 return intel_pt_bug(decoder);
1431 }
1432}
1433
1434static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1435{
1436 unsigned int fup_tx_flags;
1437 int err;
1438
1439 fup_tx_flags = decoder->packet.payload &
1440 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1441 err = intel_pt_get_next_packet(decoder);
1442 if (err)
1443 return err;
1444 if (decoder->packet.type == INTEL_PT_FUP) {
1445 decoder->fup_tx_flags = fup_tx_flags;
1446 decoder->set_fup_tx_flags = true;
1447 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1448 *no_tip = true;
1449 } else {
1450 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1451 decoder->pos);
1452 intel_pt_update_in_tx(decoder);
1453 }
1454 return 0;
1455}
1456
1457static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)
1458{
1459 timestamp |= (ref_timestamp & (0xffULL << 56));
1460
1461 if (timestamp < ref_timestamp) {
1462 if (ref_timestamp - timestamp > (1ULL << 55))
1463 timestamp += (1ULL << 56);
1464 } else {
1465 if (timestamp - ref_timestamp > (1ULL << 55))
1466 timestamp -= (1ULL << 56);
1467 }
1468
1469 return timestamp;
1470}
1471
1472static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1473{
1474 uint64_t timestamp;
1475
1476 decoder->have_tma = false;
1477
1478 if (decoder->ref_timestamp) {
1479 timestamp = intel_pt_8b_tsc(decoder->packet.payload,
1480 decoder->ref_timestamp);
1481 decoder->tsc_timestamp = timestamp;
1482 decoder->timestamp = timestamp;
1483 decoder->ref_timestamp = 0;
1484 decoder->timestamp_insn_cnt = 0;
1485 } else if (decoder->timestamp) {
1486 timestamp = decoder->packet.payload |
1487 (decoder->timestamp & (0xffULL << 56));
1488 decoder->tsc_timestamp = timestamp;
1489 if (timestamp < decoder->timestamp &&
1490 decoder->timestamp - timestamp < decoder->tsc_slip) {
1491 intel_pt_log_to("Suppressing backwards timestamp",
1492 timestamp);
1493 timestamp = decoder->timestamp;
1494 }
1495 if (timestamp < decoder->timestamp) {
1496 intel_pt_log_to("Wraparound timestamp", timestamp);
1497 timestamp += (1ULL << 56);
1498 decoder->tsc_timestamp = timestamp;
1499 }
1500 decoder->timestamp = timestamp;
1501 decoder->timestamp_insn_cnt = 0;
1502 }
1503
1504 if (decoder->last_packet_type == INTEL_PT_CYC) {
1505 decoder->cyc_ref_timestamp = decoder->timestamp;
1506 decoder->cycle_cnt = 0;
1507 decoder->have_calc_cyc_to_tsc = false;
1508 intel_pt_calc_cyc_to_tsc(decoder, false);
1509 }
1510
1511 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1512}
1513
1514static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1515{
1516 intel_pt_log("ERROR: Buffer overflow\n");
1517 intel_pt_clear_tx_flags(decoder);
1518 intel_pt_set_nr(decoder);
1519 decoder->timestamp_insn_cnt = 0;
1520 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1521 decoder->overflow = true;
1522 return -EOVERFLOW;
1523}
1524
1525static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1526{
1527 if (decoder->have_cyc)
1528 return;
1529
1530 decoder->cyc_cnt_timestamp = decoder->timestamp;
1531 decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1532}
1533
1534static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1535{
1536 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1537
1538 if (decoder->pge)
1539 intel_pt_mtc_cyc_cnt_pge(decoder);
1540}
1541
1542static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1543{
1544 uint64_t tot_cyc_cnt, tsc_delta;
1545
1546 if (decoder->have_cyc)
1547 return;
1548
1549 decoder->sample_cyc = true;
1550
1551 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)
1552 return;
1553
1554 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1555 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1556
1557 if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1558 decoder->tot_cyc_cnt = tot_cyc_cnt;
1559}
1560
1561static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1562{
1563 uint32_t ctc = decoder->packet.payload;
1564 uint32_t fc = decoder->packet.count;
1565 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1566
1567 if (!decoder->tsc_ctc_ratio_d)
1568 return;
1569
1570 if (decoder->pge && !decoder->in_psb)
1571 intel_pt_mtc_cyc_cnt_pge(decoder);
1572 else
1573 intel_pt_mtc_cyc_cnt_upd(decoder);
1574
1575 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1576 decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1577 if (decoder->tsc_ctc_mult) {
1578 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1579 } else {
1580 decoder->ctc_timestamp -= multdiv(ctc_rem,
1581 decoder->tsc_ctc_ratio_n,
1582 decoder->tsc_ctc_ratio_d);
1583 }
1584 decoder->ctc_delta = 0;
1585 decoder->have_tma = true;
1586 decoder->fixup_last_mtc = true;
1587 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n",
1588 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1589}
1590
1591static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1592{
1593 uint64_t timestamp;
1594 uint32_t mtc, mtc_delta;
1595
1596 if (!decoder->have_tma)
1597 return;
1598
1599 mtc = decoder->packet.payload;
1600
1601 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1602 decoder->fixup_last_mtc = false;
1603 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1604 &decoder->last_mtc);
1605 }
1606
1607 if (mtc > decoder->last_mtc)
1608 mtc_delta = mtc - decoder->last_mtc;
1609 else
1610 mtc_delta = mtc + 256 - decoder->last_mtc;
1611
1612 decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1613
1614 if (decoder->tsc_ctc_mult) {
1615 timestamp = decoder->ctc_timestamp +
1616 decoder->ctc_delta * decoder->tsc_ctc_mult;
1617 } else {
1618 timestamp = decoder->ctc_timestamp +
1619 multdiv(decoder->ctc_delta,
1620 decoder->tsc_ctc_ratio_n,
1621 decoder->tsc_ctc_ratio_d);
1622 }
1623
1624 if (timestamp < decoder->timestamp)
1625 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1626 timestamp, decoder->timestamp);
1627 else
1628 decoder->timestamp = timestamp;
1629
1630 intel_pt_mtc_cyc_cnt_upd(decoder);
1631
1632 decoder->timestamp_insn_cnt = 0;
1633 decoder->last_mtc = mtc;
1634
1635 if (decoder->last_packet_type == INTEL_PT_CYC) {
1636 decoder->cyc_ref_timestamp = decoder->timestamp;
1637 decoder->cycle_cnt = 0;
1638 decoder->have_calc_cyc_to_tsc = false;
1639 intel_pt_calc_cyc_to_tsc(decoder, true);
1640 }
1641
1642 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1643}
1644
1645static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1646{
1647 unsigned int cbr = decoder->packet.payload & 0xff;
1648
1649 decoder->cbr_payload = decoder->packet.payload;
1650
1651 if (decoder->cbr == cbr)
1652 return;
1653
1654 decoder->cbr = cbr;
1655 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1656
1657 intel_pt_mtc_cyc_cnt_cbr(decoder);
1658}
1659
1660static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1661{
1662 uint64_t timestamp = decoder->cyc_ref_timestamp;
1663
1664 decoder->have_cyc = true;
1665
1666 decoder->cycle_cnt += decoder->packet.payload;
1667 if (decoder->pge)
1668 decoder->tot_cyc_cnt += decoder->packet.payload;
1669 decoder->sample_cyc = true;
1670
1671 if (!decoder->cyc_ref_timestamp)
1672 return;
1673
1674 if (decoder->have_calc_cyc_to_tsc)
1675 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1676 else if (decoder->cbr)
1677 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1678 else
1679 return;
1680
1681 if (timestamp < decoder->timestamp)
1682 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1683 timestamp, decoder->timestamp);
1684 else
1685 decoder->timestamp = timestamp;
1686
1687 decoder->timestamp_insn_cnt = 0;
1688
1689 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1690}
1691
1692static void intel_pt_bbp(struct intel_pt_decoder *decoder)
1693{
1694 if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {
1695 memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));
1696 decoder->state.items.is_32_bit = false;
1697 }
1698 decoder->blk_type = decoder->packet.payload;
1699 decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);
1700 if (decoder->blk_type == INTEL_PT_GP_REGS)
1701 decoder->state.items.is_32_bit = decoder->packet.count;
1702 if (decoder->blk_type_pos < 0) {
1703 intel_pt_log("WARNING: Unknown block type %u\n",
1704 decoder->blk_type);
1705 } else if (decoder->state.items.mask[decoder->blk_type_pos]) {
1706 intel_pt_log("WARNING: Duplicate block type %u\n",
1707 decoder->blk_type);
1708 }
1709}
1710
1711static void intel_pt_bip(struct intel_pt_decoder *decoder)
1712{
1713 uint32_t id = decoder->packet.count;
1714 uint32_t bit = 1 << id;
1715 int pos = decoder->blk_type_pos;
1716
1717 if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) {
1718 intel_pt_log("WARNING: Unknown block item %u type %d\n",
1719 id, decoder->blk_type);
1720 return;
1721 }
1722
1723 if (decoder->state.items.mask[pos] & bit) {
1724 intel_pt_log("WARNING: Duplicate block item %u type %d\n",
1725 id, decoder->blk_type);
1726 }
1727
1728 decoder->state.items.mask[pos] |= bit;
1729 decoder->state.items.val[pos][id] = decoder->packet.payload;
1730}
1731
1732
1733static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1734{
1735 int err;
1736
1737 decoder->in_psb = true;
1738
1739 while (1) {
1740 err = intel_pt_get_next_packet(decoder);
1741 if (err)
1742 goto out;
1743
1744 switch (decoder->packet.type) {
1745 case INTEL_PT_PSBEND:
1746 err = 0;
1747 goto out;
1748
1749 case INTEL_PT_TIP_PGD:
1750 case INTEL_PT_TIP_PGE:
1751 case INTEL_PT_TIP:
1752 case INTEL_PT_TNT:
1753 case INTEL_PT_TRACESTOP:
1754 case INTEL_PT_BAD:
1755 case INTEL_PT_PSB:
1756 case INTEL_PT_PTWRITE:
1757 case INTEL_PT_PTWRITE_IP:
1758 case INTEL_PT_EXSTOP:
1759 case INTEL_PT_EXSTOP_IP:
1760 case INTEL_PT_MWAIT:
1761 case INTEL_PT_PWRE:
1762 case INTEL_PT_PWRX:
1763 case INTEL_PT_BBP:
1764 case INTEL_PT_BIP:
1765 case INTEL_PT_BEP:
1766 case INTEL_PT_BEP_IP:
1767 decoder->have_tma = false;
1768 intel_pt_log("ERROR: Unexpected packet\n");
1769 err = -EAGAIN;
1770 goto out;
1771
1772 case INTEL_PT_OVF:
1773 err = intel_pt_overflow(decoder);
1774 goto out;
1775
1776 case INTEL_PT_TSC:
1777 intel_pt_calc_tsc_timestamp(decoder);
1778 break;
1779
1780 case INTEL_PT_TMA:
1781 intel_pt_calc_tma(decoder);
1782 break;
1783
1784 case INTEL_PT_CBR:
1785 intel_pt_calc_cbr(decoder);
1786 break;
1787
1788 case INTEL_PT_MODE_EXEC:
1789 decoder->exec_mode = decoder->packet.payload;
1790 break;
1791
1792 case INTEL_PT_PIP:
1793 intel_pt_set_pip(decoder);
1794 break;
1795
1796 case INTEL_PT_FUP:
1797 decoder->pge = true;
1798 if (decoder->packet.count) {
1799 intel_pt_set_last_ip(decoder);
1800 decoder->psb_ip = decoder->last_ip;
1801 }
1802 break;
1803
1804 case INTEL_PT_MODE_TSX:
1805 intel_pt_update_in_tx(decoder);
1806 break;
1807
1808 case INTEL_PT_MTC:
1809 intel_pt_calc_mtc_timestamp(decoder);
1810 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1811 decoder->state.type |= INTEL_PT_INSTRUCTION;
1812 break;
1813
1814 case INTEL_PT_CYC:
1815 intel_pt_calc_cyc_timestamp(decoder);
1816 break;
1817
1818 case INTEL_PT_VMCS:
1819 case INTEL_PT_MNT:
1820 case INTEL_PT_PAD:
1821 default:
1822 break;
1823 }
1824 }
1825out:
1826 decoder->in_psb = false;
1827
1828 return err;
1829}
1830
1831static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1832{
1833 int err;
1834
1835 if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1836 decoder->tx_flags = 0;
1837 decoder->state.flags &= ~INTEL_PT_IN_TX;
1838 decoder->state.flags |= INTEL_PT_ABORT_TX;
1839 } else {
1840 decoder->state.flags |= INTEL_PT_ASYNC;
1841 }
1842
1843 while (1) {
1844 err = intel_pt_get_next_packet(decoder);
1845 if (err)
1846 return err;
1847
1848 switch (decoder->packet.type) {
1849 case INTEL_PT_TNT:
1850 case INTEL_PT_FUP:
1851 case INTEL_PT_TRACESTOP:
1852 case INTEL_PT_PSB:
1853 case INTEL_PT_TSC:
1854 case INTEL_PT_TMA:
1855 case INTEL_PT_MODE_TSX:
1856 case INTEL_PT_BAD:
1857 case INTEL_PT_PSBEND:
1858 case INTEL_PT_PTWRITE:
1859 case INTEL_PT_PTWRITE_IP:
1860 case INTEL_PT_EXSTOP:
1861 case INTEL_PT_EXSTOP_IP:
1862 case INTEL_PT_MWAIT:
1863 case INTEL_PT_PWRE:
1864 case INTEL_PT_PWRX:
1865 case INTEL_PT_BBP:
1866 case INTEL_PT_BIP:
1867 case INTEL_PT_BEP:
1868 case INTEL_PT_BEP_IP:
1869 intel_pt_log("ERROR: Missing TIP after FUP\n");
1870 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1871 decoder->pkt_step = 0;
1872 return -ENOENT;
1873
1874 case INTEL_PT_CBR:
1875 intel_pt_calc_cbr(decoder);
1876 break;
1877
1878 case INTEL_PT_OVF:
1879 return intel_pt_overflow(decoder);
1880
1881 case INTEL_PT_TIP_PGD:
1882 decoder->state.from_ip = decoder->ip;
1883 if (decoder->packet.count == 0) {
1884 decoder->state.to_ip = 0;
1885 } else {
1886 intel_pt_set_ip(decoder);
1887 decoder->state.to_ip = decoder->ip;
1888 }
1889 decoder->pge = false;
1890 decoder->continuous_period = false;
1891 decoder->state.type |= INTEL_PT_TRACE_END;
1892 intel_pt_update_nr(decoder);
1893 return 0;
1894
1895 case INTEL_PT_TIP_PGE:
1896 decoder->pge = true;
1897 intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1898 decoder->ip);
1899 decoder->state.from_ip = 0;
1900 if (decoder->packet.count == 0) {
1901 decoder->state.to_ip = 0;
1902 } else {
1903 intel_pt_set_ip(decoder);
1904 decoder->state.to_ip = decoder->ip;
1905 }
1906 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1907 intel_pt_mtc_cyc_cnt_pge(decoder);
1908 intel_pt_set_nr(decoder);
1909 return 0;
1910
1911 case INTEL_PT_TIP:
1912 decoder->state.from_ip = decoder->ip;
1913 if (decoder->packet.count == 0) {
1914 decoder->state.to_ip = 0;
1915 } else {
1916 intel_pt_set_ip(decoder);
1917 decoder->state.to_ip = decoder->ip;
1918 }
1919 intel_pt_update_nr(decoder);
1920 return 0;
1921
1922 case INTEL_PT_PIP:
1923 intel_pt_update_pip(decoder);
1924 break;
1925
1926 case INTEL_PT_MTC:
1927 intel_pt_calc_mtc_timestamp(decoder);
1928 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1929 decoder->state.type |= INTEL_PT_INSTRUCTION;
1930 break;
1931
1932 case INTEL_PT_CYC:
1933 intel_pt_calc_cyc_timestamp(decoder);
1934 break;
1935
1936 case INTEL_PT_MODE_EXEC:
1937 decoder->exec_mode = decoder->packet.payload;
1938 break;
1939
1940 case INTEL_PT_VMCS:
1941 case INTEL_PT_MNT:
1942 case INTEL_PT_PAD:
1943 break;
1944
1945 default:
1946 return intel_pt_bug(decoder);
1947 }
1948 }
1949}
1950
1951static int intel_pt_resample(struct intel_pt_decoder *decoder)
1952{
1953 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1954 decoder->state.type = INTEL_PT_INSTRUCTION;
1955 decoder->state.from_ip = decoder->ip;
1956 decoder->state.to_ip = 0;
1957 return 0;
1958}
1959
1960#define HOP_PROCESS 0
1961#define HOP_IGNORE 1
1962#define HOP_RETURN 2
1963#define HOP_AGAIN 3
1964
1965static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
1966
1967
1968static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)
1969{
1970
1971 if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {
1972 *err = intel_pt_scan_for_psb(decoder);
1973 if (*err)
1974 return HOP_RETURN;
1975 }
1976
1977 switch (decoder->packet.type) {
1978 case INTEL_PT_TNT:
1979 return HOP_IGNORE;
1980
1981 case INTEL_PT_TIP_PGD:
1982 if (!decoder->packet.count) {
1983 intel_pt_set_nr(decoder);
1984 return HOP_IGNORE;
1985 }
1986 intel_pt_set_ip(decoder);
1987 decoder->state.type |= INTEL_PT_TRACE_END;
1988 decoder->state.from_ip = 0;
1989 decoder->state.to_ip = decoder->ip;
1990 intel_pt_update_nr(decoder);
1991 return HOP_RETURN;
1992
1993 case INTEL_PT_TIP:
1994 if (!decoder->packet.count) {
1995 intel_pt_set_nr(decoder);
1996 return HOP_IGNORE;
1997 }
1998 intel_pt_set_ip(decoder);
1999 decoder->state.type = INTEL_PT_INSTRUCTION;
2000 decoder->state.from_ip = decoder->ip;
2001 decoder->state.to_ip = 0;
2002 intel_pt_update_nr(decoder);
2003 return HOP_RETURN;
2004
2005 case INTEL_PT_FUP:
2006 if (!decoder->packet.count)
2007 return HOP_IGNORE;
2008 intel_pt_set_ip(decoder);
2009 if (intel_pt_fup_event(decoder))
2010 return HOP_RETURN;
2011 if (!decoder->branch_enable)
2012 *no_tip = true;
2013 if (*no_tip) {
2014 decoder->state.type = INTEL_PT_INSTRUCTION;
2015 decoder->state.from_ip = decoder->ip;
2016 decoder->state.to_ip = 0;
2017 return HOP_RETURN;
2018 }
2019 *err = intel_pt_walk_fup_tip(decoder);
2020 if (!*err)
2021 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2022 return HOP_RETURN;
2023
2024 case INTEL_PT_PSB:
2025 decoder->state.psb_offset = decoder->pos;
2026 decoder->psb_ip = 0;
2027 decoder->last_ip = 0;
2028 decoder->have_last_ip = true;
2029 *err = intel_pt_walk_psbend(decoder);
2030 if (*err == -EAGAIN)
2031 return HOP_AGAIN;
2032 if (*err)
2033 return HOP_RETURN;
2034 decoder->state.type = INTEL_PT_PSB_EVT;
2035 if (decoder->psb_ip) {
2036 decoder->state.type |= INTEL_PT_INSTRUCTION;
2037 decoder->ip = decoder->psb_ip;
2038 }
2039 decoder->state.from_ip = decoder->psb_ip;
2040 decoder->state.to_ip = 0;
2041 return HOP_RETURN;
2042
2043 case INTEL_PT_BAD:
2044 case INTEL_PT_PAD:
2045 case INTEL_PT_TIP_PGE:
2046 case INTEL_PT_TSC:
2047 case INTEL_PT_TMA:
2048 case INTEL_PT_MODE_EXEC:
2049 case INTEL_PT_MODE_TSX:
2050 case INTEL_PT_MTC:
2051 case INTEL_PT_CYC:
2052 case INTEL_PT_VMCS:
2053 case INTEL_PT_PSBEND:
2054 case INTEL_PT_CBR:
2055 case INTEL_PT_TRACESTOP:
2056 case INTEL_PT_PIP:
2057 case INTEL_PT_OVF:
2058 case INTEL_PT_MNT:
2059 case INTEL_PT_PTWRITE:
2060 case INTEL_PT_PTWRITE_IP:
2061 case INTEL_PT_EXSTOP:
2062 case INTEL_PT_EXSTOP_IP:
2063 case INTEL_PT_MWAIT:
2064 case INTEL_PT_PWRE:
2065 case INTEL_PT_PWRX:
2066 case INTEL_PT_BBP:
2067 case INTEL_PT_BIP:
2068 case INTEL_PT_BEP:
2069 case INTEL_PT_BEP_IP:
2070 default:
2071 return HOP_PROCESS;
2072 }
2073}
2074
2075struct intel_pt_psb_info {
2076 struct intel_pt_pkt fup_packet;
2077 bool fup;
2078 int after_psbend;
2079};
2080
2081
2082static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)
2083{
2084 struct intel_pt_psb_info *data = pkt_info->data;
2085
2086 switch (pkt_info->packet.type) {
2087 case INTEL_PT_PAD:
2088 case INTEL_PT_MNT:
2089 case INTEL_PT_TSC:
2090 case INTEL_PT_TMA:
2091 case INTEL_PT_MODE_EXEC:
2092 case INTEL_PT_MODE_TSX:
2093 case INTEL_PT_MTC:
2094 case INTEL_PT_CYC:
2095 case INTEL_PT_VMCS:
2096 case INTEL_PT_CBR:
2097 case INTEL_PT_PIP:
2098 if (data->after_psbend) {
2099 data->after_psbend -= 1;
2100 if (!data->after_psbend)
2101 return 1;
2102 }
2103 break;
2104
2105 case INTEL_PT_FUP:
2106 if (data->after_psbend)
2107 return 1;
2108 if (data->fup || pkt_info->packet.count == 0)
2109 return 1;
2110 data->fup_packet = pkt_info->packet;
2111 data->fup = true;
2112 break;
2113
2114 case INTEL_PT_PSBEND:
2115 if (!data->fup)
2116 return 1;
2117
2118 data->after_psbend = 6;
2119 break;
2120
2121 case INTEL_PT_TIP_PGE:
2122
2123 if (data->after_psbend)
2124 data->fup = false;
2125 return 1;
2126
2127 case INTEL_PT_PTWRITE:
2128 case INTEL_PT_PTWRITE_IP:
2129 case INTEL_PT_EXSTOP:
2130 case INTEL_PT_EXSTOP_IP:
2131 case INTEL_PT_MWAIT:
2132 case INTEL_PT_PWRE:
2133 case INTEL_PT_PWRX:
2134 case INTEL_PT_BBP:
2135 case INTEL_PT_BIP:
2136 case INTEL_PT_BEP:
2137 case INTEL_PT_BEP_IP:
2138 if (data->after_psbend) {
2139 data->after_psbend -= 1;
2140 if (!data->after_psbend)
2141 return 1;
2142 break;
2143 }
2144 return 1;
2145
2146 case INTEL_PT_OVF:
2147 case INTEL_PT_BAD:
2148 case INTEL_PT_TNT:
2149 case INTEL_PT_TIP_PGD:
2150 case INTEL_PT_TIP:
2151 case INTEL_PT_PSB:
2152 case INTEL_PT_TRACESTOP:
2153 default:
2154 return 1;
2155 }
2156
2157 return 0;
2158}
2159
2160static int intel_pt_psb(struct intel_pt_decoder *decoder)
2161{
2162 int err;
2163
2164 decoder->last_ip = 0;
2165 decoder->psb_ip = 0;
2166 decoder->have_last_ip = true;
2167 intel_pt_clear_stack(&decoder->stack);
2168 err = intel_pt_walk_psbend(decoder);
2169 if (err)
2170 return err;
2171 decoder->state.type = INTEL_PT_PSB_EVT;
2172 decoder->state.from_ip = decoder->psb_ip;
2173 decoder->state.to_ip = 0;
2174 return 0;
2175}
2176
2177static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)
2178{
2179 int err;
2180
2181 if (decoder->ip != decoder->last_ip) {
2182 err = intel_pt_walk_fup(decoder);
2183 if (!err || err != -EAGAIN)
2184 return err;
2185 }
2186
2187 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2188 err = intel_pt_psb(decoder);
2189 if (err) {
2190 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2191 return -ENOENT;
2192 }
2193
2194 return 0;
2195}
2196
2197static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)
2198{
2199 struct intel_pt_psb_info data = { .fup = false };
2200
2201 if (!decoder->branch_enable || !decoder->pge)
2202 return false;
2203
2204 intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);
2205 if (!data.fup)
2206 return false;
2207
2208 decoder->packet = data.fup_packet;
2209 intel_pt_set_last_ip(decoder);
2210 decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;
2211
2212 *err = intel_pt_fup_in_psb(decoder);
2213
2214 return true;
2215}
2216
2217static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
2218{
2219 int last_packet_type = INTEL_PT_PAD;
2220 bool no_tip = false;
2221 int err;
2222
2223 while (1) {
2224 err = intel_pt_get_next_packet(decoder);
2225 if (err)
2226 return err;
2227next:
2228 if (decoder->cyc_threshold) {
2229 if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)
2230 decoder->sample_cyc = false;
2231 last_packet_type = decoder->packet.type;
2232 }
2233
2234 if (decoder->hop) {
2235 switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {
2236 case HOP_IGNORE:
2237 continue;
2238 case HOP_RETURN:
2239 return err;
2240 case HOP_AGAIN:
2241 goto next;
2242 default:
2243 break;
2244 }
2245 }
2246
2247 switch (decoder->packet.type) {
2248 case INTEL_PT_TNT:
2249 if (!decoder->packet.count)
2250 break;
2251 decoder->tnt = decoder->packet;
2252 decoder->pkt_state = INTEL_PT_STATE_TNT;
2253 err = intel_pt_walk_tnt(decoder);
2254 if (err == -EAGAIN)
2255 break;
2256 return err;
2257
2258 case INTEL_PT_TIP_PGD:
2259 if (decoder->packet.count != 0)
2260 intel_pt_set_last_ip(decoder);
2261 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
2262 return intel_pt_walk_tip(decoder);
2263
2264 case INTEL_PT_TIP_PGE: {
2265 decoder->pge = true;
2266 intel_pt_mtc_cyc_cnt_pge(decoder);
2267 intel_pt_set_nr(decoder);
2268 if (decoder->packet.count == 0) {
2269 intel_pt_log_at("Skipping zero TIP.PGE",
2270 decoder->pos);
2271 break;
2272 }
2273 intel_pt_set_ip(decoder);
2274 decoder->state.from_ip = 0;
2275 decoder->state.to_ip = decoder->ip;
2276 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2277
2278
2279
2280
2281 if (decoder->hop)
2282 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2283 return 0;
2284 }
2285
2286 case INTEL_PT_OVF:
2287 return intel_pt_overflow(decoder);
2288
2289 case INTEL_PT_TIP:
2290 if (decoder->packet.count != 0)
2291 intel_pt_set_last_ip(decoder);
2292 decoder->pkt_state = INTEL_PT_STATE_TIP;
2293 return intel_pt_walk_tip(decoder);
2294
2295 case INTEL_PT_FUP:
2296 if (decoder->packet.count == 0) {
2297 intel_pt_log_at("Skipping zero FUP",
2298 decoder->pos);
2299 no_tip = false;
2300 break;
2301 }
2302 intel_pt_set_last_ip(decoder);
2303 if (!decoder->branch_enable) {
2304 decoder->ip = decoder->last_ip;
2305 if (intel_pt_fup_event(decoder))
2306 return 0;
2307 no_tip = false;
2308 break;
2309 }
2310 if (decoder->set_fup_mwait)
2311 no_tip = true;
2312 if (no_tip)
2313 decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;
2314 else
2315 decoder->pkt_state = INTEL_PT_STATE_FUP;
2316 err = intel_pt_walk_fup(decoder);
2317 if (err != -EAGAIN)
2318 return err;
2319 if (no_tip) {
2320 no_tip = false;
2321 break;
2322 }
2323 return intel_pt_walk_fup_tip(decoder);
2324
2325 case INTEL_PT_TRACESTOP:
2326 decoder->pge = false;
2327 decoder->continuous_period = false;
2328 intel_pt_clear_tx_flags(decoder);
2329 decoder->have_tma = false;
2330 break;
2331
2332 case INTEL_PT_PSB:
2333 decoder->state.psb_offset = decoder->pos;
2334 decoder->psb_ip = 0;
2335 if (intel_pt_psb_with_fup(decoder, &err))
2336 return err;
2337 err = intel_pt_psb(decoder);
2338 if (err == -EAGAIN)
2339 goto next;
2340 return err;
2341
2342 case INTEL_PT_PIP:
2343 intel_pt_update_pip(decoder);
2344 break;
2345
2346 case INTEL_PT_MTC:
2347 intel_pt_calc_mtc_timestamp(decoder);
2348 if (decoder->period_type != INTEL_PT_PERIOD_MTC)
2349 break;
2350
2351
2352
2353
2354 if (!decoder->mtc_insn)
2355 break;
2356 decoder->mtc_insn = false;
2357
2358 if (!decoder->timestamp)
2359 break;
2360 decoder->state.type = INTEL_PT_INSTRUCTION;
2361 decoder->state.from_ip = decoder->ip;
2362 decoder->state.to_ip = 0;
2363 decoder->mtc_insn = false;
2364 return 0;
2365
2366 case INTEL_PT_TSC:
2367 intel_pt_calc_tsc_timestamp(decoder);
2368 break;
2369
2370 case INTEL_PT_TMA:
2371 intel_pt_calc_tma(decoder);
2372 break;
2373
2374 case INTEL_PT_CYC:
2375 intel_pt_calc_cyc_timestamp(decoder);
2376 break;
2377
2378 case INTEL_PT_CBR:
2379 intel_pt_calc_cbr(decoder);
2380 if (decoder->cbr != decoder->cbr_seen) {
2381 decoder->state.type = 0;
2382 return 0;
2383 }
2384 break;
2385
2386 case INTEL_PT_MODE_EXEC:
2387 decoder->exec_mode = decoder->packet.payload;
2388 break;
2389
2390 case INTEL_PT_MODE_TSX:
2391
2392 if (!decoder->pge || decoder->in_psb) {
2393 intel_pt_update_in_tx(decoder);
2394 break;
2395 }
2396 err = intel_pt_mode_tsx(decoder, &no_tip);
2397 if (err)
2398 return err;
2399 goto next;
2400
2401 case INTEL_PT_BAD:
2402 return intel_pt_bug(decoder);
2403
2404 case INTEL_PT_PSBEND:
2405 case INTEL_PT_VMCS:
2406 case INTEL_PT_MNT:
2407 case INTEL_PT_PAD:
2408 break;
2409
2410 case INTEL_PT_PTWRITE_IP:
2411 decoder->fup_ptw_payload = decoder->packet.payload;
2412 err = intel_pt_get_next_packet(decoder);
2413 if (err)
2414 return err;
2415 if (decoder->packet.type == INTEL_PT_FUP) {
2416 decoder->set_fup_ptw = true;
2417 no_tip = true;
2418 } else {
2419 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
2420 decoder->pos);
2421 }
2422 goto next;
2423
2424 case INTEL_PT_PTWRITE:
2425 decoder->state.type = INTEL_PT_PTW;
2426 decoder->state.from_ip = decoder->ip;
2427 decoder->state.to_ip = 0;
2428 decoder->state.ptw_payload = decoder->packet.payload;
2429 return 0;
2430
2431 case INTEL_PT_MWAIT:
2432 decoder->fup_mwait_payload = decoder->packet.payload;
2433 decoder->set_fup_mwait = true;
2434 break;
2435
2436 case INTEL_PT_PWRE:
2437 if (decoder->set_fup_mwait) {
2438 decoder->fup_pwre_payload =
2439 decoder->packet.payload;
2440 decoder->set_fup_pwre = true;
2441 break;
2442 }
2443 decoder->state.type = INTEL_PT_PWR_ENTRY;
2444 decoder->state.from_ip = decoder->ip;
2445 decoder->state.to_ip = 0;
2446 decoder->state.pwrx_payload = decoder->packet.payload;
2447 return 0;
2448
2449 case INTEL_PT_EXSTOP_IP:
2450 err = intel_pt_get_next_packet(decoder);
2451 if (err)
2452 return err;
2453 if (decoder->packet.type == INTEL_PT_FUP) {
2454 decoder->set_fup_exstop = true;
2455 no_tip = true;
2456 } else {
2457 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
2458 decoder->pos);
2459 }
2460 goto next;
2461
2462 case INTEL_PT_EXSTOP:
2463 decoder->state.type = INTEL_PT_EX_STOP;
2464 decoder->state.from_ip = decoder->ip;
2465 decoder->state.to_ip = 0;
2466 return 0;
2467
2468 case INTEL_PT_PWRX:
2469 decoder->state.type = INTEL_PT_PWR_EXIT;
2470 decoder->state.from_ip = decoder->ip;
2471 decoder->state.to_ip = 0;
2472 decoder->state.pwrx_payload = decoder->packet.payload;
2473 return 0;
2474
2475 case INTEL_PT_BBP:
2476 intel_pt_bbp(decoder);
2477 break;
2478
2479 case INTEL_PT_BIP:
2480 intel_pt_bip(decoder);
2481 break;
2482
2483 case INTEL_PT_BEP:
2484 decoder->state.type = INTEL_PT_BLK_ITEMS;
2485 decoder->state.from_ip = decoder->ip;
2486 decoder->state.to_ip = 0;
2487 return 0;
2488
2489 case INTEL_PT_BEP_IP:
2490 err = intel_pt_get_next_packet(decoder);
2491 if (err)
2492 return err;
2493 if (decoder->packet.type == INTEL_PT_FUP) {
2494 decoder->set_fup_bep = true;
2495 no_tip = true;
2496 } else {
2497 intel_pt_log_at("ERROR: Missing FUP after BEP",
2498 decoder->pos);
2499 }
2500 goto next;
2501
2502 default:
2503 return intel_pt_bug(decoder);
2504 }
2505 }
2506}
2507
2508static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
2509{
2510 return decoder->packet.count &&
2511 (decoder->have_last_ip || decoder->packet.count == 3 ||
2512 decoder->packet.count == 6);
2513}
2514
2515
2516static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
2517{
2518 int err;
2519
2520 decoder->in_psb = true;
2521
2522 while (1) {
2523 err = intel_pt_get_next_packet(decoder);
2524 if (err)
2525 goto out;
2526
2527 switch (decoder->packet.type) {
2528 case INTEL_PT_TIP_PGD:
2529 decoder->continuous_period = false;
2530 __fallthrough;
2531 case INTEL_PT_TIP_PGE:
2532 case INTEL_PT_TIP:
2533 case INTEL_PT_PTWRITE:
2534 case INTEL_PT_PTWRITE_IP:
2535 case INTEL_PT_EXSTOP:
2536 case INTEL_PT_EXSTOP_IP:
2537 case INTEL_PT_MWAIT:
2538 case INTEL_PT_PWRE:
2539 case INTEL_PT_PWRX:
2540 case INTEL_PT_BBP:
2541 case INTEL_PT_BIP:
2542 case INTEL_PT_BEP:
2543 case INTEL_PT_BEP_IP:
2544 intel_pt_log("ERROR: Unexpected packet\n");
2545 err = -ENOENT;
2546 goto out;
2547
2548 case INTEL_PT_FUP:
2549 decoder->pge = true;
2550 if (intel_pt_have_ip(decoder)) {
2551 uint64_t current_ip = decoder->ip;
2552
2553 intel_pt_set_ip(decoder);
2554 decoder->psb_ip = decoder->ip;
2555 if (current_ip)
2556 intel_pt_log_to("Setting IP",
2557 decoder->ip);
2558 }
2559 break;
2560
2561 case INTEL_PT_MTC:
2562 intel_pt_calc_mtc_timestamp(decoder);
2563 break;
2564
2565 case INTEL_PT_TSC:
2566 intel_pt_calc_tsc_timestamp(decoder);
2567 break;
2568
2569 case INTEL_PT_TMA:
2570 intel_pt_calc_tma(decoder);
2571 break;
2572
2573 case INTEL_PT_CYC:
2574 intel_pt_calc_cyc_timestamp(decoder);
2575 break;
2576
2577 case INTEL_PT_CBR:
2578 intel_pt_calc_cbr(decoder);
2579 break;
2580
2581 case INTEL_PT_PIP:
2582 intel_pt_set_pip(decoder);
2583 break;
2584
2585 case INTEL_PT_MODE_EXEC:
2586 decoder->exec_mode = decoder->packet.payload;
2587 break;
2588
2589 case INTEL_PT_MODE_TSX:
2590 intel_pt_update_in_tx(decoder);
2591 break;
2592
2593 case INTEL_PT_TRACESTOP:
2594 decoder->pge = false;
2595 decoder->continuous_period = false;
2596 intel_pt_clear_tx_flags(decoder);
2597 __fallthrough;
2598
2599 case INTEL_PT_TNT:
2600 decoder->have_tma = false;
2601 intel_pt_log("ERROR: Unexpected packet\n");
2602 if (decoder->ip)
2603 decoder->pkt_state = INTEL_PT_STATE_ERR4;
2604 else
2605 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2606 err = -ENOENT;
2607 goto out;
2608
2609 case INTEL_PT_BAD:
2610 err = intel_pt_bug(decoder);
2611 goto out;
2612
2613 case INTEL_PT_OVF:
2614 err = intel_pt_overflow(decoder);
2615 goto out;
2616
2617 case INTEL_PT_PSBEND:
2618 err = 0;
2619 goto out;
2620
2621 case INTEL_PT_PSB:
2622 case INTEL_PT_VMCS:
2623 case INTEL_PT_MNT:
2624 case INTEL_PT_PAD:
2625 default:
2626 break;
2627 }
2628 }
2629out:
2630 decoder->in_psb = false;
2631
2632 return err;
2633}
2634
2635static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
2636{
2637 int err;
2638
2639 while (1) {
2640 err = intel_pt_get_next_packet(decoder);
2641 if (err)
2642 return err;
2643
2644 switch (decoder->packet.type) {
2645 case INTEL_PT_TIP_PGD:
2646 decoder->continuous_period = false;
2647 decoder->pge = false;
2648 if (intel_pt_have_ip(decoder))
2649 intel_pt_set_ip(decoder);
2650 if (!decoder->ip)
2651 break;
2652 decoder->state.type |= INTEL_PT_TRACE_END;
2653 return 0;
2654
2655 case INTEL_PT_TIP_PGE:
2656 decoder->pge = true;
2657 intel_pt_mtc_cyc_cnt_pge(decoder);
2658 if (intel_pt_have_ip(decoder))
2659 intel_pt_set_ip(decoder);
2660 if (!decoder->ip)
2661 break;
2662 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2663 return 0;
2664
2665 case INTEL_PT_TIP:
2666 decoder->pge = true;
2667 if (intel_pt_have_ip(decoder))
2668 intel_pt_set_ip(decoder);
2669 if (!decoder->ip)
2670 break;
2671 return 0;
2672
2673 case INTEL_PT_FUP:
2674 if (intel_pt_have_ip(decoder))
2675 intel_pt_set_ip(decoder);
2676 if (decoder->ip)
2677 return 0;
2678 break;
2679
2680 case INTEL_PT_MTC:
2681 intel_pt_calc_mtc_timestamp(decoder);
2682 break;
2683
2684 case INTEL_PT_TSC:
2685 intel_pt_calc_tsc_timestamp(decoder);
2686 break;
2687
2688 case INTEL_PT_TMA:
2689 intel_pt_calc_tma(decoder);
2690 break;
2691
2692 case INTEL_PT_CYC:
2693 intel_pt_calc_cyc_timestamp(decoder);
2694 break;
2695
2696 case INTEL_PT_CBR:
2697 intel_pt_calc_cbr(decoder);
2698 break;
2699
2700 case INTEL_PT_PIP:
2701 intel_pt_set_pip(decoder);
2702 break;
2703
2704 case INTEL_PT_MODE_EXEC:
2705 decoder->exec_mode = decoder->packet.payload;
2706 break;
2707
2708 case INTEL_PT_MODE_TSX:
2709 intel_pt_update_in_tx(decoder);
2710 break;
2711
2712 case INTEL_PT_OVF:
2713 return intel_pt_overflow(decoder);
2714
2715 case INTEL_PT_BAD:
2716 return intel_pt_bug(decoder);
2717
2718 case INTEL_PT_TRACESTOP:
2719 decoder->pge = false;
2720 decoder->continuous_period = false;
2721 intel_pt_clear_tx_flags(decoder);
2722 decoder->have_tma = false;
2723 break;
2724
2725 case INTEL_PT_PSB:
2726 decoder->state.psb_offset = decoder->pos;
2727 decoder->psb_ip = 0;
2728 decoder->last_ip = 0;
2729 decoder->have_last_ip = true;
2730 intel_pt_clear_stack(&decoder->stack);
2731 err = intel_pt_walk_psb(decoder);
2732 if (err)
2733 return err;
2734 decoder->state.type = INTEL_PT_PSB_EVT;
2735 decoder->state.from_ip = decoder->psb_ip;
2736 decoder->state.to_ip = 0;
2737 return 0;
2738
2739 case INTEL_PT_TNT:
2740 case INTEL_PT_PSBEND:
2741 case INTEL_PT_VMCS:
2742 case INTEL_PT_MNT:
2743 case INTEL_PT_PAD:
2744 case INTEL_PT_PTWRITE:
2745 case INTEL_PT_PTWRITE_IP:
2746 case INTEL_PT_EXSTOP:
2747 case INTEL_PT_EXSTOP_IP:
2748 case INTEL_PT_MWAIT:
2749 case INTEL_PT_PWRE:
2750 case INTEL_PT_PWRX:
2751 case INTEL_PT_BBP:
2752 case INTEL_PT_BIP:
2753 case INTEL_PT_BEP:
2754 case INTEL_PT_BEP_IP:
2755 default:
2756 break;
2757 }
2758 }
2759}
2760
2761static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
2762{
2763 int err;
2764
2765 decoder->set_fup_tx_flags = false;
2766 decoder->set_fup_ptw = false;
2767 decoder->set_fup_mwait = false;
2768 decoder->set_fup_pwre = false;
2769 decoder->set_fup_exstop = false;
2770 decoder->set_fup_bep = false;
2771
2772 if (!decoder->branch_enable) {
2773 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2774 decoder->overflow = false;
2775 decoder->state.type = 0;
2776 return 0;
2777 }
2778
2779 intel_pt_log("Scanning for full IP\n");
2780 err = intel_pt_walk_to_ip(decoder);
2781 if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))
2782 return err;
2783
2784
2785 if (decoder->hop)
2786 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2787 else
2788 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2789 decoder->overflow = false;
2790
2791 decoder->state.from_ip = 0;
2792 decoder->state.to_ip = decoder->ip;
2793 intel_pt_log_to("Setting IP", decoder->ip);
2794
2795 return 0;
2796}
2797
2798static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
2799{
2800 const unsigned char *end = decoder->buf + decoder->len;
2801 size_t i;
2802
2803 for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
2804 if (i > decoder->len)
2805 continue;
2806 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
2807 return i;
2808 }
2809 return 0;
2810}
2811
2812static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
2813{
2814 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
2815 const char *psb = INTEL_PT_PSB_STR;
2816
2817 if (rest_psb > decoder->len ||
2818 memcmp(decoder->buf, psb + part_psb, rest_psb))
2819 return 0;
2820
2821 return rest_psb;
2822}
2823
2824static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2825 int part_psb)
2826{
2827 int rest_psb, ret;
2828
2829 decoder->pos += decoder->len;
2830 decoder->len = 0;
2831
2832 ret = intel_pt_get_next_data(decoder, false);
2833 if (ret)
2834 return ret;
2835
2836 rest_psb = intel_pt_rest_psb(decoder, part_psb);
2837 if (!rest_psb)
2838 return 0;
2839
2840 decoder->pos -= part_psb;
2841 decoder->next_buf = decoder->buf + rest_psb;
2842 decoder->next_len = decoder->len - rest_psb;
2843 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2844 decoder->buf = decoder->temp_buf;
2845 decoder->len = INTEL_PT_PSB_LEN;
2846
2847 return 0;
2848}
2849
2850static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2851{
2852 unsigned char *next;
2853 int ret;
2854
2855 intel_pt_log("Scanning for PSB\n");
2856 while (1) {
2857 if (!decoder->len) {
2858 ret = intel_pt_get_next_data(decoder, false);
2859 if (ret)
2860 return ret;
2861 }
2862
2863 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2864 INTEL_PT_PSB_LEN);
2865 if (!next) {
2866 int part_psb;
2867
2868 part_psb = intel_pt_part_psb(decoder);
2869 if (part_psb) {
2870 ret = intel_pt_get_split_psb(decoder, part_psb);
2871 if (ret)
2872 return ret;
2873 } else {
2874 decoder->pos += decoder->len;
2875 decoder->len = 0;
2876 }
2877 continue;
2878 }
2879
2880 decoder->pkt_step = next - decoder->buf;
2881 return intel_pt_get_next_packet(decoder);
2882 }
2883}
2884
2885static int intel_pt_sync(struct intel_pt_decoder *decoder)
2886{
2887 int err;
2888
2889 decoder->pge = false;
2890 decoder->continuous_period = false;
2891 decoder->have_last_ip = false;
2892 decoder->last_ip = 0;
2893 decoder->psb_ip = 0;
2894 decoder->ip = 0;
2895 intel_pt_clear_stack(&decoder->stack);
2896
2897 err = intel_pt_scan_for_psb(decoder);
2898 if (err)
2899 return err;
2900
2901 decoder->have_last_ip = true;
2902 decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2903
2904 err = intel_pt_walk_psb(decoder);
2905 if (err)
2906 return err;
2907
2908 decoder->state.type = INTEL_PT_PSB_EVT;
2909 decoder->state.from_ip = decoder->psb_ip;
2910 decoder->state.to_ip = 0;
2911
2912 if (decoder->ip) {
2913
2914
2915
2916
2917 if (decoder->hop)
2918 decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;
2919 else
2920 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2921 }
2922
2923 return 0;
2924}
2925
2926static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2927{
2928 uint64_t est = decoder->sample_insn_cnt << 1;
2929
2930 if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2931 goto out;
2932
2933 est *= decoder->max_non_turbo_ratio;
2934 est /= decoder->cbr;
2935out:
2936 return decoder->sample_timestamp + est;
2937}
2938
2939const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2940{
2941 int err;
2942
2943 do {
2944 decoder->state.type = INTEL_PT_BRANCH;
2945 decoder->state.flags = 0;
2946
2947 switch (decoder->pkt_state) {
2948 case INTEL_PT_STATE_NO_PSB:
2949 err = intel_pt_sync(decoder);
2950 break;
2951 case INTEL_PT_STATE_NO_IP:
2952 decoder->have_last_ip = false;
2953 decoder->last_ip = 0;
2954 decoder->ip = 0;
2955 __fallthrough;
2956 case INTEL_PT_STATE_ERR_RESYNC:
2957 err = intel_pt_sync_ip(decoder);
2958 break;
2959 case INTEL_PT_STATE_IN_SYNC:
2960 err = intel_pt_walk_trace(decoder);
2961 break;
2962 case INTEL_PT_STATE_TNT:
2963 case INTEL_PT_STATE_TNT_CONT:
2964 err = intel_pt_walk_tnt(decoder);
2965 if (err == -EAGAIN)
2966 err = intel_pt_walk_trace(decoder);
2967 break;
2968 case INTEL_PT_STATE_TIP:
2969 case INTEL_PT_STATE_TIP_PGD:
2970 err = intel_pt_walk_tip(decoder);
2971 break;
2972 case INTEL_PT_STATE_FUP:
2973 err = intel_pt_walk_fup(decoder);
2974 if (err == -EAGAIN)
2975 err = intel_pt_walk_fup_tip(decoder);
2976 break;
2977 case INTEL_PT_STATE_FUP_NO_TIP:
2978 err = intel_pt_walk_fup(decoder);
2979 if (err == -EAGAIN)
2980 err = intel_pt_walk_trace(decoder);
2981 break;
2982 case INTEL_PT_STATE_FUP_IN_PSB:
2983 err = intel_pt_fup_in_psb(decoder);
2984 break;
2985 case INTEL_PT_STATE_RESAMPLE:
2986 err = intel_pt_resample(decoder);
2987 break;
2988 default:
2989 err = intel_pt_bug(decoder);
2990 break;
2991 }
2992 } while (err == -ENOLINK);
2993
2994 if (err) {
2995 decoder->state.err = intel_pt_ext_err(err);
2996 decoder->state.from_ip = decoder->ip;
2997 intel_pt_update_sample_time(decoder);
2998 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
2999 intel_pt_set_nr(decoder);
3000 } else {
3001 decoder->state.err = 0;
3002 if (decoder->cbr != decoder->cbr_seen) {
3003 decoder->cbr_seen = decoder->cbr;
3004 if (!decoder->state.type) {
3005 decoder->state.from_ip = decoder->ip;
3006 decoder->state.to_ip = 0;
3007 }
3008 decoder->state.type |= INTEL_PT_CBR_CHG;
3009 decoder->state.cbr_payload = decoder->cbr_payload;
3010 decoder->state.cbr = decoder->cbr;
3011 }
3012 if (intel_pt_sample_time(decoder->pkt_state)) {
3013 intel_pt_update_sample_time(decoder);
3014 if (decoder->sample_cyc) {
3015 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
3016 decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
3017 decoder->sample_cyc = false;
3018 }
3019 }
3020
3021
3022
3023
3024 if (!decoder->have_cyc)
3025 decoder->state.flags |= INTEL_PT_SAMPLE_IPC;
3026 }
3027
3028
3029 if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)
3030 decoder->sample_timestamp = decoder->tsc_timestamp;
3031
3032 decoder->state.from_nr = decoder->nr;
3033 decoder->state.to_nr = decoder->next_nr;
3034 decoder->nr = decoder->next_nr;
3035
3036 decoder->state.timestamp = decoder->sample_timestamp;
3037 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
3038 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
3039 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
3040
3041 return &decoder->state;
3042}
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
3056{
3057 unsigned char *next;
3058
3059 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3060 if (next) {
3061 *len -= next - *buf;
3062 *buf = next;
3063 return true;
3064 }
3065 return false;
3066}
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
3081{
3082 unsigned char *next;
3083
3084 if (!*len)
3085 return false;
3086
3087 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
3088 if (next) {
3089 *len -= next - *buf;
3090 *buf = next;
3091 return true;
3092 }
3093 return false;
3094}
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
3106{
3107 const char *n = INTEL_PT_PSB_STR;
3108 unsigned char *p;
3109 size_t k;
3110
3111 if (len < INTEL_PT_PSB_LEN)
3112 return NULL;
3113
3114 k = len - INTEL_PT_PSB_LEN + 1;
3115 while (1) {
3116 p = memrchr(buf, n[0], k);
3117 if (!p)
3118 return NULL;
3119 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
3120 return p;
3121 k = p - buf;
3122 if (!k)
3123 return NULL;
3124 }
3125}
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
3141 size_t *rem)
3142{
3143 enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;
3144 struct intel_pt_pkt packet;
3145 int ret;
3146
3147 while (len) {
3148 ret = intel_pt_get_packet(buf, len, &packet, &ctx);
3149 if (ret <= 0)
3150 return false;
3151 if (packet.type == INTEL_PT_TSC) {
3152 *tsc = packet.payload;
3153 *rem = len;
3154 return true;
3155 }
3156 if (packet.type == INTEL_PT_PSBEND)
3157 return false;
3158 buf += ret;
3159 len -= ret;
3160 }
3161 return false;
3162}
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
3178{
3179 const uint64_t halfway = (1ULL << 55);
3180
3181 if (tsc1 == tsc2)
3182 return 0;
3183
3184 if (tsc1 < tsc2) {
3185 if (tsc2 - tsc1 < halfway)
3186 return -1;
3187 else
3188 return 1;
3189 } else {
3190 if (tsc1 - tsc2 < halfway)
3191 return 1;
3192 else
3193 return -1;
3194 }
3195}
3196
3197#define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210static unsigned char *adj_for_padding(unsigned char *buf_b,
3211 unsigned char *buf_a, size_t len_a)
3212{
3213 unsigned char *p = buf_b - MAX_PADDING;
3214 unsigned char *q = buf_a + len_a - MAX_PADDING;
3215 int i;
3216
3217 for (i = MAX_PADDING; i; i--, p++, q++) {
3218 if (*p != *q)
3219 break;
3220 }
3221
3222 return p;
3223}
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
3244 size_t len_a,
3245 unsigned char *buf_b,
3246 size_t len_b, bool *consecutive)
3247{
3248 uint64_t tsc_a, tsc_b;
3249 unsigned char *p;
3250 size_t len, rem_a, rem_b;
3251
3252 p = intel_pt_last_psb(buf_a, len_a);
3253 if (!p)
3254 return buf_b;
3255
3256 len = len_a - (p - buf_a);
3257 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
3258
3259 len_a -= len;
3260 p = intel_pt_last_psb(buf_a, len_a);
3261 if (!p)
3262 return buf_b;
3263 len = len_a - (p - buf_a);
3264 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
3265 return buf_b;
3266 }
3267
3268 while (1) {
3269
3270 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
3271 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
3272
3273
3274 if (!cmp && rem_b >= rem_a) {
3275 unsigned char *start;
3276
3277 *consecutive = true;
3278 start = buf_b + len_b - (rem_b - rem_a);
3279 return adj_for_padding(start, buf_a, len_a);
3280 }
3281 if (cmp < 0)
3282 return buf_b;
3283 }
3284
3285 if (!intel_pt_step_psb(&buf_b, &len_b))
3286 return buf_b + len_b;
3287 }
3288}
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
3308 unsigned char *buf_b, size_t len_b,
3309 bool have_tsc, bool *consecutive)
3310{
3311 unsigned char *found;
3312
3313
3314 if (!intel_pt_next_psb(&buf_b, &len_b))
3315 return buf_b + len_b;
3316
3317 if (!intel_pt_next_psb(&buf_a, &len_a))
3318 return buf_b;
3319
3320 if (have_tsc) {
3321 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
3322 consecutive);
3323 if (found)
3324 return found;
3325 }
3326
3327
3328
3329
3330
3331 while (len_b < len_a) {
3332 if (!intel_pt_step_psb(&buf_a, &len_a))
3333 return buf_b;
3334 }
3335
3336
3337 while (1) {
3338
3339 found = memmem(buf_a, len_a, buf_b, len_a);
3340 if (found) {
3341 *consecutive = true;
3342 return adj_for_padding(buf_b + len_a, buf_a, len_a);
3343 }
3344
3345
3346 if (!intel_pt_step_psb(&buf_a, &len_a))
3347 return buf_b;
3348 }
3349}
3350
3351
3352
3353
3354
3355
3356
3357struct fast_forward_data {
3358 uint64_t timestamp;
3359 uint64_t buf_timestamp;
3360};
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data)
3373{
3374 struct fast_forward_data *d = data;
3375 unsigned char *buf;
3376 uint64_t tsc;
3377 size_t rem;
3378 size_t len;
3379
3380 buf = (unsigned char *)buffer->buf;
3381 len = buffer->len;
3382
3383 if (!intel_pt_next_psb(&buf, &len) ||
3384 !intel_pt_next_tsc(buf, len, &tsc, &rem))
3385 return 0;
3386
3387 tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);
3388
3389 intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",
3390 tsc, buffer->ref_timestamp);
3391
3392
3393
3394
3395
3396 if (tsc < d->timestamp)
3397 d->buf_timestamp = buffer->ref_timestamp;
3398 else
3399 return 1;
3400
3401 return 0;
3402}
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)
3414{
3415 struct fast_forward_data d = { .timestamp = timestamp };
3416 unsigned char *buf;
3417 size_t len;
3418 int err;
3419
3420 intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);
3421
3422
3423 err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);
3424 if (err < 0)
3425 return err;
3426
3427
3428 if (d.buf_timestamp) {
3429 do {
3430 decoder->pos += decoder->len;
3431 decoder->len = 0;
3432 err = intel_pt_get_next_data(decoder, true);
3433
3434 if (err && err != -ENOLINK)
3435 return err;
3436 } while (decoder->buf_timestamp != d.buf_timestamp);
3437 }
3438
3439 if (!decoder->buf)
3440 return 0;
3441
3442 buf = (unsigned char *)decoder->buf;
3443 len = decoder->len;
3444
3445 if (!intel_pt_next_psb(&buf, &len))
3446 return 0;
3447
3448
3449
3450
3451
3452 do {
3453 uint64_t tsc;
3454 size_t rem;
3455
3456 if (!intel_pt_next_tsc(buf, len, &tsc, &rem))
3457 break;
3458 tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);
3459
3460
3461
3462
3463
3464
3465 if (tsc < timestamp) {
3466 intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);
3467 decoder->pos += decoder->len - len;
3468 decoder->buf = buf;
3469 decoder->len = len;
3470 intel_pt_reposition(decoder);
3471 } else {
3472 break;
3473 }
3474 } while (intel_pt_step_psb(&buf, &len));
3475
3476 return 0;
3477}
3478