2 * intel_pt_decoder.c: Intel Processor Trace support
3 * Copyright (c) 2013-2014, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
25 #include <linux/compiler.h>
30 #include "intel-pt-insn-decoder.h"
31 #include "intel-pt-pkt-decoder.h"
32 #include "intel-pt-decoder.h"
33 #include "intel-pt-log.h"
35 #define INTEL_PT_BLK_SIZE 1024
37 #define BIT63 (((uint64_t)1 << 63))
39 #define INTEL_PT_RETURN 1
41 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
42 #define INTEL_PT_MAX_LOOPS 10000
45 struct intel_pt_blk *prev;
46 uint64_t ip[INTEL_PT_BLK_SIZE];
49 struct intel_pt_stack {
50 struct intel_pt_blk *blk;
51 struct intel_pt_blk *spare;
55 enum intel_pt_pkt_state {
56 INTEL_PT_STATE_NO_PSB,
58 INTEL_PT_STATE_ERR_RESYNC,
59 INTEL_PT_STATE_IN_SYNC,
62 INTEL_PT_STATE_TIP_PGD,
64 INTEL_PT_STATE_FUP_NO_TIP,
67 #ifdef INTEL_PT_STRICT
68 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
69 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
70 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
71 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
73 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
74 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
75 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
76 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
79 struct intel_pt_decoder {
80 int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
81 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
82 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
83 uint64_t max_insn_cnt, void *data);
84 bool (*pgd_ip)(uint64_t ip, void *data);
86 struct intel_pt_state state;
87 const unsigned char *buf;
89 bool return_compression;
100 uint64_t tsc_timestamp;
101 uint64_t ref_timestamp;
103 uint64_t ctc_timestamp;
106 uint64_t cyc_ref_timestamp;
108 uint32_t tsc_ctc_ratio_n;
109 uint32_t tsc_ctc_ratio_d;
110 uint32_t tsc_ctc_mult;
112 uint32_t ctc_rem_mask;
114 struct intel_pt_stack stack;
115 enum intel_pt_pkt_state pkt_state;
116 struct intel_pt_pkt packet;
117 struct intel_pt_pkt tnt;
120 int last_packet_type;
122 unsigned int max_non_turbo_ratio;
123 double max_non_turbo_ratio_fp;
124 double cbr_cyc_to_tsc;
125 double calc_cyc_to_tsc;
126 bool have_calc_cyc_to_tsc;
128 unsigned int insn_bytes;
130 enum intel_pt_period_type period_type;
131 uint64_t tot_insn_cnt;
132 uint64_t period_insn_cnt;
133 uint64_t period_mask;
134 uint64_t period_ticks;
135 uint64_t last_masked_timestamp;
136 bool continuous_period;
138 bool set_fup_tx_flags;
139 unsigned int fup_tx_flags;
140 unsigned int tx_flags;
141 uint64_t timestamp_insn_cnt;
146 const unsigned char *next_buf;
148 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
151 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
155 for (i = 0; x != 1; i++)
161 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
163 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
166 period = intel_pt_lower_power_of_2(decoder->period);
167 decoder->period_mask = ~(period - 1);
168 decoder->period_ticks = period;
172 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
176 return (t / d) * n + ((t % d) * n) / d;
179 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
181 struct intel_pt_decoder *decoder;
183 if (!params->get_trace || !params->walk_insn)
186 decoder = zalloc(sizeof(struct intel_pt_decoder));
190 decoder->get_trace = params->get_trace;
191 decoder->walk_insn = params->walk_insn;
192 decoder->pgd_ip = params->pgd_ip;
193 decoder->data = params->data;
194 decoder->return_compression = params->return_compression;
196 decoder->period = params->period;
197 decoder->period_type = params->period_type;
199 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
200 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
202 intel_pt_setup_period(decoder);
204 decoder->mtc_shift = params->mtc_period;
205 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
207 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
208 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
210 if (!decoder->tsc_ctc_ratio_n)
211 decoder->tsc_ctc_ratio_d = 0;
213 if (decoder->tsc_ctc_ratio_d) {
214 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
215 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
216 decoder->tsc_ctc_ratio_d;
219 * Allow for timestamps appearing to backwards because a TSC
220 * packet has slipped past a MTC packet, so allow 2 MTC ticks
223 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
224 decoder->tsc_ctc_ratio_n,
225 decoder->tsc_ctc_ratio_d);
227 /* ... or 0x100 paranoia */
228 if (decoder->tsc_slip < 0x100)
229 decoder->tsc_slip = 0x100;
231 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
232 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
233 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
234 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
235 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
240 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
242 struct intel_pt_blk *blk = stack->blk;
244 stack->blk = blk->prev;
251 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
256 intel_pt_pop_blk(stack);
259 stack->pos = INTEL_PT_BLK_SIZE;
261 return stack->blk->ip[--stack->pos];
264 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
266 struct intel_pt_blk *blk;
272 blk = malloc(sizeof(struct intel_pt_blk));
277 blk->prev = stack->blk;
283 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
287 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
288 err = intel_pt_alloc_blk(stack);
293 stack->blk->ip[stack->pos++] = ip;
297 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
300 intel_pt_pop_blk(stack);
304 static void intel_pt_free_stack(struct intel_pt_stack *stack)
306 intel_pt_clear_stack(stack);
308 zfree(&stack->spare);
311 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
313 intel_pt_free_stack(&decoder->stack);
317 static int intel_pt_ext_err(int code)
321 return INTEL_PT_ERR_NOMEM;
323 return INTEL_PT_ERR_INTERN;
325 return INTEL_PT_ERR_BADPKT;
327 return INTEL_PT_ERR_NODATA;
329 return INTEL_PT_ERR_NOINSN;
331 return INTEL_PT_ERR_MISMAT;
333 return INTEL_PT_ERR_OVR;
335 return INTEL_PT_ERR_LOST;
337 return INTEL_PT_ERR_NELOOP;
339 return INTEL_PT_ERR_UNK;
343 static const char *intel_pt_err_msgs[] = {
344 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
345 [INTEL_PT_ERR_INTERN] = "Internal error",
346 [INTEL_PT_ERR_BADPKT] = "Bad packet",
347 [INTEL_PT_ERR_NODATA] = "No more data",
348 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
349 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
350 [INTEL_PT_ERR_OVR] = "Overflow packet",
351 [INTEL_PT_ERR_LOST] = "Lost trace data",
352 [INTEL_PT_ERR_UNK] = "Unknown error!",
353 [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
356 int intel_pt__strerror(int code, char *buf, size_t buflen)
358 if (code < 1 || code >= INTEL_PT_ERR_MAX)
359 code = INTEL_PT_ERR_UNK;
360 strlcpy(buf, intel_pt_err_msgs[code], buflen);
364 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
369 switch (packet->count) {
371 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
375 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
379 ip = packet->payload;
380 /* Sign-extend 6-byte ip */
381 if (ip & (uint64_t)0x800000000000ULL)
382 ip |= (uint64_t)0xffff000000000000ULL;
385 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
389 ip = packet->payload;
398 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
400 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
403 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
405 intel_pt_set_last_ip(decoder);
406 decoder->ip = decoder->last_ip;
409 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
411 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
415 static int intel_pt_bug(struct intel_pt_decoder *decoder)
417 intel_pt_log("ERROR: Internal error\n");
418 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
422 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
424 decoder->tx_flags = 0;
427 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
429 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
432 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
434 intel_pt_clear_tx_flags(decoder);
435 decoder->have_tma = false;
436 decoder->pkt_len = 1;
437 decoder->pkt_step = 1;
438 intel_pt_decoder_log_packet(decoder);
439 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
440 intel_pt_log("ERROR: Bad packet\n");
441 decoder->pkt_state = INTEL_PT_STATE_ERR1;
446 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
448 struct intel_pt_buffer buffer = { .buf = 0, };
451 decoder->pkt_step = 0;
453 intel_pt_log("Getting more data\n");
454 ret = decoder->get_trace(&buffer, decoder->data);
457 decoder->buf = buffer.buf;
458 decoder->len = buffer.len;
460 intel_pt_log("No more data\n");
463 if (!buffer.consecutive) {
465 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
466 decoder->ref_timestamp = buffer.ref_timestamp;
467 decoder->timestamp = 0;
468 decoder->have_tma = false;
469 decoder->state.trace_nr = buffer.trace_nr;
470 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
471 decoder->ref_timestamp);
478 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
480 if (!decoder->next_buf)
481 return intel_pt_get_data(decoder);
483 decoder->buf = decoder->next_buf;
484 decoder->len = decoder->next_len;
485 decoder->next_buf = 0;
486 decoder->next_len = 0;
490 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
492 unsigned char *buf = decoder->temp_buf;
493 size_t old_len, len, n;
496 old_len = decoder->len;
498 memcpy(buf, decoder->buf, len);
500 ret = intel_pt_get_data(decoder);
502 decoder->pos += old_len;
503 return ret < 0 ? ret : -EINVAL;
506 n = INTEL_PT_PKT_MAX_SZ - len;
507 if (n > decoder->len)
509 memcpy(buf + len, decoder->buf, n);
512 ret = intel_pt_get_packet(buf, len, &decoder->packet);
513 if (ret < (int)old_len) {
514 decoder->next_buf = decoder->buf;
515 decoder->next_len = decoder->len;
517 decoder->len = old_len;
518 return intel_pt_bad_packet(decoder);
521 decoder->next_buf = decoder->buf + (ret - old_len);
522 decoder->next_len = decoder->len - (ret - old_len);
530 struct intel_pt_pkt_info {
531 struct intel_pt_decoder *decoder;
532 struct intel_pt_pkt packet;
535 int last_packet_type;
539 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
541 /* Lookahead packets in current buffer */
542 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
543 intel_pt_pkt_cb_t cb, void *data)
545 struct intel_pt_pkt_info pkt_info;
546 const unsigned char *buf = decoder->buf;
547 size_t len = decoder->len;
550 pkt_info.decoder = decoder;
551 pkt_info.pos = decoder->pos;
552 pkt_info.pkt_len = decoder->pkt_step;
553 pkt_info.last_packet_type = decoder->last_packet_type;
554 pkt_info.data = data;
558 pkt_info.pos += pkt_info.pkt_len;
559 buf += pkt_info.pkt_len;
560 len -= pkt_info.pkt_len;
563 return INTEL_PT_NEED_MORE_BYTES;
565 ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
567 return INTEL_PT_NEED_MORE_BYTES;
571 pkt_info.pkt_len = ret;
572 } while (pkt_info.packet.type == INTEL_PT_PAD);
578 pkt_info.last_packet_type = pkt_info.packet.type;
582 struct intel_pt_calc_cyc_to_tsc_info {
586 uint64_t ctc_timestamp;
588 uint64_t tsc_timestamp;
593 double cbr_cyc_to_tsc;
597 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
598 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
599 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
600 * packet by copying the missing bits from the current MTC assuming the least
601 * difference between the two, and that the current MTC comes after last_mtc.
603 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
606 uint32_t first_missing_bit = 1U << (16 - mtc_shift);
607 uint32_t mask = ~(first_missing_bit - 1);
609 *last_mtc |= mtc & mask;
610 if (*last_mtc >= mtc) {
611 *last_mtc -= first_missing_bit;
616 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
618 struct intel_pt_decoder *decoder = pkt_info->decoder;
619 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
623 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
625 switch (pkt_info->packet.type) {
627 case INTEL_PT_TIP_PGE:
632 case INTEL_PT_MODE_EXEC:
633 case INTEL_PT_MODE_TSX:
634 case INTEL_PT_PSBEND:
644 mtc = pkt_info->packet.payload;
645 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
646 data->fixup_last_mtc = false;
647 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
650 if (mtc > data->last_mtc)
651 mtc_delta = mtc - data->last_mtc;
653 mtc_delta = mtc + 256 - data->last_mtc;
654 data->ctc_delta += mtc_delta << decoder->mtc_shift;
655 data->last_mtc = mtc;
657 if (decoder->tsc_ctc_mult) {
658 timestamp = data->ctc_timestamp +
659 data->ctc_delta * decoder->tsc_ctc_mult;
661 timestamp = data->ctc_timestamp +
662 multdiv(data->ctc_delta,
663 decoder->tsc_ctc_ratio_n,
664 decoder->tsc_ctc_ratio_d);
667 if (timestamp < data->timestamp)
670 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
671 data->timestamp = timestamp;
678 timestamp = pkt_info->packet.payload |
679 (data->timestamp & (0xffULL << 56));
680 if (data->from_mtc && timestamp < data->timestamp &&
681 data->timestamp - timestamp < decoder->tsc_slip)
683 if (timestamp < data->timestamp)
684 timestamp += (1ULL << 56);
685 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
688 data->tsc_timestamp = timestamp;
689 data->timestamp = timestamp;
698 if (!decoder->tsc_ctc_ratio_d)
701 ctc = pkt_info->packet.payload;
702 fc = pkt_info->packet.count;
703 ctc_rem = ctc & decoder->ctc_rem_mask;
705 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
707 data->ctc_timestamp = data->tsc_timestamp - fc;
708 if (decoder->tsc_ctc_mult) {
709 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
711 data->ctc_timestamp -=
712 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
713 decoder->tsc_ctc_ratio_d);
717 data->have_tma = true;
718 data->fixup_last_mtc = true;
723 data->cycle_cnt += pkt_info->packet.payload;
727 cbr = pkt_info->packet.payload;
728 if (data->cbr && data->cbr != cbr)
731 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
734 case INTEL_PT_TIP_PGD:
735 case INTEL_PT_TRACESTOP:
737 case INTEL_PT_BAD: /* Does not happen */
742 if (!data->cbr && decoder->cbr) {
743 data->cbr = decoder->cbr;
744 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
747 if (!data->cycle_cnt)
750 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
752 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
753 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
754 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
755 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
759 decoder->calc_cyc_to_tsc = cyc_to_tsc;
760 decoder->have_calc_cyc_to_tsc = true;
763 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
764 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
766 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
767 cyc_to_tsc, pkt_info->pos);
773 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
776 struct intel_pt_calc_cyc_to_tsc_info data = {
779 .last_mtc = decoder->last_mtc,
780 .ctc_timestamp = decoder->ctc_timestamp,
781 .ctc_delta = decoder->ctc_delta,
782 .tsc_timestamp = decoder->tsc_timestamp,
783 .timestamp = decoder->timestamp,
784 .have_tma = decoder->have_tma,
785 .fixup_last_mtc = decoder->fixup_last_mtc,
786 .from_mtc = from_mtc,
790 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
793 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
797 decoder->last_packet_type = decoder->packet.type;
800 decoder->pos += decoder->pkt_step;
801 decoder->buf += decoder->pkt_step;
802 decoder->len -= decoder->pkt_step;
805 ret = intel_pt_get_next_data(decoder);
810 ret = intel_pt_get_packet(decoder->buf, decoder->len,
812 if (ret == INTEL_PT_NEED_MORE_BYTES &&
813 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
814 ret = intel_pt_get_split_packet(decoder);
819 return intel_pt_bad_packet(decoder);
821 decoder->pkt_len = ret;
822 decoder->pkt_step = ret;
823 intel_pt_decoder_log_packet(decoder);
824 } while (decoder->packet.type == INTEL_PT_PAD);
829 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
831 uint64_t timestamp, masked_timestamp;
833 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
834 masked_timestamp = timestamp & decoder->period_mask;
835 if (decoder->continuous_period) {
836 if (masked_timestamp != decoder->last_masked_timestamp)
840 masked_timestamp = timestamp & decoder->period_mask;
841 if (masked_timestamp != decoder->last_masked_timestamp) {
842 decoder->last_masked_timestamp = masked_timestamp;
843 decoder->continuous_period = true;
846 return decoder->period_ticks - (timestamp - masked_timestamp);
849 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
851 switch (decoder->period_type) {
852 case INTEL_PT_PERIOD_INSTRUCTIONS:
853 return decoder->period - decoder->period_insn_cnt;
854 case INTEL_PT_PERIOD_TICKS:
855 return intel_pt_next_period(decoder);
856 case INTEL_PT_PERIOD_NONE:
857 case INTEL_PT_PERIOD_MTC:
863 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
865 uint64_t timestamp, masked_timestamp;
867 switch (decoder->period_type) {
868 case INTEL_PT_PERIOD_INSTRUCTIONS:
869 decoder->period_insn_cnt = 0;
871 case INTEL_PT_PERIOD_TICKS:
872 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
873 masked_timestamp = timestamp & decoder->period_mask;
874 decoder->last_masked_timestamp = masked_timestamp;
876 case INTEL_PT_PERIOD_NONE:
877 case INTEL_PT_PERIOD_MTC:
882 decoder->state.type |= INTEL_PT_INSTRUCTION;
885 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
886 struct intel_pt_insn *intel_pt_insn, uint64_t ip)
888 uint64_t max_insn_cnt, insn_cnt = 0;
891 if (!decoder->mtc_insn)
892 decoder->mtc_insn = true;
894 max_insn_cnt = intel_pt_next_sample(decoder);
896 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
897 max_insn_cnt, decoder->data);
899 decoder->tot_insn_cnt += insn_cnt;
900 decoder->timestamp_insn_cnt += insn_cnt;
901 decoder->period_insn_cnt += insn_cnt;
904 decoder->no_progress = 0;
905 decoder->pkt_state = INTEL_PT_STATE_ERR2;
906 intel_pt_log_at("ERROR: Failed to get instruction",
913 if (ip && decoder->ip == ip) {
918 if (max_insn_cnt && insn_cnt >= max_insn_cnt)
919 intel_pt_sample_insn(decoder);
921 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
922 decoder->state.type = INTEL_PT_INSTRUCTION;
923 decoder->state.from_ip = decoder->ip;
924 decoder->state.to_ip = 0;
925 decoder->ip += intel_pt_insn->length;
926 err = INTEL_PT_RETURN;
930 if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
931 /* Zero-length calls are excluded */
932 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
933 intel_pt_insn->rel) {
934 err = intel_pt_push(&decoder->stack, decoder->ip +
935 intel_pt_insn->length);
939 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
940 decoder->ret_addr = intel_pt_pop(&decoder->stack);
943 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
944 int cnt = decoder->no_progress++;
946 decoder->state.from_ip = decoder->ip;
947 decoder->ip += intel_pt_insn->length +
949 decoder->state.to_ip = decoder->ip;
950 err = INTEL_PT_RETURN;
953 * Check for being stuck in a loop. This can happen if a
954 * decoder error results in the decoder erroneously setting the
955 * ip to an address that is itself in an infinite loop that
956 * consumes no packets. When that happens, there must be an
957 * unconditional branch.
961 decoder->stuck_ip = decoder->state.to_ip;
962 decoder->stuck_ip_prd = 1;
963 decoder->stuck_ip_cnt = 1;
964 } else if (cnt > INTEL_PT_MAX_LOOPS ||
965 decoder->state.to_ip == decoder->stuck_ip) {
966 intel_pt_log_at("ERROR: Never-ending loop",
967 decoder->state.to_ip);
968 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
971 } else if (!--decoder->stuck_ip_cnt) {
972 decoder->stuck_ip_prd += 1;
973 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
974 decoder->stuck_ip = decoder->state.to_ip;
977 goto out_no_progress;
980 decoder->no_progress = 0;
982 decoder->state.insn_op = intel_pt_insn->op;
983 decoder->state.insn_len = intel_pt_insn->length;
984 memcpy(decoder->state.insn, intel_pt_insn->buf,
985 INTEL_PT_INSN_BUF_SZ);
987 if (decoder->tx_flags & INTEL_PT_IN_TX)
988 decoder->state.flags |= INTEL_PT_IN_TX;
993 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
995 struct intel_pt_insn intel_pt_insn;
999 ip = decoder->last_ip;
1002 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1003 if (err == INTEL_PT_RETURN)
1005 if (err == -EAGAIN) {
1006 if (decoder->set_fup_tx_flags) {
1007 decoder->set_fup_tx_flags = false;
1008 decoder->tx_flags = decoder->fup_tx_flags;
1009 decoder->state.type = INTEL_PT_TRANSACTION;
1010 decoder->state.from_ip = decoder->ip;
1011 decoder->state.to_ip = 0;
1012 decoder->state.flags = decoder->fup_tx_flags;
1017 decoder->set_fup_tx_flags = false;
1021 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1022 intel_pt_log_at("ERROR: Unexpected indirect branch",
1024 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1028 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1029 intel_pt_log_at("ERROR: Unexpected conditional branch",
1031 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1035 intel_pt_bug(decoder);
1039 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1041 struct intel_pt_insn intel_pt_insn;
1044 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1045 if (err == INTEL_PT_RETURN &&
1047 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1048 (decoder->state.type & INTEL_PT_BRANCH) &&
1049 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1050 /* Unconditional branch leaving filter region */
1051 decoder->no_progress = 0;
1052 decoder->pge = false;
1053 decoder->continuous_period = false;
1054 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1055 decoder->state.to_ip = 0;
1058 if (err == INTEL_PT_RETURN)
1063 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1064 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1065 decoder->pge = false;
1066 decoder->continuous_period = false;
1067 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1068 decoder->state.from_ip = decoder->ip;
1069 decoder->state.to_ip = 0;
1070 if (decoder->packet.count != 0)
1071 decoder->ip = decoder->last_ip;
1073 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1074 decoder->state.from_ip = decoder->ip;
1075 if (decoder->packet.count == 0) {
1076 decoder->state.to_ip = 0;
1078 decoder->state.to_ip = decoder->last_ip;
1079 decoder->ip = decoder->last_ip;
1085 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1086 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1089 if (decoder->pgd_ip &&
1090 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1091 decoder->pgd_ip(to_ip, decoder->data)) {
1092 /* Conditional branch leaving filter region */
1093 decoder->pge = false;
1094 decoder->continuous_period = false;
1095 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1096 decoder->ip = to_ip;
1097 decoder->state.from_ip = decoder->ip;
1098 decoder->state.to_ip = 0;
1101 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1103 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1107 return intel_pt_bug(decoder);
1110 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1112 struct intel_pt_insn intel_pt_insn;
1116 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1117 if (err == INTEL_PT_RETURN)
1122 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1123 if (!decoder->return_compression) {
1124 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1126 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1129 if (!decoder->ret_addr) {
1130 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1132 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1135 if (!(decoder->tnt.payload & BIT63)) {
1136 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1138 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1141 decoder->tnt.count -= 1;
1142 if (!decoder->tnt.count)
1143 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1144 decoder->tnt.payload <<= 1;
1145 decoder->state.from_ip = decoder->ip;
1146 decoder->ip = decoder->ret_addr;
1147 decoder->state.to_ip = decoder->ip;
1151 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1152 /* Handle deferred TIPs */
1153 err = intel_pt_get_next_packet(decoder);
1156 if (decoder->packet.type != INTEL_PT_TIP ||
1157 decoder->packet.count == 0) {
1158 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1160 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1161 decoder->pkt_step = 0;
1164 intel_pt_set_last_ip(decoder);
1165 decoder->state.from_ip = decoder->ip;
1166 decoder->state.to_ip = decoder->last_ip;
1167 decoder->ip = decoder->last_ip;
1171 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1172 decoder->tnt.count -= 1;
1173 if (!decoder->tnt.count)
1174 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1175 if (decoder->tnt.payload & BIT63) {
1176 decoder->tnt.payload <<= 1;
1177 decoder->state.from_ip = decoder->ip;
1178 decoder->ip += intel_pt_insn.length +
1180 decoder->state.to_ip = decoder->ip;
1183 /* Instruction sample for a non-taken branch */
1184 if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1185 decoder->tnt.payload <<= 1;
1186 decoder->state.type = INTEL_PT_INSTRUCTION;
1187 decoder->state.from_ip = decoder->ip;
1188 decoder->state.to_ip = 0;
1189 decoder->ip += intel_pt_insn.length;
1192 decoder->ip += intel_pt_insn.length;
1193 if (!decoder->tnt.count)
1195 decoder->tnt.payload <<= 1;
1199 return intel_pt_bug(decoder);
1203 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1205 unsigned int fup_tx_flags;
1208 fup_tx_flags = decoder->packet.payload &
1209 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1210 err = intel_pt_get_next_packet(decoder);
1213 if (decoder->packet.type == INTEL_PT_FUP) {
1214 decoder->fup_tx_flags = fup_tx_flags;
1215 decoder->set_fup_tx_flags = true;
1216 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1219 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1221 intel_pt_update_in_tx(decoder);
1226 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1230 decoder->have_tma = false;
1232 if (decoder->ref_timestamp) {
1233 timestamp = decoder->packet.payload |
1234 (decoder->ref_timestamp & (0xffULL << 56));
1235 if (timestamp < decoder->ref_timestamp) {
1236 if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1237 timestamp += (1ULL << 56);
1239 if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1240 timestamp -= (1ULL << 56);
1242 decoder->tsc_timestamp = timestamp;
1243 decoder->timestamp = timestamp;
1244 decoder->ref_timestamp = 0;
1245 decoder->timestamp_insn_cnt = 0;
1246 } else if (decoder->timestamp) {
1247 timestamp = decoder->packet.payload |
1248 (decoder->timestamp & (0xffULL << 56));
1249 decoder->tsc_timestamp = timestamp;
1250 if (timestamp < decoder->timestamp &&
1251 decoder->timestamp - timestamp < decoder->tsc_slip) {
1252 intel_pt_log_to("Suppressing backwards timestamp",
1254 timestamp = decoder->timestamp;
1256 if (timestamp < decoder->timestamp) {
1257 intel_pt_log_to("Wraparound timestamp", timestamp);
1258 timestamp += (1ULL << 56);
1259 decoder->tsc_timestamp = timestamp;
1261 decoder->timestamp = timestamp;
1262 decoder->timestamp_insn_cnt = 0;
1265 if (decoder->last_packet_type == INTEL_PT_CYC) {
1266 decoder->cyc_ref_timestamp = decoder->timestamp;
1267 decoder->cycle_cnt = 0;
1268 decoder->have_calc_cyc_to_tsc = false;
1269 intel_pt_calc_cyc_to_tsc(decoder, false);
1272 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1275 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1277 intel_pt_log("ERROR: Buffer overflow\n");
1278 intel_pt_clear_tx_flags(decoder);
1279 decoder->have_tma = false;
1281 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1282 decoder->overflow = true;
1286 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1288 uint32_t ctc = decoder->packet.payload;
1289 uint32_t fc = decoder->packet.count;
1290 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1292 if (!decoder->tsc_ctc_ratio_d)
1295 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1296 decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1297 if (decoder->tsc_ctc_mult) {
1298 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1300 decoder->ctc_timestamp -= multdiv(ctc_rem,
1301 decoder->tsc_ctc_ratio_n,
1302 decoder->tsc_ctc_ratio_d);
1304 decoder->ctc_delta = 0;
1305 decoder->have_tma = true;
1306 decoder->fixup_last_mtc = true;
1307 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n",
1308 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1311 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1314 uint32_t mtc, mtc_delta;
1316 if (!decoder->have_tma)
1319 mtc = decoder->packet.payload;
1321 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1322 decoder->fixup_last_mtc = false;
1323 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1324 &decoder->last_mtc);
1327 if (mtc > decoder->last_mtc)
1328 mtc_delta = mtc - decoder->last_mtc;
1330 mtc_delta = mtc + 256 - decoder->last_mtc;
1332 decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1334 if (decoder->tsc_ctc_mult) {
1335 timestamp = decoder->ctc_timestamp +
1336 decoder->ctc_delta * decoder->tsc_ctc_mult;
1338 timestamp = decoder->ctc_timestamp +
1339 multdiv(decoder->ctc_delta,
1340 decoder->tsc_ctc_ratio_n,
1341 decoder->tsc_ctc_ratio_d);
1344 if (timestamp < decoder->timestamp)
1345 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1346 timestamp, decoder->timestamp);
1348 decoder->timestamp = timestamp;
1350 decoder->timestamp_insn_cnt = 0;
1351 decoder->last_mtc = mtc;
1353 if (decoder->last_packet_type == INTEL_PT_CYC) {
1354 decoder->cyc_ref_timestamp = decoder->timestamp;
1355 decoder->cycle_cnt = 0;
1356 decoder->have_calc_cyc_to_tsc = false;
1357 intel_pt_calc_cyc_to_tsc(decoder, true);
1361 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1363 unsigned int cbr = decoder->packet.payload;
1365 if (decoder->cbr == cbr)
1369 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1372 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1374 uint64_t timestamp = decoder->cyc_ref_timestamp;
1376 decoder->have_cyc = true;
1378 decoder->cycle_cnt += decoder->packet.payload;
1380 if (!decoder->cyc_ref_timestamp)
1383 if (decoder->have_calc_cyc_to_tsc)
1384 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1385 else if (decoder->cbr)
1386 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1390 if (timestamp < decoder->timestamp)
1391 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1392 timestamp, decoder->timestamp);
1394 decoder->timestamp = timestamp;
1396 decoder->timestamp_insn_cnt = 0;
1399 /* Walk PSB+ packets when already in sync. */
1400 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1405 err = intel_pt_get_next_packet(decoder);
1409 switch (decoder->packet.type) {
1410 case INTEL_PT_PSBEND:
1413 case INTEL_PT_TIP_PGD:
1414 case INTEL_PT_TIP_PGE:
1417 case INTEL_PT_TRACESTOP:
1420 decoder->have_tma = false;
1421 intel_pt_log("ERROR: Unexpected packet\n");
1425 return intel_pt_overflow(decoder);
1428 intel_pt_calc_tsc_timestamp(decoder);
1432 intel_pt_calc_tma(decoder);
1436 intel_pt_calc_cbr(decoder);
1439 case INTEL_PT_MODE_EXEC:
1440 decoder->exec_mode = decoder->packet.payload;
1444 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1448 decoder->pge = true;
1449 intel_pt_set_last_ip(decoder);
1452 case INTEL_PT_MODE_TSX:
1453 intel_pt_update_in_tx(decoder);
1457 intel_pt_calc_mtc_timestamp(decoder);
1458 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1459 decoder->state.type |= INTEL_PT_INSTRUCTION;
1472 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1476 if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1477 decoder->tx_flags = 0;
1478 decoder->state.flags &= ~INTEL_PT_IN_TX;
1479 decoder->state.flags |= INTEL_PT_ABORT_TX;
1481 decoder->state.flags |= INTEL_PT_ASYNC;
1485 err = intel_pt_get_next_packet(decoder);
1489 switch (decoder->packet.type) {
1492 case INTEL_PT_TRACESTOP:
1497 case INTEL_PT_MODE_TSX:
1499 case INTEL_PT_PSBEND:
1500 intel_pt_log("ERROR: Missing TIP after FUP\n");
1501 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1505 return intel_pt_overflow(decoder);
1507 case INTEL_PT_TIP_PGD:
1508 decoder->state.from_ip = decoder->ip;
1509 decoder->state.to_ip = 0;
1510 if (decoder->packet.count != 0) {
1511 intel_pt_set_ip(decoder);
1512 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1515 decoder->pge = false;
1516 decoder->continuous_period = false;
1519 case INTEL_PT_TIP_PGE:
1520 decoder->pge = true;
1521 intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1523 decoder->state.from_ip = 0;
1524 if (decoder->packet.count == 0) {
1525 decoder->state.to_ip = 0;
1527 intel_pt_set_ip(decoder);
1528 decoder->state.to_ip = decoder->ip;
1533 decoder->state.from_ip = decoder->ip;
1534 if (decoder->packet.count == 0) {
1535 decoder->state.to_ip = 0;
1537 intel_pt_set_ip(decoder);
1538 decoder->state.to_ip = decoder->ip;
1543 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1547 intel_pt_calc_mtc_timestamp(decoder);
1548 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1549 decoder->state.type |= INTEL_PT_INSTRUCTION;
1553 intel_pt_calc_cyc_timestamp(decoder);
1556 case INTEL_PT_MODE_EXEC:
1557 decoder->exec_mode = decoder->packet.payload;
1566 return intel_pt_bug(decoder);
1571 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1573 bool no_tip = false;
1577 err = intel_pt_get_next_packet(decoder);
1581 switch (decoder->packet.type) {
1583 if (!decoder->packet.count)
1585 decoder->tnt = decoder->packet;
1586 decoder->pkt_state = INTEL_PT_STATE_TNT;
1587 err = intel_pt_walk_tnt(decoder);
1592 case INTEL_PT_TIP_PGD:
1593 if (decoder->packet.count != 0)
1594 intel_pt_set_last_ip(decoder);
1595 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1596 return intel_pt_walk_tip(decoder);
1598 case INTEL_PT_TIP_PGE: {
1599 decoder->pge = true;
1600 if (decoder->packet.count == 0) {
1601 intel_pt_log_at("Skipping zero TIP.PGE",
1605 intel_pt_set_ip(decoder);
1606 decoder->state.from_ip = 0;
1607 decoder->state.to_ip = decoder->ip;
1612 return intel_pt_overflow(decoder);
1615 if (decoder->packet.count != 0)
1616 intel_pt_set_last_ip(decoder);
1617 decoder->pkt_state = INTEL_PT_STATE_TIP;
1618 return intel_pt_walk_tip(decoder);
1621 if (decoder->packet.count == 0) {
1622 intel_pt_log_at("Skipping zero FUP",
1627 intel_pt_set_last_ip(decoder);
1628 err = intel_pt_walk_fup(decoder);
1629 if (err != -EAGAIN) {
1633 decoder->pkt_state =
1634 INTEL_PT_STATE_FUP_NO_TIP;
1636 decoder->pkt_state = INTEL_PT_STATE_FUP;
1643 return intel_pt_walk_fup_tip(decoder);
1645 case INTEL_PT_TRACESTOP:
1646 decoder->pge = false;
1647 decoder->continuous_period = false;
1648 intel_pt_clear_tx_flags(decoder);
1649 decoder->have_tma = false;
1653 intel_pt_clear_stack(&decoder->stack);
1654 err = intel_pt_walk_psbend(decoder);
1662 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1666 intel_pt_calc_mtc_timestamp(decoder);
1667 if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1670 * Ensure that there has been an instruction since the
1673 if (!decoder->mtc_insn)
1675 decoder->mtc_insn = false;
1676 /* Ensure that there is a timestamp */
1677 if (!decoder->timestamp)
1679 decoder->state.type = INTEL_PT_INSTRUCTION;
1680 decoder->state.from_ip = decoder->ip;
1681 decoder->state.to_ip = 0;
1682 decoder->mtc_insn = false;
1686 intel_pt_calc_tsc_timestamp(decoder);
1690 intel_pt_calc_tma(decoder);
1694 intel_pt_calc_cyc_timestamp(decoder);
1698 intel_pt_calc_cbr(decoder);
1701 case INTEL_PT_MODE_EXEC:
1702 decoder->exec_mode = decoder->packet.payload;
1705 case INTEL_PT_MODE_TSX:
1706 /* MODE_TSX need not be followed by FUP */
1707 if (!decoder->pge) {
1708 intel_pt_update_in_tx(decoder);
1711 err = intel_pt_mode_tsx(decoder, &no_tip);
1716 case INTEL_PT_BAD: /* Does not happen */
1717 return intel_pt_bug(decoder);
1719 case INTEL_PT_PSBEND:
1726 return intel_pt_bug(decoder);
1731 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1733 return decoder->last_ip || decoder->packet.count == 0 ||
1734 decoder->packet.count == 3 || decoder->packet.count == 6;
1737 /* Walk PSB+ packets to get in sync. */
1738 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1743 err = intel_pt_get_next_packet(decoder);
1747 switch (decoder->packet.type) {
1748 case INTEL_PT_TIP_PGD:
1749 decoder->continuous_period = false;
1751 case INTEL_PT_TIP_PGE:
1753 intel_pt_log("ERROR: Unexpected packet\n");
1757 decoder->pge = true;
1758 if (intel_pt_have_ip(decoder)) {
1759 uint64_t current_ip = decoder->ip;
1761 intel_pt_set_ip(decoder);
1763 intel_pt_log_to("Setting IP",
1769 intel_pt_calc_mtc_timestamp(decoder);
1773 intel_pt_calc_tsc_timestamp(decoder);
1777 intel_pt_calc_tma(decoder);
1781 intel_pt_calc_cyc_timestamp(decoder);
1785 intel_pt_calc_cbr(decoder);
1789 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1792 case INTEL_PT_MODE_EXEC:
1793 decoder->exec_mode = decoder->packet.payload;
1796 case INTEL_PT_MODE_TSX:
1797 intel_pt_update_in_tx(decoder);
1800 case INTEL_PT_TRACESTOP:
1801 decoder->pge = false;
1802 decoder->continuous_period = false;
1803 intel_pt_clear_tx_flags(decoder);
1807 decoder->have_tma = false;
1808 intel_pt_log("ERROR: Unexpected packet\n");
1810 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1812 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1815 case INTEL_PT_BAD: /* Does not happen */
1816 return intel_pt_bug(decoder);
1819 return intel_pt_overflow(decoder);
1821 case INTEL_PT_PSBEND:
1834 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1839 err = intel_pt_get_next_packet(decoder);
1843 switch (decoder->packet.type) {
1844 case INTEL_PT_TIP_PGD:
1845 decoder->continuous_period = false;
1847 case INTEL_PT_TIP_PGE:
1849 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1850 if (intel_pt_have_ip(decoder))
1851 intel_pt_set_ip(decoder);
1857 if (decoder->overflow) {
1858 if (intel_pt_have_ip(decoder))
1859 intel_pt_set_ip(decoder);
1863 if (decoder->packet.count)
1864 intel_pt_set_last_ip(decoder);
1868 intel_pt_calc_mtc_timestamp(decoder);
1872 intel_pt_calc_tsc_timestamp(decoder);
1876 intel_pt_calc_tma(decoder);
1880 intel_pt_calc_cyc_timestamp(decoder);
1884 intel_pt_calc_cbr(decoder);
1888 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1891 case INTEL_PT_MODE_EXEC:
1892 decoder->exec_mode = decoder->packet.payload;
1895 case INTEL_PT_MODE_TSX:
1896 intel_pt_update_in_tx(decoder);
1900 return intel_pt_overflow(decoder);
1902 case INTEL_PT_BAD: /* Does not happen */
1903 return intel_pt_bug(decoder);
1905 case INTEL_PT_TRACESTOP:
1906 decoder->pge = false;
1907 decoder->continuous_period = false;
1908 intel_pt_clear_tx_flags(decoder);
1909 decoder->have_tma = false;
1913 err = intel_pt_walk_psb(decoder);
1917 /* Do not have a sample */
1918 decoder->state.type = 0;
1924 case INTEL_PT_PSBEND:
1934 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1938 intel_pt_log("Scanning for full IP\n");
1939 err = intel_pt_walk_to_ip(decoder);
1943 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1944 decoder->overflow = false;
1946 decoder->state.from_ip = 0;
1947 decoder->state.to_ip = decoder->ip;
1948 intel_pt_log_to("Setting IP", decoder->ip);
1953 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1955 const unsigned char *end = decoder->buf + decoder->len;
1958 for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1959 if (i > decoder->len)
1961 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1967 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1969 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1970 const char *psb = INTEL_PT_PSB_STR;
1972 if (rest_psb > decoder->len ||
1973 memcmp(decoder->buf, psb + part_psb, rest_psb))
1979 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
1984 decoder->pos += decoder->len;
1987 ret = intel_pt_get_next_data(decoder);
1991 rest_psb = intel_pt_rest_psb(decoder, part_psb);
1995 decoder->pos -= part_psb;
1996 decoder->next_buf = decoder->buf + rest_psb;
1997 decoder->next_len = decoder->len - rest_psb;
1998 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
1999 decoder->buf = decoder->temp_buf;
2000 decoder->len = INTEL_PT_PSB_LEN;
2005 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2007 unsigned char *next;
2010 intel_pt_log("Scanning for PSB\n");
2012 if (!decoder->len) {
2013 ret = intel_pt_get_next_data(decoder);
2018 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2023 part_psb = intel_pt_part_psb(decoder);
2025 ret = intel_pt_get_split_psb(decoder, part_psb);
2029 decoder->pos += decoder->len;
2035 decoder->pkt_step = next - decoder->buf;
2036 return intel_pt_get_next_packet(decoder);
2040 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2044 decoder->pge = false;
2045 decoder->continuous_period = false;
2046 decoder->last_ip = 0;
2048 intel_pt_clear_stack(&decoder->stack);
2050 err = intel_pt_scan_for_psb(decoder);
2054 decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2056 err = intel_pt_walk_psb(decoder);
2061 decoder->state.type = 0; /* Do not have a sample */
2062 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2064 return intel_pt_sync_ip(decoder);
2070 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2072 uint64_t est = decoder->timestamp_insn_cnt << 1;
2074 if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2077 est *= decoder->max_non_turbo_ratio;
2078 est /= decoder->cbr;
2080 return decoder->timestamp + est;
2083 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2088 decoder->state.type = INTEL_PT_BRANCH;
2089 decoder->state.flags = 0;
2091 switch (decoder->pkt_state) {
2092 case INTEL_PT_STATE_NO_PSB:
2093 err = intel_pt_sync(decoder);
2095 case INTEL_PT_STATE_NO_IP:
2096 decoder->last_ip = 0;
2098 case INTEL_PT_STATE_ERR_RESYNC:
2099 err = intel_pt_sync_ip(decoder);
2101 case INTEL_PT_STATE_IN_SYNC:
2102 err = intel_pt_walk_trace(decoder);
2104 case INTEL_PT_STATE_TNT:
2105 err = intel_pt_walk_tnt(decoder);
2107 err = intel_pt_walk_trace(decoder);
2109 case INTEL_PT_STATE_TIP:
2110 case INTEL_PT_STATE_TIP_PGD:
2111 err = intel_pt_walk_tip(decoder);
2113 case INTEL_PT_STATE_FUP:
2114 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2115 err = intel_pt_walk_fup(decoder);
2117 err = intel_pt_walk_fup_tip(decoder);
2119 decoder->pkt_state = INTEL_PT_STATE_FUP;
2121 case INTEL_PT_STATE_FUP_NO_TIP:
2122 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2123 err = intel_pt_walk_fup(decoder);
2125 err = intel_pt_walk_trace(decoder);
2128 err = intel_pt_bug(decoder);
2131 } while (err == -ENOLINK);
2133 decoder->state.err = err ? intel_pt_ext_err(err) : 0;
2134 decoder->state.timestamp = decoder->timestamp;
2135 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2136 decoder->state.cr3 = decoder->cr3;
2137 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2140 decoder->state.from_ip = decoder->ip;
2142 return &decoder->state;
2145 static bool intel_pt_at_psb(unsigned char *buf, size_t len)
2147 if (len < INTEL_PT_PSB_LEN)
2149 return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR,
2154 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2155 * @buf: pointer to buffer pointer
2156 * @len: size of buffer
2158 * Updates the buffer pointer to point to the start of the next PSB packet if
2159 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated,
2160 * @len is adjusted accordingly.
2162 * Return: %true if a PSB packet is found, %false otherwise.
2164 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2166 unsigned char *next;
2168 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2170 *len -= next - *buf;
2178 * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2180 * @buf: pointer to buffer pointer
2181 * @len: size of buffer
2183 * Updates the buffer pointer to point to the start of the following PSB packet
2184 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2185 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly.
2187 * Return: %true if a PSB packet is found, %false otherwise.
2189 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2191 unsigned char *next;
2196 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2198 *len -= next - *buf;
2206 * intel_pt_last_psb - find the last PSB packet in a buffer.
2208 * @len: size of buffer
2210 * This function finds the last PSB in a buffer.
2212 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2214 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2216 const char *n = INTEL_PT_PSB_STR;
2220 if (len < INTEL_PT_PSB_LEN)
2223 k = len - INTEL_PT_PSB_LEN + 1;
2225 p = memrchr(buf, n[0], k);
2228 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2237 * intel_pt_next_tsc - find and return next TSC.
2239 * @len: size of buffer
2240 * @tsc: TSC value returned
2242 * Find a TSC packet in @buf and return the TSC value. This function assumes
2243 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2244 * PSBEND packet is found.
2246 * Return: %true if TSC is found, false otherwise.
2248 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc)
2250 struct intel_pt_pkt packet;
2254 ret = intel_pt_get_packet(buf, len, &packet);
2257 if (packet.type == INTEL_PT_TSC) {
2258 *tsc = packet.payload;
2261 if (packet.type == INTEL_PT_PSBEND)
2270 * intel_pt_tsc_cmp - compare 7-byte TSCs.
2271 * @tsc1: first TSC to compare
2272 * @tsc2: second TSC to compare
2274 * This function compares 7-byte TSC values allowing for the possibility that
2275 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped
2276 * around so for that purpose this function assumes the absolute difference is
2277 * less than half the maximum difference.
2279 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2282 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2284 const uint64_t halfway = (1ULL << 55);
2290 if (tsc2 - tsc1 < halfway)
2295 if (tsc1 - tsc2 < halfway)
2303 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2305 * @buf_a: first buffer
2306 * @len_a: size of first buffer
2307 * @buf_b: second buffer
2308 * @len_b: size of second buffer
2310 * If the trace contains TSC we can look at the last TSC of @buf_a and the
2311 * first TSC of @buf_b in order to determine if the buffers overlap, and then
2312 * walk forward in @buf_b until a later TSC is found. A precondition is that
2313 * @buf_a and @buf_b are positioned at a PSB.
2315 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2316 * @buf_b + @len_b if there is no non-overlapped data.
2318 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2320 unsigned char *buf_b,
2323 uint64_t tsc_a, tsc_b;
2327 p = intel_pt_last_psb(buf_a, len_a);
2329 return buf_b; /* No PSB in buf_a => no overlap */
2331 len = len_a - (p - buf_a);
2332 if (!intel_pt_next_tsc(p, len, &tsc_a)) {
2333 /* The last PSB+ in buf_a is incomplete, so go back one more */
2335 p = intel_pt_last_psb(buf_a, len_a);
2337 return buf_b; /* No full PSB+ => assume no overlap */
2338 len = len_a - (p - buf_a);
2339 if (!intel_pt_next_tsc(p, len, &tsc_a))
2340 return buf_b; /* No TSC in buf_a => assume no overlap */
2344 /* Ignore PSB+ with no TSC */
2345 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) &&
2346 intel_pt_tsc_cmp(tsc_a, tsc_b) < 0)
2347 return buf_b; /* tsc_a < tsc_b => no overlap */
2349 if (!intel_pt_step_psb(&buf_b, &len_b))
2350 return buf_b + len_b; /* No PSB in buf_b => no data */
2355 * intel_pt_find_overlap - determine start of non-overlapped trace data.
2356 * @buf_a: first buffer
2357 * @len_a: size of first buffer
2358 * @buf_b: second buffer
2359 * @len_b: size of second buffer
2360 * @have_tsc: can use TSC packets to detect overlap
2362 * When trace samples or snapshots are recorded there is the possibility that
2363 * the data overlaps. Note that, for the purposes of decoding, data is only
2364 * useful if it begins with a PSB packet.
2366 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2367 * @buf_b + @len_b if there is no non-overlapped data.
2369 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2370 unsigned char *buf_b, size_t len_b,
2373 unsigned char *found;
2375 /* Buffer 'b' must start at PSB so throw away everything before that */
2376 if (!intel_pt_next_psb(&buf_b, &len_b))
2377 return buf_b + len_b; /* No PSB */
2379 if (!intel_pt_next_psb(&buf_a, &len_a))
2380 return buf_b; /* No overlap */
2383 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b);
2389 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2390 * we can ignore the first part of buffer 'a'.
2392 while (len_b < len_a) {
2393 if (!intel_pt_step_psb(&buf_a, &len_a))
2394 return buf_b; /* No overlap */
2397 /* Now len_b >= len_a */
2398 if (len_b > len_a) {
2399 /* The leftover buffer 'b' must start at a PSB */
2400 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2401 if (!intel_pt_step_psb(&buf_a, &len_a))
2402 return buf_b; /* No overlap */
2407 /* Potential overlap so check the bytes */
2408 found = memmem(buf_a, len_a, buf_b, len_a);
2410 return buf_b + len_a;
2412 /* Try again at next PSB in buffer 'a' */
2413 if (!intel_pt_step_psb(&buf_a, &len_a))
2414 return buf_b; /* No overlap */
2416 /* The leftover buffer 'b' must start at a PSB */
2417 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
2418 if (!intel_pt_step_psb(&buf_a, &len_a))
2419 return buf_b; /* No overlap */