tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25 #include <linux/compiler.h>
  26
  27 #include "../cache.h"
  28 #include "../util.h"
  29 #include "../auxtrace.h"
  30
  31 #include "intel-pt-insn-decoder.h"
  32 #include "intel-pt-pkt-decoder.h"
  33 #include "intel-pt-decoder.h"
  34 #include "intel-pt-log.h"
  35
  36 #define INTEL_PT_BLK_SIZE 1024
  37
  38 #define BIT63 (((uint64_t)1 << 63))
  39
  40 #define INTEL_PT_RETURN 1
  41
  42 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  43 #define INTEL_PT_MAX_LOOPS 10000
  44
  45 struct intel_pt_blk {
  46         struct intel_pt_blk *prev;
  47         uint64_t ip[INTEL_PT_BLK_SIZE];
  48 };
  49
  50 struct intel_pt_stack {
  51         struct intel_pt_blk *blk;
  52         struct intel_pt_blk *spare;
  53         int pos;
  54 };
  55
  56 enum intel_pt_pkt_state {
  57         INTEL_PT_STATE_NO_PSB,
  58         INTEL_PT_STATE_NO_IP,
  59         INTEL_PT_STATE_ERR_RESYNC,
  60         INTEL_PT_STATE_IN_SYNC,
  61         INTEL_PT_STATE_TNT_CONT,
  62         INTEL_PT_STATE_TNT,
  63         INTEL_PT_STATE_TIP,
  64         INTEL_PT_STATE_TIP_PGD,
  65         INTEL_PT_STATE_FUP,
  66         INTEL_PT_STATE_FUP_NO_TIP,
  67 };
  68
  69 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  70 {
  71         switch (pkt_state) {
  72         case INTEL_PT_STATE_NO_PSB:
  73         case INTEL_PT_STATE_NO_IP:
  74         case INTEL_PT_STATE_ERR_RESYNC:
  75         case INTEL_PT_STATE_IN_SYNC:
  76         case INTEL_PT_STATE_TNT_CONT:
  77                 return true;
  78         case INTEL_PT_STATE_TNT:
  79         case INTEL_PT_STATE_TIP:
  80         case INTEL_PT_STATE_TIP_PGD:
  81         case INTEL_PT_STATE_FUP:
  82         case INTEL_PT_STATE_FUP_NO_TIP:
  83                 return false;
  84         default:
  85                 return true;
  86         };
  87 }
  88
  89 #ifdef INTEL_PT_STRICT
  90 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  91 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  92 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  93 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  94 #else
  95 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  96 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  97 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  98 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  99 #endif
 100
 101 struct intel_pt_decoder {
 102         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 103         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 104                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 105                          uint64_t max_insn_cnt, void *data);
 106         bool (*pgd_ip)(uint64_t ip, void *data);
 107         void *data;
 108         struct intel_pt_state state;
 109         const unsigned char *buf;
 110         size_t len;
 111         bool return_compression;
 112         bool branch_enable;
 113         bool mtc_insn;
 114         bool pge;
 115         bool have_tma;
 116         bool have_cyc;
 117         bool fixup_last_mtc;
 118         bool have_last_ip;
 119         enum intel_pt_param_flags flags;
 120         uint64_t pos;
 121         uint64_t last_ip;
 122         uint64_t ip;
 123         uint64_t cr3;
 124         uint64_t timestamp;
 125         uint64_t tsc_timestamp;
 126         uint64_t ref_timestamp;
 127         uint64_t sample_timestamp;
 128         uint64_t ret_addr;
 129         uint64_t ctc_timestamp;
 130         uint64_t ctc_delta;
 131         uint64_t cycle_cnt;
 132         uint64_t cyc_ref_timestamp;
 133         uint32_t last_mtc;
 134         uint32_t tsc_ctc_ratio_n;
 135         uint32_t tsc_ctc_ratio_d;
 136         uint32_t tsc_ctc_mult;
 137         uint32_t tsc_slip;
 138         uint32_t ctc_rem_mask;
 139         int mtc_shift;
 140         struct intel_pt_stack stack;
 141         enum intel_pt_pkt_state pkt_state;
 142         struct intel_pt_pkt packet;
 143         struct intel_pt_pkt tnt;
 144         int pkt_step;
 145         int pkt_len;
 146         int last_packet_type;
 147         unsigned int cbr;
 148         unsigned int cbr_seen;
 149         unsigned int max_non_turbo_ratio;
 150         double max_non_turbo_ratio_fp;
 151         double cbr_cyc_to_tsc;
 152         double calc_cyc_to_tsc;
 153         bool have_calc_cyc_to_tsc;
 154         int exec_mode;
 155         unsigned int insn_bytes;
 156         uint64_t period;
 157         enum intel_pt_period_type period_type;
 158         uint64_t tot_insn_cnt;
 159         uint64_t period_insn_cnt;
 160         uint64_t period_mask;
 161         uint64_t period_ticks;
 162         uint64_t last_masked_timestamp;
 163         bool continuous_period;
 164         bool overflow;
 165         bool set_fup_tx_flags;
 166         bool set_fup_ptw;
 167         bool set_fup_mwait;
 168         bool set_fup_pwre;
 169         bool set_fup_exstop;
 170         unsigned int fup_tx_flags;
 171         unsigned int tx_flags;
 172         uint64_t fup_ptw_payload;
 173         uint64_t fup_mwait_payload;
 174         uint64_t fup_pwre_payload;
 175         uint64_t cbr_payload;
 176         uint64_t timestamp_insn_cnt;
 177         uint64_t sample_insn_cnt;
 178         uint64_t stuck_ip;
 179         int no_progress;
 180         int stuck_ip_prd;
 181         int stuck_ip_cnt;
 182         const unsigned char *next_buf;
 183         size_t next_len;
 184         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 185 };
 186
 187 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 188 {
 189         int i;
 190
 191         for (i = 0; x != 1; i++)
 192                 x >>= 1;
 193
 194         return x << i;
 195 }
 196
 197 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 198 {
 199         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 200                 uint64_t period;
 201
 202                 period = intel_pt_lower_power_of_2(decoder->period);
 203                 decoder->period_mask  = ~(period - 1);
 204                 decoder->period_ticks = period;
 205         }
 206 }
 207
 208 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 209 {
 210         if (!d)
 211                 return 0;
 212         return (t / d) * n + ((t % d) * n) / d;
 213 }
 214
 215 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 216 {
 217         struct intel_pt_decoder *decoder;
 218
 219         if (!params->get_trace || !params->walk_insn)
 220                 return NULL;
 221
 222         decoder = zalloc(sizeof(struct intel_pt_decoder));
 223         if (!decoder)
 224                 return NULL;
 225
 226         decoder->get_trace          = params->get_trace;
 227         decoder->walk_insn          = params->walk_insn;
 228         decoder->pgd_ip             = params->pgd_ip;
 229         decoder->data               = params->data;
 230         decoder->return_compression = params->return_compression;
 231         decoder->branch_enable      = params->branch_enable;
 232
 233         decoder->flags              = params->flags;
 234
 235         decoder->period             = params->period;
 236         decoder->period_type        = params->period_type;
 237
 238         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 239         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 240
 241         intel_pt_setup_period(decoder);
 242
 243         decoder->mtc_shift = params->mtc_period;
 244         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 245
 246         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 247         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 248
 249         if (!decoder->tsc_ctc_ratio_n)
 250                 decoder->tsc_ctc_ratio_d = 0;
 251
 252         if (decoder->tsc_ctc_ratio_d) {
 253                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 254                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 255                                                 decoder->tsc_ctc_ratio_d;
 256         }
 257
 258         /*
 259          * A TSC packet can slip past MTC packets so that the timestamp appears
 260          * to go backwards. One estimate is that can be up to about 40 CPU
 261          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 262          * slippage an order of magnitude more to be on the safe side.
 263          */
 264         decoder->tsc_slip = 0x10000;
 265
 266         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 267         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 268         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 269         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 270         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 271
 272         return decoder;
 273 }
 274
 275 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 276 {
 277         struct intel_pt_blk *blk = stack->blk;
 278
 279         stack->blk = blk->prev;
 280         if (!stack->spare)
 281                 stack->spare = blk;
 282         else
 283                 free(blk);
 284 }
 285
 286 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 287 {
 288         if (!stack->pos) {
 289                 if (!stack->blk)
 290                         return 0;
 291                 intel_pt_pop_blk(stack);
 292                 if (!stack->blk)
 293                         return 0;
 294                 stack->pos = INTEL_PT_BLK_SIZE;
 295         }
 296         return stack->blk->ip[--stack->pos];
 297 }
 298
 299 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 300 {
 301         struct intel_pt_blk *blk;
 302
 303         if (stack->spare) {
 304                 blk = stack->spare;
 305                 stack->spare = NULL;
 306         } else {
 307                 blk = malloc(sizeof(struct intel_pt_blk));
 308                 if (!blk)
 309                         return -ENOMEM;
 310         }
 311
 312         blk->prev = stack->blk;
 313         stack->blk = blk;
 314         stack->pos = 0;
 315         return 0;
 316 }
 317
 318 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 319 {
 320         int err;
 321
 322         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 323                 err = intel_pt_alloc_blk(stack);
 324                 if (err)
 325                         return err;
 326         }
 327
 328         stack->blk->ip[stack->pos++] = ip;
 329         return 0;
 330 }
 331
 332 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 333 {
 334         while (stack->blk)
 335                 intel_pt_pop_blk(stack);
 336         stack->pos = 0;
 337 }
 338
 339 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 340 {
 341         intel_pt_clear_stack(stack);
 342         zfree(&stack->blk);
 343         zfree(&stack->spare);
 344 }
 345
 346 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 347 {
 348         intel_pt_free_stack(&decoder->stack);
 349         free(decoder);
 350 }
 351
 352 static int intel_pt_ext_err(int code)
 353 {
 354         switch (code) {
 355         case -ENOMEM:
 356                 return INTEL_PT_ERR_NOMEM;
 357         case -ENOSYS:
 358                 return INTEL_PT_ERR_INTERN;
 359         case -EBADMSG:
 360                 return INTEL_PT_ERR_BADPKT;
 361         case -ENODATA:
 362                 return INTEL_PT_ERR_NODATA;
 363         case -EILSEQ:
 364                 return INTEL_PT_ERR_NOINSN;
 365         case -ENOENT:
 366                 return INTEL_PT_ERR_MISMAT;
 367         case -EOVERFLOW:
 368                 return INTEL_PT_ERR_OVR;
 369         case -ENOSPC:
 370                 return INTEL_PT_ERR_LOST;
 371         case -ELOOP:
 372                 return INTEL_PT_ERR_NELOOP;
 373         default:
 374                 return INTEL_PT_ERR_UNK;
 375         }
 376 }
 377
 378 static const char *intel_pt_err_msgs[] = {
 379         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 380         [INTEL_PT_ERR_INTERN] = "Internal error",
 381         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 382         [INTEL_PT_ERR_NODATA] = "No more data",
 383         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 384         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 385         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 386         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 387         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 388         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 389 };
 390
 391 int intel_pt__strerror(int code, char *buf, size_t buflen)
 392 {
 393         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 394                 code = INTEL_PT_ERR_UNK;
 395         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 396         return 0;
 397 }
 398
 399 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 400                                  uint64_t last_ip)
 401 {
 402         uint64_t ip;
 403
 404         switch (packet->count) {
 405         case 1:
 406                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 407                      packet->payload;
 408                 break;
 409         case 2:
 410                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 411                      packet->payload;
 412                 break;
 413         case 3:
 414                 ip = packet->payload;
 415                 /* Sign-extend 6-byte ip */
 416                 if (ip & (uint64_t)0x800000000000ULL)
 417                         ip |= (uint64_t)0xffff000000000000ULL;
 418                 break;
 419         case 4:
 420                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 421                      packet->payload;
 422                 break;
 423         case 6:
 424                 ip = packet->payload;
 425                 break;
 426         default:
 427                 return 0;
 428         }
 429
 430         return ip;
 431 }
 432
 433 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 434 {
 435         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 436         decoder->have_last_ip = true;
 437 }
 438
 439 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 440 {
 441         intel_pt_set_last_ip(decoder);
 442         decoder->ip = decoder->last_ip;
 443 }
 444
 445 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 446 {
 447         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 448                             decoder->buf);
 449 }
 450
 451 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 452 {
 453         intel_pt_log("ERROR: Internal error\n");
 454         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 455         return -ENOSYS;
 456 }
 457
 458 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 459 {
 460         decoder->tx_flags = 0;
 461 }
 462
 463 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 464 {
 465         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 466 }
 467
 468 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 469 {
 470         intel_pt_clear_tx_flags(decoder);
 471         decoder->have_tma = false;
 472         decoder->pkt_len = 1;
 473         decoder->pkt_step = 1;
 474         intel_pt_decoder_log_packet(decoder);
 475         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 476                 intel_pt_log("ERROR: Bad packet\n");
 477                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 478         }
 479         return -EBADMSG;
 480 }
 481
 482 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 483 {
 484         struct intel_pt_buffer buffer = { .buf = 0, };
 485         int ret;
 486
 487         decoder->pkt_step = 0;
 488
 489         intel_pt_log("Getting more data\n");
 490         ret = decoder->get_trace(&buffer, decoder->data);
 491         if (ret)
 492                 return ret;
 493         decoder->buf = buffer.buf;
 494         decoder->len = buffer.len;
 495         if (!decoder->len) {
 496                 intel_pt_log("No more data\n");
 497                 return -ENODATA;
 498         }
 499         if (!buffer.consecutive) {
 500                 decoder->ip = 0;
 501                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 502                 decoder->ref_timestamp = buffer.ref_timestamp;
 503                 decoder->timestamp = 0;
 504                 decoder->have_tma = false;
 505                 decoder->state.trace_nr = buffer.trace_nr;
 506                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 507                              decoder->ref_timestamp);
 508                 return -ENOLINK;
 509         }
 510
 511         return 0;
 512 }
 513
 514 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 515 {
 516         if (!decoder->next_buf)
 517                 return intel_pt_get_data(decoder);
 518
 519         decoder->buf = decoder->next_buf;
 520         decoder->len = decoder->next_len;
 521         decoder->next_buf = 0;
 522         decoder->next_len = 0;
 523         return 0;
 524 }
 525
 526 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 527 {
 528         unsigned char *buf = decoder->temp_buf;
 529         size_t old_len, len, n;
 530         int ret;
 531
 532         old_len = decoder->len;
 533         len = decoder->len;
 534         memcpy(buf, decoder->buf, len);
 535
 536         ret = intel_pt_get_data(decoder);
 537         if (ret) {
 538                 decoder->pos += old_len;
 539                 return ret < 0 ? ret : -EINVAL;
 540         }
 541
 542         n = INTEL_PT_PKT_MAX_SZ - len;
 543         if (n > decoder->len)
 544                 n = decoder->len;
 545         memcpy(buf + len, decoder->buf, n);
 546         len += n;
 547
 548         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 549         if (ret < (int)old_len) {
 550                 decoder->next_buf = decoder->buf;
 551                 decoder->next_len = decoder->len;
 552                 decoder->buf = buf;
 553                 decoder->len = old_len;
 554                 return intel_pt_bad_packet(decoder);
 555         }
 556
 557         decoder->next_buf = decoder->buf + (ret - old_len);
 558         decoder->next_len = decoder->len - (ret - old_len);
 559
 560         decoder->buf = buf;
 561         decoder->len = ret;
 562
 563         return ret;
 564 }
 565
 566 struct intel_pt_pkt_info {
 567         struct intel_pt_decoder   *decoder;
 568         struct intel_pt_pkt       packet;
 569         uint64_t                  pos;
 570         int                       pkt_len;
 571         int                       last_packet_type;
 572         void                      *data;
 573 };
 574
 575 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 576
 577 /* Lookahead packets in current buffer */
 578 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 579                                   intel_pt_pkt_cb_t cb, void *data)
 580 {
 581         struct intel_pt_pkt_info pkt_info;
 582         const unsigned char *buf = decoder->buf;
 583         size_t len = decoder->len;
 584         int ret;
 585
 586         pkt_info.decoder          = decoder;
 587         pkt_info.pos              = decoder->pos;
 588         pkt_info.pkt_len          = decoder->pkt_step;
 589         pkt_info.last_packet_type = decoder->last_packet_type;
 590         pkt_info.data             = data;
 591
 592         while (1) {
 593                 do {
 594                         pkt_info.pos += pkt_info.pkt_len;
 595                         buf          += pkt_info.pkt_len;
 596                         len          -= pkt_info.pkt_len;
 597
 598                         if (!len)
 599                                 return INTEL_PT_NEED_MORE_BYTES;
 600
 601                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 602                         if (!ret)
 603                                 return INTEL_PT_NEED_MORE_BYTES;
 604                         if (ret < 0)
 605                                 return ret;
 606
 607                         pkt_info.pkt_len = ret;
 608                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 609
 610                 ret = cb(&pkt_info);
 611                 if (ret)
 612                         return 0;
 613
 614                 pkt_info.last_packet_type = pkt_info.packet.type;
 615         }
 616 }
 617
 618 struct intel_pt_calc_cyc_to_tsc_info {
 619         uint64_t        cycle_cnt;
 620         unsigned int    cbr;
 621         uint32_t        last_mtc;
 622         uint64_t        ctc_timestamp;
 623         uint64_t        ctc_delta;
 624         uint64_t        tsc_timestamp;
 625         uint64_t        timestamp;
 626         bool            have_tma;
 627         bool            fixup_last_mtc;
 628         bool            from_mtc;
 629         double          cbr_cyc_to_tsc;
 630 };
 631
 632 /*
 633  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 634  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 635  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 636  * packet by copying the missing bits from the current MTC assuming the least
 637  * difference between the two, and that the current MTC comes after last_mtc.
 638  */
 639 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 640                                     uint32_t *last_mtc)
 641 {
 642         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 643         uint32_t mask = ~(first_missing_bit - 1);
 644
 645         *last_mtc |= mtc & mask;
 646         if (*last_mtc >= mtc) {
 647                 *last_mtc -= first_missing_bit;
 648                 *last_mtc &= 0xff;
 649         }
 650 }
 651
 652 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 653 {
 654         struct intel_pt_decoder *decoder = pkt_info->decoder;
 655         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 656         uint64_t timestamp;
 657         double cyc_to_tsc;
 658         unsigned int cbr;
 659         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 660
 661         switch (pkt_info->packet.type) {
 662         case INTEL_PT_TNT:
 663         case INTEL_PT_TIP_PGE:
 664         case INTEL_PT_TIP:
 665         case INTEL_PT_FUP:
 666         case INTEL_PT_PSB:
 667         case INTEL_PT_PIP:
 668         case INTEL_PT_MODE_EXEC:
 669         case INTEL_PT_MODE_TSX:
 670         case INTEL_PT_PSBEND:
 671         case INTEL_PT_PAD:
 672         case INTEL_PT_VMCS:
 673         case INTEL_PT_MNT:
 674         case INTEL_PT_PTWRITE:
 675         case INTEL_PT_PTWRITE_IP:
 676                 return 0;
 677
 678         case INTEL_PT_MTC:
 679                 if (!data->have_tma)
 680                         return 0;
 681
 682                 mtc = pkt_info->packet.payload;
 683                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 684                         data->fixup_last_mtc = false;
 685                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 686                                                 &data->last_mtc);
 687                 }
 688                 if (mtc > data->last_mtc)
 689                         mtc_delta = mtc - data->last_mtc;
 690                 else
 691                         mtc_delta = mtc + 256 - data->last_mtc;
 692                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 693                 data->last_mtc = mtc;
 694
 695                 if (decoder->tsc_ctc_mult) {
 696                         timestamp = data->ctc_timestamp +
 697                                 data->ctc_delta * decoder->tsc_ctc_mult;
 698                 } else {
 699                         timestamp = data->ctc_timestamp +
 700                                 multdiv(data->ctc_delta,
 701                                         decoder->tsc_ctc_ratio_n,
 702                                         decoder->tsc_ctc_ratio_d);
 703                 }
 704
 705                 if (timestamp < data->timestamp)
 706                         return 1;
 707
 708                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 709                         data->timestamp = timestamp;
 710                         return 0;
 711                 }
 712
 713                 break;
 714
 715         case INTEL_PT_TSC:
 716                 /*
 717                  * For now, do not support using TSC packets - refer
 718                  * intel_pt_calc_cyc_to_tsc().
 719                  */
 720                 if (data->from_mtc)
 721                         return 1;
 722                 timestamp = pkt_info->packet.payload |
 723                             (data->timestamp & (0xffULL << 56));
 724                 if (data->from_mtc && timestamp < data->timestamp &&
 725                     data->timestamp - timestamp < decoder->tsc_slip)
 726                         return 1;
 727                 if (timestamp < data->timestamp)
 728                         timestamp += (1ULL << 56);
 729                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 730                         if (data->from_mtc)
 731                                 return 1;
 732                         data->tsc_timestamp = timestamp;
 733                         data->timestamp = timestamp;
 734                         return 0;
 735                 }
 736                 break;
 737
 738         case INTEL_PT_TMA:
 739                 if (data->from_mtc)
 740                         return 1;
 741
 742                 if (!decoder->tsc_ctc_ratio_d)
 743                         return 0;
 744
 745                 ctc = pkt_info->packet.payload;
 746                 fc = pkt_info->packet.count;
 747                 ctc_rem = ctc & decoder->ctc_rem_mask;
 748
 749                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 750
 751                 data->ctc_timestamp = data->tsc_timestamp - fc;
 752                 if (decoder->tsc_ctc_mult) {
 753                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 754                 } else {
 755                         data->ctc_timestamp -=
 756                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 757                                         decoder->tsc_ctc_ratio_d);
 758                 }
 759
 760                 data->ctc_delta = 0;
 761                 data->have_tma = true;
 762                 data->fixup_last_mtc = true;
 763
 764                 return 0;
 765
 766         case INTEL_PT_CYC:
 767                 data->cycle_cnt += pkt_info->packet.payload;
 768                 return 0;
 769
 770         case INTEL_PT_CBR:
 771                 cbr = pkt_info->packet.payload;
 772                 if (data->cbr && data->cbr != cbr)
 773                         return 1;
 774                 data->cbr = cbr;
 775                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 776                 return 0;
 777
 778         case INTEL_PT_TIP_PGD:
 779         case INTEL_PT_TRACESTOP:
 780         case INTEL_PT_EXSTOP:
 781         case INTEL_PT_EXSTOP_IP:
 782         case INTEL_PT_MWAIT:
 783         case INTEL_PT_PWRE:
 784         case INTEL_PT_PWRX:
 785         case INTEL_PT_OVF:
 786         case INTEL_PT_BAD: /* Does not happen */
 787         default:
 788                 return 1;
 789         }
 790
 791         if (!data->cbr && decoder->cbr) {
 792                 data->cbr = decoder->cbr;
 793                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 794         }
 795
 796         if (!data->cycle_cnt)
 797                 return 1;
 798
 799         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 800
 801         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 802             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 803                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 804                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 805                 return 1;
 806         }
 807
 808         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 809         decoder->have_calc_cyc_to_tsc = true;
 810
 811         if (data->cbr) {
 812                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 813                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 814         } else {
 815                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 816                              cyc_to_tsc, pkt_info->pos);
 817         }
 818
 819         return 1;
 820 }
 821
 822 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 823                                      bool from_mtc)
 824 {
 825         struct intel_pt_calc_cyc_to_tsc_info data = {
 826                 .cycle_cnt      = 0,
 827                 .cbr            = 0,
 828                 .last_mtc       = decoder->last_mtc,
 829                 .ctc_timestamp  = decoder->ctc_timestamp,
 830                 .ctc_delta      = decoder->ctc_delta,
 831                 .tsc_timestamp  = decoder->tsc_timestamp,
 832                 .timestamp      = decoder->timestamp,
 833                 .have_tma       = decoder->have_tma,
 834                 .fixup_last_mtc = decoder->fixup_last_mtc,
 835                 .from_mtc       = from_mtc,
 836                 .cbr_cyc_to_tsc = 0,
 837         };
 838
 839         /*
 840          * For now, do not support using TSC packets for at least the reasons:
 841          * 1) timing might have stopped
 842          * 2) TSC packets within PSB+ can slip against CYC packets
 843          */
 844         if (!from_mtc)
 845                 return;
 846
 847         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 848 }
 849
 850 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 851 {
 852         int ret;
 853
 854         decoder->last_packet_type = decoder->packet.type;
 855
 856         do {
 857                 decoder->pos += decoder->pkt_step;
 858                 decoder->buf += decoder->pkt_step;
 859                 decoder->len -= decoder->pkt_step;
 860
 861                 if (!decoder->len) {
 862                         ret = intel_pt_get_next_data(decoder);
 863                         if (ret)
 864                                 return ret;
 865                 }
 866
 867                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 868                                           &decoder->packet);
 869                 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
 870                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 871                         ret = intel_pt_get_split_packet(decoder);
 872                         if (ret < 0)
 873                                 return ret;
 874                 }
 875                 if (ret <= 0)
 876                         return intel_pt_bad_packet(decoder);
 877
 878                 decoder->pkt_len = ret;
 879                 decoder->pkt_step = ret;
 880                 intel_pt_decoder_log_packet(decoder);
 881         } while (decoder->packet.type == INTEL_PT_PAD);
 882
 883         return 0;
 884 }
 885
 886 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 887 {
 888         uint64_t timestamp, masked_timestamp;
 889
 890         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 891         masked_timestamp = timestamp & decoder->period_mask;
 892         if (decoder->continuous_period) {
 893                 if (masked_timestamp > decoder->last_masked_timestamp)
 894                         return 1;
 895         } else {
 896                 timestamp += 1;
 897                 masked_timestamp = timestamp & decoder->period_mask;
 898                 if (masked_timestamp > decoder->last_masked_timestamp) {
 899                         decoder->last_masked_timestamp = masked_timestamp;
 900                         decoder->continuous_period = true;
 901                 }
 902         }
 903
 904         if (masked_timestamp < decoder->last_masked_timestamp)
 905                 return decoder->period_ticks;
 906
 907         return decoder->period_ticks - (timestamp - masked_timestamp);
 908 }
 909
 910 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 911 {
 912         switch (decoder->period_type) {
 913         case INTEL_PT_PERIOD_INSTRUCTIONS:
 914                 return decoder->period - decoder->period_insn_cnt;
 915         case INTEL_PT_PERIOD_TICKS:
 916                 return intel_pt_next_period(decoder);
 917         case INTEL_PT_PERIOD_NONE:
 918         case INTEL_PT_PERIOD_MTC:
 919         default:
 920                 return 0;
 921         }
 922 }
 923
 924 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 925 {
 926         uint64_t timestamp, masked_timestamp;
 927
 928         switch (decoder->period_type) {
 929         case INTEL_PT_PERIOD_INSTRUCTIONS:
 930                 decoder->period_insn_cnt = 0;
 931                 break;
 932         case INTEL_PT_PERIOD_TICKS:
 933                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 934                 masked_timestamp = timestamp & decoder->period_mask;
 935                 if (masked_timestamp > decoder->last_masked_timestamp)
 936                         decoder->last_masked_timestamp = masked_timestamp;
 937                 else
 938                         decoder->last_masked_timestamp += decoder->period_ticks;
 939                 break;
 940         case INTEL_PT_PERIOD_NONE:
 941         case INTEL_PT_PERIOD_MTC:
 942         default:
 943                 break;
 944         }
 945
 946         decoder->state.type |= INTEL_PT_INSTRUCTION;
 947 }
 948
 949 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 950                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 951 {
 952         uint64_t max_insn_cnt, insn_cnt = 0;
 953         int err;
 954
 955         if (!decoder->mtc_insn)
 956                 decoder->mtc_insn = true;
 957
 958         max_insn_cnt = intel_pt_next_sample(decoder);
 959
 960         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 961                                  max_insn_cnt, decoder->data);
 962
 963         decoder->tot_insn_cnt += insn_cnt;
 964         decoder->timestamp_insn_cnt += insn_cnt;
 965         decoder->sample_insn_cnt += insn_cnt;
 966         decoder->period_insn_cnt += insn_cnt;
 967
 968         if (err) {
 969                 decoder->no_progress = 0;
 970                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 971                 intel_pt_log_at("ERROR: Failed to get instruction",
 972                                 decoder->ip);
 973                 if (err == -ENOENT)
 974                         return -ENOLINK;
 975                 return -EILSEQ;
 976         }
 977
 978         if (ip && decoder->ip == ip) {
 979                 err = -EAGAIN;
 980                 goto out;
 981         }
 982
 983         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 984                 intel_pt_sample_insn(decoder);
 985
 986         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 987                 decoder->state.type = INTEL_PT_INSTRUCTION;
 988                 decoder->state.from_ip = decoder->ip;
 989                 decoder->state.to_ip = 0;
 990                 decoder->ip += intel_pt_insn->length;
 991                 err = INTEL_PT_RETURN;
 992                 goto out;
 993         }
 994
 995         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 996                 /* Zero-length calls are excluded */
 997                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 998                     intel_pt_insn->rel) {
 999                         err = intel_pt_push(&decoder->stack, decoder->ip +
1000                                             intel_pt_insn->length);
1001                         if (err)
1002                                 goto out;
1003                 }
1004         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1005                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1006         }
1007
1008         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1009                 int cnt = decoder->no_progress++;
1010
1011                 decoder->state.from_ip = decoder->ip;
1012                 decoder->ip += intel_pt_insn->length +
1013                                 intel_pt_insn->rel;
1014                 decoder->state.to_ip = decoder->ip;
1015                 err = INTEL_PT_RETURN;
1016
1017                 /*
1018                  * Check for being stuck in a loop.  This can happen if a
1019                  * decoder error results in the decoder erroneously setting the
1020                  * ip to an address that is itself in an infinite loop that
1021                  * consumes no packets.  When that happens, there must be an
1022                  * unconditional branch.
1023                  */
1024                 if (cnt) {
1025                         if (cnt == 1) {
1026                                 decoder->stuck_ip = decoder->state.to_ip;
1027                                 decoder->stuck_ip_prd = 1;
1028                                 decoder->stuck_ip_cnt = 1;
1029                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
1030                                    decoder->state.to_ip == decoder->stuck_ip) {
1031                                 intel_pt_log_at("ERROR: Never-ending loop",
1032                                                 decoder->state.to_ip);
1033                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1034                                 err = -ELOOP;
1035                                 goto out;
1036                         } else if (!--decoder->stuck_ip_cnt) {
1037                                 decoder->stuck_ip_prd += 1;
1038                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1039                                 decoder->stuck_ip = decoder->state.to_ip;
1040                         }
1041                 }
1042                 goto out_no_progress;
1043         }
1044 out:
1045         decoder->no_progress = 0;
1046 out_no_progress:
1047         decoder->state.insn_op = intel_pt_insn->op;
1048         decoder->state.insn_len = intel_pt_insn->length;
1049         memcpy(decoder->state.insn, intel_pt_insn->buf,
1050                INTEL_PT_INSN_BUF_SZ);
1051
1052         if (decoder->tx_flags & INTEL_PT_IN_TX)
1053                 decoder->state.flags |= INTEL_PT_IN_TX;
1054
1055         return err;
1056 }
1057
1058 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1059 {
1060         bool ret = false;
1061
1062         if (decoder->set_fup_tx_flags) {
1063                 decoder->set_fup_tx_flags = false;
1064                 decoder->tx_flags = decoder->fup_tx_flags;
1065                 decoder->state.type = INTEL_PT_TRANSACTION;
1066                 decoder->state.from_ip = decoder->ip;
1067                 decoder->state.to_ip = 0;
1068                 decoder->state.flags = decoder->fup_tx_flags;
1069                 return true;
1070         }
1071         if (decoder->set_fup_ptw) {
1072                 decoder->set_fup_ptw = false;
1073                 decoder->state.type = INTEL_PT_PTW;
1074                 decoder->state.flags |= INTEL_PT_FUP_IP;
1075                 decoder->state.from_ip = decoder->ip;
1076                 decoder->state.to_ip = 0;
1077                 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1078                 return true;
1079         }
1080         if (decoder->set_fup_mwait) {
1081                 decoder->set_fup_mwait = false;
1082                 decoder->state.type = INTEL_PT_MWAIT_OP;
1083                 decoder->state.from_ip = decoder->ip;
1084                 decoder->state.to_ip = 0;
1085                 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1086                 ret = true;
1087         }
1088         if (decoder->set_fup_pwre) {
1089                 decoder->set_fup_pwre = false;
1090                 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1091                 decoder->state.type &= ~INTEL_PT_BRANCH;
1092                 decoder->state.from_ip = decoder->ip;
1093                 decoder->state.to_ip = 0;
1094                 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1095                 ret = true;
1096         }
1097         if (decoder->set_fup_exstop) {
1098                 decoder->set_fup_exstop = false;
1099                 decoder->state.type |= INTEL_PT_EX_STOP;
1100                 decoder->state.type &= ~INTEL_PT_BRANCH;
1101                 decoder->state.flags |= INTEL_PT_FUP_IP;
1102                 decoder->state.from_ip = decoder->ip;
1103                 decoder->state.to_ip = 0;
1104                 ret = true;
1105         }
1106         return ret;
1107 }
1108
1109 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1110                                           struct intel_pt_insn *intel_pt_insn,
1111                                           uint64_t ip, int err)
1112 {
1113         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1114                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1115                ip == decoder->ip + intel_pt_insn->length;
1116 }
1117
1118 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1119 {
1120         struct intel_pt_insn intel_pt_insn;
1121         uint64_t ip;
1122         int err;
1123
1124         ip = decoder->last_ip;
1125
1126         while (1) {
1127                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1128                 if (err == INTEL_PT_RETURN)
1129                         return 0;
1130                 if (err == -EAGAIN ||
1131                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1132                         if (intel_pt_fup_event(decoder))
1133                                 return 0;
1134                         return -EAGAIN;
1135                 }
1136                 decoder->set_fup_tx_flags = false;
1137                 if (err)
1138                         return err;
1139
1140                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1141                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1142                                         decoder->ip);
1143                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1144                         return -ENOENT;
1145                 }
1146
1147                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1148                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1149                                         decoder->ip);
1150                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1151                         return -ENOENT;
1152                 }
1153
1154                 intel_pt_bug(decoder);
1155         }
1156 }
1157
1158 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1159 {
1160         struct intel_pt_insn intel_pt_insn;
1161         int err;
1162
1163         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1164         if (err == INTEL_PT_RETURN &&
1165             decoder->pgd_ip &&
1166             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1167             (decoder->state.type & INTEL_PT_BRANCH) &&
1168             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1169                 /* Unconditional branch leaving filter region */
1170                 decoder->no_progress = 0;
1171                 decoder->pge = false;
1172                 decoder->continuous_period = false;
1173                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1174                 decoder->state.type |= INTEL_PT_TRACE_END;
1175                 return 0;
1176         }
1177         if (err == INTEL_PT_RETURN)
1178                 return 0;
1179         if (err)
1180                 return err;
1181
1182         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1183                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1184                         decoder->pge = false;
1185                         decoder->continuous_period = false;
1186                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1187                         decoder->state.from_ip = decoder->ip;
1188                         if (decoder->packet.count == 0) {
1189                                 decoder->state.to_ip = 0;
1190                         } else {
1191                                 decoder->state.to_ip = decoder->last_ip;
1192                                 decoder->ip = decoder->last_ip;
1193                         }
1194                         decoder->state.type |= INTEL_PT_TRACE_END;
1195                 } else {
1196                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1197                         decoder->state.from_ip = decoder->ip;
1198                         if (decoder->packet.count == 0) {
1199                                 decoder->state.to_ip = 0;
1200                         } else {
1201                                 decoder->state.to_ip = decoder->last_ip;
1202                                 decoder->ip = decoder->last_ip;
1203                         }
1204                 }
1205                 return 0;
1206         }
1207
1208         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1209                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1210                                  intel_pt_insn.rel;
1211
1212                 if (decoder->pgd_ip &&
1213                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1214                     decoder->pgd_ip(to_ip, decoder->data)) {
1215                         /* Conditional branch leaving filter region */
1216                         decoder->pge = false;
1217                         decoder->continuous_period = false;
1218                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1219                         decoder->ip = to_ip;
1220                         decoder->state.from_ip = decoder->ip;
1221                         decoder->state.to_ip = to_ip;
1222                         decoder->state.type |= INTEL_PT_TRACE_END;
1223                         return 0;
1224                 }
1225                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1226                                 decoder->ip);
1227                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1228                 return -ENOENT;
1229         }
1230
1231         return intel_pt_bug(decoder);
1232 }
1233
1234 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1235 {
1236         struct intel_pt_insn intel_pt_insn;
1237         int err;
1238
1239         while (1) {
1240                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1241                 if (err == INTEL_PT_RETURN)
1242                         return 0;
1243                 if (err)
1244                         return err;
1245
1246                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1247                         if (!decoder->return_compression) {
1248                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1249                                                 decoder->ip);
1250                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1251                                 return -ENOENT;
1252                         }
1253                         if (!decoder->ret_addr) {
1254                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1255                                                 decoder->ip);
1256                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1257                                 return -ENOENT;
1258                         }
1259                         if (!(decoder->tnt.payload & BIT63)) {
1260                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1261                                                 decoder->ip);
1262                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1263                                 return -ENOENT;
1264                         }
1265                         decoder->tnt.count -= 1;
1266                         if (decoder->tnt.count)
1267                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1268                         else
1269                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1270                         decoder->tnt.payload <<= 1;
1271                         decoder->state.from_ip = decoder->ip;
1272                         decoder->ip = decoder->ret_addr;
1273                         decoder->state.to_ip = decoder->ip;
1274                         return 0;
1275                 }
1276
1277                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1278                         /* Handle deferred TIPs */
1279                         err = intel_pt_get_next_packet(decoder);
1280                         if (err)
1281                                 return err;
1282                         if (decoder->packet.type != INTEL_PT_TIP ||
1283                             decoder->packet.count == 0) {
1284                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1285                                                 decoder->ip);
1286                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1287                                 decoder->pkt_step = 0;
1288                                 return -ENOENT;
1289                         }
1290                         intel_pt_set_last_ip(decoder);
1291                         decoder->state.from_ip = decoder->ip;
1292                         decoder->state.to_ip = decoder->last_ip;
1293                         decoder->ip = decoder->last_ip;
1294                         return 0;
1295                 }
1296
1297                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1298                         decoder->tnt.count -= 1;
1299                         if (decoder->tnt.count)
1300                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1301                         else
1302                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1303                         if (decoder->tnt.payload & BIT63) {
1304                                 decoder->tnt.payload <<= 1;
1305                                 decoder->state.from_ip = decoder->ip;
1306                                 decoder->ip += intel_pt_insn.length +
1307                                                intel_pt_insn.rel;
1308                                 decoder->state.to_ip = decoder->ip;
1309                                 return 0;
1310                         }
1311                         /* Instruction sample for a non-taken branch */
1312                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1313                                 decoder->tnt.payload <<= 1;
1314                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1315                                 decoder->state.from_ip = decoder->ip;
1316                                 decoder->state.to_ip = 0;
1317                                 decoder->ip += intel_pt_insn.length;
1318                                 return 0;
1319                         }
1320                         decoder->ip += intel_pt_insn.length;
1321                         if (!decoder->tnt.count)
1322                                 return -EAGAIN;
1323                         decoder->tnt.payload <<= 1;
1324                         continue;
1325                 }
1326
1327                 return intel_pt_bug(decoder);
1328         }
1329 }
1330
1331 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1332 {
1333         unsigned int fup_tx_flags;
1334         int err;
1335
1336         fup_tx_flags = decoder->packet.payload &
1337                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1338         err = intel_pt_get_next_packet(decoder);
1339         if (err)
1340                 return err;
1341         if (decoder->packet.type == INTEL_PT_FUP) {
1342                 decoder->fup_tx_flags = fup_tx_flags;
1343                 decoder->set_fup_tx_flags = true;
1344                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1345                         *no_tip = true;
1346         } else {
1347                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1348                                 decoder->pos);
1349                 intel_pt_update_in_tx(decoder);
1350         }
1351         return 0;
1352 }
1353
1354 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1355 {
1356         uint64_t timestamp;
1357
1358         decoder->have_tma = false;
1359
1360         if (decoder->ref_timestamp) {
1361                 timestamp = decoder->packet.payload |
1362                             (decoder->ref_timestamp & (0xffULL << 56));
1363                 if (timestamp < decoder->ref_timestamp) {
1364                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1365                                 timestamp += (1ULL << 56);
1366                 } else {
1367                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1368                                 timestamp -= (1ULL << 56);
1369                 }
1370                 decoder->tsc_timestamp = timestamp;
1371                 decoder->timestamp = timestamp;
1372                 decoder->ref_timestamp = 0;
1373                 decoder->timestamp_insn_cnt = 0;
1374         } else if (decoder->timestamp) {
1375                 timestamp = decoder->packet.payload |
1376                             (decoder->timestamp & (0xffULL << 56));
1377                 decoder->tsc_timestamp = timestamp;
1378                 if (timestamp < decoder->timestamp &&
1379                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1380                         intel_pt_log_to("Suppressing backwards timestamp",
1381                                         timestamp);
1382                         timestamp = decoder->timestamp;
1383                 }
1384                 if (timestamp < decoder->timestamp) {
1385                         intel_pt_log_to("Wraparound timestamp", timestamp);
1386                         timestamp += (1ULL << 56);
1387                         decoder->tsc_timestamp = timestamp;
1388                 }
1389                 decoder->timestamp = timestamp;
1390                 decoder->timestamp_insn_cnt = 0;
1391         }
1392
1393         if (decoder->last_packet_type == INTEL_PT_CYC) {
1394                 decoder->cyc_ref_timestamp = decoder->timestamp;
1395                 decoder->cycle_cnt = 0;
1396                 decoder->have_calc_cyc_to_tsc = false;
1397                 intel_pt_calc_cyc_to_tsc(decoder, false);
1398         }
1399
1400         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1401 }
1402
1403 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1404 {
1405         intel_pt_log("ERROR: Buffer overflow\n");
1406         intel_pt_clear_tx_flags(decoder);
1407         decoder->timestamp_insn_cnt = 0;
1408         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1409         decoder->overflow = true;
1410         return -EOVERFLOW;
1411 }
1412
1413 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1414 {
1415         uint32_t ctc = decoder->packet.payload;
1416         uint32_t fc = decoder->packet.count;
1417         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1418
1419         if (!decoder->tsc_ctc_ratio_d)
1420                 return;
1421
1422         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1423         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1424         if (decoder->tsc_ctc_mult) {
1425                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1426         } else {
1427                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1428                                                   decoder->tsc_ctc_ratio_n,
1429                                                   decoder->tsc_ctc_ratio_d);
1430         }
1431         decoder->ctc_delta = 0;
1432         decoder->have_tma = true;
1433         decoder->fixup_last_mtc = true;
1434         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1435                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1436 }
1437
1438 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1439 {
1440         uint64_t timestamp;
1441         uint32_t mtc, mtc_delta;
1442
1443         if (!decoder->have_tma)
1444                 return;
1445
1446         mtc = decoder->packet.payload;
1447
1448         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1449                 decoder->fixup_last_mtc = false;
1450                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1451                                         &decoder->last_mtc);
1452         }
1453
1454         if (mtc > decoder->last_mtc)
1455                 mtc_delta = mtc - decoder->last_mtc;
1456         else
1457                 mtc_delta = mtc + 256 - decoder->last_mtc;
1458
1459         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1460
1461         if (decoder->tsc_ctc_mult) {
1462                 timestamp = decoder->ctc_timestamp +
1463                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1464         } else {
1465                 timestamp = decoder->ctc_timestamp +
1466                             multdiv(decoder->ctc_delta,
1467                                     decoder->tsc_ctc_ratio_n,
1468                                     decoder->tsc_ctc_ratio_d);
1469         }
1470
1471         if (timestamp < decoder->timestamp)
1472                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1473                              timestamp, decoder->timestamp);
1474         else
1475                 decoder->timestamp = timestamp;
1476
1477         decoder->timestamp_insn_cnt = 0;
1478         decoder->last_mtc = mtc;
1479
1480         if (decoder->last_packet_type == INTEL_PT_CYC) {
1481                 decoder->cyc_ref_timestamp = decoder->timestamp;
1482                 decoder->cycle_cnt = 0;
1483                 decoder->have_calc_cyc_to_tsc = false;
1484                 intel_pt_calc_cyc_to_tsc(decoder, true);
1485         }
1486
1487         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1488 }
1489
1490 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1491 {
1492         unsigned int cbr = decoder->packet.payload & 0xff;
1493
1494         decoder->cbr_payload = decoder->packet.payload;
1495
1496         if (decoder->cbr == cbr)
1497                 return;
1498
1499         decoder->cbr = cbr;
1500         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1501 }
1502
1503 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1504 {
1505         uint64_t timestamp = decoder->cyc_ref_timestamp;
1506
1507         decoder->have_cyc = true;
1508
1509         decoder->cycle_cnt += decoder->packet.payload;
1510
1511         if (!decoder->cyc_ref_timestamp)
1512                 return;
1513
1514         if (decoder->have_calc_cyc_to_tsc)
1515                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1516         else if (decoder->cbr)
1517                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1518         else
1519                 return;
1520
1521         if (timestamp < decoder->timestamp)
1522                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1523                              timestamp, decoder->timestamp);
1524         else
1525                 decoder->timestamp = timestamp;
1526
1527         decoder->timestamp_insn_cnt = 0;
1528
1529         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1530 }
1531
1532 /* Walk PSB+ packets when already in sync. */
1533 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1534 {
1535         int err;
1536
1537         while (1) {
1538                 err = intel_pt_get_next_packet(decoder);
1539                 if (err)
1540                         return err;
1541
1542                 switch (decoder->packet.type) {
1543                 case INTEL_PT_PSBEND:
1544                         return 0;
1545
1546                 case INTEL_PT_TIP_PGD:
1547                 case INTEL_PT_TIP_PGE:
1548                 case INTEL_PT_TIP:
1549                 case INTEL_PT_TNT:
1550                 case INTEL_PT_TRACESTOP:
1551                 case INTEL_PT_BAD:
1552                 case INTEL_PT_PSB:
1553                 case INTEL_PT_PTWRITE:
1554                 case INTEL_PT_PTWRITE_IP:
1555                 case INTEL_PT_EXSTOP:
1556                 case INTEL_PT_EXSTOP_IP:
1557                 case INTEL_PT_MWAIT:
1558                 case INTEL_PT_PWRE:
1559                 case INTEL_PT_PWRX:
1560                         decoder->have_tma = false;
1561                         intel_pt_log("ERROR: Unexpected packet\n");
1562                         return -EAGAIN;
1563
1564                 case INTEL_PT_OVF:
1565                         return intel_pt_overflow(decoder);
1566
1567                 case INTEL_PT_TSC:
1568                         intel_pt_calc_tsc_timestamp(decoder);
1569                         break;
1570
1571                 case INTEL_PT_TMA:
1572                         intel_pt_calc_tma(decoder);
1573                         break;
1574
1575                 case INTEL_PT_CBR:
1576                         intel_pt_calc_cbr(decoder);
1577                         break;
1578
1579                 case INTEL_PT_MODE_EXEC:
1580                         decoder->exec_mode = decoder->packet.payload;
1581                         break;
1582
1583                 case INTEL_PT_PIP:
1584                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1585                         break;
1586
1587                 case INTEL_PT_FUP:
1588                         decoder->pge = true;
1589                         if (decoder->packet.count)
1590                                 intel_pt_set_last_ip(decoder);
1591                         break;
1592
1593                 case INTEL_PT_MODE_TSX:
1594                         intel_pt_update_in_tx(decoder);
1595                         break;
1596
1597                 case INTEL_PT_MTC:
1598                         intel_pt_calc_mtc_timestamp(decoder);
1599                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1600                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1601                         break;
1602
1603                 case INTEL_PT_CYC:
1604                 case INTEL_PT_VMCS:
1605                 case INTEL_PT_MNT:
1606                 case INTEL_PT_PAD:
1607                 default:
1608                         break;
1609                 }
1610         }
1611 }
1612
1613 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1614 {
1615         int err;
1616
1617         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1618                 decoder->tx_flags = 0;
1619                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1620                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1621         } else {
1622                 decoder->state.flags |= INTEL_PT_ASYNC;
1623         }
1624
1625         while (1) {
1626                 err = intel_pt_get_next_packet(decoder);
1627                 if (err)
1628                         return err;
1629
1630                 switch (decoder->packet.type) {
1631                 case INTEL_PT_TNT:
1632                 case INTEL_PT_FUP:
1633                 case INTEL_PT_TRACESTOP:
1634                 case INTEL_PT_PSB:
1635                 case INTEL_PT_TSC:
1636                 case INTEL_PT_TMA:
1637                 case INTEL_PT_MODE_TSX:
1638                 case INTEL_PT_BAD:
1639                 case INTEL_PT_PSBEND:
1640                 case INTEL_PT_PTWRITE:
1641                 case INTEL_PT_PTWRITE_IP:
1642                 case INTEL_PT_EXSTOP:
1643                 case INTEL_PT_EXSTOP_IP:
1644                 case INTEL_PT_MWAIT:
1645                 case INTEL_PT_PWRE:
1646                 case INTEL_PT_PWRX:
1647                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1648                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1649                         decoder->pkt_step = 0;
1650                         return -ENOENT;
1651
1652                 case INTEL_PT_CBR:
1653                         intel_pt_calc_cbr(decoder);
1654                         break;
1655
1656                 case INTEL_PT_OVF:
1657                         return intel_pt_overflow(decoder);
1658
1659                 case INTEL_PT_TIP_PGD:
1660                         decoder->state.from_ip = decoder->ip;
1661                         if (decoder->packet.count == 0) {
1662                                 decoder->state.to_ip = 0;
1663                         } else {
1664                                 intel_pt_set_ip(decoder);
1665                                 decoder->state.to_ip = decoder->ip;
1666                         }
1667                         decoder->pge = false;
1668                         decoder->continuous_period = false;
1669                         decoder->state.type |= INTEL_PT_TRACE_END;
1670                         return 0;
1671
1672                 case INTEL_PT_TIP_PGE:
1673                         decoder->pge = true;
1674                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1675                                      decoder->ip);
1676                         decoder->state.from_ip = 0;
1677                         if (decoder->packet.count == 0) {
1678                                 decoder->state.to_ip = 0;
1679                         } else {
1680                                 intel_pt_set_ip(decoder);
1681                                 decoder->state.to_ip = decoder->ip;
1682                         }
1683                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1684                         return 0;
1685
1686                 case INTEL_PT_TIP:
1687                         decoder->state.from_ip = decoder->ip;
1688                         if (decoder->packet.count == 0) {
1689                                 decoder->state.to_ip = 0;
1690                         } else {
1691                                 intel_pt_set_ip(decoder);
1692                                 decoder->state.to_ip = decoder->ip;
1693                         }
1694                         return 0;
1695
1696                 case INTEL_PT_PIP:
1697                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1698                         break;
1699
1700                 case INTEL_PT_MTC:
1701                         intel_pt_calc_mtc_timestamp(decoder);
1702                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1703                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1704                         break;
1705
1706                 case INTEL_PT_CYC:
1707                         intel_pt_calc_cyc_timestamp(decoder);
1708                         break;
1709
1710                 case INTEL_PT_MODE_EXEC:
1711                         decoder->exec_mode = decoder->packet.payload;
1712                         break;
1713
1714                 case INTEL_PT_VMCS:
1715                 case INTEL_PT_MNT:
1716                 case INTEL_PT_PAD:
1717                         break;
1718
1719                 default:
1720                         return intel_pt_bug(decoder);
1721                 }
1722         }
1723 }
1724
1725 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1726 {
1727         bool no_tip = false;
1728         int err;
1729
1730         while (1) {
1731                 err = intel_pt_get_next_packet(decoder);
1732                 if (err)
1733                         return err;
1734 next:
1735                 switch (decoder->packet.type) {
1736                 case INTEL_PT_TNT:
1737                         if (!decoder->packet.count)
1738                                 break;
1739                         decoder->tnt = decoder->packet;
1740                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1741                         err = intel_pt_walk_tnt(decoder);
1742                         if (err == -EAGAIN)
1743                                 break;
1744                         return err;
1745
1746                 case INTEL_PT_TIP_PGD:
1747                         if (decoder->packet.count != 0)
1748                                 intel_pt_set_last_ip(decoder);
1749                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1750                         return intel_pt_walk_tip(decoder);
1751
1752                 case INTEL_PT_TIP_PGE: {
1753                         decoder->pge = true;
1754                         if (decoder->packet.count == 0) {
1755                                 intel_pt_log_at("Skipping zero TIP.PGE",
1756                                                 decoder->pos);
1757                                 break;
1758                         }
1759                         intel_pt_set_ip(decoder);
1760                         decoder->state.from_ip = 0;
1761                         decoder->state.to_ip = decoder->ip;
1762                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1763                         return 0;
1764                 }
1765
1766                 case INTEL_PT_OVF:
1767                         return intel_pt_overflow(decoder);
1768
1769                 case INTEL_PT_TIP:
1770                         if (decoder->packet.count != 0)
1771                                 intel_pt_set_last_ip(decoder);
1772                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1773                         return intel_pt_walk_tip(decoder);
1774
1775                 case INTEL_PT_FUP:
1776                         if (decoder->packet.count == 0) {
1777                                 intel_pt_log_at("Skipping zero FUP",
1778                                                 decoder->pos);
1779                                 no_tip = false;
1780                                 break;
1781                         }
1782                         intel_pt_set_last_ip(decoder);
1783                         if (!decoder->branch_enable) {
1784                                 decoder->ip = decoder->last_ip;
1785                                 if (intel_pt_fup_event(decoder))
1786                                         return 0;
1787                                 no_tip = false;
1788                                 break;
1789                         }
1790                         if (decoder->set_fup_mwait)
1791                                 no_tip = true;
1792                         err = intel_pt_walk_fup(decoder);
1793                         if (err != -EAGAIN) {
1794                                 if (err)
1795                                         return err;
1796                                 if (no_tip)
1797                                         decoder->pkt_state =
1798                                                 INTEL_PT_STATE_FUP_NO_TIP;
1799                                 else
1800                                         decoder->pkt_state = INTEL_PT_STATE_FUP;
1801                                 return 0;
1802                         }
1803                         if (no_tip) {
1804                                 no_tip = false;
1805                                 break;
1806                         }
1807                         return intel_pt_walk_fup_tip(decoder);
1808
1809                 case INTEL_PT_TRACESTOP:
1810                         decoder->pge = false;
1811                         decoder->continuous_period = false;
1812                         intel_pt_clear_tx_flags(decoder);
1813                         decoder->have_tma = false;
1814                         break;
1815
1816                 case INTEL_PT_PSB:
1817                         decoder->last_ip = 0;
1818                         decoder->have_last_ip = true;
1819                         intel_pt_clear_stack(&decoder->stack);
1820                         err = intel_pt_walk_psbend(decoder);
1821                         if (err == -EAGAIN)
1822                                 goto next;
1823                         if (err)
1824                                 return err;
1825                         break;
1826
1827                 case INTEL_PT_PIP:
1828                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1829                         break;
1830
1831                 case INTEL_PT_MTC:
1832                         intel_pt_calc_mtc_timestamp(decoder);
1833                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1834                                 break;
1835                         /*
1836                          * Ensure that there has been an instruction since the
1837                          * last MTC.
1838                          */
1839                         if (!decoder->mtc_insn)
1840                                 break;
1841                         decoder->mtc_insn = false;
1842                         /* Ensure that there is a timestamp */
1843                         if (!decoder->timestamp)
1844                                 break;
1845                         decoder->state.type = INTEL_PT_INSTRUCTION;
1846                         decoder->state.from_ip = decoder->ip;
1847                         decoder->state.to_ip = 0;
1848                         decoder->mtc_insn = false;
1849                         return 0;
1850
1851                 case INTEL_PT_TSC:
1852                         intel_pt_calc_tsc_timestamp(decoder);
1853                         break;
1854
1855                 case INTEL_PT_TMA:
1856                         intel_pt_calc_tma(decoder);
1857                         break;
1858
1859                 case INTEL_PT_CYC:
1860                         intel_pt_calc_cyc_timestamp(decoder);
1861                         break;
1862
1863                 case INTEL_PT_CBR:
1864                         intel_pt_calc_cbr(decoder);
1865                         if (!decoder->branch_enable &&
1866                             decoder->cbr != decoder->cbr_seen) {
1867                                 decoder->cbr_seen = decoder->cbr;
1868                                 decoder->state.type = INTEL_PT_CBR_CHG;
1869                                 decoder->state.from_ip = decoder->ip;
1870                                 decoder->state.to_ip = 0;
1871                                 decoder->state.cbr_payload =
1872                                                         decoder->packet.payload;
1873                                 return 0;
1874                         }
1875                         break;
1876
1877                 case INTEL_PT_MODE_EXEC:
1878                         decoder->exec_mode = decoder->packet.payload;
1879                         break;
1880
1881                 case INTEL_PT_MODE_TSX:
1882                         /* MODE_TSX need not be followed by FUP */
1883                         if (!decoder->pge) {
1884                                 intel_pt_update_in_tx(decoder);
1885                                 break;
1886                         }
1887                         err = intel_pt_mode_tsx(decoder, &no_tip);
1888                         if (err)
1889                                 return err;
1890                         goto next;
1891
1892                 case INTEL_PT_BAD: /* Does not happen */
1893                         return intel_pt_bug(decoder);
1894
1895                 case INTEL_PT_PSBEND:
1896                 case INTEL_PT_VMCS:
1897                 case INTEL_PT_MNT:
1898                 case INTEL_PT_PAD:
1899                         break;
1900
1901                 case INTEL_PT_PTWRITE_IP:
1902                         decoder->fup_ptw_payload = decoder->packet.payload;
1903                         err = intel_pt_get_next_packet(decoder);
1904                         if (err)
1905                                 return err;
1906                         if (decoder->packet.type == INTEL_PT_FUP) {
1907                                 decoder->set_fup_ptw = true;
1908                                 no_tip = true;
1909                         } else {
1910                                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
1911                                                 decoder->pos);
1912                         }
1913                         goto next;
1914
1915                 case INTEL_PT_PTWRITE:
1916                         decoder->state.type = INTEL_PT_PTW;
1917                         decoder->state.from_ip = decoder->ip;
1918                         decoder->state.to_ip = 0;
1919                         decoder->state.ptw_payload = decoder->packet.payload;
1920                         return 0;
1921
1922                 case INTEL_PT_MWAIT:
1923                         decoder->fup_mwait_payload = decoder->packet.payload;
1924                         decoder->set_fup_mwait = true;
1925                         break;
1926
1927                 case INTEL_PT_PWRE:
1928                         if (decoder->set_fup_mwait) {
1929                                 decoder->fup_pwre_payload =
1930                                                         decoder->packet.payload;
1931                                 decoder->set_fup_pwre = true;
1932                                 break;
1933                         }
1934                         decoder->state.type = INTEL_PT_PWR_ENTRY;
1935                         decoder->state.from_ip = decoder->ip;
1936                         decoder->state.to_ip = 0;
1937                         decoder->state.pwrx_payload = decoder->packet.payload;
1938                         return 0;
1939
1940                 case INTEL_PT_EXSTOP_IP:
1941                         err = intel_pt_get_next_packet(decoder);
1942                         if (err)
1943                                 return err;
1944                         if (decoder->packet.type == INTEL_PT_FUP) {
1945                                 decoder->set_fup_exstop = true;
1946                                 no_tip = true;
1947                         } else {
1948                                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
1949                                                 decoder->pos);
1950                         }
1951                         goto next;
1952
1953                 case INTEL_PT_EXSTOP:
1954                         decoder->state.type = INTEL_PT_EX_STOP;
1955                         decoder->state.from_ip = decoder->ip;
1956                         decoder->state.to_ip = 0;
1957                         return 0;
1958
1959                 case INTEL_PT_PWRX:
1960                         decoder->state.type = INTEL_PT_PWR_EXIT;
1961                         decoder->state.from_ip = decoder->ip;
1962                         decoder->state.to_ip = 0;
1963                         decoder->state.pwrx_payload = decoder->packet.payload;
1964                         return 0;
1965
1966                 default:
1967                         return intel_pt_bug(decoder);
1968                 }
1969         }
1970 }
1971
1972 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1973 {
1974         return decoder->packet.count &&
1975                (decoder->have_last_ip || decoder->packet.count == 3 ||
1976                 decoder->packet.count == 6);
1977 }
1978
1979 /* Walk PSB+ packets to get in sync. */
1980 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1981 {
1982         int err;
1983
1984         while (1) {
1985                 err = intel_pt_get_next_packet(decoder);
1986                 if (err)
1987                         return err;
1988
1989                 switch (decoder->packet.type) {
1990                 case INTEL_PT_TIP_PGD:
1991                         decoder->continuous_period = false;
1992                         __fallthrough;
1993                 case INTEL_PT_TIP_PGE:
1994                 case INTEL_PT_TIP:
1995                 case INTEL_PT_PTWRITE:
1996                 case INTEL_PT_PTWRITE_IP:
1997                 case INTEL_PT_EXSTOP:
1998                 case INTEL_PT_EXSTOP_IP:
1999                 case INTEL_PT_MWAIT:
2000                 case INTEL_PT_PWRE:
2001                 case INTEL_PT_PWRX:
2002                         intel_pt_log("ERROR: Unexpected packet\n");
2003                         return -ENOENT;
2004
2005                 case INTEL_PT_FUP:
2006                         decoder->pge = true;
2007                         if (intel_pt_have_ip(decoder)) {
2008                                 uint64_t current_ip = decoder->ip;
2009
2010                                 intel_pt_set_ip(decoder);
2011                                 if (current_ip)
2012                                         intel_pt_log_to("Setting IP",
2013                                                         decoder->ip);
2014                         }
2015                         break;
2016
2017                 case INTEL_PT_MTC:
2018                         intel_pt_calc_mtc_timestamp(decoder);
2019                         break;
2020
2021                 case INTEL_PT_TSC:
2022                         intel_pt_calc_tsc_timestamp(decoder);
2023                         break;
2024
2025                 case INTEL_PT_TMA:
2026                         intel_pt_calc_tma(decoder);
2027                         break;
2028
2029                 case INTEL_PT_CYC:
2030                         intel_pt_calc_cyc_timestamp(decoder);
2031                         break;
2032
2033                 case INTEL_PT_CBR:
2034                         intel_pt_calc_cbr(decoder);
2035                         break;
2036
2037                 case INTEL_PT_PIP:
2038                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2039                         break;
2040
2041                 case INTEL_PT_MODE_EXEC:
2042                         decoder->exec_mode = decoder->packet.payload;
2043                         break;
2044
2045                 case INTEL_PT_MODE_TSX:
2046                         intel_pt_update_in_tx(decoder);
2047                         break;
2048
2049                 case INTEL_PT_TRACESTOP:
2050                         decoder->pge = false;
2051                         decoder->continuous_period = false;
2052                         intel_pt_clear_tx_flags(decoder);
2053                         __fallthrough;
2054
2055                 case INTEL_PT_TNT:
2056                         decoder->have_tma = false;
2057                         intel_pt_log("ERROR: Unexpected packet\n");
2058                         if (decoder->ip)
2059                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
2060                         else
2061                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2062                         return -ENOENT;
2063
2064                 case INTEL_PT_BAD: /* Does not happen */
2065                         return intel_pt_bug(decoder);
2066
2067                 case INTEL_PT_OVF:
2068                         return intel_pt_overflow(decoder);
2069
2070                 case INTEL_PT_PSBEND:
2071                         return 0;
2072
2073                 case INTEL_PT_PSB:
2074                 case INTEL_PT_VMCS:
2075                 case INTEL_PT_MNT:
2076                 case INTEL_PT_PAD:
2077                 default:
2078                         break;
2079                 }
2080         }
2081 }
2082
2083 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
2084 {
2085         int err;
2086
2087         while (1) {
2088                 err = intel_pt_get_next_packet(decoder);
2089                 if (err)
2090                         return err;
2091
2092                 switch (decoder->packet.type) {
2093                 case INTEL_PT_TIP_PGD:
2094                         decoder->continuous_period = false;
2095                         __fallthrough;
2096                 case INTEL_PT_TIP_PGE:
2097                 case INTEL_PT_TIP:
2098                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
2099                         if (intel_pt_have_ip(decoder))
2100                                 intel_pt_set_ip(decoder);
2101                         if (!decoder->ip)
2102                                 break;
2103                         if (decoder->packet.type == INTEL_PT_TIP_PGE)
2104                                 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2105                         if (decoder->packet.type == INTEL_PT_TIP_PGD)
2106                                 decoder->state.type |= INTEL_PT_TRACE_END;
2107                         return 0;
2108
2109                 case INTEL_PT_FUP:
2110                         if (intel_pt_have_ip(decoder))
2111                                 intel_pt_set_ip(decoder);
2112                         if (decoder->ip)
2113                                 return 0;
2114                         break;
2115
2116                 case INTEL_PT_MTC:
2117                         intel_pt_calc_mtc_timestamp(decoder);
2118                         break;
2119
2120                 case INTEL_PT_TSC:
2121                         intel_pt_calc_tsc_timestamp(decoder);
2122                         break;
2123
2124                 case INTEL_PT_TMA:
2125                         intel_pt_calc_tma(decoder);
2126                         break;
2127
2128                 case INTEL_PT_CYC:
2129                         intel_pt_calc_cyc_timestamp(decoder);
2130                         break;
2131
2132                 case INTEL_PT_CBR:
2133                         intel_pt_calc_cbr(decoder);
2134                         break;
2135
2136                 case INTEL_PT_PIP:
2137                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2138                         break;
2139
2140                 case INTEL_PT_MODE_EXEC:
2141                         decoder->exec_mode = decoder->packet.payload;
2142                         break;
2143
2144                 case INTEL_PT_MODE_TSX:
2145                         intel_pt_update_in_tx(decoder);
2146                         break;
2147
2148                 case INTEL_PT_OVF:
2149                         return intel_pt_overflow(decoder);
2150
2151                 case INTEL_PT_BAD: /* Does not happen */
2152                         return intel_pt_bug(decoder);
2153
2154                 case INTEL_PT_TRACESTOP:
2155                         decoder->pge = false;
2156                         decoder->continuous_period = false;
2157                         intel_pt_clear_tx_flags(decoder);
2158                         decoder->have_tma = false;
2159                         break;
2160
2161                 case INTEL_PT_PSB:
2162                         decoder->last_ip = 0;
2163                         decoder->have_last_ip = true;
2164                         intel_pt_clear_stack(&decoder->stack);
2165                         err = intel_pt_walk_psb(decoder);
2166                         if (err)
2167                                 return err;
2168                         if (decoder->ip) {
2169                                 /* Do not have a sample */
2170                                 decoder->state.type = 0;
2171                                 return 0;
2172                         }
2173                         break;
2174
2175                 case INTEL_PT_TNT:
2176                 case INTEL_PT_PSBEND:
2177                 case INTEL_PT_VMCS:
2178                 case INTEL_PT_MNT:
2179                 case INTEL_PT_PAD:
2180                 case INTEL_PT_PTWRITE:
2181                 case INTEL_PT_PTWRITE_IP:
2182                 case INTEL_PT_EXSTOP:
2183                 case INTEL_PT_EXSTOP_IP:
2184                 case INTEL_PT_MWAIT:
2185                 case INTEL_PT_PWRE:
2186                 case INTEL_PT_PWRX:
2187                 default:
2188                         break;
2189                 }
2190         }
2191 }
2192
2193 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
2194 {
2195         int err;
2196
2197         decoder->set_fup_tx_flags = false;
2198         decoder->set_fup_ptw = false;
2199         decoder->set_fup_mwait = false;
2200         decoder->set_fup_pwre = false;
2201         decoder->set_fup_exstop = false;
2202
2203         if (!decoder->branch_enable) {
2204                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2205                 decoder->overflow = false;
2206                 decoder->state.type = 0; /* Do not have a sample */
2207                 return 0;
2208         }
2209
2210         intel_pt_log("Scanning for full IP\n");
2211         err = intel_pt_walk_to_ip(decoder);
2212         if (err)
2213                 return err;
2214
2215         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2216         decoder->overflow = false;
2217
2218         decoder->state.from_ip = 0;
2219         decoder->state.to_ip = decoder->ip;
2220         intel_pt_log_to("Setting IP", decoder->ip);
2221
2222         return 0;
2223 }
2224
2225 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
2226 {
2227         const unsigned char *end = decoder->buf + decoder->len;
2228         size_t i;
2229
2230         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
2231                 if (i > decoder->len)
2232                         continue;
2233                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
2234                         return i;
2235         }
2236         return 0;
2237 }
2238
2239 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
2240 {
2241         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
2242         const char *psb = INTEL_PT_PSB_STR;
2243
2244         if (rest_psb > decoder->len ||
2245             memcmp(decoder->buf, psb + part_psb, rest_psb))
2246                 return 0;
2247
2248         return rest_psb;
2249 }
2250
2251 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2252                                   int part_psb)
2253 {
2254         int rest_psb, ret;
2255
2256         decoder->pos += decoder->len;
2257         decoder->len = 0;
2258
2259         ret = intel_pt_get_next_data(decoder);
2260         if (ret)
2261                 return ret;
2262
2263         rest_psb = intel_pt_rest_psb(decoder, part_psb);
2264         if (!rest_psb)
2265                 return 0;
2266
2267         decoder->pos -= part_psb;
2268         decoder->next_buf = decoder->buf + rest_psb;
2269         decoder->next_len = decoder->len - rest_psb;
2270         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2271         decoder->buf = decoder->temp_buf;
2272         decoder->len = INTEL_PT_PSB_LEN;
2273
2274         return 0;
2275 }
2276
2277 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2278 {
2279         unsigned char *next;
2280         int ret;
2281
2282         intel_pt_log("Scanning for PSB\n");
2283         while (1) {
2284                 if (!decoder->len) {
2285                         ret = intel_pt_get_next_data(decoder);
2286                         if (ret)
2287                                 return ret;
2288                 }
2289
2290                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2291                               INTEL_PT_PSB_LEN);
2292                 if (!next) {
2293                         int part_psb;
2294
2295                         part_psb = intel_pt_part_psb(decoder);
2296                         if (part_psb) {
2297                                 ret = intel_pt_get_split_psb(decoder, part_psb);
2298                                 if (ret)
2299                                         return ret;
2300                         } else {
2301                                 decoder->pos += decoder->len;
2302                                 decoder->len = 0;
2303                         }
2304                         continue;
2305                 }
2306
2307                 decoder->pkt_step = next - decoder->buf;
2308                 return intel_pt_get_next_packet(decoder);
2309         }
2310 }
2311
2312 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2313 {
2314         int err;
2315
2316         decoder->pge = false;
2317         decoder->continuous_period = false;
2318         decoder->have_last_ip = false;
2319         decoder->last_ip = 0;
2320         decoder->ip = 0;
2321         intel_pt_clear_stack(&decoder->stack);
2322
2323         err = intel_pt_scan_for_psb(decoder);
2324         if (err)
2325                 return err;
2326
2327         decoder->have_last_ip = true;
2328         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2329
2330         err = intel_pt_walk_psb(decoder);
2331         if (err)
2332                 return err;
2333
2334         if (decoder->ip) {
2335                 decoder->state.type = 0; /* Do not have a sample */
2336                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2337         } else {
2338                 return intel_pt_sync_ip(decoder);
2339         }
2340
2341         return 0;
2342 }
2343
2344 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2345 {
2346         uint64_t est = decoder->sample_insn_cnt << 1;
2347
2348         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2349                 goto out;
2350
2351         est *= decoder->max_non_turbo_ratio;
2352         est /= decoder->cbr;
2353 out:
2354         return decoder->sample_timestamp + est;
2355 }
2356
2357 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2358 {
2359         int err;
2360
2361         do {
2362                 decoder->state.type = INTEL_PT_BRANCH;
2363                 decoder->state.flags = 0;
2364
2365                 switch (decoder->pkt_state) {
2366                 case INTEL_PT_STATE_NO_PSB:
2367                         err = intel_pt_sync(decoder);
2368                         break;
2369                 case INTEL_PT_STATE_NO_IP:
2370                         decoder->have_last_ip = false;
2371                         decoder->last_ip = 0;
2372                         decoder->ip = 0;
2373                         __fallthrough;
2374                 case INTEL_PT_STATE_ERR_RESYNC:
2375                         err = intel_pt_sync_ip(decoder);
2376                         break;
2377                 case INTEL_PT_STATE_IN_SYNC:
2378                         err = intel_pt_walk_trace(decoder);
2379                         break;
2380                 case INTEL_PT_STATE_TNT:
2381                 case INTEL_PT_STATE_TNT_CONT:
2382                         err = intel_pt_walk_tnt(decoder);
2383                         if (err == -EAGAIN)
2384                                 err = intel_pt_walk_trace(decoder);
2385                         break;
2386                 case INTEL_PT_STATE_TIP:
2387                 case INTEL_PT_STATE_TIP_PGD:
2388                         err = intel_pt_walk_tip(decoder);
2389                         break;
2390                 case INTEL_PT_STATE_FUP:
2391                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2392                         err = intel_pt_walk_fup(decoder);
2393                         if (err == -EAGAIN)
2394                                 err = intel_pt_walk_fup_tip(decoder);
2395                         else if (!err)
2396                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
2397                         break;
2398                 case INTEL_PT_STATE_FUP_NO_TIP:
2399                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2400                         err = intel_pt_walk_fup(decoder);
2401                         if (err == -EAGAIN)
2402                                 err = intel_pt_walk_trace(decoder);
2403                         break;
2404                 default:
2405                         err = intel_pt_bug(decoder);
2406                         break;
2407                 }
2408         } while (err == -ENOLINK);
2409
2410         if (err) {
2411                 decoder->state.err = intel_pt_ext_err(err);
2412                 decoder->state.from_ip = decoder->ip;
2413                 decoder->sample_timestamp = decoder->timestamp;
2414                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2415         } else {
2416                 decoder->state.err = 0;
2417                 if (decoder->cbr != decoder->cbr_seen && decoder->state.type) {
2418                         decoder->cbr_seen = decoder->cbr;
2419                         decoder->state.type |= INTEL_PT_CBR_CHG;
2420                         decoder->state.cbr_payload = decoder->cbr_payload;
2421                 }
2422                 if (intel_pt_sample_time(decoder->pkt_state)) {
2423                         decoder->sample_timestamp = decoder->timestamp;
2424                         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2425                 }
2426         }
2427
2428         decoder->state.timestamp = decoder->sample_timestamp;
2429         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2430         decoder->state.cr3 = decoder->cr3;
2431         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2432
2433         return &decoder->state;
2434 }
2435
2436 /**
2437  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2438  * @buf: pointer to buffer pointer
2439  * @len: size of buffer
2440  *
2441  * Updates the buffer pointer to point to the start of the next PSB packet if
2442  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2443  * @len is adjusted accordingly.
2444  *
2445  * Return: %true if a PSB packet is found, %false otherwise.
2446  */
2447 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2448 {
2449         unsigned char *next;
2450
2451         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2452         if (next) {
2453                 *len -= next - *buf;
2454                 *buf = next;
2455                 return true;
2456         }
2457         return false;
2458 }
2459
2460 /**
2461  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2462  *                     packet.
2463  * @buf: pointer to buffer pointer
2464  * @len: size of buffer
2465  *
2466  * Updates the buffer pointer to point to the start of the following PSB packet
2467  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2468  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2469  *
2470  * Return: %true if a PSB packet is found, %false otherwise.
2471  */
2472 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2473 {
2474         unsigned char *next;
2475
2476         if (!*len)
2477                 return false;
2478
2479         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2480         if (next) {
2481                 *len -= next - *buf;
2482                 *buf = next;
2483                 return true;
2484         }
2485         return false;
2486 }
2487
2488 /**
2489  * intel_pt_last_psb - find the last PSB packet in a buffer.
2490  * @buf: buffer
2491  * @len: size of buffer
2492  *
2493  * This function finds the last PSB in a buffer.
2494  *
2495  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2496  */
2497 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2498 {
2499         const char *n = INTEL_PT_PSB_STR;
2500         unsigned char *p;
2501         size_t k;
2502
2503         if (len < INTEL_PT_PSB_LEN)
2504                 return NULL;
2505
2506         k = len - INTEL_PT_PSB_LEN + 1;
2507         while (1) {
2508                 p = memrchr(buf, n[0], k);
2509                 if (!p)
2510                         return NULL;
2511                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2512                         return p;
2513                 k = p - buf;
2514                 if (!k)
2515                         return NULL;
2516         }
2517 }
2518
2519 /**
2520  * intel_pt_next_tsc - find and return next TSC.
2521  * @buf: buffer
2522  * @len: size of buffer
2523  * @tsc: TSC value returned
2524  * @rem: returns remaining size when TSC is found
2525  *
2526  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2527  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2528  * PSBEND packet is found.
2529  *
2530  * Return: %true if TSC is found, false otherwise.
2531  */
2532 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2533                               size_t *rem)
2534 {
2535         struct intel_pt_pkt packet;
2536         int ret;
2537
2538         while (len) {
2539                 ret = intel_pt_get_packet(buf, len, &packet);
2540                 if (ret <= 0)
2541                         return false;
2542                 if (packet.type == INTEL_PT_TSC) {
2543                         *tsc = packet.payload;
2544                         *rem = len;
2545                         return true;
2546                 }
2547                 if (packet.type == INTEL_PT_PSBEND)
2548                         return false;
2549                 buf += ret;
2550                 len -= ret;
2551         }
2552         return false;
2553 }
2554
2555 /**
2556  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2557  * @tsc1: first TSC to compare
2558  * @tsc2: second TSC to compare
2559  *
2560  * This function compares 7-byte TSC values allowing for the possibility that
2561  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2562  * around so for that purpose this function assumes the absolute difference is
2563  * less than half the maximum difference.
2564  *
2565  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2566  * after @tsc2.
2567  */
2568 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2569 {
2570         const uint64_t halfway = (1ULL << 55);
2571
2572         if (tsc1 == tsc2)
2573                 return 0;
2574
2575         if (tsc1 < tsc2) {
2576                 if (tsc2 - tsc1 < halfway)
2577                         return -1;
2578                 else
2579                         return 1;
2580         } else {
2581                 if (tsc1 - tsc2 < halfway)
2582                         return 1;
2583                 else
2584                         return -1;
2585         }
2586 }
2587
2588 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
2589
2590 /**
2591  * adj_for_padding - adjust overlap to account for padding.
2592  * @buf_b: second buffer
2593  * @buf_a: first buffer
2594  * @len_a: size of first buffer
2595  *
2596  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
2597  * accordingly.
2598  *
2599  * Return: A pointer into @buf_b from where non-overlapped data starts
2600  */
2601 static unsigned char *adj_for_padding(unsigned char *buf_b,
2602                                       unsigned char *buf_a, size_t len_a)
2603 {
2604         unsigned char *p = buf_b - MAX_PADDING;
2605         unsigned char *q = buf_a + len_a - MAX_PADDING;
2606         int i;
2607
2608         for (i = MAX_PADDING; i; i--, p++, q++) {
2609                 if (*p != *q)
2610                         break;
2611         }
2612
2613         return p;
2614 }
2615
2616 /**
2617  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2618  *                             using TSC.
2619  * @buf_a: first buffer
2620  * @len_a: size of first buffer
2621  * @buf_b: second buffer
2622  * @len_b: size of second buffer
2623  * @consecutive: returns true if there is data in buf_b that is consecutive
2624  *               to buf_a
2625  *
2626  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2627  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2628  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2629  * @buf_a and @buf_b are positioned at a PSB.
2630  *
2631  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2632  * @buf_b + @len_b if there is no non-overlapped data.
2633  */
2634 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2635                                                 size_t len_a,
2636                                                 unsigned char *buf_b,
2637                                                 size_t len_b, bool *consecutive)
2638 {
2639         uint64_t tsc_a, tsc_b;
2640         unsigned char *p;
2641         size_t len, rem_a, rem_b;
2642
2643         p = intel_pt_last_psb(buf_a, len_a);
2644         if (!p)
2645                 return buf_b; /* No PSB in buf_a => no overlap */
2646
2647         len = len_a - (p - buf_a);
2648         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2649                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2650                 len_a -= len;
2651                 p = intel_pt_last_psb(buf_a, len_a);
2652                 if (!p)
2653                         return buf_b; /* No full PSB+ => assume no overlap */
2654                 len = len_a - (p - buf_a);
2655                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2656                         return buf_b; /* No TSC in buf_a => assume no overlap */
2657         }
2658
2659         while (1) {
2660                 /* Ignore PSB+ with no TSC */
2661                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2662                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2663
2664                         /* Same TSC, so buffers are consecutive */
2665                         if (!cmp && rem_b >= rem_a) {
2666                                 unsigned char *start;
2667
2668                                 *consecutive = true;
2669                                 start = buf_b + len_b - (rem_b - rem_a);
2670                                 return adj_for_padding(start, buf_a, len_a);
2671                         }
2672                         if (cmp < 0)
2673                                 return buf_b; /* tsc_a < tsc_b => no overlap */
2674                 }
2675
2676                 if (!intel_pt_step_psb(&buf_b, &len_b))
2677                         return buf_b + len_b; /* No PSB in buf_b => no data */
2678         }
2679 }
2680
2681 /**
2682  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2683  * @buf_a: first buffer
2684  * @len_a: size of first buffer
2685  * @buf_b: second buffer
2686  * @len_b: size of second buffer
2687  * @have_tsc: can use TSC packets to detect overlap
2688  * @consecutive: returns true if there is data in buf_b that is consecutive
2689  *               to buf_a
2690  *
2691  * When trace samples or snapshots are recorded there is the possibility that
2692  * the data overlaps.  Note that, for the purposes of decoding, data is only
2693  * useful if it begins with a PSB packet.
2694  *
2695  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2696  * @buf_b + @len_b if there is no non-overlapped data.
2697  */
2698 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2699                                      unsigned char *buf_b, size_t len_b,
2700                                      bool have_tsc, bool *consecutive)
2701 {
2702         unsigned char *found;
2703
2704         /* Buffer 'b' must start at PSB so throw away everything before that */
2705         if (!intel_pt_next_psb(&buf_b, &len_b))
2706                 return buf_b + len_b; /* No PSB */
2707
2708         if (!intel_pt_next_psb(&buf_a, &len_a))
2709                 return buf_b; /* No overlap */
2710
2711         if (have_tsc) {
2712                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2713                                                   consecutive);
2714                 if (found)
2715                         return found;
2716         }
2717
2718         /*
2719          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2720          * we can ignore the first part of buffer 'a'.
2721          */
2722         while (len_b < len_a) {
2723                 if (!intel_pt_step_psb(&buf_a, &len_a))
2724                         return buf_b; /* No overlap */
2725         }
2726
2727         /* Now len_b >= len_a */
2728         while (1) {
2729                 /* Potential overlap so check the bytes */
2730                 found = memmem(buf_a, len_a, buf_b, len_a);
2731                 if (found) {
2732                         *consecutive = true;
2733                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
2734                 }
2735
2736                 /* Try again at next PSB in buffer 'a' */
2737                 if (!intel_pt_step_psb(&buf_a, &len_a))
2738                         return buf_b; /* No overlap */
2739         }
2740 }