tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25 #include <linux/compiler.h>
  26
  27 #include "../cache.h"
  28 #include "../util.h"
  29 #include "../auxtrace.h"
  30
  31 #include "intel-pt-insn-decoder.h"
  32 #include "intel-pt-pkt-decoder.h"
  33 #include "intel-pt-decoder.h"
  34 #include "intel-pt-log.h"
  35
  36 #define INTEL_PT_BLK_SIZE 1024
  37
  38 #define BIT63 (((uint64_t)1 << 63))
  39
  40 #define INTEL_PT_RETURN 1
  41
  42 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  43 #define INTEL_PT_MAX_LOOPS 10000
  44
  45 struct intel_pt_blk {
  46         struct intel_pt_blk *prev;
  47         uint64_t ip[INTEL_PT_BLK_SIZE];
  48 };
  49
  50 struct intel_pt_stack {
  51         struct intel_pt_blk *blk;
  52         struct intel_pt_blk *spare;
  53         int pos;
  54 };
  55
  56 enum intel_pt_pkt_state {
  57         INTEL_PT_STATE_NO_PSB,
  58         INTEL_PT_STATE_NO_IP,
  59         INTEL_PT_STATE_ERR_RESYNC,
  60         INTEL_PT_STATE_IN_SYNC,
  61         INTEL_PT_STATE_TNT_CONT,
  62         INTEL_PT_STATE_TNT,
  63         INTEL_PT_STATE_TIP,
  64         INTEL_PT_STATE_TIP_PGD,
  65         INTEL_PT_STATE_FUP,
  66         INTEL_PT_STATE_FUP_NO_TIP,
  67 };
  68
  69 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  70 {
  71         switch (pkt_state) {
  72         case INTEL_PT_STATE_NO_PSB:
  73         case INTEL_PT_STATE_NO_IP:
  74         case INTEL_PT_STATE_ERR_RESYNC:
  75         case INTEL_PT_STATE_IN_SYNC:
  76         case INTEL_PT_STATE_TNT_CONT:
  77                 return true;
  78         case INTEL_PT_STATE_TNT:
  79         case INTEL_PT_STATE_TIP:
  80         case INTEL_PT_STATE_TIP_PGD:
  81         case INTEL_PT_STATE_FUP:
  82         case INTEL_PT_STATE_FUP_NO_TIP:
  83                 return false;
  84         default:
  85                 return true;
  86         };
  87 }
  88
  89 #ifdef INTEL_PT_STRICT
  90 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  91 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  92 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  93 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  94 #else
  95 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  96 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  97 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  98 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  99 #endif
 100
 101 struct intel_pt_decoder {
 102         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 103         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 104                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 105                          uint64_t max_insn_cnt, void *data);
 106         bool (*pgd_ip)(uint64_t ip, void *data);
 107         void *data;
 108         struct intel_pt_state state;
 109         const unsigned char *buf;
 110         size_t len;
 111         bool return_compression;
 112         bool branch_enable;
 113         bool mtc_insn;
 114         bool pge;
 115         bool have_tma;
 116         bool have_cyc;
 117         bool fixup_last_mtc;
 118         bool have_last_ip;
 119         enum intel_pt_param_flags flags;
 120         uint64_t pos;
 121         uint64_t last_ip;
 122         uint64_t ip;
 123         uint64_t cr3;
 124         uint64_t timestamp;
 125         uint64_t tsc_timestamp;
 126         uint64_t ref_timestamp;
 127         uint64_t sample_timestamp;
 128         uint64_t ret_addr;
 129         uint64_t ctc_timestamp;
 130         uint64_t ctc_delta;
 131         uint64_t cycle_cnt;
 132         uint64_t cyc_ref_timestamp;
 133         uint32_t last_mtc;
 134         uint32_t tsc_ctc_ratio_n;
 135         uint32_t tsc_ctc_ratio_d;
 136         uint32_t tsc_ctc_mult;
 137         uint32_t tsc_slip;
 138         uint32_t ctc_rem_mask;
 139         int mtc_shift;
 140         struct intel_pt_stack stack;
 141         enum intel_pt_pkt_state pkt_state;
 142         struct intel_pt_pkt packet;
 143         struct intel_pt_pkt tnt;
 144         int pkt_step;
 145         int pkt_len;
 146         int last_packet_type;
 147         unsigned int cbr;
 148         unsigned int cbr_seen;
 149         unsigned int max_non_turbo_ratio;
 150         double max_non_turbo_ratio_fp;
 151         double cbr_cyc_to_tsc;
 152         double calc_cyc_to_tsc;
 153         bool have_calc_cyc_to_tsc;
 154         int exec_mode;
 155         unsigned int insn_bytes;
 156         uint64_t period;
 157         enum intel_pt_period_type period_type;
 158         uint64_t tot_insn_cnt;
 159         uint64_t period_insn_cnt;
 160         uint64_t period_mask;
 161         uint64_t period_ticks;
 162         uint64_t last_masked_timestamp;
 163         bool continuous_period;
 164         bool overflow;
 165         bool set_fup_tx_flags;
 166         bool set_fup_ptw;
 167         bool set_fup_mwait;
 168         bool set_fup_pwre;
 169         bool set_fup_exstop;
 170         unsigned int fup_tx_flags;
 171         unsigned int tx_flags;
 172         uint64_t fup_ptw_payload;
 173         uint64_t fup_mwait_payload;
 174         uint64_t fup_pwre_payload;
 175         uint64_t cbr_payload;
 176         uint64_t timestamp_insn_cnt;
 177         uint64_t sample_insn_cnt;
 178         uint64_t stuck_ip;
 179         int no_progress;
 180         int stuck_ip_prd;
 181         int stuck_ip_cnt;
 182         const unsigned char *next_buf;
 183         size_t next_len;
 184         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 185 };
 186
 187 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 188 {
 189         int i;
 190
 191         for (i = 0; x != 1; i++)
 192                 x >>= 1;
 193
 194         return x << i;
 195 }
 196
 197 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 198 {
 199         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 200                 uint64_t period;
 201
 202                 period = intel_pt_lower_power_of_2(decoder->period);
 203                 decoder->period_mask  = ~(period - 1);
 204                 decoder->period_ticks = period;
 205         }
 206 }
 207
 208 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 209 {
 210         if (!d)
 211                 return 0;
 212         return (t / d) * n + ((t % d) * n) / d;
 213 }
 214
 215 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 216 {
 217         struct intel_pt_decoder *decoder;
 218
 219         if (!params->get_trace || !params->walk_insn)
 220                 return NULL;
 221
 222         decoder = zalloc(sizeof(struct intel_pt_decoder));
 223         if (!decoder)
 224                 return NULL;
 225
 226         decoder->get_trace          = params->get_trace;
 227         decoder->walk_insn          = params->walk_insn;
 228         decoder->pgd_ip             = params->pgd_ip;
 229         decoder->data               = params->data;
 230         decoder->return_compression = params->return_compression;
 231         decoder->branch_enable      = params->branch_enable;
 232
 233         decoder->flags              = params->flags;
 234
 235         decoder->period             = params->period;
 236         decoder->period_type        = params->period_type;
 237
 238         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 239         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 240
 241         intel_pt_setup_period(decoder);
 242
 243         decoder->mtc_shift = params->mtc_period;
 244         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 245
 246         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 247         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 248
 249         if (!decoder->tsc_ctc_ratio_n)
 250                 decoder->tsc_ctc_ratio_d = 0;
 251
 252         if (decoder->tsc_ctc_ratio_d) {
 253                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 254                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 255                                                 decoder->tsc_ctc_ratio_d;
 256         }
 257
 258         /*
 259          * A TSC packet can slip past MTC packets so that the timestamp appears
 260          * to go backwards. One estimate is that can be up to about 40 CPU
 261          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 262          * slippage an order of magnitude more to be on the safe side.
 263          */
 264         decoder->tsc_slip = 0x10000;
 265
 266         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 267         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 268         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 269         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 270         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 271
 272         return decoder;
 273 }
 274
 275 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 276 {
 277         struct intel_pt_blk *blk = stack->blk;
 278
 279         stack->blk = blk->prev;
 280         if (!stack->spare)
 281                 stack->spare = blk;
 282         else
 283                 free(blk);
 284 }
 285
 286 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 287 {
 288         if (!stack->pos) {
 289                 if (!stack->blk)
 290                         return 0;
 291                 intel_pt_pop_blk(stack);
 292                 if (!stack->blk)
 293                         return 0;
 294                 stack->pos = INTEL_PT_BLK_SIZE;
 295         }
 296         return stack->blk->ip[--stack->pos];
 297 }
 298
 299 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 300 {
 301         struct intel_pt_blk *blk;
 302
 303         if (stack->spare) {
 304                 blk = stack->spare;
 305                 stack->spare = NULL;
 306         } else {
 307                 blk = malloc(sizeof(struct intel_pt_blk));
 308                 if (!blk)
 309                         return -ENOMEM;
 310         }
 311
 312         blk->prev = stack->blk;
 313         stack->blk = blk;
 314         stack->pos = 0;
 315         return 0;
 316 }
 317
 318 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 319 {
 320         int err;
 321
 322         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 323                 err = intel_pt_alloc_blk(stack);
 324                 if (err)
 325                         return err;
 326         }
 327
 328         stack->blk->ip[stack->pos++] = ip;
 329         return 0;
 330 }
 331
 332 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 333 {
 334         while (stack->blk)
 335                 intel_pt_pop_blk(stack);
 336         stack->pos = 0;
 337 }
 338
 339 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 340 {
 341         intel_pt_clear_stack(stack);
 342         zfree(&stack->blk);
 343         zfree(&stack->spare);
 344 }
 345
 346 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 347 {
 348         intel_pt_free_stack(&decoder->stack);
 349         free(decoder);
 350 }
 351
 352 static int intel_pt_ext_err(int code)
 353 {
 354         switch (code) {
 355         case -ENOMEM:
 356                 return INTEL_PT_ERR_NOMEM;
 357         case -ENOSYS:
 358                 return INTEL_PT_ERR_INTERN;
 359         case -EBADMSG:
 360                 return INTEL_PT_ERR_BADPKT;
 361         case -ENODATA:
 362                 return INTEL_PT_ERR_NODATA;
 363         case -EILSEQ:
 364                 return INTEL_PT_ERR_NOINSN;
 365         case -ENOENT:
 366                 return INTEL_PT_ERR_MISMAT;
 367         case -EOVERFLOW:
 368                 return INTEL_PT_ERR_OVR;
 369         case -ENOSPC:
 370                 return INTEL_PT_ERR_LOST;
 371         case -ELOOP:
 372                 return INTEL_PT_ERR_NELOOP;
 373         default:
 374                 return INTEL_PT_ERR_UNK;
 375         }
 376 }
 377
 378 static const char *intel_pt_err_msgs[] = {
 379         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 380         [INTEL_PT_ERR_INTERN] = "Internal error",
 381         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 382         [INTEL_PT_ERR_NODATA] = "No more data",
 383         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 384         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 385         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 386         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 387         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 388         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 389 };
 390
 391 int intel_pt__strerror(int code, char *buf, size_t buflen)
 392 {
 393         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 394                 code = INTEL_PT_ERR_UNK;
 395         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 396         return 0;
 397 }
 398
 399 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 400                                  uint64_t last_ip)
 401 {
 402         uint64_t ip;
 403
 404         switch (packet->count) {
 405         case 1:
 406                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 407                      packet->payload;
 408                 break;
 409         case 2:
 410                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 411                      packet->payload;
 412                 break;
 413         case 3:
 414                 ip = packet->payload;
 415                 /* Sign-extend 6-byte ip */
 416                 if (ip & (uint64_t)0x800000000000ULL)
 417                         ip |= (uint64_t)0xffff000000000000ULL;
 418                 break;
 419         case 4:
 420                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 421                      packet->payload;
 422                 break;
 423         case 6:
 424                 ip = packet->payload;
 425                 break;
 426         default:
 427                 return 0;
 428         }
 429
 430         return ip;
 431 }
 432
 433 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 434 {
 435         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 436         decoder->have_last_ip = true;
 437 }
 438
 439 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 440 {
 441         intel_pt_set_last_ip(decoder);
 442         decoder->ip = decoder->last_ip;
 443 }
 444
 445 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 446 {
 447         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 448                             decoder->buf);
 449 }
 450
 451 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 452 {
 453         intel_pt_log("ERROR: Internal error\n");
 454         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 455         return -ENOSYS;
 456 }
 457
 458 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 459 {
 460         decoder->tx_flags = 0;
 461 }
 462
 463 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 464 {
 465         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 466 }
 467
 468 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 469 {
 470         intel_pt_clear_tx_flags(decoder);
 471         decoder->have_tma = false;
 472         decoder->pkt_len = 1;
 473         decoder->pkt_step = 1;
 474         intel_pt_decoder_log_packet(decoder);
 475         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 476                 intel_pt_log("ERROR: Bad packet\n");
 477                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 478         }
 479         return -EBADMSG;
 480 }
 481
 482 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 483 {
 484         struct intel_pt_buffer buffer = { .buf = 0, };
 485         int ret;
 486
 487         decoder->pkt_step = 0;
 488
 489         intel_pt_log("Getting more data\n");
 490         ret = decoder->get_trace(&buffer, decoder->data);
 491         if (ret)
 492                 return ret;
 493         decoder->buf = buffer.buf;
 494         decoder->len = buffer.len;
 495         if (!decoder->len) {
 496                 intel_pt_log("No more data\n");
 497                 return -ENODATA;
 498         }
 499         if (!buffer.consecutive) {
 500                 decoder->ip = 0;
 501                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 502                 decoder->ref_timestamp = buffer.ref_timestamp;
 503                 decoder->timestamp = 0;
 504                 decoder->have_tma = false;
 505                 decoder->state.trace_nr = buffer.trace_nr;
 506                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 507                              decoder->ref_timestamp);
 508                 return -ENOLINK;
 509         }
 510
 511         return 0;
 512 }
 513
 514 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 515 {
 516         if (!decoder->next_buf)
 517                 return intel_pt_get_data(decoder);
 518
 519         decoder->buf = decoder->next_buf;
 520         decoder->len = decoder->next_len;
 521         decoder->next_buf = 0;
 522         decoder->next_len = 0;
 523         return 0;
 524 }
 525
 526 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 527 {
 528         unsigned char *buf = decoder->temp_buf;
 529         size_t old_len, len, n;
 530         int ret;
 531
 532         old_len = decoder->len;
 533         len = decoder->len;
 534         memcpy(buf, decoder->buf, len);
 535
 536         ret = intel_pt_get_data(decoder);
 537         if (ret) {
 538                 decoder->pos += old_len;
 539                 return ret < 0 ? ret : -EINVAL;
 540         }
 541
 542         n = INTEL_PT_PKT_MAX_SZ - len;
 543         if (n > decoder->len)
 544                 n = decoder->len;
 545         memcpy(buf + len, decoder->buf, n);
 546         len += n;
 547
 548         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 549         if (ret < (int)old_len) {
 550                 decoder->next_buf = decoder->buf;
 551                 decoder->next_len = decoder->len;
 552                 decoder->buf = buf;
 553                 decoder->len = old_len;
 554                 return intel_pt_bad_packet(decoder);
 555         }
 556
 557         decoder->next_buf = decoder->buf + (ret - old_len);
 558         decoder->next_len = decoder->len - (ret - old_len);
 559
 560         decoder->buf = buf;
 561         decoder->len = ret;
 562
 563         return ret;
 564 }
 565
 566 struct intel_pt_pkt_info {
 567         struct intel_pt_decoder   *decoder;
 568         struct intel_pt_pkt       packet;
 569         uint64_t                  pos;
 570         int                       pkt_len;
 571         int                       last_packet_type;
 572         void                      *data;
 573 };
 574
 575 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 576
 577 /* Lookahead packets in current buffer */
 578 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 579                                   intel_pt_pkt_cb_t cb, void *data)
 580 {
 581         struct intel_pt_pkt_info pkt_info;
 582         const unsigned char *buf = decoder->buf;
 583         size_t len = decoder->len;
 584         int ret;
 585
 586         pkt_info.decoder          = decoder;
 587         pkt_info.pos              = decoder->pos;
 588         pkt_info.pkt_len          = decoder->pkt_step;
 589         pkt_info.last_packet_type = decoder->last_packet_type;
 590         pkt_info.data             = data;
 591
 592         while (1) {
 593                 do {
 594                         pkt_info.pos += pkt_info.pkt_len;
 595                         buf          += pkt_info.pkt_len;
 596                         len          -= pkt_info.pkt_len;
 597
 598                         if (!len)
 599                                 return INTEL_PT_NEED_MORE_BYTES;
 600
 601                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 602                         if (!ret)
 603                                 return INTEL_PT_NEED_MORE_BYTES;
 604                         if (ret < 0)
 605                                 return ret;
 606
 607                         pkt_info.pkt_len = ret;
 608                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 609
 610                 ret = cb(&pkt_info);
 611                 if (ret)
 612                         return 0;
 613
 614                 pkt_info.last_packet_type = pkt_info.packet.type;
 615         }
 616 }
 617
 618 struct intel_pt_calc_cyc_to_tsc_info {
 619         uint64_t        cycle_cnt;
 620         unsigned int    cbr;
 621         uint32_t        last_mtc;
 622         uint64_t        ctc_timestamp;
 623         uint64_t        ctc_delta;
 624         uint64_t        tsc_timestamp;
 625         uint64_t        timestamp;
 626         bool            have_tma;
 627         bool            fixup_last_mtc;
 628         bool            from_mtc;
 629         double          cbr_cyc_to_tsc;
 630 };
 631
 632 /*
 633  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 634  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 635  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 636  * packet by copying the missing bits from the current MTC assuming the least
 637  * difference between the two, and that the current MTC comes after last_mtc.
 638  */
 639 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 640                                     uint32_t *last_mtc)
 641 {
 642         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 643         uint32_t mask = ~(first_missing_bit - 1);
 644
 645         *last_mtc |= mtc & mask;
 646         if (*last_mtc >= mtc) {
 647                 *last_mtc -= first_missing_bit;
 648                 *last_mtc &= 0xff;
 649         }
 650 }
 651
 652 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 653 {
 654         struct intel_pt_decoder *decoder = pkt_info->decoder;
 655         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 656         uint64_t timestamp;
 657         double cyc_to_tsc;
 658         unsigned int cbr;
 659         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 660
 661         switch (pkt_info->packet.type) {
 662         case INTEL_PT_TNT:
 663         case INTEL_PT_TIP_PGE:
 664         case INTEL_PT_TIP:
 665         case INTEL_PT_FUP:
 666         case INTEL_PT_PSB:
 667         case INTEL_PT_PIP:
 668         case INTEL_PT_MODE_EXEC:
 669         case INTEL_PT_MODE_TSX:
 670         case INTEL_PT_PSBEND:
 671         case INTEL_PT_PAD:
 672         case INTEL_PT_VMCS:
 673         case INTEL_PT_MNT:
 674         case INTEL_PT_PTWRITE:
 675         case INTEL_PT_PTWRITE_IP:
 676                 return 0;
 677
 678         case INTEL_PT_MTC:
 679                 if (!data->have_tma)
 680                         return 0;
 681
 682                 mtc = pkt_info->packet.payload;
 683                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 684                         data->fixup_last_mtc = false;
 685                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 686                                                 &data->last_mtc);
 687                 }
 688                 if (mtc > data->last_mtc)
 689                         mtc_delta = mtc - data->last_mtc;
 690                 else
 691                         mtc_delta = mtc + 256 - data->last_mtc;
 692                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 693                 data->last_mtc = mtc;
 694
 695                 if (decoder->tsc_ctc_mult) {
 696                         timestamp = data->ctc_timestamp +
 697                                 data->ctc_delta * decoder->tsc_ctc_mult;
 698                 } else {
 699                         timestamp = data->ctc_timestamp +
 700                                 multdiv(data->ctc_delta,
 701                                         decoder->tsc_ctc_ratio_n,
 702                                         decoder->tsc_ctc_ratio_d);
 703                 }
 704
 705                 if (timestamp < data->timestamp)
 706                         return 1;
 707
 708                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 709                         data->timestamp = timestamp;
 710                         return 0;
 711                 }
 712
 713                 break;
 714
 715         case INTEL_PT_TSC:
 716                 /*
 717                  * For now, do not support using TSC packets - refer
 718                  * intel_pt_calc_cyc_to_tsc().
 719                  */
 720                 if (data->from_mtc)
 721                         return 1;
 722                 timestamp = pkt_info->packet.payload |
 723                             (data->timestamp & (0xffULL << 56));
 724                 if (data->from_mtc && timestamp < data->timestamp &&
 725                     data->timestamp - timestamp < decoder->tsc_slip)
 726                         return 1;
 727                 if (timestamp < data->timestamp)
 728                         timestamp += (1ULL << 56);
 729                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 730                         if (data->from_mtc)
 731                                 return 1;
 732                         data->tsc_timestamp = timestamp;
 733                         data->timestamp = timestamp;
 734                         return 0;
 735                 }
 736                 break;
 737
 738         case INTEL_PT_TMA:
 739                 if (data->from_mtc)
 740                         return 1;
 741
 742                 if (!decoder->tsc_ctc_ratio_d)
 743                         return 0;
 744
 745                 ctc = pkt_info->packet.payload;
 746                 fc = pkt_info->packet.count;
 747                 ctc_rem = ctc & decoder->ctc_rem_mask;
 748
 749                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 750
 751                 data->ctc_timestamp = data->tsc_timestamp - fc;
 752                 if (decoder->tsc_ctc_mult) {
 753                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 754                 } else {
 755                         data->ctc_timestamp -=
 756                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 757                                         decoder->tsc_ctc_ratio_d);
 758                 }
 759
 760                 data->ctc_delta = 0;
 761                 data->have_tma = true;
 762                 data->fixup_last_mtc = true;
 763
 764                 return 0;
 765
 766         case INTEL_PT_CYC:
 767                 data->cycle_cnt += pkt_info->packet.payload;
 768                 return 0;
 769
 770         case INTEL_PT_CBR:
 771                 cbr = pkt_info->packet.payload;
 772                 if (data->cbr && data->cbr != cbr)
 773                         return 1;
 774                 data->cbr = cbr;
 775                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 776                 return 0;
 777
 778         case INTEL_PT_TIP_PGD:
 779         case INTEL_PT_TRACESTOP:
 780         case INTEL_PT_EXSTOP:
 781         case INTEL_PT_EXSTOP_IP:
 782         case INTEL_PT_MWAIT:
 783         case INTEL_PT_PWRE:
 784         case INTEL_PT_PWRX:
 785         case INTEL_PT_OVF:
 786         case INTEL_PT_BAD: /* Does not happen */
 787         default:
 788                 return 1;
 789         }
 790
 791         if (!data->cbr && decoder->cbr) {
 792                 data->cbr = decoder->cbr;
 793                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 794         }
 795
 796         if (!data->cycle_cnt)
 797                 return 1;
 798
 799         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 800
 801         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 802             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 803                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 804                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 805                 return 1;
 806         }
 807
 808         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 809         decoder->have_calc_cyc_to_tsc = true;
 810
 811         if (data->cbr) {
 812                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 813                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 814         } else {
 815                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 816                              cyc_to_tsc, pkt_info->pos);
 817         }
 818
 819         return 1;
 820 }
 821
 822 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 823                                      bool from_mtc)
 824 {
 825         struct intel_pt_calc_cyc_to_tsc_info data = {
 826                 .cycle_cnt      = 0,
 827                 .cbr            = 0,
 828                 .last_mtc       = decoder->last_mtc,
 829                 .ctc_timestamp  = decoder->ctc_timestamp,
 830                 .ctc_delta      = decoder->ctc_delta,
 831                 .tsc_timestamp  = decoder->tsc_timestamp,
 832                 .timestamp      = decoder->timestamp,
 833                 .have_tma       = decoder->have_tma,
 834                 .fixup_last_mtc = decoder->fixup_last_mtc,
 835                 .from_mtc       = from_mtc,
 836                 .cbr_cyc_to_tsc = 0,
 837         };
 838
 839         /*
 840          * For now, do not support using TSC packets for at least the reasons:
 841          * 1) timing might have stopped
 842          * 2) TSC packets within PSB+ can slip against CYC packets
 843          */
 844         if (!from_mtc)
 845                 return;
 846
 847         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 848 }
 849
 850 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 851 {
 852         int ret;
 853
 854         decoder->last_packet_type = decoder->packet.type;
 855
 856         do {
 857                 decoder->pos += decoder->pkt_step;
 858                 decoder->buf += decoder->pkt_step;
 859                 decoder->len -= decoder->pkt_step;
 860
 861                 if (!decoder->len) {
 862                         ret = intel_pt_get_next_data(decoder);
 863                         if (ret)
 864                                 return ret;
 865                 }
 866
 867                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 868                                           &decoder->packet);
 869                 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
 870                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 871                         ret = intel_pt_get_split_packet(decoder);
 872                         if (ret < 0)
 873                                 return ret;
 874                 }
 875                 if (ret <= 0)
 876                         return intel_pt_bad_packet(decoder);
 877
 878                 decoder->pkt_len = ret;
 879                 decoder->pkt_step = ret;
 880                 intel_pt_decoder_log_packet(decoder);
 881         } while (decoder->packet.type == INTEL_PT_PAD);
 882
 883         return 0;
 884 }
 885
 886 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 887 {
 888         uint64_t timestamp, masked_timestamp;
 889
 890         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 891         masked_timestamp = timestamp & decoder->period_mask;
 892         if (decoder->continuous_period) {
 893                 if (masked_timestamp > decoder->last_masked_timestamp)
 894                         return 1;
 895         } else {
 896                 timestamp += 1;
 897                 masked_timestamp = timestamp & decoder->period_mask;
 898                 if (masked_timestamp > decoder->last_masked_timestamp) {
 899                         decoder->last_masked_timestamp = masked_timestamp;
 900                         decoder->continuous_period = true;
 901                 }
 902         }
 903
 904         if (masked_timestamp < decoder->last_masked_timestamp)
 905                 return decoder->period_ticks;
 906
 907         return decoder->period_ticks - (timestamp - masked_timestamp);
 908 }
 909
 910 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 911 {
 912         switch (decoder->period_type) {
 913         case INTEL_PT_PERIOD_INSTRUCTIONS:
 914                 return decoder->period - decoder->period_insn_cnt;
 915         case INTEL_PT_PERIOD_TICKS:
 916                 return intel_pt_next_period(decoder);
 917         case INTEL_PT_PERIOD_NONE:
 918         case INTEL_PT_PERIOD_MTC:
 919         default:
 920                 return 0;
 921         }
 922 }
 923
 924 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 925 {
 926         uint64_t timestamp, masked_timestamp;
 927
 928         switch (decoder->period_type) {
 929         case INTEL_PT_PERIOD_INSTRUCTIONS:
 930                 decoder->period_insn_cnt = 0;
 931                 break;
 932         case INTEL_PT_PERIOD_TICKS:
 933                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 934                 masked_timestamp = timestamp & decoder->period_mask;
 935                 if (masked_timestamp > decoder->last_masked_timestamp)
 936                         decoder->last_masked_timestamp = masked_timestamp;
 937                 else
 938                         decoder->last_masked_timestamp += decoder->period_ticks;
 939                 break;
 940         case INTEL_PT_PERIOD_NONE:
 941         case INTEL_PT_PERIOD_MTC:
 942         default:
 943                 break;
 944         }
 945
 946         decoder->state.type |= INTEL_PT_INSTRUCTION;
 947 }
 948
 949 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 950                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 951 {
 952         uint64_t max_insn_cnt, insn_cnt = 0;
 953         int err;
 954
 955         if (!decoder->mtc_insn)
 956                 decoder->mtc_insn = true;
 957
 958         max_insn_cnt = intel_pt_next_sample(decoder);
 959
 960         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 961                                  max_insn_cnt, decoder->data);
 962
 963         decoder->tot_insn_cnt += insn_cnt;
 964         decoder->timestamp_insn_cnt += insn_cnt;
 965         decoder->sample_insn_cnt += insn_cnt;
 966         decoder->period_insn_cnt += insn_cnt;
 967
 968         if (err) {
 969                 decoder->no_progress = 0;
 970                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 971                 intel_pt_log_at("ERROR: Failed to get instruction",
 972                                 decoder->ip);
 973                 if (err == -ENOENT)
 974                         return -ENOLINK;
 975                 return -EILSEQ;
 976         }
 977
 978         if (ip && decoder->ip == ip) {
 979                 err = -EAGAIN;
 980                 goto out;
 981         }
 982
 983         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 984                 intel_pt_sample_insn(decoder);
 985
 986         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 987                 decoder->state.type = INTEL_PT_INSTRUCTION;
 988                 decoder->state.from_ip = decoder->ip;
 989                 decoder->state.to_ip = 0;
 990                 decoder->ip += intel_pt_insn->length;
 991                 err = INTEL_PT_RETURN;
 992                 goto out;
 993         }
 994
 995         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 996                 /* Zero-length calls are excluded */
 997                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 998                     intel_pt_insn->rel) {
 999                         err = intel_pt_push(&decoder->stack, decoder->ip +
1000                                             intel_pt_insn->length);
1001                         if (err)
1002                                 goto out;
1003                 }
1004         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1005                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1006         }
1007
1008         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1009                 int cnt = decoder->no_progress++;
1010
1011                 decoder->state.from_ip = decoder->ip;
1012                 decoder->ip += intel_pt_insn->length +
1013                                 intel_pt_insn->rel;
1014                 decoder->state.to_ip = decoder->ip;
1015                 err = INTEL_PT_RETURN;
1016
1017                 /*
1018                  * Check for being stuck in a loop.  This can happen if a
1019                  * decoder error results in the decoder erroneously setting the
1020                  * ip to an address that is itself in an infinite loop that
1021                  * consumes no packets.  When that happens, there must be an
1022                  * unconditional branch.
1023                  */
1024                 if (cnt) {
1025                         if (cnt == 1) {
1026                                 decoder->stuck_ip = decoder->state.to_ip;
1027                                 decoder->stuck_ip_prd = 1;
1028                                 decoder->stuck_ip_cnt = 1;
1029                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
1030                                    decoder->state.to_ip == decoder->stuck_ip) {
1031                                 intel_pt_log_at("ERROR: Never-ending loop",
1032                                                 decoder->state.to_ip);
1033                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1034                                 err = -ELOOP;
1035                                 goto out;
1036                         } else if (!--decoder->stuck_ip_cnt) {
1037                                 decoder->stuck_ip_prd += 1;
1038                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1039                                 decoder->stuck_ip = decoder->state.to_ip;
1040                         }
1041                 }
1042                 goto out_no_progress;
1043         }
1044 out:
1045         decoder->no_progress = 0;
1046 out_no_progress:
1047         decoder->state.insn_op = intel_pt_insn->op;
1048         decoder->state.insn_len = intel_pt_insn->length;
1049         memcpy(decoder->state.insn, intel_pt_insn->buf,
1050                INTEL_PT_INSN_BUF_SZ);
1051
1052         if (decoder->tx_flags & INTEL_PT_IN_TX)
1053                 decoder->state.flags |= INTEL_PT_IN_TX;
1054
1055         return err;
1056 }
1057
1058 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1059 {
1060         bool ret = false;
1061
1062         if (decoder->set_fup_tx_flags) {
1063                 decoder->set_fup_tx_flags = false;
1064                 decoder->tx_flags = decoder->fup_tx_flags;
1065                 decoder->state.type = INTEL_PT_TRANSACTION;
1066                 decoder->state.from_ip = decoder->ip;
1067                 decoder->state.to_ip = 0;
1068                 decoder->state.flags = decoder->fup_tx_flags;
1069                 return true;
1070         }
1071         if (decoder->set_fup_ptw) {
1072                 decoder->set_fup_ptw = false;
1073                 decoder->state.type = INTEL_PT_PTW;
1074                 decoder->state.flags |= INTEL_PT_FUP_IP;
1075                 decoder->state.from_ip = decoder->ip;
1076                 decoder->state.to_ip = 0;
1077                 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1078                 return true;
1079         }
1080         if (decoder->set_fup_mwait) {
1081                 decoder->set_fup_mwait = false;
1082                 decoder->state.type = INTEL_PT_MWAIT_OP;
1083                 decoder->state.from_ip = decoder->ip;
1084                 decoder->state.to_ip = 0;
1085                 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1086                 ret = true;
1087         }
1088         if (decoder->set_fup_pwre) {
1089                 decoder->set_fup_pwre = false;
1090                 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1091                 decoder->state.type &= ~INTEL_PT_BRANCH;
1092                 decoder->state.from_ip = decoder->ip;
1093                 decoder->state.to_ip = 0;
1094                 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1095                 ret = true;
1096         }
1097         if (decoder->set_fup_exstop) {
1098                 decoder->set_fup_exstop = false;
1099                 decoder->state.type |= INTEL_PT_EX_STOP;
1100                 decoder->state.type &= ~INTEL_PT_BRANCH;
1101                 decoder->state.flags |= INTEL_PT_FUP_IP;
1102                 decoder->state.from_ip = decoder->ip;
1103                 decoder->state.to_ip = 0;
1104                 ret = true;
1105         }
1106         return ret;
1107 }
1108
1109 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1110                                           struct intel_pt_insn *intel_pt_insn,
1111                                           uint64_t ip, int err)
1112 {
1113         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1114                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1115                ip == decoder->ip + intel_pt_insn->length;
1116 }
1117
1118 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1119 {
1120         struct intel_pt_insn intel_pt_insn;
1121         uint64_t ip;
1122         int err;
1123
1124         ip = decoder->last_ip;
1125
1126         while (1) {
1127                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1128                 if (err == INTEL_PT_RETURN)
1129                         return 0;
1130                 if (err == -EAGAIN ||
1131                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1132                         if (intel_pt_fup_event(decoder))
1133                                 return 0;
1134                         return -EAGAIN;
1135                 }
1136                 decoder->set_fup_tx_flags = false;
1137                 if (err)
1138                         return err;
1139
1140                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1141                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1142                                         decoder->ip);
1143                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1144                         return -ENOENT;
1145                 }
1146
1147                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1148                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1149                                         decoder->ip);
1150                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1151                         return -ENOENT;
1152                 }
1153
1154                 intel_pt_bug(decoder);
1155         }
1156 }
1157
1158 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1159 {
1160         struct intel_pt_insn intel_pt_insn;
1161         int err;
1162
1163         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1164         if (err == INTEL_PT_RETURN &&
1165             decoder->pgd_ip &&
1166             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1167             (decoder->state.type & INTEL_PT_BRANCH) &&
1168             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1169                 /* Unconditional branch leaving filter region */
1170                 decoder->no_progress = 0;
1171                 decoder->pge = false;
1172                 decoder->continuous_period = false;
1173                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1174                 decoder->state.type |= INTEL_PT_TRACE_END;
1175                 return 0;
1176         }
1177         if (err == INTEL_PT_RETURN)
1178                 return 0;
1179         if (err)
1180                 return err;
1181
1182         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1183                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1184                         decoder->pge = false;
1185                         decoder->continuous_period = false;
1186                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1187                         decoder->state.from_ip = decoder->ip;
1188                         if (decoder->packet.count == 0) {
1189                                 decoder->state.to_ip = 0;
1190                         } else {
1191                                 decoder->state.to_ip = decoder->last_ip;
1192                                 decoder->ip = decoder->last_ip;
1193                         }
1194                         decoder->state.type |= INTEL_PT_TRACE_END;
1195                 } else {
1196                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1197                         decoder->state.from_ip = decoder->ip;
1198                         if (decoder->packet.count == 0) {
1199                                 decoder->state.to_ip = 0;
1200                         } else {
1201                                 decoder->state.to_ip = decoder->last_ip;
1202                                 decoder->ip = decoder->last_ip;
1203                         }
1204                 }
1205                 return 0;
1206         }
1207
1208         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1209                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1210                                  intel_pt_insn.rel;
1211
1212                 if (decoder->pgd_ip &&
1213                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1214                     decoder->pgd_ip(to_ip, decoder->data)) {
1215                         /* Conditional branch leaving filter region */
1216                         decoder->pge = false;
1217                         decoder->continuous_period = false;
1218                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1219                         decoder->ip = to_ip;
1220                         decoder->state.from_ip = decoder->ip;
1221                         decoder->state.to_ip = to_ip;
1222                         decoder->state.type |= INTEL_PT_TRACE_END;
1223                         return 0;
1224                 }
1225                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1226                                 decoder->ip);
1227                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1228                 return -ENOENT;
1229         }
1230
1231         return intel_pt_bug(decoder);
1232 }
1233
1234 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1235 {
1236         struct intel_pt_insn intel_pt_insn;
1237         int err;
1238
1239         while (1) {
1240                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1241                 if (err == INTEL_PT_RETURN)
1242                         return 0;
1243                 if (err)
1244                         return err;
1245
1246                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1247                         if (!decoder->return_compression) {
1248                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1249                                                 decoder->ip);
1250                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1251                                 return -ENOENT;
1252                         }
1253                         if (!decoder->ret_addr) {
1254                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1255                                                 decoder->ip);
1256                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1257                                 return -ENOENT;
1258                         }
1259                         if (!(decoder->tnt.payload & BIT63)) {
1260                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1261                                                 decoder->ip);
1262                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1263                                 return -ENOENT;
1264                         }
1265                         decoder->tnt.count -= 1;
1266                         if (decoder->tnt.count)
1267                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1268                         else
1269                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1270                         decoder->tnt.payload <<= 1;
1271                         decoder->state.from_ip = decoder->ip;
1272                         decoder->ip = decoder->ret_addr;
1273                         decoder->state.to_ip = decoder->ip;
1274                         return 0;
1275                 }
1276
1277                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1278                         /* Handle deferred TIPs */
1279                         err = intel_pt_get_next_packet(decoder);
1280                         if (err)
1281                                 return err;
1282                         if (decoder->packet.type != INTEL_PT_TIP ||
1283                             decoder->packet.count == 0) {
1284                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1285                                                 decoder->ip);
1286                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1287                                 decoder->pkt_step = 0;
1288                                 return -ENOENT;
1289                         }
1290                         intel_pt_set_last_ip(decoder);
1291                         decoder->state.from_ip = decoder->ip;
1292                         decoder->state.to_ip = decoder->last_ip;
1293                         decoder->ip = decoder->last_ip;
1294                         return 0;
1295                 }
1296
1297                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1298                         decoder->tnt.count -= 1;
1299                         if (decoder->tnt.count)
1300                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1301                         else
1302                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1303                         if (decoder->tnt.payload & BIT63) {
1304                                 decoder->tnt.payload <<= 1;
1305                                 decoder->state.from_ip = decoder->ip;
1306                                 decoder->ip += intel_pt_insn.length +
1307                                                intel_pt_insn.rel;
1308                                 decoder->state.to_ip = decoder->ip;
1309                                 return 0;
1310                         }
1311                         /* Instruction sample for a non-taken branch */
1312                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1313                                 decoder->tnt.payload <<= 1;
1314                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1315                                 decoder->state.from_ip = decoder->ip;
1316                                 decoder->state.to_ip = 0;
1317                                 decoder->ip += intel_pt_insn.length;
1318                                 return 0;
1319                         }
1320                         decoder->ip += intel_pt_insn.length;
1321                         if (!decoder->tnt.count) {
1322                                 decoder->sample_timestamp = decoder->timestamp;
1323                                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
1324                                 return -EAGAIN;
1325                         }
1326                         decoder->tnt.payload <<= 1;
1327                         continue;
1328                 }
1329
1330                 return intel_pt_bug(decoder);
1331         }
1332 }
1333
1334 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1335 {
1336         unsigned int fup_tx_flags;
1337         int err;
1338
1339         fup_tx_flags = decoder->packet.payload &
1340                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1341         err = intel_pt_get_next_packet(decoder);
1342         if (err)
1343                 return err;
1344         if (decoder->packet.type == INTEL_PT_FUP) {
1345                 decoder->fup_tx_flags = fup_tx_flags;
1346                 decoder->set_fup_tx_flags = true;
1347                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1348                         *no_tip = true;
1349         } else {
1350                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1351                                 decoder->pos);
1352                 intel_pt_update_in_tx(decoder);
1353         }
1354         return 0;
1355 }
1356
1357 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1358 {
1359         uint64_t timestamp;
1360
1361         decoder->have_tma = false;
1362
1363         if (decoder->ref_timestamp) {
1364                 timestamp = decoder->packet.payload |
1365                             (decoder->ref_timestamp & (0xffULL << 56));
1366                 if (timestamp < decoder->ref_timestamp) {
1367                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1368                                 timestamp += (1ULL << 56);
1369                 } else {
1370                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1371                                 timestamp -= (1ULL << 56);
1372                 }
1373                 decoder->tsc_timestamp = timestamp;
1374                 decoder->timestamp = timestamp;
1375                 decoder->ref_timestamp = 0;
1376                 decoder->timestamp_insn_cnt = 0;
1377         } else if (decoder->timestamp) {
1378                 timestamp = decoder->packet.payload |
1379                             (decoder->timestamp & (0xffULL << 56));
1380                 decoder->tsc_timestamp = timestamp;
1381                 if (timestamp < decoder->timestamp &&
1382                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1383                         intel_pt_log_to("Suppressing backwards timestamp",
1384                                         timestamp);
1385                         timestamp = decoder->timestamp;
1386                 }
1387                 if (timestamp < decoder->timestamp) {
1388                         intel_pt_log_to("Wraparound timestamp", timestamp);
1389                         timestamp += (1ULL << 56);
1390                         decoder->tsc_timestamp = timestamp;
1391                 }
1392                 decoder->timestamp = timestamp;
1393                 decoder->timestamp_insn_cnt = 0;
1394         }
1395
1396         if (decoder->last_packet_type == INTEL_PT_CYC) {
1397                 decoder->cyc_ref_timestamp = decoder->timestamp;
1398                 decoder->cycle_cnt = 0;
1399                 decoder->have_calc_cyc_to_tsc = false;
1400                 intel_pt_calc_cyc_to_tsc(decoder, false);
1401         }
1402
1403         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1404 }
1405
1406 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1407 {
1408         intel_pt_log("ERROR: Buffer overflow\n");
1409         intel_pt_clear_tx_flags(decoder);
1410         decoder->timestamp_insn_cnt = 0;
1411         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1412         decoder->overflow = true;
1413         return -EOVERFLOW;
1414 }
1415
1416 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1417 {
1418         uint32_t ctc = decoder->packet.payload;
1419         uint32_t fc = decoder->packet.count;
1420         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1421
1422         if (!decoder->tsc_ctc_ratio_d)
1423                 return;
1424
1425         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1426         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1427         if (decoder->tsc_ctc_mult) {
1428                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1429         } else {
1430                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1431                                                   decoder->tsc_ctc_ratio_n,
1432                                                   decoder->tsc_ctc_ratio_d);
1433         }
1434         decoder->ctc_delta = 0;
1435         decoder->have_tma = true;
1436         decoder->fixup_last_mtc = true;
1437         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1438                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1439 }
1440
1441 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1442 {
1443         uint64_t timestamp;
1444         uint32_t mtc, mtc_delta;
1445
1446         if (!decoder->have_tma)
1447                 return;
1448
1449         mtc = decoder->packet.payload;
1450
1451         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1452                 decoder->fixup_last_mtc = false;
1453                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1454                                         &decoder->last_mtc);
1455         }
1456
1457         if (mtc > decoder->last_mtc)
1458                 mtc_delta = mtc - decoder->last_mtc;
1459         else
1460                 mtc_delta = mtc + 256 - decoder->last_mtc;
1461
1462         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1463
1464         if (decoder->tsc_ctc_mult) {
1465                 timestamp = decoder->ctc_timestamp +
1466                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1467         } else {
1468                 timestamp = decoder->ctc_timestamp +
1469                             multdiv(decoder->ctc_delta,
1470                                     decoder->tsc_ctc_ratio_n,
1471                                     decoder->tsc_ctc_ratio_d);
1472         }
1473
1474         if (timestamp < decoder->timestamp)
1475                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1476                              timestamp, decoder->timestamp);
1477         else
1478                 decoder->timestamp = timestamp;
1479
1480         decoder->timestamp_insn_cnt = 0;
1481         decoder->last_mtc = mtc;
1482
1483         if (decoder->last_packet_type == INTEL_PT_CYC) {
1484                 decoder->cyc_ref_timestamp = decoder->timestamp;
1485                 decoder->cycle_cnt = 0;
1486                 decoder->have_calc_cyc_to_tsc = false;
1487                 intel_pt_calc_cyc_to_tsc(decoder, true);
1488         }
1489
1490         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1491 }
1492
1493 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1494 {
1495         unsigned int cbr = decoder->packet.payload & 0xff;
1496
1497         decoder->cbr_payload = decoder->packet.payload;
1498
1499         if (decoder->cbr == cbr)
1500                 return;
1501
1502         decoder->cbr = cbr;
1503         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1504 }
1505
1506 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1507 {
1508         uint64_t timestamp = decoder->cyc_ref_timestamp;
1509
1510         decoder->have_cyc = true;
1511
1512         decoder->cycle_cnt += decoder->packet.payload;
1513
1514         if (!decoder->cyc_ref_timestamp)
1515                 return;
1516
1517         if (decoder->have_calc_cyc_to_tsc)
1518                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1519         else if (decoder->cbr)
1520                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1521         else
1522                 return;
1523
1524         if (timestamp < decoder->timestamp)
1525                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1526                              timestamp, decoder->timestamp);
1527         else
1528                 decoder->timestamp = timestamp;
1529
1530         decoder->timestamp_insn_cnt = 0;
1531
1532         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1533 }
1534
1535 /* Walk PSB+ packets when already in sync. */
1536 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1537 {
1538         int err;
1539
1540         while (1) {
1541                 err = intel_pt_get_next_packet(decoder);
1542                 if (err)
1543                         return err;
1544
1545                 switch (decoder->packet.type) {
1546                 case INTEL_PT_PSBEND:
1547                         return 0;
1548
1549                 case INTEL_PT_TIP_PGD:
1550                 case INTEL_PT_TIP_PGE:
1551                 case INTEL_PT_TIP:
1552                 case INTEL_PT_TNT:
1553                 case INTEL_PT_TRACESTOP:
1554                 case INTEL_PT_BAD:
1555                 case INTEL_PT_PSB:
1556                 case INTEL_PT_PTWRITE:
1557                 case INTEL_PT_PTWRITE_IP:
1558                 case INTEL_PT_EXSTOP:
1559                 case INTEL_PT_EXSTOP_IP:
1560                 case INTEL_PT_MWAIT:
1561                 case INTEL_PT_PWRE:
1562                 case INTEL_PT_PWRX:
1563                         decoder->have_tma = false;
1564                         intel_pt_log("ERROR: Unexpected packet\n");
1565                         return -EAGAIN;
1566
1567                 case INTEL_PT_OVF:
1568                         return intel_pt_overflow(decoder);
1569
1570                 case INTEL_PT_TSC:
1571                         intel_pt_calc_tsc_timestamp(decoder);
1572                         break;
1573
1574                 case INTEL_PT_TMA:
1575                         intel_pt_calc_tma(decoder);
1576                         break;
1577
1578                 case INTEL_PT_CBR:
1579                         intel_pt_calc_cbr(decoder);
1580                         break;
1581
1582                 case INTEL_PT_MODE_EXEC:
1583                         decoder->exec_mode = decoder->packet.payload;
1584                         break;
1585
1586                 case INTEL_PT_PIP:
1587                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1588                         break;
1589
1590                 case INTEL_PT_FUP:
1591                         decoder->pge = true;
1592                         if (decoder->packet.count)
1593                                 intel_pt_set_last_ip(decoder);
1594                         break;
1595
1596                 case INTEL_PT_MODE_TSX:
1597                         intel_pt_update_in_tx(decoder);
1598                         break;
1599
1600                 case INTEL_PT_MTC:
1601                         intel_pt_calc_mtc_timestamp(decoder);
1602                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1603                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1604                         break;
1605
1606                 case INTEL_PT_CYC:
1607                 case INTEL_PT_VMCS:
1608                 case INTEL_PT_MNT:
1609                 case INTEL_PT_PAD:
1610                 default:
1611                         break;
1612                 }
1613         }
1614 }
1615
1616 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1617 {
1618         int err;
1619
1620         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1621                 decoder->tx_flags = 0;
1622                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1623                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1624         } else {
1625                 decoder->state.flags |= INTEL_PT_ASYNC;
1626         }
1627
1628         while (1) {
1629                 err = intel_pt_get_next_packet(decoder);
1630                 if (err)
1631                         return err;
1632
1633                 switch (decoder->packet.type) {
1634                 case INTEL_PT_TNT:
1635                 case INTEL_PT_FUP:
1636                 case INTEL_PT_TRACESTOP:
1637                 case INTEL_PT_PSB:
1638                 case INTEL_PT_TSC:
1639                 case INTEL_PT_TMA:
1640                 case INTEL_PT_MODE_TSX:
1641                 case INTEL_PT_BAD:
1642                 case INTEL_PT_PSBEND:
1643                 case INTEL_PT_PTWRITE:
1644                 case INTEL_PT_PTWRITE_IP:
1645                 case INTEL_PT_EXSTOP:
1646                 case INTEL_PT_EXSTOP_IP:
1647                 case INTEL_PT_MWAIT:
1648                 case INTEL_PT_PWRE:
1649                 case INTEL_PT_PWRX:
1650                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1651                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1652                         decoder->pkt_step = 0;
1653                         return -ENOENT;
1654
1655                 case INTEL_PT_CBR:
1656                         intel_pt_calc_cbr(decoder);
1657                         break;
1658
1659                 case INTEL_PT_OVF:
1660                         return intel_pt_overflow(decoder);
1661
1662                 case INTEL_PT_TIP_PGD:
1663                         decoder->state.from_ip = decoder->ip;
1664                         if (decoder->packet.count == 0) {
1665                                 decoder->state.to_ip = 0;
1666                         } else {
1667                                 intel_pt_set_ip(decoder);
1668                                 decoder->state.to_ip = decoder->ip;
1669                         }
1670                         decoder->pge = false;
1671                         decoder->continuous_period = false;
1672                         decoder->state.type |= INTEL_PT_TRACE_END;
1673                         return 0;
1674
1675                 case INTEL_PT_TIP_PGE:
1676                         decoder->pge = true;
1677                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1678                                      decoder->ip);
1679                         decoder->state.from_ip = 0;
1680                         if (decoder->packet.count == 0) {
1681                                 decoder->state.to_ip = 0;
1682                         } else {
1683                                 intel_pt_set_ip(decoder);
1684                                 decoder->state.to_ip = decoder->ip;
1685                         }
1686                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1687                         return 0;
1688
1689                 case INTEL_PT_TIP:
1690                         decoder->state.from_ip = decoder->ip;
1691                         if (decoder->packet.count == 0) {
1692                                 decoder->state.to_ip = 0;
1693                         } else {
1694                                 intel_pt_set_ip(decoder);
1695                                 decoder->state.to_ip = decoder->ip;
1696                         }
1697                         return 0;
1698
1699                 case INTEL_PT_PIP:
1700                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1701                         break;
1702
1703                 case INTEL_PT_MTC:
1704                         intel_pt_calc_mtc_timestamp(decoder);
1705                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1706                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1707                         break;
1708
1709                 case INTEL_PT_CYC:
1710                         intel_pt_calc_cyc_timestamp(decoder);
1711                         break;
1712
1713                 case INTEL_PT_MODE_EXEC:
1714                         decoder->exec_mode = decoder->packet.payload;
1715                         break;
1716
1717                 case INTEL_PT_VMCS:
1718                 case INTEL_PT_MNT:
1719                 case INTEL_PT_PAD:
1720                         break;
1721
1722                 default:
1723                         return intel_pt_bug(decoder);
1724                 }
1725         }
1726 }
1727
1728 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1729 {
1730         bool no_tip = false;
1731         int err;
1732
1733         while (1) {
1734                 err = intel_pt_get_next_packet(decoder);
1735                 if (err)
1736                         return err;
1737 next:
1738                 switch (decoder->packet.type) {
1739                 case INTEL_PT_TNT:
1740                         if (!decoder->packet.count)
1741                                 break;
1742                         decoder->tnt = decoder->packet;
1743                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1744                         err = intel_pt_walk_tnt(decoder);
1745                         if (err == -EAGAIN)
1746                                 break;
1747                         return err;
1748
1749                 case INTEL_PT_TIP_PGD:
1750                         if (decoder->packet.count != 0)
1751                                 intel_pt_set_last_ip(decoder);
1752                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1753                         return intel_pt_walk_tip(decoder);
1754
1755                 case INTEL_PT_TIP_PGE: {
1756                         decoder->pge = true;
1757                         if (decoder->packet.count == 0) {
1758                                 intel_pt_log_at("Skipping zero TIP.PGE",
1759                                                 decoder->pos);
1760                                 break;
1761                         }
1762                         intel_pt_set_ip(decoder);
1763                         decoder->state.from_ip = 0;
1764                         decoder->state.to_ip = decoder->ip;
1765                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1766                         return 0;
1767                 }
1768
1769                 case INTEL_PT_OVF:
1770                         return intel_pt_overflow(decoder);
1771
1772                 case INTEL_PT_TIP:
1773                         if (decoder->packet.count != 0)
1774                                 intel_pt_set_last_ip(decoder);
1775                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1776                         return intel_pt_walk_tip(decoder);
1777
1778                 case INTEL_PT_FUP:
1779                         if (decoder->packet.count == 0) {
1780                                 intel_pt_log_at("Skipping zero FUP",
1781                                                 decoder->pos);
1782                                 no_tip = false;
1783                                 break;
1784                         }
1785                         intel_pt_set_last_ip(decoder);
1786                         if (!decoder->branch_enable) {
1787                                 decoder->ip = decoder->last_ip;
1788                                 if (intel_pt_fup_event(decoder))
1789                                         return 0;
1790                                 no_tip = false;
1791                                 break;
1792                         }
1793                         if (decoder->set_fup_mwait)
1794                                 no_tip = true;
1795                         err = intel_pt_walk_fup(decoder);
1796                         if (err != -EAGAIN) {
1797                                 if (err)
1798                                         return err;
1799                                 if (no_tip)
1800                                         decoder->pkt_state =
1801                                                 INTEL_PT_STATE_FUP_NO_TIP;
1802                                 else
1803                                         decoder->pkt_state = INTEL_PT_STATE_FUP;
1804                                 return 0;
1805                         }
1806                         if (no_tip) {
1807                                 no_tip = false;
1808                                 break;
1809                         }
1810                         return intel_pt_walk_fup_tip(decoder);
1811
1812                 case INTEL_PT_TRACESTOP:
1813                         decoder->pge = false;
1814                         decoder->continuous_period = false;
1815                         intel_pt_clear_tx_flags(decoder);
1816                         decoder->have_tma = false;
1817                         break;
1818
1819                 case INTEL_PT_PSB:
1820                         decoder->last_ip = 0;
1821                         decoder->have_last_ip = true;
1822                         intel_pt_clear_stack(&decoder->stack);
1823                         err = intel_pt_walk_psbend(decoder);
1824                         if (err == -EAGAIN)
1825                                 goto next;
1826                         if (err)
1827                                 return err;
1828                         break;
1829
1830                 case INTEL_PT_PIP:
1831                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1832                         break;
1833
1834                 case INTEL_PT_MTC:
1835                         intel_pt_calc_mtc_timestamp(decoder);
1836                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1837                                 break;
1838                         /*
1839                          * Ensure that there has been an instruction since the
1840                          * last MTC.
1841                          */
1842                         if (!decoder->mtc_insn)
1843                                 break;
1844                         decoder->mtc_insn = false;
1845                         /* Ensure that there is a timestamp */
1846                         if (!decoder->timestamp)
1847                                 break;
1848                         decoder->state.type = INTEL_PT_INSTRUCTION;
1849                         decoder->state.from_ip = decoder->ip;
1850                         decoder->state.to_ip = 0;
1851                         decoder->mtc_insn = false;
1852                         return 0;
1853
1854                 case INTEL_PT_TSC:
1855                         intel_pt_calc_tsc_timestamp(decoder);
1856                         break;
1857
1858                 case INTEL_PT_TMA:
1859                         intel_pt_calc_tma(decoder);
1860                         break;
1861
1862                 case INTEL_PT_CYC:
1863                         intel_pt_calc_cyc_timestamp(decoder);
1864                         break;
1865
1866                 case INTEL_PT_CBR:
1867                         intel_pt_calc_cbr(decoder);
1868                         if (!decoder->branch_enable &&
1869                             decoder->cbr != decoder->cbr_seen) {
1870                                 decoder->cbr_seen = decoder->cbr;
1871                                 decoder->state.type = INTEL_PT_CBR_CHG;
1872                                 decoder->state.from_ip = decoder->ip;
1873                                 decoder->state.to_ip = 0;
1874                                 decoder->state.cbr_payload =
1875                                                         decoder->packet.payload;
1876                                 return 0;
1877                         }
1878                         break;
1879
1880                 case INTEL_PT_MODE_EXEC:
1881                         decoder->exec_mode = decoder->packet.payload;
1882                         break;
1883
1884                 case INTEL_PT_MODE_TSX:
1885                         /* MODE_TSX need not be followed by FUP */
1886                         if (!decoder->pge) {
1887                                 intel_pt_update_in_tx(decoder);
1888                                 break;
1889                         }
1890                         err = intel_pt_mode_tsx(decoder, &no_tip);
1891                         if (err)
1892                                 return err;
1893                         goto next;
1894
1895                 case INTEL_PT_BAD: /* Does not happen */
1896                         return intel_pt_bug(decoder);
1897
1898                 case INTEL_PT_PSBEND:
1899                 case INTEL_PT_VMCS:
1900                 case INTEL_PT_MNT:
1901                 case INTEL_PT_PAD:
1902                         break;
1903
1904                 case INTEL_PT_PTWRITE_IP:
1905                         decoder->fup_ptw_payload = decoder->packet.payload;
1906                         err = intel_pt_get_next_packet(decoder);
1907                         if (err)
1908                                 return err;
1909                         if (decoder->packet.type == INTEL_PT_FUP) {
1910                                 decoder->set_fup_ptw = true;
1911                                 no_tip = true;
1912                         } else {
1913                                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
1914                                                 decoder->pos);
1915                         }
1916                         goto next;
1917
1918                 case INTEL_PT_PTWRITE:
1919                         decoder->state.type = INTEL_PT_PTW;
1920                         decoder->state.from_ip = decoder->ip;
1921                         decoder->state.to_ip = 0;
1922                         decoder->state.ptw_payload = decoder->packet.payload;
1923                         return 0;
1924
1925                 case INTEL_PT_MWAIT:
1926                         decoder->fup_mwait_payload = decoder->packet.payload;
1927                         decoder->set_fup_mwait = true;
1928                         break;
1929
1930                 case INTEL_PT_PWRE:
1931                         if (decoder->set_fup_mwait) {
1932                                 decoder->fup_pwre_payload =
1933                                                         decoder->packet.payload;
1934                                 decoder->set_fup_pwre = true;
1935                                 break;
1936                         }
1937                         decoder->state.type = INTEL_PT_PWR_ENTRY;
1938                         decoder->state.from_ip = decoder->ip;
1939                         decoder->state.to_ip = 0;
1940                         decoder->state.pwrx_payload = decoder->packet.payload;
1941                         return 0;
1942
1943                 case INTEL_PT_EXSTOP_IP:
1944                         err = intel_pt_get_next_packet(decoder);
1945                         if (err)
1946                                 return err;
1947                         if (decoder->packet.type == INTEL_PT_FUP) {
1948                                 decoder->set_fup_exstop = true;
1949                                 no_tip = true;
1950                         } else {
1951                                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
1952                                                 decoder->pos);
1953                         }
1954                         goto next;
1955
1956                 case INTEL_PT_EXSTOP:
1957                         decoder->state.type = INTEL_PT_EX_STOP;
1958                         decoder->state.from_ip = decoder->ip;
1959                         decoder->state.to_ip = 0;
1960                         return 0;
1961
1962                 case INTEL_PT_PWRX:
1963                         decoder->state.type = INTEL_PT_PWR_EXIT;
1964                         decoder->state.from_ip = decoder->ip;
1965                         decoder->state.to_ip = 0;
1966                         decoder->state.pwrx_payload = decoder->packet.payload;
1967                         return 0;
1968
1969                 default:
1970                         return intel_pt_bug(decoder);
1971                 }
1972         }
1973 }
1974
1975 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1976 {
1977         return decoder->packet.count &&
1978                (decoder->have_last_ip || decoder->packet.count == 3 ||
1979                 decoder->packet.count == 6);
1980 }
1981
1982 /* Walk PSB+ packets to get in sync. */
1983 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1984 {
1985         int err;
1986
1987         while (1) {
1988                 err = intel_pt_get_next_packet(decoder);
1989                 if (err)
1990                         return err;
1991
1992                 switch (decoder->packet.type) {
1993                 case INTEL_PT_TIP_PGD:
1994                         decoder->continuous_period = false;
1995                         __fallthrough;
1996                 case INTEL_PT_TIP_PGE:
1997                 case INTEL_PT_TIP:
1998                 case INTEL_PT_PTWRITE:
1999                 case INTEL_PT_PTWRITE_IP:
2000                 case INTEL_PT_EXSTOP:
2001                 case INTEL_PT_EXSTOP_IP:
2002                 case INTEL_PT_MWAIT:
2003                 case INTEL_PT_PWRE:
2004                 case INTEL_PT_PWRX:
2005                         intel_pt_log("ERROR: Unexpected packet\n");
2006                         return -ENOENT;
2007
2008                 case INTEL_PT_FUP:
2009                         decoder->pge = true;
2010                         if (intel_pt_have_ip(decoder)) {
2011                                 uint64_t current_ip = decoder->ip;
2012
2013                                 intel_pt_set_ip(decoder);
2014                                 if (current_ip)
2015                                         intel_pt_log_to("Setting IP",
2016                                                         decoder->ip);
2017                         }
2018                         break;
2019
2020                 case INTEL_PT_MTC:
2021                         intel_pt_calc_mtc_timestamp(decoder);
2022                         break;
2023
2024                 case INTEL_PT_TSC:
2025                         intel_pt_calc_tsc_timestamp(decoder);
2026                         break;
2027
2028                 case INTEL_PT_TMA:
2029                         intel_pt_calc_tma(decoder);
2030                         break;
2031
2032                 case INTEL_PT_CYC:
2033                         intel_pt_calc_cyc_timestamp(decoder);
2034                         break;
2035
2036                 case INTEL_PT_CBR:
2037                         intel_pt_calc_cbr(decoder);
2038                         break;
2039
2040                 case INTEL_PT_PIP:
2041                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2042                         break;
2043
2044                 case INTEL_PT_MODE_EXEC:
2045                         decoder->exec_mode = decoder->packet.payload;
2046                         break;
2047
2048                 case INTEL_PT_MODE_TSX:
2049                         intel_pt_update_in_tx(decoder);
2050                         break;
2051
2052                 case INTEL_PT_TRACESTOP:
2053                         decoder->pge = false;
2054                         decoder->continuous_period = false;
2055                         intel_pt_clear_tx_flags(decoder);
2056                         __fallthrough;
2057
2058                 case INTEL_PT_TNT:
2059                         decoder->have_tma = false;
2060                         intel_pt_log("ERROR: Unexpected packet\n");
2061                         if (decoder->ip)
2062                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
2063                         else
2064                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2065                         return -ENOENT;
2066
2067                 case INTEL_PT_BAD: /* Does not happen */
2068                         return intel_pt_bug(decoder);
2069
2070                 case INTEL_PT_OVF:
2071                         return intel_pt_overflow(decoder);
2072
2073                 case INTEL_PT_PSBEND:
2074                         return 0;
2075
2076                 case INTEL_PT_PSB:
2077                 case INTEL_PT_VMCS:
2078                 case INTEL_PT_MNT:
2079                 case INTEL_PT_PAD:
2080                 default:
2081                         break;
2082                 }
2083         }
2084 }
2085
2086 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
2087 {
2088         int err;
2089
2090         while (1) {
2091                 err = intel_pt_get_next_packet(decoder);
2092                 if (err)
2093                         return err;
2094
2095                 switch (decoder->packet.type) {
2096                 case INTEL_PT_TIP_PGD:
2097                         decoder->continuous_period = false;
2098                         __fallthrough;
2099                 case INTEL_PT_TIP_PGE:
2100                 case INTEL_PT_TIP:
2101                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
2102                         if (intel_pt_have_ip(decoder))
2103                                 intel_pt_set_ip(decoder);
2104                         if (!decoder->ip)
2105                                 break;
2106                         if (decoder->packet.type == INTEL_PT_TIP_PGE)
2107                                 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2108                         if (decoder->packet.type == INTEL_PT_TIP_PGD)
2109                                 decoder->state.type |= INTEL_PT_TRACE_END;
2110                         return 0;
2111
2112                 case INTEL_PT_FUP:
2113                         if (intel_pt_have_ip(decoder))
2114                                 intel_pt_set_ip(decoder);
2115                         if (decoder->ip)
2116                                 return 0;
2117                         break;
2118
2119                 case INTEL_PT_MTC:
2120                         intel_pt_calc_mtc_timestamp(decoder);
2121                         break;
2122
2123                 case INTEL_PT_TSC:
2124                         intel_pt_calc_tsc_timestamp(decoder);
2125                         break;
2126
2127                 case INTEL_PT_TMA:
2128                         intel_pt_calc_tma(decoder);
2129                         break;
2130
2131                 case INTEL_PT_CYC:
2132                         intel_pt_calc_cyc_timestamp(decoder);
2133                         break;
2134
2135                 case INTEL_PT_CBR:
2136                         intel_pt_calc_cbr(decoder);
2137                         break;
2138
2139                 case INTEL_PT_PIP:
2140                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2141                         break;
2142
2143                 case INTEL_PT_MODE_EXEC:
2144                         decoder->exec_mode = decoder->packet.payload;
2145                         break;
2146
2147                 case INTEL_PT_MODE_TSX:
2148                         intel_pt_update_in_tx(decoder);
2149                         break;
2150
2151                 case INTEL_PT_OVF:
2152                         return intel_pt_overflow(decoder);
2153
2154                 case INTEL_PT_BAD: /* Does not happen */
2155                         return intel_pt_bug(decoder);
2156
2157                 case INTEL_PT_TRACESTOP:
2158                         decoder->pge = false;
2159                         decoder->continuous_period = false;
2160                         intel_pt_clear_tx_flags(decoder);
2161                         decoder->have_tma = false;
2162                         break;
2163
2164                 case INTEL_PT_PSB:
2165                         decoder->last_ip = 0;
2166                         decoder->have_last_ip = true;
2167                         intel_pt_clear_stack(&decoder->stack);
2168                         err = intel_pt_walk_psb(decoder);
2169                         if (err)
2170                                 return err;
2171                         if (decoder->ip) {
2172                                 /* Do not have a sample */
2173                                 decoder->state.type = 0;
2174                                 return 0;
2175                         }
2176                         break;
2177
2178                 case INTEL_PT_TNT:
2179                 case INTEL_PT_PSBEND:
2180                 case INTEL_PT_VMCS:
2181                 case INTEL_PT_MNT:
2182                 case INTEL_PT_PAD:
2183                 case INTEL_PT_PTWRITE:
2184                 case INTEL_PT_PTWRITE_IP:
2185                 case INTEL_PT_EXSTOP:
2186                 case INTEL_PT_EXSTOP_IP:
2187                 case INTEL_PT_MWAIT:
2188                 case INTEL_PT_PWRE:
2189                 case INTEL_PT_PWRX:
2190                 default:
2191                         break;
2192                 }
2193         }
2194 }
2195
2196 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
2197 {
2198         int err;
2199
2200         decoder->set_fup_tx_flags = false;
2201         decoder->set_fup_ptw = false;
2202         decoder->set_fup_mwait = false;
2203         decoder->set_fup_pwre = false;
2204         decoder->set_fup_exstop = false;
2205
2206         if (!decoder->branch_enable) {
2207                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2208                 decoder->overflow = false;
2209                 decoder->state.type = 0; /* Do not have a sample */
2210                 return 0;
2211         }
2212
2213         intel_pt_log("Scanning for full IP\n");
2214         err = intel_pt_walk_to_ip(decoder);
2215         if (err)
2216                 return err;
2217
2218         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2219         decoder->overflow = false;
2220
2221         decoder->state.from_ip = 0;
2222         decoder->state.to_ip = decoder->ip;
2223         intel_pt_log_to("Setting IP", decoder->ip);
2224
2225         return 0;
2226 }
2227
2228 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
2229 {
2230         const unsigned char *end = decoder->buf + decoder->len;
2231         size_t i;
2232
2233         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
2234                 if (i > decoder->len)
2235                         continue;
2236                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
2237                         return i;
2238         }
2239         return 0;
2240 }
2241
2242 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
2243 {
2244         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
2245         const char *psb = INTEL_PT_PSB_STR;
2246
2247         if (rest_psb > decoder->len ||
2248             memcmp(decoder->buf, psb + part_psb, rest_psb))
2249                 return 0;
2250
2251         return rest_psb;
2252 }
2253
2254 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2255                                   int part_psb)
2256 {
2257         int rest_psb, ret;
2258
2259         decoder->pos += decoder->len;
2260         decoder->len = 0;
2261
2262         ret = intel_pt_get_next_data(decoder);
2263         if (ret)
2264                 return ret;
2265
2266         rest_psb = intel_pt_rest_psb(decoder, part_psb);
2267         if (!rest_psb)
2268                 return 0;
2269
2270         decoder->pos -= part_psb;
2271         decoder->next_buf = decoder->buf + rest_psb;
2272         decoder->next_len = decoder->len - rest_psb;
2273         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2274         decoder->buf = decoder->temp_buf;
2275         decoder->len = INTEL_PT_PSB_LEN;
2276
2277         return 0;
2278 }
2279
2280 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2281 {
2282         unsigned char *next;
2283         int ret;
2284
2285         intel_pt_log("Scanning for PSB\n");
2286         while (1) {
2287                 if (!decoder->len) {
2288                         ret = intel_pt_get_next_data(decoder);
2289                         if (ret)
2290                                 return ret;
2291                 }
2292
2293                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2294                               INTEL_PT_PSB_LEN);
2295                 if (!next) {
2296                         int part_psb;
2297
2298                         part_psb = intel_pt_part_psb(decoder);
2299                         if (part_psb) {
2300                                 ret = intel_pt_get_split_psb(decoder, part_psb);
2301                                 if (ret)
2302                                         return ret;
2303                         } else {
2304                                 decoder->pos += decoder->len;
2305                                 decoder->len = 0;
2306                         }
2307                         continue;
2308                 }
2309
2310                 decoder->pkt_step = next - decoder->buf;
2311                 return intel_pt_get_next_packet(decoder);
2312         }
2313 }
2314
2315 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2316 {
2317         int err;
2318
2319         decoder->pge = false;
2320         decoder->continuous_period = false;
2321         decoder->have_last_ip = false;
2322         decoder->last_ip = 0;
2323         decoder->ip = 0;
2324         intel_pt_clear_stack(&decoder->stack);
2325
2326         err = intel_pt_scan_for_psb(decoder);
2327         if (err)
2328                 return err;
2329
2330         decoder->have_last_ip = true;
2331         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2332
2333         err = intel_pt_walk_psb(decoder);
2334         if (err)
2335                 return err;
2336
2337         if (decoder->ip) {
2338                 decoder->state.type = 0; /* Do not have a sample */
2339                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2340         } else {
2341                 return intel_pt_sync_ip(decoder);
2342         }
2343
2344         return 0;
2345 }
2346
2347 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2348 {
2349         uint64_t est = decoder->sample_insn_cnt << 1;
2350
2351         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2352                 goto out;
2353
2354         est *= decoder->max_non_turbo_ratio;
2355         est /= decoder->cbr;
2356 out:
2357         return decoder->sample_timestamp + est;
2358 }
2359
2360 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2361 {
2362         int err;
2363
2364         do {
2365                 decoder->state.type = INTEL_PT_BRANCH;
2366                 decoder->state.flags = 0;
2367
2368                 switch (decoder->pkt_state) {
2369                 case INTEL_PT_STATE_NO_PSB:
2370                         err = intel_pt_sync(decoder);
2371                         break;
2372                 case INTEL_PT_STATE_NO_IP:
2373                         decoder->have_last_ip = false;
2374                         decoder->last_ip = 0;
2375                         decoder->ip = 0;
2376                         __fallthrough;
2377                 case INTEL_PT_STATE_ERR_RESYNC:
2378                         err = intel_pt_sync_ip(decoder);
2379                         break;
2380                 case INTEL_PT_STATE_IN_SYNC:
2381                         err = intel_pt_walk_trace(decoder);
2382                         break;
2383                 case INTEL_PT_STATE_TNT:
2384                 case INTEL_PT_STATE_TNT_CONT:
2385                         err = intel_pt_walk_tnt(decoder);
2386                         if (err == -EAGAIN)
2387                                 err = intel_pt_walk_trace(decoder);
2388                         break;
2389                 case INTEL_PT_STATE_TIP:
2390                 case INTEL_PT_STATE_TIP_PGD:
2391                         err = intel_pt_walk_tip(decoder);
2392                         break;
2393                 case INTEL_PT_STATE_FUP:
2394                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2395                         err = intel_pt_walk_fup(decoder);
2396                         if (err == -EAGAIN)
2397                                 err = intel_pt_walk_fup_tip(decoder);
2398                         else if (!err)
2399                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
2400                         break;
2401                 case INTEL_PT_STATE_FUP_NO_TIP:
2402                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2403                         err = intel_pt_walk_fup(decoder);
2404                         if (err == -EAGAIN)
2405                                 err = intel_pt_walk_trace(decoder);
2406                         break;
2407                 default:
2408                         err = intel_pt_bug(decoder);
2409                         break;
2410                 }
2411         } while (err == -ENOLINK);
2412
2413         if (err) {
2414                 decoder->state.err = intel_pt_ext_err(err);
2415                 decoder->state.from_ip = decoder->ip;
2416                 decoder->sample_timestamp = decoder->timestamp;
2417                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2418         } else {
2419                 decoder->state.err = 0;
2420                 if (decoder->cbr != decoder->cbr_seen && decoder->state.type) {
2421                         decoder->cbr_seen = decoder->cbr;
2422                         decoder->state.type |= INTEL_PT_CBR_CHG;
2423                         decoder->state.cbr_payload = decoder->cbr_payload;
2424                 }
2425                 if (intel_pt_sample_time(decoder->pkt_state)) {
2426                         decoder->sample_timestamp = decoder->timestamp;
2427                         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2428                 }
2429         }
2430
2431         decoder->state.timestamp = decoder->sample_timestamp;
2432         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2433         decoder->state.cr3 = decoder->cr3;
2434         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2435
2436         return &decoder->state;
2437 }
2438
2439 /**
2440  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2441  * @buf: pointer to buffer pointer
2442  * @len: size of buffer
2443  *
2444  * Updates the buffer pointer to point to the start of the next PSB packet if
2445  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2446  * @len is adjusted accordingly.
2447  *
2448  * Return: %true if a PSB packet is found, %false otherwise.
2449  */
2450 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2451 {
2452         unsigned char *next;
2453
2454         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2455         if (next) {
2456                 *len -= next - *buf;
2457                 *buf = next;
2458                 return true;
2459         }
2460         return false;
2461 }
2462
2463 /**
2464  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2465  *                     packet.
2466  * @buf: pointer to buffer pointer
2467  * @len: size of buffer
2468  *
2469  * Updates the buffer pointer to point to the start of the following PSB packet
2470  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2471  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2472  *
2473  * Return: %true if a PSB packet is found, %false otherwise.
2474  */
2475 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2476 {
2477         unsigned char *next;
2478
2479         if (!*len)
2480                 return false;
2481
2482         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2483         if (next) {
2484                 *len -= next - *buf;
2485                 *buf = next;
2486                 return true;
2487         }
2488         return false;
2489 }
2490
2491 /**
2492  * intel_pt_last_psb - find the last PSB packet in a buffer.
2493  * @buf: buffer
2494  * @len: size of buffer
2495  *
2496  * This function finds the last PSB in a buffer.
2497  *
2498  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2499  */
2500 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2501 {
2502         const char *n = INTEL_PT_PSB_STR;
2503         unsigned char *p;
2504         size_t k;
2505
2506         if (len < INTEL_PT_PSB_LEN)
2507                 return NULL;
2508
2509         k = len - INTEL_PT_PSB_LEN + 1;
2510         while (1) {
2511                 p = memrchr(buf, n[0], k);
2512                 if (!p)
2513                         return NULL;
2514                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2515                         return p;
2516                 k = p - buf;
2517                 if (!k)
2518                         return NULL;
2519         }
2520 }
2521
2522 /**
2523  * intel_pt_next_tsc - find and return next TSC.
2524  * @buf: buffer
2525  * @len: size of buffer
2526  * @tsc: TSC value returned
2527  * @rem: returns remaining size when TSC is found
2528  *
2529  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2530  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2531  * PSBEND packet is found.
2532  *
2533  * Return: %true if TSC is found, false otherwise.
2534  */
2535 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2536                               size_t *rem)
2537 {
2538         struct intel_pt_pkt packet;
2539         int ret;
2540
2541         while (len) {
2542                 ret = intel_pt_get_packet(buf, len, &packet);
2543                 if (ret <= 0)
2544                         return false;
2545                 if (packet.type == INTEL_PT_TSC) {
2546                         *tsc = packet.payload;
2547                         *rem = len;
2548                         return true;
2549                 }
2550                 if (packet.type == INTEL_PT_PSBEND)
2551                         return false;
2552                 buf += ret;
2553                 len -= ret;
2554         }
2555         return false;
2556 }
2557
2558 /**
2559  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2560  * @tsc1: first TSC to compare
2561  * @tsc2: second TSC to compare
2562  *
2563  * This function compares 7-byte TSC values allowing for the possibility that
2564  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2565  * around so for that purpose this function assumes the absolute difference is
2566  * less than half the maximum difference.
2567  *
2568  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2569  * after @tsc2.
2570  */
2571 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2572 {
2573         const uint64_t halfway = (1ULL << 55);
2574
2575         if (tsc1 == tsc2)
2576                 return 0;
2577
2578         if (tsc1 < tsc2) {
2579                 if (tsc2 - tsc1 < halfway)
2580                         return -1;
2581                 else
2582                         return 1;
2583         } else {
2584                 if (tsc1 - tsc2 < halfway)
2585                         return 1;
2586                 else
2587                         return -1;
2588         }
2589 }
2590
2591 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
2592
2593 /**
2594  * adj_for_padding - adjust overlap to account for padding.
2595  * @buf_b: second buffer
2596  * @buf_a: first buffer
2597  * @len_a: size of first buffer
2598  *
2599  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
2600  * accordingly.
2601  *
2602  * Return: A pointer into @buf_b from where non-overlapped data starts
2603  */
2604 static unsigned char *adj_for_padding(unsigned char *buf_b,
2605                                       unsigned char *buf_a, size_t len_a)
2606 {
2607         unsigned char *p = buf_b - MAX_PADDING;
2608         unsigned char *q = buf_a + len_a - MAX_PADDING;
2609         int i;
2610
2611         for (i = MAX_PADDING; i; i--, p++, q++) {
2612                 if (*p != *q)
2613                         break;
2614         }
2615
2616         return p;
2617 }
2618
2619 /**
2620  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2621  *                             using TSC.
2622  * @buf_a: first buffer
2623  * @len_a: size of first buffer
2624  * @buf_b: second buffer
2625  * @len_b: size of second buffer
2626  * @consecutive: returns true if there is data in buf_b that is consecutive
2627  *               to buf_a
2628  *
2629  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2630  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2631  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2632  * @buf_a and @buf_b are positioned at a PSB.
2633  *
2634  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2635  * @buf_b + @len_b if there is no non-overlapped data.
2636  */
2637 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2638                                                 size_t len_a,
2639                                                 unsigned char *buf_b,
2640                                                 size_t len_b, bool *consecutive)
2641 {
2642         uint64_t tsc_a, tsc_b;
2643         unsigned char *p;
2644         size_t len, rem_a, rem_b;
2645
2646         p = intel_pt_last_psb(buf_a, len_a);
2647         if (!p)
2648                 return buf_b; /* No PSB in buf_a => no overlap */
2649
2650         len = len_a - (p - buf_a);
2651         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2652                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2653                 len_a -= len;
2654                 p = intel_pt_last_psb(buf_a, len_a);
2655                 if (!p)
2656                         return buf_b; /* No full PSB+ => assume no overlap */
2657                 len = len_a - (p - buf_a);
2658                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2659                         return buf_b; /* No TSC in buf_a => assume no overlap */
2660         }
2661
2662         while (1) {
2663                 /* Ignore PSB+ with no TSC */
2664                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2665                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2666
2667                         /* Same TSC, so buffers are consecutive */
2668                         if (!cmp && rem_b >= rem_a) {
2669                                 unsigned char *start;
2670
2671                                 *consecutive = true;
2672                                 start = buf_b + len_b - (rem_b - rem_a);
2673                                 return adj_for_padding(start, buf_a, len_a);
2674                         }
2675                         if (cmp < 0)
2676                                 return buf_b; /* tsc_a < tsc_b => no overlap */
2677                 }
2678
2679                 if (!intel_pt_step_psb(&buf_b, &len_b))
2680                         return buf_b + len_b; /* No PSB in buf_b => no data */
2681         }
2682 }
2683
2684 /**
2685  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2686  * @buf_a: first buffer
2687  * @len_a: size of first buffer
2688  * @buf_b: second buffer
2689  * @len_b: size of second buffer
2690  * @have_tsc: can use TSC packets to detect overlap
2691  * @consecutive: returns true if there is data in buf_b that is consecutive
2692  *               to buf_a
2693  *
2694  * When trace samples or snapshots are recorded there is the possibility that
2695  * the data overlaps.  Note that, for the purposes of decoding, data is only
2696  * useful if it begins with a PSB packet.
2697  *
2698  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2699  * @buf_b + @len_b if there is no non-overlapped data.
2700  */
2701 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2702                                      unsigned char *buf_b, size_t len_b,
2703                                      bool have_tsc, bool *consecutive)
2704 {
2705         unsigned char *found;
2706
2707         /* Buffer 'b' must start at PSB so throw away everything before that */
2708         if (!intel_pt_next_psb(&buf_b, &len_b))
2709                 return buf_b + len_b; /* No PSB */
2710
2711         if (!intel_pt_next_psb(&buf_a, &len_a))
2712                 return buf_b; /* No overlap */
2713
2714         if (have_tsc) {
2715                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2716                                                   consecutive);
2717                 if (found)
2718                         return found;
2719         }
2720
2721         /*
2722          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2723          * we can ignore the first part of buffer 'a'.
2724          */
2725         while (len_b < len_a) {
2726                 if (!intel_pt_step_psb(&buf_a, &len_a))
2727                         return buf_b; /* No overlap */
2728         }
2729
2730         /* Now len_b >= len_a */
2731         while (1) {
2732                 /* Potential overlap so check the bytes */
2733                 found = memmem(buf_a, len_a, buf_b, len_a);
2734                 if (found) {
2735                         *consecutive = true;
2736                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
2737                 }
2738
2739                 /* Try again at next PSB in buffer 'a' */
2740                 if (!intel_pt_step_psb(&buf_a, &len_a))
2741                         return buf_b; /* No overlap */
2742         }
2743 }