tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25 #include <linux/compiler.h>
  26
  27 #include "../cache.h"
  28 #include "../util.h"
  29 #include "../auxtrace.h"
  30
  31 #include "intel-pt-insn-decoder.h"
  32 #include "intel-pt-pkt-decoder.h"
  33 #include "intel-pt-decoder.h"
  34 #include "intel-pt-log.h"
  35
  36 #define INTEL_PT_BLK_SIZE 1024
  37
  38 #define BIT63 (((uint64_t)1 << 63))
  39
  40 #define INTEL_PT_RETURN 1
  41
  42 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  43 #define INTEL_PT_MAX_LOOPS 10000
  44
  45 struct intel_pt_blk {
  46         struct intel_pt_blk *prev;
  47         uint64_t ip[INTEL_PT_BLK_SIZE];
  48 };
  49
  50 struct intel_pt_stack {
  51         struct intel_pt_blk *blk;
  52         struct intel_pt_blk *spare;
  53         int pos;
  54 };
  55
  56 enum intel_pt_pkt_state {
  57         INTEL_PT_STATE_NO_PSB,
  58         INTEL_PT_STATE_NO_IP,
  59         INTEL_PT_STATE_ERR_RESYNC,
  60         INTEL_PT_STATE_IN_SYNC,
  61         INTEL_PT_STATE_TNT_CONT,
  62         INTEL_PT_STATE_TNT,
  63         INTEL_PT_STATE_TIP,
  64         INTEL_PT_STATE_TIP_PGD,
  65         INTEL_PT_STATE_FUP,
  66         INTEL_PT_STATE_FUP_NO_TIP,
  67 };
  68
  69 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  70 {
  71         switch (pkt_state) {
  72         case INTEL_PT_STATE_NO_PSB:
  73         case INTEL_PT_STATE_NO_IP:
  74         case INTEL_PT_STATE_ERR_RESYNC:
  75         case INTEL_PT_STATE_IN_SYNC:
  76         case INTEL_PT_STATE_TNT_CONT:
  77                 return true;
  78         case INTEL_PT_STATE_TNT:
  79         case INTEL_PT_STATE_TIP:
  80         case INTEL_PT_STATE_TIP_PGD:
  81         case INTEL_PT_STATE_FUP:
  82         case INTEL_PT_STATE_FUP_NO_TIP:
  83                 return false;
  84         default:
  85                 return true;
  86         };
  87 }
  88
  89 #ifdef INTEL_PT_STRICT
  90 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  91 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  92 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  93 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  94 #else
  95 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  96 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  97 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  98 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  99 #endif
 100
 101 struct intel_pt_decoder {
 102         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 103         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 104                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 105                          uint64_t max_insn_cnt, void *data);
 106         bool (*pgd_ip)(uint64_t ip, void *data);
 107         void *data;
 108         struct intel_pt_state state;
 109         const unsigned char *buf;
 110         size_t len;
 111         bool return_compression;
 112         bool branch_enable;
 113         bool mtc_insn;
 114         bool pge;
 115         bool have_tma;
 116         bool have_cyc;
 117         bool fixup_last_mtc;
 118         bool have_last_ip;
 119         bool in_psb;
 120         enum intel_pt_param_flags flags;
 121         uint64_t pos;
 122         uint64_t last_ip;
 123         uint64_t ip;
 124         uint64_t cr3;
 125         uint64_t timestamp;
 126         uint64_t tsc_timestamp;
 127         uint64_t ref_timestamp;
 128         uint64_t sample_timestamp;
 129         uint64_t ret_addr;
 130         uint64_t ctc_timestamp;
 131         uint64_t ctc_delta;
 132         uint64_t cycle_cnt;
 133         uint64_t cyc_ref_timestamp;
 134         uint32_t last_mtc;
 135         uint32_t tsc_ctc_ratio_n;
 136         uint32_t tsc_ctc_ratio_d;
 137         uint32_t tsc_ctc_mult;
 138         uint32_t tsc_slip;
 139         uint32_t ctc_rem_mask;
 140         int mtc_shift;
 141         struct intel_pt_stack stack;
 142         enum intel_pt_pkt_state pkt_state;
 143         struct intel_pt_pkt packet;
 144         struct intel_pt_pkt tnt;
 145         int pkt_step;
 146         int pkt_len;
 147         int last_packet_type;
 148         unsigned int cbr;
 149         unsigned int cbr_seen;
 150         unsigned int max_non_turbo_ratio;
 151         double max_non_turbo_ratio_fp;
 152         double cbr_cyc_to_tsc;
 153         double calc_cyc_to_tsc;
 154         bool have_calc_cyc_to_tsc;
 155         int exec_mode;
 156         unsigned int insn_bytes;
 157         uint64_t period;
 158         enum intel_pt_period_type period_type;
 159         uint64_t tot_insn_cnt;
 160         uint64_t period_insn_cnt;
 161         uint64_t period_mask;
 162         uint64_t period_ticks;
 163         uint64_t last_masked_timestamp;
 164         uint64_t tot_cyc_cnt;
 165         uint64_t sample_tot_cyc_cnt;
 166         bool continuous_period;
 167         bool overflow;
 168         bool set_fup_tx_flags;
 169         bool set_fup_ptw;
 170         bool set_fup_mwait;
 171         bool set_fup_pwre;
 172         bool set_fup_exstop;
 173         bool sample_cyc;
 174         unsigned int fup_tx_flags;
 175         unsigned int tx_flags;
 176         uint64_t fup_ptw_payload;
 177         uint64_t fup_mwait_payload;
 178         uint64_t fup_pwre_payload;
 179         uint64_t cbr_payload;
 180         uint64_t timestamp_insn_cnt;
 181         uint64_t sample_insn_cnt;
 182         uint64_t stuck_ip;
 183         int no_progress;
 184         int stuck_ip_prd;
 185         int stuck_ip_cnt;
 186         const unsigned char *next_buf;
 187         size_t next_len;
 188         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 189 };
 190
 191 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 192 {
 193         int i;
 194
 195         for (i = 0; x != 1; i++)
 196                 x >>= 1;
 197
 198         return x << i;
 199 }
 200
 201 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 202 {
 203         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 204                 uint64_t period;
 205
 206                 period = intel_pt_lower_power_of_2(decoder->period);
 207                 decoder->period_mask  = ~(period - 1);
 208                 decoder->period_ticks = period;
 209         }
 210 }
 211
 212 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 213 {
 214         if (!d)
 215                 return 0;
 216         return (t / d) * n + ((t % d) * n) / d;
 217 }
 218
 219 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 220 {
 221         struct intel_pt_decoder *decoder;
 222
 223         if (!params->get_trace || !params->walk_insn)
 224                 return NULL;
 225
 226         decoder = zalloc(sizeof(struct intel_pt_decoder));
 227         if (!decoder)
 228                 return NULL;
 229
 230         decoder->get_trace          = params->get_trace;
 231         decoder->walk_insn          = params->walk_insn;
 232         decoder->pgd_ip             = params->pgd_ip;
 233         decoder->data               = params->data;
 234         decoder->return_compression = params->return_compression;
 235         decoder->branch_enable      = params->branch_enable;
 236
 237         decoder->flags              = params->flags;
 238
 239         decoder->period             = params->period;
 240         decoder->period_type        = params->period_type;
 241
 242         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 243         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 244
 245         intel_pt_setup_period(decoder);
 246
 247         decoder->mtc_shift = params->mtc_period;
 248         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 249
 250         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 251         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 252
 253         if (!decoder->tsc_ctc_ratio_n)
 254                 decoder->tsc_ctc_ratio_d = 0;
 255
 256         if (decoder->tsc_ctc_ratio_d) {
 257                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 258                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 259                                                 decoder->tsc_ctc_ratio_d;
 260         }
 261
 262         /*
 263          * A TSC packet can slip past MTC packets so that the timestamp appears
 264          * to go backwards. One estimate is that can be up to about 40 CPU
 265          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 266          * slippage an order of magnitude more to be on the safe side.
 267          */
 268         decoder->tsc_slip = 0x10000;
 269
 270         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 271         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 272         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 273         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 274         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 275
 276         return decoder;
 277 }
 278
 279 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 280 {
 281         struct intel_pt_blk *blk = stack->blk;
 282
 283         stack->blk = blk->prev;
 284         if (!stack->spare)
 285                 stack->spare = blk;
 286         else
 287                 free(blk);
 288 }
 289
 290 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 291 {
 292         if (!stack->pos) {
 293                 if (!stack->blk)
 294                         return 0;
 295                 intel_pt_pop_blk(stack);
 296                 if (!stack->blk)
 297                         return 0;
 298                 stack->pos = INTEL_PT_BLK_SIZE;
 299         }
 300         return stack->blk->ip[--stack->pos];
 301 }
 302
 303 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 304 {
 305         struct intel_pt_blk *blk;
 306
 307         if (stack->spare) {
 308                 blk = stack->spare;
 309                 stack->spare = NULL;
 310         } else {
 311                 blk = malloc(sizeof(struct intel_pt_blk));
 312                 if (!blk)
 313                         return -ENOMEM;
 314         }
 315
 316         blk->prev = stack->blk;
 317         stack->blk = blk;
 318         stack->pos = 0;
 319         return 0;
 320 }
 321
 322 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 323 {
 324         int err;
 325
 326         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 327                 err = intel_pt_alloc_blk(stack);
 328                 if (err)
 329                         return err;
 330         }
 331
 332         stack->blk->ip[stack->pos++] = ip;
 333         return 0;
 334 }
 335
 336 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 337 {
 338         while (stack->blk)
 339                 intel_pt_pop_blk(stack);
 340         stack->pos = 0;
 341 }
 342
 343 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 344 {
 345         intel_pt_clear_stack(stack);
 346         zfree(&stack->blk);
 347         zfree(&stack->spare);
 348 }
 349
 350 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 351 {
 352         intel_pt_free_stack(&decoder->stack);
 353         free(decoder);
 354 }
 355
 356 static int intel_pt_ext_err(int code)
 357 {
 358         switch (code) {
 359         case -ENOMEM:
 360                 return INTEL_PT_ERR_NOMEM;
 361         case -ENOSYS:
 362                 return INTEL_PT_ERR_INTERN;
 363         case -EBADMSG:
 364                 return INTEL_PT_ERR_BADPKT;
 365         case -ENODATA:
 366                 return INTEL_PT_ERR_NODATA;
 367         case -EILSEQ:
 368                 return INTEL_PT_ERR_NOINSN;
 369         case -ENOENT:
 370                 return INTEL_PT_ERR_MISMAT;
 371         case -EOVERFLOW:
 372                 return INTEL_PT_ERR_OVR;
 373         case -ENOSPC:
 374                 return INTEL_PT_ERR_LOST;
 375         case -ELOOP:
 376                 return INTEL_PT_ERR_NELOOP;
 377         default:
 378                 return INTEL_PT_ERR_UNK;
 379         }
 380 }
 381
 382 static const char *intel_pt_err_msgs[] = {
 383         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 384         [INTEL_PT_ERR_INTERN] = "Internal error",
 385         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 386         [INTEL_PT_ERR_NODATA] = "No more data",
 387         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 388         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 389         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 390         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 391         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 392         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 393 };
 394
 395 int intel_pt__strerror(int code, char *buf, size_t buflen)
 396 {
 397         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 398                 code = INTEL_PT_ERR_UNK;
 399         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 400         return 0;
 401 }
 402
 403 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 404                                  uint64_t last_ip)
 405 {
 406         uint64_t ip;
 407
 408         switch (packet->count) {
 409         case 1:
 410                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 411                      packet->payload;
 412                 break;
 413         case 2:
 414                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 415                      packet->payload;
 416                 break;
 417         case 3:
 418                 ip = packet->payload;
 419                 /* Sign-extend 6-byte ip */
 420                 if (ip & (uint64_t)0x800000000000ULL)
 421                         ip |= (uint64_t)0xffff000000000000ULL;
 422                 break;
 423         case 4:
 424                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 425                      packet->payload;
 426                 break;
 427         case 6:
 428                 ip = packet->payload;
 429                 break;
 430         default:
 431                 return 0;
 432         }
 433
 434         return ip;
 435 }
 436
 437 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 438 {
 439         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 440         decoder->have_last_ip = true;
 441 }
 442
 443 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 444 {
 445         intel_pt_set_last_ip(decoder);
 446         decoder->ip = decoder->last_ip;
 447 }
 448
 449 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 450 {
 451         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 452                             decoder->buf);
 453 }
 454
 455 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 456 {
 457         intel_pt_log("ERROR: Internal error\n");
 458         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 459         return -ENOSYS;
 460 }
 461
 462 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 463 {
 464         decoder->tx_flags = 0;
 465 }
 466
 467 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 468 {
 469         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 470 }
 471
 472 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 473 {
 474         intel_pt_clear_tx_flags(decoder);
 475         decoder->have_tma = false;
 476         decoder->pkt_len = 1;
 477         decoder->pkt_step = 1;
 478         intel_pt_decoder_log_packet(decoder);
 479         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 480                 intel_pt_log("ERROR: Bad packet\n");
 481                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 482         }
 483         return -EBADMSG;
 484 }
 485
 486 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
 487 {
 488         decoder->sample_timestamp = decoder->timestamp;
 489         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
 490 }
 491
 492 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 493 {
 494         struct intel_pt_buffer buffer = { .buf = 0, };
 495         int ret;
 496
 497         decoder->pkt_step = 0;
 498
 499         intel_pt_log("Getting more data\n");
 500         ret = decoder->get_trace(&buffer, decoder->data);
 501         if (ret)
 502                 return ret;
 503         decoder->buf = buffer.buf;
 504         decoder->len = buffer.len;
 505         if (!decoder->len) {
 506                 intel_pt_log("No more data\n");
 507                 return -ENODATA;
 508         }
 509         if (!buffer.consecutive) {
 510                 decoder->ip = 0;
 511                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 512                 decoder->ref_timestamp = buffer.ref_timestamp;
 513                 decoder->timestamp = 0;
 514                 decoder->have_tma = false;
 515                 decoder->state.trace_nr = buffer.trace_nr;
 516                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 517                              decoder->ref_timestamp);
 518                 return -ENOLINK;
 519         }
 520
 521         return 0;
 522 }
 523
 524 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 525 {
 526         if (!decoder->next_buf)
 527                 return intel_pt_get_data(decoder);
 528
 529         decoder->buf = decoder->next_buf;
 530         decoder->len = decoder->next_len;
 531         decoder->next_buf = 0;
 532         decoder->next_len = 0;
 533         return 0;
 534 }
 535
 536 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 537 {
 538         unsigned char *buf = decoder->temp_buf;
 539         size_t old_len, len, n;
 540         int ret;
 541
 542         old_len = decoder->len;
 543         len = decoder->len;
 544         memcpy(buf, decoder->buf, len);
 545
 546         ret = intel_pt_get_data(decoder);
 547         if (ret) {
 548                 decoder->pos += old_len;
 549                 return ret < 0 ? ret : -EINVAL;
 550         }
 551
 552         n = INTEL_PT_PKT_MAX_SZ - len;
 553         if (n > decoder->len)
 554                 n = decoder->len;
 555         memcpy(buf + len, decoder->buf, n);
 556         len += n;
 557
 558         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 559         if (ret < (int)old_len) {
 560                 decoder->next_buf = decoder->buf;
 561                 decoder->next_len = decoder->len;
 562                 decoder->buf = buf;
 563                 decoder->len = old_len;
 564                 return intel_pt_bad_packet(decoder);
 565         }
 566
 567         decoder->next_buf = decoder->buf + (ret - old_len);
 568         decoder->next_len = decoder->len - (ret - old_len);
 569
 570         decoder->buf = buf;
 571         decoder->len = ret;
 572
 573         return ret;
 574 }
 575
 576 struct intel_pt_pkt_info {
 577         struct intel_pt_decoder   *decoder;
 578         struct intel_pt_pkt       packet;
 579         uint64_t                  pos;
 580         int                       pkt_len;
 581         int                       last_packet_type;
 582         void                      *data;
 583 };
 584
 585 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 586
 587 /* Lookahead packets in current buffer */
 588 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 589                                   intel_pt_pkt_cb_t cb, void *data)
 590 {
 591         struct intel_pt_pkt_info pkt_info;
 592         const unsigned char *buf = decoder->buf;
 593         size_t len = decoder->len;
 594         int ret;
 595
 596         pkt_info.decoder          = decoder;
 597         pkt_info.pos              = decoder->pos;
 598         pkt_info.pkt_len          = decoder->pkt_step;
 599         pkt_info.last_packet_type = decoder->last_packet_type;
 600         pkt_info.data             = data;
 601
 602         while (1) {
 603                 do {
 604                         pkt_info.pos += pkt_info.pkt_len;
 605                         buf          += pkt_info.pkt_len;
 606                         len          -= pkt_info.pkt_len;
 607
 608                         if (!len)
 609                                 return INTEL_PT_NEED_MORE_BYTES;
 610
 611                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 612                         if (!ret)
 613                                 return INTEL_PT_NEED_MORE_BYTES;
 614                         if (ret < 0)
 615                                 return ret;
 616
 617                         pkt_info.pkt_len = ret;
 618                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 619
 620                 ret = cb(&pkt_info);
 621                 if (ret)
 622                         return 0;
 623
 624                 pkt_info.last_packet_type = pkt_info.packet.type;
 625         }
 626 }
 627
 628 struct intel_pt_calc_cyc_to_tsc_info {
 629         uint64_t        cycle_cnt;
 630         unsigned int    cbr;
 631         uint32_t        last_mtc;
 632         uint64_t        ctc_timestamp;
 633         uint64_t        ctc_delta;
 634         uint64_t        tsc_timestamp;
 635         uint64_t        timestamp;
 636         bool            have_tma;
 637         bool            fixup_last_mtc;
 638         bool            from_mtc;
 639         double          cbr_cyc_to_tsc;
 640 };
 641
 642 /*
 643  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 644  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 645  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 646  * packet by copying the missing bits from the current MTC assuming the least
 647  * difference between the two, and that the current MTC comes after last_mtc.
 648  */
 649 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 650                                     uint32_t *last_mtc)
 651 {
 652         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 653         uint32_t mask = ~(first_missing_bit - 1);
 654
 655         *last_mtc |= mtc & mask;
 656         if (*last_mtc >= mtc) {
 657                 *last_mtc -= first_missing_bit;
 658                 *last_mtc &= 0xff;
 659         }
 660 }
 661
 662 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 663 {
 664         struct intel_pt_decoder *decoder = pkt_info->decoder;
 665         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 666         uint64_t timestamp;
 667         double cyc_to_tsc;
 668         unsigned int cbr;
 669         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 670
 671         switch (pkt_info->packet.type) {
 672         case INTEL_PT_TNT:
 673         case INTEL_PT_TIP_PGE:
 674         case INTEL_PT_TIP:
 675         case INTEL_PT_FUP:
 676         case INTEL_PT_PSB:
 677         case INTEL_PT_PIP:
 678         case INTEL_PT_MODE_EXEC:
 679         case INTEL_PT_MODE_TSX:
 680         case INTEL_PT_PSBEND:
 681         case INTEL_PT_PAD:
 682         case INTEL_PT_VMCS:
 683         case INTEL_PT_MNT:
 684         case INTEL_PT_PTWRITE:
 685         case INTEL_PT_PTWRITE_IP:
 686                 return 0;
 687
 688         case INTEL_PT_MTC:
 689                 if (!data->have_tma)
 690                         return 0;
 691
 692                 mtc = pkt_info->packet.payload;
 693                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 694                         data->fixup_last_mtc = false;
 695                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 696                                                 &data->last_mtc);
 697                 }
 698                 if (mtc > data->last_mtc)
 699                         mtc_delta = mtc - data->last_mtc;
 700                 else
 701                         mtc_delta = mtc + 256 - data->last_mtc;
 702                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 703                 data->last_mtc = mtc;
 704
 705                 if (decoder->tsc_ctc_mult) {
 706                         timestamp = data->ctc_timestamp +
 707                                 data->ctc_delta * decoder->tsc_ctc_mult;
 708                 } else {
 709                         timestamp = data->ctc_timestamp +
 710                                 multdiv(data->ctc_delta,
 711                                         decoder->tsc_ctc_ratio_n,
 712                                         decoder->tsc_ctc_ratio_d);
 713                 }
 714
 715                 if (timestamp < data->timestamp)
 716                         return 1;
 717
 718                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 719                         data->timestamp = timestamp;
 720                         return 0;
 721                 }
 722
 723                 break;
 724
 725         case INTEL_PT_TSC:
 726                 /*
 727                  * For now, do not support using TSC packets - refer
 728                  * intel_pt_calc_cyc_to_tsc().
 729                  */
 730                 if (data->from_mtc)
 731                         return 1;
 732                 timestamp = pkt_info->packet.payload |
 733                             (data->timestamp & (0xffULL << 56));
 734                 if (data->from_mtc && timestamp < data->timestamp &&
 735                     data->timestamp - timestamp < decoder->tsc_slip)
 736                         return 1;
 737                 if (timestamp < data->timestamp)
 738                         timestamp += (1ULL << 56);
 739                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 740                         if (data->from_mtc)
 741                                 return 1;
 742                         data->tsc_timestamp = timestamp;
 743                         data->timestamp = timestamp;
 744                         return 0;
 745                 }
 746                 break;
 747
 748         case INTEL_PT_TMA:
 749                 if (data->from_mtc)
 750                         return 1;
 751
 752                 if (!decoder->tsc_ctc_ratio_d)
 753                         return 0;
 754
 755                 ctc = pkt_info->packet.payload;
 756                 fc = pkt_info->packet.count;
 757                 ctc_rem = ctc & decoder->ctc_rem_mask;
 758
 759                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 760
 761                 data->ctc_timestamp = data->tsc_timestamp - fc;
 762                 if (decoder->tsc_ctc_mult) {
 763                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 764                 } else {
 765                         data->ctc_timestamp -=
 766                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 767                                         decoder->tsc_ctc_ratio_d);
 768                 }
 769
 770                 data->ctc_delta = 0;
 771                 data->have_tma = true;
 772                 data->fixup_last_mtc = true;
 773
 774                 return 0;
 775
 776         case INTEL_PT_CYC:
 777                 data->cycle_cnt += pkt_info->packet.payload;
 778                 return 0;
 779
 780         case INTEL_PT_CBR:
 781                 cbr = pkt_info->packet.payload;
 782                 if (data->cbr && data->cbr != cbr)
 783                         return 1;
 784                 data->cbr = cbr;
 785                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 786                 return 0;
 787
 788         case INTEL_PT_TIP_PGD:
 789         case INTEL_PT_TRACESTOP:
 790         case INTEL_PT_EXSTOP:
 791         case INTEL_PT_EXSTOP_IP:
 792         case INTEL_PT_MWAIT:
 793         case INTEL_PT_PWRE:
 794         case INTEL_PT_PWRX:
 795         case INTEL_PT_OVF:
 796         case INTEL_PT_BAD: /* Does not happen */
 797         default:
 798                 return 1;
 799         }
 800
 801         if (!data->cbr && decoder->cbr) {
 802                 data->cbr = decoder->cbr;
 803                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 804         }
 805
 806         if (!data->cycle_cnt)
 807                 return 1;
 808
 809         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 810
 811         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 812             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 813                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 814                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 815                 return 1;
 816         }
 817
 818         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 819         decoder->have_calc_cyc_to_tsc = true;
 820
 821         if (data->cbr) {
 822                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 823                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 824         } else {
 825                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 826                              cyc_to_tsc, pkt_info->pos);
 827         }
 828
 829         return 1;
 830 }
 831
 832 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 833                                      bool from_mtc)
 834 {
 835         struct intel_pt_calc_cyc_to_tsc_info data = {
 836                 .cycle_cnt      = 0,
 837                 .cbr            = 0,
 838                 .last_mtc       = decoder->last_mtc,
 839                 .ctc_timestamp  = decoder->ctc_timestamp,
 840                 .ctc_delta      = decoder->ctc_delta,
 841                 .tsc_timestamp  = decoder->tsc_timestamp,
 842                 .timestamp      = decoder->timestamp,
 843                 .have_tma       = decoder->have_tma,
 844                 .fixup_last_mtc = decoder->fixup_last_mtc,
 845                 .from_mtc       = from_mtc,
 846                 .cbr_cyc_to_tsc = 0,
 847         };
 848
 849         /*
 850          * For now, do not support using TSC packets for at least the reasons:
 851          * 1) timing might have stopped
 852          * 2) TSC packets within PSB+ can slip against CYC packets
 853          */
 854         if (!from_mtc)
 855                 return;
 856
 857         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 858 }
 859
 860 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 861 {
 862         int ret;
 863
 864         decoder->last_packet_type = decoder->packet.type;
 865
 866         do {
 867                 decoder->pos += decoder->pkt_step;
 868                 decoder->buf += decoder->pkt_step;
 869                 decoder->len -= decoder->pkt_step;
 870
 871                 if (!decoder->len) {
 872                         ret = intel_pt_get_next_data(decoder);
 873                         if (ret)
 874                                 return ret;
 875                 }
 876
 877                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 878                                           &decoder->packet);
 879                 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
 880                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 881                         ret = intel_pt_get_split_packet(decoder);
 882                         if (ret < 0)
 883                                 return ret;
 884                 }
 885                 if (ret <= 0)
 886                         return intel_pt_bad_packet(decoder);
 887
 888                 decoder->pkt_len = ret;
 889                 decoder->pkt_step = ret;
 890                 intel_pt_decoder_log_packet(decoder);
 891         } while (decoder->packet.type == INTEL_PT_PAD);
 892
 893         return 0;
 894 }
 895
 896 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 897 {
 898         uint64_t timestamp, masked_timestamp;
 899
 900         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 901         masked_timestamp = timestamp & decoder->period_mask;
 902         if (decoder->continuous_period) {
 903                 if (masked_timestamp > decoder->last_masked_timestamp)
 904                         return 1;
 905         } else {
 906                 timestamp += 1;
 907                 masked_timestamp = timestamp & decoder->period_mask;
 908                 if (masked_timestamp > decoder->last_masked_timestamp) {
 909                         decoder->last_masked_timestamp = masked_timestamp;
 910                         decoder->continuous_period = true;
 911                 }
 912         }
 913
 914         if (masked_timestamp < decoder->last_masked_timestamp)
 915                 return decoder->period_ticks;
 916
 917         return decoder->period_ticks - (timestamp - masked_timestamp);
 918 }
 919
 920 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 921 {
 922         switch (decoder->period_type) {
 923         case INTEL_PT_PERIOD_INSTRUCTIONS:
 924                 return decoder->period - decoder->period_insn_cnt;
 925         case INTEL_PT_PERIOD_TICKS:
 926                 return intel_pt_next_period(decoder);
 927         case INTEL_PT_PERIOD_NONE:
 928         case INTEL_PT_PERIOD_MTC:
 929         default:
 930                 return 0;
 931         }
 932 }
 933
 934 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 935 {
 936         uint64_t timestamp, masked_timestamp;
 937
 938         switch (decoder->period_type) {
 939         case INTEL_PT_PERIOD_INSTRUCTIONS:
 940                 decoder->period_insn_cnt = 0;
 941                 break;
 942         case INTEL_PT_PERIOD_TICKS:
 943                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 944                 masked_timestamp = timestamp & decoder->period_mask;
 945                 if (masked_timestamp > decoder->last_masked_timestamp)
 946                         decoder->last_masked_timestamp = masked_timestamp;
 947                 else
 948                         decoder->last_masked_timestamp += decoder->period_ticks;
 949                 break;
 950         case INTEL_PT_PERIOD_NONE:
 951         case INTEL_PT_PERIOD_MTC:
 952         default:
 953                 break;
 954         }
 955
 956         decoder->state.type |= INTEL_PT_INSTRUCTION;
 957 }
 958
 959 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 960                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 961 {
 962         uint64_t max_insn_cnt, insn_cnt = 0;
 963         int err;
 964
 965         if (!decoder->mtc_insn)
 966                 decoder->mtc_insn = true;
 967
 968         max_insn_cnt = intel_pt_next_sample(decoder);
 969
 970         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 971                                  max_insn_cnt, decoder->data);
 972
 973         decoder->tot_insn_cnt += insn_cnt;
 974         decoder->timestamp_insn_cnt += insn_cnt;
 975         decoder->sample_insn_cnt += insn_cnt;
 976         decoder->period_insn_cnt += insn_cnt;
 977
 978         if (err) {
 979                 decoder->no_progress = 0;
 980                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 981                 intel_pt_log_at("ERROR: Failed to get instruction",
 982                                 decoder->ip);
 983                 if (err == -ENOENT)
 984                         return -ENOLINK;
 985                 return -EILSEQ;
 986         }
 987
 988         if (ip && decoder->ip == ip) {
 989                 err = -EAGAIN;
 990                 goto out;
 991         }
 992
 993         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 994                 intel_pt_sample_insn(decoder);
 995
 996         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 997                 decoder->state.type = INTEL_PT_INSTRUCTION;
 998                 decoder->state.from_ip = decoder->ip;
 999                 decoder->state.to_ip = 0;
1000                 decoder->ip += intel_pt_insn->length;
1001                 err = INTEL_PT_RETURN;
1002                 goto out;
1003         }
1004
1005         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1006                 /* Zero-length calls are excluded */
1007                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
1008                     intel_pt_insn->rel) {
1009                         err = intel_pt_push(&decoder->stack, decoder->ip +
1010                                             intel_pt_insn->length);
1011                         if (err)
1012                                 goto out;
1013                 }
1014         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1015                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1016         }
1017
1018         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1019                 int cnt = decoder->no_progress++;
1020
1021                 decoder->state.from_ip = decoder->ip;
1022                 decoder->ip += intel_pt_insn->length +
1023                                 intel_pt_insn->rel;
1024                 decoder->state.to_ip = decoder->ip;
1025                 err = INTEL_PT_RETURN;
1026
1027                 /*
1028                  * Check for being stuck in a loop.  This can happen if a
1029                  * decoder error results in the decoder erroneously setting the
1030                  * ip to an address that is itself in an infinite loop that
1031                  * consumes no packets.  When that happens, there must be an
1032                  * unconditional branch.
1033                  */
1034                 if (cnt) {
1035                         if (cnt == 1) {
1036                                 decoder->stuck_ip = decoder->state.to_ip;
1037                                 decoder->stuck_ip_prd = 1;
1038                                 decoder->stuck_ip_cnt = 1;
1039                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
1040                                    decoder->state.to_ip == decoder->stuck_ip) {
1041                                 intel_pt_log_at("ERROR: Never-ending loop",
1042                                                 decoder->state.to_ip);
1043                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1044                                 err = -ELOOP;
1045                                 goto out;
1046                         } else if (!--decoder->stuck_ip_cnt) {
1047                                 decoder->stuck_ip_prd += 1;
1048                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1049                                 decoder->stuck_ip = decoder->state.to_ip;
1050                         }
1051                 }
1052                 goto out_no_progress;
1053         }
1054 out:
1055         decoder->no_progress = 0;
1056 out_no_progress:
1057         decoder->state.insn_op = intel_pt_insn->op;
1058         decoder->state.insn_len = intel_pt_insn->length;
1059         memcpy(decoder->state.insn, intel_pt_insn->buf,
1060                INTEL_PT_INSN_BUF_SZ);
1061
1062         if (decoder->tx_flags & INTEL_PT_IN_TX)
1063                 decoder->state.flags |= INTEL_PT_IN_TX;
1064
1065         return err;
1066 }
1067
1068 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1069 {
1070         bool ret = false;
1071
1072         if (decoder->set_fup_tx_flags) {
1073                 decoder->set_fup_tx_flags = false;
1074                 decoder->tx_flags = decoder->fup_tx_flags;
1075                 decoder->state.type = INTEL_PT_TRANSACTION;
1076                 decoder->state.from_ip = decoder->ip;
1077                 decoder->state.to_ip = 0;
1078                 decoder->state.flags = decoder->fup_tx_flags;
1079                 return true;
1080         }
1081         if (decoder->set_fup_ptw) {
1082                 decoder->set_fup_ptw = false;
1083                 decoder->state.type = INTEL_PT_PTW;
1084                 decoder->state.flags |= INTEL_PT_FUP_IP;
1085                 decoder->state.from_ip = decoder->ip;
1086                 decoder->state.to_ip = 0;
1087                 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1088                 return true;
1089         }
1090         if (decoder->set_fup_mwait) {
1091                 decoder->set_fup_mwait = false;
1092                 decoder->state.type = INTEL_PT_MWAIT_OP;
1093                 decoder->state.from_ip = decoder->ip;
1094                 decoder->state.to_ip = 0;
1095                 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1096                 ret = true;
1097         }
1098         if (decoder->set_fup_pwre) {
1099                 decoder->set_fup_pwre = false;
1100                 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1101                 decoder->state.type &= ~INTEL_PT_BRANCH;
1102                 decoder->state.from_ip = decoder->ip;
1103                 decoder->state.to_ip = 0;
1104                 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1105                 ret = true;
1106         }
1107         if (decoder->set_fup_exstop) {
1108                 decoder->set_fup_exstop = false;
1109                 decoder->state.type |= INTEL_PT_EX_STOP;
1110                 decoder->state.type &= ~INTEL_PT_BRANCH;
1111                 decoder->state.flags |= INTEL_PT_FUP_IP;
1112                 decoder->state.from_ip = decoder->ip;
1113                 decoder->state.to_ip = 0;
1114                 ret = true;
1115         }
1116         return ret;
1117 }
1118
1119 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1120                                           struct intel_pt_insn *intel_pt_insn,
1121                                           uint64_t ip, int err)
1122 {
1123         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1124                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1125                ip == decoder->ip + intel_pt_insn->length;
1126 }
1127
1128 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1129 {
1130         struct intel_pt_insn intel_pt_insn;
1131         uint64_t ip;
1132         int err;
1133
1134         ip = decoder->last_ip;
1135
1136         while (1) {
1137                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1138                 if (err == INTEL_PT_RETURN)
1139                         return 0;
1140                 if (err == -EAGAIN ||
1141                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1142                         if (intel_pt_fup_event(decoder))
1143                                 return 0;
1144                         return -EAGAIN;
1145                 }
1146                 decoder->set_fup_tx_flags = false;
1147                 if (err)
1148                         return err;
1149
1150                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1151                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1152                                         decoder->ip);
1153                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1154                         return -ENOENT;
1155                 }
1156
1157                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1158                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1159                                         decoder->ip);
1160                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1161                         return -ENOENT;
1162                 }
1163
1164                 intel_pt_bug(decoder);
1165         }
1166 }
1167
1168 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1169 {
1170         struct intel_pt_insn intel_pt_insn;
1171         int err;
1172
1173         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1174         if (err == INTEL_PT_RETURN &&
1175             decoder->pgd_ip &&
1176             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1177             (decoder->state.type & INTEL_PT_BRANCH) &&
1178             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1179                 /* Unconditional branch leaving filter region */
1180                 decoder->no_progress = 0;
1181                 decoder->pge = false;
1182                 decoder->continuous_period = false;
1183                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1184                 decoder->state.type |= INTEL_PT_TRACE_END;
1185                 return 0;
1186         }
1187         if (err == INTEL_PT_RETURN)
1188                 return 0;
1189         if (err)
1190                 return err;
1191
1192         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1193                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1194                         decoder->pge = false;
1195                         decoder->continuous_period = false;
1196                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1197                         decoder->state.from_ip = decoder->ip;
1198                         if (decoder->packet.count == 0) {
1199                                 decoder->state.to_ip = 0;
1200                         } else {
1201                                 decoder->state.to_ip = decoder->last_ip;
1202                                 decoder->ip = decoder->last_ip;
1203                         }
1204                         decoder->state.type |= INTEL_PT_TRACE_END;
1205                 } else {
1206                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1207                         decoder->state.from_ip = decoder->ip;
1208                         if (decoder->packet.count == 0) {
1209                                 decoder->state.to_ip = 0;
1210                         } else {
1211                                 decoder->state.to_ip = decoder->last_ip;
1212                                 decoder->ip = decoder->last_ip;
1213                         }
1214                 }
1215                 return 0;
1216         }
1217
1218         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1219                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1220                                  intel_pt_insn.rel;
1221
1222                 if (decoder->pgd_ip &&
1223                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1224                     decoder->pgd_ip(to_ip, decoder->data)) {
1225                         /* Conditional branch leaving filter region */
1226                         decoder->pge = false;
1227                         decoder->continuous_period = false;
1228                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1229                         decoder->ip = to_ip;
1230                         decoder->state.from_ip = decoder->ip;
1231                         decoder->state.to_ip = to_ip;
1232                         decoder->state.type |= INTEL_PT_TRACE_END;
1233                         return 0;
1234                 }
1235                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1236                                 decoder->ip);
1237                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1238                 return -ENOENT;
1239         }
1240
1241         return intel_pt_bug(decoder);
1242 }
1243
1244 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1245 {
1246         struct intel_pt_insn intel_pt_insn;
1247         int err;
1248
1249         while (1) {
1250                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1251                 if (err == INTEL_PT_RETURN)
1252                         return 0;
1253                 if (err)
1254                         return err;
1255
1256                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1257                         if (!decoder->return_compression) {
1258                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1259                                                 decoder->ip);
1260                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1261                                 return -ENOENT;
1262                         }
1263                         if (!decoder->ret_addr) {
1264                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1265                                                 decoder->ip);
1266                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1267                                 return -ENOENT;
1268                         }
1269                         if (!(decoder->tnt.payload & BIT63)) {
1270                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1271                                                 decoder->ip);
1272                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1273                                 return -ENOENT;
1274                         }
1275                         decoder->tnt.count -= 1;
1276                         if (decoder->tnt.count)
1277                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1278                         else
1279                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1280                         decoder->tnt.payload <<= 1;
1281                         decoder->state.from_ip = decoder->ip;
1282                         decoder->ip = decoder->ret_addr;
1283                         decoder->state.to_ip = decoder->ip;
1284                         return 0;
1285                 }
1286
1287                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1288                         /* Handle deferred TIPs */
1289                         err = intel_pt_get_next_packet(decoder);
1290                         if (err)
1291                                 return err;
1292                         if (decoder->packet.type != INTEL_PT_TIP ||
1293                             decoder->packet.count == 0) {
1294                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1295                                                 decoder->ip);
1296                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1297                                 decoder->pkt_step = 0;
1298                                 return -ENOENT;
1299                         }
1300                         intel_pt_set_last_ip(decoder);
1301                         decoder->state.from_ip = decoder->ip;
1302                         decoder->state.to_ip = decoder->last_ip;
1303                         decoder->ip = decoder->last_ip;
1304                         return 0;
1305                 }
1306
1307                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1308                         decoder->tnt.count -= 1;
1309                         if (decoder->tnt.count)
1310                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1311                         else
1312                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1313                         if (decoder->tnt.payload & BIT63) {
1314                                 decoder->tnt.payload <<= 1;
1315                                 decoder->state.from_ip = decoder->ip;
1316                                 decoder->ip += intel_pt_insn.length +
1317                                                intel_pt_insn.rel;
1318                                 decoder->state.to_ip = decoder->ip;
1319                                 return 0;
1320                         }
1321                         /* Instruction sample for a non-taken branch */
1322                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1323                                 decoder->tnt.payload <<= 1;
1324                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1325                                 decoder->state.from_ip = decoder->ip;
1326                                 decoder->state.to_ip = 0;
1327                                 decoder->ip += intel_pt_insn.length;
1328                                 return 0;
1329                         }
1330                         decoder->sample_cyc = false;
1331                         decoder->ip += intel_pt_insn.length;
1332                         if (!decoder->tnt.count) {
1333                                 intel_pt_update_sample_time(decoder);
1334                                 return -EAGAIN;
1335                         }
1336                         decoder->tnt.payload <<= 1;
1337                         continue;
1338                 }
1339
1340                 return intel_pt_bug(decoder);
1341         }
1342 }
1343
1344 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1345 {
1346         unsigned int fup_tx_flags;
1347         int err;
1348
1349         fup_tx_flags = decoder->packet.payload &
1350                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1351         err = intel_pt_get_next_packet(decoder);
1352         if (err)
1353                 return err;
1354         if (decoder->packet.type == INTEL_PT_FUP) {
1355                 decoder->fup_tx_flags = fup_tx_flags;
1356                 decoder->set_fup_tx_flags = true;
1357                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1358                         *no_tip = true;
1359         } else {
1360                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1361                                 decoder->pos);
1362                 intel_pt_update_in_tx(decoder);
1363         }
1364         return 0;
1365 }
1366
1367 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1368 {
1369         uint64_t timestamp;
1370
1371         decoder->have_tma = false;
1372
1373         if (decoder->ref_timestamp) {
1374                 timestamp = decoder->packet.payload |
1375                             (decoder->ref_timestamp & (0xffULL << 56));
1376                 if (timestamp < decoder->ref_timestamp) {
1377                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1378                                 timestamp += (1ULL << 56);
1379                 } else {
1380                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1381                                 timestamp -= (1ULL << 56);
1382                 }
1383                 decoder->tsc_timestamp = timestamp;
1384                 decoder->timestamp = timestamp;
1385                 decoder->ref_timestamp = 0;
1386                 decoder->timestamp_insn_cnt = 0;
1387         } else if (decoder->timestamp) {
1388                 timestamp = decoder->packet.payload |
1389                             (decoder->timestamp & (0xffULL << 56));
1390                 decoder->tsc_timestamp = timestamp;
1391                 if (timestamp < decoder->timestamp &&
1392                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1393                         intel_pt_log_to("Suppressing backwards timestamp",
1394                                         timestamp);
1395                         timestamp = decoder->timestamp;
1396                 }
1397                 if (timestamp < decoder->timestamp) {
1398                         intel_pt_log_to("Wraparound timestamp", timestamp);
1399                         timestamp += (1ULL << 56);
1400                         decoder->tsc_timestamp = timestamp;
1401                 }
1402                 decoder->timestamp = timestamp;
1403                 decoder->timestamp_insn_cnt = 0;
1404         }
1405
1406         if (decoder->last_packet_type == INTEL_PT_CYC) {
1407                 decoder->cyc_ref_timestamp = decoder->timestamp;
1408                 decoder->cycle_cnt = 0;
1409                 decoder->have_calc_cyc_to_tsc = false;
1410                 intel_pt_calc_cyc_to_tsc(decoder, false);
1411         }
1412
1413         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1414 }
1415
1416 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1417 {
1418         intel_pt_log("ERROR: Buffer overflow\n");
1419         intel_pt_clear_tx_flags(decoder);
1420         decoder->timestamp_insn_cnt = 0;
1421         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1422         decoder->overflow = true;
1423         return -EOVERFLOW;
1424 }
1425
1426 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1427 {
1428         uint32_t ctc = decoder->packet.payload;
1429         uint32_t fc = decoder->packet.count;
1430         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1431
1432         if (!decoder->tsc_ctc_ratio_d)
1433                 return;
1434
1435         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1436         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1437         if (decoder->tsc_ctc_mult) {
1438                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1439         } else {
1440                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1441                                                   decoder->tsc_ctc_ratio_n,
1442                                                   decoder->tsc_ctc_ratio_d);
1443         }
1444         decoder->ctc_delta = 0;
1445         decoder->have_tma = true;
1446         decoder->fixup_last_mtc = true;
1447         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1448                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1449 }
1450
1451 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1452 {
1453         uint64_t timestamp;
1454         uint32_t mtc, mtc_delta;
1455
1456         if (!decoder->have_tma)
1457                 return;
1458
1459         mtc = decoder->packet.payload;
1460
1461         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1462                 decoder->fixup_last_mtc = false;
1463                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1464                                         &decoder->last_mtc);
1465         }
1466
1467         if (mtc > decoder->last_mtc)
1468                 mtc_delta = mtc - decoder->last_mtc;
1469         else
1470                 mtc_delta = mtc + 256 - decoder->last_mtc;
1471
1472         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1473
1474         if (decoder->tsc_ctc_mult) {
1475                 timestamp = decoder->ctc_timestamp +
1476                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1477         } else {
1478                 timestamp = decoder->ctc_timestamp +
1479                             multdiv(decoder->ctc_delta,
1480                                     decoder->tsc_ctc_ratio_n,
1481                                     decoder->tsc_ctc_ratio_d);
1482         }
1483
1484         if (timestamp < decoder->timestamp)
1485                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1486                              timestamp, decoder->timestamp);
1487         else
1488                 decoder->timestamp = timestamp;
1489
1490         decoder->timestamp_insn_cnt = 0;
1491         decoder->last_mtc = mtc;
1492
1493         if (decoder->last_packet_type == INTEL_PT_CYC) {
1494                 decoder->cyc_ref_timestamp = decoder->timestamp;
1495                 decoder->cycle_cnt = 0;
1496                 decoder->have_calc_cyc_to_tsc = false;
1497                 intel_pt_calc_cyc_to_tsc(decoder, true);
1498         }
1499
1500         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1501 }
1502
1503 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1504 {
1505         unsigned int cbr = decoder->packet.payload & 0xff;
1506
1507         decoder->cbr_payload = decoder->packet.payload;
1508
1509         if (decoder->cbr == cbr)
1510                 return;
1511
1512         decoder->cbr = cbr;
1513         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1514 }
1515
1516 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1517 {
1518         uint64_t timestamp = decoder->cyc_ref_timestamp;
1519
1520         decoder->have_cyc = true;
1521
1522         decoder->cycle_cnt += decoder->packet.payload;
1523         if (decoder->pge)
1524                 decoder->tot_cyc_cnt += decoder->packet.payload;
1525         decoder->sample_cyc = true;
1526
1527         if (!decoder->cyc_ref_timestamp)
1528                 return;
1529
1530         if (decoder->have_calc_cyc_to_tsc)
1531                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1532         else if (decoder->cbr)
1533                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1534         else
1535                 return;
1536
1537         if (timestamp < decoder->timestamp)
1538                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1539                              timestamp, decoder->timestamp);
1540         else
1541                 decoder->timestamp = timestamp;
1542
1543         decoder->timestamp_insn_cnt = 0;
1544
1545         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1546 }
1547
1548 /* Walk PSB+ packets when already in sync. */
1549 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1550 {
1551         int err;
1552
1553         decoder->in_psb = true;
1554
1555         while (1) {
1556                 err = intel_pt_get_next_packet(decoder);
1557                 if (err)
1558                         goto out;
1559
1560                 switch (decoder->packet.type) {
1561                 case INTEL_PT_PSBEND:
1562                         err = 0;
1563                         goto out;
1564
1565                 case INTEL_PT_TIP_PGD:
1566                 case INTEL_PT_TIP_PGE:
1567                 case INTEL_PT_TIP:
1568                 case INTEL_PT_TNT:
1569                 case INTEL_PT_TRACESTOP:
1570                 case INTEL_PT_BAD:
1571                 case INTEL_PT_PSB:
1572                 case INTEL_PT_PTWRITE:
1573                 case INTEL_PT_PTWRITE_IP:
1574                 case INTEL_PT_EXSTOP:
1575                 case INTEL_PT_EXSTOP_IP:
1576                 case INTEL_PT_MWAIT:
1577                 case INTEL_PT_PWRE:
1578                 case INTEL_PT_PWRX:
1579                         decoder->have_tma = false;
1580                         intel_pt_log("ERROR: Unexpected packet\n");
1581                         err = -EAGAIN;
1582                         goto out;
1583
1584                 case INTEL_PT_OVF:
1585                         err = intel_pt_overflow(decoder);
1586                         goto out;
1587
1588                 case INTEL_PT_TSC:
1589                         intel_pt_calc_tsc_timestamp(decoder);
1590                         break;
1591
1592                 case INTEL_PT_TMA:
1593                         intel_pt_calc_tma(decoder);
1594                         break;
1595
1596                 case INTEL_PT_CBR:
1597                         intel_pt_calc_cbr(decoder);
1598                         break;
1599
1600                 case INTEL_PT_MODE_EXEC:
1601                         decoder->exec_mode = decoder->packet.payload;
1602                         break;
1603
1604                 case INTEL_PT_PIP:
1605                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1606                         break;
1607
1608                 case INTEL_PT_FUP:
1609                         decoder->pge = true;
1610                         if (decoder->packet.count)
1611                                 intel_pt_set_last_ip(decoder);
1612                         break;
1613
1614                 case INTEL_PT_MODE_TSX:
1615                         intel_pt_update_in_tx(decoder);
1616                         break;
1617
1618                 case INTEL_PT_MTC:
1619                         intel_pt_calc_mtc_timestamp(decoder);
1620                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1621                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1622                         break;
1623
1624                 case INTEL_PT_CYC:
1625                 case INTEL_PT_VMCS:
1626                 case INTEL_PT_MNT:
1627                 case INTEL_PT_PAD:
1628                 default:
1629                         break;
1630                 }
1631         }
1632 out:
1633         decoder->in_psb = false;
1634
1635         return err;
1636 }
1637
1638 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1639 {
1640         int err;
1641
1642         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1643                 decoder->tx_flags = 0;
1644                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1645                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1646         } else {
1647                 decoder->state.flags |= INTEL_PT_ASYNC;
1648         }
1649
1650         while (1) {
1651                 err = intel_pt_get_next_packet(decoder);
1652                 if (err)
1653                         return err;
1654
1655                 switch (decoder->packet.type) {
1656                 case INTEL_PT_TNT:
1657                 case INTEL_PT_FUP:
1658                 case INTEL_PT_TRACESTOP:
1659                 case INTEL_PT_PSB:
1660                 case INTEL_PT_TSC:
1661                 case INTEL_PT_TMA:
1662                 case INTEL_PT_MODE_TSX:
1663                 case INTEL_PT_BAD:
1664                 case INTEL_PT_PSBEND:
1665                 case INTEL_PT_PTWRITE:
1666                 case INTEL_PT_PTWRITE_IP:
1667                 case INTEL_PT_EXSTOP:
1668                 case INTEL_PT_EXSTOP_IP:
1669                 case INTEL_PT_MWAIT:
1670                 case INTEL_PT_PWRE:
1671                 case INTEL_PT_PWRX:
1672                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1673                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1674                         decoder->pkt_step = 0;
1675                         return -ENOENT;
1676
1677                 case INTEL_PT_CBR:
1678                         intel_pt_calc_cbr(decoder);
1679                         break;
1680
1681                 case INTEL_PT_OVF:
1682                         return intel_pt_overflow(decoder);
1683
1684                 case INTEL_PT_TIP_PGD:
1685                         decoder->state.from_ip = decoder->ip;
1686                         if (decoder->packet.count == 0) {
1687                                 decoder->state.to_ip = 0;
1688                         } else {
1689                                 intel_pt_set_ip(decoder);
1690                                 decoder->state.to_ip = decoder->ip;
1691                         }
1692                         decoder->pge = false;
1693                         decoder->continuous_period = false;
1694                         decoder->state.type |= INTEL_PT_TRACE_END;
1695                         return 0;
1696
1697                 case INTEL_PT_TIP_PGE:
1698                         decoder->pge = true;
1699                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1700                                      decoder->ip);
1701                         decoder->state.from_ip = 0;
1702                         if (decoder->packet.count == 0) {
1703                                 decoder->state.to_ip = 0;
1704                         } else {
1705                                 intel_pt_set_ip(decoder);
1706                                 decoder->state.to_ip = decoder->ip;
1707                         }
1708                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1709                         return 0;
1710
1711                 case INTEL_PT_TIP:
1712                         decoder->state.from_ip = decoder->ip;
1713                         if (decoder->packet.count == 0) {
1714                                 decoder->state.to_ip = 0;
1715                         } else {
1716                                 intel_pt_set_ip(decoder);
1717                                 decoder->state.to_ip = decoder->ip;
1718                         }
1719                         return 0;
1720
1721                 case INTEL_PT_PIP:
1722                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1723                         break;
1724
1725                 case INTEL_PT_MTC:
1726                         intel_pt_calc_mtc_timestamp(decoder);
1727                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1728                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1729                         break;
1730
1731                 case INTEL_PT_CYC:
1732                         intel_pt_calc_cyc_timestamp(decoder);
1733                         break;
1734
1735                 case INTEL_PT_MODE_EXEC:
1736                         decoder->exec_mode = decoder->packet.payload;
1737                         break;
1738
1739                 case INTEL_PT_VMCS:
1740                 case INTEL_PT_MNT:
1741                 case INTEL_PT_PAD:
1742                         break;
1743
1744                 default:
1745                         return intel_pt_bug(decoder);
1746                 }
1747         }
1748 }
1749
1750 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1751 {
1752         bool no_tip = false;
1753         int err;
1754
1755         while (1) {
1756                 err = intel_pt_get_next_packet(decoder);
1757                 if (err)
1758                         return err;
1759 next:
1760                 switch (decoder->packet.type) {
1761                 case INTEL_PT_TNT:
1762                         if (!decoder->packet.count)
1763                                 break;
1764                         decoder->tnt = decoder->packet;
1765                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1766                         err = intel_pt_walk_tnt(decoder);
1767                         if (err == -EAGAIN)
1768                                 break;
1769                         return err;
1770
1771                 case INTEL_PT_TIP_PGD:
1772                         if (decoder->packet.count != 0)
1773                                 intel_pt_set_last_ip(decoder);
1774                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1775                         return intel_pt_walk_tip(decoder);
1776
1777                 case INTEL_PT_TIP_PGE: {
1778                         decoder->pge = true;
1779                         if (decoder->packet.count == 0) {
1780                                 intel_pt_log_at("Skipping zero TIP.PGE",
1781                                                 decoder->pos);
1782                                 break;
1783                         }
1784                         intel_pt_set_ip(decoder);
1785                         decoder->state.from_ip = 0;
1786                         decoder->state.to_ip = decoder->ip;
1787                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1788                         return 0;
1789                 }
1790
1791                 case INTEL_PT_OVF:
1792                         return intel_pt_overflow(decoder);
1793
1794                 case INTEL_PT_TIP:
1795                         if (decoder->packet.count != 0)
1796                                 intel_pt_set_last_ip(decoder);
1797                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1798                         return intel_pt_walk_tip(decoder);
1799
1800                 case INTEL_PT_FUP:
1801                         if (decoder->packet.count == 0) {
1802                                 intel_pt_log_at("Skipping zero FUP",
1803                                                 decoder->pos);
1804                                 no_tip = false;
1805                                 break;
1806                         }
1807                         intel_pt_set_last_ip(decoder);
1808                         if (!decoder->branch_enable) {
1809                                 decoder->ip = decoder->last_ip;
1810                                 if (intel_pt_fup_event(decoder))
1811                                         return 0;
1812                                 no_tip = false;
1813                                 break;
1814                         }
1815                         if (decoder->set_fup_mwait)
1816                                 no_tip = true;
1817                         err = intel_pt_walk_fup(decoder);
1818                         if (err != -EAGAIN) {
1819                                 if (err)
1820                                         return err;
1821                                 if (no_tip)
1822                                         decoder->pkt_state =
1823                                                 INTEL_PT_STATE_FUP_NO_TIP;
1824                                 else
1825                                         decoder->pkt_state = INTEL_PT_STATE_FUP;
1826                                 return 0;
1827                         }
1828                         if (no_tip) {
1829                                 no_tip = false;
1830                                 break;
1831                         }
1832                         return intel_pt_walk_fup_tip(decoder);
1833
1834                 case INTEL_PT_TRACESTOP:
1835                         decoder->pge = false;
1836                         decoder->continuous_period = false;
1837                         intel_pt_clear_tx_flags(decoder);
1838                         decoder->have_tma = false;
1839                         break;
1840
1841                 case INTEL_PT_PSB:
1842                         decoder->last_ip = 0;
1843                         decoder->have_last_ip = true;
1844                         intel_pt_clear_stack(&decoder->stack);
1845                         err = intel_pt_walk_psbend(decoder);
1846                         if (err == -EAGAIN)
1847                                 goto next;
1848                         if (err)
1849                                 return err;
1850                         break;
1851
1852                 case INTEL_PT_PIP:
1853                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1854                         break;
1855
1856                 case INTEL_PT_MTC:
1857                         intel_pt_calc_mtc_timestamp(decoder);
1858                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1859                                 break;
1860                         /*
1861                          * Ensure that there has been an instruction since the
1862                          * last MTC.
1863                          */
1864                         if (!decoder->mtc_insn)
1865                                 break;
1866                         decoder->mtc_insn = false;
1867                         /* Ensure that there is a timestamp */
1868                         if (!decoder->timestamp)
1869                                 break;
1870                         decoder->state.type = INTEL_PT_INSTRUCTION;
1871                         decoder->state.from_ip = decoder->ip;
1872                         decoder->state.to_ip = 0;
1873                         decoder->mtc_insn = false;
1874                         return 0;
1875
1876                 case INTEL_PT_TSC:
1877                         intel_pt_calc_tsc_timestamp(decoder);
1878                         break;
1879
1880                 case INTEL_PT_TMA:
1881                         intel_pt_calc_tma(decoder);
1882                         break;
1883
1884                 case INTEL_PT_CYC:
1885                         intel_pt_calc_cyc_timestamp(decoder);
1886                         break;
1887
1888                 case INTEL_PT_CBR:
1889                         intel_pt_calc_cbr(decoder);
1890                         if (!decoder->branch_enable &&
1891                             decoder->cbr != decoder->cbr_seen) {
1892                                 decoder->cbr_seen = decoder->cbr;
1893                                 decoder->state.type = INTEL_PT_CBR_CHG;
1894                                 decoder->state.from_ip = decoder->ip;
1895                                 decoder->state.to_ip = 0;
1896                                 decoder->state.cbr_payload =
1897                                                         decoder->packet.payload;
1898                                 return 0;
1899                         }
1900                         break;
1901
1902                 case INTEL_PT_MODE_EXEC:
1903                         decoder->exec_mode = decoder->packet.payload;
1904                         break;
1905
1906                 case INTEL_PT_MODE_TSX:
1907                         /* MODE_TSX need not be followed by FUP */
1908                         if (!decoder->pge) {
1909                                 intel_pt_update_in_tx(decoder);
1910                                 break;
1911                         }
1912                         err = intel_pt_mode_tsx(decoder, &no_tip);
1913                         if (err)
1914                                 return err;
1915                         goto next;
1916
1917                 case INTEL_PT_BAD: /* Does not happen */
1918                         return intel_pt_bug(decoder);
1919
1920                 case INTEL_PT_PSBEND:
1921                 case INTEL_PT_VMCS:
1922                 case INTEL_PT_MNT:
1923                 case INTEL_PT_PAD:
1924                         break;
1925
1926                 case INTEL_PT_PTWRITE_IP:
1927                         decoder->fup_ptw_payload = decoder->packet.payload;
1928                         err = intel_pt_get_next_packet(decoder);
1929                         if (err)
1930                                 return err;
1931                         if (decoder->packet.type == INTEL_PT_FUP) {
1932                                 decoder->set_fup_ptw = true;
1933                                 no_tip = true;
1934                         } else {
1935                                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
1936                                                 decoder->pos);
1937                         }
1938                         goto next;
1939
1940                 case INTEL_PT_PTWRITE:
1941                         decoder->state.type = INTEL_PT_PTW;
1942                         decoder->state.from_ip = decoder->ip;
1943                         decoder->state.to_ip = 0;
1944                         decoder->state.ptw_payload = decoder->packet.payload;
1945                         return 0;
1946
1947                 case INTEL_PT_MWAIT:
1948                         decoder->fup_mwait_payload = decoder->packet.payload;
1949                         decoder->set_fup_mwait = true;
1950                         break;
1951
1952                 case INTEL_PT_PWRE:
1953                         if (decoder->set_fup_mwait) {
1954                                 decoder->fup_pwre_payload =
1955                                                         decoder->packet.payload;
1956                                 decoder->set_fup_pwre = true;
1957                                 break;
1958                         }
1959                         decoder->state.type = INTEL_PT_PWR_ENTRY;
1960                         decoder->state.from_ip = decoder->ip;
1961                         decoder->state.to_ip = 0;
1962                         decoder->state.pwrx_payload = decoder->packet.payload;
1963                         return 0;
1964
1965                 case INTEL_PT_EXSTOP_IP:
1966                         err = intel_pt_get_next_packet(decoder);
1967                         if (err)
1968                                 return err;
1969                         if (decoder->packet.type == INTEL_PT_FUP) {
1970                                 decoder->set_fup_exstop = true;
1971                                 no_tip = true;
1972                         } else {
1973                                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
1974                                                 decoder->pos);
1975                         }
1976                         goto next;
1977
1978                 case INTEL_PT_EXSTOP:
1979                         decoder->state.type = INTEL_PT_EX_STOP;
1980                         decoder->state.from_ip = decoder->ip;
1981                         decoder->state.to_ip = 0;
1982                         return 0;
1983
1984                 case INTEL_PT_PWRX:
1985                         decoder->state.type = INTEL_PT_PWR_EXIT;
1986                         decoder->state.from_ip = decoder->ip;
1987                         decoder->state.to_ip = 0;
1988                         decoder->state.pwrx_payload = decoder->packet.payload;
1989                         return 0;
1990
1991                 default:
1992                         return intel_pt_bug(decoder);
1993                 }
1994         }
1995 }
1996
1997 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1998 {
1999         return decoder->packet.count &&
2000                (decoder->have_last_ip || decoder->packet.count == 3 ||
2001                 decoder->packet.count == 6);
2002 }
2003
2004 /* Walk PSB+ packets to get in sync. */
2005 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
2006 {
2007         int err;
2008
2009         decoder->in_psb = true;
2010
2011         while (1) {
2012                 err = intel_pt_get_next_packet(decoder);
2013                 if (err)
2014                         goto out;
2015
2016                 switch (decoder->packet.type) {
2017                 case INTEL_PT_TIP_PGD:
2018                         decoder->continuous_period = false;
2019                         __fallthrough;
2020                 case INTEL_PT_TIP_PGE:
2021                 case INTEL_PT_TIP:
2022                 case INTEL_PT_PTWRITE:
2023                 case INTEL_PT_PTWRITE_IP:
2024                 case INTEL_PT_EXSTOP:
2025                 case INTEL_PT_EXSTOP_IP:
2026                 case INTEL_PT_MWAIT:
2027                 case INTEL_PT_PWRE:
2028                 case INTEL_PT_PWRX:
2029                         intel_pt_log("ERROR: Unexpected packet\n");
2030                         err = -ENOENT;
2031                         goto out;
2032
2033                 case INTEL_PT_FUP:
2034                         decoder->pge = true;
2035                         if (intel_pt_have_ip(decoder)) {
2036                                 uint64_t current_ip = decoder->ip;
2037
2038                                 intel_pt_set_ip(decoder);
2039                                 if (current_ip)
2040                                         intel_pt_log_to("Setting IP",
2041                                                         decoder->ip);
2042                         }
2043                         break;
2044
2045                 case INTEL_PT_MTC:
2046                         intel_pt_calc_mtc_timestamp(decoder);
2047                         break;
2048
2049                 case INTEL_PT_TSC:
2050                         intel_pt_calc_tsc_timestamp(decoder);
2051                         break;
2052
2053                 case INTEL_PT_TMA:
2054                         intel_pt_calc_tma(decoder);
2055                         break;
2056
2057                 case INTEL_PT_CYC:
2058                         intel_pt_calc_cyc_timestamp(decoder);
2059                         break;
2060
2061                 case INTEL_PT_CBR:
2062                         intel_pt_calc_cbr(decoder);
2063                         break;
2064
2065                 case INTEL_PT_PIP:
2066                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2067                         break;
2068
2069                 case INTEL_PT_MODE_EXEC:
2070                         decoder->exec_mode = decoder->packet.payload;
2071                         break;
2072
2073                 case INTEL_PT_MODE_TSX:
2074                         intel_pt_update_in_tx(decoder);
2075                         break;
2076
2077                 case INTEL_PT_TRACESTOP:
2078                         decoder->pge = false;
2079                         decoder->continuous_period = false;
2080                         intel_pt_clear_tx_flags(decoder);
2081                         __fallthrough;
2082
2083                 case INTEL_PT_TNT:
2084                         decoder->have_tma = false;
2085                         intel_pt_log("ERROR: Unexpected packet\n");
2086                         if (decoder->ip)
2087                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
2088                         else
2089                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2090                         err = -ENOENT;
2091                         goto out;
2092
2093                 case INTEL_PT_BAD: /* Does not happen */
2094                         err = intel_pt_bug(decoder);
2095                         goto out;
2096
2097                 case INTEL_PT_OVF:
2098                         err = intel_pt_overflow(decoder);
2099                         goto out;
2100
2101                 case INTEL_PT_PSBEND:
2102                         err = 0;
2103                         goto out;
2104
2105                 case INTEL_PT_PSB:
2106                 case INTEL_PT_VMCS:
2107                 case INTEL_PT_MNT:
2108                 case INTEL_PT_PAD:
2109                 default:
2110                         break;
2111                 }
2112         }
2113 out:
2114         decoder->in_psb = false;
2115
2116         return err;
2117 }
2118
2119 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
2120 {
2121         int err;
2122
2123         while (1) {
2124                 err = intel_pt_get_next_packet(decoder);
2125                 if (err)
2126                         return err;
2127
2128                 switch (decoder->packet.type) {
2129                 case INTEL_PT_TIP_PGD:
2130                         decoder->continuous_period = false;
2131                         __fallthrough;
2132                 case INTEL_PT_TIP_PGE:
2133                 case INTEL_PT_TIP:
2134                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
2135                         if (intel_pt_have_ip(decoder))
2136                                 intel_pt_set_ip(decoder);
2137                         if (!decoder->ip)
2138                                 break;
2139                         if (decoder->packet.type == INTEL_PT_TIP_PGE)
2140                                 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2141                         if (decoder->packet.type == INTEL_PT_TIP_PGD)
2142                                 decoder->state.type |= INTEL_PT_TRACE_END;
2143                         return 0;
2144
2145                 case INTEL_PT_FUP:
2146                         if (intel_pt_have_ip(decoder))
2147                                 intel_pt_set_ip(decoder);
2148                         if (decoder->ip)
2149                                 return 0;
2150                         break;
2151
2152                 case INTEL_PT_MTC:
2153                         intel_pt_calc_mtc_timestamp(decoder);
2154                         break;
2155
2156                 case INTEL_PT_TSC:
2157                         intel_pt_calc_tsc_timestamp(decoder);
2158                         break;
2159
2160                 case INTEL_PT_TMA:
2161                         intel_pt_calc_tma(decoder);
2162                         break;
2163
2164                 case INTEL_PT_CYC:
2165                         intel_pt_calc_cyc_timestamp(decoder);
2166                         break;
2167
2168                 case INTEL_PT_CBR:
2169                         intel_pt_calc_cbr(decoder);
2170                         break;
2171
2172                 case INTEL_PT_PIP:
2173                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2174                         break;
2175
2176                 case INTEL_PT_MODE_EXEC:
2177                         decoder->exec_mode = decoder->packet.payload;
2178                         break;
2179
2180                 case INTEL_PT_MODE_TSX:
2181                         intel_pt_update_in_tx(decoder);
2182                         break;
2183
2184                 case INTEL_PT_OVF:
2185                         return intel_pt_overflow(decoder);
2186
2187                 case INTEL_PT_BAD: /* Does not happen */
2188                         return intel_pt_bug(decoder);
2189
2190                 case INTEL_PT_TRACESTOP:
2191                         decoder->pge = false;
2192                         decoder->continuous_period = false;
2193                         intel_pt_clear_tx_flags(decoder);
2194                         decoder->have_tma = false;
2195                         break;
2196
2197                 case INTEL_PT_PSB:
2198                         decoder->last_ip = 0;
2199                         decoder->have_last_ip = true;
2200                         intel_pt_clear_stack(&decoder->stack);
2201                         err = intel_pt_walk_psb(decoder);
2202                         if (err)
2203                                 return err;
2204                         if (decoder->ip) {
2205                                 /* Do not have a sample */
2206                                 decoder->state.type = 0;
2207                                 return 0;
2208                         }
2209                         break;
2210
2211                 case INTEL_PT_TNT:
2212                 case INTEL_PT_PSBEND:
2213                 case INTEL_PT_VMCS:
2214                 case INTEL_PT_MNT:
2215                 case INTEL_PT_PAD:
2216                 case INTEL_PT_PTWRITE:
2217                 case INTEL_PT_PTWRITE_IP:
2218                 case INTEL_PT_EXSTOP:
2219                 case INTEL_PT_EXSTOP_IP:
2220                 case INTEL_PT_MWAIT:
2221                 case INTEL_PT_PWRE:
2222                 case INTEL_PT_PWRX:
2223                 default:
2224                         break;
2225                 }
2226         }
2227 }
2228
2229 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
2230 {
2231         int err;
2232
2233         decoder->set_fup_tx_flags = false;
2234         decoder->set_fup_ptw = false;
2235         decoder->set_fup_mwait = false;
2236         decoder->set_fup_pwre = false;
2237         decoder->set_fup_exstop = false;
2238
2239         if (!decoder->branch_enable) {
2240                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2241                 decoder->overflow = false;
2242                 decoder->state.type = 0; /* Do not have a sample */
2243                 return 0;
2244         }
2245
2246         intel_pt_log("Scanning for full IP\n");
2247         err = intel_pt_walk_to_ip(decoder);
2248         if (err)
2249                 return err;
2250
2251         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2252         decoder->overflow = false;
2253
2254         decoder->state.from_ip = 0;
2255         decoder->state.to_ip = decoder->ip;
2256         intel_pt_log_to("Setting IP", decoder->ip);
2257
2258         return 0;
2259 }
2260
2261 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
2262 {
2263         const unsigned char *end = decoder->buf + decoder->len;
2264         size_t i;
2265
2266         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
2267                 if (i > decoder->len)
2268                         continue;
2269                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
2270                         return i;
2271         }
2272         return 0;
2273 }
2274
2275 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
2276 {
2277         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
2278         const char *psb = INTEL_PT_PSB_STR;
2279
2280         if (rest_psb > decoder->len ||
2281             memcmp(decoder->buf, psb + part_psb, rest_psb))
2282                 return 0;
2283
2284         return rest_psb;
2285 }
2286
2287 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2288                                   int part_psb)
2289 {
2290         int rest_psb, ret;
2291
2292         decoder->pos += decoder->len;
2293         decoder->len = 0;
2294
2295         ret = intel_pt_get_next_data(decoder);
2296         if (ret)
2297                 return ret;
2298
2299         rest_psb = intel_pt_rest_psb(decoder, part_psb);
2300         if (!rest_psb)
2301                 return 0;
2302
2303         decoder->pos -= part_psb;
2304         decoder->next_buf = decoder->buf + rest_psb;
2305         decoder->next_len = decoder->len - rest_psb;
2306         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2307         decoder->buf = decoder->temp_buf;
2308         decoder->len = INTEL_PT_PSB_LEN;
2309
2310         return 0;
2311 }
2312
2313 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2314 {
2315         unsigned char *next;
2316         int ret;
2317
2318         intel_pt_log("Scanning for PSB\n");
2319         while (1) {
2320                 if (!decoder->len) {
2321                         ret = intel_pt_get_next_data(decoder);
2322                         if (ret)
2323                                 return ret;
2324                 }
2325
2326                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2327                               INTEL_PT_PSB_LEN);
2328                 if (!next) {
2329                         int part_psb;
2330
2331                         part_psb = intel_pt_part_psb(decoder);
2332                         if (part_psb) {
2333                                 ret = intel_pt_get_split_psb(decoder, part_psb);
2334                                 if (ret)
2335                                         return ret;
2336                         } else {
2337                                 decoder->pos += decoder->len;
2338                                 decoder->len = 0;
2339                         }
2340                         continue;
2341                 }
2342
2343                 decoder->pkt_step = next - decoder->buf;
2344                 return intel_pt_get_next_packet(decoder);
2345         }
2346 }
2347
2348 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2349 {
2350         int err;
2351
2352         decoder->pge = false;
2353         decoder->continuous_period = false;
2354         decoder->have_last_ip = false;
2355         decoder->last_ip = 0;
2356         decoder->ip = 0;
2357         intel_pt_clear_stack(&decoder->stack);
2358
2359         err = intel_pt_scan_for_psb(decoder);
2360         if (err)
2361                 return err;
2362
2363         decoder->have_last_ip = true;
2364         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2365
2366         err = intel_pt_walk_psb(decoder);
2367         if (err)
2368                 return err;
2369
2370         if (decoder->ip) {
2371                 decoder->state.type = 0; /* Do not have a sample */
2372                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2373         } else {
2374                 return intel_pt_sync_ip(decoder);
2375         }
2376
2377         return 0;
2378 }
2379
2380 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2381 {
2382         uint64_t est = decoder->sample_insn_cnt << 1;
2383
2384         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2385                 goto out;
2386
2387         est *= decoder->max_non_turbo_ratio;
2388         est /= decoder->cbr;
2389 out:
2390         return decoder->sample_timestamp + est;
2391 }
2392
2393 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2394 {
2395         int err;
2396
2397         do {
2398                 decoder->state.type = INTEL_PT_BRANCH;
2399                 decoder->state.flags = 0;
2400
2401                 switch (decoder->pkt_state) {
2402                 case INTEL_PT_STATE_NO_PSB:
2403                         err = intel_pt_sync(decoder);
2404                         break;
2405                 case INTEL_PT_STATE_NO_IP:
2406                         decoder->have_last_ip = false;
2407                         decoder->last_ip = 0;
2408                         decoder->ip = 0;
2409                         __fallthrough;
2410                 case INTEL_PT_STATE_ERR_RESYNC:
2411                         err = intel_pt_sync_ip(decoder);
2412                         break;
2413                 case INTEL_PT_STATE_IN_SYNC:
2414                         err = intel_pt_walk_trace(decoder);
2415                         break;
2416                 case INTEL_PT_STATE_TNT:
2417                 case INTEL_PT_STATE_TNT_CONT:
2418                         err = intel_pt_walk_tnt(decoder);
2419                         if (err == -EAGAIN)
2420                                 err = intel_pt_walk_trace(decoder);
2421                         break;
2422                 case INTEL_PT_STATE_TIP:
2423                 case INTEL_PT_STATE_TIP_PGD:
2424                         err = intel_pt_walk_tip(decoder);
2425                         break;
2426                 case INTEL_PT_STATE_FUP:
2427                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2428                         err = intel_pt_walk_fup(decoder);
2429                         if (err == -EAGAIN)
2430                                 err = intel_pt_walk_fup_tip(decoder);
2431                         else if (!err)
2432                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
2433                         break;
2434                 case INTEL_PT_STATE_FUP_NO_TIP:
2435                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2436                         err = intel_pt_walk_fup(decoder);
2437                         if (err == -EAGAIN)
2438                                 err = intel_pt_walk_trace(decoder);
2439                         break;
2440                 default:
2441                         err = intel_pt_bug(decoder);
2442                         break;
2443                 }
2444         } while (err == -ENOLINK);
2445
2446         if (err) {
2447                 decoder->state.err = intel_pt_ext_err(err);
2448                 decoder->state.from_ip = decoder->ip;
2449                 intel_pt_update_sample_time(decoder);
2450                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
2451         } else {
2452                 decoder->state.err = 0;
2453                 if (decoder->cbr != decoder->cbr_seen && decoder->state.type) {
2454                         decoder->cbr_seen = decoder->cbr;
2455                         decoder->state.type |= INTEL_PT_CBR_CHG;
2456                         decoder->state.cbr_payload = decoder->cbr_payload;
2457                 }
2458                 if (intel_pt_sample_time(decoder->pkt_state)) {
2459                         intel_pt_update_sample_time(decoder);
2460                         if (decoder->sample_cyc)
2461                                 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
2462                 }
2463         }
2464
2465         decoder->state.timestamp = decoder->sample_timestamp;
2466         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2467         decoder->state.cr3 = decoder->cr3;
2468         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2469         decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
2470
2471         return &decoder->state;
2472 }
2473
2474 /**
2475  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2476  * @buf: pointer to buffer pointer
2477  * @len: size of buffer
2478  *
2479  * Updates the buffer pointer to point to the start of the next PSB packet if
2480  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2481  * @len is adjusted accordingly.
2482  *
2483  * Return: %true if a PSB packet is found, %false otherwise.
2484  */
2485 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2486 {
2487         unsigned char *next;
2488
2489         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2490         if (next) {
2491                 *len -= next - *buf;
2492                 *buf = next;
2493                 return true;
2494         }
2495         return false;
2496 }
2497
2498 /**
2499  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2500  *                     packet.
2501  * @buf: pointer to buffer pointer
2502  * @len: size of buffer
2503  *
2504  * Updates the buffer pointer to point to the start of the following PSB packet
2505  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2506  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2507  *
2508  * Return: %true if a PSB packet is found, %false otherwise.
2509  */
2510 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2511 {
2512         unsigned char *next;
2513
2514         if (!*len)
2515                 return false;
2516
2517         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2518         if (next) {
2519                 *len -= next - *buf;
2520                 *buf = next;
2521                 return true;
2522         }
2523         return false;
2524 }
2525
2526 /**
2527  * intel_pt_last_psb - find the last PSB packet in a buffer.
2528  * @buf: buffer
2529  * @len: size of buffer
2530  *
2531  * This function finds the last PSB in a buffer.
2532  *
2533  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2534  */
2535 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2536 {
2537         const char *n = INTEL_PT_PSB_STR;
2538         unsigned char *p;
2539         size_t k;
2540
2541         if (len < INTEL_PT_PSB_LEN)
2542                 return NULL;
2543
2544         k = len - INTEL_PT_PSB_LEN + 1;
2545         while (1) {
2546                 p = memrchr(buf, n[0], k);
2547                 if (!p)
2548                         return NULL;
2549                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2550                         return p;
2551                 k = p - buf;
2552                 if (!k)
2553                         return NULL;
2554         }
2555 }
2556
2557 /**
2558  * intel_pt_next_tsc - find and return next TSC.
2559  * @buf: buffer
2560  * @len: size of buffer
2561  * @tsc: TSC value returned
2562  * @rem: returns remaining size when TSC is found
2563  *
2564  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2565  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2566  * PSBEND packet is found.
2567  *
2568  * Return: %true if TSC is found, false otherwise.
2569  */
2570 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2571                               size_t *rem)
2572 {
2573         struct intel_pt_pkt packet;
2574         int ret;
2575
2576         while (len) {
2577                 ret = intel_pt_get_packet(buf, len, &packet);
2578                 if (ret <= 0)
2579                         return false;
2580                 if (packet.type == INTEL_PT_TSC) {
2581                         *tsc = packet.payload;
2582                         *rem = len;
2583                         return true;
2584                 }
2585                 if (packet.type == INTEL_PT_PSBEND)
2586                         return false;
2587                 buf += ret;
2588                 len -= ret;
2589         }
2590         return false;
2591 }
2592
2593 /**
2594  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2595  * @tsc1: first TSC to compare
2596  * @tsc2: second TSC to compare
2597  *
2598  * This function compares 7-byte TSC values allowing for the possibility that
2599  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2600  * around so for that purpose this function assumes the absolute difference is
2601  * less than half the maximum difference.
2602  *
2603  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2604  * after @tsc2.
2605  */
2606 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2607 {
2608         const uint64_t halfway = (1ULL << 55);
2609
2610         if (tsc1 == tsc2)
2611                 return 0;
2612
2613         if (tsc1 < tsc2) {
2614                 if (tsc2 - tsc1 < halfway)
2615                         return -1;
2616                 else
2617                         return 1;
2618         } else {
2619                 if (tsc1 - tsc2 < halfway)
2620                         return 1;
2621                 else
2622                         return -1;
2623         }
2624 }
2625
2626 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
2627
2628 /**
2629  * adj_for_padding - adjust overlap to account for padding.
2630  * @buf_b: second buffer
2631  * @buf_a: first buffer
2632  * @len_a: size of first buffer
2633  *
2634  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
2635  * accordingly.
2636  *
2637  * Return: A pointer into @buf_b from where non-overlapped data starts
2638  */
2639 static unsigned char *adj_for_padding(unsigned char *buf_b,
2640                                       unsigned char *buf_a, size_t len_a)
2641 {
2642         unsigned char *p = buf_b - MAX_PADDING;
2643         unsigned char *q = buf_a + len_a - MAX_PADDING;
2644         int i;
2645
2646         for (i = MAX_PADDING; i; i--, p++, q++) {
2647                 if (*p != *q)
2648                         break;
2649         }
2650
2651         return p;
2652 }
2653
2654 /**
2655  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2656  *                             using TSC.
2657  * @buf_a: first buffer
2658  * @len_a: size of first buffer
2659  * @buf_b: second buffer
2660  * @len_b: size of second buffer
2661  * @consecutive: returns true if there is data in buf_b that is consecutive
2662  *               to buf_a
2663  *
2664  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2665  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2666  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2667  * @buf_a and @buf_b are positioned at a PSB.
2668  *
2669  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2670  * @buf_b + @len_b if there is no non-overlapped data.
2671  */
2672 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2673                                                 size_t len_a,
2674                                                 unsigned char *buf_b,
2675                                                 size_t len_b, bool *consecutive)
2676 {
2677         uint64_t tsc_a, tsc_b;
2678         unsigned char *p;
2679         size_t len, rem_a, rem_b;
2680
2681         p = intel_pt_last_psb(buf_a, len_a);
2682         if (!p)
2683                 return buf_b; /* No PSB in buf_a => no overlap */
2684
2685         len = len_a - (p - buf_a);
2686         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2687                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2688                 len_a -= len;
2689                 p = intel_pt_last_psb(buf_a, len_a);
2690                 if (!p)
2691                         return buf_b; /* No full PSB+ => assume no overlap */
2692                 len = len_a - (p - buf_a);
2693                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2694                         return buf_b; /* No TSC in buf_a => assume no overlap */
2695         }
2696
2697         while (1) {
2698                 /* Ignore PSB+ with no TSC */
2699                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2700                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2701
2702                         /* Same TSC, so buffers are consecutive */
2703                         if (!cmp && rem_b >= rem_a) {
2704                                 unsigned char *start;
2705
2706                                 *consecutive = true;
2707                                 start = buf_b + len_b - (rem_b - rem_a);
2708                                 return adj_for_padding(start, buf_a, len_a);
2709                         }
2710                         if (cmp < 0)
2711                                 return buf_b; /* tsc_a < tsc_b => no overlap */
2712                 }
2713
2714                 if (!intel_pt_step_psb(&buf_b, &len_b))
2715                         return buf_b + len_b; /* No PSB in buf_b => no data */
2716         }
2717 }
2718
2719 /**
2720  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2721  * @buf_a: first buffer
2722  * @len_a: size of first buffer
2723  * @buf_b: second buffer
2724  * @len_b: size of second buffer
2725  * @have_tsc: can use TSC packets to detect overlap
2726  * @consecutive: returns true if there is data in buf_b that is consecutive
2727  *               to buf_a
2728  *
2729  * When trace samples or snapshots are recorded there is the possibility that
2730  * the data overlaps.  Note that, for the purposes of decoding, data is only
2731  * useful if it begins with a PSB packet.
2732  *
2733  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2734  * @buf_b + @len_b if there is no non-overlapped data.
2735  */
2736 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2737                                      unsigned char *buf_b, size_t len_b,
2738                                      bool have_tsc, bool *consecutive)
2739 {
2740         unsigned char *found;
2741
2742         /* Buffer 'b' must start at PSB so throw away everything before that */
2743         if (!intel_pt_next_psb(&buf_b, &len_b))
2744                 return buf_b + len_b; /* No PSB */
2745
2746         if (!intel_pt_next_psb(&buf_a, &len_a))
2747                 return buf_b; /* No overlap */
2748
2749         if (have_tsc) {
2750                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2751                                                   consecutive);
2752                 if (found)
2753                         return found;
2754         }
2755
2756         /*
2757          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2758          * we can ignore the first part of buffer 'a'.
2759          */
2760         while (len_b < len_a) {
2761                 if (!intel_pt_step_psb(&buf_a, &len_a))
2762                         return buf_b; /* No overlap */
2763         }
2764
2765         /* Now len_b >= len_a */
2766         while (1) {
2767                 /* Potential overlap so check the bytes */
2768                 found = memmem(buf_a, len_a, buf_b, len_a);
2769                 if (found) {
2770                         *consecutive = true;
2771                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
2772                 }
2773
2774                 /* Try again at next PSB in buffer 'a' */
2775                 if (!intel_pt_step_psb(&buf_a, &len_a))
2776                         return buf_b; /* No overlap */
2777         }
2778 }