tools/perf/util/intel-pt.c

   1 /*
   2  * intel_pt.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2015, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #include <stdio.h>
  17 #include <stdbool.h>
  18 #include <errno.h>
  19 #include <linux/kernel.h>
  20 #include <linux/types.h>
  21
  22 #include "../perf.h"
  23 #include "session.h"
  24 #include "machine.h"
  25 #include "sort.h"
  26 #include "tool.h"
  27 #include "event.h"
  28 #include "evlist.h"
  29 #include "evsel.h"
  30 #include "map.h"
  31 #include "color.h"
  32 #include "util.h"
  33 #include "thread.h"
  34 #include "thread-stack.h"
  35 #include "symbol.h"
  36 #include "callchain.h"
  37 #include "dso.h"
  38 #include "debug.h"
  39 #include "auxtrace.h"
  40 #include "tsc.h"
  41 #include "intel-pt.h"
  42 #include "config.h"
  43
  44 #include "intel-pt-decoder/intel-pt-log.h"
  45 #include "intel-pt-decoder/intel-pt-decoder.h"
  46 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  47 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  48
  49 #define MAX_TIMESTAMP (~0ULL)
  50
  51 struct intel_pt {
  52         struct auxtrace auxtrace;
  53         struct auxtrace_queues queues;
  54         struct auxtrace_heap heap;
  55         u32 auxtrace_type;
  56         struct perf_session *session;
  57         struct machine *machine;
  58         struct perf_evsel *switch_evsel;
  59         struct thread *unknown_thread;
  60         bool timeless_decoding;
  61         bool sampling_mode;
  62         bool snapshot_mode;
  63         bool per_cpu_mmaps;
  64         bool have_tsc;
  65         bool data_queued;
  66         bool est_tsc;
  67         bool sync_switch;
  68         bool mispred_all;
  69         int have_sched_switch;
  70         u32 pmu_type;
  71         u64 kernel_start;
  72         u64 switch_ip;
  73         u64 ptss_ip;
  74
  75         struct perf_tsc_conversion tc;
  76         bool cap_user_time_zero;
  77
  78         struct itrace_synth_opts synth_opts;
  79
  80         bool sample_instructions;
  81         u64 instructions_sample_type;
  82         u64 instructions_sample_period;
  83         u64 instructions_id;
  84
  85         bool sample_branches;
  86         u32 branches_filter;
  87         u64 branches_sample_type;
  88         u64 branches_id;
  89
  90         bool sample_transactions;
  91         u64 transactions_sample_type;
  92         u64 transactions_id;
  93
  94         bool synth_needs_swap;
  95
  96         u64 tsc_bit;
  97         u64 mtc_bit;
  98         u64 mtc_freq_bits;
  99         u32 tsc_ctc_ratio_n;
 100         u32 tsc_ctc_ratio_d;
 101         u64 cyc_bit;
 102         u64 noretcomp_bit;
 103         unsigned max_non_turbo_ratio;
 104
 105         unsigned long num_events;
 106
 107         char *filter;
 108         struct addr_filters filts;
 109 };
 110
 111 enum switch_state {
 112         INTEL_PT_SS_NOT_TRACING,
 113         INTEL_PT_SS_UNKNOWN,
 114         INTEL_PT_SS_TRACING,
 115         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 116         INTEL_PT_SS_EXPECTING_SWITCH_IP,
 117 };
 118
 119 struct intel_pt_queue {
 120         struct intel_pt *pt;
 121         unsigned int queue_nr;
 122         struct auxtrace_buffer *buffer;
 123         void *decoder;
 124         const struct intel_pt_state *state;
 125         struct ip_callchain *chain;
 126         struct branch_stack *last_branch;
 127         struct branch_stack *last_branch_rb;
 128         size_t last_branch_pos;
 129         union perf_event *event_buf;
 130         bool on_heap;
 131         bool stop;
 132         bool step_through_buffers;
 133         bool use_buffer_pid_tid;
 134         pid_t pid, tid;
 135         int cpu;
 136         int switch_state;
 137         pid_t next_tid;
 138         struct thread *thread;
 139         bool exclude_kernel;
 140         bool have_sample;
 141         u64 time;
 142         u64 timestamp;
 143         u32 flags;
 144         u16 insn_len;
 145         u64 last_insn_cnt;
 146         char insn[INTEL_PT_INSN_BUF_SZ];
 147 };
 148
 149 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 150                           unsigned char *buf, size_t len)
 151 {
 152         struct intel_pt_pkt packet;
 153         size_t pos = 0;
 154         int ret, pkt_len, i;
 155         char desc[INTEL_PT_PKT_DESC_MAX];
 156         const char *color = PERF_COLOR_BLUE;
 157
 158         color_fprintf(stdout, color,
 159                       ". ... Intel Processor Trace data: size %zu bytes\n",
 160                       len);
 161
 162         while (len) {
 163                 ret = intel_pt_get_packet(buf, len, &packet);
 164                 if (ret > 0)
 165                         pkt_len = ret;
 166                 else
 167                         pkt_len = 1;
 168                 printf(".");
 169                 color_fprintf(stdout, color, "  %08x: ", pos);
 170                 for (i = 0; i < pkt_len; i++)
 171                         color_fprintf(stdout, color, " %02x", buf[i]);
 172                 for (; i < 16; i++)
 173                         color_fprintf(stdout, color, "   ");
 174                 if (ret > 0) {
 175                         ret = intel_pt_pkt_desc(&packet, desc,
 176                                                 INTEL_PT_PKT_DESC_MAX);
 177                         if (ret > 0)
 178                                 color_fprintf(stdout, color, " %s\n", desc);
 179                 } else {
 180                         color_fprintf(stdout, color, " Bad packet!\n");
 181                 }
 182                 pos += pkt_len;
 183                 buf += pkt_len;
 184                 len -= pkt_len;
 185         }
 186 }
 187
 188 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 189                                 size_t len)
 190 {
 191         printf(".\n");
 192         intel_pt_dump(pt, buf, len);
 193 }
 194
 195 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 196                                    struct auxtrace_buffer *b)
 197 {
 198         void *start;
 199
 200         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 201                                       pt->have_tsc);
 202         if (!start)
 203                 return -EINVAL;
 204         b->use_size = b->data + b->size - start;
 205         b->use_data = start;
 206         return 0;
 207 }
 208
 209 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,
 210                                         struct auxtrace_queue *queue,
 211                                         struct auxtrace_buffer *buffer)
 212 {
 213         if (queue->cpu == -1 && buffer->cpu != -1)
 214                 ptq->cpu = buffer->cpu;
 215
 216         ptq->pid = buffer->pid;
 217         ptq->tid = buffer->tid;
 218
 219         intel_pt_log("queue %u cpu %d pid %d tid %d\n",
 220                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 221
 222         thread__zput(ptq->thread);
 223
 224         if (ptq->tid != -1) {
 225                 if (ptq->pid != -1)
 226                         ptq->thread = machine__findnew_thread(ptq->pt->machine,
 227                                                               ptq->pid,
 228                                                               ptq->tid);
 229                 else
 230                         ptq->thread = machine__find_thread(ptq->pt->machine, -1,
 231                                                            ptq->tid);
 232         }
 233 }
 234
 235 /* This function assumes data is processed sequentially only */
 236 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 237 {
 238         struct intel_pt_queue *ptq = data;
 239         struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer;
 240         struct auxtrace_queue *queue;
 241
 242         if (ptq->stop) {
 243                 b->len = 0;
 244                 return 0;
 245         }
 246
 247         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 248 next:
 249         buffer = auxtrace_buffer__next(queue, buffer);
 250         if (!buffer) {
 251                 if (old_buffer)
 252                         auxtrace_buffer__drop_data(old_buffer);
 253                 b->len = 0;
 254                 return 0;
 255         }
 256
 257         ptq->buffer = buffer;
 258
 259         if (!buffer->data) {
 260                 int fd = perf_data_file__fd(ptq->pt->session->file);
 261
 262                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 263                 if (!buffer->data)
 264                         return -ENOMEM;
 265         }
 266
 267         if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer &&
 268             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 269                 return -ENOMEM;
 270
 271         if (buffer->use_data) {
 272                 b->len = buffer->use_size;
 273                 b->buf = buffer->use_data;
 274         } else {
 275                 b->len = buffer->size;
 276                 b->buf = buffer->data;
 277         }
 278         b->ref_timestamp = buffer->reference;
 279
 280         /*
 281          * If in snapshot mode and the buffer has no usable data, get next
 282          * buffer and again check overlap against old_buffer.
 283          */
 284         if (ptq->pt->snapshot_mode && !b->len)
 285                 goto next;
 286
 287         if (old_buffer)
 288                 auxtrace_buffer__drop_data(old_buffer);
 289
 290         if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode &&
 291                                                       !buffer->consecutive)) {
 292                 b->consecutive = false;
 293                 b->trace_nr = buffer->buffer_nr + 1;
 294         } else {
 295                 b->consecutive = true;
 296         }
 297
 298         if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid ||
 299                                         ptq->tid != buffer->tid))
 300                 intel_pt_use_buffer_pid_tid(ptq, queue, buffer);
 301
 302         if (ptq->step_through_buffers)
 303                 ptq->stop = true;
 304
 305         if (!b->len)
 306                 return intel_pt_get_trace(b, data);
 307
 308         return 0;
 309 }
 310
 311 struct intel_pt_cache_entry {
 312         struct auxtrace_cache_entry     entry;
 313         u64                             insn_cnt;
 314         u64                             byte_cnt;
 315         enum intel_pt_insn_op           op;
 316         enum intel_pt_insn_branch       branch;
 317         int                             length;
 318         int32_t                         rel;
 319         char                            insn[INTEL_PT_INSN_BUF_SZ];
 320 };
 321
 322 static int intel_pt_config_div(const char *var, const char *value, void *data)
 323 {
 324         int *d = data;
 325         long val;
 326
 327         if (!strcmp(var, "intel-pt.cache-divisor")) {
 328                 val = strtol(value, NULL, 0);
 329                 if (val > 0 && val <= INT_MAX)
 330                         *d = val;
 331         }
 332
 333         return 0;
 334 }
 335
 336 static int intel_pt_cache_divisor(void)
 337 {
 338         static int d;
 339
 340         if (d)
 341                 return d;
 342
 343         perf_config(intel_pt_config_div, &d);
 344
 345         if (!d)
 346                 d = 64;
 347
 348         return d;
 349 }
 350
 351 static unsigned int intel_pt_cache_size(struct dso *dso,
 352                                         struct machine *machine)
 353 {
 354         off_t size;
 355
 356         size = dso__data_size(dso, machine);
 357         size /= intel_pt_cache_divisor();
 358         if (size < 1000)
 359                 return 10;
 360         if (size > (1 << 21))
 361                 return 21;
 362         return 32 - __builtin_clz(size);
 363 }
 364
 365 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 366                                              struct machine *machine)
 367 {
 368         struct auxtrace_cache *c;
 369         unsigned int bits;
 370
 371         if (dso->auxtrace_cache)
 372                 return dso->auxtrace_cache;
 373
 374         bits = intel_pt_cache_size(dso, machine);
 375
 376         /* Ignoring cache creation failure */
 377         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 378
 379         dso->auxtrace_cache = c;
 380
 381         return c;
 382 }
 383
 384 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 385                               u64 offset, u64 insn_cnt, u64 byte_cnt,
 386                               struct intel_pt_insn *intel_pt_insn)
 387 {
 388         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 389         struct intel_pt_cache_entry *e;
 390         int err;
 391
 392         if (!c)
 393                 return -ENOMEM;
 394
 395         e = auxtrace_cache__alloc_entry(c);
 396         if (!e)
 397                 return -ENOMEM;
 398
 399         e->insn_cnt = insn_cnt;
 400         e->byte_cnt = byte_cnt;
 401         e->op = intel_pt_insn->op;
 402         e->branch = intel_pt_insn->branch;
 403         e->length = intel_pt_insn->length;
 404         e->rel = intel_pt_insn->rel;
 405         memcpy(e->insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);
 406
 407         err = auxtrace_cache__add(c, offset, &e->entry);
 408         if (err)
 409                 auxtrace_cache__free_entry(c, e);
 410
 411         return err;
 412 }
 413
 414 static struct intel_pt_cache_entry *
 415 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 416 {
 417         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 418
 419         if (!c)
 420                 return NULL;
 421
 422         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 423 }
 424
 425 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 426                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
 427                                    uint64_t to_ip, uint64_t max_insn_cnt,
 428                                    void *data)
 429 {
 430         struct intel_pt_queue *ptq = data;
 431         struct machine *machine = ptq->pt->machine;
 432         struct thread *thread;
 433         struct addr_location al;
 434         unsigned char buf[INTEL_PT_INSN_BUF_SZ];
 435         ssize_t len;
 436         int x86_64;
 437         u8 cpumode;
 438         u64 offset, start_offset, start_ip;
 439         u64 insn_cnt = 0;
 440         bool one_map = true;
 441
 442         intel_pt_insn->length = 0;
 443
 444         if (to_ip && *ip == to_ip)
 445                 goto out_no_cache;
 446
 447         if (*ip >= ptq->pt->kernel_start)
 448                 cpumode = PERF_RECORD_MISC_KERNEL;
 449         else
 450                 cpumode = PERF_RECORD_MISC_USER;
 451
 452         thread = ptq->thread;
 453         if (!thread) {
 454                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 455                         return -EINVAL;
 456                 thread = ptq->pt->unknown_thread;
 457         }
 458
 459         while (1) {
 460                 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
 461                 if (!al.map || !al.map->dso)
 462                         return -EINVAL;
 463
 464                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 465                     dso__data_status_seen(al.map->dso,
 466                                           DSO_DATA_STATUS_SEEN_ITRACE))
 467                         return -ENOENT;
 468
 469                 offset = al.map->map_ip(al.map, *ip);
 470
 471                 if (!to_ip && one_map) {
 472                         struct intel_pt_cache_entry *e;
 473
 474                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 475                         if (e &&
 476                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 477                                 *insn_cnt_ptr = e->insn_cnt;
 478                                 *ip += e->byte_cnt;
 479                                 intel_pt_insn->op = e->op;
 480                                 intel_pt_insn->branch = e->branch;
 481                                 intel_pt_insn->length = e->length;
 482                                 intel_pt_insn->rel = e->rel;
 483                                 memcpy(intel_pt_insn->buf, e->insn,
 484                                        INTEL_PT_INSN_BUF_SZ);
 485                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 486                                 return 0;
 487                         }
 488                 }
 489
 490                 start_offset = offset;
 491                 start_ip = *ip;
 492
 493                 /* Load maps to ensure dso->is_64_bit has been updated */
 494                 map__load(al.map);
 495
 496                 x86_64 = al.map->dso->is_64_bit;
 497
 498                 while (1) {
 499                         len = dso__data_read_offset(al.map->dso, machine,
 500                                                     offset, buf,
 501                                                     INTEL_PT_INSN_BUF_SZ);
 502                         if (len <= 0)
 503                                 return -EINVAL;
 504
 505                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 506                                 return -EINVAL;
 507
 508                         intel_pt_log_insn(intel_pt_insn, *ip);
 509
 510                         insn_cnt += 1;
 511
 512                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 513                                 goto out;
 514
 515                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 516                                 goto out_no_cache;
 517
 518                         *ip += intel_pt_insn->length;
 519
 520                         if (to_ip && *ip == to_ip)
 521                                 goto out_no_cache;
 522
 523                         if (*ip >= al.map->end)
 524                                 break;
 525
 526                         offset += intel_pt_insn->length;
 527                 }
 528                 one_map = false;
 529         }
 530 out:
 531         *insn_cnt_ptr = insn_cnt;
 532
 533         if (!one_map)
 534                 goto out_no_cache;
 535
 536         /*
 537          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 538          * entries.
 539          */
 540         if (to_ip) {
 541                 struct intel_pt_cache_entry *e;
 542
 543                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 544                 if (e)
 545                         return 0;
 546         }
 547
 548         /* Ignore cache errors */
 549         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 550                            *ip - start_ip, intel_pt_insn);
 551
 552         return 0;
 553
 554 out_no_cache:
 555         *insn_cnt_ptr = insn_cnt;
 556         return 0;
 557 }
 558
 559 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
 560                                   uint64_t offset, const char *filename)
 561 {
 562         struct addr_filter *filt;
 563         bool have_filter   = false;
 564         bool hit_tracestop = false;
 565         bool hit_filter    = false;
 566
 567         list_for_each_entry(filt, &pt->filts.head, list) {
 568                 if (filt->start)
 569                         have_filter = true;
 570
 571                 if ((filename && !filt->filename) ||
 572                     (!filename && filt->filename) ||
 573                     (filename && strcmp(filename, filt->filename)))
 574                         continue;
 575
 576                 if (!(offset >= filt->addr && offset < filt->addr + filt->size))
 577                         continue;
 578
 579                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s hit filter: %s offset %#"PRIx64" size %#"PRIx64"\n",
 580                              ip, offset, filename ? filename : "[kernel]",
 581                              filt->start ? "filter" : "stop",
 582                              filt->addr, filt->size);
 583
 584                 if (filt->start)
 585                         hit_filter = true;
 586                 else
 587                         hit_tracestop = true;
 588         }
 589
 590         if (!hit_tracestop && !hit_filter)
 591                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s is not in a filter region\n",
 592                              ip, offset, filename ? filename : "[kernel]");
 593
 594         return hit_tracestop || (have_filter && !hit_filter);
 595 }
 596
 597 static int __intel_pt_pgd_ip(uint64_t ip, void *data)
 598 {
 599         struct intel_pt_queue *ptq = data;
 600         struct thread *thread;
 601         struct addr_location al;
 602         u8 cpumode;
 603         u64 offset;
 604
 605         if (ip >= ptq->pt->kernel_start)
 606                 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
 607
 608         cpumode = PERF_RECORD_MISC_USER;
 609
 610         thread = ptq->thread;
 611         if (!thread)
 612                 return -EINVAL;
 613
 614         thread__find_addr_map(thread, cpumode, MAP__FUNCTION, ip, &al);
 615         if (!al.map || !al.map->dso)
 616                 return -EINVAL;
 617
 618         offset = al.map->map_ip(al.map, ip);
 619
 620         return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
 621                                      al.map->dso->long_name);
 622 }
 623
 624 static bool intel_pt_pgd_ip(uint64_t ip, void *data)
 625 {
 626         return __intel_pt_pgd_ip(ip, data) > 0;
 627 }
 628
 629 static bool intel_pt_get_config(struct intel_pt *pt,
 630                                 struct perf_event_attr *attr, u64 *config)
 631 {
 632         if (attr->type == pt->pmu_type) {
 633                 if (config)
 634                         *config = attr->config;
 635                 return true;
 636         }
 637
 638         return false;
 639 }
 640
 641 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 642 {
 643         struct perf_evsel *evsel;
 644
 645         evlist__for_each_entry(pt->session->evlist, evsel) {
 646                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 647                     !evsel->attr.exclude_kernel)
 648                         return false;
 649         }
 650         return true;
 651 }
 652
 653 static bool intel_pt_return_compression(struct intel_pt *pt)
 654 {
 655         struct perf_evsel *evsel;
 656         u64 config;
 657
 658         if (!pt->noretcomp_bit)
 659                 return true;
 660
 661         evlist__for_each_entry(pt->session->evlist, evsel) {
 662                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 663                     (config & pt->noretcomp_bit))
 664                         return false;
 665         }
 666         return true;
 667 }
 668
 669 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 670 {
 671         struct perf_evsel *evsel;
 672         unsigned int shift;
 673         u64 config;
 674
 675         if (!pt->mtc_freq_bits)
 676                 return 0;
 677
 678         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 679                 config >>= 1;
 680
 681         evlist__for_each_entry(pt->session->evlist, evsel) {
 682                 if (intel_pt_get_config(pt, &evsel->attr, &config))
 683                         return (config & pt->mtc_freq_bits) >> shift;
 684         }
 685         return 0;
 686 }
 687
 688 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 689 {
 690         struct perf_evsel *evsel;
 691         bool timeless_decoding = true;
 692         u64 config;
 693
 694         if (!pt->tsc_bit || !pt->cap_user_time_zero)
 695                 return true;
 696
 697         evlist__for_each_entry(pt->session->evlist, evsel) {
 698                 if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
 699                         return true;
 700                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 701                         if (config & pt->tsc_bit)
 702                                 timeless_decoding = false;
 703                         else
 704                                 return true;
 705                 }
 706         }
 707         return timeless_decoding;
 708 }
 709
 710 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 711 {
 712         struct perf_evsel *evsel;
 713
 714         evlist__for_each_entry(pt->session->evlist, evsel) {
 715                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 716                     !evsel->attr.exclude_kernel)
 717                         return true;
 718         }
 719         return false;
 720 }
 721
 722 static bool intel_pt_have_tsc(struct intel_pt *pt)
 723 {
 724         struct perf_evsel *evsel;
 725         bool have_tsc = false;
 726         u64 config;
 727
 728         if (!pt->tsc_bit)
 729                 return false;
 730
 731         evlist__for_each_entry(pt->session->evlist, evsel) {
 732                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 733                         if (config & pt->tsc_bit)
 734                                 have_tsc = true;
 735                         else
 736                                 return false;
 737                 }
 738         }
 739         return have_tsc;
 740 }
 741
 742 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 743 {
 744         u64 quot, rem;
 745
 746         quot = ns / pt->tc.time_mult;
 747         rem  = ns % pt->tc.time_mult;
 748         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 749                 pt->tc.time_mult;
 750 }
 751
 752 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 753                                                    unsigned int queue_nr)
 754 {
 755         struct intel_pt_params params = { .get_trace = 0, };
 756         struct intel_pt_queue *ptq;
 757
 758         ptq = zalloc(sizeof(struct intel_pt_queue));
 759         if (!ptq)
 760                 return NULL;
 761
 762         if (pt->synth_opts.callchain) {
 763                 size_t sz = sizeof(struct ip_callchain);
 764
 765                 sz += pt->synth_opts.callchain_sz * sizeof(u64);
 766                 ptq->chain = zalloc(sz);
 767                 if (!ptq->chain)
 768                         goto out_free;
 769         }
 770
 771         if (pt->synth_opts.last_branch) {
 772                 size_t sz = sizeof(struct branch_stack);
 773
 774                 sz += pt->synth_opts.last_branch_sz *
 775                       sizeof(struct branch_entry);
 776                 ptq->last_branch = zalloc(sz);
 777                 if (!ptq->last_branch)
 778                         goto out_free;
 779                 ptq->last_branch_rb = zalloc(sz);
 780                 if (!ptq->last_branch_rb)
 781                         goto out_free;
 782         }
 783
 784         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 785         if (!ptq->event_buf)
 786                 goto out_free;
 787
 788         ptq->pt = pt;
 789         ptq->queue_nr = queue_nr;
 790         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 791         ptq->pid = -1;
 792         ptq->tid = -1;
 793         ptq->cpu = -1;
 794         ptq->next_tid = -1;
 795
 796         params.get_trace = intel_pt_get_trace;
 797         params.walk_insn = intel_pt_walk_next_insn;
 798         params.data = ptq;
 799         params.return_compression = intel_pt_return_compression(pt);
 800         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 801         params.mtc_period = intel_pt_mtc_period(pt);
 802         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 803         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 804
 805         if (pt->filts.cnt > 0)
 806                 params.pgd_ip = intel_pt_pgd_ip;
 807
 808         if (pt->synth_opts.instructions) {
 809                 if (pt->synth_opts.period) {
 810                         switch (pt->synth_opts.period_type) {
 811                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 812                                 params.period_type =
 813                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
 814                                 params.period = pt->synth_opts.period;
 815                                 break;
 816                         case PERF_ITRACE_PERIOD_TICKS:
 817                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 818                                 params.period = pt->synth_opts.period;
 819                                 break;
 820                         case PERF_ITRACE_PERIOD_NANOSECS:
 821                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 822                                 params.period = intel_pt_ns_to_ticks(pt,
 823                                                         pt->synth_opts.period);
 824                                 break;
 825                         default:
 826                                 break;
 827                         }
 828                 }
 829
 830                 if (!params.period) {
 831                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 832                         params.period = 1;
 833                 }
 834         }
 835
 836         ptq->decoder = intel_pt_decoder_new(&params);
 837         if (!ptq->decoder)
 838                 goto out_free;
 839
 840         return ptq;
 841
 842 out_free:
 843         zfree(&ptq->event_buf);
 844         zfree(&ptq->last_branch);
 845         zfree(&ptq->last_branch_rb);
 846         zfree(&ptq->chain);
 847         free(ptq);
 848         return NULL;
 849 }
 850
 851 static void intel_pt_free_queue(void *priv)
 852 {
 853         struct intel_pt_queue *ptq = priv;
 854
 855         if (!ptq)
 856                 return;
 857         thread__zput(ptq->thread);
 858         intel_pt_decoder_free(ptq->decoder);
 859         zfree(&ptq->event_buf);
 860         zfree(&ptq->last_branch);
 861         zfree(&ptq->last_branch_rb);
 862         zfree(&ptq->chain);
 863         free(ptq);
 864 }
 865
 866 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 867                                      struct auxtrace_queue *queue)
 868 {
 869         struct intel_pt_queue *ptq = queue->priv;
 870
 871         if (queue->tid == -1 || pt->have_sched_switch) {
 872                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 873                 thread__zput(ptq->thread);
 874         }
 875
 876         if (!ptq->thread && ptq->tid != -1)
 877                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
 878
 879         if (ptq->thread) {
 880                 ptq->pid = ptq->thread->pid_;
 881                 if (queue->cpu == -1)
 882                         ptq->cpu = ptq->thread->cpu;
 883         }
 884 }
 885
 886 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
 887 {
 888         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
 889                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
 890         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
 891                 if (ptq->state->to_ip)
 892                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
 893                                      PERF_IP_FLAG_ASYNC |
 894                                      PERF_IP_FLAG_INTERRUPT;
 895                 else
 896                         ptq->flags = PERF_IP_FLAG_BRANCH |
 897                                      PERF_IP_FLAG_TRACE_END;
 898                 ptq->insn_len = 0;
 899         } else {
 900                 if (ptq->state->from_ip)
 901                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
 902                 else
 903                         ptq->flags = PERF_IP_FLAG_BRANCH |
 904                                      PERF_IP_FLAG_TRACE_BEGIN;
 905                 if (ptq->state->flags & INTEL_PT_IN_TX)
 906                         ptq->flags |= PERF_IP_FLAG_IN_TX;
 907                 ptq->insn_len = ptq->state->insn_len;
 908                 memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);
 909         }
 910 }
 911
 912 static int intel_pt_setup_queue(struct intel_pt *pt,
 913                                 struct auxtrace_queue *queue,
 914                                 unsigned int queue_nr)
 915 {
 916         struct intel_pt_queue *ptq = queue->priv;
 917
 918         if (list_empty(&queue->head))
 919                 return 0;
 920
 921         if (!ptq) {
 922                 ptq = intel_pt_alloc_queue(pt, queue_nr);
 923                 if (!ptq)
 924                         return -ENOMEM;
 925                 queue->priv = ptq;
 926
 927                 if (queue->cpu != -1)
 928                         ptq->cpu = queue->cpu;
 929                 ptq->tid = queue->tid;
 930
 931                 if (pt->sampling_mode) {
 932                         if (pt->timeless_decoding)
 933                                 ptq->step_through_buffers = true;
 934                         if (pt->timeless_decoding || !pt->have_sched_switch)
 935                                 ptq->use_buffer_pid_tid = true;
 936                 }
 937         }
 938
 939         if (!ptq->on_heap &&
 940             (!pt->sync_switch ||
 941              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
 942                 const struct intel_pt_state *state;
 943                 int ret;
 944
 945                 if (pt->timeless_decoding)
 946                         return 0;
 947
 948                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
 949                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
 950                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 951                 while (1) {
 952                         state = intel_pt_decode(ptq->decoder);
 953                         if (state->err) {
 954                                 if (state->err == INTEL_PT_ERR_NODATA) {
 955                                         intel_pt_log("queue %u has no timestamp\n",
 956                                                      queue_nr);
 957                                         return 0;
 958                                 }
 959                                 continue;
 960                         }
 961                         if (state->timestamp)
 962                                 break;
 963                 }
 964
 965                 ptq->timestamp = state->timestamp;
 966                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
 967                              queue_nr, ptq->timestamp);
 968                 ptq->state = state;
 969                 ptq->have_sample = true;
 970                 intel_pt_sample_flags(ptq);
 971                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
 972                 if (ret)
 973                         return ret;
 974                 ptq->on_heap = true;
 975         }
 976
 977         return 0;
 978 }
 979
 980 static int intel_pt_setup_queues(struct intel_pt *pt)
 981 {
 982         unsigned int i;
 983         int ret;
 984
 985         for (i = 0; i < pt->queues.nr_queues; i++) {
 986                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
 987                 if (ret)
 988                         return ret;
 989         }
 990         return 0;
 991 }
 992
 993 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
 994 {
 995         struct branch_stack *bs_src = ptq->last_branch_rb;
 996         struct branch_stack *bs_dst = ptq->last_branch;
 997         size_t nr = 0;
 998
 999         bs_dst->nr = bs_src->nr;
1000
1001         if (!bs_src->nr)
1002                 return;
1003
1004         nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
1005         memcpy(&bs_dst->entries[0],
1006                &bs_src->entries[ptq->last_branch_pos],
1007                sizeof(struct branch_entry) * nr);
1008
1009         if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
1010                 memcpy(&bs_dst->entries[nr],
1011                        &bs_src->entries[0],
1012                        sizeof(struct branch_entry) * ptq->last_branch_pos);
1013         }
1014 }
1015
1016 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
1017 {
1018         ptq->last_branch_pos = 0;
1019         ptq->last_branch_rb->nr = 0;
1020 }
1021
1022 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
1023 {
1024         const struct intel_pt_state *state = ptq->state;
1025         struct branch_stack *bs = ptq->last_branch_rb;
1026         struct branch_entry *be;
1027
1028         if (!ptq->last_branch_pos)
1029                 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
1030
1031         ptq->last_branch_pos -= 1;
1032
1033         be              = &bs->entries[ptq->last_branch_pos];
1034         be->from        = state->from_ip;
1035         be->to          = state->to_ip;
1036         be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
1037         be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
1038         /* No support for mispredict */
1039         be->flags.mispred = ptq->pt->mispred_all;
1040
1041         if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
1042                 bs->nr += 1;
1043 }
1044
1045 static int intel_pt_inject_event(union perf_event *event,
1046                                  struct perf_sample *sample, u64 type,
1047                                  bool swapped)
1048 {
1049         event->header.size = perf_event__sample_event_size(sample, type, 0);
1050         return perf_event__synthesize_sample(event, type, 0, sample, swapped);
1051 }
1052
1053 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
1054 {
1055         int ret;
1056         struct intel_pt *pt = ptq->pt;
1057         union perf_event *event = ptq->event_buf;
1058         struct perf_sample sample = { .ip = 0, };
1059         struct dummy_branch_stack {
1060                 u64                     nr;
1061                 struct branch_entry     entries;
1062         } dummy_bs;
1063
1064         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1065                 return 0;
1066
1067         if (pt->synth_opts.initial_skip &&
1068             pt->num_events++ < pt->synth_opts.initial_skip)
1069                 return 0;
1070
1071         event->sample.header.type = PERF_RECORD_SAMPLE;
1072         event->sample.header.misc = PERF_RECORD_MISC_USER;
1073         event->sample.header.size = sizeof(struct perf_event_header);
1074
1075         if (!pt->timeless_decoding)
1076                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1077
1078         sample.cpumode = PERF_RECORD_MISC_USER;
1079         sample.ip = ptq->state->from_ip;
1080         sample.pid = ptq->pid;
1081         sample.tid = ptq->tid;
1082         sample.addr = ptq->state->to_ip;
1083         sample.id = ptq->pt->branches_id;
1084         sample.stream_id = ptq->pt->branches_id;
1085         sample.period = 1;
1086         sample.cpu = ptq->cpu;
1087         sample.flags = ptq->flags;
1088         sample.insn_len = ptq->insn_len;
1089         memcpy(sample.insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1090
1091         /*
1092          * perf report cannot handle events without a branch stack when using
1093          * SORT_MODE__BRANCH so make a dummy one.
1094          */
1095         if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1096                 dummy_bs = (struct dummy_branch_stack){
1097                         .nr = 1,
1098                         .entries = {
1099                                 .from = sample.ip,
1100                                 .to = sample.addr,
1101                         },
1102                 };
1103                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1104         }
1105
1106         if (pt->synth_opts.inject) {
1107                 ret = intel_pt_inject_event(event, &sample,
1108                                             pt->branches_sample_type,
1109                                             pt->synth_needs_swap);
1110                 if (ret)
1111                         return ret;
1112         }
1113
1114         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1115         if (ret)
1116                 pr_err("Intel Processor Trace: failed to deliver branch event, error %d\n",
1117                        ret);
1118
1119         return ret;
1120 }
1121
1122 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1123 {
1124         int ret;
1125         struct intel_pt *pt = ptq->pt;
1126         union perf_event *event = ptq->event_buf;
1127         struct perf_sample sample = { .ip = 0, };
1128
1129         if (pt->synth_opts.initial_skip &&
1130             pt->num_events++ < pt->synth_opts.initial_skip)
1131                 return 0;
1132
1133         event->sample.header.type = PERF_RECORD_SAMPLE;
1134         event->sample.header.misc = PERF_RECORD_MISC_USER;
1135         event->sample.header.size = sizeof(struct perf_event_header);
1136
1137         if (!pt->timeless_decoding)
1138                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1139
1140         sample.cpumode = PERF_RECORD_MISC_USER;
1141         sample.ip = ptq->state->from_ip;
1142         sample.pid = ptq->pid;
1143         sample.tid = ptq->tid;
1144         sample.addr = ptq->state->to_ip;
1145         sample.id = ptq->pt->instructions_id;
1146         sample.stream_id = ptq->pt->instructions_id;
1147         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1148         sample.cpu = ptq->cpu;
1149         sample.flags = ptq->flags;
1150         sample.insn_len = ptq->insn_len;
1151         memcpy(sample.insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1152
1153         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1154
1155         if (pt->synth_opts.callchain) {
1156                 thread_stack__sample(ptq->thread, ptq->chain,
1157                                      pt->synth_opts.callchain_sz, sample.ip);
1158                 sample.callchain = ptq->chain;
1159         }
1160
1161         if (pt->synth_opts.last_branch) {
1162                 intel_pt_copy_last_branch_rb(ptq);
1163                 sample.branch_stack = ptq->last_branch;
1164         }
1165
1166         if (pt->synth_opts.inject) {
1167                 ret = intel_pt_inject_event(event, &sample,
1168                                             pt->instructions_sample_type,
1169                                             pt->synth_needs_swap);
1170                 if (ret)
1171                         return ret;
1172         }
1173
1174         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1175         if (ret)
1176                 pr_err("Intel Processor Trace: failed to deliver instruction event, error %d\n",
1177                        ret);
1178
1179         if (pt->synth_opts.last_branch)
1180                 intel_pt_reset_last_branch_rb(ptq);
1181
1182         return ret;
1183 }
1184
1185 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1186 {
1187         int ret;
1188         struct intel_pt *pt = ptq->pt;
1189         union perf_event *event = ptq->event_buf;
1190         struct perf_sample sample = { .ip = 0, };
1191
1192         if (pt->synth_opts.initial_skip &&
1193             pt->num_events++ < pt->synth_opts.initial_skip)
1194                 return 0;
1195
1196         event->sample.header.type = PERF_RECORD_SAMPLE;
1197         event->sample.header.misc = PERF_RECORD_MISC_USER;
1198         event->sample.header.size = sizeof(struct perf_event_header);
1199
1200         if (!pt->timeless_decoding)
1201                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1202
1203         sample.cpumode = PERF_RECORD_MISC_USER;
1204         sample.ip = ptq->state->from_ip;
1205         sample.pid = ptq->pid;
1206         sample.tid = ptq->tid;
1207         sample.addr = ptq->state->to_ip;
1208         sample.id = ptq->pt->transactions_id;
1209         sample.stream_id = ptq->pt->transactions_id;
1210         sample.period = 1;
1211         sample.cpu = ptq->cpu;
1212         sample.flags = ptq->flags;
1213         sample.insn_len = ptq->insn_len;
1214         memcpy(sample.insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1215
1216         if (pt->synth_opts.callchain) {
1217                 thread_stack__sample(ptq->thread, ptq->chain,
1218                                      pt->synth_opts.callchain_sz, sample.ip);
1219                 sample.callchain = ptq->chain;
1220         }
1221
1222         if (pt->synth_opts.last_branch) {
1223                 intel_pt_copy_last_branch_rb(ptq);
1224                 sample.branch_stack = ptq->last_branch;
1225         }
1226
1227         if (pt->synth_opts.inject) {
1228                 ret = intel_pt_inject_event(event, &sample,
1229                                             pt->transactions_sample_type,
1230                                             pt->synth_needs_swap);
1231                 if (ret)
1232                         return ret;
1233         }
1234
1235         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1236         if (ret)
1237                 pr_err("Intel Processor Trace: failed to deliver transaction event, error %d\n",
1238                        ret);
1239
1240         if (pt->synth_opts.last_branch)
1241                 intel_pt_reset_last_branch_rb(ptq);
1242
1243         return ret;
1244 }
1245
1246 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1247                                 pid_t pid, pid_t tid, u64 ip)
1248 {
1249         union perf_event event;
1250         char msg[MAX_AUXTRACE_ERROR_MSG];
1251         int err;
1252
1253         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1254
1255         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1256                              code, cpu, pid, tid, ip, msg);
1257
1258         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1259         if (err)
1260                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1261                        err);
1262
1263         return err;
1264 }
1265
1266 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1267 {
1268         struct auxtrace_queue *queue;
1269         pid_t tid = ptq->next_tid;
1270         int err;
1271
1272         if (tid == -1)
1273                 return 0;
1274
1275         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1276
1277         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1278
1279         queue = &pt->queues.queue_array[ptq->queue_nr];
1280         intel_pt_set_pid_tid_cpu(pt, queue);
1281
1282         ptq->next_tid = -1;
1283
1284         return err;
1285 }
1286
1287 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1288 {
1289         struct intel_pt *pt = ptq->pt;
1290
1291         return ip == pt->switch_ip &&
1292                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1293                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1294                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1295 }
1296
1297 static int intel_pt_sample(struct intel_pt_queue *ptq)
1298 {
1299         const struct intel_pt_state *state = ptq->state;
1300         struct intel_pt *pt = ptq->pt;
1301         int err;
1302
1303         if (!ptq->have_sample)
1304                 return 0;
1305
1306         ptq->have_sample = false;
1307
1308         if (pt->sample_instructions &&
1309             (state->type & INTEL_PT_INSTRUCTION) &&
1310             (!pt->synth_opts.initial_skip ||
1311              pt->num_events++ >= pt->synth_opts.initial_skip)) {
1312                 err = intel_pt_synth_instruction_sample(ptq);
1313                 if (err)
1314                         return err;
1315         }
1316
1317         if (pt->sample_transactions &&
1318             (state->type & INTEL_PT_TRANSACTION) &&
1319             (!pt->synth_opts.initial_skip ||
1320              pt->num_events++ >= pt->synth_opts.initial_skip)) {
1321                 err = intel_pt_synth_transaction_sample(ptq);
1322                 if (err)
1323                         return err;
1324         }
1325
1326         if (!(state->type & INTEL_PT_BRANCH))
1327                 return 0;
1328
1329         if (pt->synth_opts.callchain || pt->synth_opts.thread_stack)
1330                 thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1331                                     state->to_ip, ptq->insn_len,
1332                                     state->trace_nr);
1333         else
1334                 thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1335
1336         if (pt->sample_branches) {
1337                 err = intel_pt_synth_branch_sample(ptq);
1338                 if (err)
1339                         return err;
1340         }
1341
1342         if (pt->synth_opts.last_branch)
1343                 intel_pt_update_last_branch_rb(ptq);
1344
1345         if (!pt->sync_switch)
1346                 return 0;
1347
1348         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1349                 switch (ptq->switch_state) {
1350                 case INTEL_PT_SS_UNKNOWN:
1351                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1352                         err = intel_pt_next_tid(pt, ptq);
1353                         if (err)
1354                                 return err;
1355                         ptq->switch_state = INTEL_PT_SS_TRACING;
1356                         break;
1357                 default:
1358                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1359                         return 1;
1360                 }
1361         } else if (!state->to_ip) {
1362                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1363         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1364                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1365         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1366                    state->to_ip == pt->ptss_ip &&
1367                    (ptq->flags & PERF_IP_FLAG_CALL)) {
1368                 ptq->switch_state = INTEL_PT_SS_TRACING;
1369         }
1370
1371         return 0;
1372 }
1373
1374 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1375 {
1376         struct machine *machine = pt->machine;
1377         struct map *map;
1378         struct symbol *sym, *start;
1379         u64 ip, switch_ip = 0;
1380         const char *ptss;
1381
1382         if (ptss_ip)
1383                 *ptss_ip = 0;
1384
1385         map = machine__kernel_map(machine);
1386         if (!map)
1387                 return 0;
1388
1389         if (map__load(map))
1390                 return 0;
1391
1392         start = dso__first_symbol(map->dso, MAP__FUNCTION);
1393
1394         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1395                 if (sym->binding == STB_GLOBAL &&
1396                     !strcmp(sym->name, "__switch_to")) {
1397                         ip = map->unmap_ip(map, sym->start);
1398                         if (ip >= map->start && ip < map->end) {
1399                                 switch_ip = ip;
1400                                 break;
1401                         }
1402                 }
1403         }
1404
1405         if (!switch_ip || !ptss_ip)
1406                 return 0;
1407
1408         if (pt->have_sched_switch == 1)
1409                 ptss = "perf_trace_sched_switch";
1410         else
1411                 ptss = "__perf_event_task_sched_out";
1412
1413         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1414                 if (!strcmp(sym->name, ptss)) {
1415                         ip = map->unmap_ip(map, sym->start);
1416                         if (ip >= map->start && ip < map->end) {
1417                                 *ptss_ip = ip;
1418                                 break;
1419                         }
1420                 }
1421         }
1422
1423         return switch_ip;
1424 }
1425
1426 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1427 {
1428         const struct intel_pt_state *state = ptq->state;
1429         struct intel_pt *pt = ptq->pt;
1430         int err;
1431
1432         if (!pt->kernel_start) {
1433                 pt->kernel_start = machine__kernel_start(pt->machine);
1434                 if (pt->per_cpu_mmaps &&
1435                     (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1436                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1437                     !pt->sampling_mode) {
1438                         pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1439                         if (pt->switch_ip) {
1440                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1441                                              pt->switch_ip, pt->ptss_ip);
1442                                 pt->sync_switch = true;
1443                         }
1444                 }
1445         }
1446
1447         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1448                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1449         while (1) {
1450                 err = intel_pt_sample(ptq);
1451                 if (err)
1452                         return err;
1453
1454                 state = intel_pt_decode(ptq->decoder);
1455                 if (state->err) {
1456                         if (state->err == INTEL_PT_ERR_NODATA)
1457                                 return 1;
1458                         if (pt->sync_switch &&
1459                             state->from_ip >= pt->kernel_start) {
1460                                 pt->sync_switch = false;
1461                                 intel_pt_next_tid(pt, ptq);
1462                         }
1463                         if (pt->synth_opts.errors) {
1464                                 err = intel_pt_synth_error(pt, state->err,
1465                                                            ptq->cpu, ptq->pid,
1466                                                            ptq->tid,
1467                                                            state->from_ip);
1468                                 if (err)
1469                                         return err;
1470                         }
1471                         continue;
1472                 }
1473
1474                 ptq->state = state;
1475                 ptq->have_sample = true;
1476                 intel_pt_sample_flags(ptq);
1477
1478                 /* Use estimated TSC upon return to user space */
1479                 if (pt->est_tsc &&
1480                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1481                     state->to_ip && state->to_ip < pt->kernel_start) {
1482                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1483                                      state->timestamp, state->est_timestamp);
1484                         ptq->timestamp = state->est_timestamp;
1485                 /* Use estimated TSC in unknown switch state */
1486                 } else if (pt->sync_switch &&
1487                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1488                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
1489                            ptq->next_tid == -1) {
1490                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1491                                      state->timestamp, state->est_timestamp);
1492                         ptq->timestamp = state->est_timestamp;
1493                 } else if (state->timestamp > ptq->timestamp) {
1494                         ptq->timestamp = state->timestamp;
1495                 }
1496
1497                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1498                         *timestamp = ptq->timestamp;
1499                         return 0;
1500                 }
1501         }
1502         return 0;
1503 }
1504
1505 static inline int intel_pt_update_queues(struct intel_pt *pt)
1506 {
1507         if (pt->queues.new_data) {
1508                 pt->queues.new_data = false;
1509                 return intel_pt_setup_queues(pt);
1510         }
1511         return 0;
1512 }
1513
1514 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1515 {
1516         unsigned int queue_nr;
1517         u64 ts;
1518         int ret;
1519
1520         while (1) {
1521                 struct auxtrace_queue *queue;
1522                 struct intel_pt_queue *ptq;
1523
1524                 if (!pt->heap.heap_cnt)
1525                         return 0;
1526
1527                 if (pt->heap.heap_array[0].ordinal >= timestamp)
1528                         return 0;
1529
1530                 queue_nr = pt->heap.heap_array[0].queue_nr;
1531                 queue = &pt->queues.queue_array[queue_nr];
1532                 ptq = queue->priv;
1533
1534                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1535                              queue_nr, pt->heap.heap_array[0].ordinal,
1536                              timestamp);
1537
1538                 auxtrace_heap__pop(&pt->heap);
1539
1540                 if (pt->heap.heap_cnt) {
1541                         ts = pt->heap.heap_array[0].ordinal + 1;
1542                         if (ts > timestamp)
1543                                 ts = timestamp;
1544                 } else {
1545                         ts = timestamp;
1546                 }
1547
1548                 intel_pt_set_pid_tid_cpu(pt, queue);
1549
1550                 ret = intel_pt_run_decoder(ptq, &ts);
1551
1552                 if (ret < 0) {
1553                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
1554                         return ret;
1555                 }
1556
1557                 if (!ret) {
1558                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1559                         if (ret < 0)
1560                                 return ret;
1561                 } else {
1562                         ptq->on_heap = false;
1563                 }
1564         }
1565
1566         return 0;
1567 }
1568
1569 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1570                                             u64 time_)
1571 {
1572         struct auxtrace_queues *queues = &pt->queues;
1573         unsigned int i;
1574         u64 ts = 0;
1575
1576         for (i = 0; i < queues->nr_queues; i++) {
1577                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1578                 struct intel_pt_queue *ptq = queue->priv;
1579
1580                 if (ptq && (tid == -1 || ptq->tid == tid)) {
1581                         ptq->time = time_;
1582                         intel_pt_set_pid_tid_cpu(pt, queue);
1583                         intel_pt_run_decoder(ptq, &ts);
1584                 }
1585         }
1586         return 0;
1587 }
1588
1589 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1590 {
1591         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1592                                     sample->pid, sample->tid, 0);
1593 }
1594
1595 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1596 {
1597         unsigned i, j;
1598
1599         if (cpu < 0 || !pt->queues.nr_queues)
1600                 return NULL;
1601
1602         if ((unsigned)cpu >= pt->queues.nr_queues)
1603                 i = pt->queues.nr_queues - 1;
1604         else
1605                 i = cpu;
1606
1607         if (pt->queues.queue_array[i].cpu == cpu)
1608                 return pt->queues.queue_array[i].priv;
1609
1610         for (j = 0; i > 0; j++) {
1611                 if (pt->queues.queue_array[--i].cpu == cpu)
1612                         return pt->queues.queue_array[i].priv;
1613         }
1614
1615         for (; j < pt->queues.nr_queues; j++) {
1616                 if (pt->queues.queue_array[j].cpu == cpu)
1617                         return pt->queues.queue_array[j].priv;
1618         }
1619
1620         return NULL;
1621 }
1622
1623 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1624                                 u64 timestamp)
1625 {
1626         struct intel_pt_queue *ptq;
1627         int err;
1628
1629         if (!pt->sync_switch)
1630                 return 1;
1631
1632         ptq = intel_pt_cpu_to_ptq(pt, cpu);
1633         if (!ptq)
1634                 return 1;
1635
1636         switch (ptq->switch_state) {
1637         case INTEL_PT_SS_NOT_TRACING:
1638                 ptq->next_tid = -1;
1639                 break;
1640         case INTEL_PT_SS_UNKNOWN:
1641         case INTEL_PT_SS_TRACING:
1642                 ptq->next_tid = tid;
1643                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1644                 return 0;
1645         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1646                 if (!ptq->on_heap) {
1647                         ptq->timestamp = perf_time_to_tsc(timestamp,
1648                                                           &pt->tc);
1649                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1650                                                  ptq->timestamp);
1651                         if (err)
1652                                 return err;
1653                         ptq->on_heap = true;
1654                 }
1655                 ptq->switch_state = INTEL_PT_SS_TRACING;
1656                 break;
1657         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1658                 ptq->next_tid = tid;
1659                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1660                 break;
1661         default:
1662                 break;
1663         }
1664
1665         return 1;
1666 }
1667
1668 static int intel_pt_process_switch(struct intel_pt *pt,
1669                                    struct perf_sample *sample)
1670 {
1671         struct perf_evsel *evsel;
1672         pid_t tid;
1673         int cpu, ret;
1674
1675         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1676         if (evsel != pt->switch_evsel)
1677                 return 0;
1678
1679         tid = perf_evsel__intval(evsel, sample, "next_pid");
1680         cpu = sample->cpu;
1681
1682         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1683                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1684                      &pt->tc));
1685
1686         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1687         if (ret <= 0)
1688                 return ret;
1689
1690         return machine__set_current_tid(pt->machine, cpu, -1, tid);
1691 }
1692
1693 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1694                                    struct perf_sample *sample)
1695 {
1696         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1697         pid_t pid, tid;
1698         int cpu, ret;
1699
1700         cpu = sample->cpu;
1701
1702         if (pt->have_sched_switch == 3) {
1703                 if (!out)
1704                         return 0;
1705                 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1706                         pr_err("Expecting CPU-wide context switch event\n");
1707                         return -EINVAL;
1708                 }
1709                 pid = event->context_switch.next_prev_pid;
1710                 tid = event->context_switch.next_prev_tid;
1711         } else {
1712                 if (out)
1713                         return 0;
1714                 pid = sample->pid;
1715                 tid = sample->tid;
1716         }
1717
1718         if (tid == -1) {
1719                 pr_err("context_switch event has no tid\n");
1720                 return -EINVAL;
1721         }
1722
1723         intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1724                      cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
1725                      &pt->tc));
1726
1727         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1728         if (ret <= 0)
1729                 return ret;
1730
1731         return machine__set_current_tid(pt->machine, cpu, pid, tid);
1732 }
1733
1734 static int intel_pt_process_itrace_start(struct intel_pt *pt,
1735                                          union perf_event *event,
1736                                          struct perf_sample *sample)
1737 {
1738         if (!pt->per_cpu_mmaps)
1739                 return 0;
1740
1741         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1742                      sample->cpu, event->itrace_start.pid,
1743                      event->itrace_start.tid, sample->time,
1744                      perf_time_to_tsc(sample->time, &pt->tc));
1745
1746         return machine__set_current_tid(pt->machine, sample->cpu,
1747                                         event->itrace_start.pid,
1748                                         event->itrace_start.tid);
1749 }
1750
1751 static int intel_pt_process_event(struct perf_session *session,
1752                                   union perf_event *event,
1753                                   struct perf_sample *sample,
1754                                   struct perf_tool *tool)
1755 {
1756         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1757                                            auxtrace);
1758         u64 timestamp;
1759         int err = 0;
1760
1761         if (dump_trace)
1762                 return 0;
1763
1764         if (!tool->ordered_events) {
1765                 pr_err("Intel Processor Trace requires ordered events\n");
1766                 return -EINVAL;
1767         }
1768
1769         if (sample->time && sample->time != (u64)-1)
1770                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1771         else
1772                 timestamp = 0;
1773
1774         if (timestamp || pt->timeless_decoding) {
1775                 err = intel_pt_update_queues(pt);
1776                 if (err)
1777                         return err;
1778         }
1779
1780         if (pt->timeless_decoding) {
1781                 if (event->header.type == PERF_RECORD_EXIT) {
1782                         err = intel_pt_process_timeless_queues(pt,
1783                                                                event->fork.tid,
1784                                                                sample->time);
1785                 }
1786         } else if (timestamp) {
1787                 err = intel_pt_process_queues(pt, timestamp);
1788         }
1789         if (err)
1790                 return err;
1791
1792         if (event->header.type == PERF_RECORD_AUX &&
1793             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
1794             pt->synth_opts.errors) {
1795                 err = intel_pt_lost(pt, sample);
1796                 if (err)
1797                         return err;
1798         }
1799
1800         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
1801                 err = intel_pt_process_switch(pt, sample);
1802         else if (event->header.type == PERF_RECORD_ITRACE_START)
1803                 err = intel_pt_process_itrace_start(pt, event, sample);
1804         else if (event->header.type == PERF_RECORD_SWITCH ||
1805                  event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
1806                 err = intel_pt_context_switch(pt, event, sample);
1807
1808         intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
1809                      perf_event__name(event->header.type), event->header.type,
1810                      sample->cpu, sample->time, timestamp);
1811
1812         return err;
1813 }
1814
1815 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
1816 {
1817         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1818                                            auxtrace);
1819         int ret;
1820
1821         if (dump_trace)
1822                 return 0;
1823
1824         if (!tool->ordered_events)
1825                 return -EINVAL;
1826
1827         ret = intel_pt_update_queues(pt);
1828         if (ret < 0)
1829                 return ret;
1830
1831         if (pt->timeless_decoding)
1832                 return intel_pt_process_timeless_queues(pt, -1,
1833                                                         MAX_TIMESTAMP - 1);
1834
1835         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
1836 }
1837
1838 static void intel_pt_free_events(struct perf_session *session)
1839 {
1840         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1841                                            auxtrace);
1842         struct auxtrace_queues *queues = &pt->queues;
1843         unsigned int i;
1844
1845         for (i = 0; i < queues->nr_queues; i++) {
1846                 intel_pt_free_queue(queues->queue_array[i].priv);
1847                 queues->queue_array[i].priv = NULL;
1848         }
1849         intel_pt_log_disable();
1850         auxtrace_queues__free(queues);
1851 }
1852
1853 static void intel_pt_free(struct perf_session *session)
1854 {
1855         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1856                                            auxtrace);
1857
1858         auxtrace_heap__free(&pt->heap);
1859         intel_pt_free_events(session);
1860         session->auxtrace = NULL;
1861         thread__put(pt->unknown_thread);
1862         addr_filters__exit(&pt->filts);
1863         zfree(&pt->filter);
1864         free(pt);
1865 }
1866
1867 static int intel_pt_process_auxtrace_event(struct perf_session *session,
1868                                            union perf_event *event,
1869                                            struct perf_tool *tool __maybe_unused)
1870 {
1871         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1872                                            auxtrace);
1873
1874         if (pt->sampling_mode)
1875                 return 0;
1876
1877         if (!pt->data_queued) {
1878                 struct auxtrace_buffer *buffer;
1879                 off_t data_offset;
1880                 int fd = perf_data_file__fd(session->file);
1881                 int err;
1882
1883                 if (perf_data_file__is_pipe(session->file)) {
1884                         data_offset = 0;
1885                 } else {
1886                         data_offset = lseek(fd, 0, SEEK_CUR);
1887                         if (data_offset == -1)
1888                                 return -errno;
1889                 }
1890
1891                 err = auxtrace_queues__add_event(&pt->queues, session, event,
1892                                                  data_offset, &buffer);
1893                 if (err)
1894                         return err;
1895
1896                 /* Dump here now we have copied a piped trace out of the pipe */
1897                 if (dump_trace) {
1898                         if (auxtrace_buffer__get_data(buffer, fd)) {
1899                                 intel_pt_dump_event(pt, buffer->data,
1900                                                     buffer->size);
1901                                 auxtrace_buffer__put_data(buffer);
1902                         }
1903                 }
1904         }
1905
1906         return 0;
1907 }
1908
1909 struct intel_pt_synth {
1910         struct perf_tool dummy_tool;
1911         struct perf_session *session;
1912 };
1913
1914 static int intel_pt_event_synth(struct perf_tool *tool,
1915                                 union perf_event *event,
1916                                 struct perf_sample *sample __maybe_unused,
1917                                 struct machine *machine __maybe_unused)
1918 {
1919         struct intel_pt_synth *intel_pt_synth =
1920                         container_of(tool, struct intel_pt_synth, dummy_tool);
1921
1922         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
1923                                                  NULL);
1924 }
1925
1926 static int intel_pt_synth_event(struct perf_session *session,
1927                                 struct perf_event_attr *attr, u64 id)
1928 {
1929         struct intel_pt_synth intel_pt_synth;
1930
1931         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
1932         intel_pt_synth.session = session;
1933
1934         return perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
1935                                            &id, intel_pt_event_synth);
1936 }
1937
1938 static int intel_pt_synth_events(struct intel_pt *pt,
1939                                  struct perf_session *session)
1940 {
1941         struct perf_evlist *evlist = session->evlist;
1942         struct perf_evsel *evsel;
1943         struct perf_event_attr attr;
1944         bool found = false;
1945         u64 id;
1946         int err;
1947
1948         evlist__for_each_entry(evlist, evsel) {
1949                 if (evsel->attr.type == pt->pmu_type && evsel->ids) {
1950                         found = true;
1951                         break;
1952                 }
1953         }
1954
1955         if (!found) {
1956                 pr_debug("There are no selected events with Intel Processor Trace data\n");
1957                 return 0;
1958         }
1959
1960         memset(&attr, 0, sizeof(struct perf_event_attr));
1961         attr.size = sizeof(struct perf_event_attr);
1962         attr.type = PERF_TYPE_HARDWARE;
1963         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
1964         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1965                             PERF_SAMPLE_PERIOD;
1966         if (pt->timeless_decoding)
1967                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1968         else
1969                 attr.sample_type |= PERF_SAMPLE_TIME;
1970         if (!pt->per_cpu_mmaps)
1971                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
1972         attr.exclude_user = evsel->attr.exclude_user;
1973         attr.exclude_kernel = evsel->attr.exclude_kernel;
1974         attr.exclude_hv = evsel->attr.exclude_hv;
1975         attr.exclude_host = evsel->attr.exclude_host;
1976         attr.exclude_guest = evsel->attr.exclude_guest;
1977         attr.sample_id_all = evsel->attr.sample_id_all;
1978         attr.read_format = evsel->attr.read_format;
1979
1980         id = evsel->id[0] + 1000000000;
1981         if (!id)
1982                 id = 1;
1983
1984         if (pt->synth_opts.instructions) {
1985                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1986                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
1987                         attr.sample_period =
1988                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
1989                 else
1990                         attr.sample_period = pt->synth_opts.period;
1991                 pt->instructions_sample_period = attr.sample_period;
1992                 if (pt->synth_opts.callchain)
1993                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1994                 if (pt->synth_opts.last_branch)
1995                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1996                 pr_debug("Synthesizing 'instructions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1997                          id, (u64)attr.sample_type);
1998                 err = intel_pt_synth_event(session, &attr, id);
1999                 if (err) {
2000                         pr_err("%s: failed to synthesize 'instructions' event type\n",
2001                                __func__);
2002                         return err;
2003                 }
2004                 pt->sample_instructions = true;
2005                 pt->instructions_sample_type = attr.sample_type;
2006                 pt->instructions_id = id;
2007                 id += 1;
2008         }
2009
2010         if (pt->synth_opts.transactions) {
2011                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2012                 attr.sample_period = 1;
2013                 if (pt->synth_opts.callchain)
2014                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
2015                 if (pt->synth_opts.last_branch)
2016                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
2017                 pr_debug("Synthesizing 'transactions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2018                          id, (u64)attr.sample_type);
2019                 err = intel_pt_synth_event(session, &attr, id);
2020                 if (err) {
2021                         pr_err("%s: failed to synthesize 'transactions' event type\n",
2022                                __func__);
2023                         return err;
2024                 }
2025                 pt->sample_transactions = true;
2026                 pt->transactions_id = id;
2027                 id += 1;
2028                 evlist__for_each_entry(evlist, evsel) {
2029                         if (evsel->id && evsel->id[0] == pt->transactions_id) {
2030                                 if (evsel->name)
2031                                         zfree(&evsel->name);
2032                                 evsel->name = strdup("transactions");
2033                                 break;
2034                         }
2035                 }
2036         }
2037
2038         if (pt->synth_opts.branches) {
2039                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
2040                 attr.sample_period = 1;
2041                 attr.sample_type |= PERF_SAMPLE_ADDR;
2042                 attr.sample_type &= ~(u64)PERF_SAMPLE_CALLCHAIN;
2043                 attr.sample_type &= ~(u64)PERF_SAMPLE_BRANCH_STACK;
2044                 pr_debug("Synthesizing 'branches' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2045                          id, (u64)attr.sample_type);
2046                 err = intel_pt_synth_event(session, &attr, id);
2047                 if (err) {
2048                         pr_err("%s: failed to synthesize 'branches' event type\n",
2049                                __func__);
2050                         return err;
2051                 }
2052                 pt->sample_branches = true;
2053                 pt->branches_sample_type = attr.sample_type;
2054                 pt->branches_id = id;
2055         }
2056
2057         pt->synth_needs_swap = evsel->needs_swap;
2058
2059         return 0;
2060 }
2061
2062 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
2063 {
2064         struct perf_evsel *evsel;
2065
2066         evlist__for_each_entry_reverse(evlist, evsel) {
2067                 const char *name = perf_evsel__name(evsel);
2068
2069                 if (!strcmp(name, "sched:sched_switch"))
2070                         return evsel;
2071         }
2072
2073         return NULL;
2074 }
2075
2076 static bool intel_pt_find_switch(struct perf_evlist *evlist)
2077 {
2078         struct perf_evsel *evsel;
2079
2080         evlist__for_each_entry(evlist, evsel) {
2081                 if (evsel->attr.context_switch)
2082                         return true;
2083         }
2084
2085         return false;
2086 }
2087
2088 static int intel_pt_perf_config(const char *var, const char *value, void *data)
2089 {
2090         struct intel_pt *pt = data;
2091
2092         if (!strcmp(var, "intel-pt.mispred-all"))
2093                 pt->mispred_all = perf_config_bool(var, value);
2094
2095         return 0;
2096 }
2097
2098 static const char * const intel_pt_info_fmts[] = {
2099         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
2100         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
2101         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
2102         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
2103         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
2104         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
2105         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
2106         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
2107         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
2108         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
2109         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
2110         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
2111         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
2112         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
2113         [INTEL_PT_MAX_NONTURBO_RATIO]   = "  Max non-turbo ratio %"PRIu64"\n",
2114         [INTEL_PT_FILTER_STR_LEN]       = "  Filter string len.  %"PRIu64"\n",
2115 };
2116
2117 static void intel_pt_print_info(u64 *arr, int start, int finish)
2118 {
2119         int i;
2120
2121         if (!dump_trace)
2122                 return;
2123
2124         for (i = start; i <= finish; i++)
2125                 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
2126 }
2127
2128 static void intel_pt_print_info_str(const char *name, const char *str)
2129 {
2130         if (!dump_trace)
2131                 return;
2132
2133         fprintf(stdout, "  %-20s%s\n", name, str ? str : "");
2134 }
2135
2136 static bool intel_pt_has(struct auxtrace_info_event *auxtrace_info, int pos)
2137 {
2138         return auxtrace_info->header.size >=
2139                 sizeof(struct auxtrace_info_event) + (sizeof(u64) * (pos + 1));
2140 }
2141
2142 int intel_pt_process_auxtrace_info(union perf_event *event,
2143                                    struct perf_session *session)
2144 {
2145         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2146         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2147         struct intel_pt *pt;
2148         void *info_end;
2149         u64 *info;
2150         int err;
2151
2152         if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2153                                         min_sz)
2154                 return -EINVAL;
2155
2156         pt = zalloc(sizeof(struct intel_pt));
2157         if (!pt)
2158                 return -ENOMEM;
2159
2160         addr_filters__init(&pt->filts);
2161
2162         err = perf_config(intel_pt_perf_config, pt);
2163         if (err)
2164                 goto err_free;
2165
2166         err = auxtrace_queues__init(&pt->queues);
2167         if (err)
2168                 goto err_free;
2169
2170         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2171
2172         pt->session = session;
2173         pt->machine = &session->machines.host; /* No kvm support */
2174         pt->auxtrace_type = auxtrace_info->type;
2175         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2176         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2177         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2178         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2179         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2180         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2181         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2182         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2183         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2184         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2185         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2186                             INTEL_PT_PER_CPU_MMAPS);
2187
2188         if (intel_pt_has(auxtrace_info, INTEL_PT_CYC_BIT)) {
2189                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2190                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2191                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2192                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2193                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2194                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2195                                     INTEL_PT_CYC_BIT);
2196         }
2197
2198         if (intel_pt_has(auxtrace_info, INTEL_PT_MAX_NONTURBO_RATIO)) {
2199                 pt->max_non_turbo_ratio =
2200                         auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];
2201                 intel_pt_print_info(&auxtrace_info->priv[0],
2202                                     INTEL_PT_MAX_NONTURBO_RATIO,
2203                                     INTEL_PT_MAX_NONTURBO_RATIO);
2204         }
2205
2206         info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
2207         info_end = (void *)info + auxtrace_info->header.size;
2208
2209         if (intel_pt_has(auxtrace_info, INTEL_PT_FILTER_STR_LEN)) {
2210                 size_t len;
2211
2212                 len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];
2213                 intel_pt_print_info(&auxtrace_info->priv[0],
2214                                     INTEL_PT_FILTER_STR_LEN,
2215                                     INTEL_PT_FILTER_STR_LEN);
2216                 if (len) {
2217                         const char *filter = (const char *)info;
2218
2219                         len = roundup(len + 1, 8);
2220                         info += len >> 3;
2221                         if ((void *)info > info_end) {
2222                                 pr_err("%s: bad filter string length\n", __func__);
2223                                 err = -EINVAL;
2224                                 goto err_free_queues;
2225                         }
2226                         pt->filter = memdup(filter, len);
2227                         if (!pt->filter) {
2228                                 err = -ENOMEM;
2229                                 goto err_free_queues;
2230                         }
2231                         if (session->header.needs_swap)
2232                                 mem_bswap_64(pt->filter, len);
2233                         if (pt->filter[len - 1]) {
2234                                 pr_err("%s: filter string not null terminated\n", __func__);
2235                                 err = -EINVAL;
2236                                 goto err_free_queues;
2237                         }
2238                         err = addr_filters__parse_bare_filter(&pt->filts,
2239                                                               filter);
2240                         if (err)
2241                                 goto err_free_queues;
2242                 }
2243                 intel_pt_print_info_str("Filter string", pt->filter);
2244         }
2245
2246         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2247         pt->have_tsc = intel_pt_have_tsc(pt);
2248         pt->sampling_mode = false;
2249         pt->est_tsc = !pt->timeless_decoding;
2250
2251         pt->unknown_thread = thread__new(999999999, 999999999);
2252         if (!pt->unknown_thread) {
2253                 err = -ENOMEM;
2254                 goto err_free_queues;
2255         }
2256
2257         /*
2258          * Since this thread will not be kept in any rbtree not in a
2259          * list, initialize its list node so that at thread__put() the
2260          * current thread lifetime assuption is kept and we don't segfault
2261          * at list_del_init().
2262          */
2263         INIT_LIST_HEAD(&pt->unknown_thread->node);
2264
2265         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2266         if (err)
2267                 goto err_delete_thread;
2268         if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2269                 err = -ENOMEM;
2270                 goto err_delete_thread;
2271         }
2272
2273         pt->auxtrace.process_event = intel_pt_process_event;
2274         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2275         pt->auxtrace.flush_events = intel_pt_flush;
2276         pt->auxtrace.free_events = intel_pt_free_events;
2277         pt->auxtrace.free = intel_pt_free;
2278         session->auxtrace = &pt->auxtrace;
2279
2280         if (dump_trace)
2281                 return 0;
2282
2283         if (pt->have_sched_switch == 1) {
2284                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2285                 if (!pt->switch_evsel) {
2286                         pr_err("%s: missing sched_switch event\n", __func__);
2287                         err = -EINVAL;
2288                         goto err_delete_thread;
2289                 }
2290         } else if (pt->have_sched_switch == 2 &&
2291                    !intel_pt_find_switch(session->evlist)) {
2292                 pr_err("%s: missing context_switch attribute flag\n", __func__);
2293                 err = -EINVAL;
2294                 goto err_delete_thread;
2295         }
2296
2297         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2298                 pt->synth_opts = *session->itrace_synth_opts;
2299         } else {
2300                 itrace_synth_opts__set_default(&pt->synth_opts);
2301                 if (use_browser != -1) {
2302                         pt->synth_opts.branches = false;
2303                         pt->synth_opts.callchain = true;
2304                 }
2305                 if (session->itrace_synth_opts)
2306                         pt->synth_opts.thread_stack =
2307                                 session->itrace_synth_opts->thread_stack;
2308         }
2309
2310         if (pt->synth_opts.log)
2311                 intel_pt_log_enable();
2312
2313         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
2314         if (pt->tc.time_mult) {
2315                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2316
2317                 if (!pt->max_non_turbo_ratio)
2318                         pt->max_non_turbo_ratio =
2319                                         (tsc_freq + 50000000) / 100000000;
2320                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2321                 intel_pt_log("Maximum non-turbo ratio %u\n",
2322                              pt->max_non_turbo_ratio);
2323         }
2324
2325         if (pt->synth_opts.calls)
2326                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2327                                        PERF_IP_FLAG_TRACE_END;
2328         if (pt->synth_opts.returns)
2329                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
2330                                        PERF_IP_FLAG_TRACE_BEGIN;
2331
2332         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2333                 symbol_conf.use_callchain = true;
2334                 if (callchain_register_param(&callchain_param) < 0) {
2335                         symbol_conf.use_callchain = false;
2336                         pt->synth_opts.callchain = false;
2337                 }
2338         }
2339
2340         err = intel_pt_synth_events(pt, session);
2341         if (err)
2342                 goto err_delete_thread;
2343
2344         err = auxtrace_queues__process_index(&pt->queues, session);
2345         if (err)
2346                 goto err_delete_thread;
2347
2348         if (pt->queues.populated)
2349                 pt->data_queued = true;
2350
2351         if (pt->timeless_decoding)
2352                 pr_debug2("Intel PT decoding without timestamps\n");
2353
2354         return 0;
2355
2356 err_delete_thread:
2357         thread__zput(pt->unknown_thread);
2358 err_free_queues:
2359         intel_pt_log_disable();
2360         auxtrace_queues__free(&pt->queues);
2361         session->auxtrace = NULL;
2362 err_free:
2363         addr_filters__exit(&pt->filts);
2364         zfree(&pt->filter);
2365         free(pt);
2366         return err;
2367 }