tools/perf/util/intel-bts.c

   1 /*
   2  * intel-bts.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2015, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #include <endian.h>
  17 #include <errno.h>
  18 #include <byteswap.h>
  19 #include <inttypes.h>
  20 #include <linux/kernel.h>
  21 #include <linux/types.h>
  22 #include <linux/bitops.h>
  23 #include <linux/log2.h>
  24
  25 #include "cpumap.h"
  26 #include "color.h"
  27 #include "evsel.h"
  28 #include "evlist.h"
  29 #include "machine.h"
  30 #include "map.h"
  31 #include "symbol.h"
  32 #include "session.h"
  33 #include "util.h"
  34 #include "thread.h"
  35 #include "thread-stack.h"
  36 #include "debug.h"
  37 #include "tsc.h"
  38 #include "auxtrace.h"
  39 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  40 #include "intel-bts.h"
  41
  42 #define MAX_TIMESTAMP (~0ULL)
  43
  44 #define INTEL_BTS_ERR_NOINSN  5
  45 #define INTEL_BTS_ERR_LOST    9
  46
  47 #if __BYTE_ORDER == __BIG_ENDIAN
  48 #define le64_to_cpu bswap_64
  49 #else
  50 #define le64_to_cpu
  51 #endif
  52
  53 struct intel_bts {
  54         struct auxtrace                 auxtrace;
  55         struct auxtrace_queues          queues;
  56         struct auxtrace_heap            heap;
  57         u32                             auxtrace_type;
  58         struct perf_session             *session;
  59         struct machine                  *machine;
  60         bool                            sampling_mode;
  61         bool                            snapshot_mode;
  62         bool                            data_queued;
  63         u32                             pmu_type;
  64         struct perf_tsc_conversion      tc;
  65         bool                            cap_user_time_zero;
  66         struct itrace_synth_opts        synth_opts;
  67         bool                            sample_branches;
  68         u32                             branches_filter;
  69         u64                             branches_sample_type;
  70         u64                             branches_id;
  71         size_t                          branches_event_size;
  72         unsigned long                   num_events;
  73 };
  74
  75 struct intel_bts_queue {
  76         struct intel_bts        *bts;
  77         unsigned int            queue_nr;
  78         struct auxtrace_buffer  *buffer;
  79         bool                    on_heap;
  80         bool                    done;
  81         pid_t                   pid;
  82         pid_t                   tid;
  83         int                     cpu;
  84         u64                     time;
  85         struct intel_pt_insn    intel_pt_insn;
  86         u32                     sample_flags;
  87 };
  88
  89 struct branch {
  90         u64 from;
  91         u64 to;
  92         u64 misc;
  93 };
  94
  95 static void intel_bts_dump(struct intel_bts *bts __maybe_unused,
  96                            unsigned char *buf, size_t len)
  97 {
  98         struct branch *branch;
  99         size_t i, pos = 0, br_sz = sizeof(struct branch), sz;
 100         const char *color = PERF_COLOR_BLUE;
 101
 102         color_fprintf(stdout, color,
 103                       ". ... Intel BTS data: size %zu bytes\n",
 104                       len);
 105
 106         while (len) {
 107                 if (len >= br_sz)
 108                         sz = br_sz;
 109                 else
 110                         sz = len;
 111                 printf(".");
 112                 color_fprintf(stdout, color, "  %08x: ", pos);
 113                 for (i = 0; i < sz; i++)
 114                         color_fprintf(stdout, color, " %02x", buf[i]);
 115                 for (; i < br_sz; i++)
 116                         color_fprintf(stdout, color, "   ");
 117                 if (len >= br_sz) {
 118                         branch = (struct branch *)buf;
 119                         color_fprintf(stdout, color, " %"PRIx64" -> %"PRIx64" %s\n",
 120                                       le64_to_cpu(branch->from),
 121                                       le64_to_cpu(branch->to),
 122                                       le64_to_cpu(branch->misc) & 0x10 ?
 123                                                         "pred" : "miss");
 124                 } else {
 125                         color_fprintf(stdout, color, " Bad record!\n");
 126                 }
 127                 pos += sz;
 128                 buf += sz;
 129                 len -= sz;
 130         }
 131 }
 132
 133 static void intel_bts_dump_event(struct intel_bts *bts, unsigned char *buf,
 134                                  size_t len)
 135 {
 136         printf(".\n");
 137         intel_bts_dump(bts, buf, len);
 138 }
 139
 140 static int intel_bts_lost(struct intel_bts *bts, struct perf_sample *sample)
 141 {
 142         union perf_event event;
 143         int err;
 144
 145         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
 146                              INTEL_BTS_ERR_LOST, sample->cpu, sample->pid,
 147                              sample->tid, 0, "Lost trace data", sample->time);
 148
 149         err = perf_session__deliver_synth_event(bts->session, &event, NULL);
 150         if (err)
 151                 pr_err("Intel BTS: failed to deliver error event, error %d\n",
 152                        err);
 153
 154         return err;
 155 }
 156
 157 static struct intel_bts_queue *intel_bts_alloc_queue(struct intel_bts *bts,
 158                                                      unsigned int queue_nr)
 159 {
 160         struct intel_bts_queue *btsq;
 161
 162         btsq = zalloc(sizeof(struct intel_bts_queue));
 163         if (!btsq)
 164                 return NULL;
 165
 166         btsq->bts = bts;
 167         btsq->queue_nr = queue_nr;
 168         btsq->pid = -1;
 169         btsq->tid = -1;
 170         btsq->cpu = -1;
 171
 172         return btsq;
 173 }
 174
 175 static int intel_bts_setup_queue(struct intel_bts *bts,
 176                                  struct auxtrace_queue *queue,
 177                                  unsigned int queue_nr)
 178 {
 179         struct intel_bts_queue *btsq = queue->priv;
 180
 181         if (list_empty(&queue->head))
 182                 return 0;
 183
 184         if (!btsq) {
 185                 btsq = intel_bts_alloc_queue(bts, queue_nr);
 186                 if (!btsq)
 187                         return -ENOMEM;
 188                 queue->priv = btsq;
 189
 190                 if (queue->cpu != -1)
 191                         btsq->cpu = queue->cpu;
 192                 btsq->tid = queue->tid;
 193         }
 194
 195         if (bts->sampling_mode)
 196                 return 0;
 197
 198         if (!btsq->on_heap && !btsq->buffer) {
 199                 int ret;
 200
 201                 btsq->buffer = auxtrace_buffer__next(queue, NULL);
 202                 if (!btsq->buffer)
 203                         return 0;
 204
 205                 ret = auxtrace_heap__add(&bts->heap, queue_nr,
 206                                          btsq->buffer->reference);
 207                 if (ret)
 208                         return ret;
 209                 btsq->on_heap = true;
 210         }
 211
 212         return 0;
 213 }
 214
 215 static int intel_bts_setup_queues(struct intel_bts *bts)
 216 {
 217         unsigned int i;
 218         int ret;
 219
 220         for (i = 0; i < bts->queues.nr_queues; i++) {
 221                 ret = intel_bts_setup_queue(bts, &bts->queues.queue_array[i],
 222                                             i);
 223                 if (ret)
 224                         return ret;
 225         }
 226         return 0;
 227 }
 228
 229 static inline int intel_bts_update_queues(struct intel_bts *bts)
 230 {
 231         if (bts->queues.new_data) {
 232                 bts->queues.new_data = false;
 233                 return intel_bts_setup_queues(bts);
 234         }
 235         return 0;
 236 }
 237
 238 static unsigned char *intel_bts_find_overlap(unsigned char *buf_a, size_t len_a,
 239                                              unsigned char *buf_b, size_t len_b)
 240 {
 241         size_t offs, len;
 242
 243         if (len_a > len_b)
 244                 offs = len_a - len_b;
 245         else
 246                 offs = 0;
 247
 248         for (; offs < len_a; offs += sizeof(struct branch)) {
 249                 len = len_a - offs;
 250                 if (!memcmp(buf_a + offs, buf_b, len))
 251                         return buf_b + len;
 252         }
 253
 254         return buf_b;
 255 }
 256
 257 static int intel_bts_do_fix_overlap(struct auxtrace_queue *queue,
 258                                     struct auxtrace_buffer *b)
 259 {
 260         struct auxtrace_buffer *a;
 261         void *start;
 262
 263         if (b->list.prev == &queue->head)
 264                 return 0;
 265         a = list_entry(b->list.prev, struct auxtrace_buffer, list);
 266         start = intel_bts_find_overlap(a->data, a->size, b->data, b->size);
 267         if (!start)
 268                 return -EINVAL;
 269         b->use_size = b->data + b->size - start;
 270         b->use_data = start;
 271         return 0;
 272 }
 273
 274 static inline u8 intel_bts_cpumode(struct intel_bts *bts, uint64_t ip)
 275 {
 276         return machine__kernel_ip(bts->machine, ip) ?
 277                PERF_RECORD_MISC_KERNEL :
 278                PERF_RECORD_MISC_USER;
 279 }
 280
 281 static int intel_bts_synth_branch_sample(struct intel_bts_queue *btsq,
 282                                          struct branch *branch)
 283 {
 284         int ret;
 285         struct intel_bts *bts = btsq->bts;
 286         union perf_event event;
 287         struct perf_sample sample = { .ip = 0, };
 288
 289         if (bts->synth_opts.initial_skip &&
 290             bts->num_events++ <= bts->synth_opts.initial_skip)
 291                 return 0;
 292
 293         sample.ip = le64_to_cpu(branch->from);
 294         sample.cpumode = intel_bts_cpumode(bts, sample.ip);
 295         sample.pid = btsq->pid;
 296         sample.tid = btsq->tid;
 297         sample.addr = le64_to_cpu(branch->to);
 298         sample.id = btsq->bts->branches_id;
 299         sample.stream_id = btsq->bts->branches_id;
 300         sample.period = 1;
 301         sample.cpu = btsq->cpu;
 302         sample.flags = btsq->sample_flags;
 303         sample.insn_len = btsq->intel_pt_insn.length;
 304         memcpy(sample.insn, btsq->intel_pt_insn.buf, INTEL_PT_INSN_BUF_SZ);
 305
 306         event.sample.header.type = PERF_RECORD_SAMPLE;
 307         event.sample.header.misc = sample.cpumode;
 308         event.sample.header.size = sizeof(struct perf_event_header);
 309
 310         if (bts->synth_opts.inject) {
 311                 event.sample.header.size = bts->branches_event_size;
 312                 ret = perf_event__synthesize_sample(&event,
 313                                                     bts->branches_sample_type,
 314                                                     0, &sample);
 315                 if (ret)
 316                         return ret;
 317         }
 318
 319         ret = perf_session__deliver_synth_event(bts->session, &event, &sample);
 320         if (ret)
 321                 pr_err("Intel BTS: failed to deliver branch event, error %d\n",
 322                        ret);
 323
 324         return ret;
 325 }
 326
 327 static int intel_bts_get_next_insn(struct intel_bts_queue *btsq, u64 ip)
 328 {
 329         struct machine *machine = btsq->bts->machine;
 330         struct thread *thread;
 331         unsigned char buf[INTEL_PT_INSN_BUF_SZ];
 332         ssize_t len;
 333         bool x86_64;
 334         int err = -1;
 335
 336         thread = machine__find_thread(machine, -1, btsq->tid);
 337         if (!thread)
 338                 return -1;
 339
 340         len = thread__memcpy(thread, machine, buf, ip, INTEL_PT_INSN_BUF_SZ, &x86_64);
 341         if (len <= 0)
 342                 goto out_put;
 343
 344         if (intel_pt_get_insn(buf, len, x86_64, &btsq->intel_pt_insn))
 345                 goto out_put;
 346
 347         err = 0;
 348 out_put:
 349         thread__put(thread);
 350         return err;
 351 }
 352
 353 static int intel_bts_synth_error(struct intel_bts *bts, int cpu, pid_t pid,
 354                                  pid_t tid, u64 ip)
 355 {
 356         union perf_event event;
 357         int err;
 358
 359         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
 360                              INTEL_BTS_ERR_NOINSN, cpu, pid, tid, ip,
 361                              "Failed to get instruction", 0);
 362
 363         err = perf_session__deliver_synth_event(bts->session, &event, NULL);
 364         if (err)
 365                 pr_err("Intel BTS: failed to deliver error event, error %d\n",
 366                        err);
 367
 368         return err;
 369 }
 370
 371 static int intel_bts_get_branch_type(struct intel_bts_queue *btsq,
 372                                      struct branch *branch)
 373 {
 374         int err;
 375
 376         if (!branch->from) {
 377                 if (branch->to)
 378                         btsq->sample_flags = PERF_IP_FLAG_BRANCH |
 379                                              PERF_IP_FLAG_TRACE_BEGIN;
 380                 else
 381                         btsq->sample_flags = 0;
 382                 btsq->intel_pt_insn.length = 0;
 383         } else if (!branch->to) {
 384                 btsq->sample_flags = PERF_IP_FLAG_BRANCH |
 385                                      PERF_IP_FLAG_TRACE_END;
 386                 btsq->intel_pt_insn.length = 0;
 387         } else {
 388                 err = intel_bts_get_next_insn(btsq, branch->from);
 389                 if (err) {
 390                         btsq->sample_flags = 0;
 391                         btsq->intel_pt_insn.length = 0;
 392                         if (!btsq->bts->synth_opts.errors)
 393                                 return 0;
 394                         err = intel_bts_synth_error(btsq->bts, btsq->cpu,
 395                                                     btsq->pid, btsq->tid,
 396                                                     branch->from);
 397                         return err;
 398                 }
 399                 btsq->sample_flags = intel_pt_insn_type(btsq->intel_pt_insn.op);
 400                 /* Check for an async branch into the kernel */
 401                 if (!machine__kernel_ip(btsq->bts->machine, branch->from) &&
 402                     machine__kernel_ip(btsq->bts->machine, branch->to) &&
 403                     btsq->sample_flags != (PERF_IP_FLAG_BRANCH |
 404                                            PERF_IP_FLAG_CALL |
 405                                            PERF_IP_FLAG_SYSCALLRET))
 406                         btsq->sample_flags = PERF_IP_FLAG_BRANCH |
 407                                              PERF_IP_FLAG_CALL |
 408                                              PERF_IP_FLAG_ASYNC |
 409                                              PERF_IP_FLAG_INTERRUPT;
 410         }
 411
 412         return 0;
 413 }
 414
 415 static int intel_bts_process_buffer(struct intel_bts_queue *btsq,
 416                                     struct auxtrace_buffer *buffer,
 417                                     struct thread *thread)
 418 {
 419         struct branch *branch;
 420         size_t sz, bsz = sizeof(struct branch);
 421         u32 filter = btsq->bts->branches_filter;
 422         int err = 0;
 423
 424         if (buffer->use_data) {
 425                 sz = buffer->use_size;
 426                 branch = buffer->use_data;
 427         } else {
 428                 sz = buffer->size;
 429                 branch = buffer->data;
 430         }
 431
 432         if (!btsq->bts->sample_branches)
 433                 return 0;
 434
 435         for (; sz > bsz; branch += 1, sz -= bsz) {
 436                 if (!branch->from && !branch->to)
 437                         continue;
 438                 intel_bts_get_branch_type(btsq, branch);
 439                 if (btsq->bts->synth_opts.thread_stack)
 440                         thread_stack__event(thread, btsq->cpu, btsq->sample_flags,
 441                                             le64_to_cpu(branch->from),
 442                                             le64_to_cpu(branch->to),
 443                                             btsq->intel_pt_insn.length,
 444                                             buffer->buffer_nr + 1);
 445                 if (filter && !(filter & btsq->sample_flags))
 446                         continue;
 447                 err = intel_bts_synth_branch_sample(btsq, branch);
 448                 if (err)
 449                         break;
 450         }
 451         return err;
 452 }
 453
 454 static int intel_bts_process_queue(struct intel_bts_queue *btsq, u64 *timestamp)
 455 {
 456         struct auxtrace_buffer *buffer = btsq->buffer, *old_buffer = buffer;
 457         struct auxtrace_queue *queue;
 458         struct thread *thread;
 459         int err;
 460
 461         if (btsq->done)
 462                 return 1;
 463
 464         if (btsq->pid == -1) {
 465                 thread = machine__find_thread(btsq->bts->machine, -1,
 466                                               btsq->tid);
 467                 if (thread)
 468                         btsq->pid = thread->pid_;
 469         } else {
 470                 thread = machine__findnew_thread(btsq->bts->machine, btsq->pid,
 471                                                  btsq->tid);
 472         }
 473
 474         queue = &btsq->bts->queues.queue_array[btsq->queue_nr];
 475
 476         if (!buffer)
 477                 buffer = auxtrace_buffer__next(queue, NULL);
 478
 479         if (!buffer) {
 480                 if (!btsq->bts->sampling_mode)
 481                         btsq->done = 1;
 482                 err = 1;
 483                 goto out_put;
 484         }
 485
 486         /* Currently there is no support for split buffers */
 487         if (buffer->consecutive) {
 488                 err = -EINVAL;
 489                 goto out_put;
 490         }
 491
 492         if (!buffer->data) {
 493                 int fd = perf_data__fd(btsq->bts->session->data);
 494
 495                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 496                 if (!buffer->data) {
 497                         err = -ENOMEM;
 498                         goto out_put;
 499                 }
 500         }
 501
 502         if (btsq->bts->snapshot_mode && !buffer->consecutive &&
 503             intel_bts_do_fix_overlap(queue, buffer)) {
 504                 err = -ENOMEM;
 505                 goto out_put;
 506         }
 507
 508         if (!btsq->bts->synth_opts.callchain &&
 509             !btsq->bts->synth_opts.thread_stack && thread &&
 510             (!old_buffer || btsq->bts->sampling_mode ||
 511              (btsq->bts->snapshot_mode && !buffer->consecutive)))
 512                 thread_stack__set_trace_nr(thread, btsq->cpu, buffer->buffer_nr + 1);
 513
 514         err = intel_bts_process_buffer(btsq, buffer, thread);
 515
 516         auxtrace_buffer__drop_data(buffer);
 517
 518         btsq->buffer = auxtrace_buffer__next(queue, buffer);
 519         if (btsq->buffer) {
 520                 if (timestamp)
 521                         *timestamp = btsq->buffer->reference;
 522         } else {
 523                 if (!btsq->bts->sampling_mode)
 524                         btsq->done = 1;
 525         }
 526 out_put:
 527         thread__put(thread);
 528         return err;
 529 }
 530
 531 static int intel_bts_flush_queue(struct intel_bts_queue *btsq)
 532 {
 533         u64 ts = 0;
 534         int ret;
 535
 536         while (1) {
 537                 ret = intel_bts_process_queue(btsq, &ts);
 538                 if (ret < 0)
 539                         return ret;
 540                 if (ret)
 541                         break;
 542         }
 543         return 0;
 544 }
 545
 546 static int intel_bts_process_tid_exit(struct intel_bts *bts, pid_t tid)
 547 {
 548         struct auxtrace_queues *queues = &bts->queues;
 549         unsigned int i;
 550
 551         for (i = 0; i < queues->nr_queues; i++) {
 552                 struct auxtrace_queue *queue = &bts->queues.queue_array[i];
 553                 struct intel_bts_queue *btsq = queue->priv;
 554
 555                 if (btsq && btsq->tid == tid)
 556                         return intel_bts_flush_queue(btsq);
 557         }
 558         return 0;
 559 }
 560
 561 static int intel_bts_process_queues(struct intel_bts *bts, u64 timestamp)
 562 {
 563         while (1) {
 564                 unsigned int queue_nr;
 565                 struct auxtrace_queue *queue;
 566                 struct intel_bts_queue *btsq;
 567                 u64 ts = 0;
 568                 int ret;
 569
 570                 if (!bts->heap.heap_cnt)
 571                         return 0;
 572
 573                 if (bts->heap.heap_array[0].ordinal > timestamp)
 574                         return 0;
 575
 576                 queue_nr = bts->heap.heap_array[0].queue_nr;
 577                 queue = &bts->queues.queue_array[queue_nr];
 578                 btsq = queue->priv;
 579
 580                 auxtrace_heap__pop(&bts->heap);
 581
 582                 ret = intel_bts_process_queue(btsq, &ts);
 583                 if (ret < 0) {
 584                         auxtrace_heap__add(&bts->heap, queue_nr, ts);
 585                         return ret;
 586                 }
 587
 588                 if (!ret) {
 589                         ret = auxtrace_heap__add(&bts->heap, queue_nr, ts);
 590                         if (ret < 0)
 591                                 return ret;
 592                 } else {
 593                         btsq->on_heap = false;
 594                 }
 595         }
 596
 597         return 0;
 598 }
 599
 600 static int intel_bts_process_event(struct perf_session *session,
 601                                    union perf_event *event,
 602                                    struct perf_sample *sample,
 603                                    struct perf_tool *tool)
 604 {
 605         struct intel_bts *bts = container_of(session->auxtrace, struct intel_bts,
 606                                              auxtrace);
 607         u64 timestamp;
 608         int err;
 609
 610         if (dump_trace)
 611                 return 0;
 612
 613         if (!tool->ordered_events) {
 614                 pr_err("Intel BTS requires ordered events\n");
 615                 return -EINVAL;
 616         }
 617
 618         if (sample->time && sample->time != (u64)-1)
 619                 timestamp = perf_time_to_tsc(sample->time, &bts->tc);
 620         else
 621                 timestamp = 0;
 622
 623         err = intel_bts_update_queues(bts);
 624         if (err)
 625                 return err;
 626
 627         err = intel_bts_process_queues(bts, timestamp);
 628         if (err)
 629                 return err;
 630         if (event->header.type == PERF_RECORD_EXIT) {
 631                 err = intel_bts_process_tid_exit(bts, event->fork.tid);
 632                 if (err)
 633                         return err;
 634         }
 635
 636         if (event->header.type == PERF_RECORD_AUX &&
 637             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
 638             bts->synth_opts.errors)
 639                 err = intel_bts_lost(bts, sample);
 640
 641         return err;
 642 }
 643
 644 static int intel_bts_process_auxtrace_event(struct perf_session *session,
 645                                             union perf_event *event,
 646                                             struct perf_tool *tool __maybe_unused)
 647 {
 648         struct intel_bts *bts = container_of(session->auxtrace, struct intel_bts,
 649                                              auxtrace);
 650
 651         if (bts->sampling_mode)
 652                 return 0;
 653
 654         if (!bts->data_queued) {
 655                 struct auxtrace_buffer *buffer;
 656                 off_t data_offset;
 657                 int fd = perf_data__fd(session->data);
 658                 int err;
 659
 660                 if (perf_data__is_pipe(session->data)) {
 661                         data_offset = 0;
 662                 } else {
 663                         data_offset = lseek(fd, 0, SEEK_CUR);
 664                         if (data_offset == -1)
 665                                 return -errno;
 666                 }
 667
 668                 err = auxtrace_queues__add_event(&bts->queues, session, event,
 669                                                  data_offset, &buffer);
 670                 if (err)
 671                         return err;
 672
 673                 /* Dump here now we have copied a piped trace out of the pipe */
 674                 if (dump_trace) {
 675                         if (auxtrace_buffer__get_data(buffer, fd)) {
 676                                 intel_bts_dump_event(bts, buffer->data,
 677                                                      buffer->size);
 678                                 auxtrace_buffer__put_data(buffer);
 679                         }
 680                 }
 681         }
 682
 683         return 0;
 684 }
 685
 686 static int intel_bts_flush(struct perf_session *session,
 687                            struct perf_tool *tool __maybe_unused)
 688 {
 689         struct intel_bts *bts = container_of(session->auxtrace, struct intel_bts,
 690                                              auxtrace);
 691         int ret;
 692
 693         if (dump_trace || bts->sampling_mode)
 694                 return 0;
 695
 696         if (!tool->ordered_events)
 697                 return -EINVAL;
 698
 699         ret = intel_bts_update_queues(bts);
 700         if (ret < 0)
 701                 return ret;
 702
 703         return intel_bts_process_queues(bts, MAX_TIMESTAMP);
 704 }
 705
 706 static void intel_bts_free_queue(void *priv)
 707 {
 708         struct intel_bts_queue *btsq = priv;
 709
 710         if (!btsq)
 711                 return;
 712         free(btsq);
 713 }
 714
 715 static void intel_bts_free_events(struct perf_session *session)
 716 {
 717         struct intel_bts *bts = container_of(session->auxtrace, struct intel_bts,
 718                                              auxtrace);
 719         struct auxtrace_queues *queues = &bts->queues;
 720         unsigned int i;
 721
 722         for (i = 0; i < queues->nr_queues; i++) {
 723                 intel_bts_free_queue(queues->queue_array[i].priv);
 724                 queues->queue_array[i].priv = NULL;
 725         }
 726         auxtrace_queues__free(queues);
 727 }
 728
 729 static void intel_bts_free(struct perf_session *session)
 730 {
 731         struct intel_bts *bts = container_of(session->auxtrace, struct intel_bts,
 732                                              auxtrace);
 733
 734         auxtrace_heap__free(&bts->heap);
 735         intel_bts_free_events(session);
 736         session->auxtrace = NULL;
 737         free(bts);
 738 }
 739
 740 struct intel_bts_synth {
 741         struct perf_tool dummy_tool;
 742         struct perf_session *session;
 743 };
 744
 745 static int intel_bts_event_synth(struct perf_tool *tool,
 746                                  union perf_event *event,
 747                                  struct perf_sample *sample __maybe_unused,
 748                                  struct machine *machine __maybe_unused)
 749 {
 750         struct intel_bts_synth *intel_bts_synth =
 751                         container_of(tool, struct intel_bts_synth, dummy_tool);
 752
 753         return perf_session__deliver_synth_event(intel_bts_synth->session,
 754                                                  event, NULL);
 755 }
 756
 757 static int intel_bts_synth_event(struct perf_session *session,
 758                                  struct perf_event_attr *attr, u64 id)
 759 {
 760         struct intel_bts_synth intel_bts_synth;
 761
 762         memset(&intel_bts_synth, 0, sizeof(struct intel_bts_synth));
 763         intel_bts_synth.session = session;
 764
 765         return perf_event__synthesize_attr(&intel_bts_synth.dummy_tool, attr, 1,
 766                                            &id, intel_bts_event_synth);
 767 }
 768
 769 static int intel_bts_synth_events(struct intel_bts *bts,
 770                                   struct perf_session *session)
 771 {
 772         struct perf_evlist *evlist = session->evlist;
 773         struct perf_evsel *evsel;
 774         struct perf_event_attr attr;
 775         bool found = false;
 776         u64 id;
 777         int err;
 778
 779         evlist__for_each_entry(evlist, evsel) {
 780                 if (evsel->attr.type == bts->pmu_type && evsel->ids) {
 781                         found = true;
 782                         break;
 783                 }
 784         }
 785
 786         if (!found) {
 787                 pr_debug("There are no selected events with Intel BTS data\n");
 788                 return 0;
 789         }
 790
 791         memset(&attr, 0, sizeof(struct perf_event_attr));
 792         attr.size = sizeof(struct perf_event_attr);
 793         attr.type = PERF_TYPE_HARDWARE;
 794         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
 795         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
 796                             PERF_SAMPLE_PERIOD;
 797         attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
 798         attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
 799         attr.exclude_user = evsel->attr.exclude_user;
 800         attr.exclude_kernel = evsel->attr.exclude_kernel;
 801         attr.exclude_hv = evsel->attr.exclude_hv;
 802         attr.exclude_host = evsel->attr.exclude_host;
 803         attr.exclude_guest = evsel->attr.exclude_guest;
 804         attr.sample_id_all = evsel->attr.sample_id_all;
 805         attr.read_format = evsel->attr.read_format;
 806
 807         id = evsel->id[0] + 1000000000;
 808         if (!id)
 809                 id = 1;
 810
 811         if (bts->synth_opts.branches) {
 812                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
 813                 attr.sample_period = 1;
 814                 attr.sample_type |= PERF_SAMPLE_ADDR;
 815                 pr_debug("Synthesizing 'branches' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
 816                          id, (u64)attr.sample_type);
 817                 err = intel_bts_synth_event(session, &attr, id);
 818                 if (err) {
 819                         pr_err("%s: failed to synthesize 'branches' event type\n",
 820                                __func__);
 821                         return err;
 822                 }
 823                 bts->sample_branches = true;
 824                 bts->branches_sample_type = attr.sample_type;
 825                 bts->branches_id = id;
 826                 /*
 827                  * We only use sample types from PERF_SAMPLE_MASK so we can use
 828                  * __perf_evsel__sample_size() here.
 829                  */
 830                 bts->branches_event_size = sizeof(struct sample_event) +
 831                                 __perf_evsel__sample_size(attr.sample_type);
 832         }
 833
 834         return 0;
 835 }
 836
 837 static const char * const intel_bts_info_fmts[] = {
 838         [INTEL_BTS_PMU_TYPE]            = "  PMU Type           %"PRId64"\n",
 839         [INTEL_BTS_TIME_SHIFT]          = "  Time Shift         %"PRIu64"\n",
 840         [INTEL_BTS_TIME_MULT]           = "  Time Muliplier     %"PRIu64"\n",
 841         [INTEL_BTS_TIME_ZERO]           = "  Time Zero          %"PRIu64"\n",
 842         [INTEL_BTS_CAP_USER_TIME_ZERO]  = "  Cap Time Zero      %"PRId64"\n",
 843         [INTEL_BTS_SNAPSHOT_MODE]       = "  Snapshot mode      %"PRId64"\n",
 844 };
 845
 846 static void intel_bts_print_info(u64 *arr, int start, int finish)
 847 {
 848         int i;
 849
 850         if (!dump_trace)
 851                 return;
 852
 853         for (i = start; i <= finish; i++)
 854                 fprintf(stdout, intel_bts_info_fmts[i], arr[i]);
 855 }
 856
 857 int intel_bts_process_auxtrace_info(union perf_event *event,
 858                                     struct perf_session *session)
 859 {
 860         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
 861         size_t min_sz = sizeof(u64) * INTEL_BTS_SNAPSHOT_MODE;
 862         struct intel_bts *bts;
 863         int err;
 864
 865         if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
 866                                         min_sz)
 867                 return -EINVAL;
 868
 869         bts = zalloc(sizeof(struct intel_bts));
 870         if (!bts)
 871                 return -ENOMEM;
 872
 873         err = auxtrace_queues__init(&bts->queues);
 874         if (err)
 875                 goto err_free;
 876
 877         bts->session = session;
 878         bts->machine = &session->machines.host; /* No kvm support */
 879         bts->auxtrace_type = auxtrace_info->type;
 880         bts->pmu_type = auxtrace_info->priv[INTEL_BTS_PMU_TYPE];
 881         bts->tc.time_shift = auxtrace_info->priv[INTEL_BTS_TIME_SHIFT];
 882         bts->tc.time_mult = auxtrace_info->priv[INTEL_BTS_TIME_MULT];
 883         bts->tc.time_zero = auxtrace_info->priv[INTEL_BTS_TIME_ZERO];
 884         bts->cap_user_time_zero =
 885                         auxtrace_info->priv[INTEL_BTS_CAP_USER_TIME_ZERO];
 886         bts->snapshot_mode = auxtrace_info->priv[INTEL_BTS_SNAPSHOT_MODE];
 887
 888         bts->sampling_mode = false;
 889
 890         bts->auxtrace.process_event = intel_bts_process_event;
 891         bts->auxtrace.process_auxtrace_event = intel_bts_process_auxtrace_event;
 892         bts->auxtrace.flush_events = intel_bts_flush;
 893         bts->auxtrace.free_events = intel_bts_free_events;
 894         bts->auxtrace.free = intel_bts_free;
 895         session->auxtrace = &bts->auxtrace;
 896
 897         intel_bts_print_info(&auxtrace_info->priv[0], INTEL_BTS_PMU_TYPE,
 898                              INTEL_BTS_SNAPSHOT_MODE);
 899
 900         if (dump_trace)
 901                 return 0;
 902
 903         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
 904                 bts->synth_opts = *session->itrace_synth_opts;
 905         } else {
 906                 itrace_synth_opts__set_default(&bts->synth_opts,
 907                                 session->itrace_synth_opts->default_no_sample);
 908                 if (session->itrace_synth_opts)
 909                         bts->synth_opts.thread_stack =
 910                                 session->itrace_synth_opts->thread_stack;
 911         }
 912
 913         if (bts->synth_opts.calls)
 914                 bts->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
 915                                         PERF_IP_FLAG_TRACE_END;
 916         if (bts->synth_opts.returns)
 917                 bts->branches_filter |= PERF_IP_FLAG_RETURN |
 918                                         PERF_IP_FLAG_TRACE_BEGIN;
 919
 920         err = intel_bts_synth_events(bts, session);
 921         if (err)
 922                 goto err_free_queues;
 923
 924         err = auxtrace_queues__process_index(&bts->queues, session);
 925         if (err)
 926                 goto err_free_queues;
 927
 928         if (bts->queues.populated)
 929                 bts->data_queued = true;
 930
 931         return 0;
 932
 933 err_free_queues:
 934         auxtrace_queues__free(&bts->queues);
 935         session->auxtrace = NULL;
 936 err_free:
 937         free(bts);
 938         return err;
 939 }