tools/perf/util/cs-etm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright(C) 2015-2018 Linaro Limited.
   4  *
   5  * Author: Tor Jeremiassen <tor@ti.com>
   6  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   7  */
   8
   9 #include <linux/bitops.h>
  10 #include <linux/err.h>
  11 #include <linux/kernel.h>
  12 #include <linux/log2.h>
  13 #include <linux/types.h>
  14
  15 #include <opencsd/ocsd_if_types.h>
  16 #include <stdlib.h>
  17
  18 #include "auxtrace.h"
  19 #include "color.h"
  20 #include "cs-etm.h"
  21 #include "cs-etm-decoder/cs-etm-decoder.h"
  22 #include "debug.h"
  23 #include "evlist.h"
  24 #include "intlist.h"
  25 #include "machine.h"
  26 #include "map.h"
  27 #include "perf.h"
  28 #include "symbol.h"
  29 #include "thread.h"
  30 #include "thread_map.h"
  31 #include "thread-stack.h"
  32 #include "util.h"
  33
  34 #define MAX_TIMESTAMP (~0ULL)
  35
  36 struct cs_etm_auxtrace {
  37         struct auxtrace auxtrace;
  38         struct auxtrace_queues queues;
  39         struct auxtrace_heap heap;
  40         struct itrace_synth_opts synth_opts;
  41         struct perf_session *session;
  42         struct machine *machine;
  43         struct thread *unknown_thread;
  44
  45         u8 timeless_decoding;
  46         u8 snapshot_mode;
  47         u8 data_queued;
  48         u8 sample_branches;
  49         u8 sample_instructions;
  50
  51         int num_cpu;
  52         u32 auxtrace_type;
  53         u64 branches_sample_type;
  54         u64 branches_id;
  55         u64 instructions_sample_type;
  56         u64 instructions_sample_period;
  57         u64 instructions_id;
  58         u64 **metadata;
  59         u64 kernel_start;
  60         unsigned int pmu_type;
  61 };
  62
  63 struct cs_etm_traceid_queue {
  64         u8 trace_chan_id;
  65         u64 period_instructions;
  66         size_t last_branch_pos;
  67         union perf_event *event_buf;
  68         struct thread *thread;
  69         struct branch_stack *last_branch;
  70         struct branch_stack *last_branch_rb;
  71         struct cs_etm_packet *prev_packet;
  72         struct cs_etm_packet *packet;
  73         struct cs_etm_packet_queue packet_queue;
  74 };
  75
  76 struct cs_etm_queue {
  77         struct cs_etm_auxtrace *etm;
  78         struct cs_etm_decoder *decoder;
  79         struct auxtrace_buffer *buffer;
  80         unsigned int queue_nr;
  81         pid_t pid, tid;
  82         u64 offset;
  83         const unsigned char *buf;
  84         size_t buf_len, buf_used;
  85         struct cs_etm_traceid_queue *traceid_queues;
  86 };
  87
  88 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm);
  89 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
  90                                            pid_t tid);
  91
  92 /* PTMs ETMIDR [11:8] set to b0011 */
  93 #define ETMIDR_PTM_VERSION 0x00000300
  94
  95 static u32 cs_etm__get_v7_protocol_version(u32 etmidr)
  96 {
  97         etmidr &= ETMIDR_PTM_VERSION;
  98
  99         if (etmidr == ETMIDR_PTM_VERSION)
 100                 return CS_ETM_PROTO_PTM;
 101
 102         return CS_ETM_PROTO_ETMV3;
 103 }
 104
 105 static int cs_etm__get_magic(u8 trace_chan_id, u64 *magic)
 106 {
 107         struct int_node *inode;
 108         u64 *metadata;
 109
 110         inode = intlist__find(traceid_list, trace_chan_id);
 111         if (!inode)
 112                 return -EINVAL;
 113
 114         metadata = inode->priv;
 115         *magic = metadata[CS_ETM_MAGIC];
 116         return 0;
 117 }
 118
 119 int cs_etm__get_cpu(u8 trace_chan_id, int *cpu)
 120 {
 121         struct int_node *inode;
 122         u64 *metadata;
 123
 124         inode = intlist__find(traceid_list, trace_chan_id);
 125         if (!inode)
 126                 return -EINVAL;
 127
 128         metadata = inode->priv;
 129         *cpu = (int)metadata[CS_ETM_CPU];
 130         return 0;
 131 }
 132
 133 static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue)
 134 {
 135         int i;
 136
 137         queue->head = 0;
 138         queue->tail = 0;
 139         queue->packet_count = 0;
 140         for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) {
 141                 queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN;
 142                 queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR;
 143                 queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR;
 144                 queue->packet_buffer[i].instr_count = 0;
 145                 queue->packet_buffer[i].last_instr_taken_branch = false;
 146                 queue->packet_buffer[i].last_instr_size = 0;
 147                 queue->packet_buffer[i].last_instr_type = 0;
 148                 queue->packet_buffer[i].last_instr_subtype = 0;
 149                 queue->packet_buffer[i].last_instr_cond = 0;
 150                 queue->packet_buffer[i].flags = 0;
 151                 queue->packet_buffer[i].exception_number = UINT32_MAX;
 152                 queue->packet_buffer[i].trace_chan_id = UINT8_MAX;
 153                 queue->packet_buffer[i].cpu = INT_MIN;
 154         }
 155 }
 156
 157 static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq,
 158                                       struct cs_etm_traceid_queue *tidq,
 159                                       u8 trace_chan_id)
 160 {
 161         int rc = -ENOMEM;
 162         struct cs_etm_auxtrace *etm = etmq->etm;
 163
 164         cs_etm__clear_packet_queue(&tidq->packet_queue);
 165
 166         tidq->trace_chan_id = trace_chan_id;
 167
 168         tidq->packet = zalloc(sizeof(struct cs_etm_packet));
 169         if (!tidq->packet)
 170                 goto out;
 171
 172         tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));
 173         if (!tidq->prev_packet)
 174                 goto out_free;
 175
 176         if (etm->synth_opts.last_branch) {
 177                 size_t sz = sizeof(struct branch_stack);
 178
 179                 sz += etm->synth_opts.last_branch_sz *
 180                       sizeof(struct branch_entry);
 181                 tidq->last_branch = zalloc(sz);
 182                 if (!tidq->last_branch)
 183                         goto out_free;
 184                 tidq->last_branch_rb = zalloc(sz);
 185                 if (!tidq->last_branch_rb)
 186                         goto out_free;
 187         }
 188
 189         tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 190         if (!tidq->event_buf)
 191                 goto out_free;
 192
 193         return 0;
 194
 195 out_free:
 196         zfree(&tidq->last_branch_rb);
 197         zfree(&tidq->last_branch);
 198         zfree(&tidq->prev_packet);
 199         zfree(&tidq->packet);
 200 out:
 201         return rc;
 202 }
 203
 204 static struct cs_etm_traceid_queue
 205 *cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
 206 {
 207         struct cs_etm_traceid_queue *tidq;
 208         struct cs_etm_auxtrace *etm = etmq->etm;
 209
 210         if (!etm->timeless_decoding)
 211                 return NULL;
 212
 213         tidq = etmq->traceid_queues;
 214
 215         if (tidq)
 216                 return tidq;
 217
 218         tidq = malloc(sizeof(*tidq));
 219         if (!tidq)
 220                 return NULL;
 221
 222         memset(tidq, 0, sizeof(*tidq));
 223
 224         if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))
 225                 goto out_free;
 226
 227         etmq->traceid_queues = tidq;
 228
 229         return etmq->traceid_queues;
 230
 231 out_free:
 232         free(tidq);
 233
 234         return NULL;
 235 }
 236
 237 struct cs_etm_packet_queue
 238 *cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id)
 239 {
 240         struct cs_etm_traceid_queue *tidq;
 241
 242         tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);
 243         if (tidq)
 244                 return &tidq->packet_queue;
 245
 246         return NULL;
 247 }
 248
 249 static void cs_etm__packet_dump(const char *pkt_string)
 250 {
 251         const char *color = PERF_COLOR_BLUE;
 252         int len = strlen(pkt_string);
 253
 254         if (len && (pkt_string[len-1] == '\n'))
 255                 color_fprintf(stdout, color, "  %s", pkt_string);
 256         else
 257                 color_fprintf(stdout, color, "  %s\n", pkt_string);
 258
 259         fflush(stdout);
 260 }
 261
 262 static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params,
 263                                           struct cs_etm_auxtrace *etm, int idx,
 264                                           u32 etmidr)
 265 {
 266         u64 **metadata = etm->metadata;
 267
 268         t_params[idx].protocol = cs_etm__get_v7_protocol_version(etmidr);
 269         t_params[idx].etmv3.reg_ctrl = metadata[idx][CS_ETM_ETMCR];
 270         t_params[idx].etmv3.reg_trc_id = metadata[idx][CS_ETM_ETMTRACEIDR];
 271 }
 272
 273 static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params,
 274                                           struct cs_etm_auxtrace *etm, int idx)
 275 {
 276         u64 **metadata = etm->metadata;
 277
 278         t_params[idx].protocol = CS_ETM_PROTO_ETMV4i;
 279         t_params[idx].etmv4.reg_idr0 = metadata[idx][CS_ETMV4_TRCIDR0];
 280         t_params[idx].etmv4.reg_idr1 = metadata[idx][CS_ETMV4_TRCIDR1];
 281         t_params[idx].etmv4.reg_idr2 = metadata[idx][CS_ETMV4_TRCIDR2];
 282         t_params[idx].etmv4.reg_idr8 = metadata[idx][CS_ETMV4_TRCIDR8];
 283         t_params[idx].etmv4.reg_configr = metadata[idx][CS_ETMV4_TRCCONFIGR];
 284         t_params[idx].etmv4.reg_traceidr = metadata[idx][CS_ETMV4_TRCTRACEIDR];
 285 }
 286
 287 static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params,
 288                                      struct cs_etm_auxtrace *etm)
 289 {
 290         int i;
 291         u32 etmidr;
 292         u64 architecture;
 293
 294         for (i = 0; i < etm->num_cpu; i++) {
 295                 architecture = etm->metadata[i][CS_ETM_MAGIC];
 296
 297                 switch (architecture) {
 298                 case __perf_cs_etmv3_magic:
 299                         etmidr = etm->metadata[i][CS_ETM_ETMIDR];
 300                         cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr);
 301                         break;
 302                 case __perf_cs_etmv4_magic:
 303                         cs_etm__set_trace_param_etmv4(t_params, etm, i);
 304                         break;
 305                 default:
 306                         return -EINVAL;
 307                 }
 308         }
 309
 310         return 0;
 311 }
 312
 313 static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params,
 314                                        struct cs_etm_queue *etmq,
 315                                        enum cs_etm_decoder_operation mode)
 316 {
 317         int ret = -EINVAL;
 318
 319         if (!(mode < CS_ETM_OPERATION_MAX))
 320                 goto out;
 321
 322         d_params->packet_printer = cs_etm__packet_dump;
 323         d_params->operation = mode;
 324         d_params->data = etmq;
 325         d_params->formatted = true;
 326         d_params->fsyncs = false;
 327         d_params->hsyncs = false;
 328         d_params->frame_aligned = true;
 329
 330         ret = 0;
 331 out:
 332         return ret;
 333 }
 334
 335 static void cs_etm__dump_event(struct cs_etm_auxtrace *etm,
 336                                struct auxtrace_buffer *buffer)
 337 {
 338         int ret;
 339         const char *color = PERF_COLOR_BLUE;
 340         struct cs_etm_decoder_params d_params;
 341         struct cs_etm_trace_params *t_params;
 342         struct cs_etm_decoder *decoder;
 343         size_t buffer_used = 0;
 344
 345         fprintf(stdout, "\n");
 346         color_fprintf(stdout, color,
 347                      ". ... CoreSight ETM Trace data: size %zu bytes\n",
 348                      buffer->size);
 349
 350         /* Use metadata to fill in trace parameters for trace decoder */
 351         t_params = zalloc(sizeof(*t_params) * etm->num_cpu);
 352
 353         if (!t_params)
 354                 return;
 355
 356         if (cs_etm__init_trace_params(t_params, etm))
 357                 goto out_free;
 358
 359         /* Set decoder parameters to simply print the trace packets */
 360         if (cs_etm__init_decoder_params(&d_params, NULL,
 361                                         CS_ETM_OPERATION_PRINT))
 362                 goto out_free;
 363
 364         decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
 365
 366         if (!decoder)
 367                 goto out_free;
 368         do {
 369                 size_t consumed;
 370
 371                 ret = cs_etm_decoder__process_data_block(
 372                                 decoder, buffer->offset,
 373                                 &((u8 *)buffer->data)[buffer_used],
 374                                 buffer->size - buffer_used, &consumed);
 375                 if (ret)
 376                         break;
 377
 378                 buffer_used += consumed;
 379         } while (buffer_used < buffer->size);
 380
 381         cs_etm_decoder__free(decoder);
 382
 383 out_free:
 384         zfree(&t_params);
 385 }
 386
 387 static int cs_etm__flush_events(struct perf_session *session,
 388                                 struct perf_tool *tool)
 389 {
 390         int ret;
 391         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
 392                                                    struct cs_etm_auxtrace,
 393                                                    auxtrace);
 394         if (dump_trace)
 395                 return 0;
 396
 397         if (!tool->ordered_events)
 398                 return -EINVAL;
 399
 400         if (!etm->timeless_decoding)
 401                 return -EINVAL;
 402
 403         ret = cs_etm__update_queues(etm);
 404
 405         if (ret < 0)
 406                 return ret;
 407
 408         return cs_etm__process_timeless_queues(etm, -1);
 409 }
 410
 411 static void cs_etm__free_queue(void *priv)
 412 {
 413         struct cs_etm_queue *etmq = priv;
 414
 415         if (!etmq)
 416                 return;
 417
 418         thread__zput(etmq->traceid_queues->thread);
 419         cs_etm_decoder__free(etmq->decoder);
 420         zfree(&etmq->traceid_queues->event_buf);
 421         zfree(&etmq->traceid_queues->last_branch);
 422         zfree(&etmq->traceid_queues->last_branch_rb);
 423         zfree(&etmq->traceid_queues->prev_packet);
 424         zfree(&etmq->traceid_queues->packet);
 425         zfree(&etmq->traceid_queues);
 426         free(etmq);
 427 }
 428
 429 static void cs_etm__free_events(struct perf_session *session)
 430 {
 431         unsigned int i;
 432         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 433                                                    struct cs_etm_auxtrace,
 434                                                    auxtrace);
 435         struct auxtrace_queues *queues = &aux->queues;
 436
 437         for (i = 0; i < queues->nr_queues; i++) {
 438                 cs_etm__free_queue(queues->queue_array[i].priv);
 439                 queues->queue_array[i].priv = NULL;
 440         }
 441
 442         auxtrace_queues__free(queues);
 443 }
 444
 445 static void cs_etm__free(struct perf_session *session)
 446 {
 447         int i;
 448         struct int_node *inode, *tmp;
 449         struct cs_etm_auxtrace *aux = container_of(session->auxtrace,
 450                                                    struct cs_etm_auxtrace,
 451                                                    auxtrace);
 452         cs_etm__free_events(session);
 453         session->auxtrace = NULL;
 454
 455         /* First remove all traceID/metadata nodes for the RB tree */
 456         intlist__for_each_entry_safe(inode, tmp, traceid_list)
 457                 intlist__remove(traceid_list, inode);
 458         /* Then the RB tree itself */
 459         intlist__delete(traceid_list);
 460
 461         for (i = 0; i < aux->num_cpu; i++)
 462                 zfree(&aux->metadata[i]);
 463
 464         thread__zput(aux->unknown_thread);
 465         zfree(&aux->metadata);
 466         zfree(&aux);
 467 }
 468
 469 static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address)
 470 {
 471         struct machine *machine;
 472
 473         machine = etmq->etm->machine;
 474
 475         if (address >= etmq->etm->kernel_start) {
 476                 if (machine__is_host(machine))
 477                         return PERF_RECORD_MISC_KERNEL;
 478                 else
 479                         return PERF_RECORD_MISC_GUEST_KERNEL;
 480         } else {
 481                 if (machine__is_host(machine))
 482                         return PERF_RECORD_MISC_USER;
 483                 else if (perf_guest)
 484                         return PERF_RECORD_MISC_GUEST_USER;
 485                 else
 486                         return PERF_RECORD_MISC_HYPERVISOR;
 487         }
 488 }
 489
 490 static u32 cs_etm__mem_access(struct cs_etm_queue *etmq, u64 address,
 491                               size_t size, u8 *buffer)
 492 {
 493         u8  cpumode;
 494         u64 offset;
 495         int len;
 496         struct   thread *thread;
 497         struct   machine *machine;
 498         struct   addr_location al;
 499
 500         if (!etmq)
 501                 return 0;
 502
 503         machine = etmq->etm->machine;
 504         cpumode = cs_etm__cpu_mode(etmq, address);
 505
 506         thread = etmq->traceid_queues->thread;
 507         if (!thread) {
 508                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 509                         return 0;
 510                 thread = etmq->etm->unknown_thread;
 511         }
 512
 513         if (!thread__find_map(thread, cpumode, address, &al) || !al.map->dso)
 514                 return 0;
 515
 516         if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 517             dso__data_status_seen(al.map->dso, DSO_DATA_STATUS_SEEN_ITRACE))
 518                 return 0;
 519
 520         offset = al.map->map_ip(al.map, address);
 521
 522         map__load(al.map);
 523
 524         len = dso__data_read_offset(al.map->dso, machine, offset, buffer, size);
 525
 526         if (len <= 0)
 527                 return 0;
 528
 529         return len;
 530 }
 531
 532 static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm)
 533 {
 534         struct cs_etm_decoder_params d_params;
 535         struct cs_etm_trace_params  *t_params = NULL;
 536         struct cs_etm_queue *etmq;
 537
 538         etmq = zalloc(sizeof(*etmq));
 539         if (!etmq)
 540                 return NULL;
 541
 542         /* Use metadata to fill in trace parameters for trace decoder */
 543         t_params = zalloc(sizeof(*t_params) * etm->num_cpu);
 544
 545         if (!t_params)
 546                 goto out_free;
 547
 548         if (cs_etm__init_trace_params(t_params, etm))
 549                 goto out_free;
 550
 551         /* Set decoder parameters to decode trace packets */
 552         if (cs_etm__init_decoder_params(&d_params, etmq,
 553                                         CS_ETM_OPERATION_DECODE))
 554                 goto out_free;
 555
 556         etmq->decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);
 557
 558         if (!etmq->decoder)
 559                 goto out_free;
 560
 561         /*
 562          * Register a function to handle all memory accesses required by
 563          * the trace decoder library.
 564          */
 565         if (cs_etm_decoder__add_mem_access_cb(etmq->decoder,
 566                                               0x0L, ((u64) -1L),
 567                                               cs_etm__mem_access))
 568                 goto out_free_decoder;
 569
 570         zfree(&t_params);
 571         return etmq;
 572
 573 out_free_decoder:
 574         cs_etm_decoder__free(etmq->decoder);
 575 out_free:
 576         free(etmq);
 577
 578         return NULL;
 579 }
 580
 581 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,
 582                                struct auxtrace_queue *queue,
 583                                unsigned int queue_nr)
 584 {
 585         int ret = 0;
 586         struct cs_etm_queue *etmq = queue->priv;
 587
 588         if (list_empty(&queue->head) || etmq)
 589                 goto out;
 590
 591         etmq = cs_etm__alloc_queue(etm);
 592
 593         if (!etmq) {
 594                 ret = -ENOMEM;
 595                 goto out;
 596         }
 597
 598         queue->priv = etmq;
 599         etmq->etm = etm;
 600         etmq->queue_nr = queue_nr;
 601         etmq->tid = queue->tid;
 602         etmq->pid = -1;
 603         etmq->offset = 0;
 604
 605 out:
 606         return ret;
 607 }
 608
 609 static int cs_etm__setup_queues(struct cs_etm_auxtrace *etm)
 610 {
 611         unsigned int i;
 612         int ret;
 613
 614         if (!etm->kernel_start)
 615                 etm->kernel_start = machine__kernel_start(etm->machine);
 616
 617         for (i = 0; i < etm->queues.nr_queues; i++) {
 618                 ret = cs_etm__setup_queue(etm, &etm->queues.queue_array[i], i);
 619                 if (ret)
 620                         return ret;
 621         }
 622
 623         return 0;
 624 }
 625
 626 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm)
 627 {
 628         if (etm->queues.new_data) {
 629                 etm->queues.new_data = false;
 630                 return cs_etm__setup_queues(etm);
 631         }
 632
 633         return 0;
 634 }
 635
 636 static inline
 637 void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq,
 638                                  struct cs_etm_traceid_queue *tidq)
 639 {
 640         struct branch_stack *bs_src = tidq->last_branch_rb;
 641         struct branch_stack *bs_dst = tidq->last_branch;
 642         size_t nr = 0;
 643
 644         /*
 645          * Set the number of records before early exit: ->nr is used to
 646          * determine how many branches to copy from ->entries.
 647          */
 648         bs_dst->nr = bs_src->nr;
 649
 650         /*
 651          * Early exit when there is nothing to copy.
 652          */
 653         if (!bs_src->nr)
 654                 return;
 655
 656         /*
 657          * As bs_src->entries is a circular buffer, we need to copy from it in
 658          * two steps.  First, copy the branches from the most recently inserted
 659          * branch ->last_branch_pos until the end of bs_src->entries buffer.
 660          */
 661         nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;
 662         memcpy(&bs_dst->entries[0],
 663                &bs_src->entries[tidq->last_branch_pos],
 664                sizeof(struct branch_entry) * nr);
 665
 666         /*
 667          * If we wrapped around at least once, the branches from the beginning
 668          * of the bs_src->entries buffer and until the ->last_branch_pos element
 669          * are older valid branches: copy them over.  The total number of
 670          * branches copied over will be equal to the number of branches asked by
 671          * the user in last_branch_sz.
 672          */
 673         if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {
 674                 memcpy(&bs_dst->entries[nr],
 675                        &bs_src->entries[0],
 676                        sizeof(struct branch_entry) * tidq->last_branch_pos);
 677         }
 678 }
 679
 680 static inline
 681 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)
 682 {
 683         tidq->last_branch_pos = 0;
 684         tidq->last_branch_rb->nr = 0;
 685 }
 686
 687 static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq,
 688                                          u64 addr) {
 689         u8 instrBytes[2];
 690
 691         cs_etm__mem_access(etmq, addr, ARRAY_SIZE(instrBytes), instrBytes);
 692         /*
 693          * T32 instruction size is indicated by bits[15:11] of the first
 694          * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
 695          * denote a 32-bit instruction.
 696          */
 697         return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2;
 698 }
 699
 700 static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet)
 701 {
 702         /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
 703         if (packet->sample_type == CS_ETM_DISCONTINUITY)
 704                 return 0;
 705
 706         return packet->start_addr;
 707 }
 708
 709 static inline
 710 u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet)
 711 {
 712         /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
 713         if (packet->sample_type == CS_ETM_DISCONTINUITY)
 714                 return 0;
 715
 716         return packet->end_addr - packet->last_instr_size;
 717 }
 718
 719 static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq,
 720                                      const struct cs_etm_packet *packet,
 721                                      u64 offset)
 722 {
 723         if (packet->isa == CS_ETM_ISA_T32) {
 724                 u64 addr = packet->start_addr;
 725
 726                 while (offset > 0) {
 727                         addr += cs_etm__t32_instr_size(etmq, addr);
 728                         offset--;
 729                 }
 730                 return addr;
 731         }
 732
 733         /* Assume a 4 byte instruction size (A32/A64) */
 734         return packet->start_addr + offset * 4;
 735 }
 736
 737 static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq,
 738                                           struct cs_etm_traceid_queue *tidq)
 739 {
 740         struct branch_stack *bs = tidq->last_branch_rb;
 741         struct branch_entry *be;
 742
 743         /*
 744          * The branches are recorded in a circular buffer in reverse
 745          * chronological order: we start recording from the last element of the
 746          * buffer down.  After writing the first element of the stack, move the
 747          * insert position back to the end of the buffer.
 748          */
 749         if (!tidq->last_branch_pos)
 750                 tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;
 751
 752         tidq->last_branch_pos -= 1;
 753
 754         be       = &bs->entries[tidq->last_branch_pos];
 755         be->from = cs_etm__last_executed_instr(tidq->prev_packet);
 756         be->to   = cs_etm__first_executed_instr(tidq->packet);
 757         /* No support for mispredict */
 758         be->flags.mispred = 0;
 759         be->flags.predicted = 1;
 760
 761         /*
 762          * Increment bs->nr until reaching the number of last branches asked by
 763          * the user on the command line.
 764          */
 765         if (bs->nr < etmq->etm->synth_opts.last_branch_sz)
 766                 bs->nr += 1;
 767 }
 768
 769 static int cs_etm__inject_event(union perf_event *event,
 770                                struct perf_sample *sample, u64 type)
 771 {
 772         event->header.size = perf_event__sample_event_size(sample, type, 0);
 773         return perf_event__synthesize_sample(event, type, 0, sample);
 774 }
 775
 776
 777 static int
 778 cs_etm__get_trace(struct cs_etm_queue *etmq)
 779 {
 780         struct auxtrace_buffer *aux_buffer = etmq->buffer;
 781         struct auxtrace_buffer *old_buffer = aux_buffer;
 782         struct auxtrace_queue *queue;
 783
 784         queue = &etmq->etm->queues.queue_array[etmq->queue_nr];
 785
 786         aux_buffer = auxtrace_buffer__next(queue, aux_buffer);
 787
 788         /* If no more data, drop the previous auxtrace_buffer and return */
 789         if (!aux_buffer) {
 790                 if (old_buffer)
 791                         auxtrace_buffer__drop_data(old_buffer);
 792                 etmq->buf_len = 0;
 793                 return 0;
 794         }
 795
 796         etmq->buffer = aux_buffer;
 797
 798         /* If the aux_buffer doesn't have data associated, try to load it */
 799         if (!aux_buffer->data) {
 800                 /* get the file desc associated with the perf data file */
 801                 int fd = perf_data__fd(etmq->etm->session->data);
 802
 803                 aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd);
 804                 if (!aux_buffer->data)
 805                         return -ENOMEM;
 806         }
 807
 808         /* If valid, drop the previous buffer */
 809         if (old_buffer)
 810                 auxtrace_buffer__drop_data(old_buffer);
 811
 812         etmq->buf_used = 0;
 813         etmq->buf_len = aux_buffer->size;
 814         etmq->buf = aux_buffer->data;
 815
 816         return etmq->buf_len;
 817 }
 818
 819 static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm,
 820                                     struct auxtrace_queue *queue)
 821 {
 822         struct cs_etm_traceid_queue *tidq;
 823         struct cs_etm_queue *etmq = queue->priv;
 824
 825         tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
 826
 827         /* CPU-wide tracing isn't supported yet */
 828         if (queue->tid == -1)
 829                 return;
 830
 831         if ((!tidq->thread) && (etmq->tid != -1))
 832                 tidq->thread = machine__find_thread(etm->machine, -1,
 833                                                     etmq->tid);
 834
 835         if (tidq->thread)
 836                 etmq->pid = tidq->thread->pid_;
 837 }
 838
 839 static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq,
 840                                             struct cs_etm_traceid_queue *tidq,
 841                                             u64 addr, u64 period)
 842 {
 843         int ret = 0;
 844         struct cs_etm_auxtrace *etm = etmq->etm;
 845         union perf_event *event = tidq->event_buf;
 846         struct perf_sample sample = {.ip = 0,};
 847
 848         event->sample.header.type = PERF_RECORD_SAMPLE;
 849         event->sample.header.misc = cs_etm__cpu_mode(etmq, addr);
 850         event->sample.header.size = sizeof(struct perf_event_header);
 851
 852         sample.ip = addr;
 853         sample.pid = etmq->pid;
 854         sample.tid = etmq->tid;
 855         sample.id = etmq->etm->instructions_id;
 856         sample.stream_id = etmq->etm->instructions_id;
 857         sample.period = period;
 858         sample.cpu = tidq->packet->cpu;
 859         sample.flags = tidq->prev_packet->flags;
 860         sample.insn_len = 1;
 861         sample.cpumode = event->sample.header.misc;
 862
 863         if (etm->synth_opts.last_branch) {
 864                 cs_etm__copy_last_branch_rb(etmq, tidq);
 865                 sample.branch_stack = tidq->last_branch;
 866         }
 867
 868         if (etm->synth_opts.inject) {
 869                 ret = cs_etm__inject_event(event, &sample,
 870                                            etm->instructions_sample_type);
 871                 if (ret)
 872                         return ret;
 873         }
 874
 875         ret = perf_session__deliver_synth_event(etm->session, event, &sample);
 876
 877         if (ret)
 878                 pr_err(
 879                         "CS ETM Trace: failed to deliver instruction event, error %d\n",
 880                         ret);
 881
 882         if (etm->synth_opts.last_branch)
 883                 cs_etm__reset_last_branch_rb(tidq);
 884
 885         return ret;
 886 }
 887
 888 /*
 889  * The cs etm packet encodes an instruction range between a branch target
 890  * and the next taken branch. Generate sample accordingly.
 891  */
 892 static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq,
 893                                        struct cs_etm_traceid_queue *tidq)
 894 {
 895         int ret = 0;
 896         struct cs_etm_auxtrace *etm = etmq->etm;
 897         struct perf_sample sample = {.ip = 0,};
 898         union perf_event *event = tidq->event_buf;
 899         struct dummy_branch_stack {
 900                 u64                     nr;
 901                 struct branch_entry     entries;
 902         } dummy_bs;
 903         u64 ip;
 904
 905         ip = cs_etm__last_executed_instr(tidq->prev_packet);
 906
 907         event->sample.header.type = PERF_RECORD_SAMPLE;
 908         event->sample.header.misc = cs_etm__cpu_mode(etmq, ip);
 909         event->sample.header.size = sizeof(struct perf_event_header);
 910
 911         sample.ip = ip;
 912         sample.pid = etmq->pid;
 913         sample.tid = etmq->tid;
 914         sample.addr = cs_etm__first_executed_instr(tidq->packet);
 915         sample.id = etmq->etm->branches_id;
 916         sample.stream_id = etmq->etm->branches_id;
 917         sample.period = 1;
 918         sample.cpu = tidq->packet->cpu;
 919         sample.flags = tidq->prev_packet->flags;
 920         sample.cpumode = event->sample.header.misc;
 921
 922         /*
 923          * perf report cannot handle events without a branch stack
 924          */
 925         if (etm->synth_opts.last_branch) {
 926                 dummy_bs = (struct dummy_branch_stack){
 927                         .nr = 1,
 928                         .entries = {
 929                                 .from = sample.ip,
 930                                 .to = sample.addr,
 931                         },
 932                 };
 933                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
 934         }
 935
 936         if (etm->synth_opts.inject) {
 937                 ret = cs_etm__inject_event(event, &sample,
 938                                            etm->branches_sample_type);
 939                 if (ret)
 940                         return ret;
 941         }
 942
 943         ret = perf_session__deliver_synth_event(etm->session, event, &sample);
 944
 945         if (ret)
 946                 pr_err(
 947                 "CS ETM Trace: failed to deliver instruction event, error %d\n",
 948                 ret);
 949
 950         return ret;
 951 }
 952
 953 struct cs_etm_synth {
 954         struct perf_tool dummy_tool;
 955         struct perf_session *session;
 956 };
 957
 958 static int cs_etm__event_synth(struct perf_tool *tool,
 959                                union perf_event *event,
 960                                struct perf_sample *sample __maybe_unused,
 961                                struct machine *machine __maybe_unused)
 962 {
 963         struct cs_etm_synth *cs_etm_synth =
 964                       container_of(tool, struct cs_etm_synth, dummy_tool);
 965
 966         return perf_session__deliver_synth_event(cs_etm_synth->session,
 967                                                  event, NULL);
 968 }
 969
 970 static int cs_etm__synth_event(struct perf_session *session,
 971                                struct perf_event_attr *attr, u64 id)
 972 {
 973         struct cs_etm_synth cs_etm_synth;
 974
 975         memset(&cs_etm_synth, 0, sizeof(struct cs_etm_synth));
 976         cs_etm_synth.session = session;
 977
 978         return perf_event__synthesize_attr(&cs_etm_synth.dummy_tool, attr, 1,
 979                                            &id, cs_etm__event_synth);
 980 }
 981
 982 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,
 983                                 struct perf_session *session)
 984 {
 985         struct perf_evlist *evlist = session->evlist;
 986         struct perf_evsel *evsel;
 987         struct perf_event_attr attr;
 988         bool found = false;
 989         u64 id;
 990         int err;
 991
 992         evlist__for_each_entry(evlist, evsel) {
 993                 if (evsel->attr.type == etm->pmu_type) {
 994                         found = true;
 995                         break;
 996                 }
 997         }
 998
 999         if (!found) {
1000                 pr_debug("No selected events with CoreSight Trace data\n");
1001                 return 0;
1002         }
1003
1004         memset(&attr, 0, sizeof(struct perf_event_attr));
1005         attr.size = sizeof(struct perf_event_attr);
1006         attr.type = PERF_TYPE_HARDWARE;
1007         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
1008         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1009                             PERF_SAMPLE_PERIOD;
1010         if (etm->timeless_decoding)
1011                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1012         else
1013                 attr.sample_type |= PERF_SAMPLE_TIME;
1014
1015         attr.exclude_user = evsel->attr.exclude_user;
1016         attr.exclude_kernel = evsel->attr.exclude_kernel;
1017         attr.exclude_hv = evsel->attr.exclude_hv;
1018         attr.exclude_host = evsel->attr.exclude_host;
1019         attr.exclude_guest = evsel->attr.exclude_guest;
1020         attr.sample_id_all = evsel->attr.sample_id_all;
1021         attr.read_format = evsel->attr.read_format;
1022
1023         /* create new id val to be a fixed offset from evsel id */
1024         id = evsel->id[0] + 1000000000;
1025
1026         if (!id)
1027                 id = 1;
1028
1029         if (etm->synth_opts.branches) {
1030                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1031                 attr.sample_period = 1;
1032                 attr.sample_type |= PERF_SAMPLE_ADDR;
1033                 err = cs_etm__synth_event(session, &attr, id);
1034                 if (err)
1035                         return err;
1036                 etm->sample_branches = true;
1037                 etm->branches_sample_type = attr.sample_type;
1038                 etm->branches_id = id;
1039                 id += 1;
1040                 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
1041         }
1042
1043         if (etm->synth_opts.last_branch)
1044                 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1045
1046         if (etm->synth_opts.instructions) {
1047                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1048                 attr.sample_period = etm->synth_opts.period;
1049                 etm->instructions_sample_period = attr.sample_period;
1050                 err = cs_etm__synth_event(session, &attr, id);
1051                 if (err)
1052                         return err;
1053                 etm->sample_instructions = true;
1054                 etm->instructions_sample_type = attr.sample_type;
1055                 etm->instructions_id = id;
1056                 id += 1;
1057         }
1058
1059         return 0;
1060 }
1061
1062 static int cs_etm__sample(struct cs_etm_queue *etmq,
1063                           struct cs_etm_traceid_queue *tidq)
1064 {
1065         struct cs_etm_auxtrace *etm = etmq->etm;
1066         struct cs_etm_packet *tmp;
1067         int ret;
1068         u64 instrs_executed = tidq->packet->instr_count;
1069
1070         tidq->period_instructions += instrs_executed;
1071
1072         /*
1073          * Record a branch when the last instruction in
1074          * PREV_PACKET is a branch.
1075          */
1076         if (etm->synth_opts.last_branch &&
1077             tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1078             tidq->prev_packet->last_instr_taken_branch)
1079                 cs_etm__update_last_branch_rb(etmq, tidq);
1080
1081         if (etm->sample_instructions &&
1082             tidq->period_instructions >= etm->instructions_sample_period) {
1083                 /*
1084                  * Emit instruction sample periodically
1085                  * TODO: allow period to be defined in cycles and clock time
1086                  */
1087
1088                 /* Get number of instructions executed after the sample point */
1089                 u64 instrs_over = tidq->period_instructions -
1090                         etm->instructions_sample_period;
1091
1092                 /*
1093                  * Calculate the address of the sampled instruction (-1 as
1094                  * sample is reported as though instruction has just been
1095                  * executed, but PC has not advanced to next instruction)
1096                  */
1097                 u64 offset = (instrs_executed - instrs_over - 1);
1098                 u64 addr = cs_etm__instr_addr(etmq, tidq->packet, offset);
1099
1100                 ret = cs_etm__synth_instruction_sample(
1101                         etmq, tidq, addr, etm->instructions_sample_period);
1102                 if (ret)
1103                         return ret;
1104
1105                 /* Carry remaining instructions into next sample period */
1106                 tidq->period_instructions = instrs_over;
1107         }
1108
1109         if (etm->sample_branches) {
1110                 bool generate_sample = false;
1111
1112                 /* Generate sample for tracing on packet */
1113                 if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1114                         generate_sample = true;
1115
1116                 /* Generate sample for branch taken packet */
1117                 if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&
1118                     tidq->prev_packet->last_instr_taken_branch)
1119                         generate_sample = true;
1120
1121                 if (generate_sample) {
1122                         ret = cs_etm__synth_branch_sample(etmq, tidq);
1123                         if (ret)
1124                                 return ret;
1125                 }
1126         }
1127
1128         if (etm->sample_branches || etm->synth_opts.last_branch) {
1129                 /*
1130                  * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1131                  * the next incoming packet.
1132                  */
1133                 tmp = tidq->packet;
1134                 tidq->packet = tidq->prev_packet;
1135                 tidq->prev_packet = tmp;
1136         }
1137
1138         return 0;
1139 }
1140
1141 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)
1142 {
1143         /*
1144          * When the exception packet is inserted, whether the last instruction
1145          * in previous range packet is taken branch or not, we need to force
1146          * to set 'prev_packet->last_instr_taken_branch' to true.  This ensures
1147          * to generate branch sample for the instruction range before the
1148          * exception is trapped to kernel or before the exception returning.
1149          *
1150          * The exception packet includes the dummy address values, so don't
1151          * swap PACKET with PREV_PACKET.  This keeps PREV_PACKET to be useful
1152          * for generating instruction and branch samples.
1153          */
1154         if (tidq->prev_packet->sample_type == CS_ETM_RANGE)
1155                 tidq->prev_packet->last_instr_taken_branch = true;
1156
1157         return 0;
1158 }
1159
1160 static int cs_etm__flush(struct cs_etm_queue *etmq,
1161                          struct cs_etm_traceid_queue *tidq)
1162 {
1163         int err = 0;
1164         struct cs_etm_auxtrace *etm = etmq->etm;
1165         struct cs_etm_packet *tmp;
1166
1167         /* Handle start tracing packet */
1168         if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)
1169                 goto swap_packet;
1170
1171         if (etmq->etm->synth_opts.last_branch &&
1172             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1173                 /*
1174                  * Generate a last branch event for the branches left in the
1175                  * circular buffer at the end of the trace.
1176                  *
1177                  * Use the address of the end of the last reported execution
1178                  * range
1179                  */
1180                 u64 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1181
1182                 err = cs_etm__synth_instruction_sample(
1183                         etmq, tidq, addr,
1184                         tidq->period_instructions);
1185                 if (err)
1186                         return err;
1187
1188                 tidq->period_instructions = 0;
1189
1190         }
1191
1192         if (etm->sample_branches &&
1193             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1194                 err = cs_etm__synth_branch_sample(etmq, tidq);
1195                 if (err)
1196                         return err;
1197         }
1198
1199 swap_packet:
1200         if (etm->sample_branches || etm->synth_opts.last_branch) {
1201                 /*
1202                  * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
1203                  * the next incoming packet.
1204                  */
1205                 tmp = tidq->packet;
1206                 tidq->packet = tidq->prev_packet;
1207                 tidq->prev_packet = tmp;
1208         }
1209
1210         return err;
1211 }
1212
1213 static int cs_etm__end_block(struct cs_etm_queue *etmq,
1214                              struct cs_etm_traceid_queue *tidq)
1215 {
1216         int err;
1217
1218         /*
1219          * It has no new packet coming and 'etmq->packet' contains the stale
1220          * packet which was set at the previous time with packets swapping;
1221          * so skip to generate branch sample to avoid stale packet.
1222          *
1223          * For this case only flush branch stack and generate a last branch
1224          * event for the branches left in the circular buffer at the end of
1225          * the trace.
1226          */
1227         if (etmq->etm->synth_opts.last_branch &&
1228             tidq->prev_packet->sample_type == CS_ETM_RANGE) {
1229                 /*
1230                  * Use the address of the end of the last reported execution
1231                  * range.
1232                  */
1233                 u64 addr = cs_etm__last_executed_instr(tidq->prev_packet);
1234
1235                 err = cs_etm__synth_instruction_sample(
1236                         etmq, tidq, addr,
1237                         tidq->period_instructions);
1238                 if (err)
1239                         return err;
1240
1241                 tidq->period_instructions = 0;
1242         }
1243
1244         return 0;
1245 }
1246 /*
1247  * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
1248  *                         if need be.
1249  * Returns:     < 0     if error
1250  *              = 0     if no more auxtrace_buffer to read
1251  *              > 0     if the current buffer isn't empty yet
1252  */
1253 static int cs_etm__get_data_block(struct cs_etm_queue *etmq)
1254 {
1255         int ret;
1256
1257         if (!etmq->buf_len) {
1258                 ret = cs_etm__get_trace(etmq);
1259                 if (ret <= 0)
1260                         return ret;
1261                 /*
1262                  * We cannot assume consecutive blocks in the data file
1263                  * are contiguous, reset the decoder to force re-sync.
1264                  */
1265                 ret = cs_etm_decoder__reset(etmq->decoder);
1266                 if (ret)
1267                         return ret;
1268         }
1269
1270         return etmq->buf_len;
1271 }
1272
1273 static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq,
1274                                  struct cs_etm_packet *packet,
1275                                  u64 end_addr)
1276 {
1277         u16 instr16;
1278         u32 instr32;
1279         u64 addr;
1280
1281         switch (packet->isa) {
1282         case CS_ETM_ISA_T32:
1283                 /*
1284                  * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
1285                  *
1286                  *  b'15         b'8
1287                  * +-----------------+--------+
1288                  * | 1 1 0 1 1 1 1 1 |  imm8  |
1289                  * +-----------------+--------+
1290                  *
1291                  * According to the specifiction, it only defines SVC for T32
1292                  * with 16 bits instruction and has no definition for 32bits;
1293                  * so below only read 2 bytes as instruction size for T32.
1294                  */
1295                 addr = end_addr - 2;
1296                 cs_etm__mem_access(etmq, addr, sizeof(instr16), (u8 *)&instr16);
1297                 if ((instr16 & 0xFF00) == 0xDF00)
1298                         return true;
1299
1300                 break;
1301         case CS_ETM_ISA_A32:
1302                 /*
1303                  * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
1304                  *
1305                  *  b'31 b'28 b'27 b'24
1306                  * +---------+---------+-------------------------+
1307                  * |  !1111  | 1 1 1 1 |        imm24            |
1308                  * +---------+---------+-------------------------+
1309                  */
1310                 addr = end_addr - 4;
1311                 cs_etm__mem_access(etmq, addr, sizeof(instr32), (u8 *)&instr32);
1312                 if ((instr32 & 0x0F000000) == 0x0F000000 &&
1313                     (instr32 & 0xF0000000) != 0xF0000000)
1314                         return true;
1315
1316                 break;
1317         case CS_ETM_ISA_A64:
1318                 /*
1319                  * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
1320                  *
1321                  *  b'31               b'21           b'4     b'0
1322                  * +-----------------------+---------+-----------+
1323                  * | 1 1 0 1 0 1 0 0 0 0 0 |  imm16  | 0 0 0 0 1 |
1324                  * +-----------------------+---------+-----------+
1325                  */
1326                 addr = end_addr - 4;
1327                 cs_etm__mem_access(etmq, addr, sizeof(instr32), (u8 *)&instr32);
1328                 if ((instr32 & 0xFFE0001F) == 0xd4000001)
1329                         return true;
1330
1331                 break;
1332         case CS_ETM_ISA_UNKNOWN:
1333         default:
1334                 break;
1335         }
1336
1337         return false;
1338 }
1339
1340 static bool cs_etm__is_syscall(struct cs_etm_queue *etmq,
1341                                struct cs_etm_traceid_queue *tidq, u64 magic)
1342 {
1343         struct cs_etm_packet *packet = tidq->packet;
1344         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1345
1346         if (magic == __perf_cs_etmv3_magic)
1347                 if (packet->exception_number == CS_ETMV3_EXC_SVC)
1348                         return true;
1349
1350         /*
1351          * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
1352          * HVC cases; need to check if it's SVC instruction based on
1353          * packet address.
1354          */
1355         if (magic == __perf_cs_etmv4_magic) {
1356                 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1357                     cs_etm__is_svc_instr(etmq, prev_packet,
1358                                          prev_packet->end_addr))
1359                         return true;
1360         }
1361
1362         return false;
1363 }
1364
1365 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,
1366                                        u64 magic)
1367 {
1368         struct cs_etm_packet *packet = tidq->packet;
1369
1370         if (magic == __perf_cs_etmv3_magic)
1371                 if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT ||
1372                     packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT ||
1373                     packet->exception_number == CS_ETMV3_EXC_PE_RESET ||
1374                     packet->exception_number == CS_ETMV3_EXC_IRQ ||
1375                     packet->exception_number == CS_ETMV3_EXC_FIQ)
1376                         return true;
1377
1378         if (magic == __perf_cs_etmv4_magic)
1379                 if (packet->exception_number == CS_ETMV4_EXC_RESET ||
1380                     packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT ||
1381                     packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR ||
1382                     packet->exception_number == CS_ETMV4_EXC_INST_DEBUG ||
1383                     packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG ||
1384                     packet->exception_number == CS_ETMV4_EXC_IRQ ||
1385                     packet->exception_number == CS_ETMV4_EXC_FIQ)
1386                         return true;
1387
1388         return false;
1389 }
1390
1391 static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq,
1392                                       struct cs_etm_traceid_queue *tidq,
1393                                       u64 magic)
1394 {
1395         struct cs_etm_packet *packet = tidq->packet;
1396         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1397
1398         if (magic == __perf_cs_etmv3_magic)
1399                 if (packet->exception_number == CS_ETMV3_EXC_SMC ||
1400                     packet->exception_number == CS_ETMV3_EXC_HYP ||
1401                     packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE ||
1402                     packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR ||
1403                     packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT ||
1404                     packet->exception_number == CS_ETMV3_EXC_DATA_FAULT ||
1405                     packet->exception_number == CS_ETMV3_EXC_GENERIC)
1406                         return true;
1407
1408         if (magic == __perf_cs_etmv4_magic) {
1409                 if (packet->exception_number == CS_ETMV4_EXC_TRAP ||
1410                     packet->exception_number == CS_ETMV4_EXC_ALIGNMENT ||
1411                     packet->exception_number == CS_ETMV4_EXC_INST_FAULT ||
1412                     packet->exception_number == CS_ETMV4_EXC_DATA_FAULT)
1413                         return true;
1414
1415                 /*
1416                  * For CS_ETMV4_EXC_CALL, except SVC other instructions
1417                  * (SMC, HVC) are taken as sync exceptions.
1418                  */
1419                 if (packet->exception_number == CS_ETMV4_EXC_CALL &&
1420                     !cs_etm__is_svc_instr(etmq, prev_packet,
1421                                           prev_packet->end_addr))
1422                         return true;
1423
1424                 /*
1425                  * ETMv4 has 5 bits for exception number; if the numbers
1426                  * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
1427                  * they are implementation defined exceptions.
1428                  *
1429                  * For this case, simply take it as sync exception.
1430                  */
1431                 if (packet->exception_number > CS_ETMV4_EXC_FIQ &&
1432                     packet->exception_number <= CS_ETMV4_EXC_END)
1433                         return true;
1434         }
1435
1436         return false;
1437 }
1438
1439 static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq,
1440                                     struct cs_etm_traceid_queue *tidq)
1441 {
1442         struct cs_etm_packet *packet = tidq->packet;
1443         struct cs_etm_packet *prev_packet = tidq->prev_packet;
1444         u64 magic;
1445         int ret;
1446
1447         switch (packet->sample_type) {
1448         case CS_ETM_RANGE:
1449                 /*
1450                  * Immediate branch instruction without neither link nor
1451                  * return flag, it's normal branch instruction within
1452                  * the function.
1453                  */
1454                 if (packet->last_instr_type == OCSD_INSTR_BR &&
1455                     packet->last_instr_subtype == OCSD_S_INSTR_NONE) {
1456                         packet->flags = PERF_IP_FLAG_BRANCH;
1457
1458                         if (packet->last_instr_cond)
1459                                 packet->flags |= PERF_IP_FLAG_CONDITIONAL;
1460                 }
1461
1462                 /*
1463                  * Immediate branch instruction with link (e.g. BL), this is
1464                  * branch instruction for function call.
1465                  */
1466                 if (packet->last_instr_type == OCSD_INSTR_BR &&
1467                     packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
1468                         packet->flags = PERF_IP_FLAG_BRANCH |
1469                                         PERF_IP_FLAG_CALL;
1470
1471                 /*
1472                  * Indirect branch instruction with link (e.g. BLR), this is
1473                  * branch instruction for function call.
1474                  */
1475                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1476                     packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK)
1477                         packet->flags = PERF_IP_FLAG_BRANCH |
1478                                         PERF_IP_FLAG_CALL;
1479
1480                 /*
1481                  * Indirect branch instruction with subtype of
1482                  * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
1483                  * function return for A32/T32.
1484                  */
1485                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1486                     packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET)
1487                         packet->flags = PERF_IP_FLAG_BRANCH |
1488                                         PERF_IP_FLAG_RETURN;
1489
1490                 /*
1491                  * Indirect branch instruction without link (e.g. BR), usually
1492                  * this is used for function return, especially for functions
1493                  * within dynamic link lib.
1494                  */
1495                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1496                     packet->last_instr_subtype == OCSD_S_INSTR_NONE)
1497                         packet->flags = PERF_IP_FLAG_BRANCH |
1498                                         PERF_IP_FLAG_RETURN;
1499
1500                 /* Return instruction for function return. */
1501                 if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT &&
1502                     packet->last_instr_subtype == OCSD_S_INSTR_V8_RET)
1503                         packet->flags = PERF_IP_FLAG_BRANCH |
1504                                         PERF_IP_FLAG_RETURN;
1505
1506                 /*
1507                  * Decoder might insert a discontinuity in the middle of
1508                  * instruction packets, fixup prev_packet with flag
1509                  * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
1510                  */
1511                 if (prev_packet->sample_type == CS_ETM_DISCONTINUITY)
1512                         prev_packet->flags |= PERF_IP_FLAG_BRANCH |
1513                                               PERF_IP_FLAG_TRACE_BEGIN;
1514
1515                 /*
1516                  * If the previous packet is an exception return packet
1517                  * and the return address just follows SVC instuction,
1518                  * it needs to calibrate the previous packet sample flags
1519                  * as PERF_IP_FLAG_SYSCALLRET.
1520                  */
1521                 if (prev_packet->flags == (PERF_IP_FLAG_BRANCH |
1522                                            PERF_IP_FLAG_RETURN |
1523                                            PERF_IP_FLAG_INTERRUPT) &&
1524                     cs_etm__is_svc_instr(etmq, packet, packet->start_addr))
1525                         prev_packet->flags = PERF_IP_FLAG_BRANCH |
1526                                              PERF_IP_FLAG_RETURN |
1527                                              PERF_IP_FLAG_SYSCALLRET;
1528                 break;
1529         case CS_ETM_DISCONTINUITY:
1530                 /*
1531                  * The trace is discontinuous, if the previous packet is
1532                  * instruction packet, set flag PERF_IP_FLAG_TRACE_END
1533                  * for previous packet.
1534                  */
1535                 if (prev_packet->sample_type == CS_ETM_RANGE)
1536                         prev_packet->flags |= PERF_IP_FLAG_BRANCH |
1537                                               PERF_IP_FLAG_TRACE_END;
1538                 break;
1539         case CS_ETM_EXCEPTION:
1540                 ret = cs_etm__get_magic(packet->trace_chan_id, &magic);
1541                 if (ret)
1542                         return ret;
1543
1544                 /* The exception is for system call. */
1545                 if (cs_etm__is_syscall(etmq, tidq, magic))
1546                         packet->flags = PERF_IP_FLAG_BRANCH |
1547                                         PERF_IP_FLAG_CALL |
1548                                         PERF_IP_FLAG_SYSCALLRET;
1549                 /*
1550                  * The exceptions are triggered by external signals from bus,
1551                  * interrupt controller, debug module, PE reset or halt.
1552                  */
1553                 else if (cs_etm__is_async_exception(tidq, magic))
1554                         packet->flags = PERF_IP_FLAG_BRANCH |
1555                                         PERF_IP_FLAG_CALL |
1556                                         PERF_IP_FLAG_ASYNC |
1557                                         PERF_IP_FLAG_INTERRUPT;
1558                 /*
1559                  * Otherwise, exception is caused by trap, instruction &
1560                  * data fault, or alignment errors.
1561                  */
1562                 else if (cs_etm__is_sync_exception(etmq, tidq, magic))
1563                         packet->flags = PERF_IP_FLAG_BRANCH |
1564                                         PERF_IP_FLAG_CALL |
1565                                         PERF_IP_FLAG_INTERRUPT;
1566
1567                 /*
1568                  * When the exception packet is inserted, since exception
1569                  * packet is not used standalone for generating samples
1570                  * and it's affiliation to the previous instruction range
1571                  * packet; so set previous range packet flags to tell perf
1572                  * it is an exception taken branch.
1573                  */
1574                 if (prev_packet->sample_type == CS_ETM_RANGE)
1575                         prev_packet->flags = packet->flags;
1576                 break;
1577         case CS_ETM_EXCEPTION_RET:
1578                 /*
1579                  * When the exception return packet is inserted, since
1580                  * exception return packet is not used standalone for
1581                  * generating samples and it's affiliation to the previous
1582                  * instruction range packet; so set previous range packet
1583                  * flags to tell perf it is an exception return branch.
1584                  *
1585                  * The exception return can be for either system call or
1586                  * other exception types; unfortunately the packet doesn't
1587                  * contain exception type related info so we cannot decide
1588                  * the exception type purely based on exception return packet.
1589                  * If we record the exception number from exception packet and
1590                  * reuse it for excpetion return packet, this is not reliable
1591                  * due the trace can be discontinuity or the interrupt can
1592                  * be nested, thus the recorded exception number cannot be
1593                  * used for exception return packet for these two cases.
1594                  *
1595                  * For exception return packet, we only need to distinguish the
1596                  * packet is for system call or for other types.  Thus the
1597                  * decision can be deferred when receive the next packet which
1598                  * contains the return address, based on the return address we
1599                  * can read out the previous instruction and check if it's a
1600                  * system call instruction and then calibrate the sample flag
1601                  * as needed.
1602                  */
1603                 if (prev_packet->sample_type == CS_ETM_RANGE)
1604                         prev_packet->flags = PERF_IP_FLAG_BRANCH |
1605                                              PERF_IP_FLAG_RETURN |
1606                                              PERF_IP_FLAG_INTERRUPT;
1607                 break;
1608         case CS_ETM_EMPTY:
1609         default:
1610                 break;
1611         }
1612
1613         return 0;
1614 }
1615
1616 static int cs_etm__decode_data_block(struct cs_etm_queue *etmq)
1617 {
1618         int ret = 0;
1619         size_t processed = 0;
1620
1621         /*
1622          * Packets are decoded and added to the decoder's packet queue
1623          * until the decoder packet processing callback has requested that
1624          * processing stops or there is nothing left in the buffer.  Normal
1625          * operations that stop processing are a timestamp packet or a full
1626          * decoder buffer queue.
1627          */
1628         ret = cs_etm_decoder__process_data_block(etmq->decoder,
1629                                                  etmq->offset,
1630                                                  &etmq->buf[etmq->buf_used],
1631                                                  etmq->buf_len,
1632                                                  &processed);
1633         if (ret)
1634                 goto out;
1635
1636         etmq->offset += processed;
1637         etmq->buf_used += processed;
1638         etmq->buf_len -= processed;
1639
1640 out:
1641         return ret;
1642 }
1643
1644 static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq,
1645                                          struct cs_etm_traceid_queue *tidq)
1646 {
1647         int ret;
1648         struct cs_etm_packet_queue *packet_queue;
1649
1650         packet_queue = &tidq->packet_queue;
1651
1652         /* Process each packet in this chunk */
1653         while (1) {
1654                 ret = cs_etm_decoder__get_packet(packet_queue,
1655                                                  tidq->packet);
1656                 if (ret <= 0)
1657                         /*
1658                          * Stop processing this chunk on
1659                          * end of data or error
1660                          */
1661                         break;
1662
1663                 /*
1664                  * Since packet addresses are swapped in packet
1665                  * handling within below switch() statements,
1666                  * thus setting sample flags must be called
1667                  * prior to switch() statement to use address
1668                  * information before packets swapping.
1669                  */
1670                 ret = cs_etm__set_sample_flags(etmq, tidq);
1671                 if (ret < 0)
1672                         break;
1673
1674                 switch (tidq->packet->sample_type) {
1675                 case CS_ETM_RANGE:
1676                         /*
1677                          * If the packet contains an instruction
1678                          * range, generate instruction sequence
1679                          * events.
1680                          */
1681                         cs_etm__sample(etmq, tidq);
1682                         break;
1683                 case CS_ETM_EXCEPTION:
1684                 case CS_ETM_EXCEPTION_RET:
1685                         /*
1686                          * If the exception packet is coming,
1687                          * make sure the previous instruction
1688                          * range packet to be handled properly.
1689                          */
1690                         cs_etm__exception(tidq);
1691                         break;
1692                 case CS_ETM_DISCONTINUITY:
1693                         /*
1694                          * Discontinuity in trace, flush
1695                          * previous branch stack
1696                          */
1697                         cs_etm__flush(etmq, tidq);
1698                         break;
1699                 case CS_ETM_EMPTY:
1700                         /*
1701                          * Should not receive empty packet,
1702                          * report error.
1703                          */
1704                         pr_err("CS ETM Trace: empty packet\n");
1705                         return -EINVAL;
1706                 default:
1707                         break;
1708                 }
1709         }
1710
1711         return ret;
1712 }
1713
1714 static int cs_etm__run_decoder(struct cs_etm_queue *etmq)
1715 {
1716         int err = 0;
1717         struct cs_etm_traceid_queue *tidq;
1718
1719         tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);
1720         if (!tidq)
1721                 return -EINVAL;
1722
1723         /* Go through each buffer in the queue and decode them one by one */
1724         while (1) {
1725                 err = cs_etm__get_data_block(etmq);
1726                 if (err <= 0)
1727                         return err;
1728
1729                 /* Run trace decoder until buffer consumed or end of trace */
1730                 do {
1731                         err = cs_etm__decode_data_block(etmq);
1732                         if (err)
1733                                 return err;
1734
1735                         /*
1736                          * Process each packet in this chunk, nothing to do if
1737                          * an error occurs other than hoping the next one will
1738                          * be better.
1739                          */
1740                         err = cs_etm__process_traceid_queue(etmq, tidq);
1741
1742                 } while (etmq->buf_len);
1743
1744                 if (err == 0)
1745                         /* Flush any remaining branch stack entries */
1746                         err = cs_etm__end_block(etmq, tidq);
1747         }
1748
1749         return err;
1750 }
1751
1752 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
1753                                            pid_t tid)
1754 {
1755         unsigned int i;
1756         struct auxtrace_queues *queues = &etm->queues;
1757
1758         for (i = 0; i < queues->nr_queues; i++) {
1759                 struct auxtrace_queue *queue = &etm->queues.queue_array[i];
1760                 struct cs_etm_queue *etmq = queue->priv;
1761
1762                 if (etmq && ((tid == -1) || (etmq->tid == tid))) {
1763                         cs_etm__set_pid_tid_cpu(etm, queue);
1764                         cs_etm__run_decoder(etmq);
1765                 }
1766         }
1767
1768         return 0;
1769 }
1770
1771 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,
1772                                         union perf_event *event)
1773 {
1774         struct thread *th;
1775
1776         if (etm->timeless_decoding)
1777                 return 0;
1778
1779         /*
1780          * Add the tid/pid to the log so that we can get a match when
1781          * we get a contextID from the decoder.
1782          */
1783         th = machine__findnew_thread(etm->machine,
1784                                      event->itrace_start.pid,
1785                                      event->itrace_start.tid);
1786         if (!th)
1787                 return -ENOMEM;
1788
1789         thread__put(th);
1790
1791         return 0;
1792 }
1793
1794 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,
1795                                            union perf_event *event)
1796 {
1797         struct thread *th;
1798         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1799
1800         /*
1801          * Context switch in per-thread mode are irrelevant since perf
1802          * will start/stop tracing as the process is scheduled.
1803          */
1804         if (etm->timeless_decoding)
1805                 return 0;
1806
1807         /*
1808          * SWITCH_IN events carry the next process to be switched out while
1809          * SWITCH_OUT events carry the process to be switched in.  As such
1810          * we don't care about IN events.
1811          */
1812         if (!out)
1813                 return 0;
1814
1815         /*
1816          * Add the tid/pid to the log so that we can get a match when
1817          * we get a contextID from the decoder.
1818          */
1819         th = machine__findnew_thread(etm->machine,
1820                                      event->context_switch.next_prev_pid,
1821                                      event->context_switch.next_prev_tid);
1822         if (!th)
1823                 return -ENOMEM;
1824
1825         thread__put(th);
1826
1827         return 0;
1828 }
1829
1830 static int cs_etm__process_event(struct perf_session *session,
1831                                  union perf_event *event,
1832                                  struct perf_sample *sample,
1833                                  struct perf_tool *tool)
1834 {
1835         int err = 0;
1836         u64 timestamp;
1837         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
1838                                                    struct cs_etm_auxtrace,
1839                                                    auxtrace);
1840
1841         if (dump_trace)
1842                 return 0;
1843
1844         if (!tool->ordered_events) {
1845                 pr_err("CoreSight ETM Trace requires ordered events\n");
1846                 return -EINVAL;
1847         }
1848
1849         if (!etm->timeless_decoding)
1850                 return -EINVAL;
1851
1852         if (sample->time && (sample->time != (u64) -1))
1853                 timestamp = sample->time;
1854         else
1855                 timestamp = 0;
1856
1857         if (timestamp || etm->timeless_decoding) {
1858                 err = cs_etm__update_queues(etm);
1859                 if (err)
1860                         return err;
1861         }
1862
1863         if (event->header.type == PERF_RECORD_EXIT)
1864                 return cs_etm__process_timeless_queues(etm,
1865                                                        event->fork.tid);
1866
1867         if (event->header.type == PERF_RECORD_ITRACE_START)
1868                 return cs_etm__process_itrace_start(etm, event);
1869         else if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
1870                 return cs_etm__process_switch_cpu_wide(etm, event);
1871
1872         return 0;
1873 }
1874
1875 static int cs_etm__process_auxtrace_event(struct perf_session *session,
1876                                           union perf_event *event,
1877                                           struct perf_tool *tool __maybe_unused)
1878 {
1879         struct cs_etm_auxtrace *etm = container_of(session->auxtrace,
1880                                                    struct cs_etm_auxtrace,
1881                                                    auxtrace);
1882         if (!etm->data_queued) {
1883                 struct auxtrace_buffer *buffer;
1884                 off_t  data_offset;
1885                 int fd = perf_data__fd(session->data);
1886                 bool is_pipe = perf_data__is_pipe(session->data);
1887                 int err;
1888
1889                 if (is_pipe)
1890                         data_offset = 0;
1891                 else {
1892                         data_offset = lseek(fd, 0, SEEK_CUR);
1893                         if (data_offset == -1)
1894                                 return -errno;
1895                 }
1896
1897                 err = auxtrace_queues__add_event(&etm->queues, session,
1898                                                  event, data_offset, &buffer);
1899                 if (err)
1900                         return err;
1901
1902                 if (dump_trace)
1903                         if (auxtrace_buffer__get_data(buffer, fd)) {
1904                                 cs_etm__dump_event(etm, buffer);
1905                                 auxtrace_buffer__put_data(buffer);
1906                         }
1907         }
1908
1909         return 0;
1910 }
1911
1912 static bool cs_etm__is_timeless_decoding(struct cs_etm_auxtrace *etm)
1913 {
1914         struct perf_evsel *evsel;
1915         struct perf_evlist *evlist = etm->session->evlist;
1916         bool timeless_decoding = true;
1917
1918         /*
1919          * Circle through the list of event and complain if we find one
1920          * with the time bit set.
1921          */
1922         evlist__for_each_entry(evlist, evsel) {
1923                 if ((evsel->attr.sample_type & PERF_SAMPLE_TIME))
1924                         timeless_decoding = false;
1925         }
1926
1927         return timeless_decoding;
1928 }
1929
1930 static const char * const cs_etm_global_header_fmts[] = {
1931         [CS_HEADER_VERSION_0]   = "     Header version                 %llx\n",
1932         [CS_PMU_TYPE_CPUS]      = "     PMU type/num cpus              %llx\n",
1933         [CS_ETM_SNAPSHOT]       = "     Snapshot                       %llx\n",
1934 };
1935
1936 static const char * const cs_etm_priv_fmts[] = {
1937         [CS_ETM_MAGIC]          = "     Magic number                   %llx\n",
1938         [CS_ETM_CPU]            = "     CPU                            %lld\n",
1939         [CS_ETM_ETMCR]          = "     ETMCR                          %llx\n",
1940         [CS_ETM_ETMTRACEIDR]    = "     ETMTRACEIDR                    %llx\n",
1941         [CS_ETM_ETMCCER]        = "     ETMCCER                        %llx\n",
1942         [CS_ETM_ETMIDR]         = "     ETMIDR                         %llx\n",
1943 };
1944
1945 static const char * const cs_etmv4_priv_fmts[] = {
1946         [CS_ETM_MAGIC]          = "     Magic number                   %llx\n",
1947         [CS_ETM_CPU]            = "     CPU                            %lld\n",
1948         [CS_ETMV4_TRCCONFIGR]   = "     TRCCONFIGR                     %llx\n",
1949         [CS_ETMV4_TRCTRACEIDR]  = "     TRCTRACEIDR                    %llx\n",
1950         [CS_ETMV4_TRCIDR0]      = "     TRCIDR0                        %llx\n",
1951         [CS_ETMV4_TRCIDR1]      = "     TRCIDR1                        %llx\n",
1952         [CS_ETMV4_TRCIDR2]      = "     TRCIDR2                        %llx\n",
1953         [CS_ETMV4_TRCIDR8]      = "     TRCIDR8                        %llx\n",
1954         [CS_ETMV4_TRCAUTHSTATUS] = "    TRCAUTHSTATUS                  %llx\n",
1955 };
1956
1957 static void cs_etm__print_auxtrace_info(u64 *val, int num)
1958 {
1959         int i, j, cpu = 0;
1960
1961         for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++)
1962                 fprintf(stdout, cs_etm_global_header_fmts[i], val[i]);
1963
1964         for (i = CS_HEADER_VERSION_0_MAX; cpu < num; cpu++) {
1965                 if (val[i] == __perf_cs_etmv3_magic)
1966                         for (j = 0; j < CS_ETM_PRIV_MAX; j++, i++)
1967                                 fprintf(stdout, cs_etm_priv_fmts[j], val[i]);
1968                 else if (val[i] == __perf_cs_etmv4_magic)
1969                         for (j = 0; j < CS_ETMV4_PRIV_MAX; j++, i++)
1970                                 fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]);
1971                 else
1972                         /* failure.. return */
1973                         return;
1974         }
1975 }
1976
1977 int cs_etm__process_auxtrace_info(union perf_event *event,
1978                                   struct perf_session *session)
1979 {
1980         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
1981         struct cs_etm_auxtrace *etm = NULL;
1982         struct int_node *inode;
1983         unsigned int pmu_type;
1984         int event_header_size = sizeof(struct perf_event_header);
1985         int info_header_size;
1986         int total_size = auxtrace_info->header.size;
1987         int priv_size = 0;
1988         int num_cpu;
1989         int err = 0, idx = -1;
1990         int i, j, k;
1991         u64 *ptr, *hdr = NULL;
1992         u64 **metadata = NULL;
1993
1994         /*
1995          * sizeof(auxtrace_info_event::type) +
1996          * sizeof(auxtrace_info_event::reserved) == 8
1997          */
1998         info_header_size = 8;
1999
2000         if (total_size < (event_header_size + info_header_size))
2001                 return -EINVAL;
2002
2003         priv_size = total_size - event_header_size - info_header_size;
2004
2005         /* First the global part */
2006         ptr = (u64 *) auxtrace_info->priv;
2007
2008         /* Look for version '0' of the header */
2009         if (ptr[0] != 0)
2010                 return -EINVAL;
2011
2012         hdr = zalloc(sizeof(*hdr) * CS_HEADER_VERSION_0_MAX);
2013         if (!hdr)
2014                 return -ENOMEM;
2015
2016         /* Extract header information - see cs-etm.h for format */
2017         for (i = 0; i < CS_HEADER_VERSION_0_MAX; i++)
2018                 hdr[i] = ptr[i];
2019         num_cpu = hdr[CS_PMU_TYPE_CPUS] & 0xffffffff;
2020         pmu_type = (unsigned int) ((hdr[CS_PMU_TYPE_CPUS] >> 32) &
2021                                     0xffffffff);
2022
2023         /*
2024          * Create an RB tree for traceID-metadata tuple.  Since the conversion
2025          * has to be made for each packet that gets decoded, optimizing access
2026          * in anything other than a sequential array is worth doing.
2027          */
2028         traceid_list = intlist__new(NULL);
2029         if (!traceid_list) {
2030                 err = -ENOMEM;
2031                 goto err_free_hdr;
2032         }
2033
2034         metadata = zalloc(sizeof(*metadata) * num_cpu);
2035         if (!metadata) {
2036                 err = -ENOMEM;
2037                 goto err_free_traceid_list;
2038         }
2039
2040         /*
2041          * The metadata is stored in the auxtrace_info section and encodes
2042          * the configuration of the ARM embedded trace macrocell which is
2043          * required by the trace decoder to properly decode the trace due
2044          * to its highly compressed nature.
2045          */
2046         for (j = 0; j < num_cpu; j++) {
2047                 if (ptr[i] == __perf_cs_etmv3_magic) {
2048                         metadata[j] = zalloc(sizeof(*metadata[j]) *
2049                                              CS_ETM_PRIV_MAX);
2050                         if (!metadata[j]) {
2051                                 err = -ENOMEM;
2052                                 goto err_free_metadata;
2053                         }
2054                         for (k = 0; k < CS_ETM_PRIV_MAX; k++)
2055                                 metadata[j][k] = ptr[i + k];
2056
2057                         /* The traceID is our handle */
2058                         idx = metadata[j][CS_ETM_ETMTRACEIDR];
2059                         i += CS_ETM_PRIV_MAX;
2060                 } else if (ptr[i] == __perf_cs_etmv4_magic) {
2061                         metadata[j] = zalloc(sizeof(*metadata[j]) *
2062                                              CS_ETMV4_PRIV_MAX);
2063                         if (!metadata[j]) {
2064                                 err = -ENOMEM;
2065                                 goto err_free_metadata;
2066                         }
2067                         for (k = 0; k < CS_ETMV4_PRIV_MAX; k++)
2068                                 metadata[j][k] = ptr[i + k];
2069
2070                         /* The traceID is our handle */
2071                         idx = metadata[j][CS_ETMV4_TRCTRACEIDR];
2072                         i += CS_ETMV4_PRIV_MAX;
2073                 }
2074
2075                 /* Get an RB node for this CPU */
2076                 inode = intlist__findnew(traceid_list, idx);
2077
2078                 /* Something went wrong, no need to continue */
2079                 if (!inode) {
2080                         err = PTR_ERR(inode);
2081                         goto err_free_metadata;
2082                 }
2083
2084                 /*
2085                  * The node for that CPU should not be taken.
2086                  * Back out if that's the case.
2087                  */
2088                 if (inode->priv) {
2089                         err = -EINVAL;
2090                         goto err_free_metadata;
2091                 }
2092                 /* All good, associate the traceID with the metadata pointer */
2093                 inode->priv = metadata[j];
2094         }
2095
2096         /*
2097          * Each of CS_HEADER_VERSION_0_MAX, CS_ETM_PRIV_MAX and
2098          * CS_ETMV4_PRIV_MAX mark how many double words are in the
2099          * global metadata, and each cpu's metadata respectively.
2100          * The following tests if the correct number of double words was
2101          * present in the auxtrace info section.
2102          */
2103         if (i * 8 != priv_size) {
2104                 err = -EINVAL;
2105                 goto err_free_metadata;
2106         }
2107
2108         etm = zalloc(sizeof(*etm));
2109
2110         if (!etm) {
2111                 err = -ENOMEM;
2112                 goto err_free_metadata;
2113         }
2114
2115         err = auxtrace_queues__init(&etm->queues);
2116         if (err)
2117                 goto err_free_etm;
2118
2119         etm->session = session;
2120         etm->machine = &session->machines.host;
2121
2122         etm->num_cpu = num_cpu;
2123         etm->pmu_type = pmu_type;
2124         etm->snapshot_mode = (hdr[CS_ETM_SNAPSHOT] != 0);
2125         etm->metadata = metadata;
2126         etm->auxtrace_type = auxtrace_info->type;
2127         etm->timeless_decoding = cs_etm__is_timeless_decoding(etm);
2128
2129         etm->auxtrace.process_event = cs_etm__process_event;
2130         etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;
2131         etm->auxtrace.flush_events = cs_etm__flush_events;
2132         etm->auxtrace.free_events = cs_etm__free_events;
2133         etm->auxtrace.free = cs_etm__free;
2134         session->auxtrace = &etm->auxtrace;
2135
2136         etm->unknown_thread = thread__new(999999999, 999999999);
2137         if (!etm->unknown_thread)
2138                 goto err_free_queues;
2139
2140         /*
2141          * Initialize list node so that at thread__zput() we can avoid
2142          * segmentation fault at list_del_init().
2143          */
2144         INIT_LIST_HEAD(&etm->unknown_thread->node);
2145
2146         err = thread__set_comm(etm->unknown_thread, "unknown", 0);
2147         if (err)
2148                 goto err_delete_thread;
2149
2150         if (thread__init_map_groups(etm->unknown_thread, etm->machine))
2151                 goto err_delete_thread;
2152
2153         if (dump_trace) {
2154                 cs_etm__print_auxtrace_info(auxtrace_info->priv, num_cpu);
2155                 return 0;
2156         }
2157
2158         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2159                 etm->synth_opts = *session->itrace_synth_opts;
2160         } else {
2161                 itrace_synth_opts__set_default(&etm->synth_opts,
2162                                 session->itrace_synth_opts->default_no_sample);
2163                 etm->synth_opts.callchain = false;
2164         }
2165
2166         err = cs_etm__synth_events(etm, session);
2167         if (err)
2168                 goto err_delete_thread;
2169
2170         err = auxtrace_queues__process_index(&etm->queues, session);
2171         if (err)
2172                 goto err_delete_thread;
2173
2174         etm->data_queued = etm->queues.populated;
2175
2176         return 0;
2177
2178 err_delete_thread:
2179         thread__zput(etm->unknown_thread);
2180 err_free_queues:
2181         auxtrace_queues__free(&etm->queues);
2182         session->auxtrace = NULL;
2183 err_free_etm:
2184         zfree(&etm);
2185 err_free_metadata:
2186         /* No need to check @metadata[j], free(NULL) is supported */
2187         for (j = 0; j < num_cpu; j++)
2188                 free(metadata[j]);
2189         zfree(&metadata);
2190 err_free_traceid_list:
2191         intlist__delete(traceid_list);
2192 err_free_hdr:
2193         zfree(&hdr);
2194
2195         return -EINVAL;
2196 }