drivers/hwtracing/coresight/coresight-tmc-etf.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright(C) 2016 Linaro Limited. All rights reserved.
   4  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   5  */
   6
   7 #include <linux/circ_buf.h>
   8 #include <linux/coresight.h>
   9 #include <linux/perf_event.h>
  10 #include <linux/slab.h>
  11 #include "coresight-priv.h"
  12 #include "coresight-tmc.h"
  13 #include "coresight-etm-perf.h"
  14
  15 static int tmc_set_etf_buffer(struct coresight_device *csdev,
  16                               struct perf_output_handle *handle);
  17
  18 static void __tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
  19 {
  20         CS_UNLOCK(drvdata->base);
  21
  22         /* Wait for TMCSReady bit to be set */
  23         tmc_wait_for_tmcready(drvdata);
  24
  25         writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
  26         writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
  27                        TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
  28                        TMC_FFCR_TRIGON_TRIGIN,
  29                        drvdata->base + TMC_FFCR);
  30
  31         writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
  32         tmc_enable_hw(drvdata);
  33
  34         CS_LOCK(drvdata->base);
  35 }
  36
  37 static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
  38 {
  39         int rc = coresight_claim_device(drvdata->base);
  40
  41         if (rc)
  42                 return rc;
  43
  44         __tmc_etb_enable_hw(drvdata);
  45         return 0;
  46 }
  47
  48 static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
  49 {
  50         char *bufp;
  51         u32 read_data, lost;
  52
  53         /* Check if the buffer wrapped around. */
  54         lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL;
  55         bufp = drvdata->buf;
  56         drvdata->len = 0;
  57         while (1) {
  58                 read_data = readl_relaxed(drvdata->base + TMC_RRD);
  59                 if (read_data == 0xFFFFFFFF)
  60                         break;
  61                 memcpy(bufp, &read_data, 4);
  62                 bufp += 4;
  63                 drvdata->len += 4;
  64         }
  65
  66         if (lost)
  67                 coresight_insert_barrier_packet(drvdata->buf);
  68         return;
  69 }
  70
  71 static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
  72 {
  73         CS_UNLOCK(drvdata->base);
  74
  75         tmc_flush_and_stop(drvdata);
  76         /*
  77          * When operating in sysFS mode the content of the buffer needs to be
  78          * read before the TMC is disabled.
  79          */
  80         if (drvdata->mode == CS_MODE_SYSFS)
  81                 tmc_etb_dump_hw(drvdata);
  82         tmc_disable_hw(drvdata);
  83
  84         CS_LOCK(drvdata->base);
  85 }
  86
  87 static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
  88 {
  89         __tmc_etb_disable_hw(drvdata);
  90         coresight_disclaim_device(drvdata->base);
  91 }
  92
  93 static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
  94 {
  95         CS_UNLOCK(drvdata->base);
  96
  97         /* Wait for TMCSReady bit to be set */
  98         tmc_wait_for_tmcready(drvdata);
  99
 100         writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
 101         writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
 102                        drvdata->base + TMC_FFCR);
 103         writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
 104         tmc_enable_hw(drvdata);
 105
 106         CS_LOCK(drvdata->base);
 107 }
 108
 109 static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
 110 {
 111         int rc = coresight_claim_device(drvdata->base);
 112
 113         if (rc)
 114                 return rc;
 115
 116         __tmc_etf_enable_hw(drvdata);
 117         return 0;
 118 }
 119
 120 static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
 121 {
 122         CS_UNLOCK(drvdata->base);
 123
 124         tmc_flush_and_stop(drvdata);
 125         tmc_disable_hw(drvdata);
 126         coresight_disclaim_device_unlocked(drvdata->base);
 127         CS_LOCK(drvdata->base);
 128 }
 129
 130 /*
 131  * Return the available trace data in the buffer from @pos, with
 132  * a maximum limit of @len, updating the @bufpp on where to
 133  * find it.
 134  */
 135 ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata,
 136                                 loff_t pos, size_t len, char **bufpp)
 137 {
 138         ssize_t actual = len;
 139
 140         /* Adjust the len to available size @pos */
 141         if (pos + actual > drvdata->len)
 142                 actual = drvdata->len - pos;
 143         if (actual > 0)
 144                 *bufpp = drvdata->buf + pos;
 145         return actual;
 146 }
 147
 148 static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
 149 {
 150         int ret = 0;
 151         bool used = false;
 152         char *buf = NULL;
 153         unsigned long flags;
 154         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 155
 156         /*
 157          * If we don't have a buffer release the lock and allocate memory.
 158          * Otherwise keep the lock and move along.
 159          */
 160         spin_lock_irqsave(&drvdata->spinlock, flags);
 161         if (!drvdata->buf) {
 162                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 163
 164                 /* Allocating the memory here while outside of the spinlock */
 165                 buf = kzalloc(drvdata->size, GFP_KERNEL);
 166                 if (!buf)
 167                         return -ENOMEM;
 168
 169                 /* Let's try again */
 170                 spin_lock_irqsave(&drvdata->spinlock, flags);
 171         }
 172
 173         if (drvdata->reading) {
 174                 ret = -EBUSY;
 175                 goto out;
 176         }
 177
 178         /*
 179          * In sysFS mode we can have multiple writers per sink.  Since this
 180          * sink is already enabled no memory is needed and the HW need not be
 181          * touched.
 182          */
 183         if (drvdata->mode == CS_MODE_SYSFS)
 184                 goto out;
 185
 186         /*
 187          * If drvdata::buf isn't NULL, memory was allocated for a previous
 188          * trace run but wasn't read.  If so simply zero-out the memory.
 189          * Otherwise use the memory allocated above.
 190          *
 191          * The memory is freed when users read the buffer using the
 192          * /dev/xyz.{etf|etb} interface.  See tmc_read_unprepare_etf() for
 193          * details.
 194          */
 195         if (drvdata->buf) {
 196                 memset(drvdata->buf, 0, drvdata->size);
 197         } else {
 198                 used = true;
 199                 drvdata->buf = buf;
 200         }
 201
 202         ret = tmc_etb_enable_hw(drvdata);
 203         if (!ret)
 204                 drvdata->mode = CS_MODE_SYSFS;
 205         else
 206                 /* Free up the buffer if we failed to enable */
 207                 used = false;
 208 out:
 209         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 210
 211         /* Free memory outside the spinlock if need be */
 212         if (!used)
 213                 kfree(buf);
 214
 215         return ret;
 216 }
 217
 218 static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data)
 219 {
 220         int ret = 0;
 221         unsigned long flags;
 222         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 223         struct perf_output_handle *handle = data;
 224
 225         spin_lock_irqsave(&drvdata->spinlock, flags);
 226         do {
 227                 ret = -EINVAL;
 228                 if (drvdata->reading)
 229                         break;
 230                 /*
 231                  * In Perf mode there can be only one writer per sink.  There
 232                  * is also no need to continue if the ETB/ETF is already
 233                  * operated from sysFS.
 234                  */
 235                 if (drvdata->mode != CS_MODE_DISABLED)
 236                         break;
 237
 238                 ret = tmc_set_etf_buffer(csdev, handle);
 239                 if (ret)
 240                         break;
 241                 ret  = tmc_etb_enable_hw(drvdata);
 242                 if (!ret)
 243                         drvdata->mode = CS_MODE_PERF;
 244         } while (0);
 245         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 246
 247         return ret;
 248 }
 249
 250 static int tmc_enable_etf_sink(struct coresight_device *csdev,
 251                                u32 mode, void *data)
 252 {
 253         int ret;
 254         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 255
 256         switch (mode) {
 257         case CS_MODE_SYSFS:
 258                 ret = tmc_enable_etf_sink_sysfs(csdev);
 259                 break;
 260         case CS_MODE_PERF:
 261                 ret = tmc_enable_etf_sink_perf(csdev, data);
 262                 break;
 263         /* We shouldn't be here */
 264         default:
 265                 ret = -EINVAL;
 266                 break;
 267         }
 268
 269         if (ret)
 270                 return ret;
 271
 272         dev_dbg(drvdata->dev, "TMC-ETB/ETF enabled\n");
 273         return 0;
 274 }
 275
 276 static void tmc_disable_etf_sink(struct coresight_device *csdev)
 277 {
 278         unsigned long flags;
 279         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 280
 281         spin_lock_irqsave(&drvdata->spinlock, flags);
 282         if (drvdata->reading) {
 283                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 284                 return;
 285         }
 286
 287         /* Disable the TMC only if it needs to */
 288         if (drvdata->mode != CS_MODE_DISABLED) {
 289                 tmc_etb_disable_hw(drvdata);
 290                 drvdata->mode = CS_MODE_DISABLED;
 291         }
 292
 293         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 294
 295         dev_dbg(drvdata->dev, "TMC-ETB/ETF disabled\n");
 296 }
 297
 298 static int tmc_enable_etf_link(struct coresight_device *csdev,
 299                                int inport, int outport)
 300 {
 301         int ret;
 302         unsigned long flags;
 303         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 304
 305         spin_lock_irqsave(&drvdata->spinlock, flags);
 306         if (drvdata->reading) {
 307                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 308                 return -EBUSY;
 309         }
 310
 311         ret = tmc_etf_enable_hw(drvdata);
 312         if (!ret)
 313                 drvdata->mode = CS_MODE_SYSFS;
 314         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 315
 316         if (!ret)
 317                 dev_dbg(drvdata->dev, "TMC-ETF enabled\n");
 318         return ret;
 319 }
 320
 321 static void tmc_disable_etf_link(struct coresight_device *csdev,
 322                                  int inport, int outport)
 323 {
 324         unsigned long flags;
 325         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 326
 327         spin_lock_irqsave(&drvdata->spinlock, flags);
 328         if (drvdata->reading) {
 329                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 330                 return;
 331         }
 332
 333         tmc_etf_disable_hw(drvdata);
 334         drvdata->mode = CS_MODE_DISABLED;
 335         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 336
 337         dev_dbg(drvdata->dev, "TMC-ETF disabled\n");
 338 }
 339
 340 static void *tmc_alloc_etf_buffer(struct coresight_device *csdev, int cpu,
 341                                   void **pages, int nr_pages, bool overwrite)
 342 {
 343         int node;
 344         struct cs_buffers *buf;
 345
 346         if (cpu == -1)
 347                 cpu = smp_processor_id();
 348         node = cpu_to_node(cpu);
 349
 350         /* Allocate memory structure for interaction with Perf */
 351         buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
 352         if (!buf)
 353                 return NULL;
 354
 355         buf->snapshot = overwrite;
 356         buf->nr_pages = nr_pages;
 357         buf->data_pages = pages;
 358
 359         return buf;
 360 }
 361
 362 static void tmc_free_etf_buffer(void *config)
 363 {
 364         struct cs_buffers *buf = config;
 365
 366         kfree(buf);
 367 }
 368
 369 static int tmc_set_etf_buffer(struct coresight_device *csdev,
 370                               struct perf_output_handle *handle)
 371 {
 372         int ret = 0;
 373         unsigned long head;
 374         struct cs_buffers *buf = etm_perf_sink_config(handle);
 375
 376         if (!buf)
 377                 return -EINVAL;
 378
 379         /* wrap head around to the amount of space we have */
 380         head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
 381
 382         /* find the page to write to */
 383         buf->cur = head / PAGE_SIZE;
 384
 385         /* and offset within that page */
 386         buf->offset = head % PAGE_SIZE;
 387
 388         local_set(&buf->data_size, 0);
 389
 390         return ret;
 391 }
 392
 393 static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev,
 394                                   struct perf_output_handle *handle,
 395                                   void *sink_config)
 396 {
 397         bool lost = false;
 398         int i, cur;
 399         const u32 *barrier;
 400         u32 *buf_ptr;
 401         u64 read_ptr, write_ptr;
 402         u32 status;
 403         unsigned long offset, to_read;
 404         struct cs_buffers *buf = sink_config;
 405         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 406
 407         if (!buf)
 408                 return 0;
 409
 410         /* This shouldn't happen */
 411         if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF))
 412                 return 0;
 413
 414         CS_UNLOCK(drvdata->base);
 415
 416         tmc_flush_and_stop(drvdata);
 417
 418         read_ptr = tmc_read_rrp(drvdata);
 419         write_ptr = tmc_read_rwp(drvdata);
 420
 421         /*
 422          * Get a hold of the status register and see if a wrap around
 423          * has occurred.  If so adjust things accordingly.
 424          */
 425         status = readl_relaxed(drvdata->base + TMC_STS);
 426         if (status & TMC_STS_FULL) {
 427                 lost = true;
 428                 to_read = drvdata->size;
 429         } else {
 430                 to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
 431         }
 432
 433         /*
 434          * The TMC RAM buffer may be bigger than the space available in the
 435          * perf ring buffer (handle->size).  If so advance the RRP so that we
 436          * get the latest trace data.
 437          */
 438         if (to_read > handle->size) {
 439                 u32 mask = 0;
 440
 441                 /*
 442                  * The value written to RRP must be byte-address aligned to
 443                  * the width of the trace memory databus _and_ to a frame
 444                  * boundary (16 byte), whichever is the biggest. For example,
 445                  * for 32-bit, 64-bit and 128-bit wide trace memory, the four
 446                  * LSBs must be 0s. For 256-bit wide trace memory, the five
 447                  * LSBs must be 0s.
 448                  */
 449                 switch (drvdata->memwidth) {
 450                 case TMC_MEM_INTF_WIDTH_32BITS:
 451                 case TMC_MEM_INTF_WIDTH_64BITS:
 452                 case TMC_MEM_INTF_WIDTH_128BITS:
 453                         mask = GENMASK(31, 4);
 454                         break;
 455                 case TMC_MEM_INTF_WIDTH_256BITS:
 456                         mask = GENMASK(31, 5);
 457                         break;
 458                 }
 459
 460                 /*
 461                  * Make sure the new size is aligned in accordance with the
 462                  * requirement explained above.
 463                  */
 464                 to_read = handle->size & mask;
 465                 /* Move the RAM read pointer up */
 466                 read_ptr = (write_ptr + drvdata->size) - to_read;
 467                 /* Make sure we are still within our limits */
 468                 if (read_ptr > (drvdata->size - 1))
 469                         read_ptr -= drvdata->size;
 470                 /* Tell the HW */
 471                 tmc_write_rrp(drvdata, read_ptr);
 472                 lost = true;
 473         }
 474
 475         if (lost)
 476                 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
 477
 478         cur = buf->cur;
 479         offset = buf->offset;
 480         barrier = barrier_pkt;
 481
 482         /* for every byte to read */
 483         for (i = 0; i < to_read; i += 4) {
 484                 buf_ptr = buf->data_pages[cur] + offset;
 485                 *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
 486
 487                 if (lost && *barrier) {
 488                         *buf_ptr = *barrier;
 489                         barrier++;
 490                 }
 491
 492                 offset += 4;
 493                 if (offset >= PAGE_SIZE) {
 494                         offset = 0;
 495                         cur++;
 496                         /* wrap around at the end of the buffer */
 497                         cur &= buf->nr_pages - 1;
 498                 }
 499         }
 500
 501         /* In snapshot mode we have to update the head */
 502         if (buf->snapshot) {
 503                 handle->head = (cur * PAGE_SIZE) + offset;
 504                 to_read = buf->nr_pages << PAGE_SHIFT;
 505         }
 506         CS_LOCK(drvdata->base);
 507
 508         return to_read;
 509 }
 510
 511 static const struct coresight_ops_sink tmc_etf_sink_ops = {
 512         .enable         = tmc_enable_etf_sink,
 513         .disable        = tmc_disable_etf_sink,
 514         .alloc_buffer   = tmc_alloc_etf_buffer,
 515         .free_buffer    = tmc_free_etf_buffer,
 516         .update_buffer  = tmc_update_etf_buffer,
 517 };
 518
 519 static const struct coresight_ops_link tmc_etf_link_ops = {
 520         .enable         = tmc_enable_etf_link,
 521         .disable        = tmc_disable_etf_link,
 522 };
 523
 524 const struct coresight_ops tmc_etb_cs_ops = {
 525         .sink_ops       = &tmc_etf_sink_ops,
 526 };
 527
 528 const struct coresight_ops tmc_etf_cs_ops = {
 529         .sink_ops       = &tmc_etf_sink_ops,
 530         .link_ops       = &tmc_etf_link_ops,
 531 };
 532
 533 int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
 534 {
 535         enum tmc_mode mode;
 536         int ret = 0;
 537         unsigned long flags;
 538
 539         /* config types are set a boot time and never change */
 540         if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
 541                          drvdata->config_type != TMC_CONFIG_TYPE_ETF))
 542                 return -EINVAL;
 543
 544         spin_lock_irqsave(&drvdata->spinlock, flags);
 545
 546         if (drvdata->reading) {
 547                 ret = -EBUSY;
 548                 goto out;
 549         }
 550
 551         /* There is no point in reading a TMC in HW FIFO mode */
 552         mode = readl_relaxed(drvdata->base + TMC_MODE);
 553         if (mode != TMC_MODE_CIRCULAR_BUFFER) {
 554                 ret = -EINVAL;
 555                 goto out;
 556         }
 557
 558         /* Don't interfere if operated from Perf */
 559         if (drvdata->mode == CS_MODE_PERF) {
 560                 ret = -EINVAL;
 561                 goto out;
 562         }
 563
 564         /* If drvdata::buf is NULL the trace data has been read already */
 565         if (drvdata->buf == NULL) {
 566                 ret = -EINVAL;
 567                 goto out;
 568         }
 569
 570         /* Disable the TMC if need be */
 571         if (drvdata->mode == CS_MODE_SYSFS)
 572                 __tmc_etb_disable_hw(drvdata);
 573
 574         drvdata->reading = true;
 575 out:
 576         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 577
 578         return ret;
 579 }
 580
 581 int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
 582 {
 583         char *buf = NULL;
 584         enum tmc_mode mode;
 585         unsigned long flags;
 586
 587         /* config types are set a boot time and never change */
 588         if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
 589                          drvdata->config_type != TMC_CONFIG_TYPE_ETF))
 590                 return -EINVAL;
 591
 592         spin_lock_irqsave(&drvdata->spinlock, flags);
 593
 594         /* There is no point in reading a TMC in HW FIFO mode */
 595         mode = readl_relaxed(drvdata->base + TMC_MODE);
 596         if (mode != TMC_MODE_CIRCULAR_BUFFER) {
 597                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 598                 return -EINVAL;
 599         }
 600
 601         /* Re-enable the TMC if need be */
 602         if (drvdata->mode == CS_MODE_SYSFS) {
 603                 /*
 604                  * The trace run will continue with the same allocated trace
 605                  * buffer. As such zero-out the buffer so that we don't end
 606                  * up with stale data.
 607                  *
 608                  * Since the tracer is still enabled drvdata::buf
 609                  * can't be NULL.
 610                  */
 611                 memset(drvdata->buf, 0, drvdata->size);
 612                 __tmc_etb_enable_hw(drvdata);
 613         } else {
 614                 /*
 615                  * The ETB/ETF is not tracing and the buffer was just read.
 616                  * As such prepare to free the trace buffer.
 617                  */
 618                 buf = drvdata->buf;
 619                 drvdata->buf = NULL;
 620         }
 621
 622         drvdata->reading = false;
 623         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 624
 625         /*
 626          * Free allocated memory outside of the spinlock.  There is no need
 627          * to assert the validity of 'buf' since calling kfree(NULL) is safe.
 628          */
 629         kfree(buf);
 630
 631         return 0;
 632 }