1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright(C) 2016 Linaro Limited. All rights reserved.
4 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
7 #include <linux/atomic.h>
8 #include <linux/circ_buf.h>
9 #include <linux/coresight.h>
10 #include <linux/perf_event.h>
11 #include <linux/slab.h>
12 #include "coresight-priv.h"
13 #include "coresight-tmc.h"
14 #include "coresight-etm-perf.h"
16 static int tmc_set_etf_buffer(struct coresight_device *csdev,
17 struct perf_output_handle *handle);
19 static void __tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
21 CS_UNLOCK(drvdata->base);
23 /* Wait for TMCSReady bit to be set */
24 tmc_wait_for_tmcready(drvdata);
26 writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
27 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
28 TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
29 TMC_FFCR_TRIGON_TRIGIN,
30 drvdata->base + TMC_FFCR);
32 writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
33 tmc_enable_hw(drvdata);
35 CS_LOCK(drvdata->base);
38 static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
40 int rc = coresight_claim_device(drvdata->base);
45 __tmc_etb_enable_hw(drvdata);
49 static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
54 /* Check if the buffer wrapped around. */
55 lost = readl_relaxed(drvdata->base + TMC_STS) & TMC_STS_FULL;
59 read_data = readl_relaxed(drvdata->base + TMC_RRD);
60 if (read_data == 0xFFFFFFFF)
62 memcpy(bufp, &read_data, 4);
68 coresight_insert_barrier_packet(drvdata->buf);
72 static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
74 CS_UNLOCK(drvdata->base);
76 tmc_flush_and_stop(drvdata);
78 * When operating in sysFS mode the content of the buffer needs to be
79 * read before the TMC is disabled.
81 if (drvdata->mode == CS_MODE_SYSFS)
82 tmc_etb_dump_hw(drvdata);
83 tmc_disable_hw(drvdata);
85 CS_LOCK(drvdata->base);
88 static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
90 __tmc_etb_disable_hw(drvdata);
91 coresight_disclaim_device(drvdata->base);
94 static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
96 CS_UNLOCK(drvdata->base);
98 /* Wait for TMCSReady bit to be set */
99 tmc_wait_for_tmcready(drvdata);
101 writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
102 writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
103 drvdata->base + TMC_FFCR);
104 writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
105 tmc_enable_hw(drvdata);
107 CS_LOCK(drvdata->base);
110 static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
112 int rc = coresight_claim_device(drvdata->base);
117 __tmc_etf_enable_hw(drvdata);
121 static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
123 CS_UNLOCK(drvdata->base);
125 tmc_flush_and_stop(drvdata);
126 tmc_disable_hw(drvdata);
127 coresight_disclaim_device_unlocked(drvdata->base);
128 CS_LOCK(drvdata->base);
132 * Return the available trace data in the buffer from @pos, with
133 * a maximum limit of @len, updating the @bufpp on where to
136 ssize_t tmc_etb_get_sysfs_trace(struct tmc_drvdata *drvdata,
137 loff_t pos, size_t len, char **bufpp)
139 ssize_t actual = len;
141 /* Adjust the len to available size @pos */
142 if (pos + actual > drvdata->len)
143 actual = drvdata->len - pos;
145 *bufpp = drvdata->buf + pos;
149 static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
155 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
158 * If we don't have a buffer release the lock and allocate memory.
159 * Otherwise keep the lock and move along.
161 spin_lock_irqsave(&drvdata->spinlock, flags);
163 spin_unlock_irqrestore(&drvdata->spinlock, flags);
165 /* Allocating the memory here while outside of the spinlock */
166 buf = kzalloc(drvdata->size, GFP_KERNEL);
170 /* Let's try again */
171 spin_lock_irqsave(&drvdata->spinlock, flags);
174 if (drvdata->reading) {
180 * In sysFS mode we can have multiple writers per sink. Since this
181 * sink is already enabled no memory is needed and the HW need not be
184 if (drvdata->mode == CS_MODE_SYSFS) {
185 atomic_inc(csdev->refcnt);
190 * If drvdata::buf isn't NULL, memory was allocated for a previous
191 * trace run but wasn't read. If so simply zero-out the memory.
192 * Otherwise use the memory allocated above.
194 * The memory is freed when users read the buffer using the
195 * /dev/xyz.{etf|etb} interface. See tmc_read_unprepare_etf() for
199 memset(drvdata->buf, 0, drvdata->size);
205 ret = tmc_etb_enable_hw(drvdata);
207 drvdata->mode = CS_MODE_SYSFS;
208 atomic_inc(csdev->refcnt);
210 /* Free up the buffer if we failed to enable */
214 spin_unlock_irqrestore(&drvdata->spinlock, flags);
216 /* Free memory outside the spinlock if need be */
223 static int tmc_enable_etf_sink_perf(struct coresight_device *csdev, void *data)
228 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
229 struct perf_output_handle *handle = data;
231 spin_lock_irqsave(&drvdata->spinlock, flags);
234 if (drvdata->reading)
237 * No need to continue if the ETB/ETF is already operated
240 if (drvdata->mode == CS_MODE_SYSFS) {
245 /* Get a handle on the pid of the process to monitor */
246 pid = task_pid_nr(handle->event->owner);
248 if (drvdata->pid != -1 && drvdata->pid != pid) {
253 ret = tmc_set_etf_buffer(csdev, handle);
258 * No HW configuration is needed if the sink is already in
259 * use for this session.
261 if (drvdata->pid == pid) {
262 atomic_inc(csdev->refcnt);
266 ret = tmc_etb_enable_hw(drvdata);
268 /* Associate with monitored process. */
270 drvdata->mode = CS_MODE_PERF;
271 atomic_inc(csdev->refcnt);
274 spin_unlock_irqrestore(&drvdata->spinlock, flags);
279 static int tmc_enable_etf_sink(struct coresight_device *csdev,
280 u32 mode, void *data)
286 ret = tmc_enable_etf_sink_sysfs(csdev);
289 ret = tmc_enable_etf_sink_perf(csdev, data);
291 /* We shouldn't be here */
300 dev_dbg(&csdev->dev, "TMC-ETB/ETF enabled\n");
304 static int tmc_disable_etf_sink(struct coresight_device *csdev)
307 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
309 spin_lock_irqsave(&drvdata->spinlock, flags);
311 if (drvdata->reading) {
312 spin_unlock_irqrestore(&drvdata->spinlock, flags);
316 if (atomic_dec_return(csdev->refcnt)) {
317 spin_unlock_irqrestore(&drvdata->spinlock, flags);
321 /* Complain if we (somehow) got out of sync */
322 WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
323 tmc_etb_disable_hw(drvdata);
324 /* Dissociate from monitored process. */
326 drvdata->mode = CS_MODE_DISABLED;
328 spin_unlock_irqrestore(&drvdata->spinlock, flags);
330 dev_dbg(&csdev->dev, "TMC-ETB/ETF disabled\n");
334 static int tmc_enable_etf_link(struct coresight_device *csdev,
335 int inport, int outport)
339 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
341 spin_lock_irqsave(&drvdata->spinlock, flags);
342 if (drvdata->reading) {
343 spin_unlock_irqrestore(&drvdata->spinlock, flags);
347 ret = tmc_etf_enable_hw(drvdata);
349 drvdata->mode = CS_MODE_SYSFS;
350 spin_unlock_irqrestore(&drvdata->spinlock, flags);
353 dev_dbg(&csdev->dev, "TMC-ETF enabled\n");
357 static void tmc_disable_etf_link(struct coresight_device *csdev,
358 int inport, int outport)
361 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
363 spin_lock_irqsave(&drvdata->spinlock, flags);
364 if (drvdata->reading) {
365 spin_unlock_irqrestore(&drvdata->spinlock, flags);
369 tmc_etf_disable_hw(drvdata);
370 drvdata->mode = CS_MODE_DISABLED;
371 spin_unlock_irqrestore(&drvdata->spinlock, flags);
373 dev_dbg(&csdev->dev, "TMC-ETF disabled\n");
376 static void *tmc_alloc_etf_buffer(struct coresight_device *csdev,
377 struct perf_event *event, void **pages,
378 int nr_pages, bool overwrite)
381 struct cs_buffers *buf;
383 node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
385 /* Allocate memory structure for interaction with Perf */
386 buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
390 buf->snapshot = overwrite;
391 buf->nr_pages = nr_pages;
392 buf->data_pages = pages;
397 static void tmc_free_etf_buffer(void *config)
399 struct cs_buffers *buf = config;
404 static int tmc_set_etf_buffer(struct coresight_device *csdev,
405 struct perf_output_handle *handle)
409 struct cs_buffers *buf = etm_perf_sink_config(handle);
414 /* wrap head around to the amount of space we have */
415 head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
417 /* find the page to write to */
418 buf->cur = head / PAGE_SIZE;
420 /* and offset within that page */
421 buf->offset = head % PAGE_SIZE;
423 local_set(&buf->data_size, 0);
428 static unsigned long tmc_update_etf_buffer(struct coresight_device *csdev,
429 struct perf_output_handle *handle,
436 u64 read_ptr, write_ptr;
438 unsigned long offset, to_read = 0, flags;
439 struct cs_buffers *buf = sink_config;
440 struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
445 /* This shouldn't happen */
446 if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF))
449 spin_lock_irqsave(&drvdata->spinlock, flags);
451 /* Don't do anything if another tracer is using this sink */
452 if (atomic_read(csdev->refcnt) != 1)
455 CS_UNLOCK(drvdata->base);
457 tmc_flush_and_stop(drvdata);
459 read_ptr = tmc_read_rrp(drvdata);
460 write_ptr = tmc_read_rwp(drvdata);
463 * Get a hold of the status register and see if a wrap around
464 * has occurred. If so adjust things accordingly.
466 status = readl_relaxed(drvdata->base + TMC_STS);
467 if (status & TMC_STS_FULL) {
469 to_read = drvdata->size;
471 to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
475 * The TMC RAM buffer may be bigger than the space available in the
476 * perf ring buffer (handle->size). If so advance the RRP so that we
477 * get the latest trace data. In snapshot mode none of that matters
478 * since we are expected to clobber stale data in favour of the latest
481 if (!buf->snapshot && to_read > handle->size) {
482 u32 mask = tmc_get_memwidth_mask(drvdata);
485 * Make sure the new size is aligned in accordance with the
486 * requirement explained in function tmc_get_memwidth_mask().
488 to_read = handle->size & mask;
489 /* Move the RAM read pointer up */
490 read_ptr = (write_ptr + drvdata->size) - to_read;
491 /* Make sure we are still within our limits */
492 if (read_ptr > (drvdata->size - 1))
493 read_ptr -= drvdata->size;
495 tmc_write_rrp(drvdata, read_ptr);
500 * Don't set the TRUNCATED flag in snapshot mode because 1) the
501 * captured buffer is expected to be truncated and 2) a full buffer
502 * prevents the event from being re-enabled by the perf core,
503 * resulting in stale data being send to user space.
505 if (!buf->snapshot && lost)
506 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
509 offset = buf->offset;
510 barrier = barrier_pkt;
512 /* for every byte to read */
513 for (i = 0; i < to_read; i += 4) {
514 buf_ptr = buf->data_pages[cur] + offset;
515 *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
517 if (lost && *barrier) {
523 if (offset >= PAGE_SIZE) {
526 /* wrap around at the end of the buffer */
527 cur &= buf->nr_pages - 1;
532 * In snapshot mode we simply increment the head by the number of byte
533 * that were written. User space function cs_etm_find_snapshot() will
534 * figure out how many bytes to get from the AUX buffer based on the
535 * position of the head.
538 handle->head += to_read;
540 CS_LOCK(drvdata->base);
542 spin_unlock_irqrestore(&drvdata->spinlock, flags);
547 static const struct coresight_ops_sink tmc_etf_sink_ops = {
548 .enable = tmc_enable_etf_sink,
549 .disable = tmc_disable_etf_sink,
550 .alloc_buffer = tmc_alloc_etf_buffer,
551 .free_buffer = tmc_free_etf_buffer,
552 .update_buffer = tmc_update_etf_buffer,
555 static const struct coresight_ops_link tmc_etf_link_ops = {
556 .enable = tmc_enable_etf_link,
557 .disable = tmc_disable_etf_link,
560 const struct coresight_ops tmc_etb_cs_ops = {
561 .sink_ops = &tmc_etf_sink_ops,
564 const struct coresight_ops tmc_etf_cs_ops = {
565 .sink_ops = &tmc_etf_sink_ops,
566 .link_ops = &tmc_etf_link_ops,
569 int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
575 /* config types are set a boot time and never change */
576 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
577 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
580 spin_lock_irqsave(&drvdata->spinlock, flags);
582 if (drvdata->reading) {
587 /* There is no point in reading a TMC in HW FIFO mode */
588 mode = readl_relaxed(drvdata->base + TMC_MODE);
589 if (mode != TMC_MODE_CIRCULAR_BUFFER) {
594 /* Don't interfere if operated from Perf */
595 if (drvdata->mode == CS_MODE_PERF) {
600 /* If drvdata::buf is NULL the trace data has been read already */
601 if (drvdata->buf == NULL) {
606 /* Disable the TMC if need be */
607 if (drvdata->mode == CS_MODE_SYSFS)
608 __tmc_etb_disable_hw(drvdata);
610 drvdata->reading = true;
612 spin_unlock_irqrestore(&drvdata->spinlock, flags);
617 int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
623 /* config types are set a boot time and never change */
624 if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
625 drvdata->config_type != TMC_CONFIG_TYPE_ETF))
628 spin_lock_irqsave(&drvdata->spinlock, flags);
630 /* There is no point in reading a TMC in HW FIFO mode */
631 mode = readl_relaxed(drvdata->base + TMC_MODE);
632 if (mode != TMC_MODE_CIRCULAR_BUFFER) {
633 spin_unlock_irqrestore(&drvdata->spinlock, flags);
637 /* Re-enable the TMC if need be */
638 if (drvdata->mode == CS_MODE_SYSFS) {
640 * The trace run will continue with the same allocated trace
641 * buffer. As such zero-out the buffer so that we don't end
642 * up with stale data.
644 * Since the tracer is still enabled drvdata::buf
647 memset(drvdata->buf, 0, drvdata->size);
648 __tmc_etb_enable_hw(drvdata);
651 * The ETB/ETF is not tracing and the buffer was just read.
652 * As such prepare to free the trace buffer.
658 drvdata->reading = false;
659 spin_unlock_irqrestore(&drvdata->spinlock, flags);
662 * Free allocated memory outside of the spinlock. There is no need
663 * to assert the validity of 'buf' since calling kfree(NULL) is safe.