1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Performance event support - Freescale Embedded Performance Monitor
5 * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
6 * Copyright 2010 Freescale Semiconductor, Inc.
8 #include <linux/kernel.h>
9 #include <linux/sched.h>
10 #include <linux/perf_event.h>
11 #include <linux/percpu.h>
12 #include <linux/hardirq.h>
13 #include <asm/reg_fsl_emb.h>
15 #include <asm/machdep.h>
16 #include <asm/firmware.h>
17 #include <asm/ptrace.h>
19 struct cpu_hw_events {
23 struct perf_event *event[MAX_HWEVENTS];
25 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
27 static struct fsl_emb_pmu *ppmu;
29 /* Number of perf_events counting hardware events */
30 static atomic_t num_events;
31 /* Used to avoid races in calling reserve/release_pmc_hardware */
32 static DEFINE_MUTEX(pmc_reserve_mutex);
35 * If interrupts were soft-disabled when a PMU interrupt occurs, treat
38 static inline int perf_intr_is_nmi(struct pt_regs *regs)
41 return (regs->softe & IRQS_DISABLED);
47 static void perf_event_interrupt(struct pt_regs *regs);
50 * Read one performance monitor counter (PMC).
52 static unsigned long read_pmc(int idx)
58 val = mfpmr(PMRN_PMC0);
61 val = mfpmr(PMRN_PMC1);
64 val = mfpmr(PMRN_PMC2);
67 val = mfpmr(PMRN_PMC3);
70 val = mfpmr(PMRN_PMC4);
73 val = mfpmr(PMRN_PMC5);
76 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
85 static void write_pmc(int idx, unsigned long val)
89 mtpmr(PMRN_PMC0, val);
92 mtpmr(PMRN_PMC1, val);
95 mtpmr(PMRN_PMC2, val);
98 mtpmr(PMRN_PMC3, val);
101 mtpmr(PMRN_PMC4, val);
104 mtpmr(PMRN_PMC5, val);
107 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
114 * Write one local control A register
116 static void write_pmlca(int idx, unsigned long val)
120 mtpmr(PMRN_PMLCA0, val);
123 mtpmr(PMRN_PMLCA1, val);
126 mtpmr(PMRN_PMLCA2, val);
129 mtpmr(PMRN_PMLCA3, val);
132 mtpmr(PMRN_PMLCA4, val);
135 mtpmr(PMRN_PMLCA5, val);
138 printk(KERN_ERR "oops trying to write PMLCA%d\n", idx);
145 * Write one local control B register
147 static void write_pmlcb(int idx, unsigned long val)
151 mtpmr(PMRN_PMLCB0, val);
154 mtpmr(PMRN_PMLCB1, val);
157 mtpmr(PMRN_PMLCB2, val);
160 mtpmr(PMRN_PMLCB3, val);
163 mtpmr(PMRN_PMLCB4, val);
166 mtpmr(PMRN_PMLCB5, val);
169 printk(KERN_ERR "oops trying to write PMLCB%d\n", idx);
175 static void fsl_emb_pmu_read(struct perf_event *event)
177 s64 val, delta, prev;
179 if (event->hw.state & PERF_HES_STOPPED)
183 * Performance monitor interrupts come even when interrupts
184 * are soft-disabled, as long as interrupts are hard-enabled.
185 * Therefore we treat them like NMIs.
188 prev = local64_read(&event->hw.prev_count);
190 val = read_pmc(event->hw.idx);
191 } while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
193 /* The counters are only 32 bits wide */
194 delta = (val - prev) & 0xfffffffful;
195 local64_add(delta, &event->count);
196 local64_sub(delta, &event->hw.period_left);
200 * Disable all events to prevent PMU interrupts and to allow
201 * events to be added or removed.
203 static void fsl_emb_pmu_disable(struct pmu *pmu)
205 struct cpu_hw_events *cpuhw;
208 local_irq_save(flags);
209 cpuhw = this_cpu_ptr(&cpu_hw_events);
211 if (!cpuhw->disabled) {
215 * Check if we ever enabled the PMU on this cpu.
217 if (!cpuhw->pmcs_enabled) {
219 cpuhw->pmcs_enabled = 1;
222 if (atomic_read(&num_events)) {
224 * Set the 'freeze all counters' bit, and disable
225 * interrupts. The barrier is to make sure the
226 * mtpmr has been executed and the PMU has frozen
227 * the events before we return.
230 mtpmr(PMRN_PMGC0, PMGC0_FAC);
234 local_irq_restore(flags);
238 * Re-enable all events if disable == 0.
239 * If we were previously disabled and events were added, then
240 * put the new config on the PMU.
242 static void fsl_emb_pmu_enable(struct pmu *pmu)
244 struct cpu_hw_events *cpuhw;
247 local_irq_save(flags);
248 cpuhw = this_cpu_ptr(&cpu_hw_events);
249 if (!cpuhw->disabled)
253 ppc_set_pmu_inuse(cpuhw->n_events != 0);
255 if (cpuhw->n_events > 0) {
256 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
261 local_irq_restore(flags);
264 static int collect_events(struct perf_event *group, int max_count,
265 struct perf_event *ctrs[])
268 struct perf_event *event;
270 if (!is_software_event(group)) {
276 for_each_sibling_event(event, group) {
277 if (!is_software_event(event) &&
278 event->state != PERF_EVENT_STATE_OFF) {
288 /* context locked on entry */
289 static int fsl_emb_pmu_add(struct perf_event *event, int flags)
291 struct cpu_hw_events *cpuhw;
293 int num_counters = ppmu->n_counter;
297 perf_pmu_disable(event->pmu);
298 cpuhw = &get_cpu_var(cpu_hw_events);
300 if (event->hw.config & FSL_EMB_EVENT_RESTRICTED)
301 num_counters = ppmu->n_restricted;
304 * Allocate counters from top-down, so that restricted-capable
305 * counters are kept free as long as possible.
307 for (i = num_counters - 1; i >= 0; i--) {
318 cpuhw->event[i] = event;
322 if (event->hw.sample_period) {
323 s64 left = local64_read(&event->hw.period_left);
324 if (left < 0x80000000L)
325 val = 0x80000000L - left;
327 local64_set(&event->hw.prev_count, val);
329 if (unlikely(!(flags & PERF_EF_START))) {
330 event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
333 event->hw.state &= ~(PERF_HES_STOPPED | PERF_HES_UPTODATE);
337 perf_event_update_userpage(event);
339 write_pmlcb(i, event->hw.config >> 32);
340 write_pmlca(i, event->hw.config_base);
344 put_cpu_var(cpu_hw_events);
345 perf_pmu_enable(event->pmu);
349 /* context locked on entry */
350 static void fsl_emb_pmu_del(struct perf_event *event, int flags)
352 struct cpu_hw_events *cpuhw;
353 int i = event->hw.idx;
355 perf_pmu_disable(event->pmu);
359 fsl_emb_pmu_read(event);
361 cpuhw = &get_cpu_var(cpu_hw_events);
363 WARN_ON(event != cpuhw->event[event->hw.idx]);
369 cpuhw->event[i] = NULL;
373 * TODO: if at least one restricted event exists, and we
374 * just freed up a non-restricted-capable counter, and
375 * there is a restricted-capable counter occupied by
376 * a non-restricted event, migrate that event to the
383 perf_pmu_enable(event->pmu);
384 put_cpu_var(cpu_hw_events);
387 static void fsl_emb_pmu_start(struct perf_event *event, int ef_flags)
393 if (event->hw.idx < 0 || !event->hw.sample_period)
396 if (!(event->hw.state & PERF_HES_STOPPED))
399 if (ef_flags & PERF_EF_RELOAD)
400 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
402 local_irq_save(flags);
403 perf_pmu_disable(event->pmu);
406 left = local64_read(&event->hw.period_left);
408 if (left < 0x80000000L)
409 val = 0x80000000L - left;
410 write_pmc(event->hw.idx, val);
412 perf_event_update_userpage(event);
413 perf_pmu_enable(event->pmu);
414 local_irq_restore(flags);
417 static void fsl_emb_pmu_stop(struct perf_event *event, int ef_flags)
421 if (event->hw.idx < 0 || !event->hw.sample_period)
424 if (event->hw.state & PERF_HES_STOPPED)
427 local_irq_save(flags);
428 perf_pmu_disable(event->pmu);
430 fsl_emb_pmu_read(event);
431 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
432 write_pmc(event->hw.idx, 0);
434 perf_event_update_userpage(event);
435 perf_pmu_enable(event->pmu);
436 local_irq_restore(flags);
440 * Release the PMU if this is the last perf_event.
442 static void hw_perf_event_destroy(struct perf_event *event)
444 if (!atomic_add_unless(&num_events, -1, 1)) {
445 mutex_lock(&pmc_reserve_mutex);
446 if (atomic_dec_return(&num_events) == 0)
447 release_pmc_hardware();
448 mutex_unlock(&pmc_reserve_mutex);
453 * Translate a generic cache event_id config to a raw event_id code.
455 static int hw_perf_cache_event(u64 config, u64 *eventp)
457 unsigned long type, op, result;
460 if (!ppmu->cache_events)
464 type = config & 0xff;
465 op = (config >> 8) & 0xff;
466 result = (config >> 16) & 0xff;
468 if (type >= PERF_COUNT_HW_CACHE_MAX ||
469 op >= PERF_COUNT_HW_CACHE_OP_MAX ||
470 result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
473 ev = (*ppmu->cache_events)[type][op][result];
482 static int fsl_emb_pmu_event_init(struct perf_event *event)
485 struct perf_event *events[MAX_HWEVENTS];
491 if (ppmu->n_counter > MAX_HWEVENTS) {
492 WARN(1, "No. of perf counters (%d) is higher than max array size(%d)\n",
493 ppmu->n_counter, MAX_HWEVENTS);
494 ppmu->n_counter = MAX_HWEVENTS;
497 switch (event->attr.type) {
498 case PERF_TYPE_HARDWARE:
499 ev = event->attr.config;
500 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
502 ev = ppmu->generic_events[ev];
505 case PERF_TYPE_HW_CACHE:
506 err = hw_perf_cache_event(event->attr.config, &ev);
512 ev = event->attr.config;
519 event->hw.config = ppmu->xlate_event(ev);
520 if (!(event->hw.config & FSL_EMB_EVENT_VALID))
524 * If this is in a group, check if it can go on with all the
525 * other hardware events in the group. We assume the event
526 * hasn't been linked into its leader's sibling list at this point.
529 if (event->group_leader != event) {
530 n = collect_events(event->group_leader,
531 ppmu->n_counter - 1, events);
536 if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) {
538 for (i = 0; i < n; i++) {
539 if (events[i]->hw.config & FSL_EMB_EVENT_RESTRICTED)
543 if (num_restricted >= ppmu->n_restricted)
549 event->hw.config_base = PMLCA_CE | PMLCA_FCM1 |
550 (u32)((ev << 16) & PMLCA_EVENT_MASK);
552 if (event->attr.exclude_user)
553 event->hw.config_base |= PMLCA_FCU;
554 if (event->attr.exclude_kernel)
555 event->hw.config_base |= PMLCA_FCS;
556 if (event->attr.exclude_idle)
559 event->hw.last_period = event->hw.sample_period;
560 local64_set(&event->hw.period_left, event->hw.last_period);
563 * See if we need to reserve the PMU.
564 * If no events are currently in use, then we have to take a
565 * mutex to ensure that we don't race with another task doing
566 * reserve_pmc_hardware or release_pmc_hardware.
569 if (!atomic_inc_not_zero(&num_events)) {
570 mutex_lock(&pmc_reserve_mutex);
571 if (atomic_read(&num_events) == 0 &&
572 reserve_pmc_hardware(perf_event_interrupt))
575 atomic_inc(&num_events);
576 mutex_unlock(&pmc_reserve_mutex);
578 mtpmr(PMRN_PMGC0, PMGC0_FAC);
581 event->destroy = hw_perf_event_destroy;
586 static struct pmu fsl_emb_pmu = {
587 .pmu_enable = fsl_emb_pmu_enable,
588 .pmu_disable = fsl_emb_pmu_disable,
589 .event_init = fsl_emb_pmu_event_init,
590 .add = fsl_emb_pmu_add,
591 .del = fsl_emb_pmu_del,
592 .start = fsl_emb_pmu_start,
593 .stop = fsl_emb_pmu_stop,
594 .read = fsl_emb_pmu_read,
598 * A counter has overflowed; update its count and record
599 * things if requested. Note that interrupts are hard-disabled
600 * here so there is no possibility of being interrupted.
602 static void record_and_restart(struct perf_event *event, unsigned long val,
603 struct pt_regs *regs)
605 u64 period = event->hw.sample_period;
606 s64 prev, delta, left;
609 if (event->hw.state & PERF_HES_STOPPED) {
610 write_pmc(event->hw.idx, 0);
614 /* we don't have to worry about interrupts here */
615 prev = local64_read(&event->hw.prev_count);
616 delta = (val - prev) & 0xfffffffful;
617 local64_add(delta, &event->count);
620 * See if the total period for this event has expired,
621 * and update for the next period.
624 left = local64_read(&event->hw.period_left) - delta;
631 event->hw.last_period = event->hw.sample_period;
633 if (left < 0x80000000LL)
634 val = 0x80000000LL - left;
637 write_pmc(event->hw.idx, val);
638 local64_set(&event->hw.prev_count, val);
639 local64_set(&event->hw.period_left, left);
640 perf_event_update_userpage(event);
643 * Finally record data if requested.
646 struct perf_sample_data data;
648 perf_sample_data_init(&data, 0, event->hw.last_period);
650 if (perf_event_overflow(event, &data, regs))
651 fsl_emb_pmu_stop(event, 0);
655 static void perf_event_interrupt(struct pt_regs *regs)
658 struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
659 struct perf_event *event;
664 nmi = perf_intr_is_nmi(regs);
670 for (i = 0; i < ppmu->n_counter; ++i) {
671 event = cpuhw->event[i];
676 /* event has overflowed */
678 record_and_restart(event, val, regs);
681 * Disabled counter is negative,
682 * reset it just in case.
689 /* PMM will keep counters frozen until we return from the interrupt. */
690 mtmsr(mfmsr() | MSR_PMM);
691 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
700 void hw_perf_event_setup(int cpu)
702 struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
704 memset(cpuhw, 0, sizeof(*cpuhw));
707 int register_fsl_emb_pmu(struct fsl_emb_pmu *pmu)
710 return -EBUSY; /* something's already registered */
713 pr_info("%s performance monitor hardware support registered\n",
716 perf_pmu_register(&fsl_emb_pmu, "cpu", PERF_TYPE_RAW);