1 /* Copyright (c) 2015-2017 The Linux Foundation. All rights reserved.
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
12 #include <linux/acpi.h>
13 #include <linux/bitops.h>
14 #include <linux/bug.h>
15 #include <linux/cpuhotplug.h>
16 #include <linux/cpumask.h>
17 #include <linux/device.h>
18 #include <linux/errno.h>
19 #include <linux/interrupt.h>
20 #include <linux/irq.h>
21 #include <linux/kernel.h>
22 #include <linux/list.h>
23 #include <linux/percpu.h>
24 #include <linux/perf_event.h>
25 #include <linux/platform_device.h>
26 #include <linux/smp.h>
27 #include <linux/spinlock.h>
28 #include <linux/sysfs.h>
29 #include <linux/types.h>
31 #include <asm/barrier.h>
32 #include <asm/local64.h>
33 #include <asm/sysreg.h>
37 #define L2PMCR_NUM_EV_SHIFT 11
38 #define L2PMCR_NUM_EV_MASK 0x1F
41 #define L2PMCNTENCLR 0x403
42 #define L2PMCNTENSET 0x404
43 #define L2PMINTENCLR 0x405
44 #define L2PMINTENSET 0x406
45 #define L2PMOVSCLR 0x407
46 #define L2PMOVSSET 0x408
47 #define L2PMCCNTCR 0x409
48 #define L2PMCCNTR 0x40A
49 #define L2PMCCNTSR 0x40C
50 #define L2PMRESR 0x410
51 #define IA_L2PMXEVCNTCR_BASE 0x420
52 #define IA_L2PMXEVCNTR_BASE 0x421
53 #define IA_L2PMXEVFILTER_BASE 0x423
54 #define IA_L2PMXEVTYPER_BASE 0x424
56 #define IA_L2_REG_OFFSET 0x10
58 #define L2PMXEVFILTER_SUFILTER_ALL 0x000E0000
59 #define L2PMXEVFILTER_ORGFILTER_IDINDEP 0x00000004
60 #define L2PMXEVFILTER_ORGFILTER_ALL 0x00000003
62 #define L2EVTYPER_REG_SHIFT 3
64 #define L2PMRESR_GROUP_BITS 8
65 #define L2PMRESR_GROUP_MASK GENMASK(7, 0)
67 #define L2CYCLE_CTR_BIT 31
68 #define L2CYCLE_CTR_RAW_CODE 0xFE
70 #define L2PMCR_RESET_ALL 0x6
71 #define L2PMCR_COUNTERS_ENABLE 0x1
72 #define L2PMCR_COUNTERS_DISABLE 0x0
74 #define L2PMRESR_EN BIT_ULL(63)
76 #define L2_EVT_MASK 0x00000FFF
77 #define L2_EVT_CODE_MASK 0x00000FF0
78 #define L2_EVT_GRP_MASK 0x0000000F
79 #define L2_EVT_CODE_SHIFT 4
80 #define L2_EVT_GRP_SHIFT 0
82 #define L2_EVT_CODE(event) (((event) & L2_EVT_CODE_MASK) >> L2_EVT_CODE_SHIFT)
83 #define L2_EVT_GROUP(event) (((event) & L2_EVT_GRP_MASK) >> L2_EVT_GRP_SHIFT)
85 #define L2_EVT_GROUP_MAX 7
87 #define L2_COUNTER_RELOAD BIT_ULL(31)
88 #define L2_CYCLE_COUNTER_RELOAD BIT_ULL(63)
90 #define L2CPUSRSELR_EL1 sys_reg(3, 3, 15, 0, 6)
91 #define L2CPUSRDR_EL1 sys_reg(3, 3, 15, 0, 7)
93 #define reg_idx(reg, i) (((i) * IA_L2_REG_OFFSET) + reg##_BASE)
98 #define L2_EVENT_CYCLES 0xfe
99 #define L2_EVENT_DCACHE_OPS 0x400
100 #define L2_EVENT_ICACHE_OPS 0x401
101 #define L2_EVENT_TLBI 0x402
102 #define L2_EVENT_BARRIERS 0x403
103 #define L2_EVENT_TOTAL_READS 0x405
104 #define L2_EVENT_TOTAL_WRITES 0x406
105 #define L2_EVENT_TOTAL_REQUESTS 0x407
106 #define L2_EVENT_LDREX 0x420
107 #define L2_EVENT_STREX 0x421
108 #define L2_EVENT_CLREX 0x422
110 static DEFINE_RAW_SPINLOCK(l2_access_lock);
113 * set_l2_indirect_reg: write value to an L2 register
114 * @reg: Address of L2 register.
115 * @value: Value to be written to register.
117 * Use architecturally required barriers for ordering between system register
120 static void set_l2_indirect_reg(u64 reg, u64 val)
124 raw_spin_lock_irqsave(&l2_access_lock, flags);
125 write_sysreg_s(reg, L2CPUSRSELR_EL1);
127 write_sysreg_s(val, L2CPUSRDR_EL1);
129 raw_spin_unlock_irqrestore(&l2_access_lock, flags);
133 * get_l2_indirect_reg: read an L2 register value
134 * @reg: Address of L2 register.
136 * Use architecturally required barriers for ordering between system register
139 static u64 get_l2_indirect_reg(u64 reg)
144 raw_spin_lock_irqsave(&l2_access_lock, flags);
145 write_sysreg_s(reg, L2CPUSRSELR_EL1);
147 val = read_sysreg_s(L2CPUSRDR_EL1);
148 raw_spin_unlock_irqrestore(&l2_access_lock, flags);
156 * Aggregate PMU. Implements the core pmu functions and manages
160 struct hlist_node node;
165 struct platform_device *pdev;
166 struct cluster_pmu * __percpu *pmu_cluster;
167 struct list_head clusters;
171 * The cache is made up of one or more clusters, each cluster has its own PMU.
172 * Each cluster is associated with one or more CPUs.
173 * This structure represents one of the hardware PMUs.
175 * Events can be envisioned as a 2-dimensional array. Each column represents
176 * a group of events. There are 8 groups. Only one entry from each
177 * group can be in use at a time.
179 * Events are specified as 0xCCG, where CC is 2 hex digits specifying
180 * the code (array row) and G specifies the group (column).
182 * In addition there is a cycle counter event specified by L2CYCLE_CTR_RAW_CODE
183 * which is outside the above scheme.
186 struct list_head next;
187 struct perf_event *events[MAX_L2_CTRS];
188 struct l2cache_pmu *l2cache_pmu;
189 DECLARE_BITMAP(used_counters, MAX_L2_CTRS);
190 DECLARE_BITMAP(used_groups, L2_EVT_GROUP_MAX + 1);
193 /* The CPU that is used for collecting events on this cluster */
195 /* All the CPUs associated with this cluster */
196 cpumask_t cluster_cpus;
200 #define to_l2cache_pmu(p) (container_of(p, struct l2cache_pmu, pmu))
202 static u32 l2_cycle_ctr_idx;
203 static u32 l2_counter_present_mask;
205 static inline u32 idx_to_reg_bit(u32 idx)
207 if (idx == l2_cycle_ctr_idx)
208 return BIT(L2CYCLE_CTR_BIT);
213 static inline struct cluster_pmu *get_cluster_pmu(
214 struct l2cache_pmu *l2cache_pmu, int cpu)
216 return *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu);
219 static void cluster_pmu_reset(void)
221 /* Reset all counters */
222 set_l2_indirect_reg(L2PMCR, L2PMCR_RESET_ALL);
223 set_l2_indirect_reg(L2PMCNTENCLR, l2_counter_present_mask);
224 set_l2_indirect_reg(L2PMINTENCLR, l2_counter_present_mask);
225 set_l2_indirect_reg(L2PMOVSCLR, l2_counter_present_mask);
228 static inline void cluster_pmu_enable(void)
230 set_l2_indirect_reg(L2PMCR, L2PMCR_COUNTERS_ENABLE);
233 static inline void cluster_pmu_disable(void)
235 set_l2_indirect_reg(L2PMCR, L2PMCR_COUNTERS_DISABLE);
238 static inline void cluster_pmu_counter_set_value(u32 idx, u64 value)
240 if (idx == l2_cycle_ctr_idx)
241 set_l2_indirect_reg(L2PMCCNTR, value);
243 set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx), value);
246 static inline u64 cluster_pmu_counter_get_value(u32 idx)
250 if (idx == l2_cycle_ctr_idx)
251 value = get_l2_indirect_reg(L2PMCCNTR);
253 value = get_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx));
258 static inline void cluster_pmu_counter_enable(u32 idx)
260 set_l2_indirect_reg(L2PMCNTENSET, idx_to_reg_bit(idx));
263 static inline void cluster_pmu_counter_disable(u32 idx)
265 set_l2_indirect_reg(L2PMCNTENCLR, idx_to_reg_bit(idx));
268 static inline void cluster_pmu_counter_enable_interrupt(u32 idx)
270 set_l2_indirect_reg(L2PMINTENSET, idx_to_reg_bit(idx));
273 static inline void cluster_pmu_counter_disable_interrupt(u32 idx)
275 set_l2_indirect_reg(L2PMINTENCLR, idx_to_reg_bit(idx));
278 static inline void cluster_pmu_set_evccntcr(u32 val)
280 set_l2_indirect_reg(L2PMCCNTCR, val);
283 static inline void cluster_pmu_set_evcntcr(u32 ctr, u32 val)
285 set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTCR, ctr), val);
288 static inline void cluster_pmu_set_evtyper(u32 ctr, u32 val)
290 set_l2_indirect_reg(reg_idx(IA_L2PMXEVTYPER, ctr), val);
293 static void cluster_pmu_set_resr(struct cluster_pmu *cluster,
294 u32 event_group, u32 event_cc)
301 shift = L2PMRESR_GROUP_BITS * event_group;
302 field = ((u64)(event_cc & L2PMRESR_GROUP_MASK) << shift);
304 spin_lock_irqsave(&cluster->pmu_lock, flags);
306 resr_val = get_l2_indirect_reg(L2PMRESR);
307 resr_val &= ~(L2PMRESR_GROUP_MASK << shift);
309 resr_val |= L2PMRESR_EN;
310 set_l2_indirect_reg(L2PMRESR, resr_val);
312 spin_unlock_irqrestore(&cluster->pmu_lock, flags);
316 * Hardware allows filtering of events based on the originating
317 * CPU. Turn this off by setting filter bits to allow events from
318 * all CPUS, subunits and ID independent events in this cluster.
320 static inline void cluster_pmu_set_evfilter_sys_mode(u32 ctr)
322 u32 val = L2PMXEVFILTER_SUFILTER_ALL |
323 L2PMXEVFILTER_ORGFILTER_IDINDEP |
324 L2PMXEVFILTER_ORGFILTER_ALL;
326 set_l2_indirect_reg(reg_idx(IA_L2PMXEVFILTER, ctr), val);
329 static inline u32 cluster_pmu_getreset_ovsr(void)
331 u32 result = get_l2_indirect_reg(L2PMOVSSET);
333 set_l2_indirect_reg(L2PMOVSCLR, result);
337 static inline bool cluster_pmu_has_overflowed(u32 ovsr)
339 return !!(ovsr & l2_counter_present_mask);
342 static inline bool cluster_pmu_counter_has_overflowed(u32 ovsr, u32 idx)
344 return !!(ovsr & idx_to_reg_bit(idx));
347 static void l2_cache_event_update(struct perf_event *event)
349 struct hw_perf_event *hwc = &event->hw;
350 u64 delta, prev, now;
354 prev = local64_read(&hwc->prev_count);
355 now = cluster_pmu_counter_get_value(idx);
356 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
359 * The cycle counter is 64-bit, but all other counters are
360 * 32-bit, and we must handle 32-bit overflow explicitly.
363 if (idx != l2_cycle_ctr_idx)
366 local64_add(delta, &event->count);
369 static void l2_cache_cluster_set_period(struct cluster_pmu *cluster,
370 struct hw_perf_event *hwc)
376 * We limit the max period to half the max counter value so
377 * that even in the case of extreme interrupt latency the
378 * counter will (hopefully) not wrap past its initial value.
380 if (idx == l2_cycle_ctr_idx)
381 new = L2_CYCLE_COUNTER_RELOAD;
383 new = L2_COUNTER_RELOAD;
385 local64_set(&hwc->prev_count, new);
386 cluster_pmu_counter_set_value(idx, new);
389 static int l2_cache_get_event_idx(struct cluster_pmu *cluster,
390 struct perf_event *event)
392 struct hw_perf_event *hwc = &event->hw;
394 int num_ctrs = cluster->l2cache_pmu->num_counters - 1;
397 if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) {
398 if (test_and_set_bit(l2_cycle_ctr_idx, cluster->used_counters))
401 return l2_cycle_ctr_idx;
404 idx = find_first_zero_bit(cluster->used_counters, num_ctrs);
406 /* The counters are all in use. */
410 * Check for column exclusion: event column already in use by another
411 * event. This is for events which are not in the same group.
412 * Conflicting events in the same group are detected in event_init.
414 group = L2_EVT_GROUP(hwc->config_base);
415 if (test_bit(group, cluster->used_groups))
418 set_bit(idx, cluster->used_counters);
419 set_bit(group, cluster->used_groups);
424 static void l2_cache_clear_event_idx(struct cluster_pmu *cluster,
425 struct perf_event *event)
427 struct hw_perf_event *hwc = &event->hw;
430 clear_bit(idx, cluster->used_counters);
431 if (hwc->config_base != L2CYCLE_CTR_RAW_CODE)
432 clear_bit(L2_EVT_GROUP(hwc->config_base), cluster->used_groups);
435 static irqreturn_t l2_cache_handle_irq(int irq_num, void *data)
437 struct cluster_pmu *cluster = data;
438 int num_counters = cluster->l2cache_pmu->num_counters;
442 ovsr = cluster_pmu_getreset_ovsr();
443 if (!cluster_pmu_has_overflowed(ovsr))
446 for_each_set_bit(idx, cluster->used_counters, num_counters) {
447 struct perf_event *event = cluster->events[idx];
448 struct hw_perf_event *hwc;
450 if (WARN_ON_ONCE(!event))
453 if (!cluster_pmu_counter_has_overflowed(ovsr, idx))
456 l2_cache_event_update(event);
459 l2_cache_cluster_set_period(cluster, hwc);
466 * Implementation of abstract pmu functionality required by
467 * the core perf events code.
470 static void l2_cache_pmu_enable(struct pmu *pmu)
473 * Although there is only one PMU (per socket) controlling multiple
474 * physical PMUs (per cluster), because we do not support per-task mode
475 * each event is associated with a CPU. Each event has pmu_enable
476 * called on its CPU, so here it is only necessary to enable the
477 * counters for the current CPU.
480 cluster_pmu_enable();
483 static void l2_cache_pmu_disable(struct pmu *pmu)
485 cluster_pmu_disable();
488 static int l2_cache_event_init(struct perf_event *event)
490 struct hw_perf_event *hwc = &event->hw;
491 struct cluster_pmu *cluster;
492 struct perf_event *sibling;
493 struct l2cache_pmu *l2cache_pmu;
495 if (event->attr.type != event->pmu->type)
498 l2cache_pmu = to_l2cache_pmu(event->pmu);
500 if (hwc->sample_period) {
501 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
502 "Sampling not supported\n");
506 if (event->cpu < 0) {
507 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
508 "Per-task mode not supported\n");
512 if (((L2_EVT_GROUP(event->attr.config) > L2_EVT_GROUP_MAX) ||
513 ((event->attr.config & ~L2_EVT_MASK) != 0)) &&
514 (event->attr.config != L2CYCLE_CTR_RAW_CODE)) {
515 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
516 "Invalid config %llx\n",
521 /* Don't allow groups with mixed PMUs, except for s/w events */
522 if (event->group_leader->pmu != event->pmu &&
523 !is_software_event(event->group_leader)) {
524 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
525 "Can't create mixed PMU group\n");
529 for_each_sibling_event(sibling, event->group_leader) {
530 if (sibling->pmu != event->pmu &&
531 !is_software_event(sibling)) {
532 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
533 "Can't create mixed PMU group\n");
538 cluster = get_cluster_pmu(l2cache_pmu, event->cpu);
540 /* CPU has not been initialised */
541 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
542 "CPU%d not associated with L2 cluster\n", event->cpu);
546 /* Ensure all events in a group are on the same cpu */
547 if ((event->group_leader != event) &&
548 (cluster->on_cpu != event->group_leader->cpu)) {
549 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
550 "Can't create group on CPUs %d and %d",
551 event->cpu, event->group_leader->cpu);
555 if ((event != event->group_leader) &&
556 !is_software_event(event->group_leader) &&
557 (L2_EVT_GROUP(event->group_leader->attr.config) ==
558 L2_EVT_GROUP(event->attr.config))) {
559 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
560 "Column exclusion: conflicting events %llx %llx\n",
561 event->group_leader->attr.config,
566 for_each_sibling_event(sibling, event->group_leader) {
567 if ((sibling != event) &&
568 !is_software_event(sibling) &&
569 (L2_EVT_GROUP(sibling->attr.config) ==
570 L2_EVT_GROUP(event->attr.config))) {
571 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev,
572 "Column exclusion: conflicting events %llx %llx\n",
573 sibling->attr.config,
580 hwc->config_base = event->attr.config;
583 * Ensure all events are on the same cpu so all events are in the
584 * same cpu context, to avoid races on pmu_enable etc.
586 event->cpu = cluster->on_cpu;
591 static void l2_cache_event_start(struct perf_event *event, int flags)
593 struct cluster_pmu *cluster;
594 struct hw_perf_event *hwc = &event->hw;
597 u32 event_cc, event_group;
601 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
603 l2_cache_cluster_set_period(cluster, hwc);
605 if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) {
606 cluster_pmu_set_evccntcr(0);
608 config = hwc->config_base;
609 event_cc = L2_EVT_CODE(config);
610 event_group = L2_EVT_GROUP(config);
612 cluster_pmu_set_evcntcr(idx, 0);
613 cluster_pmu_set_evtyper(idx, event_group);
614 cluster_pmu_set_resr(cluster, event_group, event_cc);
615 cluster_pmu_set_evfilter_sys_mode(idx);
618 cluster_pmu_counter_enable_interrupt(idx);
619 cluster_pmu_counter_enable(idx);
622 static void l2_cache_event_stop(struct perf_event *event, int flags)
624 struct hw_perf_event *hwc = &event->hw;
627 if (hwc->state & PERF_HES_STOPPED)
630 cluster_pmu_counter_disable_interrupt(idx);
631 cluster_pmu_counter_disable(idx);
633 if (flags & PERF_EF_UPDATE)
634 l2_cache_event_update(event);
635 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
638 static int l2_cache_event_add(struct perf_event *event, int flags)
640 struct hw_perf_event *hwc = &event->hw;
643 struct cluster_pmu *cluster;
645 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
647 idx = l2_cache_get_event_idx(cluster, event);
652 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
653 cluster->events[idx] = event;
654 local64_set(&hwc->prev_count, 0);
656 if (flags & PERF_EF_START)
657 l2_cache_event_start(event, flags);
659 /* Propagate changes to the userspace mapping. */
660 perf_event_update_userpage(event);
665 static void l2_cache_event_del(struct perf_event *event, int flags)
667 struct hw_perf_event *hwc = &event->hw;
668 struct cluster_pmu *cluster;
671 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu);
673 l2_cache_event_stop(event, flags | PERF_EF_UPDATE);
674 cluster->events[idx] = NULL;
675 l2_cache_clear_event_idx(cluster, event);
677 perf_event_update_userpage(event);
680 static void l2_cache_event_read(struct perf_event *event)
682 l2_cache_event_update(event);
685 static ssize_t l2_cache_pmu_cpumask_show(struct device *dev,
686 struct device_attribute *attr,
689 struct l2cache_pmu *l2cache_pmu = to_l2cache_pmu(dev_get_drvdata(dev));
691 return cpumap_print_to_pagebuf(true, buf, &l2cache_pmu->cpumask);
694 static struct device_attribute l2_cache_pmu_cpumask_attr =
695 __ATTR(cpumask, S_IRUGO, l2_cache_pmu_cpumask_show, NULL);
697 static struct attribute *l2_cache_pmu_cpumask_attrs[] = {
698 &l2_cache_pmu_cpumask_attr.attr,
702 static struct attribute_group l2_cache_pmu_cpumask_group = {
703 .attrs = l2_cache_pmu_cpumask_attrs,
706 /* CCG format for perf RAW codes. */
707 PMU_FORMAT_ATTR(l2_code, "config:4-11");
708 PMU_FORMAT_ATTR(l2_group, "config:0-3");
709 PMU_FORMAT_ATTR(event, "config:0-11");
711 static struct attribute *l2_cache_pmu_formats[] = {
712 &format_attr_l2_code.attr,
713 &format_attr_l2_group.attr,
714 &format_attr_event.attr,
718 static struct attribute_group l2_cache_pmu_format_group = {
720 .attrs = l2_cache_pmu_formats,
723 static ssize_t l2cache_pmu_event_show(struct device *dev,
724 struct device_attribute *attr, char *page)
726 struct perf_pmu_events_attr *pmu_attr;
728 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
729 return sprintf(page, "event=0x%02llx\n", pmu_attr->id);
732 #define L2CACHE_EVENT_ATTR(_name, _id) \
733 (&((struct perf_pmu_events_attr[]) { \
734 { .attr = __ATTR(_name, 0444, l2cache_pmu_event_show, NULL), \
738 static struct attribute *l2_cache_pmu_events[] = {
739 L2CACHE_EVENT_ATTR(cycles, L2_EVENT_CYCLES),
740 L2CACHE_EVENT_ATTR(dcache-ops, L2_EVENT_DCACHE_OPS),
741 L2CACHE_EVENT_ATTR(icache-ops, L2_EVENT_ICACHE_OPS),
742 L2CACHE_EVENT_ATTR(tlbi, L2_EVENT_TLBI),
743 L2CACHE_EVENT_ATTR(barriers, L2_EVENT_BARRIERS),
744 L2CACHE_EVENT_ATTR(total-reads, L2_EVENT_TOTAL_READS),
745 L2CACHE_EVENT_ATTR(total-writes, L2_EVENT_TOTAL_WRITES),
746 L2CACHE_EVENT_ATTR(total-requests, L2_EVENT_TOTAL_REQUESTS),
747 L2CACHE_EVENT_ATTR(ldrex, L2_EVENT_LDREX),
748 L2CACHE_EVENT_ATTR(strex, L2_EVENT_STREX),
749 L2CACHE_EVENT_ATTR(clrex, L2_EVENT_CLREX),
753 static struct attribute_group l2_cache_pmu_events_group = {
755 .attrs = l2_cache_pmu_events,
758 static const struct attribute_group *l2_cache_pmu_attr_grps[] = {
759 &l2_cache_pmu_format_group,
760 &l2_cache_pmu_cpumask_group,
761 &l2_cache_pmu_events_group,
766 * Generic device handlers
769 static const struct acpi_device_id l2_cache_pmu_acpi_match[] = {
774 static int get_num_counters(void)
778 val = get_l2_indirect_reg(L2PMCR);
781 * Read number of counters from L2PMCR and add 1
782 * for the cycle counter.
784 return ((val >> L2PMCR_NUM_EV_SHIFT) & L2PMCR_NUM_EV_MASK) + 1;
787 static struct cluster_pmu *l2_cache_associate_cpu_with_cluster(
788 struct l2cache_pmu *l2cache_pmu, int cpu)
792 struct cluster_pmu *cluster = NULL;
795 * This assumes that the cluster_id is in MPIDR[aff1] for
796 * single-threaded cores, and MPIDR[aff2] for multi-threaded
797 * cores. This logic will have to be updated if this changes.
799 mpidr = read_cpuid_mpidr();
800 if (mpidr & MPIDR_MT_BITMASK)
801 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
803 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
805 list_for_each_entry(cluster, &l2cache_pmu->clusters, next) {
806 if (cluster->cluster_id != cpu_cluster_id)
809 dev_info(&l2cache_pmu->pdev->dev,
810 "CPU%d associated with cluster %d\n", cpu,
811 cluster->cluster_id);
812 cpumask_set_cpu(cpu, &cluster->cluster_cpus);
813 *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu) = cluster;
820 static int l2cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
822 struct cluster_pmu *cluster;
823 struct l2cache_pmu *l2cache_pmu;
825 l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node);
826 cluster = get_cluster_pmu(l2cache_pmu, cpu);
828 /* First time this CPU has come online */
829 cluster = l2_cache_associate_cpu_with_cluster(l2cache_pmu, cpu);
831 /* Only if broken firmware doesn't list every cluster */
832 WARN_ONCE(1, "No L2 cache cluster for CPU%d\n", cpu);
837 /* If another CPU is managing this cluster, we're done */
838 if (cluster->on_cpu != -1)
842 * All CPUs on this cluster were down, use this one.
843 * Reset to put it into sane state.
845 cluster->on_cpu = cpu;
846 cpumask_set_cpu(cpu, &l2cache_pmu->cpumask);
849 WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(cpu)));
850 enable_irq(cluster->irq);
855 static int l2cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
857 struct cluster_pmu *cluster;
858 struct l2cache_pmu *l2cache_pmu;
859 cpumask_t cluster_online_cpus;
862 l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node);
863 cluster = get_cluster_pmu(l2cache_pmu, cpu);
867 /* If this CPU is not managing the cluster, we're done */
868 if (cluster->on_cpu != cpu)
871 /* Give up ownership of cluster */
872 cpumask_clear_cpu(cpu, &l2cache_pmu->cpumask);
873 cluster->on_cpu = -1;
875 /* Any other CPU for this cluster which is still online */
876 cpumask_and(&cluster_online_cpus, &cluster->cluster_cpus,
878 target = cpumask_any_but(&cluster_online_cpus, cpu);
879 if (target >= nr_cpu_ids) {
880 disable_irq(cluster->irq);
884 perf_pmu_migrate_context(&l2cache_pmu->pmu, cpu, target);
885 cluster->on_cpu = target;
886 cpumask_set_cpu(target, &l2cache_pmu->cpumask);
887 WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(target)));
892 static int l2_cache_pmu_probe_cluster(struct device *dev, void *data)
894 struct platform_device *pdev = to_platform_device(dev->parent);
895 struct platform_device *sdev = to_platform_device(dev);
896 struct l2cache_pmu *l2cache_pmu = data;
897 struct cluster_pmu *cluster;
898 struct acpi_device *device;
899 unsigned long fw_cluster_id;
903 if (acpi_bus_get_device(ACPI_HANDLE(dev), &device))
906 if (kstrtoul(device->pnp.unique_id, 10, &fw_cluster_id) < 0) {
907 dev_err(&pdev->dev, "unable to read ACPI uid\n");
911 cluster = devm_kzalloc(&pdev->dev, sizeof(*cluster), GFP_KERNEL);
915 INIT_LIST_HEAD(&cluster->next);
916 list_add(&cluster->next, &l2cache_pmu->clusters);
917 cluster->cluster_id = fw_cluster_id;
919 irq = platform_get_irq(sdev, 0);
922 "Failed to get valid irq for cluster %ld\n",
926 irq_set_status_flags(irq, IRQ_NOAUTOEN);
929 cluster->l2cache_pmu = l2cache_pmu;
930 cluster->on_cpu = -1;
932 err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq,
933 IRQF_NOBALANCING | IRQF_NO_THREAD,
934 "l2-cache-pmu", cluster);
937 "Unable to request IRQ%d for L2 PMU counters\n", irq);
942 "Registered L2 cache PMU cluster %ld\n", fw_cluster_id);
944 spin_lock_init(&cluster->pmu_lock);
946 l2cache_pmu->num_pmus++;
951 static int l2_cache_pmu_probe(struct platform_device *pdev)
954 struct l2cache_pmu *l2cache_pmu;
957 devm_kzalloc(&pdev->dev, sizeof(*l2cache_pmu), GFP_KERNEL);
961 INIT_LIST_HEAD(&l2cache_pmu->clusters);
963 platform_set_drvdata(pdev, l2cache_pmu);
964 l2cache_pmu->pmu = (struct pmu) {
965 /* suffix is instance id for future use with multiple sockets */
967 .task_ctx_nr = perf_invalid_context,
968 .pmu_enable = l2_cache_pmu_enable,
969 .pmu_disable = l2_cache_pmu_disable,
970 .event_init = l2_cache_event_init,
971 .add = l2_cache_event_add,
972 .del = l2_cache_event_del,
973 .start = l2_cache_event_start,
974 .stop = l2_cache_event_stop,
975 .read = l2_cache_event_read,
976 .attr_groups = l2_cache_pmu_attr_grps,
977 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
980 l2cache_pmu->num_counters = get_num_counters();
981 l2cache_pmu->pdev = pdev;
982 l2cache_pmu->pmu_cluster = devm_alloc_percpu(&pdev->dev,
983 struct cluster_pmu *);
984 if (!l2cache_pmu->pmu_cluster)
987 l2_cycle_ctr_idx = l2cache_pmu->num_counters - 1;
988 l2_counter_present_mask = GENMASK(l2cache_pmu->num_counters - 2, 0) |
989 BIT(L2CYCLE_CTR_BIT);
991 cpumask_clear(&l2cache_pmu->cpumask);
993 /* Read cluster info and initialize each cluster */
994 err = device_for_each_child(&pdev->dev, l2cache_pmu,
995 l2_cache_pmu_probe_cluster);
999 if (l2cache_pmu->num_pmus == 0) {
1000 dev_err(&pdev->dev, "No hardware L2 cache PMUs found\n");
1004 err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
1005 &l2cache_pmu->node);
1007 dev_err(&pdev->dev, "Error %d registering hotplug", err);
1011 err = perf_pmu_register(&l2cache_pmu->pmu, l2cache_pmu->pmu.name, -1);
1013 dev_err(&pdev->dev, "Error %d registering L2 cache PMU\n", err);
1014 goto out_unregister;
1017 dev_info(&pdev->dev, "Registered L2 cache PMU using %d HW PMUs\n",
1018 l2cache_pmu->num_pmus);
1023 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
1024 &l2cache_pmu->node);
1028 static int l2_cache_pmu_remove(struct platform_device *pdev)
1030 struct l2cache_pmu *l2cache_pmu =
1031 to_l2cache_pmu(platform_get_drvdata(pdev));
1033 perf_pmu_unregister(&l2cache_pmu->pmu);
1034 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
1035 &l2cache_pmu->node);
1039 static struct platform_driver l2_cache_pmu_driver = {
1041 .name = "qcom-l2cache-pmu",
1042 .acpi_match_table = ACPI_PTR(l2_cache_pmu_acpi_match),
1044 .probe = l2_cache_pmu_probe,
1045 .remove = l2_cache_pmu_remove,
1048 static int __init register_l2_cache_pmu_driver(void)
1052 err = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE,
1053 "AP_PERF_ARM_QCOM_L2_ONLINE",
1054 l2cache_pmu_online_cpu,
1055 l2cache_pmu_offline_cpu);
1059 return platform_driver_register(&l2_cache_pmu_driver);
1061 device_initcall(register_l2_cache_pmu_driver);