1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2018 Google, Inc. */
4 #include "gasket_interrupt.h"
6 #include "gasket_constants.h"
7 #include "gasket_core.h"
8 #include "gasket_sysfs.h"
9 #include <linux/device.h>
10 #include <linux/interrupt.h>
11 #include <linux/printk.h>
12 #ifdef GASKET_KERNEL_TRACE_SUPPORT
13 #define CREATE_TRACE_POINTS
14 #include <trace/events/gasket_interrupt.h>
16 #define trace_gasket_interrupt_event(x, ...)
18 /* Retry attempts if the requested number of interrupts aren't available. */
19 #define MSIX_RETRY_COUNT 3
21 /* Instance interrupt management data. */
22 struct gasket_interrupt_data {
23 /* The name associated with this interrupt data. */
26 /* Interrupt type. See gasket_interrupt_type in gasket_core.h */
29 /* The PCI device [if any] associated with the owning device. */
30 struct pci_dev *pci_dev;
32 /* Set to 1 if MSI-X has successfully been configred, 0 otherwise. */
35 /* The number of interrupts requested by the owning device. */
38 /* A pointer to the interrupt descriptor struct for this device. */
39 const struct gasket_interrupt_desc *interrupts;
41 /* The index of the bar into which interrupts should be mapped. */
42 int interrupt_bar_index;
44 /* The width of a single interrupt in a packed interrupt register. */
48 * Design-wise, these elements should be bundled together, but
49 * pci_enable_msix's interface requires that they be managed
50 * individually (requires array of struct msix_entry).
53 /* The number of successfully configured interrupts. */
56 /* The MSI-X data for each requested/configured interrupt. */
57 struct msix_entry *msix_entries;
59 /* The eventfd "callback" data for each interrupt. */
60 struct eventfd_ctx **eventfd_ctxs;
62 /* The number of times each interrupt has been called. */
63 ulong *interrupt_counts;
65 /* Linux IRQ number. */
69 /* Structures to display interrupt counts in sysfs. */
70 enum interrupt_sysfs_attribute_type {
71 ATTR_INTERRUPT_COUNTS,
74 /* Set up device registers for interrupt handling. */
75 static void gasket_interrupt_setup(struct gasket_dev *gasket_dev)
81 struct gasket_interrupt_data *interrupt_data =
82 gasket_dev->interrupt_data;
84 if (!interrupt_data) {
85 dev_dbg(gasket_dev->dev, "Interrupt data is not initialized\n");
89 dev_dbg(gasket_dev->dev, "Running interrupt setup\n");
91 /* Setup the MSIX table. */
93 for (i = 0; i < interrupt_data->num_interrupts; i++) {
95 * If the interrupt is not packed, we can write the index into
96 * the register directly. If not, we need to deal with a read-
97 * modify-write and shift based on the packing index.
99 dev_dbg(gasket_dev->dev,
100 "Setting up interrupt index %d with index 0x%llx and packing %d\n",
101 interrupt_data->interrupts[i].index,
102 interrupt_data->interrupts[i].reg,
103 interrupt_data->interrupts[i].packing);
104 if (interrupt_data->interrupts[i].packing == UNPACKED) {
105 value = interrupt_data->interrupts[i].index;
107 switch (interrupt_data->interrupts[i].packing) {
112 pack_shift = interrupt_data->pack_width;
115 pack_shift = 2 * interrupt_data->pack_width;
118 pack_shift = 3 * interrupt_data->pack_width;
121 dev_dbg(gasket_dev->dev,
122 "Found interrupt description with unknown enum %d\n",
123 interrupt_data->interrupts[i].packing);
127 mask = ~(0xFFFF << pack_shift);
128 value = gasket_dev_read_64(gasket_dev,
129 interrupt_data->interrupt_bar_index,
130 interrupt_data->interrupts[i].reg);
132 value |= interrupt_data->interrupts[i].index
135 gasket_dev_write_64(gasket_dev, value,
136 interrupt_data->interrupt_bar_index,
137 interrupt_data->interrupts[i].reg);
142 gasket_handle_interrupt(struct gasket_interrupt_data *interrupt_data,
145 struct eventfd_ctx *ctx;
147 trace_gasket_interrupt_event(interrupt_data->name, interrupt_index);
148 ctx = interrupt_data->eventfd_ctxs[interrupt_index];
150 eventfd_signal(ctx, 1);
152 ++(interrupt_data->interrupt_counts[interrupt_index]);
155 static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id)
157 struct gasket_interrupt_data *interrupt_data = dev_id;
161 /* If this linear lookup is a problem, we can maintain a map/hash. */
162 for (i = 0; i < interrupt_data->num_interrupts; i++) {
163 if (interrupt_data->msix_entries[i].vector == irq) {
164 interrupt = interrupt_data->msix_entries[i].entry;
168 if (interrupt == -1) {
169 pr_err("Received unknown irq %d\n", irq);
172 gasket_handle_interrupt(interrupt_data, interrupt);
177 gasket_interrupt_msix_init(struct gasket_interrupt_data *interrupt_data)
182 interrupt_data->msix_entries =
183 kcalloc(interrupt_data->num_interrupts,
184 sizeof(*interrupt_data->msix_entries), GFP_KERNEL);
185 if (!interrupt_data->msix_entries)
188 for (i = 0; i < interrupt_data->num_interrupts; i++) {
189 interrupt_data->msix_entries[i].entry = i;
190 interrupt_data->msix_entries[i].vector = 0;
191 interrupt_data->eventfd_ctxs[i] = NULL;
194 /* Retry MSIX_RETRY_COUNT times if not enough IRQs are available. */
195 for (i = 0; i < MSIX_RETRY_COUNT && ret > 0; i++)
196 ret = pci_enable_msix_exact(interrupt_data->pci_dev,
197 interrupt_data->msix_entries,
198 interrupt_data->num_interrupts);
201 return ret > 0 ? -EBUSY : ret;
202 interrupt_data->msix_configured = 1;
204 for (i = 0; i < interrupt_data->num_interrupts; i++) {
205 ret = request_irq(interrupt_data->msix_entries[i].vector,
206 gasket_msix_interrupt_handler, 0,
207 interrupt_data->name, interrupt_data);
210 dev_err(&interrupt_data->pci_dev->dev,
211 "Cannot get IRQ for interrupt %d, vector %d; "
213 i, interrupt_data->msix_entries[i].vector, ret);
217 interrupt_data->num_configured++;
224 * On QCM DragonBoard, we exit gasket_interrupt_msix_init() and kernel interrupt
225 * setup code with MSIX vectors masked. This is wrong because nothing else in
226 * the driver will normally touch the MSIX vectors.
228 * As a temporary hack, force unmasking there.
230 * TODO: Figure out why QCM kernel doesn't unmask the MSIX vectors, after
231 * gasket_interrupt_msix_init(), and remove this code.
233 static void force_msix_interrupt_unmasking(struct gasket_dev *gasket_dev)
236 #define MSIX_VECTOR_SIZE 16
237 #define MSIX_MASK_BIT_OFFSET 12
238 #define APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE 0x46800
239 for (i = 0; i < gasket_dev->interrupt_data->num_configured; i++) {
240 /* Check if the MSIX vector is unmasked */
241 ulong location = APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE +
242 MSIX_MASK_BIT_OFFSET + i * MSIX_VECTOR_SIZE;
244 gasket_dev_read_32(gasket_dev,
245 gasket_dev->interrupt_data->interrupt_bar_index,
249 /* Unmask the msix vector (clear 32 bits) */
250 gasket_dev_write_32(gasket_dev, 0,
251 gasket_dev->interrupt_data->interrupt_bar_index,
254 #undef MSIX_VECTOR_SIZE
255 #undef MSIX_MASK_BIT_OFFSET
256 #undef APEX_BAR2_REG_KERNEL_HIB_MSIX_TABLE
259 static ssize_t interrupt_sysfs_show(struct device *device,
260 struct device_attribute *attr, char *buf)
263 ssize_t written = 0, total_written = 0;
264 struct gasket_interrupt_data *interrupt_data;
265 struct gasket_dev *gasket_dev;
266 struct gasket_sysfs_attribute *gasket_attr;
267 enum interrupt_sysfs_attribute_type sysfs_type;
269 gasket_dev = gasket_sysfs_get_device_data(device);
271 dev_dbg(device, "No sysfs mapping found for device\n");
275 gasket_attr = gasket_sysfs_get_attr(device, attr);
277 dev_dbg(device, "No sysfs attr data found for device\n");
278 gasket_sysfs_put_device_data(device, gasket_dev);
282 sysfs_type = (enum interrupt_sysfs_attribute_type)
283 gasket_attr->data.attr_type;
284 interrupt_data = gasket_dev->interrupt_data;
285 switch (sysfs_type) {
286 case ATTR_INTERRUPT_COUNTS:
287 for (i = 0; i < interrupt_data->num_interrupts; ++i) {
289 scnprintf(buf, PAGE_SIZE - total_written,
291 interrupt_data->interrupt_counts[i]);
292 total_written += written;
298 dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n",
304 gasket_sysfs_put_attr(device, gasket_attr);
305 gasket_sysfs_put_device_data(device, gasket_dev);
309 static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = {
310 GASKET_SYSFS_RO(interrupt_counts, interrupt_sysfs_show,
311 ATTR_INTERRUPT_COUNTS),
312 GASKET_END_OF_ATTR_ARRAY,
315 int gasket_interrupt_init(struct gasket_dev *gasket_dev)
318 struct gasket_interrupt_data *interrupt_data;
319 const struct gasket_driver_desc *driver_desc =
320 gasket_get_driver_desc(gasket_dev);
322 interrupt_data = kzalloc(sizeof(*interrupt_data), GFP_KERNEL);
325 gasket_dev->interrupt_data = interrupt_data;
326 interrupt_data->name = driver_desc->name;
327 interrupt_data->type = driver_desc->interrupt_type;
328 interrupt_data->pci_dev = gasket_dev->pci_dev;
329 interrupt_data->num_interrupts = driver_desc->num_interrupts;
330 interrupt_data->interrupts = driver_desc->interrupts;
331 interrupt_data->interrupt_bar_index = driver_desc->interrupt_bar_index;
332 interrupt_data->pack_width = driver_desc->interrupt_pack_width;
333 interrupt_data->num_configured = 0;
335 interrupt_data->eventfd_ctxs =
336 kcalloc(driver_desc->num_interrupts,
337 sizeof(*interrupt_data->eventfd_ctxs), GFP_KERNEL);
338 if (!interrupt_data->eventfd_ctxs) {
339 kfree(interrupt_data);
343 interrupt_data->interrupt_counts =
344 kcalloc(driver_desc->num_interrupts,
345 sizeof(*interrupt_data->interrupt_counts), GFP_KERNEL);
346 if (!interrupt_data->interrupt_counts) {
347 kfree(interrupt_data->eventfd_ctxs);
348 kfree(interrupt_data);
352 switch (interrupt_data->type) {
354 ret = gasket_interrupt_msix_init(interrupt_data);
357 force_msix_interrupt_unmasking(gasket_dev);
365 /* Failing to setup interrupts will cause the device to report
366 * GASKET_STATUS_LAMED. But it is not fatal.
368 dev_warn(gasket_dev->dev,
369 "Couldn't initialize interrupts: %d\n", ret);
373 gasket_interrupt_setup(gasket_dev);
374 gasket_sysfs_create_entries(gasket_dev->dev_info.device,
375 interrupt_sysfs_attrs);
381 gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data)
385 for (i = 0; i < interrupt_data->num_configured; i++)
386 free_irq(interrupt_data->msix_entries[i].vector,
388 interrupt_data->num_configured = 0;
390 if (interrupt_data->msix_configured)
391 pci_disable_msix(interrupt_data->pci_dev);
392 interrupt_data->msix_configured = 0;
393 kfree(interrupt_data->msix_entries);
396 int gasket_interrupt_reinit(struct gasket_dev *gasket_dev)
400 if (!gasket_dev->interrupt_data) {
401 dev_dbg(gasket_dev->dev,
402 "Attempted to reinit uninitialized interrupt data\n");
406 switch (gasket_dev->interrupt_data->type) {
408 gasket_interrupt_msix_cleanup(gasket_dev->interrupt_data);
409 ret = gasket_interrupt_msix_init(gasket_dev->interrupt_data);
412 force_msix_interrupt_unmasking(gasket_dev);
420 /* Failing to setup interrupts will cause the device
421 * to report GASKET_STATUS_LAMED, but is not fatal.
423 dev_warn(gasket_dev->dev, "Couldn't reinit interrupts: %d\n",
428 gasket_interrupt_setup(gasket_dev);
433 /* See gasket_interrupt.h for description. */
434 int gasket_interrupt_reset_counts(struct gasket_dev *gasket_dev)
436 dev_dbg(gasket_dev->dev, "Clearing interrupt counts\n");
437 memset(gasket_dev->interrupt_data->interrupt_counts, 0,
438 gasket_dev->interrupt_data->num_interrupts *
439 sizeof(*gasket_dev->interrupt_data->interrupt_counts));
443 /* See gasket_interrupt.h for description. */
444 void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev)
446 struct gasket_interrupt_data *interrupt_data =
447 gasket_dev->interrupt_data;
449 * It is possible to get an error code from gasket_interrupt_init
450 * before interrupt_data has been allocated, so check it.
455 switch (interrupt_data->type) {
457 gasket_interrupt_msix_cleanup(interrupt_data);
464 kfree(interrupt_data->interrupt_counts);
465 kfree(interrupt_data->eventfd_ctxs);
466 kfree(interrupt_data);
467 gasket_dev->interrupt_data = NULL;
470 int gasket_interrupt_system_status(struct gasket_dev *gasket_dev)
472 if (!gasket_dev->interrupt_data) {
473 dev_dbg(gasket_dev->dev, "Interrupt data is null\n");
474 return GASKET_STATUS_DEAD;
477 if (gasket_dev->interrupt_data->num_configured !=
478 gasket_dev->interrupt_data->num_interrupts) {
479 dev_dbg(gasket_dev->dev,
480 "Not all interrupts were configured\n");
481 return GASKET_STATUS_LAMED;
484 return GASKET_STATUS_ALIVE;
487 int gasket_interrupt_set_eventfd(struct gasket_interrupt_data *interrupt_data,
488 int interrupt, int event_fd)
490 struct eventfd_ctx *ctx = eventfd_ctx_fdget(event_fd);
495 if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts)
498 interrupt_data->eventfd_ctxs[interrupt] = ctx;
502 int gasket_interrupt_clear_eventfd(struct gasket_interrupt_data *interrupt_data,
505 if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts)
508 interrupt_data->eventfd_ctxs[interrupt] = NULL;