u32 nr_redist_regions;
u64 flags;
bool has_rss;
- unsigned int irq_nr;
- struct partition_desc *ppi_descs[16];
+ unsigned int ppi_nr;
+ struct partition_desc **ppi_descs;
};
static struct gic_chip_data gic_data __read_mostly;
static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
+#define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
+#define GIC_LINE_NR max(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U)
+#define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer)
+
/*
* The behaviours of RPR and PMR registers differ depending on the value of
* SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the
static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
/* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */
-static refcount_t ppi_nmi_refs[16];
+static refcount_t *ppi_nmi_refs;
static struct gic_kvm_info gic_v3_kvm_info;
static DEFINE_PER_CPU(bool, has_rss);
/* Our default, arbitrary priority value. Linux only uses one anyway. */
#define DEFAULT_PMR_VALUE 0xf0
+enum gic_intid_range {
+ PPI_RANGE,
+ SPI_RANGE,
+ EPPI_RANGE,
+ ESPI_RANGE,
+ LPI_RANGE,
+ __INVALID_RANGE__
+};
+
+static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)
+{
+ switch (hwirq) {
+ case 16 ... 31:
+ return PPI_RANGE;
+ case 32 ... 1019:
+ return SPI_RANGE;
+ case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63):
+ return EPPI_RANGE;
+ case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023):
+ return ESPI_RANGE;
+ case 8192 ... GENMASK(23, 0):
+ return LPI_RANGE;
+ default:
+ return __INVALID_RANGE__;
+ }
+}
+
+static enum gic_intid_range get_intid_range(struct irq_data *d)
+{
+ return __get_intid_range(d->hwirq);
+}
+
static inline unsigned int gic_irq(struct irq_data *d)
{
return d->hwirq;
static inline int gic_irq_in_rdist(struct irq_data *d)
{
- return gic_irq(d) < 32;
+ enum gic_intid_range range = get_intid_range(d);
+ return range == PPI_RANGE || range == EPPI_RANGE;
}
static inline void __iomem *gic_dist_base(struct irq_data *d)
{
- if (gic_irq_in_rdist(d)) /* SGI+PPI -> SGI_base for this CPU */
+ switch (get_intid_range(d)) {
+ case PPI_RANGE:
+ case EPPI_RANGE:
+ /* SGI+PPI -> SGI_base for this CPU */
return gic_data_rdist_sgi_base();
- if (d->hwirq <= 1023) /* SPI -> dist_base */
+ case SPI_RANGE:
+ case ESPI_RANGE:
+ /* SPI -> dist_base */
return gic_data.dist_base;
- return NULL;
+ default:
+ return NULL;
+ }
}
static void gic_do_wait_for_rwp(void __iomem *base)
/*
* Routines to disable, enable, EOI and route interrupts
*/
+static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index)
+{
+ switch (get_intid_range(d)) {
+ case PPI_RANGE:
+ case SPI_RANGE:
+ *index = d->hwirq;
+ return offset;
+ case EPPI_RANGE:
+ /*
+ * Contrary to the ESPI range, the EPPI range is contiguous
+ * to the PPI range in the registers, so let's adjust the
+ * displacement accordingly. Consistency is overrated.
+ */
+ *index = d->hwirq - EPPI_BASE_INTID + 32;
+ return offset;
+ case ESPI_RANGE:
+ *index = d->hwirq - ESPI_BASE_INTID;
+ switch (offset) {
+ case GICD_ISENABLER:
+ return GICD_ISENABLERnE;
+ case GICD_ICENABLER:
+ return GICD_ICENABLERnE;
+ case GICD_ISPENDR:
+ return GICD_ISPENDRnE;
+ case GICD_ICPENDR:
+ return GICD_ICPENDRnE;
+ case GICD_ISACTIVER:
+ return GICD_ISACTIVERnE;
+ case GICD_ICACTIVER:
+ return GICD_ICACTIVERnE;
+ case GICD_IPRIORITYR:
+ return GICD_IPRIORITYRnE;
+ case GICD_ICFGR:
+ return GICD_ICFGRnE;
+ case GICD_IROUTER:
+ return GICD_IROUTERnE;
+ default:
+ break;
+ }
+ break;
+ default:
+ break;
+ }
+
+ WARN_ON(1);
+ *index = d->hwirq;
+ return offset;
+}
+
static int gic_peek_irq(struct irq_data *d, u32 offset)
{
- u32 mask = 1 << (gic_irq(d) % 32);
void __iomem *base;
+ u32 index, mask;
+
+ offset = convert_offset_index(d, offset, &index);
+ mask = 1 << (index % 32);
if (gic_irq_in_rdist(d))
base = gic_data_rdist_sgi_base();
else
base = gic_data.dist_base;
- return !!(readl_relaxed(base + offset + (gic_irq(d) / 32) * 4) & mask);
+ return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);
}
static void gic_poke_irq(struct irq_data *d, u32 offset)
{
- u32 mask = 1 << (gic_irq(d) % 32);
void (*rwp_wait)(void);
void __iomem *base;
+ u32 index, mask;
+
+ offset = convert_offset_index(d, offset, &index);
+ mask = 1 << (index % 32);
if (gic_irq_in_rdist(d)) {
base = gic_data_rdist_sgi_base();
rwp_wait = gic_dist_wait_for_rwp;
}
- writel_relaxed(mask, base + offset + (gic_irq(d) / 32) * 4);
+ writel_relaxed(mask, base + offset + (index / 32) * 4);
rwp_wait();
}
{
u32 reg;
- if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
+ if (d->hwirq >= 8192) /* PPI/SPI only */
return -EINVAL;
switch (which) {
static int gic_irq_get_irqchip_state(struct irq_data *d,
enum irqchip_irq_state which, bool *val)
{
- if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */
+ if (d->hwirq >= 8192) /* PPI/SPI only */
return -EINVAL;
switch (which) {
static void gic_irq_set_prio(struct irq_data *d, u8 prio)
{
void __iomem *base = gic_dist_base(d);
+ u32 offset, index;
+
+ offset = convert_offset_index(d, GICD_IPRIORITYR, &index);
- writeb_relaxed(prio, base + GICD_IPRIORITYR + gic_irq(d));
+ writeb_relaxed(prio, base + offset + index);
+}
+
+static u32 gic_get_ppi_index(struct irq_data *d)
+{
+ switch (get_intid_range(d)) {
+ case PPI_RANGE:
+ return d->hwirq - 16;
+ case EPPI_RANGE:
+ return d->hwirq - EPPI_BASE_INTID + 16;
+ default:
+ unreachable();
+ }
}
static int gic_irq_nmi_setup(struct irq_data *d)
return -EINVAL;
/* desc lock should already be held */
- if (gic_irq(d) < 32) {
+ if (gic_irq_in_rdist(d)) {
+ u32 idx = gic_get_ppi_index(d);
+
/* Setting up PPI as NMI, only switch handler for first NMI */
- if (!refcount_inc_not_zero(&ppi_nmi_refs[gic_irq(d) - 16])) {
- refcount_set(&ppi_nmi_refs[gic_irq(d) - 16], 1);
+ if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) {
+ refcount_set(&ppi_nmi_refs[idx], 1);
desc->handle_irq = handle_percpu_devid_fasteoi_nmi;
}
} else {
return;
/* desc lock should already be held */
- if (gic_irq(d) < 32) {
+ if (gic_irq_in_rdist(d)) {
+ u32 idx = gic_get_ppi_index(d);
+
/* Tearing down NMI, only switch handler for last NMI */
- if (refcount_dec_and_test(&ppi_nmi_refs[gic_irq(d) - 16]))
+ if (refcount_dec_and_test(&ppi_nmi_refs[idx]))
desc->handle_irq = handle_percpu_devid_irq;
} else {
desc->handle_irq = handle_fasteoi_irq;
static int gic_set_type(struct irq_data *d, unsigned int type)
{
+ enum gic_intid_range range;
unsigned int irq = gic_irq(d);
void (*rwp_wait)(void);
void __iomem *base;
+ u32 offset, index;
+ int ret;
/* Interrupt configuration for SGIs can't be changed */
if (irq < 16)
return -EINVAL;
+ range = get_intid_range(d);
+
/* SPIs have restrictions on the supported types */
- if (irq >= 32 && type != IRQ_TYPE_LEVEL_HIGH &&
- type != IRQ_TYPE_EDGE_RISING)
+ if ((range == SPI_RANGE || range == ESPI_RANGE) &&
+ type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
return -EINVAL;
if (gic_irq_in_rdist(d)) {
rwp_wait = gic_dist_wait_for_rwp;
}
- return gic_configure_irq(irq, type, base, rwp_wait);
+ offset = convert_offset_index(d, GICD_ICFGR, &index);
+
+ ret = gic_configure_irq(index, type, base + offset, rwp_wait);
+ if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) {
+ /* Misconfigured PPIs are usually not fatal */
+ pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq);
+ ret = 0;
+ }
+
+ return ret;
}
static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
gic_arch_enable_irqs();
}
- if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) {
+ /* Check for special IDs first */
+ if ((irqnr >= 1020 && irqnr <= 1023))
+ return;
+
+ /* Treat anything but SGIs in a uniform way */
+ if (likely(irqnr > 15)) {
int err;
if (static_branch_likely(&supports_deactivate_key))
* do the right thing if the kernel is running in secure mode,
* but that's not the intended use case anyway.
*/
- for (i = 32; i < gic_data.irq_nr; i += 32)
+ for (i = 32; i < GIC_LINE_NR; i += 32)
writel_relaxed(~0, base + GICD_IGROUPR + i / 8);
- gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp);
+ /* Extended SPI range, not handled by the GICv2/GICv3 common code */
+ for (i = 0; i < GIC_ESPI_NR; i += 32) {
+ writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8);
+ writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8);
+ }
+
+ for (i = 0; i < GIC_ESPI_NR; i += 32)
+ writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8);
+
+ for (i = 0; i < GIC_ESPI_NR; i += 16)
+ writel_relaxed(0, base + GICD_ICFGRnE + i / 4);
+
+ for (i = 0; i < GIC_ESPI_NR; i += 4)
+ writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i);
+
+ /* Now do the common stuff, and wait for the distributor to drain */
+ gic_dist_config(base, GIC_LINE_NR, gic_dist_wait_for_rwp);
/* Enable distributor with ARE, Group1 */
writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
* enabled.
*/
affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
- for (i = 32; i < gic_data.irq_nr; i++)
+ for (i = 32; i < GIC_LINE_NR; i++)
gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
+
+ for (i = 0; i < GIC_ESPI_NR; i++)
+ gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8);
}
static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
return -ENODEV;
}
-static int __gic_update_vlpi_properties(struct redist_region *region,
- void __iomem *ptr)
+static int __gic_update_rdist_properties(struct redist_region *region,
+ void __iomem *ptr)
{
u64 typer = gic_read_typer(ptr + GICR_TYPER);
gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);
gic_data.rdists.has_direct_lpi &= !!(typer & GICR_TYPER_DirectLPIS);
+ gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr);
return 1;
}
-static void gic_update_vlpi_properties(void)
+static void gic_update_rdist_properties(void)
{
- gic_iterate_rdists(__gic_update_vlpi_properties);
+ gic_data.ppi_nr = UINT_MAX;
+ gic_iterate_rdists(__gic_update_rdist_properties);
+ if (WARN_ON(gic_data.ppi_nr == UINT_MAX))
+ gic_data.ppi_nr = 0;
+ pr_info("%d PPIs implemented\n", gic_data.ppi_nr);
pr_info("%sVLPI support, %sdirect LPI support\n",
!gic_data.rdists.has_vlpis ? "no " : "",
!gic_data.rdists.has_direct_lpi ? "no " : "");
static void gic_cpu_init(void)
{
void __iomem *rbase;
+ int i;
/* Register ourselves with the rest of the world */
if (gic_populate_rdist())
gic_enable_redist(true);
+ WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) &&
+ !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange),
+ "Distributor has extended ranges, but CPU%d doesn't\n",
+ smp_processor_id());
+
rbase = gic_data_rdist_sgi_base();
/* Configure SGIs/PPIs as non-secure Group-1 */
- writel_relaxed(~0, rbase + GICR_IGROUPR0);
+ for (i = 0; i < gic_data.ppi_nr + 16; i += 32)
+ writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8);
- gic_cpu_config(rbase, gic_redist_wait_for_rwp);
+ gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp);
/* initialise system registers */
gic_cpu_sys_reg_init();
bool force)
{
unsigned int cpu;
+ u32 offset, index;
void __iomem *reg;
int enabled;
u64 val;
if (enabled)
gic_mask_irq(d);
- reg = gic_dist_base(d) + GICD_IROUTER + (gic_irq(d) * 8);
+ offset = convert_offset_index(d, GICD_IROUTER, &index);
+ reg = gic_dist_base(d) + offset + (index * 8);
val = gic_mpidr_to_affinity(cpu_logical_map(cpu));
gic_write_irouter(val, reg);
IRQCHIP_MASK_ON_SUSPEND,
};
-#define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
-
static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
irq_hw_number_t hw)
{
if (static_branch_likely(&supports_deactivate_key))
chip = &gic_eoimode1_chip;
- /* SGIs are private to the core kernel */
- if (hw < 16)
- return -EPERM;
- /* Nothing here */
- if (hw >= gic_data.irq_nr && hw < 8192)
- return -EPERM;
- /* Off limits */
- if (hw >= GIC_ID_NR)
- return -EPERM;
-
- /* PPIs */
- if (hw < 32) {
+ switch (__get_intid_range(hw)) {
+ case PPI_RANGE:
+ case EPPI_RANGE:
irq_set_percpu_devid(irq);
irq_domain_set_info(d, irq, hw, chip, d->host_data,
handle_percpu_devid_irq, NULL, NULL);
irq_set_status_flags(irq, IRQ_NOAUTOEN);
- }
- /* SPIs */
- if (hw >= 32 && hw < gic_data.irq_nr) {
+ break;
+
+ case SPI_RANGE:
+ case ESPI_RANGE:
irq_domain_set_info(d, irq, hw, chip, d->host_data,
handle_fasteoi_irq, NULL, NULL);
irq_set_probe(irq);
irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
- }
- /* LPIs */
- if (hw >= 8192 && hw < GIC_ID_NR) {
+ break;
+
+ case LPI_RANGE:
if (!gic_dist_supports_lpis())
return -EPERM;
irq_domain_set_info(d, irq, hw, chip, d->host_data,
handle_fasteoi_irq, NULL, NULL);
+ break;
+
+ default:
+ return -EPERM;
}
return 0;
*hwirq = fwspec->param[1] + 32;
break;
case 1: /* PPI */
- case GIC_IRQ_TYPE_PARTITION:
*hwirq = fwspec->param[1] + 16;
break;
+ case 2: /* ESPI */
+ *hwirq = fwspec->param[1] + ESPI_BASE_INTID;
+ break;
+ case 3: /* EPPI */
+ *hwirq = fwspec->param[1] + EPPI_BASE_INTID;
+ break;
case GIC_IRQ_TYPE_LPI: /* LPI */
*hwirq = fwspec->param[1];
break;
+ case GIC_IRQ_TYPE_PARTITION:
+ *hwirq = fwspec->param[1];
+ if (fwspec->param[1] >= 16)
+ *hwirq += EPPI_BASE_INTID - 16;
+ else
+ *hwirq += 16;
+ break;
default:
return -EINVAL;
}
* then we need to match the partition domain.
*/
if (fwspec->param_count >= 4 &&
- fwspec->param[0] == 1 && fwspec->param[3] != 0)
+ fwspec->param[0] == 1 && fwspec->param[3] != 0 &&
+ gic_data.ppi_descs)
return d == partition_get_domain(gic_data.ppi_descs[fwspec->param[1]]);
return d == gic_data.domain;
struct device_node *np;
int ret;
+ if (!gic_data.ppi_descs)
+ return -ENOMEM;
+
np = of_find_node_by_phandle(fwspec->param[3]);
if (WARN_ON(!np))
return -EINVAL;
return true;
}
+static bool gic_enable_quirk_hip06_07(void *data)
+{
+ struct gic_chip_data *d = data;
+
+ /*
+ * HIP06 GICD_IIDR clashes with GIC-600 product number (despite
+ * not being an actual ARM implementation). The saving grace is
+ * that GIC-600 doesn't have ESPI, so nothing to do in that case.
+ * HIP07 doesn't even have a proper IIDR, and still pretends to
+ * have ESPI. In both cases, put them right.
+ */
+ if (d->rdists.gicd_typer & GICD_TYPER_ESPI) {
+ /* Zero both ESPI and the RES0 field next to it... */
+ d->rdists.gicd_typer &= ~GENMASK(9, 8);
+ return true;
+ }
+
+ return false;
+}
+
+static const struct gic_quirk gic_quirks[] = {
+ {
+ .desc = "GICv3: Qualcomm MSM8996 broken firmware",
+ .compatible = "qcom,msm8996-gic-v3",
+ .init = gic_enable_quirk_msm8996,
+ },
+ {
+ .desc = "GICv3: HIP06 erratum 161010803",
+ .iidr = 0x0204043b,
+ .mask = 0xffffffff,
+ .init = gic_enable_quirk_hip06_07,
+ },
+ {
+ .desc = "GICv3: HIP07 erratum 161010803",
+ .iidr = 0x00000000,
+ .mask = 0xffffffff,
+ .init = gic_enable_quirk_hip06_07,
+ },
+ {
+ }
+};
+
static void gic_enable_nmi_support(void)
{
int i;
- for (i = 0; i < 16; i++)
+ if (!gic_prio_masking_enabled())
+ return;
+
+ if (gic_has_group0() && !gic_dist_security_disabled()) {
+ pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n");
+ return;
+ }
+
+ ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL);
+ if (!ppi_nmi_refs)
+ return;
+
+ for (i = 0; i < gic_data.ppi_nr; i++)
refcount_set(&ppi_nmi_refs[i], 0);
static_branch_enable(&supports_pseudo_nmis);
struct fwnode_handle *handle)
{
u32 typer;
- int gic_irqs;
int err;
if (!is_hyp_mode_available())
/*
* Find out how many interrupts are supported.
- * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI)
*/
typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
gic_data.rdists.gicd_typer = typer;
- gic_irqs = GICD_TYPER_IRQS(typer);
- if (gic_irqs > 1020)
- gic_irqs = 1020;
- gic_data.irq_nr = gic_irqs;
+ gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR),
+ gic_quirks, &gic_data);
+
+ pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32);
+ pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR);
gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
&gic_data);
irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);
set_handle_irq(gic_handle_irq);
- gic_update_vlpi_properties();
+ gic_update_rdist_properties();
gic_smp_init();
gic_dist_init();
gicv2m_init(handle, gic_data.domain);
}
- if (gic_prio_masking_enabled()) {
- if (!gic_has_group0() || gic_dist_security_disabled())
- gic_enable_nmi_support();
- else
- pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n");
- }
+ gic_enable_nmi_support();
return 0;
if (!parts_node)
return;
+ gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL);
+ if (!gic_data.ppi_descs)
+ return;
+
nr_parts = of_get_child_count(parts_node);
if (!nr_parts)
part_idx++;
}
- for (i = 0; i < 16; i++) {
+ for (i = 0; i < gic_data.ppi_nr; i++) {
unsigned int irq;
struct partition_desc *desc;
struct irq_fwspec ppi_fwspec = {
gic_set_kvm_info(&gic_v3_kvm_info);
}
-static const struct gic_quirk gic_quirks[] = {
- {
- .desc = "GICv3: Qualcomm MSM8996 broken firmware",
- .compatible = "qcom,msm8996-gic-v3",
- .init = gic_enable_quirk_msm8996,
- },
- {
- }
-};
-
static int __init gic_of_init(struct device_node *node, struct device_node *parent)
{
void __iomem *dist_base;
if (err)
goto out_redist_unmap;
- domain_handle = irq_domain_alloc_fwnode(acpi_data.dist_base);
+ domain_handle = irq_domain_alloc_fwnode(&dist->base_address);
if (!domain_handle) {
err = -ENOMEM;
goto out_redist_unmap;
projects / linux.git / blobdiff