drivers/irqchip/irq-gic-v3.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #define pr_fmt(fmt)     "GICv3: " fmt
   8
   9 #include <linux/acpi.h>
  10 #include <linux/cpu.h>
  11 #include <linux/cpu_pm.h>
  12 #include <linux/delay.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/irqdomain.h>
  15 #include <linux/of.h>
  16 #include <linux/of_address.h>
  17 #include <linux/of_irq.h>
  18 #include <linux/percpu.h>
  19 #include <linux/refcount.h>
  20 #include <linux/slab.h>
  21
  22 #include <linux/irqchip.h>
  23 #include <linux/irqchip/arm-gic-common.h>
  24 #include <linux/irqchip/arm-gic-v3.h>
  25 #include <linux/irqchip/irq-partition-percpu.h>
  26
  27 #include <asm/cputype.h>
  28 #include <asm/exception.h>
  29 #include <asm/smp_plat.h>
  30 #include <asm/virt.h>
  31
  32 #include "irq-gic-common.h"
  33
  34 #define GICD_INT_NMI_PRI        (GICD_INT_DEF_PRI & ~0x80)
  35
  36 #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996     (1ULL << 0)
  37
  38 struct redist_region {
  39         void __iomem            *redist_base;
  40         phys_addr_t             phys_base;
  41         bool                    single_redist;
  42 };
  43
  44 struct gic_chip_data {
  45         struct fwnode_handle    *fwnode;
  46         void __iomem            *dist_base;
  47         struct redist_region    *redist_regions;
  48         struct rdists           rdists;
  49         struct irq_domain       *domain;
  50         u64                     redist_stride;
  51         u32                     nr_redist_regions;
  52         u64                     flags;
  53         bool                    has_rss;
  54         unsigned int            ppi_nr;
  55         struct partition_desc   **ppi_descs;
  56 };
  57
  58 static struct gic_chip_data gic_data __read_mostly;
  59 static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);
  60
  61 #define GIC_ID_NR       (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
  62 #define GIC_LINE_NR     min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U)
  63 #define GIC_ESPI_NR     GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer)
  64
  65 /*
  66  * The behaviours of RPR and PMR registers differ depending on the value of
  67  * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the
  68  * distributor and redistributors depends on whether security is enabled in the
  69  * GIC.
  70  *
  71  * When security is enabled, non-secure priority values from the (re)distributor
  72  * are presented to the GIC CPUIF as follow:
  73  *     (GIC_(R)DIST_PRI[irq] >> 1) | 0x80;
  74  *
  75  * If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure
  76  * EL1 are subject to a similar operation thus matching the priorities presented
  77  * from the (re)distributor when security is enabled.
  78  *
  79  * see GICv3/GICv4 Architecture Specification (IHI0069D):
  80  * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt
  81  *   priorities.
  82  * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1
  83  *   interrupt.
  84  *
  85  * For now, we only support pseudo-NMIs if we have non-secure view of
  86  * priorities.
  87  */
  88 static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);
  89
  90 /*
  91  * Global static key controlling whether an update to PMR allowing more
  92  * interrupts requires to be propagated to the redistributor (DSB SY).
  93  * And this needs to be exported for modules to be able to enable
  94  * interrupts...
  95  */
  96 DEFINE_STATIC_KEY_FALSE(gic_pmr_sync);
  97 EXPORT_SYMBOL(gic_pmr_sync);
  98
  99 /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */
 100 static refcount_t *ppi_nmi_refs;
 101
 102 static struct gic_kvm_info gic_v3_kvm_info;
 103 static DEFINE_PER_CPU(bool, has_rss);
 104
 105 #define MPIDR_RS(mpidr)                 (((mpidr) & 0xF0UL) >> 4)
 106 #define gic_data_rdist()                (this_cpu_ptr(gic_data.rdists.rdist))
 107 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
 108 #define gic_data_rdist_sgi_base()       (gic_data_rdist_rd_base() + SZ_64K)
 109
 110 /* Our default, arbitrary priority value. Linux only uses one anyway. */
 111 #define DEFAULT_PMR_VALUE       0xf0
 112
 113 enum gic_intid_range {
 114         PPI_RANGE,
 115         SPI_RANGE,
 116         EPPI_RANGE,
 117         ESPI_RANGE,
 118         LPI_RANGE,
 119         __INVALID_RANGE__
 120 };
 121
 122 static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)
 123 {
 124         switch (hwirq) {
 125         case 16 ... 31:
 126                 return PPI_RANGE;
 127         case 32 ... 1019:
 128                 return SPI_RANGE;
 129         case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63):
 130                 return EPPI_RANGE;
 131         case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023):
 132                 return ESPI_RANGE;
 133         case 8192 ... GENMASK(23, 0):
 134                 return LPI_RANGE;
 135         default:
 136                 return __INVALID_RANGE__;
 137         }
 138 }
 139
 140 static enum gic_intid_range get_intid_range(struct irq_data *d)
 141 {
 142         return __get_intid_range(d->hwirq);
 143 }
 144
 145 static inline unsigned int gic_irq(struct irq_data *d)
 146 {
 147         return d->hwirq;
 148 }
 149
 150 static inline int gic_irq_in_rdist(struct irq_data *d)
 151 {
 152         enum gic_intid_range range = get_intid_range(d);
 153         return range == PPI_RANGE || range == EPPI_RANGE;
 154 }
 155
 156 static inline void __iomem *gic_dist_base(struct irq_data *d)
 157 {
 158         switch (get_intid_range(d)) {
 159         case PPI_RANGE:
 160         case EPPI_RANGE:
 161                 /* SGI+PPI -> SGI_base for this CPU */
 162                 return gic_data_rdist_sgi_base();
 163
 164         case SPI_RANGE:
 165         case ESPI_RANGE:
 166                 /* SPI -> dist_base */
 167                 return gic_data.dist_base;
 168
 169         default:
 170                 return NULL;
 171         }
 172 }
 173
 174 static void gic_do_wait_for_rwp(void __iomem *base)
 175 {
 176         u32 count = 1000000;    /* 1s! */
 177
 178         while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) {
 179                 count--;
 180                 if (!count) {
 181                         pr_err_ratelimited("RWP timeout, gone fishing\n");
 182                         return;
 183                 }
 184                 cpu_relax();
 185                 udelay(1);
 186         }
 187 }
 188
 189 /* Wait for completion of a distributor change */
 190 static void gic_dist_wait_for_rwp(void)
 191 {
 192         gic_do_wait_for_rwp(gic_data.dist_base);
 193 }
 194
 195 /* Wait for completion of a redistributor change */
 196 static void gic_redist_wait_for_rwp(void)
 197 {
 198         gic_do_wait_for_rwp(gic_data_rdist_rd_base());
 199 }
 200
 201 #ifdef CONFIG_ARM64
 202
 203 static u64 __maybe_unused gic_read_iar(void)
 204 {
 205         if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154))
 206                 return gic_read_iar_cavium_thunderx();
 207         else
 208                 return gic_read_iar_common();
 209 }
 210 #endif
 211
 212 static void gic_enable_redist(bool enable)
 213 {
 214         void __iomem *rbase;
 215         u32 count = 1000000;    /* 1s! */
 216         u32 val;
 217
 218         if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996)
 219                 return;
 220
 221         rbase = gic_data_rdist_rd_base();
 222
 223         val = readl_relaxed(rbase + GICR_WAKER);
 224         if (enable)
 225                 /* Wake up this CPU redistributor */
 226                 val &= ~GICR_WAKER_ProcessorSleep;
 227         else
 228                 val |= GICR_WAKER_ProcessorSleep;
 229         writel_relaxed(val, rbase + GICR_WAKER);
 230
 231         if (!enable) {          /* Check that GICR_WAKER is writeable */
 232                 val = readl_relaxed(rbase + GICR_WAKER);
 233                 if (!(val & GICR_WAKER_ProcessorSleep))
 234                         return; /* No PM support in this redistributor */
 235         }
 236
 237         while (--count) {
 238                 val = readl_relaxed(rbase + GICR_WAKER);
 239                 if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep))
 240                         break;
 241                 cpu_relax();
 242                 udelay(1);
 243         }
 244         if (!count)
 245                 pr_err_ratelimited("redistributor failed to %s...\n",
 246                                    enable ? "wakeup" : "sleep");
 247 }
 248
 249 /*
 250  * Routines to disable, enable, EOI and route interrupts
 251  */
 252 static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index)
 253 {
 254         switch (get_intid_range(d)) {
 255         case PPI_RANGE:
 256         case SPI_RANGE:
 257                 *index = d->hwirq;
 258                 return offset;
 259         case EPPI_RANGE:
 260                 /*
 261                  * Contrary to the ESPI range, the EPPI range is contiguous
 262                  * to the PPI range in the registers, so let's adjust the
 263                  * displacement accordingly. Consistency is overrated.
 264                  */
 265                 *index = d->hwirq - EPPI_BASE_INTID + 32;
 266                 return offset;
 267         case ESPI_RANGE:
 268                 *index = d->hwirq - ESPI_BASE_INTID;
 269                 switch (offset) {
 270                 case GICD_ISENABLER:
 271                         return GICD_ISENABLERnE;
 272                 case GICD_ICENABLER:
 273                         return GICD_ICENABLERnE;
 274                 case GICD_ISPENDR:
 275                         return GICD_ISPENDRnE;
 276                 case GICD_ICPENDR:
 277                         return GICD_ICPENDRnE;
 278                 case GICD_ISACTIVER:
 279                         return GICD_ISACTIVERnE;
 280                 case GICD_ICACTIVER:
 281                         return GICD_ICACTIVERnE;
 282                 case GICD_IPRIORITYR:
 283                         return GICD_IPRIORITYRnE;
 284                 case GICD_ICFGR:
 285                         return GICD_ICFGRnE;
 286                 case GICD_IROUTER:
 287                         return GICD_IROUTERnE;
 288                 default:
 289                         break;
 290                 }
 291                 break;
 292         default:
 293                 break;
 294         }
 295
 296         WARN_ON(1);
 297         *index = d->hwirq;
 298         return offset;
 299 }
 300
 301 static int gic_peek_irq(struct irq_data *d, u32 offset)
 302 {
 303         void __iomem *base;
 304         u32 index, mask;
 305
 306         offset = convert_offset_index(d, offset, &index);
 307         mask = 1 << (index % 32);
 308
 309         if (gic_irq_in_rdist(d))
 310                 base = gic_data_rdist_sgi_base();
 311         else
 312                 base = gic_data.dist_base;
 313
 314         return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);
 315 }
 316
 317 static void gic_poke_irq(struct irq_data *d, u32 offset)
 318 {
 319         void (*rwp_wait)(void);
 320         void __iomem *base;
 321         u32 index, mask;
 322
 323         offset = convert_offset_index(d, offset, &index);
 324         mask = 1 << (index % 32);
 325
 326         if (gic_irq_in_rdist(d)) {
 327                 base = gic_data_rdist_sgi_base();
 328                 rwp_wait = gic_redist_wait_for_rwp;
 329         } else {
 330                 base = gic_data.dist_base;
 331                 rwp_wait = gic_dist_wait_for_rwp;
 332         }
 333
 334         writel_relaxed(mask, base + offset + (index / 32) * 4);
 335         rwp_wait();
 336 }
 337
 338 static void gic_mask_irq(struct irq_data *d)
 339 {
 340         gic_poke_irq(d, GICD_ICENABLER);
 341 }
 342
 343 static void gic_eoimode1_mask_irq(struct irq_data *d)
 344 {
 345         gic_mask_irq(d);
 346         /*
 347          * When masking a forwarded interrupt, make sure it is
 348          * deactivated as well.
 349          *
 350          * This ensures that an interrupt that is getting
 351          * disabled/masked will not get "stuck", because there is
 352          * noone to deactivate it (guest is being terminated).
 353          */
 354         if (irqd_is_forwarded_to_vcpu(d))
 355                 gic_poke_irq(d, GICD_ICACTIVER);
 356 }
 357
 358 static void gic_unmask_irq(struct irq_data *d)
 359 {
 360         gic_poke_irq(d, GICD_ISENABLER);
 361 }
 362
 363 static inline bool gic_supports_nmi(void)
 364 {
 365         return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&
 366                static_branch_likely(&supports_pseudo_nmis);
 367 }
 368
 369 static int gic_irq_set_irqchip_state(struct irq_data *d,
 370                                      enum irqchip_irq_state which, bool val)
 371 {
 372         u32 reg;
 373
 374         if (d->hwirq >= 8192) /* PPI/SPI only */
 375                 return -EINVAL;
 376
 377         switch (which) {
 378         case IRQCHIP_STATE_PENDING:
 379                 reg = val ? GICD_ISPENDR : GICD_ICPENDR;
 380                 break;
 381
 382         case IRQCHIP_STATE_ACTIVE:
 383                 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
 384                 break;
 385
 386         case IRQCHIP_STATE_MASKED:
 387                 reg = val ? GICD_ICENABLER : GICD_ISENABLER;
 388                 break;
 389
 390         default:
 391                 return -EINVAL;
 392         }
 393
 394         gic_poke_irq(d, reg);
 395         return 0;
 396 }
 397
 398 static int gic_irq_get_irqchip_state(struct irq_data *d,
 399                                      enum irqchip_irq_state which, bool *val)
 400 {
 401         if (d->hwirq >= 8192) /* PPI/SPI only */
 402                 return -EINVAL;
 403
 404         switch (which) {
 405         case IRQCHIP_STATE_PENDING:
 406                 *val = gic_peek_irq(d, GICD_ISPENDR);
 407                 break;
 408
 409         case IRQCHIP_STATE_ACTIVE:
 410                 *val = gic_peek_irq(d, GICD_ISACTIVER);
 411                 break;
 412
 413         case IRQCHIP_STATE_MASKED:
 414                 *val = !gic_peek_irq(d, GICD_ISENABLER);
 415                 break;
 416
 417         default:
 418                 return -EINVAL;
 419         }
 420
 421         return 0;
 422 }
 423
 424 static void gic_irq_set_prio(struct irq_data *d, u8 prio)
 425 {
 426         void __iomem *base = gic_dist_base(d);
 427         u32 offset, index;
 428
 429         offset = convert_offset_index(d, GICD_IPRIORITYR, &index);
 430
 431         writeb_relaxed(prio, base + offset + index);
 432 }
 433
 434 static u32 gic_get_ppi_index(struct irq_data *d)
 435 {
 436         switch (get_intid_range(d)) {
 437         case PPI_RANGE:
 438                 return d->hwirq - 16;
 439         case EPPI_RANGE:
 440                 return d->hwirq - EPPI_BASE_INTID + 16;
 441         default:
 442                 unreachable();
 443         }
 444 }
 445
 446 static int gic_irq_nmi_setup(struct irq_data *d)
 447 {
 448         struct irq_desc *desc = irq_to_desc(d->irq);
 449
 450         if (!gic_supports_nmi())
 451                 return -EINVAL;
 452
 453         if (gic_peek_irq(d, GICD_ISENABLER)) {
 454                 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
 455                 return -EINVAL;
 456         }
 457
 458         /*
 459          * A secondary irq_chip should be in charge of LPI request,
 460          * it should not be possible to get there
 461          */
 462         if (WARN_ON(gic_irq(d) >= 8192))
 463                 return -EINVAL;
 464
 465         /* desc lock should already be held */
 466         if (gic_irq_in_rdist(d)) {
 467                 u32 idx = gic_get_ppi_index(d);
 468
 469                 /* Setting up PPI as NMI, only switch handler for first NMI */
 470                 if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) {
 471                         refcount_set(&ppi_nmi_refs[idx], 1);
 472                         desc->handle_irq = handle_percpu_devid_fasteoi_nmi;
 473                 }
 474         } else {
 475                 desc->handle_irq = handle_fasteoi_nmi;
 476         }
 477
 478         gic_irq_set_prio(d, GICD_INT_NMI_PRI);
 479
 480         return 0;
 481 }
 482
 483 static void gic_irq_nmi_teardown(struct irq_data *d)
 484 {
 485         struct irq_desc *desc = irq_to_desc(d->irq);
 486
 487         if (WARN_ON(!gic_supports_nmi()))
 488                 return;
 489
 490         if (gic_peek_irq(d, GICD_ISENABLER)) {
 491                 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
 492                 return;
 493         }
 494
 495         /*
 496          * A secondary irq_chip should be in charge of LPI request,
 497          * it should not be possible to get there
 498          */
 499         if (WARN_ON(gic_irq(d) >= 8192))
 500                 return;
 501
 502         /* desc lock should already be held */
 503         if (gic_irq_in_rdist(d)) {
 504                 u32 idx = gic_get_ppi_index(d);
 505
 506                 /* Tearing down NMI, only switch handler for last NMI */
 507                 if (refcount_dec_and_test(&ppi_nmi_refs[idx]))
 508                         desc->handle_irq = handle_percpu_devid_irq;
 509         } else {
 510                 desc->handle_irq = handle_fasteoi_irq;
 511         }
 512
 513         gic_irq_set_prio(d, GICD_INT_DEF_PRI);
 514 }
 515
 516 static void gic_eoi_irq(struct irq_data *d)
 517 {
 518         gic_write_eoir(gic_irq(d));
 519 }
 520
 521 static void gic_eoimode1_eoi_irq(struct irq_data *d)
 522 {
 523         /*
 524          * No need to deactivate an LPI, or an interrupt that
 525          * is is getting forwarded to a vcpu.
 526          */
 527         if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
 528                 return;
 529         gic_write_dir(gic_irq(d));
 530 }
 531
 532 static int gic_set_type(struct irq_data *d, unsigned int type)
 533 {
 534         enum gic_intid_range range;
 535         unsigned int irq = gic_irq(d);
 536         void (*rwp_wait)(void);
 537         void __iomem *base;
 538         u32 offset, index;
 539         int ret;
 540
 541         /* Interrupt configuration for SGIs can't be changed */
 542         if (irq < 16)
 543                 return -EINVAL;
 544
 545         range = get_intid_range(d);
 546
 547         /* SPIs have restrictions on the supported types */
 548         if ((range == SPI_RANGE || range == ESPI_RANGE) &&
 549             type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
 550                 return -EINVAL;
 551
 552         if (gic_irq_in_rdist(d)) {
 553                 base = gic_data_rdist_sgi_base();
 554                 rwp_wait = gic_redist_wait_for_rwp;
 555         } else {
 556                 base = gic_data.dist_base;
 557                 rwp_wait = gic_dist_wait_for_rwp;
 558         }
 559
 560         offset = convert_offset_index(d, GICD_ICFGR, &index);
 561
 562         ret = gic_configure_irq(index, type, base + offset, rwp_wait);
 563         if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) {
 564                 /* Misconfigured PPIs are usually not fatal */
 565                 pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq);
 566                 ret = 0;
 567         }
 568
 569         return ret;
 570 }
 571
 572 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
 573 {
 574         if (vcpu)
 575                 irqd_set_forwarded_to_vcpu(d);
 576         else
 577                 irqd_clr_forwarded_to_vcpu(d);
 578         return 0;
 579 }
 580
 581 static u64 gic_mpidr_to_affinity(unsigned long mpidr)
 582 {
 583         u64 aff;
 584
 585         aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
 586                MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
 587                MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8  |
 588                MPIDR_AFFINITY_LEVEL(mpidr, 0));
 589
 590         return aff;
 591 }
 592
 593 static void gic_deactivate_unhandled(u32 irqnr)
 594 {
 595         if (static_branch_likely(&supports_deactivate_key)) {
 596                 if (irqnr < 8192)
 597                         gic_write_dir(irqnr);
 598         } else {
 599                 gic_write_eoir(irqnr);
 600         }
 601 }
 602
 603 static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
 604 {
 605         bool irqs_enabled = interrupts_enabled(regs);
 606         int err;
 607
 608         if (irqs_enabled)
 609                 nmi_enter();
 610
 611         if (static_branch_likely(&supports_deactivate_key))
 612                 gic_write_eoir(irqnr);
 613         /*
 614          * Leave the PSR.I bit set to prevent other NMIs to be
 615          * received while handling this one.
 616          * PSR.I will be restored when we ERET to the
 617          * interrupted context.
 618          */
 619         err = handle_domain_nmi(gic_data.domain, irqnr, regs);
 620         if (err)
 621                 gic_deactivate_unhandled(irqnr);
 622
 623         if (irqs_enabled)
 624                 nmi_exit();
 625 }
 626
 627 static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
 628 {
 629         u32 irqnr;
 630
 631         irqnr = gic_read_iar();
 632
 633         if (gic_supports_nmi() &&
 634             unlikely(gic_read_rpr() == GICD_INT_NMI_PRI)) {
 635                 gic_handle_nmi(irqnr, regs);
 636                 return;
 637         }
 638
 639         if (gic_prio_masking_enabled()) {
 640                 gic_pmr_mask_irqs();
 641                 gic_arch_enable_irqs();
 642         }
 643
 644         /* Check for special IDs first */
 645         if ((irqnr >= 1020 && irqnr <= 1023))
 646                 return;
 647
 648         /* Treat anything but SGIs in a uniform way */
 649         if (likely(irqnr > 15)) {
 650                 int err;
 651
 652                 if (static_branch_likely(&supports_deactivate_key))
 653                         gic_write_eoir(irqnr);
 654                 else
 655                         isb();
 656
 657                 err = handle_domain_irq(gic_data.domain, irqnr, regs);
 658                 if (err) {
 659                         WARN_ONCE(true, "Unexpected interrupt received!\n");
 660                         gic_deactivate_unhandled(irqnr);
 661                 }
 662                 return;
 663         }
 664         if (irqnr < 16) {
 665                 gic_write_eoir(irqnr);
 666                 if (static_branch_likely(&supports_deactivate_key))
 667                         gic_write_dir(irqnr);
 668 #ifdef CONFIG_SMP
 669                 /*
 670                  * Unlike GICv2, we don't need an smp_rmb() here.
 671                  * The control dependency from gic_read_iar to
 672                  * the ISB in gic_write_eoir is enough to ensure
 673                  * that any shared data read by handle_IPI will
 674                  * be read after the ACK.
 675                  */
 676                 handle_IPI(irqnr, regs);
 677 #else
 678                 WARN_ONCE(true, "Unexpected SGI received!\n");
 679 #endif
 680         }
 681 }
 682
 683 static u32 gic_get_pribits(void)
 684 {
 685         u32 pribits;
 686
 687         pribits = gic_read_ctlr();
 688         pribits &= ICC_CTLR_EL1_PRI_BITS_MASK;
 689         pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT;
 690         pribits++;
 691
 692         return pribits;
 693 }
 694
 695 static bool gic_has_group0(void)
 696 {
 697         u32 val;
 698         u32 old_pmr;
 699
 700         old_pmr = gic_read_pmr();
 701
 702         /*
 703          * Let's find out if Group0 is under control of EL3 or not by
 704          * setting the highest possible, non-zero priority in PMR.
 705          *
 706          * If SCR_EL3.FIQ is set, the priority gets shifted down in
 707          * order for the CPU interface to set bit 7, and keep the
 708          * actual priority in the non-secure range. In the process, it
 709          * looses the least significant bit and the actual priority
 710          * becomes 0x80. Reading it back returns 0, indicating that
 711          * we're don't have access to Group0.
 712          */
 713         gic_write_pmr(BIT(8 - gic_get_pribits()));
 714         val = gic_read_pmr();
 715
 716         gic_write_pmr(old_pmr);
 717
 718         return val != 0;
 719 }
 720
 721 static void __init gic_dist_init(void)
 722 {
 723         unsigned int i;
 724         u64 affinity;
 725         void __iomem *base = gic_data.dist_base;
 726
 727         /* Disable the distributor */
 728         writel_relaxed(0, base + GICD_CTLR);
 729         gic_dist_wait_for_rwp();
 730
 731         /*
 732          * Configure SPIs as non-secure Group-1. This will only matter
 733          * if the GIC only has a single security state. This will not
 734          * do the right thing if the kernel is running in secure mode,
 735          * but that's not the intended use case anyway.
 736          */
 737         for (i = 32; i < GIC_LINE_NR; i += 32)
 738                 writel_relaxed(~0, base + GICD_IGROUPR + i / 8);
 739
 740         /* Extended SPI range, not handled by the GICv2/GICv3 common code */
 741         for (i = 0; i < GIC_ESPI_NR; i += 32) {
 742                 writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8);
 743                 writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8);
 744         }
 745
 746         for (i = 0; i < GIC_ESPI_NR; i += 32)
 747                 writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8);
 748
 749         for (i = 0; i < GIC_ESPI_NR; i += 16)
 750                 writel_relaxed(0, base + GICD_ICFGRnE + i / 4);
 751
 752         for (i = 0; i < GIC_ESPI_NR; i += 4)
 753                 writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i);
 754
 755         /* Now do the common stuff, and wait for the distributor to drain */
 756         gic_dist_config(base, GIC_LINE_NR, gic_dist_wait_for_rwp);
 757
 758         /* Enable distributor with ARE, Group1 */
 759         writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
 760                        base + GICD_CTLR);
 761
 762         /*
 763          * Set all global interrupts to the boot CPU only. ARE must be
 764          * enabled.
 765          */
 766         affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
 767         for (i = 32; i < GIC_LINE_NR; i++)
 768                 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);
 769
 770         for (i = 0; i < GIC_ESPI_NR; i++)
 771                 gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8);
 772 }
 773
 774 static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
 775 {
 776         int ret = -ENODEV;
 777         int i;
 778
 779         for (i = 0; i < gic_data.nr_redist_regions; i++) {
 780                 void __iomem *ptr = gic_data.redist_regions[i].redist_base;
 781                 u64 typer;
 782                 u32 reg;
 783
 784                 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
 785                 if (reg != GIC_PIDR2_ARCH_GICv3 &&
 786                     reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
 787                         pr_warn("No redistributor present @%p\n", ptr);
 788                         break;
 789                 }
 790
 791                 do {
 792                         typer = gic_read_typer(ptr + GICR_TYPER);
 793                         ret = fn(gic_data.redist_regions + i, ptr);
 794                         if (!ret)
 795                                 return 0;
 796
 797                         if (gic_data.redist_regions[i].single_redist)
 798                                 break;
 799
 800                         if (gic_data.redist_stride) {
 801                                 ptr += gic_data.redist_stride;
 802                         } else {
 803                                 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
 804                                 if (typer & GICR_TYPER_VLPIS)
 805                                         ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
 806                         }
 807                 } while (!(typer & GICR_TYPER_LAST));
 808         }
 809
 810         return ret ? -ENODEV : 0;
 811 }
 812
 813 static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
 814 {
 815         unsigned long mpidr = cpu_logical_map(smp_processor_id());
 816         u64 typer;
 817         u32 aff;
 818
 819         /*
 820          * Convert affinity to a 32bit value that can be matched to
 821          * GICR_TYPER bits [63:32].
 822          */
 823         aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
 824                MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
 825                MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
 826                MPIDR_AFFINITY_LEVEL(mpidr, 0));
 827
 828         typer = gic_read_typer(ptr + GICR_TYPER);
 829         if ((typer >> 32) == aff) {
 830                 u64 offset = ptr - region->redist_base;
 831                 gic_data_rdist_rd_base() = ptr;
 832                 gic_data_rdist()->phys_base = region->phys_base + offset;
 833
 834                 pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
 835                         smp_processor_id(), mpidr,
 836                         (int)(region - gic_data.redist_regions),
 837                         &gic_data_rdist()->phys_base);
 838                 return 0;
 839         }
 840
 841         /* Try next one */
 842         return 1;
 843 }
 844
 845 static int gic_populate_rdist(void)
 846 {
 847         if (gic_iterate_rdists(__gic_populate_rdist) == 0)
 848                 return 0;
 849
 850         /* We couldn't even deal with ourselves... */
 851         WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
 852              smp_processor_id(),
 853              (unsigned long)cpu_logical_map(smp_processor_id()));
 854         return -ENODEV;
 855 }
 856
 857 static int __gic_update_rdist_properties(struct redist_region *region,
 858                                          void __iomem *ptr)
 859 {
 860         u64 typer = gic_read_typer(ptr + GICR_TYPER);
 861         gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);
 862         gic_data.rdists.has_direct_lpi &= !!(typer & GICR_TYPER_DirectLPIS);
 863         gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr);
 864
 865         return 1;
 866 }
 867
 868 static void gic_update_rdist_properties(void)
 869 {
 870         gic_data.ppi_nr = UINT_MAX;
 871         gic_iterate_rdists(__gic_update_rdist_properties);
 872         if (WARN_ON(gic_data.ppi_nr == UINT_MAX))
 873                 gic_data.ppi_nr = 0;
 874         pr_info("%d PPIs implemented\n", gic_data.ppi_nr);
 875         pr_info("%sVLPI support, %sdirect LPI support\n",
 876                 !gic_data.rdists.has_vlpis ? "no " : "",
 877                 !gic_data.rdists.has_direct_lpi ? "no " : "");
 878 }
 879
 880 /* Check whether it's single security state view */
 881 static inline bool gic_dist_security_disabled(void)
 882 {
 883         return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS;
 884 }
 885
 886 static void gic_cpu_sys_reg_init(void)
 887 {
 888         int i, cpu = smp_processor_id();
 889         u64 mpidr = cpu_logical_map(cpu);
 890         u64 need_rss = MPIDR_RS(mpidr);
 891         bool group0;
 892         u32 pribits;
 893
 894         /*
 895          * Need to check that the SRE bit has actually been set. If
 896          * not, it means that SRE is disabled at EL2. We're going to
 897          * die painfully, and there is nothing we can do about it.
 898          *
 899          * Kindly inform the luser.
 900          */
 901         if (!gic_enable_sre())
 902                 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
 903
 904         pribits = gic_get_pribits();
 905
 906         group0 = gic_has_group0();
 907
 908         /* Set priority mask register */
 909         if (!gic_prio_masking_enabled()) {
 910                 write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
 911         } else {
 912                 /*
 913                  * Mismatch configuration with boot CPU, the system is likely
 914                  * to die as interrupt masking will not work properly on all
 915                  * CPUs
 916                  */
 917                 WARN_ON(gic_supports_nmi() && group0 &&
 918                         !gic_dist_security_disabled());
 919         }
 920
 921         /*
 922          * Some firmwares hand over to the kernel with the BPR changed from
 923          * its reset value (and with a value large enough to prevent
 924          * any pre-emptive interrupts from working at all). Writing a zero
 925          * to BPR restores is reset value.
 926          */
 927         gic_write_bpr1(0);
 928
 929         if (static_branch_likely(&supports_deactivate_key)) {
 930                 /* EOI drops priority only (mode 1) */
 931                 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
 932         } else {
 933                 /* EOI deactivates interrupt too (mode 0) */
 934                 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
 935         }
 936
 937         /* Always whack Group0 before Group1 */
 938         if (group0) {
 939                 switch(pribits) {
 940                 case 8:
 941                 case 7:
 942                         write_gicreg(0, ICC_AP0R3_EL1);
 943                         write_gicreg(0, ICC_AP0R2_EL1);
 944                 /* Fall through */
 945                 case 6:
 946                         write_gicreg(0, ICC_AP0R1_EL1);
 947                 /* Fall through */
 948                 case 5:
 949                 case 4:
 950                         write_gicreg(0, ICC_AP0R0_EL1);
 951                 }
 952
 953                 isb();
 954         }
 955
 956         switch(pribits) {
 957         case 8:
 958         case 7:
 959                 write_gicreg(0, ICC_AP1R3_EL1);
 960                 write_gicreg(0, ICC_AP1R2_EL1);
 961                 /* Fall through */
 962         case 6:
 963                 write_gicreg(0, ICC_AP1R1_EL1);
 964                 /* Fall through */
 965         case 5:
 966         case 4:
 967                 write_gicreg(0, ICC_AP1R0_EL1);
 968         }
 969
 970         isb();
 971
 972         /* ... and let's hit the road... */
 973         gic_write_grpen1(1);
 974
 975         /* Keep the RSS capability status in per_cpu variable */
 976         per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS);
 977
 978         /* Check all the CPUs have capable of sending SGIs to other CPUs */
 979         for_each_online_cpu(i) {
 980                 bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu);
 981
 982                 need_rss |= MPIDR_RS(cpu_logical_map(i));
 983                 if (need_rss && (!have_rss))
 984                         pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n",
 985                                 cpu, (unsigned long)mpidr,
 986                                 i, (unsigned long)cpu_logical_map(i));
 987         }
 988
 989         /**
 990          * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0,
 991          * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED
 992          * UNPREDICTABLE choice of :
 993          *   - The write is ignored.
 994          *   - The RS field is treated as 0.
 995          */
 996         if (need_rss && (!gic_data.has_rss))
 997                 pr_crit_once("RSS is required but GICD doesn't support it\n");
 998 }
 999
1000 static bool gicv3_nolpi;
1001
1002 static int __init gicv3_nolpi_cfg(char *buf)
1003 {
1004         return strtobool(buf, &gicv3_nolpi);
1005 }
1006 early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg);
1007
1008 static int gic_dist_supports_lpis(void)
1009 {
1010         return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) &&
1011                 !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) &&
1012                 !gicv3_nolpi);
1013 }
1014
1015 static void gic_cpu_init(void)
1016 {
1017         void __iomem *rbase;
1018         int i;
1019
1020         /* Register ourselves with the rest of the world */
1021         if (gic_populate_rdist())
1022                 return;
1023
1024         gic_enable_redist(true);
1025
1026         WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) &&
1027              !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange),
1028              "Distributor has extended ranges, but CPU%d doesn't\n",
1029              smp_processor_id());
1030
1031         rbase = gic_data_rdist_sgi_base();
1032
1033         /* Configure SGIs/PPIs as non-secure Group-1 */
1034         for (i = 0; i < gic_data.ppi_nr + 16; i += 32)
1035                 writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8);
1036
1037         gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp);
1038
1039         /* initialise system registers */
1040         gic_cpu_sys_reg_init();
1041 }
1042
1043 #ifdef CONFIG_SMP
1044
1045 #define MPIDR_TO_SGI_RS(mpidr)  (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT)
1046 #define MPIDR_TO_SGI_CLUSTER_ID(mpidr)  ((mpidr) & ~0xFUL)
1047
1048 static int gic_starting_cpu(unsigned int cpu)
1049 {
1050         gic_cpu_init();
1051
1052         if (gic_dist_supports_lpis())
1053                 its_cpu_init();
1054
1055         return 0;
1056 }
1057
1058 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
1059                                    unsigned long cluster_id)
1060 {
1061         int next_cpu, cpu = *base_cpu;
1062         unsigned long mpidr = cpu_logical_map(cpu);
1063         u16 tlist = 0;
1064
1065         while (cpu < nr_cpu_ids) {
1066                 tlist |= 1 << (mpidr & 0xf);
1067
1068                 next_cpu = cpumask_next(cpu, mask);
1069                 if (next_cpu >= nr_cpu_ids)
1070                         goto out;
1071                 cpu = next_cpu;
1072
1073                 mpidr = cpu_logical_map(cpu);
1074
1075                 if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) {
1076                         cpu--;
1077                         goto out;
1078                 }
1079         }
1080 out:
1081         *base_cpu = cpu;
1082         return tlist;
1083 }
1084
1085 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
1086         (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
1087                 << ICC_SGI1R_AFFINITY_## level ##_SHIFT)
1088
1089 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
1090 {
1091         u64 val;
1092
1093         val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3)     |
1094                MPIDR_TO_SGI_AFFINITY(cluster_id, 2)     |
1095                irq << ICC_SGI1R_SGI_ID_SHIFT            |
1096                MPIDR_TO_SGI_AFFINITY(cluster_id, 1)     |
1097                MPIDR_TO_SGI_RS(cluster_id)              |
1098                tlist << ICC_SGI1R_TARGET_LIST_SHIFT);
1099
1100         pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
1101         gic_write_sgi1r(val);
1102 }
1103
1104 static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
1105 {
1106         int cpu;
1107
1108         if (WARN_ON(irq >= 16))
1109                 return;
1110
1111         /*
1112          * Ensure that stores to Normal memory are visible to the
1113          * other CPUs before issuing the IPI.
1114          */
1115         wmb();
1116
1117         for_each_cpu(cpu, mask) {
1118                 u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu));
1119                 u16 tlist;
1120
1121                 tlist = gic_compute_target_list(&cpu, mask, cluster_id);
1122                 gic_send_sgi(cluster_id, tlist, irq);
1123         }
1124
1125         /* Force the above writes to ICC_SGI1R_EL1 to be executed */
1126         isb();
1127 }
1128
1129 static void gic_smp_init(void)
1130 {
1131         set_smp_cross_call(gic_raise_softirq);
1132         cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
1133                                   "irqchip/arm/gicv3:starting",
1134                                   gic_starting_cpu, NULL);
1135 }
1136
1137 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1138                             bool force)
1139 {
1140         unsigned int cpu;
1141         u32 offset, index;
1142         void __iomem *reg;
1143         int enabled;
1144         u64 val;
1145
1146         if (force)
1147                 cpu = cpumask_first(mask_val);
1148         else
1149                 cpu = cpumask_any_and(mask_val, cpu_online_mask);
1150
1151         if (cpu >= nr_cpu_ids)
1152                 return -EINVAL;
1153
1154         if (gic_irq_in_rdist(d))
1155                 return -EINVAL;
1156
1157         /* If interrupt was enabled, disable it first */
1158         enabled = gic_peek_irq(d, GICD_ISENABLER);
1159         if (enabled)
1160                 gic_mask_irq(d);
1161
1162         offset = convert_offset_index(d, GICD_IROUTER, &index);
1163         reg = gic_dist_base(d) + offset + (index * 8);
1164         val = gic_mpidr_to_affinity(cpu_logical_map(cpu));
1165
1166         gic_write_irouter(val, reg);
1167
1168         /*
1169          * If the interrupt was enabled, enabled it again. Otherwise,
1170          * just wait for the distributor to have digested our changes.
1171          */
1172         if (enabled)
1173                 gic_unmask_irq(d);
1174         else
1175                 gic_dist_wait_for_rwp();
1176
1177         irq_data_update_effective_affinity(d, cpumask_of(cpu));
1178
1179         return IRQ_SET_MASK_OK_DONE;
1180 }
1181 #else
1182 #define gic_set_affinity        NULL
1183 #define gic_smp_init()          do { } while(0)
1184 #endif
1185
1186 #ifdef CONFIG_CPU_PM
1187 static int gic_cpu_pm_notifier(struct notifier_block *self,
1188                                unsigned long cmd, void *v)
1189 {
1190         if (cmd == CPU_PM_EXIT) {
1191                 if (gic_dist_security_disabled())
1192                         gic_enable_redist(true);
1193                 gic_cpu_sys_reg_init();
1194         } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) {
1195                 gic_write_grpen1(0);
1196                 gic_enable_redist(false);
1197         }
1198         return NOTIFY_OK;
1199 }
1200
1201 static struct notifier_block gic_cpu_pm_notifier_block = {
1202         .notifier_call = gic_cpu_pm_notifier,
1203 };
1204
1205 static void gic_cpu_pm_init(void)
1206 {
1207         cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
1208 }
1209
1210 #else
1211 static inline void gic_cpu_pm_init(void) { }
1212 #endif /* CONFIG_CPU_PM */
1213
1214 static struct irq_chip gic_chip = {
1215         .name                   = "GICv3",
1216         .irq_mask               = gic_mask_irq,
1217         .irq_unmask             = gic_unmask_irq,
1218         .irq_eoi                = gic_eoi_irq,
1219         .irq_set_type           = gic_set_type,
1220         .irq_set_affinity       = gic_set_affinity,
1221         .irq_get_irqchip_state  = gic_irq_get_irqchip_state,
1222         .irq_set_irqchip_state  = gic_irq_set_irqchip_state,
1223         .irq_nmi_setup          = gic_irq_nmi_setup,
1224         .irq_nmi_teardown       = gic_irq_nmi_teardown,
1225         .flags                  = IRQCHIP_SET_TYPE_MASKED |
1226                                   IRQCHIP_SKIP_SET_WAKE |
1227                                   IRQCHIP_MASK_ON_SUSPEND,
1228 };
1229
1230 static struct irq_chip gic_eoimode1_chip = {
1231         .name                   = "GICv3",
1232         .irq_mask               = gic_eoimode1_mask_irq,
1233         .irq_unmask             = gic_unmask_irq,
1234         .irq_eoi                = gic_eoimode1_eoi_irq,
1235         .irq_set_type           = gic_set_type,
1236         .irq_set_affinity       = gic_set_affinity,
1237         .irq_get_irqchip_state  = gic_irq_get_irqchip_state,
1238         .irq_set_irqchip_state  = gic_irq_set_irqchip_state,
1239         .irq_set_vcpu_affinity  = gic_irq_set_vcpu_affinity,
1240         .irq_nmi_setup          = gic_irq_nmi_setup,
1241         .irq_nmi_teardown       = gic_irq_nmi_teardown,
1242         .flags                  = IRQCHIP_SET_TYPE_MASKED |
1243                                   IRQCHIP_SKIP_SET_WAKE |
1244                                   IRQCHIP_MASK_ON_SUSPEND,
1245 };
1246
1247 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
1248                               irq_hw_number_t hw)
1249 {
1250         struct irq_chip *chip = &gic_chip;
1251
1252         if (static_branch_likely(&supports_deactivate_key))
1253                 chip = &gic_eoimode1_chip;
1254
1255         switch (__get_intid_range(hw)) {
1256         case PPI_RANGE:
1257         case EPPI_RANGE:
1258                 irq_set_percpu_devid(irq);
1259                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
1260                                     handle_percpu_devid_irq, NULL, NULL);
1261                 irq_set_status_flags(irq, IRQ_NOAUTOEN);
1262                 break;
1263
1264         case SPI_RANGE:
1265         case ESPI_RANGE:
1266                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
1267                                     handle_fasteoi_irq, NULL, NULL);
1268                 irq_set_probe(irq);
1269                 irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq)));
1270                 break;
1271
1272         case LPI_RANGE:
1273                 if (!gic_dist_supports_lpis())
1274                         return -EPERM;
1275                 irq_domain_set_info(d, irq, hw, chip, d->host_data,
1276                                     handle_fasteoi_irq, NULL, NULL);
1277                 break;
1278
1279         default:
1280                 return -EPERM;
1281         }
1282
1283         return 0;
1284 }
1285
1286 #define GIC_IRQ_TYPE_PARTITION  (GIC_IRQ_TYPE_LPI + 1)
1287
1288 static int gic_irq_domain_translate(struct irq_domain *d,
1289                                     struct irq_fwspec *fwspec,
1290                                     unsigned long *hwirq,
1291                                     unsigned int *type)
1292 {
1293         if (is_of_node(fwspec->fwnode)) {
1294                 if (fwspec->param_count < 3)
1295                         return -EINVAL;
1296
1297                 switch (fwspec->param[0]) {
1298                 case 0:                 /* SPI */
1299                         *hwirq = fwspec->param[1] + 32;
1300                         break;
1301                 case 1:                 /* PPI */
1302                         *hwirq = fwspec->param[1] + 16;
1303                         break;
1304                 case 2:                 /* ESPI */
1305                         *hwirq = fwspec->param[1] + ESPI_BASE_INTID;
1306                         break;
1307                 case 3:                 /* EPPI */
1308                         *hwirq = fwspec->param[1] + EPPI_BASE_INTID;
1309                         break;
1310                 case GIC_IRQ_TYPE_LPI:  /* LPI */
1311                         *hwirq = fwspec->param[1];
1312                         break;
1313                 case GIC_IRQ_TYPE_PARTITION:
1314                         *hwirq = fwspec->param[1];
1315                         if (fwspec->param[1] >= 16)
1316                                 *hwirq += EPPI_BASE_INTID - 16;
1317                         else
1318                                 *hwirq += 16;
1319                         break;
1320                 default:
1321                         return -EINVAL;
1322                 }
1323
1324                 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
1325
1326                 /*
1327                  * Make it clear that broken DTs are... broken.
1328                  * Partitionned PPIs are an unfortunate exception.
1329                  */
1330                 WARN_ON(*type == IRQ_TYPE_NONE &&
1331                         fwspec->param[0] != GIC_IRQ_TYPE_PARTITION);
1332                 return 0;
1333         }
1334
1335         if (is_fwnode_irqchip(fwspec->fwnode)) {
1336                 if(fwspec->param_count != 2)
1337                         return -EINVAL;
1338
1339                 *hwirq = fwspec->param[0];
1340                 *type = fwspec->param[1];
1341
1342                 WARN_ON(*type == IRQ_TYPE_NONE);
1343                 return 0;
1344         }
1345
1346         return -EINVAL;
1347 }
1348
1349 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1350                                 unsigned int nr_irqs, void *arg)
1351 {
1352         int i, ret;
1353         irq_hw_number_t hwirq;
1354         unsigned int type = IRQ_TYPE_NONE;
1355         struct irq_fwspec *fwspec = arg;
1356
1357         ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
1358         if (ret)
1359                 return ret;
1360
1361         for (i = 0; i < nr_irqs; i++) {
1362                 ret = gic_irq_domain_map(domain, virq + i, hwirq + i);
1363                 if (ret)
1364                         return ret;
1365         }
1366
1367         return 0;
1368 }
1369
1370 static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1371                                 unsigned int nr_irqs)
1372 {
1373         int i;
1374
1375         for (i = 0; i < nr_irqs; i++) {
1376                 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
1377                 irq_set_handler(virq + i, NULL);
1378                 irq_domain_reset_irq_data(d);
1379         }
1380 }
1381
1382 static int gic_irq_domain_select(struct irq_domain *d,
1383                                  struct irq_fwspec *fwspec,
1384                                  enum irq_domain_bus_token bus_token)
1385 {
1386         /* Not for us */
1387         if (fwspec->fwnode != d->fwnode)
1388                 return 0;
1389
1390         /* If this is not DT, then we have a single domain */
1391         if (!is_of_node(fwspec->fwnode))
1392                 return 1;
1393
1394         /*
1395          * If this is a PPI and we have a 4th (non-null) parameter,
1396          * then we need to match the partition domain.
1397          */
1398         if (fwspec->param_count >= 4 &&
1399             fwspec->param[0] == 1 && fwspec->param[3] != 0 &&
1400             gic_data.ppi_descs)
1401                 return d == partition_get_domain(gic_data.ppi_descs[fwspec->param[1]]);
1402
1403         return d == gic_data.domain;
1404 }
1405
1406 static const struct irq_domain_ops gic_irq_domain_ops = {
1407         .translate = gic_irq_domain_translate,
1408         .alloc = gic_irq_domain_alloc,
1409         .free = gic_irq_domain_free,
1410         .select = gic_irq_domain_select,
1411 };
1412
1413 static int partition_domain_translate(struct irq_domain *d,
1414                                       struct irq_fwspec *fwspec,
1415                                       unsigned long *hwirq,
1416                                       unsigned int *type)
1417 {
1418         struct device_node *np;
1419         int ret;
1420
1421         if (!gic_data.ppi_descs)
1422                 return -ENOMEM;
1423
1424         np = of_find_node_by_phandle(fwspec->param[3]);
1425         if (WARN_ON(!np))
1426                 return -EINVAL;
1427
1428         ret = partition_translate_id(gic_data.ppi_descs[fwspec->param[1]],
1429                                      of_node_to_fwnode(np));
1430         if (ret < 0)
1431                 return ret;
1432
1433         *hwirq = ret;
1434         *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
1435
1436         return 0;
1437 }
1438
1439 static const struct irq_domain_ops partition_domain_ops = {
1440         .translate = partition_domain_translate,
1441         .select = gic_irq_domain_select,
1442 };
1443
1444 static bool gic_enable_quirk_msm8996(void *data)
1445 {
1446         struct gic_chip_data *d = data;
1447
1448         d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996;
1449
1450         return true;
1451 }
1452
1453 static bool gic_enable_quirk_hip06_07(void *data)
1454 {
1455         struct gic_chip_data *d = data;
1456
1457         /*
1458          * HIP06 GICD_IIDR clashes with GIC-600 product number (despite
1459          * not being an actual ARM implementation). The saving grace is
1460          * that GIC-600 doesn't have ESPI, so nothing to do in that case.
1461          * HIP07 doesn't even have a proper IIDR, and still pretends to
1462          * have ESPI. In both cases, put them right.
1463          */
1464         if (d->rdists.gicd_typer & GICD_TYPER_ESPI) {
1465                 /* Zero both ESPI and the RES0 field next to it... */
1466                 d->rdists.gicd_typer &= ~GENMASK(9, 8);
1467                 return true;
1468         }
1469
1470         return false;
1471 }
1472
1473 static const struct gic_quirk gic_quirks[] = {
1474         {
1475                 .desc   = "GICv3: Qualcomm MSM8996 broken firmware",
1476                 .compatible = "qcom,msm8996-gic-v3",
1477                 .init   = gic_enable_quirk_msm8996,
1478         },
1479         {
1480                 .desc   = "GICv3: HIP06 erratum 161010803",
1481                 .iidr   = 0x0204043b,
1482                 .mask   = 0xffffffff,
1483                 .init   = gic_enable_quirk_hip06_07,
1484         },
1485         {
1486                 .desc   = "GICv3: HIP07 erratum 161010803",
1487                 .iidr   = 0x00000000,
1488                 .mask   = 0xffffffff,
1489                 .init   = gic_enable_quirk_hip06_07,
1490         },
1491         {
1492         }
1493 };
1494
1495 static void gic_enable_nmi_support(void)
1496 {
1497         int i;
1498
1499         if (!gic_prio_masking_enabled())
1500                 return;
1501
1502         if (gic_has_group0() && !gic_dist_security_disabled()) {
1503                 pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n");
1504                 return;
1505         }
1506
1507         ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL);
1508         if (!ppi_nmi_refs)
1509                 return;
1510
1511         for (i = 0; i < gic_data.ppi_nr; i++)
1512                 refcount_set(&ppi_nmi_refs[i], 0);
1513
1514         /*
1515          * Linux itself doesn't use 1:N distribution, so has no need to
1516          * set PMHE. The only reason to have it set is if EL3 requires it
1517          * (and we can't change it).
1518          */
1519         if (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK)
1520                 static_branch_enable(&gic_pmr_sync);
1521
1522         pr_info("%s ICC_PMR_EL1 synchronisation\n",
1523                 static_branch_unlikely(&gic_pmr_sync) ? "Forcing" : "Relaxing");
1524
1525         static_branch_enable(&supports_pseudo_nmis);
1526
1527         if (static_branch_likely(&supports_deactivate_key))
1528                 gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI;
1529         else
1530                 gic_chip.flags |= IRQCHIP_SUPPORTS_NMI;
1531 }
1532
1533 static int __init gic_init_bases(void __iomem *dist_base,
1534                                  struct redist_region *rdist_regs,
1535                                  u32 nr_redist_regions,
1536                                  u64 redist_stride,
1537                                  struct fwnode_handle *handle)
1538 {
1539         u32 typer;
1540         int err;
1541
1542         if (!is_hyp_mode_available())
1543                 static_branch_disable(&supports_deactivate_key);
1544
1545         if (static_branch_likely(&supports_deactivate_key))
1546                 pr_info("GIC: Using split EOI/Deactivate mode\n");
1547
1548         gic_data.fwnode = handle;
1549         gic_data.dist_base = dist_base;
1550         gic_data.redist_regions = rdist_regs;
1551         gic_data.nr_redist_regions = nr_redist_regions;
1552         gic_data.redist_stride = redist_stride;
1553
1554         /*
1555          * Find out how many interrupts are supported.
1556          */
1557         typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
1558         gic_data.rdists.gicd_typer = typer;
1559
1560         gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR),
1561                           gic_quirks, &gic_data);
1562
1563         pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32);
1564         pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR);
1565         gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
1566                                                  &gic_data);
1567         irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);
1568         gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
1569         gic_data.rdists.has_vlpis = true;
1570         gic_data.rdists.has_direct_lpi = true;
1571
1572         if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
1573                 err = -ENOMEM;
1574                 goto out_free;
1575         }
1576
1577         gic_data.has_rss = !!(typer & GICD_TYPER_RSS);
1578         pr_info("Distributor has %sRange Selector support\n",
1579                 gic_data.has_rss ? "" : "no ");
1580
1581         if (typer & GICD_TYPER_MBIS) {
1582                 err = mbi_init(handle, gic_data.domain);
1583                 if (err)
1584                         pr_err("Failed to initialize MBIs\n");
1585         }
1586
1587         set_handle_irq(gic_handle_irq);
1588
1589         gic_update_rdist_properties();
1590
1591         gic_smp_init();
1592         gic_dist_init();
1593         gic_cpu_init();
1594         gic_cpu_pm_init();
1595
1596         if (gic_dist_supports_lpis()) {
1597                 its_init(handle, &gic_data.rdists, gic_data.domain);
1598                 its_cpu_init();
1599         } else {
1600                 if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
1601                         gicv2m_init(handle, gic_data.domain);
1602         }
1603
1604         gic_enable_nmi_support();
1605
1606         return 0;
1607
1608 out_free:
1609         if (gic_data.domain)
1610                 irq_domain_remove(gic_data.domain);
1611         free_percpu(gic_data.rdists.rdist);
1612         return err;
1613 }
1614
1615 static int __init gic_validate_dist_version(void __iomem *dist_base)
1616 {
1617         u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
1618
1619         if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4)
1620                 return -ENODEV;
1621
1622         return 0;
1623 }
1624
1625 /* Create all possible partitions at boot time */
1626 static void __init gic_populate_ppi_partitions(struct device_node *gic_node)
1627 {
1628         struct device_node *parts_node, *child_part;
1629         int part_idx = 0, i;
1630         int nr_parts;
1631         struct partition_affinity *parts;
1632
1633         parts_node = of_get_child_by_name(gic_node, "ppi-partitions");
1634         if (!parts_node)
1635                 return;
1636
1637         gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL);
1638         if (!gic_data.ppi_descs)
1639                 return;
1640
1641         nr_parts = of_get_child_count(parts_node);
1642
1643         if (!nr_parts)
1644                 goto out_put_node;
1645
1646         parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL);
1647         if (WARN_ON(!parts))
1648                 goto out_put_node;
1649
1650         for_each_child_of_node(parts_node, child_part) {
1651                 struct partition_affinity *part;
1652                 int n;
1653
1654                 part = &parts[part_idx];
1655
1656                 part->partition_id = of_node_to_fwnode(child_part);
1657
1658                 pr_info("GIC: PPI partition %pOFn[%d] { ",
1659                         child_part, part_idx);
1660
1661                 n = of_property_count_elems_of_size(child_part, "affinity",
1662                                                     sizeof(u32));
1663                 WARN_ON(n <= 0);
1664
1665                 for (i = 0; i < n; i++) {
1666                         int err, cpu;
1667                         u32 cpu_phandle;
1668                         struct device_node *cpu_node;
1669
1670                         err = of_property_read_u32_index(child_part, "affinity",
1671                                                          i, &cpu_phandle);
1672                         if (WARN_ON(err))
1673                                 continue;
1674
1675                         cpu_node = of_find_node_by_phandle(cpu_phandle);
1676                         if (WARN_ON(!cpu_node))
1677                                 continue;
1678
1679                         cpu = of_cpu_node_to_id(cpu_node);
1680                         if (WARN_ON(cpu < 0))
1681                                 continue;
1682
1683                         pr_cont("%pOF[%d] ", cpu_node, cpu);
1684
1685                         cpumask_set_cpu(cpu, &part->mask);
1686                 }
1687
1688                 pr_cont("}\n");
1689                 part_idx++;
1690         }
1691
1692         for (i = 0; i < gic_data.ppi_nr; i++) {
1693                 unsigned int irq;
1694                 struct partition_desc *desc;
1695                 struct irq_fwspec ppi_fwspec = {
1696                         .fwnode         = gic_data.fwnode,
1697                         .param_count    = 3,
1698                         .param          = {
1699                                 [0]     = GIC_IRQ_TYPE_PARTITION,
1700                                 [1]     = i,
1701                                 [2]     = IRQ_TYPE_NONE,
1702                         },
1703                 };
1704
1705                 irq = irq_create_fwspec_mapping(&ppi_fwspec);
1706                 if (WARN_ON(!irq))
1707                         continue;
1708                 desc = partition_create_desc(gic_data.fwnode, parts, nr_parts,
1709                                              irq, &partition_domain_ops);
1710                 if (WARN_ON(!desc))
1711                         continue;
1712
1713                 gic_data.ppi_descs[i] = desc;
1714         }
1715
1716 out_put_node:
1717         of_node_put(parts_node);
1718 }
1719
1720 static void __init gic_of_setup_kvm_info(struct device_node *node)
1721 {
1722         int ret;
1723         struct resource r;
1724         u32 gicv_idx;
1725
1726         gic_v3_kvm_info.type = GIC_V3;
1727
1728         gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
1729         if (!gic_v3_kvm_info.maint_irq)
1730                 return;
1731
1732         if (of_property_read_u32(node, "#redistributor-regions",
1733                                  &gicv_idx))
1734                 gicv_idx = 1;
1735
1736         gicv_idx += 3;  /* Also skip GICD, GICC, GICH */
1737         ret = of_address_to_resource(node, gicv_idx, &r);
1738         if (!ret)
1739                 gic_v3_kvm_info.vcpu = r;
1740
1741         gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
1742         gic_set_kvm_info(&gic_v3_kvm_info);
1743 }
1744
1745 static int __init gic_of_init(struct device_node *node, struct device_node *parent)
1746 {
1747         void __iomem *dist_base;
1748         struct redist_region *rdist_regs;
1749         u64 redist_stride;
1750         u32 nr_redist_regions;
1751         int err, i;
1752
1753         dist_base = of_iomap(node, 0);
1754         if (!dist_base) {
1755                 pr_err("%pOF: unable to map gic dist registers\n", node);
1756                 return -ENXIO;
1757         }
1758
1759         err = gic_validate_dist_version(dist_base);
1760         if (err) {
1761                 pr_err("%pOF: no distributor detected, giving up\n", node);
1762                 goto out_unmap_dist;
1763         }
1764
1765         if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
1766                 nr_redist_regions = 1;
1767
1768         rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs),
1769                              GFP_KERNEL);
1770         if (!rdist_regs) {
1771                 err = -ENOMEM;
1772                 goto out_unmap_dist;
1773         }
1774
1775         for (i = 0; i < nr_redist_regions; i++) {
1776                 struct resource res;
1777                 int ret;
1778
1779                 ret = of_address_to_resource(node, 1 + i, &res);
1780                 rdist_regs[i].redist_base = of_iomap(node, 1 + i);
1781                 if (ret || !rdist_regs[i].redist_base) {
1782                         pr_err("%pOF: couldn't map region %d\n", node, i);
1783                         err = -ENODEV;
1784                         goto out_unmap_rdist;
1785                 }
1786                 rdist_regs[i].phys_base = res.start;
1787         }
1788
1789         if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
1790                 redist_stride = 0;
1791
1792         gic_enable_of_quirks(node, gic_quirks, &gic_data);
1793
1794         err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions,
1795                              redist_stride, &node->fwnode);
1796         if (err)
1797                 goto out_unmap_rdist;
1798
1799         gic_populate_ppi_partitions(node);
1800
1801         if (static_branch_likely(&supports_deactivate_key))
1802                 gic_of_setup_kvm_info(node);
1803         return 0;
1804
1805 out_unmap_rdist:
1806         for (i = 0; i < nr_redist_regions; i++)
1807                 if (rdist_regs[i].redist_base)
1808                         iounmap(rdist_regs[i].redist_base);
1809         kfree(rdist_regs);
1810 out_unmap_dist:
1811         iounmap(dist_base);
1812         return err;
1813 }
1814
1815 IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);
1816
1817 #ifdef CONFIG_ACPI
1818 static struct
1819 {
1820         void __iomem *dist_base;
1821         struct redist_region *redist_regs;
1822         u32 nr_redist_regions;
1823         bool single_redist;
1824         u32 maint_irq;
1825         int maint_irq_mode;
1826         phys_addr_t vcpu_base;
1827 } acpi_data __initdata;
1828
1829 static void __init
1830 gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base)
1831 {
1832         static int count = 0;
1833
1834         acpi_data.redist_regs[count].phys_base = phys_base;
1835         acpi_data.redist_regs[count].redist_base = redist_base;
1836         acpi_data.redist_regs[count].single_redist = acpi_data.single_redist;
1837         count++;
1838 }
1839
1840 static int __init
1841 gic_acpi_parse_madt_redist(union acpi_subtable_headers *header,
1842                            const unsigned long end)
1843 {
1844         struct acpi_madt_generic_redistributor *redist =
1845                         (struct acpi_madt_generic_redistributor *)header;
1846         void __iomem *redist_base;
1847
1848         redist_base = ioremap(redist->base_address, redist->length);
1849         if (!redist_base) {
1850                 pr_err("Couldn't map GICR region @%llx\n", redist->base_address);
1851                 return -ENOMEM;
1852         }
1853
1854         gic_acpi_register_redist(redist->base_address, redist_base);
1855         return 0;
1856 }
1857
1858 static int __init
1859 gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header,
1860                          const unsigned long end)
1861 {
1862         struct acpi_madt_generic_interrupt *gicc =
1863                                 (struct acpi_madt_generic_interrupt *)header;
1864         u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
1865         u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2;
1866         void __iomem *redist_base;
1867
1868         /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */
1869         if (!(gicc->flags & ACPI_MADT_ENABLED))
1870                 return 0;
1871
1872         redist_base = ioremap(gicc->gicr_base_address, size);
1873         if (!redist_base)
1874                 return -ENOMEM;
1875
1876         gic_acpi_register_redist(gicc->gicr_base_address, redist_base);
1877         return 0;
1878 }
1879
1880 static int __init gic_acpi_collect_gicr_base(void)
1881 {
1882         acpi_tbl_entry_handler redist_parser;
1883         enum acpi_madt_type type;
1884
1885         if (acpi_data.single_redist) {
1886                 type = ACPI_MADT_TYPE_GENERIC_INTERRUPT;
1887                 redist_parser = gic_acpi_parse_madt_gicc;
1888         } else {
1889                 type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR;
1890                 redist_parser = gic_acpi_parse_madt_redist;
1891         }
1892
1893         /* Collect redistributor base addresses in GICR entries */
1894         if (acpi_table_parse_madt(type, redist_parser, 0) > 0)
1895                 return 0;
1896
1897         pr_info("No valid GICR entries exist\n");
1898         return -ENODEV;
1899 }
1900
1901 static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header,
1902                                   const unsigned long end)
1903 {
1904         /* Subtable presence means that redist exists, that's it */
1905         return 0;
1906 }
1907
1908 static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header,
1909                                       const unsigned long end)
1910 {
1911         struct acpi_madt_generic_interrupt *gicc =
1912                                 (struct acpi_madt_generic_interrupt *)header;
1913
1914         /*
1915          * If GICC is enabled and has valid gicr base address, then it means
1916          * GICR base is presented via GICC
1917          */
1918         if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address)
1919                 return 0;
1920
1921         /*
1922          * It's perfectly valid firmware can pass disabled GICC entry, driver
1923          * should not treat as errors, skip the entry instead of probe fail.
1924          */
1925         if (!(gicc->flags & ACPI_MADT_ENABLED))
1926                 return 0;
1927
1928         return -ENODEV;
1929 }
1930
1931 static int __init gic_acpi_count_gicr_regions(void)
1932 {
1933         int count;
1934
1935         /*
1936          * Count how many redistributor regions we have. It is not allowed
1937          * to mix redistributor description, GICR and GICC subtables have to be
1938          * mutually exclusive.
1939          */
1940         count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR,
1941                                       gic_acpi_match_gicr, 0);
1942         if (count > 0) {
1943                 acpi_data.single_redist = false;
1944                 return count;
1945         }
1946
1947         count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
1948                                       gic_acpi_match_gicc, 0);
1949         if (count > 0)
1950                 acpi_data.single_redist = true;
1951
1952         return count;
1953 }
1954
1955 static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header,
1956                                            struct acpi_probe_entry *ape)
1957 {
1958         struct acpi_madt_generic_distributor *dist;
1959         int count;
1960
1961         dist = (struct acpi_madt_generic_distributor *)header;
1962         if (dist->version != ape->driver_data)
1963                 return false;
1964
1965         /* We need to do that exercise anyway, the sooner the better */
1966         count = gic_acpi_count_gicr_regions();
1967         if (count <= 0)
1968                 return false;
1969
1970         acpi_data.nr_redist_regions = count;
1971         return true;
1972 }
1973
1974 static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header,
1975                                                 const unsigned long end)
1976 {
1977         struct acpi_madt_generic_interrupt *gicc =
1978                 (struct acpi_madt_generic_interrupt *)header;
1979         int maint_irq_mode;
1980         static int first_madt = true;
1981
1982         /* Skip unusable CPUs */
1983         if (!(gicc->flags & ACPI_MADT_ENABLED))
1984                 return 0;
1985
1986         maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ?
1987                 ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE;
1988
1989         if (first_madt) {
1990                 first_madt = false;
1991
1992                 acpi_data.maint_irq = gicc->vgic_interrupt;
1993                 acpi_data.maint_irq_mode = maint_irq_mode;
1994                 acpi_data.vcpu_base = gicc->gicv_base_address;
1995
1996                 return 0;
1997         }
1998
1999         /*
2000          * The maintenance interrupt and GICV should be the same for every CPU
2001          */
2002         if ((acpi_data.maint_irq != gicc->vgic_interrupt) ||
2003             (acpi_data.maint_irq_mode != maint_irq_mode) ||
2004             (acpi_data.vcpu_base != gicc->gicv_base_address))
2005                 return -EINVAL;
2006
2007         return 0;
2008 }
2009
2010 static bool __init gic_acpi_collect_virt_info(void)
2011 {
2012         int count;
2013
2014         count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
2015                                       gic_acpi_parse_virt_madt_gicc, 0);
2016
2017         return (count > 0);
2018 }
2019
2020 #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K)
2021 #define ACPI_GICV2_VCTRL_MEM_SIZE       (SZ_4K)
2022 #define ACPI_GICV2_VCPU_MEM_SIZE        (SZ_8K)
2023
2024 static void __init gic_acpi_setup_kvm_info(void)
2025 {
2026         int irq;
2027
2028         if (!gic_acpi_collect_virt_info()) {
2029                 pr_warn("Unable to get hardware information used for virtualization\n");
2030                 return;
2031         }
2032
2033         gic_v3_kvm_info.type = GIC_V3;
2034
2035         irq = acpi_register_gsi(NULL, acpi_data.maint_irq,
2036                                 acpi_data.maint_irq_mode,
2037                                 ACPI_ACTIVE_HIGH);
2038         if (irq <= 0)
2039                 return;
2040
2041         gic_v3_kvm_info.maint_irq = irq;
2042
2043         if (acpi_data.vcpu_base) {
2044                 struct resource *vcpu = &gic_v3_kvm_info.vcpu;
2045
2046                 vcpu->flags = IORESOURCE_MEM;
2047                 vcpu->start = acpi_data.vcpu_base;
2048                 vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1;
2049         }
2050
2051         gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
2052         gic_set_kvm_info(&gic_v3_kvm_info);
2053 }
2054
2055 static int __init
2056 gic_acpi_init(struct acpi_subtable_header *header, const unsigned long end)
2057 {
2058         struct acpi_madt_generic_distributor *dist;
2059         struct fwnode_handle *domain_handle;
2060         size_t size;
2061         int i, err;
2062
2063         /* Get distributor base address */
2064         dist = (struct acpi_madt_generic_distributor *)header;
2065         acpi_data.dist_base = ioremap(dist->base_address,
2066                                       ACPI_GICV3_DIST_MEM_SIZE);
2067         if (!acpi_data.dist_base) {
2068                 pr_err("Unable to map GICD registers\n");
2069                 return -ENOMEM;
2070         }
2071
2072         err = gic_validate_dist_version(acpi_data.dist_base);
2073         if (err) {
2074                 pr_err("No distributor detected at @%p, giving up\n",
2075                        acpi_data.dist_base);
2076                 goto out_dist_unmap;
2077         }
2078
2079         size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions;
2080         acpi_data.redist_regs = kzalloc(size, GFP_KERNEL);
2081         if (!acpi_data.redist_regs) {
2082                 err = -ENOMEM;
2083                 goto out_dist_unmap;
2084         }
2085
2086         err = gic_acpi_collect_gicr_base();
2087         if (err)
2088                 goto out_redist_unmap;
2089
2090         domain_handle = irq_domain_alloc_fwnode(&dist->base_address);
2091         if (!domain_handle) {
2092                 err = -ENOMEM;
2093                 goto out_redist_unmap;
2094         }
2095
2096         err = gic_init_bases(acpi_data.dist_base, acpi_data.redist_regs,
2097                              acpi_data.nr_redist_regions, 0, domain_handle);
2098         if (err)
2099                 goto out_fwhandle_free;
2100
2101         acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, domain_handle);
2102
2103         if (static_branch_likely(&supports_deactivate_key))
2104                 gic_acpi_setup_kvm_info();
2105
2106         return 0;
2107
2108 out_fwhandle_free:
2109         irq_domain_free_fwnode(domain_handle);
2110 out_redist_unmap:
2111         for (i = 0; i < acpi_data.nr_redist_regions; i++)
2112                 if (acpi_data.redist_regs[i].redist_base)
2113                         iounmap(acpi_data.redist_regs[i].redist_base);
2114         kfree(acpi_data.redist_regs);
2115 out_dist_unmap:
2116         iounmap(acpi_data.dist_base);
2117         return err;
2118 }
2119 IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2120                      acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3,
2121                      gic_acpi_init);
2122 IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2123                      acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4,
2124                      gic_acpi_init);
2125 IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
2126                      acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE,
2127                      gic_acpi_init);
2128 #endif