drivers/irqchip/irq-gic-v3-its.c

   1 /*
   2  * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/acpi.h>
  19 #include <linux/acpi_iort.h>
  20 #include <linux/bitmap.h>
  21 #include <linux/cpu.h>
  22 #include <linux/delay.h>
  23 #include <linux/dma-iommu.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/irqdomain.h>
  26 #include <linux/list.h>
  27 #include <linux/list_sort.h>
  28 #include <linux/log2.h>
  29 #include <linux/mm.h>
  30 #include <linux/msi.h>
  31 #include <linux/of.h>
  32 #include <linux/of_address.h>
  33 #include <linux/of_irq.h>
  34 #include <linux/of_pci.h>
  35 #include <linux/of_platform.h>
  36 #include <linux/percpu.h>
  37 #include <linux/slab.h>
  38 #include <linux/syscore_ops.h>
  39
  40 #include <linux/irqchip.h>
  41 #include <linux/irqchip/arm-gic-v3.h>
  42 #include <linux/irqchip/arm-gic-v4.h>
  43
  44 #include <asm/cputype.h>
  45 #include <asm/exception.h>
  46
  47 #include "irq-gic-common.h"
  48
  49 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING           (1ULL << 0)
  50 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375       (1ULL << 1)
  51 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144       (1ULL << 2)
  52 #define ITS_FLAGS_SAVE_SUSPEND_STATE            (1ULL << 3)
  53
  54 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING     (1 << 0)
  55
  56 static u32 lpi_id_bits;
  57
  58 /*
  59  * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
  60  * deal with (one configuration byte per interrupt). PENDBASE has to
  61  * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
  62  */
  63 #define LPI_NRBITS              lpi_id_bits
  64 #define LPI_PROPBASE_SZ         ALIGN(BIT(LPI_NRBITS), SZ_64K)
  65 #define LPI_PENDBASE_SZ         ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
  66
  67 #define LPI_PROP_DEFAULT_PRIO   0xa0
  68
  69 /*
  70  * Collection structure - just an ID, and a redistributor address to
  71  * ping. We use one per CPU as a bag of interrupts assigned to this
  72  * CPU.
  73  */
  74 struct its_collection {
  75         u64                     target_address;
  76         u16                     col_id;
  77 };
  78
  79 /*
  80  * The ITS_BASER structure - contains memory information, cached
  81  * value of BASER register configuration and ITS page size.
  82  */
  83 struct its_baser {
  84         void            *base;
  85         u64             val;
  86         u32             order;
  87         u32             psz;
  88 };
  89
  90 struct its_device;
  91
  92 /*
  93  * The ITS structure - contains most of the infrastructure, with the
  94  * top-level MSI domain, the command queue, the collections, and the
  95  * list of devices writing to it.
  96  */
  97 struct its_node {
  98         raw_spinlock_t          lock;
  99         struct list_head        entry;
 100         void __iomem            *base;
 101         phys_addr_t             phys_base;
 102         struct its_cmd_block    *cmd_base;
 103         struct its_cmd_block    *cmd_write;
 104         struct its_baser        tables[GITS_BASER_NR_REGS];
 105         struct its_collection   *collections;
 106         struct fwnode_handle    *fwnode_handle;
 107         u64                     (*get_msi_base)(struct its_device *its_dev);
 108         u64                     cbaser_save;
 109         u32                     ctlr_save;
 110         struct list_head        its_device_list;
 111         u64                     flags;
 112         unsigned long           list_nr;
 113         u32                     ite_size;
 114         u32                     device_ids;
 115         int                     numa_node;
 116         unsigned int            msi_domain_flags;
 117         u32                     pre_its_base; /* for Socionext Synquacer */
 118         bool                    is_v4;
 119         int                     vlpi_redist_offset;
 120 };
 121
 122 #define ITS_ITT_ALIGN           SZ_256
 123
 124 /* The maximum number of VPEID bits supported by VLPI commands */
 125 #define ITS_MAX_VPEID_BITS      (16)
 126 #define ITS_MAX_VPEID           (1 << (ITS_MAX_VPEID_BITS))
 127
 128 /* Convert page order to size in bytes */
 129 #define PAGE_ORDER_TO_SIZE(o)   (PAGE_SIZE << (o))
 130
 131 struct event_lpi_map {
 132         unsigned long           *lpi_map;
 133         u16                     *col_map;
 134         irq_hw_number_t         lpi_base;
 135         int                     nr_lpis;
 136         struct mutex            vlpi_lock;
 137         struct its_vm           *vm;
 138         struct its_vlpi_map     *vlpi_maps;
 139         int                     nr_vlpis;
 140 };
 141
 142 /*
 143  * The ITS view of a device - belongs to an ITS, owns an interrupt
 144  * translation table, and a list of interrupts.  If it some of its
 145  * LPIs are injected into a guest (GICv4), the event_map.vm field
 146  * indicates which one.
 147  */
 148 struct its_device {
 149         struct list_head        entry;
 150         struct its_node         *its;
 151         struct event_lpi_map    event_map;
 152         void                    *itt;
 153         u32                     nr_ites;
 154         u32                     device_id;
 155 };
 156
 157 static struct {
 158         raw_spinlock_t          lock;
 159         struct its_device       *dev;
 160         struct its_vpe          **vpes;
 161         int                     next_victim;
 162 } vpe_proxy;
 163
 164 static LIST_HEAD(its_nodes);
 165 static DEFINE_RAW_SPINLOCK(its_lock);
 166 static struct rdists *gic_rdists;
 167 static struct irq_domain *its_parent;
 168
 169 static unsigned long its_list_map;
 170 static u16 vmovp_seq_num;
 171 static DEFINE_RAW_SPINLOCK(vmovp_lock);
 172
 173 static DEFINE_IDA(its_vpeid_ida);
 174
 175 #define gic_data_rdist()                (raw_cpu_ptr(gic_rdists->rdist))
 176 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
 177 #define gic_data_rdist_vlpi_base()      (gic_data_rdist_rd_base() + SZ_128K)
 178
 179 static struct its_collection *dev_event_to_col(struct its_device *its_dev,
 180                                                u32 event)
 181 {
 182         struct its_node *its = its_dev->its;
 183
 184         return its->collections + its_dev->event_map.col_map[event];
 185 }
 186
 187 static struct its_collection *valid_col(struct its_collection *col)
 188 {
 189         if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(0, 15)))
 190                 return NULL;
 191
 192         return col;
 193 }
 194
 195 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
 196 {
 197         if (valid_col(its->collections + vpe->col_idx))
 198                 return vpe;
 199
 200         return NULL;
 201 }
 202
 203 /*
 204  * ITS command descriptors - parameters to be encoded in a command
 205  * block.
 206  */
 207 struct its_cmd_desc {
 208         union {
 209                 struct {
 210                         struct its_device *dev;
 211                         u32 event_id;
 212                 } its_inv_cmd;
 213
 214                 struct {
 215                         struct its_device *dev;
 216                         u32 event_id;
 217                 } its_clear_cmd;
 218
 219                 struct {
 220                         struct its_device *dev;
 221                         u32 event_id;
 222                 } its_int_cmd;
 223
 224                 struct {
 225                         struct its_device *dev;
 226                         int valid;
 227                 } its_mapd_cmd;
 228
 229                 struct {
 230                         struct its_collection *col;
 231                         int valid;
 232                 } its_mapc_cmd;
 233
 234                 struct {
 235                         struct its_device *dev;
 236                         u32 phys_id;
 237                         u32 event_id;
 238                 } its_mapti_cmd;
 239
 240                 struct {
 241                         struct its_device *dev;
 242                         struct its_collection *col;
 243                         u32 event_id;
 244                 } its_movi_cmd;
 245
 246                 struct {
 247                         struct its_device *dev;
 248                         u32 event_id;
 249                 } its_discard_cmd;
 250
 251                 struct {
 252                         struct its_collection *col;
 253                 } its_invall_cmd;
 254
 255                 struct {
 256                         struct its_vpe *vpe;
 257                 } its_vinvall_cmd;
 258
 259                 struct {
 260                         struct its_vpe *vpe;
 261                         struct its_collection *col;
 262                         bool valid;
 263                 } its_vmapp_cmd;
 264
 265                 struct {
 266                         struct its_vpe *vpe;
 267                         struct its_device *dev;
 268                         u32 virt_id;
 269                         u32 event_id;
 270                         bool db_enabled;
 271                 } its_vmapti_cmd;
 272
 273                 struct {
 274                         struct its_vpe *vpe;
 275                         struct its_device *dev;
 276                         u32 event_id;
 277                         bool db_enabled;
 278                 } its_vmovi_cmd;
 279
 280                 struct {
 281                         struct its_vpe *vpe;
 282                         struct its_collection *col;
 283                         u16 seq_num;
 284                         u16 its_list;
 285                 } its_vmovp_cmd;
 286         };
 287 };
 288
 289 /*
 290  * The ITS command block, which is what the ITS actually parses.
 291  */
 292 struct its_cmd_block {
 293         u64     raw_cmd[4];
 294 };
 295
 296 #define ITS_CMD_QUEUE_SZ                SZ_64K
 297 #define ITS_CMD_QUEUE_NR_ENTRIES        (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
 298
 299 typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
 300                                                     struct its_cmd_block *,
 301                                                     struct its_cmd_desc *);
 302
 303 typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
 304                                               struct its_cmd_block *,
 305                                               struct its_cmd_desc *);
 306
 307 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
 308 {
 309         u64 mask = GENMASK_ULL(h, l);
 310         *raw_cmd &= ~mask;
 311         *raw_cmd |= (val << l) & mask;
 312 }
 313
 314 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
 315 {
 316         its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
 317 }
 318
 319 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
 320 {
 321         its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
 322 }
 323
 324 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
 325 {
 326         its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
 327 }
 328
 329 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
 330 {
 331         its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
 332 }
 333
 334 static void its_encode_size(struct its_cmd_block *cmd, u8 size)
 335 {
 336         its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
 337 }
 338
 339 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
 340 {
 341         its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
 342 }
 343
 344 static void its_encode_valid(struct its_cmd_block *cmd, int valid)
 345 {
 346         its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
 347 }
 348
 349 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
 350 {
 351         its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
 352 }
 353
 354 static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
 355 {
 356         its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
 357 }
 358
 359 static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
 360 {
 361         its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
 362 }
 363
 364 static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
 365 {
 366         its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
 367 }
 368
 369 static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
 370 {
 371         its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
 372 }
 373
 374 static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
 375 {
 376         its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
 377 }
 378
 379 static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
 380 {
 381         its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
 382 }
 383
 384 static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
 385 {
 386         its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
 387 }
 388
 389 static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
 390 {
 391         its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
 392 }
 393
 394 static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
 395 {
 396         its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
 397 }
 398
 399 static inline void its_fixup_cmd(struct its_cmd_block *cmd)
 400 {
 401         /* Let's fixup BE commands */
 402         cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]);
 403         cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]);
 404         cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]);
 405         cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]);
 406 }
 407
 408 static struct its_collection *its_build_mapd_cmd(struct its_node *its,
 409                                                  struct its_cmd_block *cmd,
 410                                                  struct its_cmd_desc *desc)
 411 {
 412         unsigned long itt_addr;
 413         u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
 414
 415         itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
 416         itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
 417
 418         its_encode_cmd(cmd, GITS_CMD_MAPD);
 419         its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
 420         its_encode_size(cmd, size - 1);
 421         its_encode_itt(cmd, itt_addr);
 422         its_encode_valid(cmd, desc->its_mapd_cmd.valid);
 423
 424         its_fixup_cmd(cmd);
 425
 426         return NULL;
 427 }
 428
 429 static struct its_collection *its_build_mapc_cmd(struct its_node *its,
 430                                                  struct its_cmd_block *cmd,
 431                                                  struct its_cmd_desc *desc)
 432 {
 433         its_encode_cmd(cmd, GITS_CMD_MAPC);
 434         its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
 435         its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
 436         its_encode_valid(cmd, desc->its_mapc_cmd.valid);
 437
 438         its_fixup_cmd(cmd);
 439
 440         return desc->its_mapc_cmd.col;
 441 }
 442
 443 static struct its_collection *its_build_mapti_cmd(struct its_node *its,
 444                                                   struct its_cmd_block *cmd,
 445                                                   struct its_cmd_desc *desc)
 446 {
 447         struct its_collection *col;
 448
 449         col = dev_event_to_col(desc->its_mapti_cmd.dev,
 450                                desc->its_mapti_cmd.event_id);
 451
 452         its_encode_cmd(cmd, GITS_CMD_MAPTI);
 453         its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
 454         its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
 455         its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
 456         its_encode_collection(cmd, col->col_id);
 457
 458         its_fixup_cmd(cmd);
 459
 460         return valid_col(col);
 461 }
 462
 463 static struct its_collection *its_build_movi_cmd(struct its_node *its,
 464                                                  struct its_cmd_block *cmd,
 465                                                  struct its_cmd_desc *desc)
 466 {
 467         struct its_collection *col;
 468
 469         col = dev_event_to_col(desc->its_movi_cmd.dev,
 470                                desc->its_movi_cmd.event_id);
 471
 472         its_encode_cmd(cmd, GITS_CMD_MOVI);
 473         its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
 474         its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
 475         its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
 476
 477         its_fixup_cmd(cmd);
 478
 479         return valid_col(col);
 480 }
 481
 482 static struct its_collection *its_build_discard_cmd(struct its_node *its,
 483                                                     struct its_cmd_block *cmd,
 484                                                     struct its_cmd_desc *desc)
 485 {
 486         struct its_collection *col;
 487
 488         col = dev_event_to_col(desc->its_discard_cmd.dev,
 489                                desc->its_discard_cmd.event_id);
 490
 491         its_encode_cmd(cmd, GITS_CMD_DISCARD);
 492         its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
 493         its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
 494
 495         its_fixup_cmd(cmd);
 496
 497         return valid_col(col);
 498 }
 499
 500 static struct its_collection *its_build_inv_cmd(struct its_node *its,
 501                                                 struct its_cmd_block *cmd,
 502                                                 struct its_cmd_desc *desc)
 503 {
 504         struct its_collection *col;
 505
 506         col = dev_event_to_col(desc->its_inv_cmd.dev,
 507                                desc->its_inv_cmd.event_id);
 508
 509         its_encode_cmd(cmd, GITS_CMD_INV);
 510         its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
 511         its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
 512
 513         its_fixup_cmd(cmd);
 514
 515         return valid_col(col);
 516 }
 517
 518 static struct its_collection *its_build_int_cmd(struct its_node *its,
 519                                                 struct its_cmd_block *cmd,
 520                                                 struct its_cmd_desc *desc)
 521 {
 522         struct its_collection *col;
 523
 524         col = dev_event_to_col(desc->its_int_cmd.dev,
 525                                desc->its_int_cmd.event_id);
 526
 527         its_encode_cmd(cmd, GITS_CMD_INT);
 528         its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
 529         its_encode_event_id(cmd, desc->its_int_cmd.event_id);
 530
 531         its_fixup_cmd(cmd);
 532
 533         return valid_col(col);
 534 }
 535
 536 static struct its_collection *its_build_clear_cmd(struct its_node *its,
 537                                                   struct its_cmd_block *cmd,
 538                                                   struct its_cmd_desc *desc)
 539 {
 540         struct its_collection *col;
 541
 542         col = dev_event_to_col(desc->its_clear_cmd.dev,
 543                                desc->its_clear_cmd.event_id);
 544
 545         its_encode_cmd(cmd, GITS_CMD_CLEAR);
 546         its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
 547         its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
 548
 549         its_fixup_cmd(cmd);
 550
 551         return valid_col(col);
 552 }
 553
 554 static struct its_collection *its_build_invall_cmd(struct its_node *its,
 555                                                    struct its_cmd_block *cmd,
 556                                                    struct its_cmd_desc *desc)
 557 {
 558         its_encode_cmd(cmd, GITS_CMD_INVALL);
 559         its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
 560
 561         its_fixup_cmd(cmd);
 562
 563         return NULL;
 564 }
 565
 566 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
 567                                              struct its_cmd_block *cmd,
 568                                              struct its_cmd_desc *desc)
 569 {
 570         its_encode_cmd(cmd, GITS_CMD_VINVALL);
 571         its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
 572
 573         its_fixup_cmd(cmd);
 574
 575         return valid_vpe(its, desc->its_vinvall_cmd.vpe);
 576 }
 577
 578 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
 579                                            struct its_cmd_block *cmd,
 580                                            struct its_cmd_desc *desc)
 581 {
 582         unsigned long vpt_addr;
 583         u64 target;
 584
 585         vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
 586         target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
 587
 588         its_encode_cmd(cmd, GITS_CMD_VMAPP);
 589         its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
 590         its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
 591         its_encode_target(cmd, target);
 592         its_encode_vpt_addr(cmd, vpt_addr);
 593         its_encode_vpt_size(cmd, LPI_NRBITS - 1);
 594
 595         its_fixup_cmd(cmd);
 596
 597         return valid_vpe(its, desc->its_vmapp_cmd.vpe);
 598 }
 599
 600 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
 601                                             struct its_cmd_block *cmd,
 602                                             struct its_cmd_desc *desc)
 603 {
 604         u32 db;
 605
 606         if (desc->its_vmapti_cmd.db_enabled)
 607                 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
 608         else
 609                 db = 1023;
 610
 611         its_encode_cmd(cmd, GITS_CMD_VMAPTI);
 612         its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
 613         its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
 614         its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
 615         its_encode_db_phys_id(cmd, db);
 616         its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
 617
 618         its_fixup_cmd(cmd);
 619
 620         return valid_vpe(its, desc->its_vmapti_cmd.vpe);
 621 }
 622
 623 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
 624                                            struct its_cmd_block *cmd,
 625                                            struct its_cmd_desc *desc)
 626 {
 627         u32 db;
 628
 629         if (desc->its_vmovi_cmd.db_enabled)
 630                 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
 631         else
 632                 db = 1023;
 633
 634         its_encode_cmd(cmd, GITS_CMD_VMOVI);
 635         its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
 636         its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
 637         its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
 638         its_encode_db_phys_id(cmd, db);
 639         its_encode_db_valid(cmd, true);
 640
 641         its_fixup_cmd(cmd);
 642
 643         return valid_vpe(its, desc->its_vmovi_cmd.vpe);
 644 }
 645
 646 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
 647                                            struct its_cmd_block *cmd,
 648                                            struct its_cmd_desc *desc)
 649 {
 650         u64 target;
 651
 652         target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
 653         its_encode_cmd(cmd, GITS_CMD_VMOVP);
 654         its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
 655         its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
 656         its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
 657         its_encode_target(cmd, target);
 658
 659         its_fixup_cmd(cmd);
 660
 661         return valid_vpe(its, desc->its_vmovp_cmd.vpe);
 662 }
 663
 664 static u64 its_cmd_ptr_to_offset(struct its_node *its,
 665                                  struct its_cmd_block *ptr)
 666 {
 667         return (ptr - its->cmd_base) * sizeof(*ptr);
 668 }
 669
 670 static int its_queue_full(struct its_node *its)
 671 {
 672         int widx;
 673         int ridx;
 674
 675         widx = its->cmd_write - its->cmd_base;
 676         ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
 677
 678         /* This is incredibly unlikely to happen, unless the ITS locks up. */
 679         if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
 680                 return 1;
 681
 682         return 0;
 683 }
 684
 685 static struct its_cmd_block *its_allocate_entry(struct its_node *its)
 686 {
 687         struct its_cmd_block *cmd;
 688         u32 count = 1000000;    /* 1s! */
 689
 690         while (its_queue_full(its)) {
 691                 count--;
 692                 if (!count) {
 693                         pr_err_ratelimited("ITS queue not draining\n");
 694                         return NULL;
 695                 }
 696                 cpu_relax();
 697                 udelay(1);
 698         }
 699
 700         cmd = its->cmd_write++;
 701
 702         /* Handle queue wrapping */
 703         if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
 704                 its->cmd_write = its->cmd_base;
 705
 706         /* Clear command  */
 707         cmd->raw_cmd[0] = 0;
 708         cmd->raw_cmd[1] = 0;
 709         cmd->raw_cmd[2] = 0;
 710         cmd->raw_cmd[3] = 0;
 711
 712         return cmd;
 713 }
 714
 715 static struct its_cmd_block *its_post_commands(struct its_node *its)
 716 {
 717         u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
 718
 719         writel_relaxed(wr, its->base + GITS_CWRITER);
 720
 721         return its->cmd_write;
 722 }
 723
 724 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
 725 {
 726         /*
 727          * Make sure the commands written to memory are observable by
 728          * the ITS.
 729          */
 730         if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
 731                 gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
 732         else
 733                 dsb(ishst);
 734 }
 735
 736 static int its_wait_for_range_completion(struct its_node *its,
 737                                          struct its_cmd_block *from,
 738                                          struct its_cmd_block *to)
 739 {
 740         u64 rd_idx, from_idx, to_idx;
 741         u32 count = 1000000;    /* 1s! */
 742
 743         from_idx = its_cmd_ptr_to_offset(its, from);
 744         to_idx = its_cmd_ptr_to_offset(its, to);
 745
 746         while (1) {
 747                 rd_idx = readl_relaxed(its->base + GITS_CREADR);
 748
 749                 /* Direct case */
 750                 if (from_idx < to_idx && rd_idx >= to_idx)
 751                         break;
 752
 753                 /* Wrapped case */
 754                 if (from_idx >= to_idx && rd_idx >= to_idx && rd_idx < from_idx)
 755                         break;
 756
 757                 count--;
 758                 if (!count) {
 759                         pr_err_ratelimited("ITS queue timeout (%llu %llu %llu)\n",
 760                                            from_idx, to_idx, rd_idx);
 761                         return -1;
 762                 }
 763                 cpu_relax();
 764                 udelay(1);
 765         }
 766
 767         return 0;
 768 }
 769
 770 /* Warning, macro hell follows */
 771 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn)       \
 772 void name(struct its_node *its,                                         \
 773           buildtype builder,                                            \
 774           struct its_cmd_desc *desc)                                    \
 775 {                                                                       \
 776         struct its_cmd_block *cmd, *sync_cmd, *next_cmd;                \
 777         synctype *sync_obj;                                             \
 778         unsigned long flags;                                            \
 779                                                                         \
 780         raw_spin_lock_irqsave(&its->lock, flags);                       \
 781                                                                         \
 782         cmd = its_allocate_entry(its);                                  \
 783         if (!cmd) {             /* We're soooooo screewed... */         \
 784                 raw_spin_unlock_irqrestore(&its->lock, flags);          \
 785                 return;                                                 \
 786         }                                                               \
 787         sync_obj = builder(its, cmd, desc);                             \
 788         its_flush_cmd(its, cmd);                                        \
 789                                                                         \
 790         if (sync_obj) {                                                 \
 791                 sync_cmd = its_allocate_entry(its);                     \
 792                 if (!sync_cmd)                                          \
 793                         goto post;                                      \
 794                                                                         \
 795                 buildfn(its, sync_cmd, sync_obj);                       \
 796                 its_flush_cmd(its, sync_cmd);                           \
 797         }                                                               \
 798                                                                         \
 799 post:                                                                   \
 800         next_cmd = its_post_commands(its);                              \
 801         raw_spin_unlock_irqrestore(&its->lock, flags);                  \
 802                                                                         \
 803         if (its_wait_for_range_completion(its, cmd, next_cmd))          \
 804                 pr_err_ratelimited("ITS cmd %ps failed\n", builder);    \
 805 }
 806
 807 static void its_build_sync_cmd(struct its_node *its,
 808                                struct its_cmd_block *sync_cmd,
 809                                struct its_collection *sync_col)
 810 {
 811         its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
 812         its_encode_target(sync_cmd, sync_col->target_address);
 813
 814         its_fixup_cmd(sync_cmd);
 815 }
 816
 817 static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
 818                              struct its_collection, its_build_sync_cmd)
 819
 820 static void its_build_vsync_cmd(struct its_node *its,
 821                                 struct its_cmd_block *sync_cmd,
 822                                 struct its_vpe *sync_vpe)
 823 {
 824         its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
 825         its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
 826
 827         its_fixup_cmd(sync_cmd);
 828 }
 829
 830 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
 831                              struct its_vpe, its_build_vsync_cmd)
 832
 833 static void its_send_int(struct its_device *dev, u32 event_id)
 834 {
 835         struct its_cmd_desc desc;
 836
 837         desc.its_int_cmd.dev = dev;
 838         desc.its_int_cmd.event_id = event_id;
 839
 840         its_send_single_command(dev->its, its_build_int_cmd, &desc);
 841 }
 842
 843 static void its_send_clear(struct its_device *dev, u32 event_id)
 844 {
 845         struct its_cmd_desc desc;
 846
 847         desc.its_clear_cmd.dev = dev;
 848         desc.its_clear_cmd.event_id = event_id;
 849
 850         its_send_single_command(dev->its, its_build_clear_cmd, &desc);
 851 }
 852
 853 static void its_send_inv(struct its_device *dev, u32 event_id)
 854 {
 855         struct its_cmd_desc desc;
 856
 857         desc.its_inv_cmd.dev = dev;
 858         desc.its_inv_cmd.event_id = event_id;
 859
 860         its_send_single_command(dev->its, its_build_inv_cmd, &desc);
 861 }
 862
 863 static void its_send_mapd(struct its_device *dev, int valid)
 864 {
 865         struct its_cmd_desc desc;
 866
 867         desc.its_mapd_cmd.dev = dev;
 868         desc.its_mapd_cmd.valid = !!valid;
 869
 870         its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
 871 }
 872
 873 static void its_send_mapc(struct its_node *its, struct its_collection *col,
 874                           int valid)
 875 {
 876         struct its_cmd_desc desc;
 877
 878         desc.its_mapc_cmd.col = col;
 879         desc.its_mapc_cmd.valid = !!valid;
 880
 881         its_send_single_command(its, its_build_mapc_cmd, &desc);
 882 }
 883
 884 static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
 885 {
 886         struct its_cmd_desc desc;
 887
 888         desc.its_mapti_cmd.dev = dev;
 889         desc.its_mapti_cmd.phys_id = irq_id;
 890         desc.its_mapti_cmd.event_id = id;
 891
 892         its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
 893 }
 894
 895 static void its_send_movi(struct its_device *dev,
 896                           struct its_collection *col, u32 id)
 897 {
 898         struct its_cmd_desc desc;
 899
 900         desc.its_movi_cmd.dev = dev;
 901         desc.its_movi_cmd.col = col;
 902         desc.its_movi_cmd.event_id = id;
 903
 904         its_send_single_command(dev->its, its_build_movi_cmd, &desc);
 905 }
 906
 907 static void its_send_discard(struct its_device *dev, u32 id)
 908 {
 909         struct its_cmd_desc desc;
 910
 911         desc.its_discard_cmd.dev = dev;
 912         desc.its_discard_cmd.event_id = id;
 913
 914         its_send_single_command(dev->its, its_build_discard_cmd, &desc);
 915 }
 916
 917 static void its_send_invall(struct its_node *its, struct its_collection *col)
 918 {
 919         struct its_cmd_desc desc;
 920
 921         desc.its_invall_cmd.col = col;
 922
 923         its_send_single_command(its, its_build_invall_cmd, &desc);
 924 }
 925
 926 static void its_send_vmapti(struct its_device *dev, u32 id)
 927 {
 928         struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id];
 929         struct its_cmd_desc desc;
 930
 931         desc.its_vmapti_cmd.vpe = map->vpe;
 932         desc.its_vmapti_cmd.dev = dev;
 933         desc.its_vmapti_cmd.virt_id = map->vintid;
 934         desc.its_vmapti_cmd.event_id = id;
 935         desc.its_vmapti_cmd.db_enabled = map->db_enabled;
 936
 937         its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
 938 }
 939
 940 static void its_send_vmovi(struct its_device *dev, u32 id)
 941 {
 942         struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id];
 943         struct its_cmd_desc desc;
 944
 945         desc.its_vmovi_cmd.vpe = map->vpe;
 946         desc.its_vmovi_cmd.dev = dev;
 947         desc.its_vmovi_cmd.event_id = id;
 948         desc.its_vmovi_cmd.db_enabled = map->db_enabled;
 949
 950         its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
 951 }
 952
 953 static void its_send_vmapp(struct its_node *its,
 954                            struct its_vpe *vpe, bool valid)
 955 {
 956         struct its_cmd_desc desc;
 957
 958         desc.its_vmapp_cmd.vpe = vpe;
 959         desc.its_vmapp_cmd.valid = valid;
 960         desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
 961
 962         its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
 963 }
 964
 965 static void its_send_vmovp(struct its_vpe *vpe)
 966 {
 967         struct its_cmd_desc desc;
 968         struct its_node *its;
 969         unsigned long flags;
 970         int col_id = vpe->col_idx;
 971
 972         desc.its_vmovp_cmd.vpe = vpe;
 973         desc.its_vmovp_cmd.its_list = (u16)its_list_map;
 974
 975         if (!its_list_map) {
 976                 its = list_first_entry(&its_nodes, struct its_node, entry);
 977                 desc.its_vmovp_cmd.seq_num = 0;
 978                 desc.its_vmovp_cmd.col = &its->collections[col_id];
 979                 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
 980                 return;
 981         }
 982
 983         /*
 984          * Yet another marvel of the architecture. If using the
 985          * its_list "feature", we need to make sure that all ITSs
 986          * receive all VMOVP commands in the same order. The only way
 987          * to guarantee this is to make vmovp a serialization point.
 988          *
 989          * Wall <-- Head.
 990          */
 991         raw_spin_lock_irqsave(&vmovp_lock, flags);
 992
 993         desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
 994
 995         /* Emit VMOVPs */
 996         list_for_each_entry(its, &its_nodes, entry) {
 997                 if (!its->is_v4)
 998                         continue;
 999
1000                 if (!vpe->its_vm->vlpi_count[its->list_nr])
1001                         continue;
1002
1003                 desc.its_vmovp_cmd.col = &its->collections[col_id];
1004                 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1005         }
1006
1007         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1008 }
1009
1010 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
1011 {
1012         struct its_cmd_desc desc;
1013
1014         desc.its_vinvall_cmd.vpe = vpe;
1015         its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1016 }
1017
1018 /*
1019  * irqchip functions - assumes MSI, mostly.
1020  */
1021
1022 static inline u32 its_get_event_id(struct irq_data *d)
1023 {
1024         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1025         return d->hwirq - its_dev->event_map.lpi_base;
1026 }
1027
1028 static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1029 {
1030         irq_hw_number_t hwirq;
1031         struct page *prop_page;
1032         u8 *cfg;
1033
1034         if (irqd_is_forwarded_to_vcpu(d)) {
1035                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1036                 u32 event = its_get_event_id(d);
1037                 struct its_vlpi_map *map;
1038
1039                 prop_page = its_dev->event_map.vm->vprop_page;
1040                 map = &its_dev->event_map.vlpi_maps[event];
1041                 hwirq = map->vintid;
1042
1043                 /* Remember the updated property */
1044                 map->properties &= ~clr;
1045                 map->properties |= set | LPI_PROP_GROUP1;
1046         } else {
1047                 prop_page = gic_rdists->prop_page;
1048                 hwirq = d->hwirq;
1049         }
1050
1051         cfg = page_address(prop_page) + hwirq - 8192;
1052         *cfg &= ~clr;
1053         *cfg |= set | LPI_PROP_GROUP1;
1054
1055         /*
1056          * Make the above write visible to the redistributors.
1057          * And yes, we're flushing exactly: One. Single. Byte.
1058          * Humpf...
1059          */
1060         if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
1061                 gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
1062         else
1063                 dsb(ishst);
1064 }
1065
1066 static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1067 {
1068         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1069
1070         lpi_write_config(d, clr, set);
1071         its_send_inv(its_dev, its_get_event_id(d));
1072 }
1073
1074 static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1075 {
1076         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1077         u32 event = its_get_event_id(d);
1078
1079         if (its_dev->event_map.vlpi_maps[event].db_enabled == enable)
1080                 return;
1081
1082         its_dev->event_map.vlpi_maps[event].db_enabled = enable;
1083
1084         /*
1085          * More fun with the architecture:
1086          *
1087          * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1088          * value or to 1023, depending on the enable bit. But that
1089          * would be issueing a mapping for an /existing/ DevID+EventID
1090          * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1091          * to the /same/ vPE, using this opportunity to adjust the
1092          * doorbell. Mouahahahaha. We loves it, Precious.
1093          */
1094         its_send_vmovi(its_dev, event);
1095 }
1096
1097 static void its_mask_irq(struct irq_data *d)
1098 {
1099         if (irqd_is_forwarded_to_vcpu(d))
1100                 its_vlpi_set_doorbell(d, false);
1101
1102         lpi_update_config(d, LPI_PROP_ENABLED, 0);
1103 }
1104
1105 static void its_unmask_irq(struct irq_data *d)
1106 {
1107         if (irqd_is_forwarded_to_vcpu(d))
1108                 its_vlpi_set_doorbell(d, true);
1109
1110         lpi_update_config(d, 0, LPI_PROP_ENABLED);
1111 }
1112
1113 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1114                             bool force)
1115 {
1116         unsigned int cpu;
1117         const struct cpumask *cpu_mask = cpu_online_mask;
1118         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1119         struct its_collection *target_col;
1120         u32 id = its_get_event_id(d);
1121
1122         /* A forwarded interrupt should use irq_set_vcpu_affinity */
1123         if (irqd_is_forwarded_to_vcpu(d))
1124                 return -EINVAL;
1125
1126        /* lpi cannot be routed to a redistributor that is on a foreign node */
1127         if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
1128                 if (its_dev->its->numa_node >= 0) {
1129                         cpu_mask = cpumask_of_node(its_dev->its->numa_node);
1130                         if (!cpumask_intersects(mask_val, cpu_mask))
1131                                 return -EINVAL;
1132                 }
1133         }
1134
1135         cpu = cpumask_any_and(mask_val, cpu_mask);
1136
1137         if (cpu >= nr_cpu_ids)
1138                 return -EINVAL;
1139
1140         /* don't set the affinity when the target cpu is same as current one */
1141         if (cpu != its_dev->event_map.col_map[id]) {
1142                 target_col = &its_dev->its->collections[cpu];
1143                 its_send_movi(its_dev, target_col, id);
1144                 its_dev->event_map.col_map[id] = cpu;
1145                 irq_data_update_effective_affinity(d, cpumask_of(cpu));
1146         }
1147
1148         return IRQ_SET_MASK_OK_DONE;
1149 }
1150
1151 static u64 its_irq_get_msi_base(struct its_device *its_dev)
1152 {
1153         struct its_node *its = its_dev->its;
1154
1155         return its->phys_base + GITS_TRANSLATER;
1156 }
1157
1158 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1159 {
1160         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1161         struct its_node *its;
1162         u64 addr;
1163
1164         its = its_dev->its;
1165         addr = its->get_msi_base(its_dev);
1166
1167         msg->address_lo         = lower_32_bits(addr);
1168         msg->address_hi         = upper_32_bits(addr);
1169         msg->data               = its_get_event_id(d);
1170
1171         iommu_dma_map_msi_msg(d->irq, msg);
1172 }
1173
1174 static int its_irq_set_irqchip_state(struct irq_data *d,
1175                                      enum irqchip_irq_state which,
1176                                      bool state)
1177 {
1178         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1179         u32 event = its_get_event_id(d);
1180
1181         if (which != IRQCHIP_STATE_PENDING)
1182                 return -EINVAL;
1183
1184         if (state)
1185                 its_send_int(its_dev, event);
1186         else
1187                 its_send_clear(its_dev, event);
1188
1189         return 0;
1190 }
1191
1192 static void its_map_vm(struct its_node *its, struct its_vm *vm)
1193 {
1194         unsigned long flags;
1195
1196         /* Not using the ITS list? Everything is always mapped. */
1197         if (!its_list_map)
1198                 return;
1199
1200         raw_spin_lock_irqsave(&vmovp_lock, flags);
1201
1202         /*
1203          * If the VM wasn't mapped yet, iterate over the vpes and get
1204          * them mapped now.
1205          */
1206         vm->vlpi_count[its->list_nr]++;
1207
1208         if (vm->vlpi_count[its->list_nr] == 1) {
1209                 int i;
1210
1211                 for (i = 0; i < vm->nr_vpes; i++) {
1212                         struct its_vpe *vpe = vm->vpes[i];
1213                         struct irq_data *d = irq_get_irq_data(vpe->irq);
1214
1215                         /* Map the VPE to the first possible CPU */
1216                         vpe->col_idx = cpumask_first(cpu_online_mask);
1217                         its_send_vmapp(its, vpe, true);
1218                         its_send_vinvall(its, vpe);
1219                         irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
1220                 }
1221         }
1222
1223         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1224 }
1225
1226 static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
1227 {
1228         unsigned long flags;
1229
1230         /* Not using the ITS list? Everything is always mapped. */
1231         if (!its_list_map)
1232                 return;
1233
1234         raw_spin_lock_irqsave(&vmovp_lock, flags);
1235
1236         if (!--vm->vlpi_count[its->list_nr]) {
1237                 int i;
1238
1239                 for (i = 0; i < vm->nr_vpes; i++)
1240                         its_send_vmapp(its, vm->vpes[i], false);
1241         }
1242
1243         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1244 }
1245
1246 static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1247 {
1248         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1249         u32 event = its_get_event_id(d);
1250         int ret = 0;
1251
1252         if (!info->map)
1253                 return -EINVAL;
1254
1255         mutex_lock(&its_dev->event_map.vlpi_lock);
1256
1257         if (!its_dev->event_map.vm) {
1258                 struct its_vlpi_map *maps;
1259
1260                 maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
1261                                GFP_KERNEL);
1262                 if (!maps) {
1263                         ret = -ENOMEM;
1264                         goto out;
1265                 }
1266
1267                 its_dev->event_map.vm = info->map->vm;
1268                 its_dev->event_map.vlpi_maps = maps;
1269         } else if (its_dev->event_map.vm != info->map->vm) {
1270                 ret = -EINVAL;
1271                 goto out;
1272         }
1273
1274         /* Get our private copy of the mapping information */
1275         its_dev->event_map.vlpi_maps[event] = *info->map;
1276
1277         if (irqd_is_forwarded_to_vcpu(d)) {
1278                 /* Already mapped, move it around */
1279                 its_send_vmovi(its_dev, event);
1280         } else {
1281                 /* Ensure all the VPEs are mapped on this ITS */
1282                 its_map_vm(its_dev->its, info->map->vm);
1283
1284                 /*
1285                  * Flag the interrupt as forwarded so that we can
1286                  * start poking the virtual property table.
1287                  */
1288                 irqd_set_forwarded_to_vcpu(d);
1289
1290                 /* Write out the property to the prop table */
1291                 lpi_write_config(d, 0xff, info->map->properties);
1292
1293                 /* Drop the physical mapping */
1294                 its_send_discard(its_dev, event);
1295
1296                 /* and install the virtual one */
1297                 its_send_vmapti(its_dev, event);
1298
1299                 /* Increment the number of VLPIs */
1300                 its_dev->event_map.nr_vlpis++;
1301         }
1302
1303 out:
1304         mutex_unlock(&its_dev->event_map.vlpi_lock);
1305         return ret;
1306 }
1307
1308 static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
1309 {
1310         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1311         u32 event = its_get_event_id(d);
1312         int ret = 0;
1313
1314         mutex_lock(&its_dev->event_map.vlpi_lock);
1315
1316         if (!its_dev->event_map.vm ||
1317             !its_dev->event_map.vlpi_maps[event].vm) {
1318                 ret = -EINVAL;
1319                 goto out;
1320         }
1321
1322         /* Copy our mapping information to the incoming request */
1323         *info->map = its_dev->event_map.vlpi_maps[event];
1324
1325 out:
1326         mutex_unlock(&its_dev->event_map.vlpi_lock);
1327         return ret;
1328 }
1329
1330 static int its_vlpi_unmap(struct irq_data *d)
1331 {
1332         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1333         u32 event = its_get_event_id(d);
1334         int ret = 0;
1335
1336         mutex_lock(&its_dev->event_map.vlpi_lock);
1337
1338         if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) {
1339                 ret = -EINVAL;
1340                 goto out;
1341         }
1342
1343         /* Drop the virtual mapping */
1344         its_send_discard(its_dev, event);
1345
1346         /* and restore the physical one */
1347         irqd_clr_forwarded_to_vcpu(d);
1348         its_send_mapti(its_dev, d->hwirq, event);
1349         lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO |
1350                                     LPI_PROP_ENABLED |
1351                                     LPI_PROP_GROUP1));
1352
1353         /* Potentially unmap the VM from this ITS */
1354         its_unmap_vm(its_dev->its, its_dev->event_map.vm);
1355
1356         /*
1357          * Drop the refcount and make the device available again if
1358          * this was the last VLPI.
1359          */
1360         if (!--its_dev->event_map.nr_vlpis) {
1361                 its_dev->event_map.vm = NULL;
1362                 kfree(its_dev->event_map.vlpi_maps);
1363         }
1364
1365 out:
1366         mutex_unlock(&its_dev->event_map.vlpi_lock);
1367         return ret;
1368 }
1369
1370 static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
1371 {
1372         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1373
1374         if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1375                 return -EINVAL;
1376
1377         if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
1378                 lpi_update_config(d, 0xff, info->config);
1379         else
1380                 lpi_write_config(d, 0xff, info->config);
1381         its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
1382
1383         return 0;
1384 }
1385
1386 static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
1387 {
1388         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1389         struct its_cmd_info *info = vcpu_info;
1390
1391         /* Need a v4 ITS */
1392         if (!its_dev->its->is_v4)
1393                 return -EINVAL;
1394
1395         /* Unmap request? */
1396         if (!info)
1397                 return its_vlpi_unmap(d);
1398
1399         switch (info->cmd_type) {
1400         case MAP_VLPI:
1401                 return its_vlpi_map(d, info);
1402
1403         case GET_VLPI:
1404                 return its_vlpi_get(d, info);
1405
1406         case PROP_UPDATE_VLPI:
1407         case PROP_UPDATE_AND_INV_VLPI:
1408                 return its_vlpi_prop_update(d, info);
1409
1410         default:
1411                 return -EINVAL;
1412         }
1413 }
1414
1415 static struct irq_chip its_irq_chip = {
1416         .name                   = "ITS",
1417         .irq_mask               = its_mask_irq,
1418         .irq_unmask             = its_unmask_irq,
1419         .irq_eoi                = irq_chip_eoi_parent,
1420         .irq_set_affinity       = its_set_affinity,
1421         .irq_compose_msi_msg    = its_irq_compose_msi_msg,
1422         .irq_set_irqchip_state  = its_irq_set_irqchip_state,
1423         .irq_set_vcpu_affinity  = its_irq_set_vcpu_affinity,
1424 };
1425
1426
1427 /*
1428  * How we allocate LPIs:
1429  *
1430  * lpi_range_list contains ranges of LPIs that are to available to
1431  * allocate from. To allocate LPIs, just pick the first range that
1432  * fits the required allocation, and reduce it by the required
1433  * amount. Once empty, remove the range from the list.
1434  *
1435  * To free a range of LPIs, add a free range to the list, sort it and
1436  * merge the result if the new range happens to be adjacent to an
1437  * already free block.
1438  *
1439  * The consequence of the above is that allocation is cost is low, but
1440  * freeing is expensive. We assumes that freeing rarely occurs.
1441  */
1442 #define ITS_MAX_LPI_NRBITS      16 /* 64K LPIs */
1443
1444 static DEFINE_MUTEX(lpi_range_lock);
1445 static LIST_HEAD(lpi_range_list);
1446
1447 struct lpi_range {
1448         struct list_head        entry;
1449         u32                     base_id;
1450         u32                     span;
1451 };
1452
1453 static struct lpi_range *mk_lpi_range(u32 base, u32 span)
1454 {
1455         struct lpi_range *range;
1456
1457         range = kzalloc(sizeof(*range), GFP_KERNEL);
1458         if (range) {
1459                 INIT_LIST_HEAD(&range->entry);
1460                 range->base_id = base;
1461                 range->span = span;
1462         }
1463
1464         return range;
1465 }
1466
1467 static int lpi_range_cmp(void *priv, struct list_head *a, struct list_head *b)
1468 {
1469         struct lpi_range *ra, *rb;
1470
1471         ra = container_of(a, struct lpi_range, entry);
1472         rb = container_of(b, struct lpi_range, entry);
1473
1474         return rb->base_id - ra->base_id;
1475 }
1476
1477 static void merge_lpi_ranges(void)
1478 {
1479         struct lpi_range *range, *tmp;
1480
1481         list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
1482                 if (!list_is_last(&range->entry, &lpi_range_list) &&
1483                     (tmp->base_id == (range->base_id + range->span))) {
1484                         tmp->base_id = range->base_id;
1485                         tmp->span += range->span;
1486                         list_del(&range->entry);
1487                         kfree(range);
1488                 }
1489         }
1490 }
1491
1492 static int alloc_lpi_range(u32 nr_lpis, u32 *base)
1493 {
1494         struct lpi_range *range, *tmp;
1495         int err = -ENOSPC;
1496
1497         mutex_lock(&lpi_range_lock);
1498
1499         list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
1500                 if (range->span >= nr_lpis) {
1501                         *base = range->base_id;
1502                         range->base_id += nr_lpis;
1503                         range->span -= nr_lpis;
1504
1505                         if (range->span == 0) {
1506                                 list_del(&range->entry);
1507                                 kfree(range);
1508                         }
1509
1510                         err = 0;
1511                         break;
1512                 }
1513         }
1514
1515         mutex_unlock(&lpi_range_lock);
1516
1517         pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
1518         return err;
1519 }
1520
1521 static int free_lpi_range(u32 base, u32 nr_lpis)
1522 {
1523         struct lpi_range *new;
1524         int err = 0;
1525
1526         mutex_lock(&lpi_range_lock);
1527
1528         new = mk_lpi_range(base, nr_lpis);
1529         if (!new) {
1530                 err = -ENOMEM;
1531                 goto out;
1532         }
1533
1534         list_add(&new->entry, &lpi_range_list);
1535         list_sort(NULL, &lpi_range_list, lpi_range_cmp);
1536         merge_lpi_ranges();
1537 out:
1538         mutex_unlock(&lpi_range_lock);
1539         return err;
1540 }
1541
1542 static int __init its_lpi_init(u32 id_bits)
1543 {
1544         u32 lpis = (1UL << id_bits) - 8192;
1545         u32 numlpis;
1546         int err;
1547
1548         numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
1549
1550         if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
1551                 lpis = numlpis;
1552                 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
1553                         lpis);
1554         }
1555
1556         /*
1557          * Initializing the allocator is just the same as freeing the
1558          * full range of LPIs.
1559          */
1560         err = free_lpi_range(8192, lpis);
1561         pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
1562         return err;
1563 }
1564
1565 static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
1566 {
1567         unsigned long *bitmap = NULL;
1568         int err = 0;
1569
1570         do {
1571                 err = alloc_lpi_range(nr_irqs, base);
1572                 if (!err)
1573                         break;
1574
1575                 nr_irqs /= 2;
1576         } while (nr_irqs > 0);
1577
1578         if (err)
1579                 goto out;
1580
1581         bitmap = kcalloc(BITS_TO_LONGS(nr_irqs), sizeof (long), GFP_ATOMIC);
1582         if (!bitmap)
1583                 goto out;
1584
1585         *nr_ids = nr_irqs;
1586
1587 out:
1588         if (!bitmap)
1589                 *base = *nr_ids = 0;
1590
1591         return bitmap;
1592 }
1593
1594 static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
1595 {
1596         WARN_ON(free_lpi_range(base, nr_ids));
1597         kfree(bitmap);
1598 }
1599
1600 static struct page *its_allocate_prop_table(gfp_t gfp_flags)
1601 {
1602         struct page *prop_page;
1603
1604         prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
1605         if (!prop_page)
1606                 return NULL;
1607
1608         /* Priority 0xa0, Group-1, disabled */
1609         memset(page_address(prop_page),
1610                LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1,
1611                LPI_PROPBASE_SZ);
1612
1613         /* Make sure the GIC will observe the written configuration */
1614         gic_flush_dcache_to_poc(page_address(prop_page), LPI_PROPBASE_SZ);
1615
1616         return prop_page;
1617 }
1618
1619 static void its_free_prop_table(struct page *prop_page)
1620 {
1621         free_pages((unsigned long)page_address(prop_page),
1622                    get_order(LPI_PROPBASE_SZ));
1623 }
1624
1625 static int __init its_alloc_lpi_tables(void)
1626 {
1627         phys_addr_t paddr;
1628
1629         lpi_id_bits = min_t(u32, GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
1630                                 ITS_MAX_LPI_NRBITS);
1631         gic_rdists->prop_page = its_allocate_prop_table(GFP_NOWAIT);
1632         if (!gic_rdists->prop_page) {
1633                 pr_err("Failed to allocate PROPBASE\n");
1634                 return -ENOMEM;
1635         }
1636
1637         paddr = page_to_phys(gic_rdists->prop_page);
1638         pr_info("GIC: using LPI property table @%pa\n", &paddr);
1639
1640         return its_lpi_init(lpi_id_bits);
1641 }
1642
1643 static const char *its_base_type_string[] = {
1644         [GITS_BASER_TYPE_DEVICE]        = "Devices",
1645         [GITS_BASER_TYPE_VCPU]          = "Virtual CPUs",
1646         [GITS_BASER_TYPE_RESERVED3]     = "Reserved (3)",
1647         [GITS_BASER_TYPE_COLLECTION]    = "Interrupt Collections",
1648         [GITS_BASER_TYPE_RESERVED5]     = "Reserved (5)",
1649         [GITS_BASER_TYPE_RESERVED6]     = "Reserved (6)",
1650         [GITS_BASER_TYPE_RESERVED7]     = "Reserved (7)",
1651 };
1652
1653 static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
1654 {
1655         u32 idx = baser - its->tables;
1656
1657         return gits_read_baser(its->base + GITS_BASER + (idx << 3));
1658 }
1659
1660 static void its_write_baser(struct its_node *its, struct its_baser *baser,
1661                             u64 val)
1662 {
1663         u32 idx = baser - its->tables;
1664
1665         gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
1666         baser->val = its_read_baser(its, baser);
1667 }
1668
1669 static int its_setup_baser(struct its_node *its, struct its_baser *baser,
1670                            u64 cache, u64 shr, u32 psz, u32 order,
1671                            bool indirect)
1672 {
1673         u64 val = its_read_baser(its, baser);
1674         u64 esz = GITS_BASER_ENTRY_SIZE(val);
1675         u64 type = GITS_BASER_TYPE(val);
1676         u64 baser_phys, tmp;
1677         u32 alloc_pages;
1678         void *base;
1679
1680 retry_alloc_baser:
1681         alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
1682         if (alloc_pages > GITS_BASER_PAGES_MAX) {
1683                 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
1684                         &its->phys_base, its_base_type_string[type],
1685                         alloc_pages, GITS_BASER_PAGES_MAX);
1686                 alloc_pages = GITS_BASER_PAGES_MAX;
1687                 order = get_order(GITS_BASER_PAGES_MAX * psz);
1688         }
1689
1690         base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
1691         if (!base)
1692                 return -ENOMEM;
1693
1694         baser_phys = virt_to_phys(base);
1695
1696         /* Check if the physical address of the memory is above 48bits */
1697         if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
1698
1699                 /* 52bit PA is supported only when PageSize=64K */
1700                 if (psz != SZ_64K) {
1701                         pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
1702                         free_pages((unsigned long)base, order);
1703                         return -ENXIO;
1704                 }
1705
1706                 /* Convert 52bit PA to 48bit field */
1707                 baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
1708         }
1709
1710 retry_baser:
1711         val = (baser_phys                                        |
1712                 (type << GITS_BASER_TYPE_SHIFT)                  |
1713                 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT)       |
1714                 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT)    |
1715                 cache                                            |
1716                 shr                                              |
1717                 GITS_BASER_VALID);
1718
1719         val |=  indirect ? GITS_BASER_INDIRECT : 0x0;
1720
1721         switch (psz) {
1722         case SZ_4K:
1723                 val |= GITS_BASER_PAGE_SIZE_4K;
1724                 break;
1725         case SZ_16K:
1726                 val |= GITS_BASER_PAGE_SIZE_16K;
1727                 break;
1728         case SZ_64K:
1729                 val |= GITS_BASER_PAGE_SIZE_64K;
1730                 break;
1731         }
1732
1733         its_write_baser(its, baser, val);
1734         tmp = baser->val;
1735
1736         if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
1737                 /*
1738                  * Shareability didn't stick. Just use
1739                  * whatever the read reported, which is likely
1740                  * to be the only thing this redistributor
1741                  * supports. If that's zero, make it
1742                  * non-cacheable as well.
1743                  */
1744                 shr = tmp & GITS_BASER_SHAREABILITY_MASK;
1745                 if (!shr) {
1746                         cache = GITS_BASER_nC;
1747                         gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
1748                 }
1749                 goto retry_baser;
1750         }
1751
1752         if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) {
1753                 /*
1754                  * Page size didn't stick. Let's try a smaller
1755                  * size and retry. If we reach 4K, then
1756                  * something is horribly wrong...
1757                  */
1758                 free_pages((unsigned long)base, order);
1759                 baser->base = NULL;
1760
1761                 switch (psz) {
1762                 case SZ_16K:
1763                         psz = SZ_4K;
1764                         goto retry_alloc_baser;
1765                 case SZ_64K:
1766                         psz = SZ_16K;
1767                         goto retry_alloc_baser;
1768                 }
1769         }
1770
1771         if (val != tmp) {
1772                 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
1773                        &its->phys_base, its_base_type_string[type],
1774                        val, tmp);
1775                 free_pages((unsigned long)base, order);
1776                 return -ENXIO;
1777         }
1778
1779         baser->order = order;
1780         baser->base = base;
1781         baser->psz = psz;
1782         tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
1783
1784         pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
1785                 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
1786                 its_base_type_string[type],
1787                 (unsigned long)virt_to_phys(base),
1788                 indirect ? "indirect" : "flat", (int)esz,
1789                 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
1790
1791         return 0;
1792 }
1793
1794 static bool its_parse_indirect_baser(struct its_node *its,
1795                                      struct its_baser *baser,
1796                                      u32 psz, u32 *order, u32 ids)
1797 {
1798         u64 tmp = its_read_baser(its, baser);
1799         u64 type = GITS_BASER_TYPE(tmp);
1800         u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
1801         u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
1802         u32 new_order = *order;
1803         bool indirect = false;
1804
1805         /* No need to enable Indirection if memory requirement < (psz*2)bytes */
1806         if ((esz << ids) > (psz * 2)) {
1807                 /*
1808                  * Find out whether hw supports a single or two-level table by
1809                  * table by reading bit at offset '62' after writing '1' to it.
1810                  */
1811                 its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
1812                 indirect = !!(baser->val & GITS_BASER_INDIRECT);
1813
1814                 if (indirect) {
1815                         /*
1816                          * The size of the lvl2 table is equal to ITS page size
1817                          * which is 'psz'. For computing lvl1 table size,
1818                          * subtract ID bits that sparse lvl2 table from 'ids'
1819                          * which is reported by ITS hardware times lvl1 table
1820                          * entry size.
1821                          */
1822                         ids -= ilog2(psz / (int)esz);
1823                         esz = GITS_LVL1_ENTRY_SIZE;
1824                 }
1825         }
1826
1827         /*
1828          * Allocate as many entries as required to fit the
1829          * range of device IDs that the ITS can grok... The ID
1830          * space being incredibly sparse, this results in a
1831          * massive waste of memory if two-level device table
1832          * feature is not supported by hardware.
1833          */
1834         new_order = max_t(u32, get_order(esz << ids), new_order);
1835         if (new_order >= MAX_ORDER) {
1836                 new_order = MAX_ORDER - 1;
1837                 ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
1838                 pr_warn("ITS@%pa: %s Table too large, reduce ids %u->%u\n",
1839                         &its->phys_base, its_base_type_string[type],
1840                         its->device_ids, ids);
1841         }
1842
1843         *order = new_order;
1844
1845         return indirect;
1846 }
1847
1848 static void its_free_tables(struct its_node *its)
1849 {
1850         int i;
1851
1852         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
1853                 if (its->tables[i].base) {
1854                         free_pages((unsigned long)its->tables[i].base,
1855                                    its->tables[i].order);
1856                         its->tables[i].base = NULL;
1857                 }
1858         }
1859 }
1860
1861 static int its_alloc_tables(struct its_node *its)
1862 {
1863         u64 shr = GITS_BASER_InnerShareable;
1864         u64 cache = GITS_BASER_RaWaWb;
1865         u32 psz = SZ_64K;
1866         int err, i;
1867
1868         if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
1869                 /* erratum 24313: ignore memory access type */
1870                 cache = GITS_BASER_nCnB;
1871
1872         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
1873                 struct its_baser *baser = its->tables + i;
1874                 u64 val = its_read_baser(its, baser);
1875                 u64 type = GITS_BASER_TYPE(val);
1876                 u32 order = get_order(psz);
1877                 bool indirect = false;
1878
1879                 switch (type) {
1880                 case GITS_BASER_TYPE_NONE:
1881                         continue;
1882
1883                 case GITS_BASER_TYPE_DEVICE:
1884                         indirect = its_parse_indirect_baser(its, baser,
1885                                                             psz, &order,
1886                                                             its->device_ids);
1887                 case GITS_BASER_TYPE_VCPU:
1888                         indirect = its_parse_indirect_baser(its, baser,
1889                                                             psz, &order,
1890                                                             ITS_MAX_VPEID_BITS);
1891                         break;
1892                 }
1893
1894                 err = its_setup_baser(its, baser, cache, shr, psz, order, indirect);
1895                 if (err < 0) {
1896                         its_free_tables(its);
1897                         return err;
1898                 }
1899
1900                 /* Update settings which will be used for next BASERn */
1901                 psz = baser->psz;
1902                 cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
1903                 shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
1904         }
1905
1906         return 0;
1907 }
1908
1909 static int its_alloc_collections(struct its_node *its)
1910 {
1911         int i;
1912
1913         its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
1914                                    GFP_KERNEL);
1915         if (!its->collections)
1916                 return -ENOMEM;
1917
1918         for (i = 0; i < nr_cpu_ids; i++)
1919                 its->collections[i].target_address = ~0ULL;
1920
1921         return 0;
1922 }
1923
1924 static struct page *its_allocate_pending_table(gfp_t gfp_flags)
1925 {
1926         struct page *pend_page;
1927         /*
1928          * The pending pages have to be at least 64kB aligned,
1929          * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below.
1930          */
1931         pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
1932                                 get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
1933         if (!pend_page)
1934                 return NULL;
1935
1936         /* Make sure the GIC will observe the zero-ed page */
1937         gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
1938
1939         return pend_page;
1940 }
1941
1942 static void its_free_pending_table(struct page *pt)
1943 {
1944         free_pages((unsigned long)page_address(pt),
1945                    get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
1946 }
1947
1948 static void its_cpu_init_lpis(void)
1949 {
1950         void __iomem *rbase = gic_data_rdist_rd_base();
1951         struct page *pend_page;
1952         u64 val, tmp;
1953
1954         /* If we didn't allocate the pending table yet, do it now */
1955         pend_page = gic_data_rdist()->pend_page;
1956         if (!pend_page) {
1957                 phys_addr_t paddr;
1958
1959                 pend_page = its_allocate_pending_table(GFP_NOWAIT);
1960                 if (!pend_page) {
1961                         pr_err("Failed to allocate PENDBASE for CPU%d\n",
1962                                smp_processor_id());
1963                         return;
1964                 }
1965
1966                 paddr = page_to_phys(pend_page);
1967                 pr_info("CPU%d: using LPI pending table @%pa\n",
1968                         smp_processor_id(), &paddr);
1969                 gic_data_rdist()->pend_page = pend_page;
1970         }
1971
1972         /* set PROPBASE */
1973         val = (page_to_phys(gic_rdists->prop_page) |
1974                GICR_PROPBASER_InnerShareable |
1975                GICR_PROPBASER_RaWaWb |
1976                ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
1977
1978         gicr_write_propbaser(val, rbase + GICR_PROPBASER);
1979         tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
1980
1981         if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
1982                 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
1983                         /*
1984                          * The HW reports non-shareable, we must
1985                          * remove the cacheability attributes as
1986                          * well.
1987                          */
1988                         val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
1989                                  GICR_PROPBASER_CACHEABILITY_MASK);
1990                         val |= GICR_PROPBASER_nC;
1991                         gicr_write_propbaser(val, rbase + GICR_PROPBASER);
1992                 }
1993                 pr_info_once("GIC: using cache flushing for LPI property table\n");
1994                 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
1995         }
1996
1997         /* set PENDBASE */
1998         val = (page_to_phys(pend_page) |
1999                GICR_PENDBASER_InnerShareable |
2000                GICR_PENDBASER_RaWaWb);
2001
2002         gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
2003         tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
2004
2005         if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
2006                 /*
2007                  * The HW reports non-shareable, we must remove the
2008                  * cacheability attributes as well.
2009                  */
2010                 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
2011                          GICR_PENDBASER_CACHEABILITY_MASK);
2012                 val |= GICR_PENDBASER_nC;
2013                 gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
2014         }
2015
2016         /* Enable LPIs */
2017         val = readl_relaxed(rbase + GICR_CTLR);
2018         val |= GICR_CTLR_ENABLE_LPIS;
2019         writel_relaxed(val, rbase + GICR_CTLR);
2020
2021         /* Make sure the GIC has seen the above */
2022         dsb(sy);
2023 }
2024
2025 static void its_cpu_init_collection(struct its_node *its)
2026 {
2027         int cpu = smp_processor_id();
2028         u64 target;
2029
2030         /* avoid cross node collections and its mapping */
2031         if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
2032                 struct device_node *cpu_node;
2033
2034                 cpu_node = of_get_cpu_node(cpu, NULL);
2035                 if (its->numa_node != NUMA_NO_NODE &&
2036                         its->numa_node != of_node_to_nid(cpu_node))
2037                         return;
2038         }
2039
2040         /*
2041          * We now have to bind each collection to its target
2042          * redistributor.
2043          */
2044         if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
2045                 /*
2046                  * This ITS wants the physical address of the
2047                  * redistributor.
2048                  */
2049                 target = gic_data_rdist()->phys_base;
2050         } else {
2051                 /* This ITS wants a linear CPU number. */
2052                 target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2053                 target = GICR_TYPER_CPU_NUMBER(target) << 16;
2054         }
2055
2056         /* Perform collection mapping */
2057         its->collections[cpu].target_address = target;
2058         its->collections[cpu].col_id = cpu;
2059
2060         its_send_mapc(its, &its->collections[cpu], 1);
2061         its_send_invall(its, &its->collections[cpu]);
2062 }
2063
2064 static void its_cpu_init_collections(void)
2065 {
2066         struct its_node *its;
2067
2068         raw_spin_lock(&its_lock);
2069
2070         list_for_each_entry(its, &its_nodes, entry)
2071                 its_cpu_init_collection(its);
2072
2073         raw_spin_unlock(&its_lock);
2074 }
2075
2076 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
2077 {
2078         struct its_device *its_dev = NULL, *tmp;
2079         unsigned long flags;
2080
2081         raw_spin_lock_irqsave(&its->lock, flags);
2082
2083         list_for_each_entry(tmp, &its->its_device_list, entry) {
2084                 if (tmp->device_id == dev_id) {
2085                         its_dev = tmp;
2086                         break;
2087                 }
2088         }
2089
2090         raw_spin_unlock_irqrestore(&its->lock, flags);
2091
2092         return its_dev;
2093 }
2094
2095 static struct its_baser *its_get_baser(struct its_node *its, u32 type)
2096 {
2097         int i;
2098
2099         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2100                 if (GITS_BASER_TYPE(its->tables[i].val) == type)
2101                         return &its->tables[i];
2102         }
2103
2104         return NULL;
2105 }
2106
2107 static bool its_alloc_table_entry(struct its_baser *baser, u32 id)
2108 {
2109         struct page *page;
2110         u32 esz, idx;
2111         __le64 *table;
2112
2113         /* Don't allow device id that exceeds single, flat table limit */
2114         esz = GITS_BASER_ENTRY_SIZE(baser->val);
2115         if (!(baser->val & GITS_BASER_INDIRECT))
2116                 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
2117
2118         /* Compute 1st level table index & check if that exceeds table limit */
2119         idx = id >> ilog2(baser->psz / esz);
2120         if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
2121                 return false;
2122
2123         table = baser->base;
2124
2125         /* Allocate memory for 2nd level table */
2126         if (!table[idx]) {
2127                 page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(baser->psz));
2128                 if (!page)
2129                         return false;
2130
2131                 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
2132                 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
2133                         gic_flush_dcache_to_poc(page_address(page), baser->psz);
2134
2135                 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
2136
2137                 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
2138                 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
2139                         gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
2140
2141                 /* Ensure updated table contents are visible to ITS hardware */
2142                 dsb(sy);
2143         }
2144
2145         return true;
2146 }
2147
2148 static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
2149 {
2150         struct its_baser *baser;
2151
2152         baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
2153
2154         /* Don't allow device id that exceeds ITS hardware limit */
2155         if (!baser)
2156                 return (ilog2(dev_id) < its->device_ids);
2157
2158         return its_alloc_table_entry(baser, dev_id);
2159 }
2160
2161 static bool its_alloc_vpe_table(u32 vpe_id)
2162 {
2163         struct its_node *its;
2164
2165         /*
2166          * Make sure the L2 tables are allocated on *all* v4 ITSs. We
2167          * could try and only do it on ITSs corresponding to devices
2168          * that have interrupts targeted at this VPE, but the
2169          * complexity becomes crazy (and you have tons of memory
2170          * anyway, right?).
2171          */
2172         list_for_each_entry(its, &its_nodes, entry) {
2173                 struct its_baser *baser;
2174
2175                 if (!its->is_v4)
2176                         continue;
2177
2178                 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
2179                 if (!baser)
2180                         return false;
2181
2182                 if (!its_alloc_table_entry(baser, vpe_id))
2183                         return false;
2184         }
2185
2186         return true;
2187 }
2188
2189 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
2190                                             int nvecs, bool alloc_lpis)
2191 {
2192         struct its_device *dev;
2193         unsigned long *lpi_map = NULL;
2194         unsigned long flags;
2195         u16 *col_map = NULL;
2196         void *itt;
2197         int lpi_base;
2198         int nr_lpis;
2199         int nr_ites;
2200         int sz;
2201
2202         if (!its_alloc_device_table(its, dev_id))
2203                 return NULL;
2204
2205         if (WARN_ON(!is_power_of_2(nvecs)))
2206                 nvecs = roundup_pow_of_two(nvecs);
2207
2208         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2209         /*
2210          * Even if the device wants a single LPI, the ITT must be
2211          * sized as a power of two (and you need at least one bit...).
2212          */
2213         nr_ites = max(2, nvecs);
2214         sz = nr_ites * its->ite_size;
2215         sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
2216         itt = kzalloc(sz, GFP_KERNEL);
2217         if (alloc_lpis) {
2218                 lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
2219                 if (lpi_map)
2220                         col_map = kcalloc(nr_lpis, sizeof(*col_map),
2221                                           GFP_KERNEL);
2222         } else {
2223                 col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
2224                 nr_lpis = 0;
2225                 lpi_base = 0;
2226         }
2227
2228         if (!dev || !itt ||  !col_map || (!lpi_map && alloc_lpis)) {
2229                 kfree(dev);
2230                 kfree(itt);
2231                 kfree(lpi_map);
2232                 kfree(col_map);
2233                 return NULL;
2234         }
2235
2236         gic_flush_dcache_to_poc(itt, sz);
2237
2238         dev->its = its;
2239         dev->itt = itt;
2240         dev->nr_ites = nr_ites;
2241         dev->event_map.lpi_map = lpi_map;
2242         dev->event_map.col_map = col_map;
2243         dev->event_map.lpi_base = lpi_base;
2244         dev->event_map.nr_lpis = nr_lpis;
2245         mutex_init(&dev->event_map.vlpi_lock);
2246         dev->device_id = dev_id;
2247         INIT_LIST_HEAD(&dev->entry);
2248
2249         raw_spin_lock_irqsave(&its->lock, flags);
2250         list_add(&dev->entry, &its->its_device_list);
2251         raw_spin_unlock_irqrestore(&its->lock, flags);
2252
2253         /* Map device to its ITT */
2254         its_send_mapd(dev, 1);
2255
2256         return dev;
2257 }
2258
2259 static void its_free_device(struct its_device *its_dev)
2260 {
2261         unsigned long flags;
2262
2263         raw_spin_lock_irqsave(&its_dev->its->lock, flags);
2264         list_del(&its_dev->entry);
2265         raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
2266         kfree(its_dev->itt);
2267         kfree(its_dev);
2268 }
2269
2270 static int its_alloc_device_irq(struct its_device *dev, irq_hw_number_t *hwirq)
2271 {
2272         int idx;
2273
2274         idx = find_first_zero_bit(dev->event_map.lpi_map,
2275                                   dev->event_map.nr_lpis);
2276         if (idx == dev->event_map.nr_lpis)
2277                 return -ENOSPC;
2278
2279         *hwirq = dev->event_map.lpi_base + idx;
2280         set_bit(idx, dev->event_map.lpi_map);
2281
2282         return 0;
2283 }
2284
2285 static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
2286                            int nvec, msi_alloc_info_t *info)
2287 {
2288         struct its_node *its;
2289         struct its_device *its_dev;
2290         struct msi_domain_info *msi_info;
2291         u32 dev_id;
2292
2293         /*
2294          * We ignore "dev" entierely, and rely on the dev_id that has
2295          * been passed via the scratchpad. This limits this domain's
2296          * usefulness to upper layers that definitely know that they
2297          * are built on top of the ITS.
2298          */
2299         dev_id = info->scratchpad[0].ul;
2300
2301         msi_info = msi_get_domain_info(domain);
2302         its = msi_info->data;
2303
2304         if (!gic_rdists->has_direct_lpi &&
2305             vpe_proxy.dev &&
2306             vpe_proxy.dev->its == its &&
2307             dev_id == vpe_proxy.dev->device_id) {
2308                 /* Bad luck. Get yourself a better implementation */
2309                 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
2310                           dev_id);
2311                 return -EINVAL;
2312         }
2313
2314         its_dev = its_find_device(its, dev_id);
2315         if (its_dev) {
2316                 /*
2317                  * We already have seen this ID, probably through
2318                  * another alias (PCI bridge of some sort). No need to
2319                  * create the device.
2320                  */
2321                 pr_debug("Reusing ITT for devID %x\n", dev_id);
2322                 goto out;
2323         }
2324
2325         its_dev = its_create_device(its, dev_id, nvec, true);
2326         if (!its_dev)
2327                 return -ENOMEM;
2328
2329         pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
2330 out:
2331         info->scratchpad[0].ptr = its_dev;
2332         return 0;
2333 }
2334
2335 static struct msi_domain_ops its_msi_domain_ops = {
2336         .msi_prepare    = its_msi_prepare,
2337 };
2338
2339 static int its_irq_gic_domain_alloc(struct irq_domain *domain,
2340                                     unsigned int virq,
2341                                     irq_hw_number_t hwirq)
2342 {
2343         struct irq_fwspec fwspec;
2344
2345         if (irq_domain_get_of_node(domain->parent)) {
2346                 fwspec.fwnode = domain->parent->fwnode;
2347                 fwspec.param_count = 3;
2348                 fwspec.param[0] = GIC_IRQ_TYPE_LPI;
2349                 fwspec.param[1] = hwirq;
2350                 fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
2351         } else if (is_fwnode_irqchip(domain->parent->fwnode)) {
2352                 fwspec.fwnode = domain->parent->fwnode;
2353                 fwspec.param_count = 2;
2354                 fwspec.param[0] = hwirq;
2355                 fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
2356         } else {
2357                 return -EINVAL;
2358         }
2359
2360         return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
2361 }
2362
2363 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
2364                                 unsigned int nr_irqs, void *args)
2365 {
2366         msi_alloc_info_t *info = args;
2367         struct its_device *its_dev = info->scratchpad[0].ptr;
2368         irq_hw_number_t hwirq;
2369         int err;
2370         int i;
2371
2372         for (i = 0; i < nr_irqs; i++) {
2373                 err = its_alloc_device_irq(its_dev, &hwirq);
2374                 if (err)
2375                         return err;
2376
2377                 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq);
2378                 if (err)
2379                         return err;
2380
2381                 irq_domain_set_hwirq_and_chip(domain, virq + i,
2382                                               hwirq, &its_irq_chip, its_dev);
2383                 irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq + i)));
2384                 pr_debug("ID:%d pID:%d vID:%d\n",
2385                          (int)(hwirq - its_dev->event_map.lpi_base),
2386                          (int) hwirq, virq + i);
2387         }
2388
2389         return 0;
2390 }
2391
2392 static int its_irq_domain_activate(struct irq_domain *domain,
2393                                    struct irq_data *d, bool reserve)
2394 {
2395         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
2396         u32 event = its_get_event_id(d);
2397         const struct cpumask *cpu_mask = cpu_online_mask;
2398         int cpu;
2399
2400         /* get the cpu_mask of local node */
2401         if (its_dev->its->numa_node >= 0)
2402                 cpu_mask = cpumask_of_node(its_dev->its->numa_node);
2403
2404         /* Bind the LPI to the first possible CPU */
2405         cpu = cpumask_first_and(cpu_mask, cpu_online_mask);
2406         if (cpu >= nr_cpu_ids) {
2407                 if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144)
2408                         return -EINVAL;
2409
2410                 cpu = cpumask_first(cpu_online_mask);
2411         }
2412
2413         its_dev->event_map.col_map[event] = cpu;
2414         irq_data_update_effective_affinity(d, cpumask_of(cpu));
2415
2416         /* Map the GIC IRQ and event to the device */
2417         its_send_mapti(its_dev, d->hwirq, event);
2418         return 0;
2419 }
2420
2421 static void its_irq_domain_deactivate(struct irq_domain *domain,
2422                                       struct irq_data *d)
2423 {
2424         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
2425         u32 event = its_get_event_id(d);
2426
2427         /* Stop the delivery of interrupts */
2428         its_send_discard(its_dev, event);
2429 }
2430
2431 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
2432                                 unsigned int nr_irqs)
2433 {
2434         struct irq_data *d = irq_domain_get_irq_data(domain, virq);
2435         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
2436         int i;
2437
2438         for (i = 0; i < nr_irqs; i++) {
2439                 struct irq_data *data = irq_domain_get_irq_data(domain,
2440                                                                 virq + i);
2441                 u32 event = its_get_event_id(data);
2442
2443                 /* Mark interrupt index as unused */
2444                 clear_bit(event, its_dev->event_map.lpi_map);
2445
2446                 /* Nuke the entry in the domain */
2447                 irq_domain_reset_irq_data(data);
2448         }
2449
2450         /* If all interrupts have been freed, start mopping the floor */
2451         if (bitmap_empty(its_dev->event_map.lpi_map,
2452                          its_dev->event_map.nr_lpis)) {
2453                 its_lpi_free(its_dev->event_map.lpi_map,
2454                              its_dev->event_map.lpi_base,
2455                              its_dev->event_map.nr_lpis);
2456                 kfree(its_dev->event_map.col_map);
2457
2458                 /* Unmap device/itt */
2459                 its_send_mapd(its_dev, 0);
2460                 its_free_device(its_dev);
2461         }
2462
2463         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
2464 }
2465
2466 static const struct irq_domain_ops its_domain_ops = {
2467         .alloc                  = its_irq_domain_alloc,
2468         .free                   = its_irq_domain_free,
2469         .activate               = its_irq_domain_activate,
2470         .deactivate             = its_irq_domain_deactivate,
2471 };
2472
2473 /*
2474  * This is insane.
2475  *
2476  * If a GICv4 doesn't implement Direct LPIs (which is extremely
2477  * likely), the only way to perform an invalidate is to use a fake
2478  * device to issue an INV command, implying that the LPI has first
2479  * been mapped to some event on that device. Since this is not exactly
2480  * cheap, we try to keep that mapping around as long as possible, and
2481  * only issue an UNMAP if we're short on available slots.
2482  *
2483  * Broken by design(tm).
2484  */
2485 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
2486 {
2487         /* Already unmapped? */
2488         if (vpe->vpe_proxy_event == -1)
2489                 return;
2490
2491         its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
2492         vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
2493
2494         /*
2495          * We don't track empty slots at all, so let's move the
2496          * next_victim pointer if we can quickly reuse that slot
2497          * instead of nuking an existing entry. Not clear that this is
2498          * always a win though, and this might just generate a ripple
2499          * effect... Let's just hope VPEs don't migrate too often.
2500          */
2501         if (vpe_proxy.vpes[vpe_proxy.next_victim])
2502                 vpe_proxy.next_victim = vpe->vpe_proxy_event;
2503
2504         vpe->vpe_proxy_event = -1;
2505 }
2506
2507 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
2508 {
2509         if (!gic_rdists->has_direct_lpi) {
2510                 unsigned long flags;
2511
2512                 raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
2513                 its_vpe_db_proxy_unmap_locked(vpe);
2514                 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
2515         }
2516 }
2517
2518 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
2519 {
2520         /* Already mapped? */
2521         if (vpe->vpe_proxy_event != -1)
2522                 return;
2523
2524         /* This slot was already allocated. Kick the other VPE out. */
2525         if (vpe_proxy.vpes[vpe_proxy.next_victim])
2526                 its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
2527
2528         /* Map the new VPE instead */
2529         vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
2530         vpe->vpe_proxy_event = vpe_proxy.next_victim;
2531         vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
2532
2533         vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
2534         its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
2535 }
2536
2537 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
2538 {
2539         unsigned long flags;
2540         struct its_collection *target_col;
2541
2542         if (gic_rdists->has_direct_lpi) {
2543                 void __iomem *rdbase;
2544
2545                 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
2546                 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
2547                 while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
2548                         cpu_relax();
2549
2550                 return;
2551         }
2552
2553         raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
2554
2555         its_vpe_db_proxy_map_locked(vpe);
2556
2557         target_col = &vpe_proxy.dev->its->collections[to];
2558         its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
2559         vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
2560
2561         raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
2562 }
2563
2564 static int its_vpe_set_affinity(struct irq_data *d,
2565                                 const struct cpumask *mask_val,
2566                                 bool force)
2567 {
2568         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2569         int cpu = cpumask_first(mask_val);
2570
2571         /*
2572          * Changing affinity is mega expensive, so let's be as lazy as
2573          * we can and only do it if we really have to. Also, if mapped
2574          * into the proxy device, we need to move the doorbell
2575          * interrupt to its new location.
2576          */
2577         if (vpe->col_idx != cpu) {
2578                 int from = vpe->col_idx;
2579
2580                 vpe->col_idx = cpu;
2581                 its_send_vmovp(vpe);
2582                 its_vpe_db_proxy_move(vpe, from, cpu);
2583         }
2584
2585         irq_data_update_effective_affinity(d, cpumask_of(cpu));
2586
2587         return IRQ_SET_MASK_OK_DONE;
2588 }
2589
2590 static void its_vpe_schedule(struct its_vpe *vpe)
2591 {
2592         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2593         u64 val;
2594
2595         /* Schedule the VPE */
2596         val  = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
2597                 GENMASK_ULL(51, 12);
2598         val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
2599         val |= GICR_VPROPBASER_RaWb;
2600         val |= GICR_VPROPBASER_InnerShareable;
2601         gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2602
2603         val  = virt_to_phys(page_address(vpe->vpt_page)) &
2604                 GENMASK_ULL(51, 16);
2605         val |= GICR_VPENDBASER_RaWaWb;
2606         val |= GICR_VPENDBASER_NonShareable;
2607         /*
2608          * There is no good way of finding out if the pending table is
2609          * empty as we can race against the doorbell interrupt very
2610          * easily. So in the end, vpe->pending_last is only an
2611          * indication that the vcpu has something pending, not one
2612          * that the pending table is empty. A good implementation
2613          * would be able to read its coarse map pretty quickly anyway,
2614          * making this a tolerable issue.
2615          */
2616         val |= GICR_VPENDBASER_PendingLast;
2617         val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
2618         val |= GICR_VPENDBASER_Valid;
2619         gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
2620 }
2621
2622 static void its_vpe_deschedule(struct its_vpe *vpe)
2623 {
2624         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2625         u32 count = 1000000;    /* 1s! */
2626         bool clean;
2627         u64 val;
2628
2629         /* We're being scheduled out */
2630         val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
2631         val &= ~GICR_VPENDBASER_Valid;
2632         gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
2633
2634         do {
2635                 val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
2636                 clean = !(val & GICR_VPENDBASER_Dirty);
2637                 if (!clean) {
2638                         count--;
2639                         cpu_relax();
2640                         udelay(1);
2641                 }
2642         } while (!clean && count);
2643
2644         if (unlikely(!clean && !count)) {
2645                 pr_err_ratelimited("ITS virtual pending table not cleaning\n");
2646                 vpe->idai = false;
2647                 vpe->pending_last = true;
2648         } else {
2649                 vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
2650                 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
2651         }
2652 }
2653
2654 static void its_vpe_invall(struct its_vpe *vpe)
2655 {
2656         struct its_node *its;
2657
2658         list_for_each_entry(its, &its_nodes, entry) {
2659                 if (!its->is_v4)
2660                         continue;
2661
2662                 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
2663                         continue;
2664
2665                 /*
2666                  * Sending a VINVALL to a single ITS is enough, as all
2667                  * we need is to reach the redistributors.
2668                  */
2669                 its_send_vinvall(its, vpe);
2670                 return;
2671         }
2672 }
2673
2674 static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
2675 {
2676         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2677         struct its_cmd_info *info = vcpu_info;
2678
2679         switch (info->cmd_type) {
2680         case SCHEDULE_VPE:
2681                 its_vpe_schedule(vpe);
2682                 return 0;
2683
2684         case DESCHEDULE_VPE:
2685                 its_vpe_deschedule(vpe);
2686                 return 0;
2687
2688         case INVALL_VPE:
2689                 its_vpe_invall(vpe);
2690                 return 0;
2691
2692         default:
2693                 return -EINVAL;
2694         }
2695 }
2696
2697 static void its_vpe_send_cmd(struct its_vpe *vpe,
2698                              void (*cmd)(struct its_device *, u32))
2699 {
2700         unsigned long flags;
2701
2702         raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
2703
2704         its_vpe_db_proxy_map_locked(vpe);
2705         cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
2706
2707         raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
2708 }
2709
2710 static void its_vpe_send_inv(struct irq_data *d)
2711 {
2712         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2713
2714         if (gic_rdists->has_direct_lpi) {
2715                 void __iomem *rdbase;
2716
2717                 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
2718                 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_INVLPIR);
2719                 while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
2720                         cpu_relax();
2721         } else {
2722                 its_vpe_send_cmd(vpe, its_send_inv);
2723         }
2724 }
2725
2726 static void its_vpe_mask_irq(struct irq_data *d)
2727 {
2728         /*
2729          * We need to unmask the LPI, which is described by the parent
2730          * irq_data. Instead of calling into the parent (which won't
2731          * exactly do the right thing, let's simply use the
2732          * parent_data pointer. Yes, I'm naughty.
2733          */
2734         lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
2735         its_vpe_send_inv(d);
2736 }
2737
2738 static void its_vpe_unmask_irq(struct irq_data *d)
2739 {
2740         /* Same hack as above... */
2741         lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
2742         its_vpe_send_inv(d);
2743 }
2744
2745 static int its_vpe_set_irqchip_state(struct irq_data *d,
2746                                      enum irqchip_irq_state which,
2747                                      bool state)
2748 {
2749         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2750
2751         if (which != IRQCHIP_STATE_PENDING)
2752                 return -EINVAL;
2753
2754         if (gic_rdists->has_direct_lpi) {
2755                 void __iomem *rdbase;
2756
2757                 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
2758                 if (state) {
2759                         gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
2760                 } else {
2761                         gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
2762                         while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
2763                                 cpu_relax();
2764                 }
2765         } else {
2766                 if (state)
2767                         its_vpe_send_cmd(vpe, its_send_int);
2768                 else
2769                         its_vpe_send_cmd(vpe, its_send_clear);
2770         }
2771
2772         return 0;
2773 }
2774
2775 static struct irq_chip its_vpe_irq_chip = {
2776         .name                   = "GICv4-vpe",
2777         .irq_mask               = its_vpe_mask_irq,
2778         .irq_unmask             = its_vpe_unmask_irq,
2779         .irq_eoi                = irq_chip_eoi_parent,
2780         .irq_set_affinity       = its_vpe_set_affinity,
2781         .irq_set_irqchip_state  = its_vpe_set_irqchip_state,
2782         .irq_set_vcpu_affinity  = its_vpe_set_vcpu_affinity,
2783 };
2784
2785 static int its_vpe_id_alloc(void)
2786 {
2787         return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL);
2788 }
2789
2790 static void its_vpe_id_free(u16 id)
2791 {
2792         ida_simple_remove(&its_vpeid_ida, id);
2793 }
2794
2795 static int its_vpe_init(struct its_vpe *vpe)
2796 {
2797         struct page *vpt_page;
2798         int vpe_id;
2799
2800         /* Allocate vpe_id */
2801         vpe_id = its_vpe_id_alloc();
2802         if (vpe_id < 0)
2803                 return vpe_id;
2804
2805         /* Allocate VPT */
2806         vpt_page = its_allocate_pending_table(GFP_KERNEL);
2807         if (!vpt_page) {
2808                 its_vpe_id_free(vpe_id);
2809                 return -ENOMEM;
2810         }
2811
2812         if (!its_alloc_vpe_table(vpe_id)) {
2813                 its_vpe_id_free(vpe_id);
2814                 its_free_pending_table(vpe->vpt_page);
2815                 return -ENOMEM;
2816         }
2817
2818         vpe->vpe_id = vpe_id;
2819         vpe->vpt_page = vpt_page;
2820         vpe->vpe_proxy_event = -1;
2821
2822         return 0;
2823 }
2824
2825 static void its_vpe_teardown(struct its_vpe *vpe)
2826 {
2827         its_vpe_db_proxy_unmap(vpe);
2828         its_vpe_id_free(vpe->vpe_id);
2829         its_free_pending_table(vpe->vpt_page);
2830 }
2831
2832 static void its_vpe_irq_domain_free(struct irq_domain *domain,
2833                                     unsigned int virq,
2834                                     unsigned int nr_irqs)
2835 {
2836         struct its_vm *vm = domain->host_data;
2837         int i;
2838
2839         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
2840
2841         for (i = 0; i < nr_irqs; i++) {
2842                 struct irq_data *data = irq_domain_get_irq_data(domain,
2843                                                                 virq + i);
2844                 struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
2845
2846                 BUG_ON(vm != vpe->its_vm);
2847
2848                 clear_bit(data->hwirq, vm->db_bitmap);
2849                 its_vpe_teardown(vpe);
2850                 irq_domain_reset_irq_data(data);
2851         }
2852
2853         if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
2854                 its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
2855                 its_free_prop_table(vm->vprop_page);
2856         }
2857 }
2858
2859 static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
2860                                     unsigned int nr_irqs, void *args)
2861 {
2862         struct its_vm *vm = args;
2863         unsigned long *bitmap;
2864         struct page *vprop_page;
2865         int base, nr_ids, i, err = 0;
2866
2867         BUG_ON(!vm);
2868
2869         bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
2870         if (!bitmap)
2871                 return -ENOMEM;
2872
2873         if (nr_ids < nr_irqs) {
2874                 its_lpi_free(bitmap, base, nr_ids);
2875                 return -ENOMEM;
2876         }
2877
2878         vprop_page = its_allocate_prop_table(GFP_KERNEL);
2879         if (!vprop_page) {
2880                 its_lpi_free(bitmap, base, nr_ids);
2881                 return -ENOMEM;
2882         }
2883
2884         vm->db_bitmap = bitmap;
2885         vm->db_lpi_base = base;
2886         vm->nr_db_lpis = nr_ids;
2887         vm->vprop_page = vprop_page;
2888
2889         for (i = 0; i < nr_irqs; i++) {
2890                 vm->vpes[i]->vpe_db_lpi = base + i;
2891                 err = its_vpe_init(vm->vpes[i]);
2892                 if (err)
2893                         break;
2894                 err = its_irq_gic_domain_alloc(domain, virq + i,
2895                                                vm->vpes[i]->vpe_db_lpi);
2896                 if (err)
2897                         break;
2898                 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
2899                                               &its_vpe_irq_chip, vm->vpes[i]);
2900                 set_bit(i, bitmap);
2901         }
2902
2903         if (err) {
2904                 if (i > 0)
2905                         its_vpe_irq_domain_free(domain, virq, i - 1);
2906
2907                 its_lpi_free(bitmap, base, nr_ids);
2908                 its_free_prop_table(vprop_page);
2909         }
2910
2911         return err;
2912 }
2913
2914 static int its_vpe_irq_domain_activate(struct irq_domain *domain,
2915                                        struct irq_data *d, bool reserve)
2916 {
2917         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2918         struct its_node *its;
2919
2920         /* If we use the list map, we issue VMAPP on demand... */
2921         if (its_list_map)
2922                 return 0;
2923
2924         /* Map the VPE to the first possible CPU */
2925         vpe->col_idx = cpumask_first(cpu_online_mask);
2926
2927         list_for_each_entry(its, &its_nodes, entry) {
2928                 if (!its->is_v4)
2929                         continue;
2930
2931                 its_send_vmapp(its, vpe, true);
2932                 its_send_vinvall(its, vpe);
2933         }
2934
2935         irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
2936
2937         return 0;
2938 }
2939
2940 static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
2941                                           struct irq_data *d)
2942 {
2943         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
2944         struct its_node *its;
2945
2946         /*
2947          * If we use the list map, we unmap the VPE once no VLPIs are
2948          * associated with the VM.
2949          */
2950         if (its_list_map)
2951                 return;
2952
2953         list_for_each_entry(its, &its_nodes, entry) {
2954                 if (!its->is_v4)
2955                         continue;
2956
2957                 its_send_vmapp(its, vpe, false);
2958         }
2959 }
2960
2961 static const struct irq_domain_ops its_vpe_domain_ops = {
2962         .alloc                  = its_vpe_irq_domain_alloc,
2963         .free                   = its_vpe_irq_domain_free,
2964         .activate               = its_vpe_irq_domain_activate,
2965         .deactivate             = its_vpe_irq_domain_deactivate,
2966 };
2967
2968 static int its_force_quiescent(void __iomem *base)
2969 {
2970         u32 count = 1000000;    /* 1s */
2971         u32 val;
2972
2973         val = readl_relaxed(base + GITS_CTLR);
2974         /*
2975          * GIC architecture specification requires the ITS to be both
2976          * disabled and quiescent for writes to GITS_BASER<n> or
2977          * GITS_CBASER to not have UNPREDICTABLE results.
2978          */
2979         if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
2980                 return 0;
2981
2982         /* Disable the generation of all interrupts to this ITS */
2983         val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
2984         writel_relaxed(val, base + GITS_CTLR);
2985
2986         /* Poll GITS_CTLR and wait until ITS becomes quiescent */
2987         while (1) {
2988                 val = readl_relaxed(base + GITS_CTLR);
2989                 if (val & GITS_CTLR_QUIESCENT)
2990                         return 0;
2991
2992                 count--;
2993                 if (!count)
2994                         return -EBUSY;
2995
2996                 cpu_relax();
2997                 udelay(1);
2998         }
2999 }
3000
3001 static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
3002 {
3003         struct its_node *its = data;
3004
3005         /* erratum 22375: only alloc 8MB table size */
3006         its->device_ids = 0x14;         /* 20 bits, 8MB */
3007         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
3008
3009         return true;
3010 }
3011
3012 static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
3013 {
3014         struct its_node *its = data;
3015
3016         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
3017
3018         return true;
3019 }
3020
3021 static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
3022 {
3023         struct its_node *its = data;
3024
3025         /* On QDF2400, the size of the ITE is 16Bytes */
3026         its->ite_size = 16;
3027
3028         return true;
3029 }
3030
3031 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
3032 {
3033         struct its_node *its = its_dev->its;
3034
3035         /*
3036          * The Socionext Synquacer SoC has a so-called 'pre-ITS',
3037          * which maps 32-bit writes targeted at a separate window of
3038          * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
3039          * with device ID taken from bits [device_id_bits + 1:2] of
3040          * the window offset.
3041          */
3042         return its->pre_its_base + (its_dev->device_id << 2);
3043 }
3044
3045 static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
3046 {
3047         struct its_node *its = data;
3048         u32 pre_its_window[2];
3049         u32 ids;
3050
3051         if (!fwnode_property_read_u32_array(its->fwnode_handle,
3052                                            "socionext,synquacer-pre-its",
3053                                            pre_its_window,
3054                                            ARRAY_SIZE(pre_its_window))) {
3055
3056                 its->pre_its_base = pre_its_window[0];
3057                 its->get_msi_base = its_irq_get_msi_base_pre_its;
3058
3059                 ids = ilog2(pre_its_window[1]) - 2;
3060                 if (its->device_ids > ids)
3061                         its->device_ids = ids;
3062
3063                 /* the pre-ITS breaks isolation, so disable MSI remapping */
3064                 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP;
3065                 return true;
3066         }
3067         return false;
3068 }
3069
3070 static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
3071 {
3072         struct its_node *its = data;
3073
3074         /*
3075          * Hip07 insists on using the wrong address for the VLPI
3076          * page. Trick it into doing the right thing...
3077          */
3078         its->vlpi_redist_offset = SZ_128K;
3079         return true;
3080 }
3081
3082 static const struct gic_quirk its_quirks[] = {
3083 #ifdef CONFIG_CAVIUM_ERRATUM_22375
3084         {
3085                 .desc   = "ITS: Cavium errata 22375, 24313",
3086                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
3087                 .mask   = 0xffff0fff,
3088                 .init   = its_enable_quirk_cavium_22375,
3089         },
3090 #endif
3091 #ifdef CONFIG_CAVIUM_ERRATUM_23144
3092         {
3093                 .desc   = "ITS: Cavium erratum 23144",
3094                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
3095                 .mask   = 0xffff0fff,
3096                 .init   = its_enable_quirk_cavium_23144,
3097         },
3098 #endif
3099 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
3100         {
3101                 .desc   = "ITS: QDF2400 erratum 0065",
3102                 .iidr   = 0x00001070, /* QDF2400 ITS rev 1.x */
3103                 .mask   = 0xffffffff,
3104                 .init   = its_enable_quirk_qdf2400_e0065,
3105         },
3106 #endif
3107 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
3108         {
3109                 /*
3110                  * The Socionext Synquacer SoC incorporates ARM's own GIC-500
3111                  * implementation, but with a 'pre-ITS' added that requires
3112                  * special handling in software.
3113                  */
3114                 .desc   = "ITS: Socionext Synquacer pre-ITS",
3115                 .iidr   = 0x0001143b,
3116                 .mask   = 0xffffffff,
3117                 .init   = its_enable_quirk_socionext_synquacer,
3118         },
3119 #endif
3120 #ifdef CONFIG_HISILICON_ERRATUM_161600802
3121         {
3122                 .desc   = "ITS: Hip07 erratum 161600802",
3123                 .iidr   = 0x00000004,
3124                 .mask   = 0xffffffff,
3125                 .init   = its_enable_quirk_hip07_161600802,
3126         },
3127 #endif
3128         {
3129         }
3130 };
3131
3132 static void its_enable_quirks(struct its_node *its)
3133 {
3134         u32 iidr = readl_relaxed(its->base + GITS_IIDR);
3135
3136         gic_enable_quirks(iidr, its_quirks, its);
3137 }
3138
3139 static int its_save_disable(void)
3140 {
3141         struct its_node *its;
3142         int err = 0;
3143
3144         raw_spin_lock(&its_lock);
3145         list_for_each_entry(its, &its_nodes, entry) {
3146                 void __iomem *base;
3147
3148                 if (!(its->flags & ITS_FLAGS_SAVE_SUSPEND_STATE))
3149                         continue;
3150
3151                 base = its->base;
3152                 its->ctlr_save = readl_relaxed(base + GITS_CTLR);
3153                 err = its_force_quiescent(base);
3154                 if (err) {
3155                         pr_err("ITS@%pa: failed to quiesce: %d\n",
3156                                &its->phys_base, err);
3157                         writel_relaxed(its->ctlr_save, base + GITS_CTLR);
3158                         goto err;
3159                 }
3160
3161                 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
3162         }
3163
3164 err:
3165         if (err) {
3166                 list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
3167                         void __iomem *base;
3168
3169                         if (!(its->flags & ITS_FLAGS_SAVE_SUSPEND_STATE))
3170                                 continue;
3171
3172                         base = its->base;
3173                         writel_relaxed(its->ctlr_save, base + GITS_CTLR);
3174                 }
3175         }
3176         raw_spin_unlock(&its_lock);
3177
3178         return err;
3179 }
3180
3181 static void its_restore_enable(void)
3182 {
3183         struct its_node *its;
3184         int ret;
3185
3186         raw_spin_lock(&its_lock);
3187         list_for_each_entry(its, &its_nodes, entry) {
3188                 void __iomem *base;
3189                 int i;
3190
3191                 if (!(its->flags & ITS_FLAGS_SAVE_SUSPEND_STATE))
3192                         continue;
3193
3194                 base = its->base;
3195
3196                 /*
3197                  * Make sure that the ITS is disabled. If it fails to quiesce,
3198                  * don't restore it since writing to CBASER or BASER<n>
3199                  * registers is undefined according to the GIC v3 ITS
3200                  * Specification.
3201                  */
3202                 ret = its_force_quiescent(base);
3203                 if (ret) {
3204                         pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
3205                                &its->phys_base, ret);
3206                         continue;
3207                 }
3208
3209                 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
3210
3211                 /*
3212                  * Writing CBASER resets CREADR to 0, so make CWRITER and
3213                  * cmd_write line up with it.
3214                  */
3215                 its->cmd_write = its->cmd_base;
3216                 gits_write_cwriter(0, base + GITS_CWRITER);
3217
3218                 /* Restore GITS_BASER from the value cache. */
3219                 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
3220                         struct its_baser *baser = &its->tables[i];
3221
3222                         if (!(baser->val & GITS_BASER_VALID))
3223                                 continue;
3224
3225                         its_write_baser(its, baser, baser->val);
3226                 }
3227                 writel_relaxed(its->ctlr_save, base + GITS_CTLR);
3228
3229                 /*
3230                  * Reinit the collection if it's stored in the ITS. This is
3231                  * indicated by the col_id being less than the HCC field.
3232                  * CID < HCC as specified in the GIC v3 Documentation.
3233                  */
3234                 if (its->collections[smp_processor_id()].col_id <
3235                     GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
3236                         its_cpu_init_collection(its);
3237         }
3238         raw_spin_unlock(&its_lock);
3239 }
3240
3241 static struct syscore_ops its_syscore_ops = {
3242         .suspend = its_save_disable,
3243         .resume = its_restore_enable,
3244 };
3245
3246 static int its_init_domain(struct fwnode_handle *handle, struct its_node *its)
3247 {
3248         struct irq_domain *inner_domain;
3249         struct msi_domain_info *info;
3250
3251         info = kzalloc(sizeof(*info), GFP_KERNEL);
3252         if (!info)
3253                 return -ENOMEM;
3254
3255         inner_domain = irq_domain_create_tree(handle, &its_domain_ops, its);
3256         if (!inner_domain) {
3257                 kfree(info);
3258                 return -ENOMEM;
3259         }
3260
3261         inner_domain->parent = its_parent;
3262         irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
3263         inner_domain->flags |= its->msi_domain_flags;
3264         info->ops = &its_msi_domain_ops;
3265         info->data = its;
3266         inner_domain->host_data = info;
3267
3268         return 0;
3269 }
3270
3271 static int its_init_vpe_domain(void)
3272 {
3273         struct its_node *its;
3274         u32 devid;
3275         int entries;
3276
3277         if (gic_rdists->has_direct_lpi) {
3278                 pr_info("ITS: Using DirectLPI for VPE invalidation\n");
3279                 return 0;
3280         }
3281
3282         /* Any ITS will do, even if not v4 */
3283         its = list_first_entry(&its_nodes, struct its_node, entry);
3284
3285         entries = roundup_pow_of_two(nr_cpu_ids);
3286         vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
3287                                  GFP_KERNEL);
3288         if (!vpe_proxy.vpes) {
3289                 pr_err("ITS: Can't allocate GICv4 proxy device array\n");
3290                 return -ENOMEM;
3291         }
3292
3293         /* Use the last possible DevID */
3294         devid = GENMASK(its->device_ids - 1, 0);
3295         vpe_proxy.dev = its_create_device(its, devid, entries, false);
3296         if (!vpe_proxy.dev) {
3297                 kfree(vpe_proxy.vpes);
3298                 pr_err("ITS: Can't allocate GICv4 proxy device\n");
3299                 return -ENOMEM;
3300         }
3301
3302         BUG_ON(entries > vpe_proxy.dev->nr_ites);
3303
3304         raw_spin_lock_init(&vpe_proxy.lock);
3305         vpe_proxy.next_victim = 0;
3306         pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
3307                 devid, vpe_proxy.dev->nr_ites);
3308
3309         return 0;
3310 }
3311
3312 static int __init its_compute_its_list_map(struct resource *res,
3313                                            void __iomem *its_base)
3314 {
3315         int its_number;
3316         u32 ctlr;
3317
3318         /*
3319          * This is assumed to be done early enough that we're
3320          * guaranteed to be single-threaded, hence no
3321          * locking. Should this change, we should address
3322          * this.
3323          */
3324         its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
3325         if (its_number >= GICv4_ITS_LIST_MAX) {
3326                 pr_err("ITS@%pa: No ITSList entry available!\n",
3327                        &res->start);
3328                 return -EINVAL;
3329         }
3330
3331         ctlr = readl_relaxed(its_base + GITS_CTLR);
3332         ctlr &= ~GITS_CTLR_ITS_NUMBER;
3333         ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
3334         writel_relaxed(ctlr, its_base + GITS_CTLR);
3335         ctlr = readl_relaxed(its_base + GITS_CTLR);
3336         if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
3337                 its_number = ctlr & GITS_CTLR_ITS_NUMBER;
3338                 its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
3339         }
3340
3341         if (test_and_set_bit(its_number, &its_list_map)) {
3342                 pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
3343                        &res->start, its_number);
3344                 return -EINVAL;
3345         }
3346
3347         return its_number;
3348 }
3349
3350 static int __init its_probe_one(struct resource *res,
3351                                 struct fwnode_handle *handle, int numa_node)
3352 {
3353         struct its_node *its;
3354         void __iomem *its_base;
3355         u32 val, ctlr;
3356         u64 baser, tmp, typer;
3357         int err;
3358
3359         its_base = ioremap(res->start, resource_size(res));
3360         if (!its_base) {
3361                 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
3362                 return -ENOMEM;
3363         }
3364
3365         val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
3366         if (val != 0x30 && val != 0x40) {
3367                 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
3368                 err = -ENODEV;
3369                 goto out_unmap;
3370         }
3371
3372         err = its_force_quiescent(its_base);
3373         if (err) {
3374                 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
3375                 goto out_unmap;
3376         }
3377
3378         pr_info("ITS %pR\n", res);
3379
3380         its = kzalloc(sizeof(*its), GFP_KERNEL);
3381         if (!its) {
3382                 err = -ENOMEM;
3383                 goto out_unmap;
3384         }
3385
3386         raw_spin_lock_init(&its->lock);
3387         INIT_LIST_HEAD(&its->entry);
3388         INIT_LIST_HEAD(&its->its_device_list);
3389         typer = gic_read_typer(its_base + GITS_TYPER);
3390         its->base = its_base;
3391         its->phys_base = res->start;
3392         its->ite_size = GITS_TYPER_ITT_ENTRY_SIZE(typer);
3393         its->device_ids = GITS_TYPER_DEVBITS(typer);
3394         its->is_v4 = !!(typer & GITS_TYPER_VLPIS);
3395         if (its->is_v4) {
3396                 if (!(typer & GITS_TYPER_VMOVP)) {
3397                         err = its_compute_its_list_map(res, its_base);
3398                         if (err < 0)
3399                                 goto out_free_its;
3400
3401                         its->list_nr = err;
3402
3403                         pr_info("ITS@%pa: Using ITS number %d\n",
3404                                 &res->start, err);
3405                 } else {
3406                         pr_info("ITS@%pa: Single VMOVP capable\n", &res->start);
3407                 }
3408         }
3409
3410         its->numa_node = numa_node;
3411
3412         its->cmd_base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
3413                                                 get_order(ITS_CMD_QUEUE_SZ));
3414         if (!its->cmd_base) {
3415                 err = -ENOMEM;
3416                 goto out_free_its;
3417         }
3418         its->cmd_write = its->cmd_base;
3419         its->fwnode_handle = handle;
3420         its->get_msi_base = its_irq_get_msi_base;
3421         its->msi_domain_flags = IRQ_DOMAIN_FLAG_MSI_REMAP;
3422
3423         its_enable_quirks(its);
3424
3425         err = its_alloc_tables(its);
3426         if (err)
3427                 goto out_free_cmd;
3428
3429         err = its_alloc_collections(its);
3430         if (err)
3431                 goto out_free_tables;
3432
3433         baser = (virt_to_phys(its->cmd_base)    |
3434                  GITS_CBASER_RaWaWb             |
3435                  GITS_CBASER_InnerShareable     |
3436                  (ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
3437                  GITS_CBASER_VALID);
3438
3439         gits_write_cbaser(baser, its->base + GITS_CBASER);
3440         tmp = gits_read_cbaser(its->base + GITS_CBASER);
3441
3442         if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
3443                 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
3444                         /*
3445                          * The HW reports non-shareable, we must
3446                          * remove the cacheability attributes as
3447                          * well.
3448                          */
3449                         baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
3450                                    GITS_CBASER_CACHEABILITY_MASK);
3451                         baser |= GITS_CBASER_nC;
3452                         gits_write_cbaser(baser, its->base + GITS_CBASER);
3453                 }
3454                 pr_info("ITS: using cache flushing for cmd queue\n");
3455                 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
3456         }
3457
3458         gits_write_cwriter(0, its->base + GITS_CWRITER);
3459         ctlr = readl_relaxed(its->base + GITS_CTLR);
3460         ctlr |= GITS_CTLR_ENABLE;
3461         if (its->is_v4)
3462                 ctlr |= GITS_CTLR_ImDe;
3463         writel_relaxed(ctlr, its->base + GITS_CTLR);
3464
3465         if (GITS_TYPER_HCC(typer))
3466                 its->flags |= ITS_FLAGS_SAVE_SUSPEND_STATE;
3467
3468         err = its_init_domain(handle, its);
3469         if (err)
3470                 goto out_free_tables;
3471
3472         raw_spin_lock(&its_lock);
3473         list_add(&its->entry, &its_nodes);
3474         raw_spin_unlock(&its_lock);
3475
3476         return 0;
3477
3478 out_free_tables:
3479         its_free_tables(its);
3480 out_free_cmd:
3481         free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
3482 out_free_its:
3483         kfree(its);
3484 out_unmap:
3485         iounmap(its_base);
3486         pr_err("ITS@%pa: failed probing (%d)\n", &res->start, err);
3487         return err;
3488 }
3489
3490 static bool gic_rdists_supports_plpis(void)
3491 {
3492         return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
3493 }
3494
3495 static int redist_disable_lpis(void)
3496 {
3497         void __iomem *rbase = gic_data_rdist_rd_base();
3498         u64 timeout = USEC_PER_SEC;
3499         u64 val;
3500
3501         /*
3502          * If coming via a CPU hotplug event, we don't need to disable
3503          * LPIs before trying to re-enable them. They are already
3504          * configured and all is well in the world. Detect this case
3505          * by checking the allocation of the pending table for the
3506          * current CPU.
3507          */
3508         if (gic_data_rdist()->pend_page)
3509                 return 0;
3510
3511         if (!gic_rdists_supports_plpis()) {
3512                 pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
3513                 return -ENXIO;
3514         }
3515
3516         val = readl_relaxed(rbase + GICR_CTLR);
3517         if (!(val & GICR_CTLR_ENABLE_LPIS))
3518                 return 0;
3519
3520         pr_warn("CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
3521                 smp_processor_id());
3522         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3523
3524         /* Disable LPIs */
3525         val &= ~GICR_CTLR_ENABLE_LPIS;
3526         writel_relaxed(val, rbase + GICR_CTLR);
3527
3528         /* Make sure any change to GICR_CTLR is observable by the GIC */
3529         dsb(sy);
3530
3531         /*
3532          * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
3533          * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
3534          * Error out if we time out waiting for RWP to clear.
3535          */
3536         while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
3537                 if (!timeout) {
3538                         pr_err("CPU%d: Timeout while disabling LPIs\n",
3539                                smp_processor_id());
3540                         return -ETIMEDOUT;
3541                 }
3542                 udelay(1);
3543                 timeout--;
3544         }
3545
3546         /*
3547          * After it has been written to 1, it is IMPLEMENTATION
3548          * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
3549          * cleared to 0. Error out if clearing the bit failed.
3550          */
3551         if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
3552                 pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
3553                 return -EBUSY;
3554         }
3555
3556         return 0;
3557 }
3558
3559 int its_cpu_init(void)
3560 {
3561         if (!list_empty(&its_nodes)) {
3562                 int ret;
3563
3564                 ret = redist_disable_lpis();
3565                 if (ret)
3566                         return ret;
3567
3568                 its_cpu_init_lpis();
3569                 its_cpu_init_collections();
3570         }
3571
3572         return 0;
3573 }
3574
3575 static const struct of_device_id its_device_id[] = {
3576         {       .compatible     = "arm,gic-v3-its",     },
3577         {},
3578 };
3579
3580 static int __init its_of_probe(struct device_node *node)
3581 {
3582         struct device_node *np;
3583         struct resource res;
3584
3585         for (np = of_find_matching_node(node, its_device_id); np;
3586              np = of_find_matching_node(np, its_device_id)) {
3587                 if (!of_device_is_available(np))
3588                         continue;
3589                 if (!of_property_read_bool(np, "msi-controller")) {
3590                         pr_warn("%pOF: no msi-controller property, ITS ignored\n",
3591                                 np);
3592                         continue;
3593                 }
3594
3595                 if (of_address_to_resource(np, 0, &res)) {
3596                         pr_warn("%pOF: no regs?\n", np);
3597                         continue;
3598                 }
3599
3600                 its_probe_one(&res, &np->fwnode, of_node_to_nid(np));
3601         }
3602         return 0;
3603 }
3604
3605 #ifdef CONFIG_ACPI
3606
3607 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
3608
3609 #ifdef CONFIG_ACPI_NUMA
3610 struct its_srat_map {
3611         /* numa node id */
3612         u32     numa_node;
3613         /* GIC ITS ID */
3614         u32     its_id;
3615 };
3616
3617 static struct its_srat_map *its_srat_maps __initdata;
3618 static int its_in_srat __initdata;
3619
3620 static int __init acpi_get_its_numa_node(u32 its_id)
3621 {
3622         int i;
3623
3624         for (i = 0; i < its_in_srat; i++) {
3625                 if (its_id == its_srat_maps[i].its_id)
3626                         return its_srat_maps[i].numa_node;
3627         }
3628         return NUMA_NO_NODE;
3629 }
3630
3631 static int __init gic_acpi_match_srat_its(struct acpi_subtable_header *header,
3632                                           const unsigned long end)
3633 {
3634         return 0;
3635 }
3636
3637 static int __init gic_acpi_parse_srat_its(struct acpi_subtable_header *header,
3638                          const unsigned long end)
3639 {
3640         int node;
3641         struct acpi_srat_gic_its_affinity *its_affinity;
3642
3643         its_affinity = (struct acpi_srat_gic_its_affinity *)header;
3644         if (!its_affinity)
3645                 return -EINVAL;
3646
3647         if (its_affinity->header.length < sizeof(*its_affinity)) {
3648                 pr_err("SRAT: Invalid header length %d in ITS affinity\n",
3649                         its_affinity->header.length);
3650                 return -EINVAL;
3651         }
3652
3653         node = acpi_map_pxm_to_node(its_affinity->proximity_domain);
3654
3655         if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
3656                 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
3657                 return 0;
3658         }
3659
3660         its_srat_maps[its_in_srat].numa_node = node;
3661         its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
3662         its_in_srat++;
3663         pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
3664                 its_affinity->proximity_domain, its_affinity->its_id, node);
3665
3666         return 0;
3667 }
3668
3669 static void __init acpi_table_parse_srat_its(void)
3670 {
3671         int count;
3672
3673         count = acpi_table_parse_entries(ACPI_SIG_SRAT,
3674                         sizeof(struct acpi_table_srat),
3675                         ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
3676                         gic_acpi_match_srat_its, 0);
3677         if (count <= 0)
3678                 return;
3679
3680         its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
3681                                       GFP_KERNEL);
3682         if (!its_srat_maps) {
3683                 pr_warn("SRAT: Failed to allocate memory for its_srat_maps!\n");
3684                 return;
3685         }
3686
3687         acpi_table_parse_entries(ACPI_SIG_SRAT,
3688                         sizeof(struct acpi_table_srat),
3689                         ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
3690                         gic_acpi_parse_srat_its, 0);
3691 }
3692
3693 /* free the its_srat_maps after ITS probing */
3694 static void __init acpi_its_srat_maps_free(void)
3695 {
3696         kfree(its_srat_maps);
3697 }
3698 #else
3699 static void __init acpi_table_parse_srat_its(void)      { }
3700 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
3701 static void __init acpi_its_srat_maps_free(void) { }
3702 #endif
3703
3704 static int __init gic_acpi_parse_madt_its(struct acpi_subtable_header *header,
3705                                           const unsigned long end)
3706 {
3707         struct acpi_madt_generic_translator *its_entry;
3708         struct fwnode_handle *dom_handle;
3709         struct resource res;
3710         int err;
3711
3712         its_entry = (struct acpi_madt_generic_translator *)header;
3713         memset(&res, 0, sizeof(res));
3714         res.start = its_entry->base_address;
3715         res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
3716         res.flags = IORESOURCE_MEM;
3717
3718         dom_handle = irq_domain_alloc_fwnode((void *)its_entry->base_address);
3719         if (!dom_handle) {
3720                 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
3721                        &res.start);
3722                 return -ENOMEM;
3723         }
3724
3725         err = iort_register_domain_token(its_entry->translation_id, res.start,
3726                                          dom_handle);
3727         if (err) {
3728                 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
3729                        &res.start, its_entry->translation_id);
3730                 goto dom_err;
3731         }
3732
3733         err = its_probe_one(&res, dom_handle,
3734                         acpi_get_its_numa_node(its_entry->translation_id));
3735         if (!err)
3736                 return 0;
3737
3738         iort_deregister_domain_token(its_entry->translation_id);
3739 dom_err:
3740         irq_domain_free_fwnode(dom_handle);
3741         return err;
3742 }
3743
3744 static void __init its_acpi_probe(void)
3745 {
3746         acpi_table_parse_srat_its();
3747         acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
3748                               gic_acpi_parse_madt_its, 0);
3749         acpi_its_srat_maps_free();
3750 }
3751 #else
3752 static void __init its_acpi_probe(void) { }
3753 #endif
3754
3755 int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
3756                     struct irq_domain *parent_domain)
3757 {
3758         struct device_node *of_node;
3759         struct its_node *its;
3760         bool has_v4 = false;
3761         int err;
3762
3763         its_parent = parent_domain;
3764         of_node = to_of_node(handle);
3765         if (of_node)
3766                 its_of_probe(of_node);
3767         else
3768                 its_acpi_probe();
3769
3770         if (list_empty(&its_nodes)) {
3771                 pr_warn("ITS: No ITS available, not enabling LPIs\n");
3772                 return -ENXIO;
3773         }
3774
3775         gic_rdists = rdists;
3776         err = its_alloc_lpi_tables();
3777         if (err)
3778                 return err;
3779
3780         list_for_each_entry(its, &its_nodes, entry)
3781                 has_v4 |= its->is_v4;
3782
3783         if (has_v4 & rdists->has_vlpis) {
3784                 if (its_init_vpe_domain() ||
3785                     its_init_v4(parent_domain, &its_vpe_domain_ops)) {
3786                         rdists->has_vlpis = false;
3787                         pr_err("ITS: Disabling GICv4 support\n");
3788                 }
3789         }
3790
3791         register_syscore_ops(&its_syscore_ops);
3792
3793         return 0;
3794 }