drivers/iommu/arm-smmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * IOMMU API for ARM architected SMMU implementations.
   4  *
   5  * Copyright (C) 2013 ARM Limited
   6  *
   7  * Author: Will Deacon <will.deacon@arm.com>
   8  *
   9  * This driver currently supports:
  10  *      - SMMUv1 and v2 implementations
  11  *      - Stream-matching and stream-indexing
  12  *      - v7/v8 long-descriptor format
  13  *      - Non-secure access to the SMMU
  14  *      - Context fault reporting
  15  *      - Extended Stream ID (16 bit)
  16  */
  17
  18 #define pr_fmt(fmt) "arm-smmu: " fmt
  19
  20 #include <linux/acpi.h>
  21 #include <linux/acpi_iort.h>
  22 #include <linux/bitfield.h>
  23 #include <linux/delay.h>
  24 #include <linux/dma-iommu.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/err.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/io.h>
  29 #include <linux/io-64-nonatomic-hi-lo.h>
  30 #include <linux/io-pgtable.h>
  31 #include <linux/iopoll.h>
  32 #include <linux/init.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/of.h>
  35 #include <linux/of_address.h>
  36 #include <linux/of_device.h>
  37 #include <linux/of_iommu.h>
  38 #include <linux/pci.h>
  39 #include <linux/platform_device.h>
  40 #include <linux/pm_runtime.h>
  41 #include <linux/slab.h>
  42
  43 #include <linux/amba/bus.h>
  44 #include <linux/fsl/mc.h>
  45
  46 #include "arm-smmu.h"
  47
  48 /*
  49  * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
  50  * global register space are still, in fact, using a hypervisor to mediate it
  51  * by trapping and emulating register accesses. Sadly, some deployed versions
  52  * of said trapping code have bugs wherein they go horribly wrong for stores
  53  * using r31 (i.e. XZR/WZR) as the source register.
  54  */
  55 #define QCOM_DUMMY_VAL -1
  56
  57 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
  58
  59 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
  60 #define ARM_MMU500_ACR_S2CRB_TLBEN      (1 << 10)
  61 #define ARM_MMU500_ACR_SMTNMB_TLBEN     (1 << 8)
  62
  63 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
  64 #define TLB_SPIN_COUNT                  10
  65
  66 #define MSI_IOVA_BASE                   0x8000000
  67 #define MSI_IOVA_LENGTH                 0x100000
  68
  69 static int force_stage;
  70 /*
  71  * not really modular, but the easiest way to keep compat with existing
  72  * bootargs behaviour is to continue using module_param() here.
  73  */
  74 module_param(force_stage, int, S_IRUGO);
  75 MODULE_PARM_DESC(force_stage,
  76         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
  77 static bool disable_bypass =
  78         IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
  79 module_param(disable_bypass, bool, S_IRUGO);
  80 MODULE_PARM_DESC(disable_bypass,
  81         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
  82
  83 struct arm_smmu_s2cr {
  84         struct iommu_group              *group;
  85         int                             count;
  86         enum arm_smmu_s2cr_type         type;
  87         enum arm_smmu_s2cr_privcfg      privcfg;
  88         u8                              cbndx;
  89 };
  90
  91 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
  92         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
  93 }
  94
  95 struct arm_smmu_smr {
  96         u16                             mask;
  97         u16                             id;
  98         bool                            valid;
  99 };
 100
 101 struct arm_smmu_cb {
 102         u64                             ttbr[2];
 103         u32                             tcr[2];
 104         u32                             mair[2];
 105         struct arm_smmu_cfg             *cfg;
 106 };
 107
 108 struct arm_smmu_master_cfg {
 109         struct arm_smmu_device          *smmu;
 110         s16                             smendx[];
 111 };
 112 #define INVALID_SMENDX                  -1
 113 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 114 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 115 #define fwspec_smendx(fw, i) \
 116         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 117 #define for_each_cfg_sme(fw, i, idx) \
 118         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 119
 120 enum arm_smmu_context_fmt {
 121         ARM_SMMU_CTX_FMT_NONE,
 122         ARM_SMMU_CTX_FMT_AARCH64,
 123         ARM_SMMU_CTX_FMT_AARCH32_L,
 124         ARM_SMMU_CTX_FMT_AARCH32_S,
 125 };
 126
 127 struct arm_smmu_cfg {
 128         u8                              cbndx;
 129         u8                              irptndx;
 130         union {
 131                 u16                     asid;
 132                 u16                     vmid;
 133         };
 134         enum arm_smmu_cbar_type         cbar;
 135         enum arm_smmu_context_fmt       fmt;
 136 };
 137 #define INVALID_IRPTNDX                 0xff
 138
 139 enum arm_smmu_domain_stage {
 140         ARM_SMMU_DOMAIN_S1 = 0,
 141         ARM_SMMU_DOMAIN_S2,
 142         ARM_SMMU_DOMAIN_NESTED,
 143         ARM_SMMU_DOMAIN_BYPASS,
 144 };
 145
 146 struct arm_smmu_domain {
 147         struct arm_smmu_device          *smmu;
 148         struct io_pgtable_ops           *pgtbl_ops;
 149         const struct iommu_gather_ops   *tlb_ops;
 150         struct arm_smmu_cfg             cfg;
 151         enum arm_smmu_domain_stage      stage;
 152         bool                            non_strict;
 153         struct mutex                    init_mutex; /* Protects smmu pointer */
 154         spinlock_t                      cb_lock; /* Serialises ATS1* ops and TLB syncs */
 155         struct iommu_domain             domain;
 156 };
 157
 158 static int arm_smmu_gr0_ns(int offset)
 159 {
 160         switch(offset) {
 161         case ARM_SMMU_GR0_sCR0:
 162         case ARM_SMMU_GR0_sACR:
 163         case ARM_SMMU_GR0_sGFSR:
 164         case ARM_SMMU_GR0_sGFSYNR0:
 165         case ARM_SMMU_GR0_sGFSYNR1:
 166         case ARM_SMMU_GR0_sGFSYNR2:
 167                 return offset + 0x400;
 168         default:
 169                 return offset;
 170         }
 171 }
 172
 173 static void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)
 174 {
 175         return smmu->base + (n << smmu->pgshift);
 176 }
 177
 178 static u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)
 179 {
 180         if ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) && page == 0)
 181                 offset = arm_smmu_gr0_ns(offset);
 182
 183         return readl_relaxed(arm_smmu_page(smmu, page) + offset);
 184 }
 185
 186 static void arm_smmu_writel(struct arm_smmu_device *smmu, int page, int offset,
 187                             u32 val)
 188 {
 189         if ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) && page == 0)
 190                 offset = arm_smmu_gr0_ns(offset);
 191
 192         writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
 193 }
 194
 195 static u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset)
 196 {
 197         return readq_relaxed(arm_smmu_page(smmu, page) + offset);
 198 }
 199
 200 static void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, int offset,
 201                             u64 val)
 202 {
 203         writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
 204 }
 205
 206 #define ARM_SMMU_GR0            0
 207 #define ARM_SMMU_GR1            1
 208 #define ARM_SMMU_CB(s, n)       ((s)->numpage + (n))
 209
 210 #define arm_smmu_gr0_read(s, o)         \
 211         arm_smmu_readl((s), ARM_SMMU_GR0, (o))
 212 #define arm_smmu_gr0_write(s, o, v)     \
 213         arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v))
 214
 215 #define arm_smmu_gr1_read(s, o)         \
 216         arm_smmu_readl((s), ARM_SMMU_GR1, (o))
 217 #define arm_smmu_gr1_write(s, o, v)     \
 218         arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v))
 219
 220 #define arm_smmu_cb_read(s, n, o)       \
 221         arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o))
 222 #define arm_smmu_cb_write(s, n, o, v)   \
 223         arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v))
 224 #define arm_smmu_cb_readq(s, n, o)      \
 225         arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o))
 226 #define arm_smmu_cb_writeq(s, n, o, v)  \
 227         arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v))
 228
 229 struct arm_smmu_option_prop {
 230         u32 opt;
 231         const char *prop;
 232 };
 233
 234 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 235
 236 static bool using_legacy_binding, using_generic_binding;
 237
 238 static struct arm_smmu_option_prop arm_smmu_options[] = {
 239         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 240         { 0, NULL},
 241 };
 242
 243 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
 244 {
 245         if (pm_runtime_enabled(smmu->dev))
 246                 return pm_runtime_get_sync(smmu->dev);
 247
 248         return 0;
 249 }
 250
 251 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
 252 {
 253         if (pm_runtime_enabled(smmu->dev))
 254                 pm_runtime_put(smmu->dev);
 255 }
 256
 257 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 258 {
 259         return container_of(dom, struct arm_smmu_domain, domain);
 260 }
 261
 262 static void parse_driver_options(struct arm_smmu_device *smmu)
 263 {
 264         int i = 0;
 265
 266         do {
 267                 if (of_property_read_bool(smmu->dev->of_node,
 268                                                 arm_smmu_options[i].prop)) {
 269                         smmu->options |= arm_smmu_options[i].opt;
 270                         dev_notice(smmu->dev, "option %s\n",
 271                                 arm_smmu_options[i].prop);
 272                 }
 273         } while (arm_smmu_options[++i].opt);
 274 }
 275
 276 static struct device_node *dev_get_dev_node(struct device *dev)
 277 {
 278         if (dev_is_pci(dev)) {
 279                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 280
 281                 while (!pci_is_root_bus(bus))
 282                         bus = bus->parent;
 283                 return of_node_get(bus->bridge->parent->of_node);
 284         }
 285
 286         return of_node_get(dev->of_node);
 287 }
 288
 289 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 290 {
 291         *((__be32 *)data) = cpu_to_be32(alias);
 292         return 0; /* Continue walking */
 293 }
 294
 295 static int __find_legacy_master_phandle(struct device *dev, void *data)
 296 {
 297         struct of_phandle_iterator *it = *(void **)data;
 298         struct device_node *np = it->node;
 299         int err;
 300
 301         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 302                             "#stream-id-cells", 0)
 303                 if (it->node == np) {
 304                         *(void **)data = dev;
 305                         return 1;
 306                 }
 307         it->node = np;
 308         return err == -ENOENT ? 0 : err;
 309 }
 310
 311 static struct platform_driver arm_smmu_driver;
 312 static struct iommu_ops arm_smmu_ops;
 313
 314 static int arm_smmu_register_legacy_master(struct device *dev,
 315                                            struct arm_smmu_device **smmu)
 316 {
 317         struct device *smmu_dev;
 318         struct device_node *np;
 319         struct of_phandle_iterator it;
 320         void *data = &it;
 321         u32 *sids;
 322         __be32 pci_sid;
 323         int err;
 324
 325         np = dev_get_dev_node(dev);
 326         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 327                 of_node_put(np);
 328                 return -ENODEV;
 329         }
 330
 331         it.node = np;
 332         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 333                                      __find_legacy_master_phandle);
 334         smmu_dev = data;
 335         of_node_put(np);
 336         if (err == 0)
 337                 return -ENODEV;
 338         if (err < 0)
 339                 return err;
 340
 341         if (dev_is_pci(dev)) {
 342                 /* "mmu-masters" assumes Stream ID == Requester ID */
 343                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 344                                        &pci_sid);
 345                 it.cur = &pci_sid;
 346                 it.cur_count = 1;
 347         }
 348
 349         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 350                                 &arm_smmu_ops);
 351         if (err)
 352                 return err;
 353
 354         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 355         if (!sids)
 356                 return -ENOMEM;
 357
 358         *smmu = dev_get_drvdata(smmu_dev);
 359         of_phandle_iterator_args(&it, sids, it.cur_count);
 360         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 361         kfree(sids);
 362         return err;
 363 }
 364
 365 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 366 {
 367         int idx;
 368
 369         do {
 370                 idx = find_next_zero_bit(map, end, start);
 371                 if (idx == end)
 372                         return -ENOSPC;
 373         } while (test_and_set_bit(idx, map));
 374
 375         return idx;
 376 }
 377
 378 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 379 {
 380         clear_bit(idx, map);
 381 }
 382
 383 /* Wait for any pending TLB invalidations to complete */
 384 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
 385                                 int sync, int status)
 386 {
 387         unsigned int spin_cnt, delay;
 388         u32 reg;
 389
 390         arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
 391         for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 392                 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 393                         reg = arm_smmu_readl(smmu, page, status);
 394                         if (!(reg & sTLBGSTATUS_GSACTIVE))
 395                                 return;
 396                         cpu_relax();
 397                 }
 398                 udelay(delay);
 399         }
 400         dev_err_ratelimited(smmu->dev,
 401                             "TLB sync timed out -- SMMU may be deadlocked\n");
 402 }
 403
 404 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 405 {
 406         unsigned long flags;
 407
 408         spin_lock_irqsave(&smmu->global_sync_lock, flags);
 409         __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
 410                             ARM_SMMU_GR0_sTLBGSTATUS);
 411         spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
 412 }
 413
 414 static void arm_smmu_tlb_sync_context(void *cookie)
 415 {
 416         struct arm_smmu_domain *smmu_domain = cookie;
 417         struct arm_smmu_device *smmu = smmu_domain->smmu;
 418         unsigned long flags;
 419
 420         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
 421         __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
 422                             ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
 423         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
 424 }
 425
 426 static void arm_smmu_tlb_sync_vmid(void *cookie)
 427 {
 428         struct arm_smmu_domain *smmu_domain = cookie;
 429
 430         arm_smmu_tlb_sync_global(smmu_domain->smmu);
 431 }
 432
 433 static void arm_smmu_tlb_inv_context_s1(void *cookie)
 434 {
 435         struct arm_smmu_domain *smmu_domain = cookie;
 436         /*
 437          * The TLBI write may be relaxed, so ensure that PTEs cleared by the
 438          * current CPU are visible beforehand.
 439          */
 440         wmb();
 441         arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
 442                           ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
 443         arm_smmu_tlb_sync_context(cookie);
 444 }
 445
 446 static void arm_smmu_tlb_inv_context_s2(void *cookie)
 447 {
 448         struct arm_smmu_domain *smmu_domain = cookie;
 449         struct arm_smmu_device *smmu = smmu_domain->smmu;
 450
 451         /* See above */
 452         wmb();
 453         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 454         arm_smmu_tlb_sync_global(smmu);
 455 }
 456
 457 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
 458                                       size_t granule, bool leaf, void *cookie)
 459 {
 460         struct arm_smmu_domain *smmu_domain = cookie;
 461         struct arm_smmu_device *smmu = smmu_domain->smmu;
 462         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 463         int reg, idx = cfg->cbndx;
 464
 465         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 466                 wmb();
 467
 468         reg = leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 469
 470         if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 471                 iova = (iova >> 12) << 12;
 472                 iova |= cfg->asid;
 473                 do {
 474                         arm_smmu_cb_write(smmu, idx, reg, iova);
 475                         iova += granule;
 476                 } while (size -= granule);
 477         } else {
 478                 iova >>= 12;
 479                 iova |= (u64)cfg->asid << 48;
 480                 do {
 481                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 482                         iova += granule >> 12;
 483                 } while (size -= granule);
 484         }
 485 }
 486
 487 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
 488                                       size_t granule, bool leaf, void *cookie)
 489 {
 490         struct arm_smmu_domain *smmu_domain = cookie;
 491         struct arm_smmu_device *smmu = smmu_domain->smmu;
 492         int reg, idx = smmu_domain->cfg.cbndx;
 493
 494         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 495                 wmb();
 496
 497         reg = leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : ARM_SMMU_CB_S2_TLBIIPAS2;
 498         iova >>= 12;
 499         do {
 500                 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
 501                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 502                 else
 503                         arm_smmu_cb_write(smmu, idx, reg, iova);
 504                 iova += granule >> 12;
 505         } while (size -= granule);
 506 }
 507
 508 /*
 509  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 510  * almost negligible, but the benefit of getting the first one in as far ahead
 511  * of the sync as possible is significant, hence we don't just make this a
 512  * no-op and set .tlb_sync to arm_smmu_inv_context_s2() as you might think.
 513  */
 514 static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size,
 515                                          size_t granule, bool leaf, void *cookie)
 516 {
 517         struct arm_smmu_domain *smmu_domain = cookie;
 518         struct arm_smmu_device *smmu = smmu_domain->smmu;
 519
 520         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 521                 wmb();
 522
 523         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 524 }
 525
 526 static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = {
 527         .tlb_flush_all  = arm_smmu_tlb_inv_context_s1,
 528         .tlb_add_flush  = arm_smmu_tlb_inv_range_s1,
 529         .tlb_sync       = arm_smmu_tlb_sync_context,
 530 };
 531
 532 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v2 = {
 533         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 534         .tlb_add_flush  = arm_smmu_tlb_inv_range_s2,
 535         .tlb_sync       = arm_smmu_tlb_sync_context,
 536 };
 537
 538 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = {
 539         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 540         .tlb_add_flush  = arm_smmu_tlb_inv_vmid_nosync,
 541         .tlb_sync       = arm_smmu_tlb_sync_vmid,
 542 };
 543
 544 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 545 {
 546         u32 fsr, fsynr, cbfrsynra;
 547         unsigned long iova;
 548         struct iommu_domain *domain = dev;
 549         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 550         struct arm_smmu_device *smmu = smmu_domain->smmu;
 551         int idx = smmu_domain->cfg.cbndx;
 552
 553         fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
 554         if (!(fsr & FSR_FAULT))
 555                 return IRQ_NONE;
 556
 557         fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
 558         iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
 559         cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
 560
 561         dev_err_ratelimited(smmu->dev,
 562         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
 563                             fsr, iova, fsynr, cbfrsynra, idx);
 564
 565         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
 566         return IRQ_HANDLED;
 567 }
 568
 569 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 570 {
 571         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 572         struct arm_smmu_device *smmu = dev;
 573
 574         gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
 575         gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
 576         gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
 577         gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
 578
 579         if (!gfsr)
 580                 return IRQ_NONE;
 581
 582         dev_err_ratelimited(smmu->dev,
 583                 "Unexpected global fault, this could be serious\n");
 584         dev_err_ratelimited(smmu->dev,
 585                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 586                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 587
 588         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
 589         return IRQ_HANDLED;
 590 }
 591
 592 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 593                                        struct io_pgtable_cfg *pgtbl_cfg)
 594 {
 595         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 596         struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
 597         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 598
 599         cb->cfg = cfg;
 600
 601         /* TCR */
 602         if (stage1) {
 603                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 604                         cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
 605                 } else {
 606                         cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 607                         cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 608                         cb->tcr[1] |= FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM);
 609                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 610                                 cb->tcr[1] |= TCR2_AS;
 611                 }
 612         } else {
 613                 cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 614         }
 615
 616         /* TTBRs */
 617         if (stage1) {
 618                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 619                         cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 620                         cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 621                 } else {
 622                         cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 623                         cb->ttbr[0] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
 624                         cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 625                         cb->ttbr[1] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
 626                 }
 627         } else {
 628                 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 629         }
 630
 631         /* MAIRs (stage-1 only) */
 632         if (stage1) {
 633                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 634                         cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
 635                         cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
 636                 } else {
 637                         cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 638                         cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 639                 }
 640         }
 641 }
 642
 643 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
 644 {
 645         u32 reg;
 646         bool stage1;
 647         struct arm_smmu_cb *cb = &smmu->cbs[idx];
 648         struct arm_smmu_cfg *cfg = cb->cfg;
 649
 650         /* Unassigned context banks only need disabling */
 651         if (!cfg) {
 652                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
 653                 return;
 654         }
 655
 656         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 657
 658         /* CBA2R */
 659         if (smmu->version > ARM_SMMU_V1) {
 660                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 661                         reg = CBA2R_VA64;
 662                 else
 663                         reg = 0;
 664                 /* 16-bit VMIDs live in CBA2R */
 665                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 666                         reg |= FIELD_PREP(CBA2R_VMID16, cfg->vmid);
 667
 668                 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
 669         }
 670
 671         /* CBAR */
 672         reg = FIELD_PREP(CBAR_TYPE, cfg->cbar);
 673         if (smmu->version < ARM_SMMU_V2)
 674                 reg |= FIELD_PREP(CBAR_IRPTNDX, cfg->irptndx);
 675
 676         /*
 677          * Use the weakest shareability/memory types, so they are
 678          * overridden by the ttbcr/pte.
 679          */
 680         if (stage1) {
 681                 reg |= FIELD_PREP(CBAR_S1_BPSHCFG, CBAR_S1_BPSHCFG_NSH) |
 682                         FIELD_PREP(CBAR_S1_MEMATTR, CBAR_S1_MEMATTR_WB);
 683         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 684                 /* 8-bit VMIDs live in CBAR */
 685                 reg |= FIELD_PREP(CBAR_VMID, cfg->vmid);
 686         }
 687         arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
 688
 689         /*
 690          * TCR
 691          * We must write this before the TTBRs, since it determines the
 692          * access behaviour of some fields (in particular, ASID[15:8]).
 693          */
 694         if (stage1 && smmu->version > ARM_SMMU_V1)
 695                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
 696         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
 697
 698         /* TTBRs */
 699         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 700                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
 701                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 702                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
 703         } else {
 704                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 705                 if (stage1)
 706                         arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
 707                                            cb->ttbr[1]);
 708         }
 709
 710         /* MAIRs (stage-1 only) */
 711         if (stage1) {
 712                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
 713                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
 714         }
 715
 716         /* SCTLR */
 717         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 718         if (stage1)
 719                 reg |= SCTLR_S1_ASIDPNE;
 720         if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 721                 reg |= SCTLR_E;
 722
 723         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
 724 }
 725
 726 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 727                                         struct arm_smmu_device *smmu)
 728 {
 729         int irq, start, ret = 0;
 730         unsigned long ias, oas;
 731         struct io_pgtable_ops *pgtbl_ops;
 732         struct io_pgtable_cfg pgtbl_cfg;
 733         enum io_pgtable_fmt fmt;
 734         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 735         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 736
 737         mutex_lock(&smmu_domain->init_mutex);
 738         if (smmu_domain->smmu)
 739                 goto out_unlock;
 740
 741         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
 742                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
 743                 smmu_domain->smmu = smmu;
 744                 goto out_unlock;
 745         }
 746
 747         /*
 748          * Mapping the requested stage onto what we support is surprisingly
 749          * complicated, mainly because the spec allows S1+S2 SMMUs without
 750          * support for nested translation. That means we end up with the
 751          * following table:
 752          *
 753          * Requested        Supported        Actual
 754          *     S1               N              S1
 755          *     S1             S1+S2            S1
 756          *     S1               S2             S2
 757          *     S1               S1             S1
 758          *     N                N              N
 759          *     N              S1+S2            S2
 760          *     N                S2             S2
 761          *     N                S1             S1
 762          *
 763          * Note that you can't actually request stage-2 mappings.
 764          */
 765         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 766                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 767         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 768                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 769
 770         /*
 771          * Choosing a suitable context format is even more fiddly. Until we
 772          * grow some way for the caller to express a preference, and/or move
 773          * the decision into the io-pgtable code where it arguably belongs,
 774          * just aim for the closest thing to the rest of the system, and hope
 775          * that the hardware isn't esoteric enough that we can't assume AArch64
 776          * support to be a superset of AArch32 support...
 777          */
 778         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 779                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 780         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 781             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 782             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 783             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 784                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 785         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 786             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 787                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 788                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 789                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 790
 791         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 792                 ret = -EINVAL;
 793                 goto out_unlock;
 794         }
 795
 796         switch (smmu_domain->stage) {
 797         case ARM_SMMU_DOMAIN_S1:
 798                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 799                 start = smmu->num_s2_context_banks;
 800                 ias = smmu->va_size;
 801                 oas = smmu->ipa_size;
 802                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 803                         fmt = ARM_64_LPAE_S1;
 804                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 805                         fmt = ARM_32_LPAE_S1;
 806                         ias = min(ias, 32UL);
 807                         oas = min(oas, 40UL);
 808                 } else {
 809                         fmt = ARM_V7S;
 810                         ias = min(ias, 32UL);
 811                         oas = min(oas, 32UL);
 812                 }
 813                 smmu_domain->tlb_ops = &arm_smmu_s1_tlb_ops;
 814                 break;
 815         case ARM_SMMU_DOMAIN_NESTED:
 816                 /*
 817                  * We will likely want to change this if/when KVM gets
 818                  * involved.
 819                  */
 820         case ARM_SMMU_DOMAIN_S2:
 821                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 822                 start = 0;
 823                 ias = smmu->ipa_size;
 824                 oas = smmu->pa_size;
 825                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 826                         fmt = ARM_64_LPAE_S2;
 827                 } else {
 828                         fmt = ARM_32_LPAE_S2;
 829                         ias = min(ias, 40UL);
 830                         oas = min(oas, 40UL);
 831                 }
 832                 if (smmu->version == ARM_SMMU_V2)
 833                         smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v2;
 834                 else
 835                         smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v1;
 836                 break;
 837         default:
 838                 ret = -EINVAL;
 839                 goto out_unlock;
 840         }
 841         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 842                                       smmu->num_context_banks);
 843         if (ret < 0)
 844                 goto out_unlock;
 845
 846         cfg->cbndx = ret;
 847         if (smmu->version < ARM_SMMU_V2) {
 848                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 849                 cfg->irptndx %= smmu->num_context_irqs;
 850         } else {
 851                 cfg->irptndx = cfg->cbndx;
 852         }
 853
 854         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
 855                 cfg->vmid = cfg->cbndx + 1 + smmu->cavium_id_base;
 856         else
 857                 cfg->asid = cfg->cbndx + smmu->cavium_id_base;
 858
 859         pgtbl_cfg = (struct io_pgtable_cfg) {
 860                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 861                 .ias            = ias,
 862                 .oas            = oas,
 863                 .coherent_walk  = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
 864                 .tlb            = smmu_domain->tlb_ops,
 865                 .iommu_dev      = smmu->dev,
 866         };
 867
 868         if (smmu_domain->non_strict)
 869                 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
 870
 871         smmu_domain->smmu = smmu;
 872         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 873         if (!pgtbl_ops) {
 874                 ret = -ENOMEM;
 875                 goto out_clear_smmu;
 876         }
 877
 878         /* Update the domain's page sizes to reflect the page table format */
 879         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 880         domain->geometry.aperture_end = (1UL << ias) - 1;
 881         domain->geometry.force_aperture = true;
 882
 883         /* Initialise the context bank with our page table cfg */
 884         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 885         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 886
 887         /*
 888          * Request context fault interrupt. Do this last to avoid the
 889          * handler seeing a half-initialised domain state.
 890          */
 891         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 892         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 893                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 894         if (ret < 0) {
 895                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 896                         cfg->irptndx, irq);
 897                 cfg->irptndx = INVALID_IRPTNDX;
 898         }
 899
 900         mutex_unlock(&smmu_domain->init_mutex);
 901
 902         /* Publish page table ops for map/unmap */
 903         smmu_domain->pgtbl_ops = pgtbl_ops;
 904         return 0;
 905
 906 out_clear_smmu:
 907         smmu_domain->smmu = NULL;
 908 out_unlock:
 909         mutex_unlock(&smmu_domain->init_mutex);
 910         return ret;
 911 }
 912
 913 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 914 {
 915         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 916         struct arm_smmu_device *smmu = smmu_domain->smmu;
 917         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 918         int ret, irq;
 919
 920         if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
 921                 return;
 922
 923         ret = arm_smmu_rpm_get(smmu);
 924         if (ret < 0)
 925                 return;
 926
 927         /*
 928          * Disable the context bank and free the page tables before freeing
 929          * it.
 930          */
 931         smmu->cbs[cfg->cbndx].cfg = NULL;
 932         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 933
 934         if (cfg->irptndx != INVALID_IRPTNDX) {
 935                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 936                 devm_free_irq(smmu->dev, irq, domain);
 937         }
 938
 939         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 940         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 941
 942         arm_smmu_rpm_put(smmu);
 943 }
 944
 945 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 946 {
 947         struct arm_smmu_domain *smmu_domain;
 948
 949         if (type != IOMMU_DOMAIN_UNMANAGED &&
 950             type != IOMMU_DOMAIN_DMA &&
 951             type != IOMMU_DOMAIN_IDENTITY)
 952                 return NULL;
 953         /*
 954          * Allocate the domain and initialise some of its data structures.
 955          * We can't really do anything meaningful until we've added a
 956          * master.
 957          */
 958         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
 959         if (!smmu_domain)
 960                 return NULL;
 961
 962         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
 963             iommu_get_dma_cookie(&smmu_domain->domain))) {
 964                 kfree(smmu_domain);
 965                 return NULL;
 966         }
 967
 968         mutex_init(&smmu_domain->init_mutex);
 969         spin_lock_init(&smmu_domain->cb_lock);
 970
 971         return &smmu_domain->domain;
 972 }
 973
 974 static void arm_smmu_domain_free(struct iommu_domain *domain)
 975 {
 976         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 977
 978         /*
 979          * Free the domain resources. We assume that all devices have
 980          * already been detached.
 981          */
 982         iommu_put_dma_cookie(domain);
 983         arm_smmu_destroy_domain_context(domain);
 984         kfree(smmu_domain);
 985 }
 986
 987 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
 988 {
 989         struct arm_smmu_smr *smr = smmu->smrs + idx;
 990         u32 reg = FIELD_PREP(SMR_ID, smr->id) | FIELD_PREP(SMR_MASK, smr->mask);
 991
 992         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
 993                 reg |= SMR_VALID;
 994         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
 995 }
 996
 997 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
 998 {
 999         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1000         u32 reg = FIELD_PREP(S2CR_TYPE, s2cr->type) |
1001                   FIELD_PREP(S2CR_CBNDX, s2cr->cbndx) |
1002                   FIELD_PREP(S2CR_PRIVCFG, s2cr->privcfg);
1003
1004         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1005             smmu->smrs[idx].valid)
1006                 reg |= S2CR_EXIDVALID;
1007         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
1008 }
1009
1010 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1011 {
1012         arm_smmu_write_s2cr(smmu, idx);
1013         if (smmu->smrs)
1014                 arm_smmu_write_smr(smmu, idx);
1015 }
1016
1017 /*
1018  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1019  * should be called after sCR0 is written.
1020  */
1021 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1022 {
1023         u32 smr;
1024
1025         if (!smmu->smrs)
1026                 return;
1027
1028         /*
1029          * SMR.ID bits may not be preserved if the corresponding MASK
1030          * bits are set, so check each one separately. We can reject
1031          * masters later if they try to claim IDs outside these masks.
1032          */
1033         smr = FIELD_PREP(SMR_ID, smmu->streamid_mask);
1034         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);
1035         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));
1036         smmu->streamid_mask = FIELD_GET(SMR_ID, smr);
1037
1038         smr = FIELD_PREP(SMR_MASK, smmu->streamid_mask);
1039         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);
1040         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));
1041         smmu->smr_mask_mask = FIELD_GET(SMR_MASK, smr);
1042 }
1043
1044 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1045 {
1046         struct arm_smmu_smr *smrs = smmu->smrs;
1047         int i, free_idx = -ENOSPC;
1048
1049         /* Stream indexing is blissfully easy */
1050         if (!smrs)
1051                 return id;
1052
1053         /* Validating SMRs is... less so */
1054         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1055                 if (!smrs[i].valid) {
1056                         /*
1057                          * Note the first free entry we come across, which
1058                          * we'll claim in the end if nothing else matches.
1059                          */
1060                         if (free_idx < 0)
1061                                 free_idx = i;
1062                         continue;
1063                 }
1064                 /*
1065                  * If the new entry is _entirely_ matched by an existing entry,
1066                  * then reuse that, with the guarantee that there also cannot
1067                  * be any subsequent conflicting entries. In normal use we'd
1068                  * expect simply identical entries for this case, but there's
1069                  * no harm in accommodating the generalisation.
1070                  */
1071                 if ((mask & smrs[i].mask) == mask &&
1072                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1073                         return i;
1074                 /*
1075                  * If the new entry has any other overlap with an existing one,
1076                  * though, then there always exists at least one stream ID
1077                  * which would cause a conflict, and we can't allow that risk.
1078                  */
1079                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1080                         return -EINVAL;
1081         }
1082
1083         return free_idx;
1084 }
1085
1086 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1087 {
1088         if (--smmu->s2crs[idx].count)
1089                 return false;
1090
1091         smmu->s2crs[idx] = s2cr_init_val;
1092         if (smmu->smrs)
1093                 smmu->smrs[idx].valid = false;
1094
1095         return true;
1096 }
1097
1098 static int arm_smmu_master_alloc_smes(struct device *dev)
1099 {
1100         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1101         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1102         struct arm_smmu_device *smmu = cfg->smmu;
1103         struct arm_smmu_smr *smrs = smmu->smrs;
1104         struct iommu_group *group;
1105         int i, idx, ret;
1106
1107         mutex_lock(&smmu->stream_map_mutex);
1108         /* Figure out a viable stream map entry allocation */
1109         for_each_cfg_sme(fwspec, i, idx) {
1110                 u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1111                 u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
1112
1113                 if (idx != INVALID_SMENDX) {
1114                         ret = -EEXIST;
1115                         goto out_err;
1116                 }
1117
1118                 ret = arm_smmu_find_sme(smmu, sid, mask);
1119                 if (ret < 0)
1120                         goto out_err;
1121
1122                 idx = ret;
1123                 if (smrs && smmu->s2crs[idx].count == 0) {
1124                         smrs[idx].id = sid;
1125                         smrs[idx].mask = mask;
1126                         smrs[idx].valid = true;
1127                 }
1128                 smmu->s2crs[idx].count++;
1129                 cfg->smendx[i] = (s16)idx;
1130         }
1131
1132         group = iommu_group_get_for_dev(dev);
1133         if (!group)
1134                 group = ERR_PTR(-ENOMEM);
1135         if (IS_ERR(group)) {
1136                 ret = PTR_ERR(group);
1137                 goto out_err;
1138         }
1139         iommu_group_put(group);
1140
1141         /* It worked! Now, poke the actual hardware */
1142         for_each_cfg_sme(fwspec, i, idx) {
1143                 arm_smmu_write_sme(smmu, idx);
1144                 smmu->s2crs[idx].group = group;
1145         }
1146
1147         mutex_unlock(&smmu->stream_map_mutex);
1148         return 0;
1149
1150 out_err:
1151         while (i--) {
1152                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1153                 cfg->smendx[i] = INVALID_SMENDX;
1154         }
1155         mutex_unlock(&smmu->stream_map_mutex);
1156         return ret;
1157 }
1158
1159 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1160 {
1161         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1162         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1163         int i, idx;
1164
1165         mutex_lock(&smmu->stream_map_mutex);
1166         for_each_cfg_sme(fwspec, i, idx) {
1167                 if (arm_smmu_free_sme(smmu, idx))
1168                         arm_smmu_write_sme(smmu, idx);
1169                 cfg->smendx[i] = INVALID_SMENDX;
1170         }
1171         mutex_unlock(&smmu->stream_map_mutex);
1172 }
1173
1174 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1175                                       struct iommu_fwspec *fwspec)
1176 {
1177         struct arm_smmu_device *smmu = smmu_domain->smmu;
1178         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1179         u8 cbndx = smmu_domain->cfg.cbndx;
1180         enum arm_smmu_s2cr_type type;
1181         int i, idx;
1182
1183         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1184                 type = S2CR_TYPE_BYPASS;
1185         else
1186                 type = S2CR_TYPE_TRANS;
1187
1188         for_each_cfg_sme(fwspec, i, idx) {
1189                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1190                         continue;
1191
1192                 s2cr[idx].type = type;
1193                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1194                 s2cr[idx].cbndx = cbndx;
1195                 arm_smmu_write_s2cr(smmu, idx);
1196         }
1197         return 0;
1198 }
1199
1200 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1201 {
1202         int ret;
1203         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1204         struct arm_smmu_device *smmu;
1205         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1206
1207         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1208                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1209                 return -ENXIO;
1210         }
1211
1212         /*
1213          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1214          * domains between of_xlate() and add_device() - we have no way to cope
1215          * with that, so until ARM gets converted to rely on groups and default
1216          * domains, just say no (but more politely than by dereferencing NULL).
1217          * This should be at least a WARN_ON once that's sorted.
1218          */
1219         if (!fwspec->iommu_priv)
1220                 return -ENODEV;
1221
1222         smmu = fwspec_smmu(fwspec);
1223
1224         ret = arm_smmu_rpm_get(smmu);
1225         if (ret < 0)
1226                 return ret;
1227
1228         /* Ensure that the domain is finalised */
1229         ret = arm_smmu_init_domain_context(domain, smmu);
1230         if (ret < 0)
1231                 goto rpm_put;
1232
1233         /*
1234          * Sanity check the domain. We don't support domains across
1235          * different SMMUs.
1236          */
1237         if (smmu_domain->smmu != smmu) {
1238                 dev_err(dev,
1239                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1240                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1241                 ret = -EINVAL;
1242                 goto rpm_put;
1243         }
1244
1245         /* Looks ok, so add the device to the domain */
1246         ret = arm_smmu_domain_add_master(smmu_domain, fwspec);
1247
1248 rpm_put:
1249         arm_smmu_rpm_put(smmu);
1250         return ret;
1251 }
1252
1253 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1254                         phys_addr_t paddr, size_t size, int prot)
1255 {
1256         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1257         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1258         int ret;
1259
1260         if (!ops)
1261                 return -ENODEV;
1262
1263         arm_smmu_rpm_get(smmu);
1264         ret = ops->map(ops, iova, paddr, size, prot);
1265         arm_smmu_rpm_put(smmu);
1266
1267         return ret;
1268 }
1269
1270 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1271                              size_t size)
1272 {
1273         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1274         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1275         size_t ret;
1276
1277         if (!ops)
1278                 return 0;
1279
1280         arm_smmu_rpm_get(smmu);
1281         ret = ops->unmap(ops, iova, size);
1282         arm_smmu_rpm_put(smmu);
1283
1284         return ret;
1285 }
1286
1287 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1288 {
1289         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1290         struct arm_smmu_device *smmu = smmu_domain->smmu;
1291
1292         if (smmu_domain->tlb_ops) {
1293                 arm_smmu_rpm_get(smmu);
1294                 smmu_domain->tlb_ops->tlb_flush_all(smmu_domain);
1295                 arm_smmu_rpm_put(smmu);
1296         }
1297 }
1298
1299 static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
1300 {
1301         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1302         struct arm_smmu_device *smmu = smmu_domain->smmu;
1303
1304         if (smmu_domain->tlb_ops) {
1305                 arm_smmu_rpm_get(smmu);
1306                 smmu_domain->tlb_ops->tlb_sync(smmu_domain);
1307                 arm_smmu_rpm_put(smmu);
1308         }
1309 }
1310
1311 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1312                                               dma_addr_t iova)
1313 {
1314         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1315         struct arm_smmu_device *smmu = smmu_domain->smmu;
1316         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1317         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1318         struct device *dev = smmu->dev;
1319         void __iomem *reg;
1320         u32 tmp;
1321         u64 phys;
1322         unsigned long va, flags;
1323         int ret, idx = cfg->cbndx;
1324
1325         ret = arm_smmu_rpm_get(smmu);
1326         if (ret < 0)
1327                 return 0;
1328
1329         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1330         va = iova & ~0xfffUL;
1331         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1332                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1333         else
1334                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1335
1336         reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1337         if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) {
1338                 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1339                 dev_err(dev,
1340                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1341                         &iova);
1342                 return ops->iova_to_phys(ops, iova);
1343         }
1344
1345         phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1346         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1347         if (phys & CB_PAR_F) {
1348                 dev_err(dev, "translation fault!\n");
1349                 dev_err(dev, "PAR = 0x%llx\n", phys);
1350                 return 0;
1351         }
1352
1353         arm_smmu_rpm_put(smmu);
1354
1355         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1356 }
1357
1358 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1359                                         dma_addr_t iova)
1360 {
1361         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1362         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1363
1364         if (domain->type == IOMMU_DOMAIN_IDENTITY)
1365                 return iova;
1366
1367         if (!ops)
1368                 return 0;
1369
1370         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1371                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1372                 return arm_smmu_iova_to_phys_hard(domain, iova);
1373
1374         return ops->iova_to_phys(ops, iova);
1375 }
1376
1377 static bool arm_smmu_capable(enum iommu_cap cap)
1378 {
1379         switch (cap) {
1380         case IOMMU_CAP_CACHE_COHERENCY:
1381                 /*
1382                  * Return true here as the SMMU can always send out coherent
1383                  * requests.
1384                  */
1385                 return true;
1386         case IOMMU_CAP_NOEXEC:
1387                 return true;
1388         default:
1389                 return false;
1390         }
1391 }
1392
1393 static int arm_smmu_match_node(struct device *dev, const void *data)
1394 {
1395         return dev->fwnode == data;
1396 }
1397
1398 static
1399 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1400 {
1401         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1402                                                 fwnode, arm_smmu_match_node);
1403         put_device(dev);
1404         return dev ? dev_get_drvdata(dev) : NULL;
1405 }
1406
1407 static int arm_smmu_add_device(struct device *dev)
1408 {
1409         struct arm_smmu_device *smmu;
1410         struct arm_smmu_master_cfg *cfg;
1411         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1412         int i, ret;
1413
1414         if (using_legacy_binding) {
1415                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1416
1417                 /*
1418                  * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1419                  * will allocate/initialise a new one. Thus we need to update fwspec for
1420                  * later use.
1421                  */
1422                 fwspec = dev_iommu_fwspec_get(dev);
1423                 if (ret)
1424                         goto out_free;
1425         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1426                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1427         } else {
1428                 return -ENODEV;
1429         }
1430
1431         ret = -EINVAL;
1432         for (i = 0; i < fwspec->num_ids; i++) {
1433                 u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1434                 u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
1435
1436                 if (sid & ~smmu->streamid_mask) {
1437                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1438                                 sid, smmu->streamid_mask);
1439                         goto out_free;
1440                 }
1441                 if (mask & ~smmu->smr_mask_mask) {
1442                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1443                                 mask, smmu->smr_mask_mask);
1444                         goto out_free;
1445                 }
1446         }
1447
1448         ret = -ENOMEM;
1449         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1450                       GFP_KERNEL);
1451         if (!cfg)
1452                 goto out_free;
1453
1454         cfg->smmu = smmu;
1455         fwspec->iommu_priv = cfg;
1456         while (i--)
1457                 cfg->smendx[i] = INVALID_SMENDX;
1458
1459         ret = arm_smmu_rpm_get(smmu);
1460         if (ret < 0)
1461                 goto out_cfg_free;
1462
1463         ret = arm_smmu_master_alloc_smes(dev);
1464         arm_smmu_rpm_put(smmu);
1465
1466         if (ret)
1467                 goto out_cfg_free;
1468
1469         iommu_device_link(&smmu->iommu, dev);
1470
1471         device_link_add(dev, smmu->dev,
1472                         DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1473
1474         return 0;
1475
1476 out_cfg_free:
1477         kfree(cfg);
1478 out_free:
1479         iommu_fwspec_free(dev);
1480         return ret;
1481 }
1482
1483 static void arm_smmu_remove_device(struct device *dev)
1484 {
1485         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1486         struct arm_smmu_master_cfg *cfg;
1487         struct arm_smmu_device *smmu;
1488         int ret;
1489
1490         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1491                 return;
1492
1493         cfg  = fwspec->iommu_priv;
1494         smmu = cfg->smmu;
1495
1496         ret = arm_smmu_rpm_get(smmu);
1497         if (ret < 0)
1498                 return;
1499
1500         iommu_device_unlink(&smmu->iommu, dev);
1501         arm_smmu_master_free_smes(fwspec);
1502
1503         arm_smmu_rpm_put(smmu);
1504
1505         iommu_group_remove_device(dev);
1506         kfree(fwspec->iommu_priv);
1507         iommu_fwspec_free(dev);
1508 }
1509
1510 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1511 {
1512         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1513         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1514         struct iommu_group *group = NULL;
1515         int i, idx;
1516
1517         for_each_cfg_sme(fwspec, i, idx) {
1518                 if (group && smmu->s2crs[idx].group &&
1519                     group != smmu->s2crs[idx].group)
1520                         return ERR_PTR(-EINVAL);
1521
1522                 group = smmu->s2crs[idx].group;
1523         }
1524
1525         if (group)
1526                 return iommu_group_ref_get(group);
1527
1528         if (dev_is_pci(dev))
1529                 group = pci_device_group(dev);
1530         else if (dev_is_fsl_mc(dev))
1531                 group = fsl_mc_device_group(dev);
1532         else
1533                 group = generic_device_group(dev);
1534
1535         return group;
1536 }
1537
1538 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1539                                     enum iommu_attr attr, void *data)
1540 {
1541         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1542
1543         switch(domain->type) {
1544         case IOMMU_DOMAIN_UNMANAGED:
1545                 switch (attr) {
1546                 case DOMAIN_ATTR_NESTING:
1547                         *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1548                         return 0;
1549                 default:
1550                         return -ENODEV;
1551                 }
1552                 break;
1553         case IOMMU_DOMAIN_DMA:
1554                 switch (attr) {
1555                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1556                         *(int *)data = smmu_domain->non_strict;
1557                         return 0;
1558                 default:
1559                         return -ENODEV;
1560                 }
1561                 break;
1562         default:
1563                 return -EINVAL;
1564         }
1565 }
1566
1567 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1568                                     enum iommu_attr attr, void *data)
1569 {
1570         int ret = 0;
1571         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1572
1573         mutex_lock(&smmu_domain->init_mutex);
1574
1575         switch(domain->type) {
1576         case IOMMU_DOMAIN_UNMANAGED:
1577                 switch (attr) {
1578                 case DOMAIN_ATTR_NESTING:
1579                         if (smmu_domain->smmu) {
1580                                 ret = -EPERM;
1581                                 goto out_unlock;
1582                         }
1583
1584                         if (*(int *)data)
1585                                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1586                         else
1587                                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1588                         break;
1589                 default:
1590                         ret = -ENODEV;
1591                 }
1592                 break;
1593         case IOMMU_DOMAIN_DMA:
1594                 switch (attr) {
1595                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1596                         smmu_domain->non_strict = *(int *)data;
1597                         break;
1598                 default:
1599                         ret = -ENODEV;
1600                 }
1601                 break;
1602         default:
1603                 ret = -EINVAL;
1604         }
1605 out_unlock:
1606         mutex_unlock(&smmu_domain->init_mutex);
1607         return ret;
1608 }
1609
1610 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1611 {
1612         u32 mask, fwid = 0;
1613
1614         if (args->args_count > 0)
1615                 fwid |= FIELD_PREP(SMR_ID, args->args[0]);
1616
1617         if (args->args_count > 1)
1618                 fwid |= FIELD_PREP(SMR_MASK, args->args[1]);
1619         else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1620                 fwid |= FIELD_PREP(SMR_MASK, mask);
1621
1622         return iommu_fwspec_add_ids(dev, &fwid, 1);
1623 }
1624
1625 static void arm_smmu_get_resv_regions(struct device *dev,
1626                                       struct list_head *head)
1627 {
1628         struct iommu_resv_region *region;
1629         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1630
1631         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1632                                          prot, IOMMU_RESV_SW_MSI);
1633         if (!region)
1634                 return;
1635
1636         list_add_tail(&region->list, head);
1637
1638         iommu_dma_get_resv_regions(dev, head);
1639 }
1640
1641 static void arm_smmu_put_resv_regions(struct device *dev,
1642                                       struct list_head *head)
1643 {
1644         struct iommu_resv_region *entry, *next;
1645
1646         list_for_each_entry_safe(entry, next, head, list)
1647                 kfree(entry);
1648 }
1649
1650 static struct iommu_ops arm_smmu_ops = {
1651         .capable                = arm_smmu_capable,
1652         .domain_alloc           = arm_smmu_domain_alloc,
1653         .domain_free            = arm_smmu_domain_free,
1654         .attach_dev             = arm_smmu_attach_dev,
1655         .map                    = arm_smmu_map,
1656         .unmap                  = arm_smmu_unmap,
1657         .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
1658         .iotlb_sync             = arm_smmu_iotlb_sync,
1659         .iova_to_phys           = arm_smmu_iova_to_phys,
1660         .add_device             = arm_smmu_add_device,
1661         .remove_device          = arm_smmu_remove_device,
1662         .device_group           = arm_smmu_device_group,
1663         .domain_get_attr        = arm_smmu_domain_get_attr,
1664         .domain_set_attr        = arm_smmu_domain_set_attr,
1665         .of_xlate               = arm_smmu_of_xlate,
1666         .get_resv_regions       = arm_smmu_get_resv_regions,
1667         .put_resv_regions       = arm_smmu_put_resv_regions,
1668         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1669 };
1670
1671 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1672 {
1673         int i;
1674         u32 reg, major;
1675
1676         /* clear global FSR */
1677         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1678         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1679
1680         /*
1681          * Reset stream mapping groups: Initial values mark all SMRn as
1682          * invalid and all S2CRn as bypass unless overridden.
1683          */
1684         for (i = 0; i < smmu->num_mapping_groups; ++i)
1685                 arm_smmu_write_sme(smmu, i);
1686
1687         if (smmu->model == ARM_MMU500) {
1688                 /*
1689                  * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1690                  * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1691                  * bit is only present in MMU-500r2 onwards.
1692                  */
1693                 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID7);
1694                 major = FIELD_GET(ID7_MAJOR, reg);
1695                 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR);
1696                 if (major >= 2)
1697                         reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1698                 /*
1699                  * Allow unmatched Stream IDs to allocate bypass
1700                  * TLB entries for reduced latency.
1701                  */
1702                 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN;
1703                 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, reg);
1704         }
1705
1706         /* Make sure all context banks are disabled and clear CB_FSR  */
1707         for (i = 0; i < smmu->num_context_banks; ++i) {
1708                 arm_smmu_write_context_bank(smmu, i);
1709                 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, FSR_FAULT);
1710                 /*
1711                  * Disable MMU-500's not-particularly-beneficial next-page
1712                  * prefetcher for the sake of errata #841119 and #826419.
1713                  */
1714                 if (smmu->model == ARM_MMU500) {
1715                         reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
1716                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1717                         arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_ACTLR, reg);
1718                 }
1719         }
1720
1721         /* Invalidate the TLB, just in case */
1722         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1723         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1724
1725         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1726
1727         /* Enable fault reporting */
1728         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1729
1730         /* Disable TLB broadcasting. */
1731         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1732
1733         /* Enable client access, handling unmatched streams as appropriate */
1734         reg &= ~sCR0_CLIENTPD;
1735         if (disable_bypass)
1736                 reg |= sCR0_USFCFG;
1737         else
1738                 reg &= ~sCR0_USFCFG;
1739
1740         /* Disable forced broadcasting */
1741         reg &= ~sCR0_FB;
1742
1743         /* Don't upgrade barriers */
1744         reg &= ~(sCR0_BSU);
1745
1746         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1747                 reg |= sCR0_VMID16EN;
1748
1749         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1750                 reg |= sCR0_EXIDENABLE;
1751
1752         /* Push the button */
1753         arm_smmu_tlb_sync_global(smmu);
1754         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1755 }
1756
1757 static int arm_smmu_id_size_to_bits(int size)
1758 {
1759         switch (size) {
1760         case 0:
1761                 return 32;
1762         case 1:
1763                 return 36;
1764         case 2:
1765                 return 40;
1766         case 3:
1767                 return 42;
1768         case 4:
1769                 return 44;
1770         case 5:
1771         default:
1772                 return 48;
1773         }
1774 }
1775
1776 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1777 {
1778         unsigned int size;
1779         u32 id;
1780         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1781         int i;
1782
1783         dev_notice(smmu->dev, "probing hardware configuration...\n");
1784         dev_notice(smmu->dev, "SMMUv%d with:\n",
1785                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1786
1787         /* ID0 */
1788         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1789
1790         /* Restrict available stages based on module parameter */
1791         if (force_stage == 1)
1792                 id &= ~(ID0_S2TS | ID0_NTS);
1793         else if (force_stage == 2)
1794                 id &= ~(ID0_S1TS | ID0_NTS);
1795
1796         if (id & ID0_S1TS) {
1797                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1798                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1799         }
1800
1801         if (id & ID0_S2TS) {
1802                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1803                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1804         }
1805
1806         if (id & ID0_NTS) {
1807                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1808                 dev_notice(smmu->dev, "\tnested translation\n");
1809         }
1810
1811         if (!(smmu->features &
1812                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1813                 dev_err(smmu->dev, "\tno translation support!\n");
1814                 return -ENODEV;
1815         }
1816
1817         if ((id & ID0_S1TS) &&
1818                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1819                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1820                 dev_notice(smmu->dev, "\taddress translation ops\n");
1821         }
1822
1823         /*
1824          * In order for DMA API calls to work properly, we must defer to what
1825          * the FW says about coherency, regardless of what the hardware claims.
1826          * Fortunately, this also opens up a workaround for systems where the
1827          * ID register value has ended up configured incorrectly.
1828          */
1829         cttw_reg = !!(id & ID0_CTTW);
1830         if (cttw_fw || cttw_reg)
1831                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1832                            cttw_fw ? "" : "non-");
1833         if (cttw_fw != cttw_reg)
1834                 dev_notice(smmu->dev,
1835                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1836
1837         /* Max. number of entries we have for stream matching/indexing */
1838         if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1839                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1840                 size = 1 << 16;
1841         } else {
1842                 size = 1 << FIELD_GET(ID0_NUMSIDB, id);
1843         }
1844         smmu->streamid_mask = size - 1;
1845         if (id & ID0_SMS) {
1846                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1847                 size = FIELD_GET(ID0_NUMSMRG, id);
1848                 if (size == 0) {
1849                         dev_err(smmu->dev,
1850                                 "stream-matching supported, but no SMRs present!\n");
1851                         return -ENODEV;
1852                 }
1853
1854                 /* Zero-initialised to mark as invalid */
1855                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1856                                           GFP_KERNEL);
1857                 if (!smmu->smrs)
1858                         return -ENOMEM;
1859
1860                 dev_notice(smmu->dev,
1861                            "\tstream matching with %u register groups", size);
1862         }
1863         /* s2cr->type == 0 means translation, so initialise explicitly */
1864         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1865                                          GFP_KERNEL);
1866         if (!smmu->s2crs)
1867                 return -ENOMEM;
1868         for (i = 0; i < size; i++)
1869                 smmu->s2crs[i] = s2cr_init_val;
1870
1871         smmu->num_mapping_groups = size;
1872         mutex_init(&smmu->stream_map_mutex);
1873         spin_lock_init(&smmu->global_sync_lock);
1874
1875         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1876                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1877                 if (!(id & ID0_PTFS_NO_AARCH32S))
1878                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1879         }
1880
1881         /* ID1 */
1882         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1883         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1884
1885         /* Check for size mismatch of SMMU address space from mapped region */
1886         size = 1 << (FIELD_GET(ID1_NUMPAGENDXB, id) + 1);
1887         if (smmu->numpage != 2 * size << smmu->pgshift)
1888                 dev_warn(smmu->dev,
1889                         "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1890                         2 * size << smmu->pgshift, smmu->numpage);
1891         /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1892         smmu->numpage = size;
1893
1894         smmu->num_s2_context_banks = FIELD_GET(ID1_NUMS2CB, id);
1895         smmu->num_context_banks = FIELD_GET(ID1_NUMCB, id);
1896         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1897                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1898                 return -ENODEV;
1899         }
1900         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1901                    smmu->num_context_banks, smmu->num_s2_context_banks);
1902         /*
1903          * Cavium CN88xx erratum #27704.
1904          * Ensure ASID and VMID allocation is unique across all SMMUs in
1905          * the system.
1906          */
1907         if (smmu->model == CAVIUM_SMMUV2) {
1908                 smmu->cavium_id_base =
1909                         atomic_add_return(smmu->num_context_banks,
1910                                           &cavium_smmu_context_count);
1911                 smmu->cavium_id_base -= smmu->num_context_banks;
1912                 dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
1913         }
1914         smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1915                                  sizeof(*smmu->cbs), GFP_KERNEL);
1916         if (!smmu->cbs)
1917                 return -ENOMEM;
1918
1919         /* ID2 */
1920         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1921         size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_IAS, id));
1922         smmu->ipa_size = size;
1923
1924         /* The output mask is also applied for bypass */
1925         size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_OAS, id));
1926         smmu->pa_size = size;
1927
1928         if (id & ID2_VMID16)
1929                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1930
1931         /*
1932          * What the page table walker can address actually depends on which
1933          * descriptor format is in use, but since a) we don't know that yet,
1934          * and b) it can vary per context bank, this will have to do...
1935          */
1936         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1937                 dev_warn(smmu->dev,
1938                          "failed to set DMA mask for table walker\n");
1939
1940         if (smmu->version < ARM_SMMU_V2) {
1941                 smmu->va_size = smmu->ipa_size;
1942                 if (smmu->version == ARM_SMMU_V1_64K)
1943                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1944         } else {
1945                 size = FIELD_GET(ID2_UBS, id);
1946                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1947                 if (id & ID2_PTFS_4K)
1948                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1949                 if (id & ID2_PTFS_16K)
1950                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1951                 if (id & ID2_PTFS_64K)
1952                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1953         }
1954
1955         /* Now we've corralled the various formats, what'll it do? */
1956         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1957                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1958         if (smmu->features &
1959             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1960                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1961         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1962                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1963         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1964                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1965
1966         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1967                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1968         else
1969                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1970         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1971                    smmu->pgsize_bitmap);
1972
1973
1974         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1975                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1976                            smmu->va_size, smmu->ipa_size);
1977
1978         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1979                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1980                            smmu->ipa_size, smmu->pa_size);
1981
1982         return 0;
1983 }
1984
1985 struct arm_smmu_match_data {
1986         enum arm_smmu_arch_version version;
1987         enum arm_smmu_implementation model;
1988 };
1989
1990 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1991 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1992
1993 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1994 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1995 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1996 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1997 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1998 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1999
2000 static const struct of_device_id arm_smmu_of_match[] = {
2001         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
2002         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
2003         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
2004         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
2005         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
2006         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
2007         { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
2008         { },
2009 };
2010
2011 #ifdef CONFIG_ACPI
2012 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
2013 {
2014         int ret = 0;
2015
2016         switch (model) {
2017         case ACPI_IORT_SMMU_V1:
2018         case ACPI_IORT_SMMU_CORELINK_MMU400:
2019                 smmu->version = ARM_SMMU_V1;
2020                 smmu->model = GENERIC_SMMU;
2021                 break;
2022         case ACPI_IORT_SMMU_CORELINK_MMU401:
2023                 smmu->version = ARM_SMMU_V1_64K;
2024                 smmu->model = GENERIC_SMMU;
2025                 break;
2026         case ACPI_IORT_SMMU_V2:
2027                 smmu->version = ARM_SMMU_V2;
2028                 smmu->model = GENERIC_SMMU;
2029                 break;
2030         case ACPI_IORT_SMMU_CORELINK_MMU500:
2031                 smmu->version = ARM_SMMU_V2;
2032                 smmu->model = ARM_MMU500;
2033                 break;
2034         case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
2035                 smmu->version = ARM_SMMU_V2;
2036                 smmu->model = CAVIUM_SMMUV2;
2037                 break;
2038         default:
2039                 ret = -ENODEV;
2040         }
2041
2042         return ret;
2043 }
2044
2045 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2046                                       struct arm_smmu_device *smmu)
2047 {
2048         struct device *dev = smmu->dev;
2049         struct acpi_iort_node *node =
2050                 *(struct acpi_iort_node **)dev_get_platdata(dev);
2051         struct acpi_iort_smmu *iort_smmu;
2052         int ret;
2053
2054         /* Retrieve SMMU1/2 specific data */
2055         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2056
2057         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2058         if (ret < 0)
2059                 return ret;
2060
2061         /* Ignore the configuration access interrupt */
2062         smmu->num_global_irqs = 1;
2063
2064         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2065                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2066
2067         return 0;
2068 }
2069 #else
2070 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2071                                              struct arm_smmu_device *smmu)
2072 {
2073         return -ENODEV;
2074 }
2075 #endif
2076
2077 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2078                                     struct arm_smmu_device *smmu)
2079 {
2080         const struct arm_smmu_match_data *data;
2081         struct device *dev = &pdev->dev;
2082         bool legacy_binding;
2083
2084         if (of_property_read_u32(dev->of_node, "#global-interrupts",
2085                                  &smmu->num_global_irqs)) {
2086                 dev_err(dev, "missing #global-interrupts property\n");
2087                 return -ENODEV;
2088         }
2089
2090         data = of_device_get_match_data(dev);
2091         smmu->version = data->version;
2092         smmu->model = data->model;
2093
2094         parse_driver_options(smmu);
2095
2096         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2097         if (legacy_binding && !using_generic_binding) {
2098                 if (!using_legacy_binding)
2099                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2100                 using_legacy_binding = true;
2101         } else if (!legacy_binding && !using_legacy_binding) {
2102                 using_generic_binding = true;
2103         } else {
2104                 dev_err(dev, "not probing due to mismatched DT properties\n");
2105                 return -ENODEV;
2106         }
2107
2108         if (of_dma_is_coherent(dev->of_node))
2109                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2110
2111         return 0;
2112 }
2113
2114 static void arm_smmu_bus_init(void)
2115 {
2116         /* Oh, for a proper bus abstraction */
2117         if (!iommu_present(&platform_bus_type))
2118                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2119 #ifdef CONFIG_ARM_AMBA
2120         if (!iommu_present(&amba_bustype))
2121                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2122 #endif
2123 #ifdef CONFIG_PCI
2124         if (!iommu_present(&pci_bus_type)) {
2125                 pci_request_acs();
2126                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2127         }
2128 #endif
2129 #ifdef CONFIG_FSL_MC_BUS
2130         if (!iommu_present(&fsl_mc_bus_type))
2131                 bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
2132 #endif
2133 }
2134
2135 static int arm_smmu_device_probe(struct platform_device *pdev)
2136 {
2137         struct resource *res;
2138         resource_size_t ioaddr;
2139         struct arm_smmu_device *smmu;
2140         struct device *dev = &pdev->dev;
2141         int num_irqs, i, err;
2142
2143         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2144         if (!smmu) {
2145                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2146                 return -ENOMEM;
2147         }
2148         smmu->dev = dev;
2149
2150         if (dev->of_node)
2151                 err = arm_smmu_device_dt_probe(pdev, smmu);
2152         else
2153                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2154
2155         if (err)
2156                 return err;
2157
2158         smmu = arm_smmu_impl_init(smmu);
2159         if (IS_ERR(smmu))
2160                 return PTR_ERR(smmu);
2161
2162         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2163         ioaddr = res->start;
2164         smmu->base = devm_ioremap_resource(dev, res);
2165         if (IS_ERR(smmu->base))
2166                 return PTR_ERR(smmu->base);
2167         /*
2168          * The resource size should effectively match the value of SMMU_TOP;
2169          * stash that temporarily until we know PAGESIZE to validate it with.
2170          */
2171         smmu->numpage = resource_size(res);
2172
2173         num_irqs = 0;
2174         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2175                 num_irqs++;
2176                 if (num_irqs > smmu->num_global_irqs)
2177                         smmu->num_context_irqs++;
2178         }
2179
2180         if (!smmu->num_context_irqs) {
2181                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2182                         num_irqs, smmu->num_global_irqs + 1);
2183                 return -ENODEV;
2184         }
2185
2186         smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2187                                   GFP_KERNEL);
2188         if (!smmu->irqs) {
2189                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2190                 return -ENOMEM;
2191         }
2192
2193         for (i = 0; i < num_irqs; ++i) {
2194                 int irq = platform_get_irq(pdev, i);
2195
2196                 if (irq < 0) {
2197                         dev_err(dev, "failed to get irq index %d\n", i);
2198                         return -ENODEV;
2199                 }
2200                 smmu->irqs[i] = irq;
2201         }
2202
2203         err = devm_clk_bulk_get_all(dev, &smmu->clks);
2204         if (err < 0) {
2205                 dev_err(dev, "failed to get clocks %d\n", err);
2206                 return err;
2207         }
2208         smmu->num_clks = err;
2209
2210         err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2211         if (err)
2212                 return err;
2213
2214         err = arm_smmu_device_cfg_probe(smmu);
2215         if (err)
2216                 return err;
2217
2218         if (smmu->version == ARM_SMMU_V2) {
2219                 if (smmu->num_context_banks > smmu->num_context_irqs) {
2220                         dev_err(dev,
2221                               "found only %d context irq(s) but %d required\n",
2222                               smmu->num_context_irqs, smmu->num_context_banks);
2223                         return -ENODEV;
2224                 }
2225
2226                 /* Ignore superfluous interrupts */
2227                 smmu->num_context_irqs = smmu->num_context_banks;
2228         }
2229
2230         for (i = 0; i < smmu->num_global_irqs; ++i) {
2231                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2232                                        arm_smmu_global_fault,
2233                                        IRQF_SHARED,
2234                                        "arm-smmu global fault",
2235                                        smmu);
2236                 if (err) {
2237                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2238                                 i, smmu->irqs[i]);
2239                         return err;
2240                 }
2241         }
2242
2243         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2244                                      "smmu.%pa", &ioaddr);
2245         if (err) {
2246                 dev_err(dev, "Failed to register iommu in sysfs\n");
2247                 return err;
2248         }
2249
2250         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2251         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2252
2253         err = iommu_device_register(&smmu->iommu);
2254         if (err) {
2255                 dev_err(dev, "Failed to register iommu\n");
2256                 return err;
2257         }
2258
2259         platform_set_drvdata(pdev, smmu);
2260         arm_smmu_device_reset(smmu);
2261         arm_smmu_test_smr_masks(smmu);
2262
2263         /*
2264          * We want to avoid touching dev->power.lock in fastpaths unless
2265          * it's really going to do something useful - pm_runtime_enabled()
2266          * can serve as an ideal proxy for that decision. So, conditionally
2267          * enable pm_runtime.
2268          */
2269         if (dev->pm_domain) {
2270                 pm_runtime_set_active(dev);
2271                 pm_runtime_enable(dev);
2272         }
2273
2274         /*
2275          * For ACPI and generic DT bindings, an SMMU will be probed before
2276          * any device which might need it, so we want the bus ops in place
2277          * ready to handle default domain setup as soon as any SMMU exists.
2278          */
2279         if (!using_legacy_binding)
2280                 arm_smmu_bus_init();
2281
2282         return 0;
2283 }
2284
2285 /*
2286  * With the legacy DT binding in play, though, we have no guarantees about
2287  * probe order, but then we're also not doing default domains, so we can
2288  * delay setting bus ops until we're sure every possible SMMU is ready,
2289  * and that way ensure that no add_device() calls get missed.
2290  */
2291 static int arm_smmu_legacy_bus_init(void)
2292 {
2293         if (using_legacy_binding)
2294                 arm_smmu_bus_init();
2295         return 0;
2296 }
2297 device_initcall_sync(arm_smmu_legacy_bus_init);
2298
2299 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2300 {
2301         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2302
2303         if (!smmu)
2304                 return;
2305
2306         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2307                 dev_err(&pdev->dev, "removing device with active domains!\n");
2308
2309         arm_smmu_rpm_get(smmu);
2310         /* Turn the thing off */
2311         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, sCR0_CLIENTPD);
2312         arm_smmu_rpm_put(smmu);
2313
2314         if (pm_runtime_enabled(smmu->dev))
2315                 pm_runtime_force_suspend(smmu->dev);
2316         else
2317                 clk_bulk_disable(smmu->num_clks, smmu->clks);
2318
2319         clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2320 }
2321
2322 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2323 {
2324         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2325         int ret;
2326
2327         ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2328         if (ret)
2329                 return ret;
2330
2331         arm_smmu_device_reset(smmu);
2332
2333         return 0;
2334 }
2335
2336 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2337 {
2338         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2339
2340         clk_bulk_disable(smmu->num_clks, smmu->clks);
2341
2342         return 0;
2343 }
2344
2345 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2346 {
2347         if (pm_runtime_suspended(dev))
2348                 return 0;
2349
2350         return arm_smmu_runtime_resume(dev);
2351 }
2352
2353 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2354 {
2355         if (pm_runtime_suspended(dev))
2356                 return 0;
2357
2358         return arm_smmu_runtime_suspend(dev);
2359 }
2360
2361 static const struct dev_pm_ops arm_smmu_pm_ops = {
2362         SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2363         SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2364                            arm_smmu_runtime_resume, NULL)
2365 };
2366
2367 static struct platform_driver arm_smmu_driver = {
2368         .driver = {
2369                 .name                   = "arm-smmu",
2370                 .of_match_table         = of_match_ptr(arm_smmu_of_match),
2371                 .pm                     = &arm_smmu_pm_ops,
2372                 .suppress_bind_attrs    = true,
2373         },
2374         .probe  = arm_smmu_device_probe,
2375         .shutdown = arm_smmu_device_shutdown,
2376 };
2377 builtin_platform_driver(arm_smmu_driver);