drivers/iommu/arm-smmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * IOMMU API for ARM architected SMMU implementations.
   4  *
   5  * Copyright (C) 2013 ARM Limited
   6  *
   7  * Author: Will Deacon <will.deacon@arm.com>
   8  *
   9  * This driver currently supports:
  10  *      - SMMUv1 and v2 implementations
  11  *      - Stream-matching and stream-indexing
  12  *      - v7/v8 long-descriptor format
  13  *      - Non-secure access to the SMMU
  14  *      - Context fault reporting
  15  *      - Extended Stream ID (16 bit)
  16  */
  17
  18 #define pr_fmt(fmt) "arm-smmu: " fmt
  19
  20 #include <linux/acpi.h>
  21 #include <linux/acpi_iort.h>
  22 #include <linux/atomic.h>
  23 #include <linux/bitfield.h>
  24 #include <linux/delay.h>
  25 #include <linux/dma-iommu.h>
  26 #include <linux/dma-mapping.h>
  27 #include <linux/err.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/io.h>
  30 #include <linux/io-64-nonatomic-hi-lo.h>
  31 #include <linux/io-pgtable.h>
  32 #include <linux/iommu.h>
  33 #include <linux/iopoll.h>
  34 #include <linux/init.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/of.h>
  37 #include <linux/of_address.h>
  38 #include <linux/of_device.h>
  39 #include <linux/of_iommu.h>
  40 #include <linux/pci.h>
  41 #include <linux/platform_device.h>
  42 #include <linux/pm_runtime.h>
  43 #include <linux/slab.h>
  44 #include <linux/spinlock.h>
  45
  46 #include <linux/amba/bus.h>
  47 #include <linux/fsl/mc.h>
  48
  49 #include "arm-smmu-regs.h"
  50
  51 /*
  52  * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
  53  * global register space are still, in fact, using a hypervisor to mediate it
  54  * by trapping and emulating register accesses. Sadly, some deployed versions
  55  * of said trapping code have bugs wherein they go horribly wrong for stores
  56  * using r31 (i.e. XZR/WZR) as the source register.
  57  */
  58 #define QCOM_DUMMY_VAL -1
  59
  60 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
  61
  62 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
  63 #define ARM_MMU500_ACR_S2CRB_TLBEN      (1 << 10)
  64 #define ARM_MMU500_ACR_SMTNMB_TLBEN     (1 << 8)
  65
  66 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
  67 #define TLB_SPIN_COUNT                  10
  68
  69 /* Maximum number of context banks per SMMU */
  70 #define ARM_SMMU_MAX_CBS                128
  71
  72 /* SMMU global address space */
  73 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  74 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  75
  76 /*
  77  * SMMU global address space with conditional offset to access secure
  78  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  79  * nsGFSYNR0: 0x450)
  80  */
  81 #define ARM_SMMU_GR0_NS(smmu)                                           \
  82         ((smmu)->base +                                                 \
  83                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  84                         ? 0x400 : 0))
  85
  86 /*
  87  * Some 64-bit registers only make sense to write atomically, but in such
  88  * cases all the data relevant to AArch32 formats lies within the lower word,
  89  * therefore this actually makes more sense than it might first appear.
  90  */
  91 #ifdef CONFIG_64BIT
  92 #define smmu_write_atomic_lq            writeq_relaxed
  93 #else
  94 #define smmu_write_atomic_lq            writel_relaxed
  95 #endif
  96
  97 /* Translation context bank */
  98 #define ARM_SMMU_CB(smmu, n)    ((smmu)->base + (((smmu)->numpage + (n)) << (smmu)->pgshift))
  99
 100 #define MSI_IOVA_BASE                   0x8000000
 101 #define MSI_IOVA_LENGTH                 0x100000
 102
 103 static int force_stage;
 104 /*
 105  * not really modular, but the easiest way to keep compat with existing
 106  * bootargs behaviour is to continue using module_param() here.
 107  */
 108 module_param(force_stage, int, S_IRUGO);
 109 MODULE_PARM_DESC(force_stage,
 110         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 111 static bool disable_bypass =
 112         IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
 113 module_param(disable_bypass, bool, S_IRUGO);
 114 MODULE_PARM_DESC(disable_bypass,
 115         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 116
 117 enum arm_smmu_arch_version {
 118         ARM_SMMU_V1,
 119         ARM_SMMU_V1_64K,
 120         ARM_SMMU_V2,
 121 };
 122
 123 enum arm_smmu_implementation {
 124         GENERIC_SMMU,
 125         ARM_MMU500,
 126         CAVIUM_SMMUV2,
 127         QCOM_SMMUV2,
 128 };
 129
 130 struct arm_smmu_s2cr {
 131         struct iommu_group              *group;
 132         int                             count;
 133         enum arm_smmu_s2cr_type         type;
 134         enum arm_smmu_s2cr_privcfg      privcfg;
 135         u8                              cbndx;
 136 };
 137
 138 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 139         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 140 }
 141
 142 struct arm_smmu_smr {
 143         u16                             mask;
 144         u16                             id;
 145         bool                            valid;
 146 };
 147
 148 struct arm_smmu_cb {
 149         u64                             ttbr[2];
 150         u32                             tcr[2];
 151         u32                             mair[2];
 152         struct arm_smmu_cfg             *cfg;
 153 };
 154
 155 struct arm_smmu_master_cfg {
 156         struct arm_smmu_device          *smmu;
 157         s16                             smendx[];
 158 };
 159 #define INVALID_SMENDX                  -1
 160 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 161 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 162 #define fwspec_smendx(fw, i) \
 163         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 164 #define for_each_cfg_sme(fw, i, idx) \
 165         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 166
 167 struct arm_smmu_device {
 168         struct device                   *dev;
 169
 170         void __iomem                    *base;
 171         unsigned int                    numpage;
 172         unsigned int                    pgshift;
 173
 174 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 175 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 176 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 177 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 178 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 179 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 180 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 181 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 182 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 183 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 184 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 185 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 186 #define ARM_SMMU_FEAT_EXIDS             (1 << 12)
 187         u32                             features;
 188
 189 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 190         u32                             options;
 191         enum arm_smmu_arch_version      version;
 192         enum arm_smmu_implementation    model;
 193
 194         u32                             num_context_banks;
 195         u32                             num_s2_context_banks;
 196         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 197         struct arm_smmu_cb              *cbs;
 198         atomic_t                        irptndx;
 199
 200         u32                             num_mapping_groups;
 201         u16                             streamid_mask;
 202         u16                             smr_mask_mask;
 203         struct arm_smmu_smr             *smrs;
 204         struct arm_smmu_s2cr            *s2crs;
 205         struct mutex                    stream_map_mutex;
 206
 207         unsigned long                   va_size;
 208         unsigned long                   ipa_size;
 209         unsigned long                   pa_size;
 210         unsigned long                   pgsize_bitmap;
 211
 212         u32                             num_global_irqs;
 213         u32                             num_context_irqs;
 214         unsigned int                    *irqs;
 215         struct clk_bulk_data            *clks;
 216         int                             num_clks;
 217
 218         u32                             cavium_id_base; /* Specific to Cavium */
 219
 220         spinlock_t                      global_sync_lock;
 221
 222         /* IOMMU core code handle */
 223         struct iommu_device             iommu;
 224 };
 225
 226 enum arm_smmu_context_fmt {
 227         ARM_SMMU_CTX_FMT_NONE,
 228         ARM_SMMU_CTX_FMT_AARCH64,
 229         ARM_SMMU_CTX_FMT_AARCH32_L,
 230         ARM_SMMU_CTX_FMT_AARCH32_S,
 231 };
 232
 233 struct arm_smmu_cfg {
 234         u8                              cbndx;
 235         u8                              irptndx;
 236         union {
 237                 u16                     asid;
 238                 u16                     vmid;
 239         };
 240         enum arm_smmu_cbar_type         cbar;
 241         enum arm_smmu_context_fmt       fmt;
 242 };
 243 #define INVALID_IRPTNDX                 0xff
 244
 245 enum arm_smmu_domain_stage {
 246         ARM_SMMU_DOMAIN_S1 = 0,
 247         ARM_SMMU_DOMAIN_S2,
 248         ARM_SMMU_DOMAIN_NESTED,
 249         ARM_SMMU_DOMAIN_BYPASS,
 250 };
 251
 252 struct arm_smmu_domain {
 253         struct arm_smmu_device          *smmu;
 254         struct io_pgtable_ops           *pgtbl_ops;
 255         const struct iommu_gather_ops   *tlb_ops;
 256         struct arm_smmu_cfg             cfg;
 257         enum arm_smmu_domain_stage      stage;
 258         bool                            non_strict;
 259         struct mutex                    init_mutex; /* Protects smmu pointer */
 260         spinlock_t                      cb_lock; /* Serialises ATS1* ops and TLB syncs */
 261         struct iommu_domain             domain;
 262 };
 263
 264 struct arm_smmu_option_prop {
 265         u32 opt;
 266         const char *prop;
 267 };
 268
 269 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 270
 271 static bool using_legacy_binding, using_generic_binding;
 272
 273 static struct arm_smmu_option_prop arm_smmu_options[] = {
 274         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 275         { 0, NULL},
 276 };
 277
 278 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
 279 {
 280         if (pm_runtime_enabled(smmu->dev))
 281                 return pm_runtime_get_sync(smmu->dev);
 282
 283         return 0;
 284 }
 285
 286 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
 287 {
 288         if (pm_runtime_enabled(smmu->dev))
 289                 pm_runtime_put(smmu->dev);
 290 }
 291
 292 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 293 {
 294         return container_of(dom, struct arm_smmu_domain, domain);
 295 }
 296
 297 static void parse_driver_options(struct arm_smmu_device *smmu)
 298 {
 299         int i = 0;
 300
 301         do {
 302                 if (of_property_read_bool(smmu->dev->of_node,
 303                                                 arm_smmu_options[i].prop)) {
 304                         smmu->options |= arm_smmu_options[i].opt;
 305                         dev_notice(smmu->dev, "option %s\n",
 306                                 arm_smmu_options[i].prop);
 307                 }
 308         } while (arm_smmu_options[++i].opt);
 309 }
 310
 311 static struct device_node *dev_get_dev_node(struct device *dev)
 312 {
 313         if (dev_is_pci(dev)) {
 314                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 315
 316                 while (!pci_is_root_bus(bus))
 317                         bus = bus->parent;
 318                 return of_node_get(bus->bridge->parent->of_node);
 319         }
 320
 321         return of_node_get(dev->of_node);
 322 }
 323
 324 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 325 {
 326         *((__be32 *)data) = cpu_to_be32(alias);
 327         return 0; /* Continue walking */
 328 }
 329
 330 static int __find_legacy_master_phandle(struct device *dev, void *data)
 331 {
 332         struct of_phandle_iterator *it = *(void **)data;
 333         struct device_node *np = it->node;
 334         int err;
 335
 336         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 337                             "#stream-id-cells", 0)
 338                 if (it->node == np) {
 339                         *(void **)data = dev;
 340                         return 1;
 341                 }
 342         it->node = np;
 343         return err == -ENOENT ? 0 : err;
 344 }
 345
 346 static struct platform_driver arm_smmu_driver;
 347 static struct iommu_ops arm_smmu_ops;
 348
 349 static int arm_smmu_register_legacy_master(struct device *dev,
 350                                            struct arm_smmu_device **smmu)
 351 {
 352         struct device *smmu_dev;
 353         struct device_node *np;
 354         struct of_phandle_iterator it;
 355         void *data = &it;
 356         u32 *sids;
 357         __be32 pci_sid;
 358         int err;
 359
 360         np = dev_get_dev_node(dev);
 361         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 362                 of_node_put(np);
 363                 return -ENODEV;
 364         }
 365
 366         it.node = np;
 367         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 368                                      __find_legacy_master_phandle);
 369         smmu_dev = data;
 370         of_node_put(np);
 371         if (err == 0)
 372                 return -ENODEV;
 373         if (err < 0)
 374                 return err;
 375
 376         if (dev_is_pci(dev)) {
 377                 /* "mmu-masters" assumes Stream ID == Requester ID */
 378                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 379                                        &pci_sid);
 380                 it.cur = &pci_sid;
 381                 it.cur_count = 1;
 382         }
 383
 384         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 385                                 &arm_smmu_ops);
 386         if (err)
 387                 return err;
 388
 389         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 390         if (!sids)
 391                 return -ENOMEM;
 392
 393         *smmu = dev_get_drvdata(smmu_dev);
 394         of_phandle_iterator_args(&it, sids, it.cur_count);
 395         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 396         kfree(sids);
 397         return err;
 398 }
 399
 400 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 401 {
 402         int idx;
 403
 404         do {
 405                 idx = find_next_zero_bit(map, end, start);
 406                 if (idx == end)
 407                         return -ENOSPC;
 408         } while (test_and_set_bit(idx, map));
 409
 410         return idx;
 411 }
 412
 413 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 414 {
 415         clear_bit(idx, map);
 416 }
 417
 418 /* Wait for any pending TLB invalidations to complete */
 419 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu,
 420                                 void __iomem *sync, void __iomem *status)
 421 {
 422         unsigned int spin_cnt, delay;
 423
 424         writel_relaxed(QCOM_DUMMY_VAL, sync);
 425         for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 426                 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 427                         if (!(readl_relaxed(status) & sTLBGSTATUS_GSACTIVE))
 428                                 return;
 429                         cpu_relax();
 430                 }
 431                 udelay(delay);
 432         }
 433         dev_err_ratelimited(smmu->dev,
 434                             "TLB sync timed out -- SMMU may be deadlocked\n");
 435 }
 436
 437 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 438 {
 439         void __iomem *base = ARM_SMMU_GR0(smmu);
 440         unsigned long flags;
 441
 442         spin_lock_irqsave(&smmu->global_sync_lock, flags);
 443         __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC,
 444                             base + ARM_SMMU_GR0_sTLBGSTATUS);
 445         spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
 446 }
 447
 448 static void arm_smmu_tlb_sync_context(void *cookie)
 449 {
 450         struct arm_smmu_domain *smmu_domain = cookie;
 451         struct arm_smmu_device *smmu = smmu_domain->smmu;
 452         void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx);
 453         unsigned long flags;
 454
 455         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
 456         __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC,
 457                             base + ARM_SMMU_CB_TLBSTATUS);
 458         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
 459 }
 460
 461 static void arm_smmu_tlb_sync_vmid(void *cookie)
 462 {
 463         struct arm_smmu_domain *smmu_domain = cookie;
 464
 465         arm_smmu_tlb_sync_global(smmu_domain->smmu);
 466 }
 467
 468 static void arm_smmu_tlb_inv_context_s1(void *cookie)
 469 {
 470         struct arm_smmu_domain *smmu_domain = cookie;
 471         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 472         void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
 473
 474         /*
 475          * NOTE: this is not a relaxed write; it needs to guarantee that PTEs
 476          * cleared by the current CPU are visible to the SMMU before the TLBI.
 477          */
 478         writel(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID);
 479         arm_smmu_tlb_sync_context(cookie);
 480 }
 481
 482 static void arm_smmu_tlb_inv_context_s2(void *cookie)
 483 {
 484         struct arm_smmu_domain *smmu_domain = cookie;
 485         struct arm_smmu_device *smmu = smmu_domain->smmu;
 486         void __iomem *base = ARM_SMMU_GR0(smmu);
 487
 488         /* NOTE: see above */
 489         writel(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
 490         arm_smmu_tlb_sync_global(smmu);
 491 }
 492
 493 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
 494                                       size_t granule, bool leaf, void *cookie)
 495 {
 496         struct arm_smmu_domain *smmu_domain = cookie;
 497         struct arm_smmu_device *smmu = smmu_domain->smmu;
 498         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 499         void __iomem *reg = ARM_SMMU_CB(smmu, cfg->cbndx);
 500
 501         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 502                 wmb();
 503
 504         reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 505
 506         if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 507                 iova = (iova >> 12) << 12;
 508                 iova |= cfg->asid;
 509                 do {
 510                         writel_relaxed(iova, reg);
 511                         iova += granule;
 512                 } while (size -= granule);
 513         } else {
 514                 iova >>= 12;
 515                 iova |= (u64)cfg->asid << 48;
 516                 do {
 517                         writeq_relaxed(iova, reg);
 518                         iova += granule >> 12;
 519                 } while (size -= granule);
 520         }
 521 }
 522
 523 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
 524                                       size_t granule, bool leaf, void *cookie)
 525 {
 526         struct arm_smmu_domain *smmu_domain = cookie;
 527         struct arm_smmu_device *smmu = smmu_domain->smmu;
 528         void __iomem *reg = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx);
 529
 530         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 531                 wmb();
 532
 533         reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : ARM_SMMU_CB_S2_TLBIIPAS2;
 534         iova >>= 12;
 535         do {
 536                 smmu_write_atomic_lq(iova, reg);
 537                 iova += granule >> 12;
 538         } while (size -= granule);
 539 }
 540
 541 /*
 542  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 543  * almost negligible, but the benefit of getting the first one in as far ahead
 544  * of the sync as possible is significant, hence we don't just make this a
 545  * no-op and set .tlb_sync to arm_smmu_inv_context_s2() as you might think.
 546  */
 547 static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size,
 548                                          size_t granule, bool leaf, void *cookie)
 549 {
 550         struct arm_smmu_domain *smmu_domain = cookie;
 551         void __iomem *base = ARM_SMMU_GR0(smmu_domain->smmu);
 552
 553         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 554                 wmb();
 555
 556         writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
 557 }
 558
 559 static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = {
 560         .tlb_flush_all  = arm_smmu_tlb_inv_context_s1,
 561         .tlb_add_flush  = arm_smmu_tlb_inv_range_s1,
 562         .tlb_sync       = arm_smmu_tlb_sync_context,
 563 };
 564
 565 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v2 = {
 566         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 567         .tlb_add_flush  = arm_smmu_tlb_inv_range_s2,
 568         .tlb_sync       = arm_smmu_tlb_sync_context,
 569 };
 570
 571 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = {
 572         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 573         .tlb_add_flush  = arm_smmu_tlb_inv_vmid_nosync,
 574         .tlb_sync       = arm_smmu_tlb_sync_vmid,
 575 };
 576
 577 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 578 {
 579         u32 fsr, fsynr, cbfrsynra;
 580         unsigned long iova;
 581         struct iommu_domain *domain = dev;
 582         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 583         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 584         struct arm_smmu_device *smmu = smmu_domain->smmu;
 585         void __iomem *gr1_base = ARM_SMMU_GR1(smmu);
 586         void __iomem *cb_base;
 587
 588         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
 589         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 590
 591         if (!(fsr & FSR_FAULT))
 592                 return IRQ_NONE;
 593
 594         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 595         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 596         cbfrsynra = readl_relaxed(gr1_base + ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx));
 597
 598         dev_err_ratelimited(smmu->dev,
 599         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
 600                             fsr, iova, fsynr, cbfrsynra, cfg->cbndx);
 601
 602         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 603         return IRQ_HANDLED;
 604 }
 605
 606 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 607 {
 608         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 609         struct arm_smmu_device *smmu = dev;
 610         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 611
 612         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 613         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 614         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 615         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 616
 617         if (!gfsr)
 618                 return IRQ_NONE;
 619
 620         dev_err_ratelimited(smmu->dev,
 621                 "Unexpected global fault, this could be serious\n");
 622         dev_err_ratelimited(smmu->dev,
 623                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 624                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 625
 626         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 627         return IRQ_HANDLED;
 628 }
 629
 630 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 631                                        struct io_pgtable_cfg *pgtbl_cfg)
 632 {
 633         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 634         struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
 635         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 636
 637         cb->cfg = cfg;
 638
 639         /* TCR */
 640         if (stage1) {
 641                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 642                         cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
 643                 } else {
 644                         cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 645                         cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 646                         cb->tcr[1] |= FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM);
 647                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 648                                 cb->tcr[1] |= TCR2_AS;
 649                 }
 650         } else {
 651                 cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 652         }
 653
 654         /* TTBRs */
 655         if (stage1) {
 656                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 657                         cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 658                         cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 659                 } else {
 660                         cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 661                         cb->ttbr[0] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
 662                         cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 663                         cb->ttbr[1] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
 664                 }
 665         } else {
 666                 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 667         }
 668
 669         /* MAIRs (stage-1 only) */
 670         if (stage1) {
 671                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 672                         cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
 673                         cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
 674                 } else {
 675                         cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 676                         cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 677                 }
 678         }
 679 }
 680
 681 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
 682 {
 683         u32 reg;
 684         bool stage1;
 685         struct arm_smmu_cb *cb = &smmu->cbs[idx];
 686         struct arm_smmu_cfg *cfg = cb->cfg;
 687         void __iomem *cb_base, *gr1_base;
 688
 689         cb_base = ARM_SMMU_CB(smmu, idx);
 690
 691         /* Unassigned context banks only need disabling */
 692         if (!cfg) {
 693                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
 694                 return;
 695         }
 696
 697         gr1_base = ARM_SMMU_GR1(smmu);
 698         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 699
 700         /* CBA2R */
 701         if (smmu->version > ARM_SMMU_V1) {
 702                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 703                         reg = CBA2R_VA64;
 704                 else
 705                         reg = 0;
 706                 /* 16-bit VMIDs live in CBA2R */
 707                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 708                         reg |= FIELD_PREP(CBA2R_VMID16, cfg->vmid);
 709
 710                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(idx));
 711         }
 712
 713         /* CBAR */
 714         reg = FIELD_PREP(CBAR_TYPE, cfg->cbar);
 715         if (smmu->version < ARM_SMMU_V2)
 716                 reg |= FIELD_PREP(CBAR_IRPTNDX, cfg->irptndx);
 717
 718         /*
 719          * Use the weakest shareability/memory types, so they are
 720          * overridden by the ttbcr/pte.
 721          */
 722         if (stage1) {
 723                 reg |= FIELD_PREP(CBAR_S1_BPSHCFG, CBAR_S1_BPSHCFG_NSH) |
 724                         FIELD_PREP(CBAR_S1_MEMATTR, CBAR_S1_MEMATTR_WB);
 725         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 726                 /* 8-bit VMIDs live in CBAR */
 727                 reg |= FIELD_PREP(CBAR_VMID, cfg->vmid);
 728         }
 729         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(idx));
 730
 731         /*
 732          * TCR
 733          * We must write this before the TTBRs, since it determines the
 734          * access behaviour of some fields (in particular, ASID[15:8]).
 735          */
 736         if (stage1 && smmu->version > ARM_SMMU_V1)
 737                 writel_relaxed(cb->tcr[1], cb_base + ARM_SMMU_CB_TCR2);
 738         writel_relaxed(cb->tcr[0], cb_base + ARM_SMMU_CB_TCR);
 739
 740         /* TTBRs */
 741         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 742                 writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 743                 writel_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
 744                 writel_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
 745         } else {
 746                 writeq_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
 747                 if (stage1)
 748                         writeq_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
 749         }
 750
 751         /* MAIRs (stage-1 only) */
 752         if (stage1) {
 753                 writel_relaxed(cb->mair[0], cb_base + ARM_SMMU_CB_S1_MAIR0);
 754                 writel_relaxed(cb->mair[1], cb_base + ARM_SMMU_CB_S1_MAIR1);
 755         }
 756
 757         /* SCTLR */
 758         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 759         if (stage1)
 760                 reg |= SCTLR_S1_ASIDPNE;
 761         if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 762                 reg |= SCTLR_E;
 763
 764         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 765 }
 766
 767 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 768                                         struct arm_smmu_device *smmu)
 769 {
 770         int irq, start, ret = 0;
 771         unsigned long ias, oas;
 772         struct io_pgtable_ops *pgtbl_ops;
 773         struct io_pgtable_cfg pgtbl_cfg;
 774         enum io_pgtable_fmt fmt;
 775         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 776         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 777
 778         mutex_lock(&smmu_domain->init_mutex);
 779         if (smmu_domain->smmu)
 780                 goto out_unlock;
 781
 782         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
 783                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
 784                 smmu_domain->smmu = smmu;
 785                 goto out_unlock;
 786         }
 787
 788         /*
 789          * Mapping the requested stage onto what we support is surprisingly
 790          * complicated, mainly because the spec allows S1+S2 SMMUs without
 791          * support for nested translation. That means we end up with the
 792          * following table:
 793          *
 794          * Requested        Supported        Actual
 795          *     S1               N              S1
 796          *     S1             S1+S2            S1
 797          *     S1               S2             S2
 798          *     S1               S1             S1
 799          *     N                N              N
 800          *     N              S1+S2            S2
 801          *     N                S2             S2
 802          *     N                S1             S1
 803          *
 804          * Note that you can't actually request stage-2 mappings.
 805          */
 806         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 807                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 808         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 809                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 810
 811         /*
 812          * Choosing a suitable context format is even more fiddly. Until we
 813          * grow some way for the caller to express a preference, and/or move
 814          * the decision into the io-pgtable code where it arguably belongs,
 815          * just aim for the closest thing to the rest of the system, and hope
 816          * that the hardware isn't esoteric enough that we can't assume AArch64
 817          * support to be a superset of AArch32 support...
 818          */
 819         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 820                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 821         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 822             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 823             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 824             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 825                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 826         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 827             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 828                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 829                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 830                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 831
 832         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 833                 ret = -EINVAL;
 834                 goto out_unlock;
 835         }
 836
 837         switch (smmu_domain->stage) {
 838         case ARM_SMMU_DOMAIN_S1:
 839                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 840                 start = smmu->num_s2_context_banks;
 841                 ias = smmu->va_size;
 842                 oas = smmu->ipa_size;
 843                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 844                         fmt = ARM_64_LPAE_S1;
 845                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 846                         fmt = ARM_32_LPAE_S1;
 847                         ias = min(ias, 32UL);
 848                         oas = min(oas, 40UL);
 849                 } else {
 850                         fmt = ARM_V7S;
 851                         ias = min(ias, 32UL);
 852                         oas = min(oas, 32UL);
 853                 }
 854                 smmu_domain->tlb_ops = &arm_smmu_s1_tlb_ops;
 855                 break;
 856         case ARM_SMMU_DOMAIN_NESTED:
 857                 /*
 858                  * We will likely want to change this if/when KVM gets
 859                  * involved.
 860                  */
 861         case ARM_SMMU_DOMAIN_S2:
 862                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 863                 start = 0;
 864                 ias = smmu->ipa_size;
 865                 oas = smmu->pa_size;
 866                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 867                         fmt = ARM_64_LPAE_S2;
 868                 } else {
 869                         fmt = ARM_32_LPAE_S2;
 870                         ias = min(ias, 40UL);
 871                         oas = min(oas, 40UL);
 872                 }
 873                 if (smmu->version == ARM_SMMU_V2)
 874                         smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v2;
 875                 else
 876                         smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v1;
 877                 break;
 878         default:
 879                 ret = -EINVAL;
 880                 goto out_unlock;
 881         }
 882         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 883                                       smmu->num_context_banks);
 884         if (ret < 0)
 885                 goto out_unlock;
 886
 887         cfg->cbndx = ret;
 888         if (smmu->version < ARM_SMMU_V2) {
 889                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 890                 cfg->irptndx %= smmu->num_context_irqs;
 891         } else {
 892                 cfg->irptndx = cfg->cbndx;
 893         }
 894
 895         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
 896                 cfg->vmid = cfg->cbndx + 1 + smmu->cavium_id_base;
 897         else
 898                 cfg->asid = cfg->cbndx + smmu->cavium_id_base;
 899
 900         pgtbl_cfg = (struct io_pgtable_cfg) {
 901                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 902                 .ias            = ias,
 903                 .oas            = oas,
 904                 .coherent_walk  = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
 905                 .tlb            = smmu_domain->tlb_ops,
 906                 .iommu_dev      = smmu->dev,
 907         };
 908
 909         if (smmu_domain->non_strict)
 910                 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
 911
 912         smmu_domain->smmu = smmu;
 913         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 914         if (!pgtbl_ops) {
 915                 ret = -ENOMEM;
 916                 goto out_clear_smmu;
 917         }
 918
 919         /* Update the domain's page sizes to reflect the page table format */
 920         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 921         domain->geometry.aperture_end = (1UL << ias) - 1;
 922         domain->geometry.force_aperture = true;
 923
 924         /* Initialise the context bank with our page table cfg */
 925         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 926         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 927
 928         /*
 929          * Request context fault interrupt. Do this last to avoid the
 930          * handler seeing a half-initialised domain state.
 931          */
 932         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 933         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 934                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 935         if (ret < 0) {
 936                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 937                         cfg->irptndx, irq);
 938                 cfg->irptndx = INVALID_IRPTNDX;
 939         }
 940
 941         mutex_unlock(&smmu_domain->init_mutex);
 942
 943         /* Publish page table ops for map/unmap */
 944         smmu_domain->pgtbl_ops = pgtbl_ops;
 945         return 0;
 946
 947 out_clear_smmu:
 948         smmu_domain->smmu = NULL;
 949 out_unlock:
 950         mutex_unlock(&smmu_domain->init_mutex);
 951         return ret;
 952 }
 953
 954 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 955 {
 956         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 957         struct arm_smmu_device *smmu = smmu_domain->smmu;
 958         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 959         int ret, irq;
 960
 961         if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
 962                 return;
 963
 964         ret = arm_smmu_rpm_get(smmu);
 965         if (ret < 0)
 966                 return;
 967
 968         /*
 969          * Disable the context bank and free the page tables before freeing
 970          * it.
 971          */
 972         smmu->cbs[cfg->cbndx].cfg = NULL;
 973         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 974
 975         if (cfg->irptndx != INVALID_IRPTNDX) {
 976                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 977                 devm_free_irq(smmu->dev, irq, domain);
 978         }
 979
 980         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 981         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 982
 983         arm_smmu_rpm_put(smmu);
 984 }
 985
 986 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 987 {
 988         struct arm_smmu_domain *smmu_domain;
 989
 990         if (type != IOMMU_DOMAIN_UNMANAGED &&
 991             type != IOMMU_DOMAIN_DMA &&
 992             type != IOMMU_DOMAIN_IDENTITY)
 993                 return NULL;
 994         /*
 995          * Allocate the domain and initialise some of its data structures.
 996          * We can't really do anything meaningful until we've added a
 997          * master.
 998          */
 999         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1000         if (!smmu_domain)
1001                 return NULL;
1002
1003         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1004             iommu_get_dma_cookie(&smmu_domain->domain))) {
1005                 kfree(smmu_domain);
1006                 return NULL;
1007         }
1008
1009         mutex_init(&smmu_domain->init_mutex);
1010         spin_lock_init(&smmu_domain->cb_lock);
1011
1012         return &smmu_domain->domain;
1013 }
1014
1015 static void arm_smmu_domain_free(struct iommu_domain *domain)
1016 {
1017         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1018
1019         /*
1020          * Free the domain resources. We assume that all devices have
1021          * already been detached.
1022          */
1023         iommu_put_dma_cookie(domain);
1024         arm_smmu_destroy_domain_context(domain);
1025         kfree(smmu_domain);
1026 }
1027
1028 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1029 {
1030         struct arm_smmu_smr *smr = smmu->smrs + idx;
1031         u32 reg = FIELD_PREP(SMR_ID, smr->id) | FIELD_PREP(SMR_MASK, smr->mask);
1032
1033         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
1034                 reg |= SMR_VALID;
1035         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1036 }
1037
1038 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1039 {
1040         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1041         u32 reg = FIELD_PREP(S2CR_TYPE, s2cr->type) |
1042                   FIELD_PREP(S2CR_CBNDX, s2cr->cbndx) |
1043                   FIELD_PREP(S2CR_PRIVCFG, s2cr->privcfg);
1044
1045         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1046             smmu->smrs[idx].valid)
1047                 reg |= S2CR_EXIDVALID;
1048         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1049 }
1050
1051 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1052 {
1053         arm_smmu_write_s2cr(smmu, idx);
1054         if (smmu->smrs)
1055                 arm_smmu_write_smr(smmu, idx);
1056 }
1057
1058 /*
1059  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1060  * should be called after sCR0 is written.
1061  */
1062 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1063 {
1064         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1065         u32 smr;
1066
1067         if (!smmu->smrs)
1068                 return;
1069
1070         /*
1071          * SMR.ID bits may not be preserved if the corresponding MASK
1072          * bits are set, so check each one separately. We can reject
1073          * masters later if they try to claim IDs outside these masks.
1074          */
1075         smr = FIELD_PREP(SMR_ID, smmu->streamid_mask);
1076         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1077         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1078         smmu->streamid_mask = FIELD_GET(SMR_ID, smr);
1079
1080         smr = FIELD_PREP(SMR_MASK, smmu->streamid_mask);
1081         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1082         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1083         smmu->smr_mask_mask = FIELD_GET(SMR_MASK, smr);
1084 }
1085
1086 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1087 {
1088         struct arm_smmu_smr *smrs = smmu->smrs;
1089         int i, free_idx = -ENOSPC;
1090
1091         /* Stream indexing is blissfully easy */
1092         if (!smrs)
1093                 return id;
1094
1095         /* Validating SMRs is... less so */
1096         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1097                 if (!smrs[i].valid) {
1098                         /*
1099                          * Note the first free entry we come across, which
1100                          * we'll claim in the end if nothing else matches.
1101                          */
1102                         if (free_idx < 0)
1103                                 free_idx = i;
1104                         continue;
1105                 }
1106                 /*
1107                  * If the new entry is _entirely_ matched by an existing entry,
1108                  * then reuse that, with the guarantee that there also cannot
1109                  * be any subsequent conflicting entries. In normal use we'd
1110                  * expect simply identical entries for this case, but there's
1111                  * no harm in accommodating the generalisation.
1112                  */
1113                 if ((mask & smrs[i].mask) == mask &&
1114                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1115                         return i;
1116                 /*
1117                  * If the new entry has any other overlap with an existing one,
1118                  * though, then there always exists at least one stream ID
1119                  * which would cause a conflict, and we can't allow that risk.
1120                  */
1121                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1122                         return -EINVAL;
1123         }
1124
1125         return free_idx;
1126 }
1127
1128 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1129 {
1130         if (--smmu->s2crs[idx].count)
1131                 return false;
1132
1133         smmu->s2crs[idx] = s2cr_init_val;
1134         if (smmu->smrs)
1135                 smmu->smrs[idx].valid = false;
1136
1137         return true;
1138 }
1139
1140 static int arm_smmu_master_alloc_smes(struct device *dev)
1141 {
1142         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1143         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1144         struct arm_smmu_device *smmu = cfg->smmu;
1145         struct arm_smmu_smr *smrs = smmu->smrs;
1146         struct iommu_group *group;
1147         int i, idx, ret;
1148
1149         mutex_lock(&smmu->stream_map_mutex);
1150         /* Figure out a viable stream map entry allocation */
1151         for_each_cfg_sme(fwspec, i, idx) {
1152                 u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1153                 u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
1154
1155                 if (idx != INVALID_SMENDX) {
1156                         ret = -EEXIST;
1157                         goto out_err;
1158                 }
1159
1160                 ret = arm_smmu_find_sme(smmu, sid, mask);
1161                 if (ret < 0)
1162                         goto out_err;
1163
1164                 idx = ret;
1165                 if (smrs && smmu->s2crs[idx].count == 0) {
1166                         smrs[idx].id = sid;
1167                         smrs[idx].mask = mask;
1168                         smrs[idx].valid = true;
1169                 }
1170                 smmu->s2crs[idx].count++;
1171                 cfg->smendx[i] = (s16)idx;
1172         }
1173
1174         group = iommu_group_get_for_dev(dev);
1175         if (!group)
1176                 group = ERR_PTR(-ENOMEM);
1177         if (IS_ERR(group)) {
1178                 ret = PTR_ERR(group);
1179                 goto out_err;
1180         }
1181         iommu_group_put(group);
1182
1183         /* It worked! Now, poke the actual hardware */
1184         for_each_cfg_sme(fwspec, i, idx) {
1185                 arm_smmu_write_sme(smmu, idx);
1186                 smmu->s2crs[idx].group = group;
1187         }
1188
1189         mutex_unlock(&smmu->stream_map_mutex);
1190         return 0;
1191
1192 out_err:
1193         while (i--) {
1194                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1195                 cfg->smendx[i] = INVALID_SMENDX;
1196         }
1197         mutex_unlock(&smmu->stream_map_mutex);
1198         return ret;
1199 }
1200
1201 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1202 {
1203         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1204         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1205         int i, idx;
1206
1207         mutex_lock(&smmu->stream_map_mutex);
1208         for_each_cfg_sme(fwspec, i, idx) {
1209                 if (arm_smmu_free_sme(smmu, idx))
1210                         arm_smmu_write_sme(smmu, idx);
1211                 cfg->smendx[i] = INVALID_SMENDX;
1212         }
1213         mutex_unlock(&smmu->stream_map_mutex);
1214 }
1215
1216 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1217                                       struct iommu_fwspec *fwspec)
1218 {
1219         struct arm_smmu_device *smmu = smmu_domain->smmu;
1220         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1221         u8 cbndx = smmu_domain->cfg.cbndx;
1222         enum arm_smmu_s2cr_type type;
1223         int i, idx;
1224
1225         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1226                 type = S2CR_TYPE_BYPASS;
1227         else
1228                 type = S2CR_TYPE_TRANS;
1229
1230         for_each_cfg_sme(fwspec, i, idx) {
1231                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1232                         continue;
1233
1234                 s2cr[idx].type = type;
1235                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1236                 s2cr[idx].cbndx = cbndx;
1237                 arm_smmu_write_s2cr(smmu, idx);
1238         }
1239         return 0;
1240 }
1241
1242 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1243 {
1244         int ret;
1245         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1246         struct arm_smmu_device *smmu;
1247         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1248
1249         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1250                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1251                 return -ENXIO;
1252         }
1253
1254         /*
1255          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1256          * domains between of_xlate() and add_device() - we have no way to cope
1257          * with that, so until ARM gets converted to rely on groups and default
1258          * domains, just say no (but more politely than by dereferencing NULL).
1259          * This should be at least a WARN_ON once that's sorted.
1260          */
1261         if (!fwspec->iommu_priv)
1262                 return -ENODEV;
1263
1264         smmu = fwspec_smmu(fwspec);
1265
1266         ret = arm_smmu_rpm_get(smmu);
1267         if (ret < 0)
1268                 return ret;
1269
1270         /* Ensure that the domain is finalised */
1271         ret = arm_smmu_init_domain_context(domain, smmu);
1272         if (ret < 0)
1273                 goto rpm_put;
1274
1275         /*
1276          * Sanity check the domain. We don't support domains across
1277          * different SMMUs.
1278          */
1279         if (smmu_domain->smmu != smmu) {
1280                 dev_err(dev,
1281                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1282                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1283                 ret = -EINVAL;
1284                 goto rpm_put;
1285         }
1286
1287         /* Looks ok, so add the device to the domain */
1288         ret = arm_smmu_domain_add_master(smmu_domain, fwspec);
1289
1290 rpm_put:
1291         arm_smmu_rpm_put(smmu);
1292         return ret;
1293 }
1294
1295 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1296                         phys_addr_t paddr, size_t size, int prot)
1297 {
1298         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1299         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1300         int ret;
1301
1302         if (!ops)
1303                 return -ENODEV;
1304
1305         arm_smmu_rpm_get(smmu);
1306         ret = ops->map(ops, iova, paddr, size, prot);
1307         arm_smmu_rpm_put(smmu);
1308
1309         return ret;
1310 }
1311
1312 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1313                              size_t size)
1314 {
1315         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1316         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1317         size_t ret;
1318
1319         if (!ops)
1320                 return 0;
1321
1322         arm_smmu_rpm_get(smmu);
1323         ret = ops->unmap(ops, iova, size);
1324         arm_smmu_rpm_put(smmu);
1325
1326         return ret;
1327 }
1328
1329 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1330 {
1331         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1332         struct arm_smmu_device *smmu = smmu_domain->smmu;
1333
1334         if (smmu_domain->tlb_ops) {
1335                 arm_smmu_rpm_get(smmu);
1336                 smmu_domain->tlb_ops->tlb_flush_all(smmu_domain);
1337                 arm_smmu_rpm_put(smmu);
1338         }
1339 }
1340
1341 static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
1342 {
1343         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1344         struct arm_smmu_device *smmu = smmu_domain->smmu;
1345
1346         if (smmu_domain->tlb_ops) {
1347                 arm_smmu_rpm_get(smmu);
1348                 smmu_domain->tlb_ops->tlb_sync(smmu_domain);
1349                 arm_smmu_rpm_put(smmu);
1350         }
1351 }
1352
1353 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1354                                               dma_addr_t iova)
1355 {
1356         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1357         struct arm_smmu_device *smmu = smmu_domain->smmu;
1358         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1359         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1360         struct device *dev = smmu->dev;
1361         void __iomem *cb_base;
1362         u32 tmp;
1363         u64 phys;
1364         unsigned long va, flags;
1365         int ret;
1366
1367         ret = arm_smmu_rpm_get(smmu);
1368         if (ret < 0)
1369                 return 0;
1370
1371         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
1372
1373         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1374         /* ATS1 registers can only be written atomically */
1375         va = iova & ~0xfffUL;
1376         if (smmu->version == ARM_SMMU_V2)
1377                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1378         else /* Register is only 32-bit in v1 */
1379                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1380
1381         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1382                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1383                 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1384                 dev_err(dev,
1385                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1386                         &iova);
1387                 return ops->iova_to_phys(ops, iova);
1388         }
1389
1390         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1391         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1392         if (phys & CB_PAR_F) {
1393                 dev_err(dev, "translation fault!\n");
1394                 dev_err(dev, "PAR = 0x%llx\n", phys);
1395                 return 0;
1396         }
1397
1398         arm_smmu_rpm_put(smmu);
1399
1400         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1401 }
1402
1403 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1404                                         dma_addr_t iova)
1405 {
1406         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1407         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1408
1409         if (domain->type == IOMMU_DOMAIN_IDENTITY)
1410                 return iova;
1411
1412         if (!ops)
1413                 return 0;
1414
1415         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1416                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1417                 return arm_smmu_iova_to_phys_hard(domain, iova);
1418
1419         return ops->iova_to_phys(ops, iova);
1420 }
1421
1422 static bool arm_smmu_capable(enum iommu_cap cap)
1423 {
1424         switch (cap) {
1425         case IOMMU_CAP_CACHE_COHERENCY:
1426                 /*
1427                  * Return true here as the SMMU can always send out coherent
1428                  * requests.
1429                  */
1430                 return true;
1431         case IOMMU_CAP_NOEXEC:
1432                 return true;
1433         default:
1434                 return false;
1435         }
1436 }
1437
1438 static int arm_smmu_match_node(struct device *dev, const void *data)
1439 {
1440         return dev->fwnode == data;
1441 }
1442
1443 static
1444 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1445 {
1446         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1447                                                 fwnode, arm_smmu_match_node);
1448         put_device(dev);
1449         return dev ? dev_get_drvdata(dev) : NULL;
1450 }
1451
1452 static int arm_smmu_add_device(struct device *dev)
1453 {
1454         struct arm_smmu_device *smmu;
1455         struct arm_smmu_master_cfg *cfg;
1456         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1457         int i, ret;
1458
1459         if (using_legacy_binding) {
1460                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1461
1462                 /*
1463                  * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1464                  * will allocate/initialise a new one. Thus we need to update fwspec for
1465                  * later use.
1466                  */
1467                 fwspec = dev_iommu_fwspec_get(dev);
1468                 if (ret)
1469                         goto out_free;
1470         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1471                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1472         } else {
1473                 return -ENODEV;
1474         }
1475
1476         ret = -EINVAL;
1477         for (i = 0; i < fwspec->num_ids; i++) {
1478                 u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1479                 u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
1480
1481                 if (sid & ~smmu->streamid_mask) {
1482                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1483                                 sid, smmu->streamid_mask);
1484                         goto out_free;
1485                 }
1486                 if (mask & ~smmu->smr_mask_mask) {
1487                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1488                                 mask, smmu->smr_mask_mask);
1489                         goto out_free;
1490                 }
1491         }
1492
1493         ret = -ENOMEM;
1494         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1495                       GFP_KERNEL);
1496         if (!cfg)
1497                 goto out_free;
1498
1499         cfg->smmu = smmu;
1500         fwspec->iommu_priv = cfg;
1501         while (i--)
1502                 cfg->smendx[i] = INVALID_SMENDX;
1503
1504         ret = arm_smmu_rpm_get(smmu);
1505         if (ret < 0)
1506                 goto out_cfg_free;
1507
1508         ret = arm_smmu_master_alloc_smes(dev);
1509         arm_smmu_rpm_put(smmu);
1510
1511         if (ret)
1512                 goto out_cfg_free;
1513
1514         iommu_device_link(&smmu->iommu, dev);
1515
1516         device_link_add(dev, smmu->dev,
1517                         DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1518
1519         return 0;
1520
1521 out_cfg_free:
1522         kfree(cfg);
1523 out_free:
1524         iommu_fwspec_free(dev);
1525         return ret;
1526 }
1527
1528 static void arm_smmu_remove_device(struct device *dev)
1529 {
1530         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1531         struct arm_smmu_master_cfg *cfg;
1532         struct arm_smmu_device *smmu;
1533         int ret;
1534
1535         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1536                 return;
1537
1538         cfg  = fwspec->iommu_priv;
1539         smmu = cfg->smmu;
1540
1541         ret = arm_smmu_rpm_get(smmu);
1542         if (ret < 0)
1543                 return;
1544
1545         iommu_device_unlink(&smmu->iommu, dev);
1546         arm_smmu_master_free_smes(fwspec);
1547
1548         arm_smmu_rpm_put(smmu);
1549
1550         iommu_group_remove_device(dev);
1551         kfree(fwspec->iommu_priv);
1552         iommu_fwspec_free(dev);
1553 }
1554
1555 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1556 {
1557         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1558         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1559         struct iommu_group *group = NULL;
1560         int i, idx;
1561
1562         for_each_cfg_sme(fwspec, i, idx) {
1563                 if (group && smmu->s2crs[idx].group &&
1564                     group != smmu->s2crs[idx].group)
1565                         return ERR_PTR(-EINVAL);
1566
1567                 group = smmu->s2crs[idx].group;
1568         }
1569
1570         if (group)
1571                 return iommu_group_ref_get(group);
1572
1573         if (dev_is_pci(dev))
1574                 group = pci_device_group(dev);
1575         else if (dev_is_fsl_mc(dev))
1576                 group = fsl_mc_device_group(dev);
1577         else
1578                 group = generic_device_group(dev);
1579
1580         return group;
1581 }
1582
1583 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1584                                     enum iommu_attr attr, void *data)
1585 {
1586         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1587
1588         switch(domain->type) {
1589         case IOMMU_DOMAIN_UNMANAGED:
1590                 switch (attr) {
1591                 case DOMAIN_ATTR_NESTING:
1592                         *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1593                         return 0;
1594                 default:
1595                         return -ENODEV;
1596                 }
1597                 break;
1598         case IOMMU_DOMAIN_DMA:
1599                 switch (attr) {
1600                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1601                         *(int *)data = smmu_domain->non_strict;
1602                         return 0;
1603                 default:
1604                         return -ENODEV;
1605                 }
1606                 break;
1607         default:
1608                 return -EINVAL;
1609         }
1610 }
1611
1612 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1613                                     enum iommu_attr attr, void *data)
1614 {
1615         int ret = 0;
1616         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1617
1618         mutex_lock(&smmu_domain->init_mutex);
1619
1620         switch(domain->type) {
1621         case IOMMU_DOMAIN_UNMANAGED:
1622                 switch (attr) {
1623                 case DOMAIN_ATTR_NESTING:
1624                         if (smmu_domain->smmu) {
1625                                 ret = -EPERM;
1626                                 goto out_unlock;
1627                         }
1628
1629                         if (*(int *)data)
1630                                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1631                         else
1632                                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1633                         break;
1634                 default:
1635                         ret = -ENODEV;
1636                 }
1637                 break;
1638         case IOMMU_DOMAIN_DMA:
1639                 switch (attr) {
1640                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1641                         smmu_domain->non_strict = *(int *)data;
1642                         break;
1643                 default:
1644                         ret = -ENODEV;
1645                 }
1646                 break;
1647         default:
1648                 ret = -EINVAL;
1649         }
1650 out_unlock:
1651         mutex_unlock(&smmu_domain->init_mutex);
1652         return ret;
1653 }
1654
1655 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1656 {
1657         u32 mask, fwid = 0;
1658
1659         if (args->args_count > 0)
1660                 fwid |= FIELD_PREP(SMR_ID, args->args[0]);
1661
1662         if (args->args_count > 1)
1663                 fwid |= FIELD_PREP(SMR_MASK, args->args[1]);
1664         else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1665                 fwid |= FIELD_PREP(SMR_MASK, mask);
1666
1667         return iommu_fwspec_add_ids(dev, &fwid, 1);
1668 }
1669
1670 static void arm_smmu_get_resv_regions(struct device *dev,
1671                                       struct list_head *head)
1672 {
1673         struct iommu_resv_region *region;
1674         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1675
1676         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1677                                          prot, IOMMU_RESV_SW_MSI);
1678         if (!region)
1679                 return;
1680
1681         list_add_tail(&region->list, head);
1682
1683         iommu_dma_get_resv_regions(dev, head);
1684 }
1685
1686 static void arm_smmu_put_resv_regions(struct device *dev,
1687                                       struct list_head *head)
1688 {
1689         struct iommu_resv_region *entry, *next;
1690
1691         list_for_each_entry_safe(entry, next, head, list)
1692                 kfree(entry);
1693 }
1694
1695 static struct iommu_ops arm_smmu_ops = {
1696         .capable                = arm_smmu_capable,
1697         .domain_alloc           = arm_smmu_domain_alloc,
1698         .domain_free            = arm_smmu_domain_free,
1699         .attach_dev             = arm_smmu_attach_dev,
1700         .map                    = arm_smmu_map,
1701         .unmap                  = arm_smmu_unmap,
1702         .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
1703         .iotlb_sync             = arm_smmu_iotlb_sync,
1704         .iova_to_phys           = arm_smmu_iova_to_phys,
1705         .add_device             = arm_smmu_add_device,
1706         .remove_device          = arm_smmu_remove_device,
1707         .device_group           = arm_smmu_device_group,
1708         .domain_get_attr        = arm_smmu_domain_get_attr,
1709         .domain_set_attr        = arm_smmu_domain_set_attr,
1710         .of_xlate               = arm_smmu_of_xlate,
1711         .get_resv_regions       = arm_smmu_get_resv_regions,
1712         .put_resv_regions       = arm_smmu_put_resv_regions,
1713         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1714 };
1715
1716 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1717 {
1718         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1719         int i;
1720         u32 reg, major;
1721
1722         /* clear global FSR */
1723         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1724         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1725
1726         /*
1727          * Reset stream mapping groups: Initial values mark all SMRn as
1728          * invalid and all S2CRn as bypass unless overridden.
1729          */
1730         for (i = 0; i < smmu->num_mapping_groups; ++i)
1731                 arm_smmu_write_sme(smmu, i);
1732
1733         if (smmu->model == ARM_MMU500) {
1734                 /*
1735                  * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1736                  * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1737                  * bit is only present in MMU-500r2 onwards.
1738                  */
1739                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1740                 major = FIELD_GET(ID7_MAJOR, reg);
1741                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1742                 if (major >= 2)
1743                         reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1744                 /*
1745                  * Allow unmatched Stream IDs to allocate bypass
1746                  * TLB entries for reduced latency.
1747                  */
1748                 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN;
1749                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1750         }
1751
1752         /* Make sure all context banks are disabled and clear CB_FSR  */
1753         for (i = 0; i < smmu->num_context_banks; ++i) {
1754                 void __iomem *cb_base = ARM_SMMU_CB(smmu, i);
1755
1756                 arm_smmu_write_context_bank(smmu, i);
1757                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1758                 /*
1759                  * Disable MMU-500's not-particularly-beneficial next-page
1760                  * prefetcher for the sake of errata #841119 and #826419.
1761                  */
1762                 if (smmu->model == ARM_MMU500) {
1763                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1764                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1765                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1766                 }
1767         }
1768
1769         /* Invalidate the TLB, just in case */
1770         writel_relaxed(QCOM_DUMMY_VAL, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1771         writel_relaxed(QCOM_DUMMY_VAL, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1772
1773         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1774
1775         /* Enable fault reporting */
1776         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1777
1778         /* Disable TLB broadcasting. */
1779         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1780
1781         /* Enable client access, handling unmatched streams as appropriate */
1782         reg &= ~sCR0_CLIENTPD;
1783         if (disable_bypass)
1784                 reg |= sCR0_USFCFG;
1785         else
1786                 reg &= ~sCR0_USFCFG;
1787
1788         /* Disable forced broadcasting */
1789         reg &= ~sCR0_FB;
1790
1791         /* Don't upgrade barriers */
1792         reg &= ~(sCR0_BSU);
1793
1794         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1795                 reg |= sCR0_VMID16EN;
1796
1797         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1798                 reg |= sCR0_EXIDENABLE;
1799
1800         /* Push the button */
1801         arm_smmu_tlb_sync_global(smmu);
1802         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1803 }
1804
1805 static int arm_smmu_id_size_to_bits(int size)
1806 {
1807         switch (size) {
1808         case 0:
1809                 return 32;
1810         case 1:
1811                 return 36;
1812         case 2:
1813                 return 40;
1814         case 3:
1815                 return 42;
1816         case 4:
1817                 return 44;
1818         case 5:
1819         default:
1820                 return 48;
1821         }
1822 }
1823
1824 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1825 {
1826         unsigned int size;
1827         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1828         u32 id;
1829         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1830         int i;
1831
1832         dev_notice(smmu->dev, "probing hardware configuration...\n");
1833         dev_notice(smmu->dev, "SMMUv%d with:\n",
1834                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1835
1836         /* ID0 */
1837         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1838
1839         /* Restrict available stages based on module parameter */
1840         if (force_stage == 1)
1841                 id &= ~(ID0_S2TS | ID0_NTS);
1842         else if (force_stage == 2)
1843                 id &= ~(ID0_S1TS | ID0_NTS);
1844
1845         if (id & ID0_S1TS) {
1846                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1847                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1848         }
1849
1850         if (id & ID0_S2TS) {
1851                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1852                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1853         }
1854
1855         if (id & ID0_NTS) {
1856                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1857                 dev_notice(smmu->dev, "\tnested translation\n");
1858         }
1859
1860         if (!(smmu->features &
1861                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1862                 dev_err(smmu->dev, "\tno translation support!\n");
1863                 return -ENODEV;
1864         }
1865
1866         if ((id & ID0_S1TS) &&
1867                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1868                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1869                 dev_notice(smmu->dev, "\taddress translation ops\n");
1870         }
1871
1872         /*
1873          * In order for DMA API calls to work properly, we must defer to what
1874          * the FW says about coherency, regardless of what the hardware claims.
1875          * Fortunately, this also opens up a workaround for systems where the
1876          * ID register value has ended up configured incorrectly.
1877          */
1878         cttw_reg = !!(id & ID0_CTTW);
1879         if (cttw_fw || cttw_reg)
1880                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1881                            cttw_fw ? "" : "non-");
1882         if (cttw_fw != cttw_reg)
1883                 dev_notice(smmu->dev,
1884                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1885
1886         /* Max. number of entries we have for stream matching/indexing */
1887         if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1888                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1889                 size = 1 << 16;
1890         } else {
1891                 size = 1 << FIELD_GET(ID0_NUMSIDB, id);
1892         }
1893         smmu->streamid_mask = size - 1;
1894         if (id & ID0_SMS) {
1895                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1896                 size = FIELD_GET(ID0_NUMSMRG, id);
1897                 if (size == 0) {
1898                         dev_err(smmu->dev,
1899                                 "stream-matching supported, but no SMRs present!\n");
1900                         return -ENODEV;
1901                 }
1902
1903                 /* Zero-initialised to mark as invalid */
1904                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1905                                           GFP_KERNEL);
1906                 if (!smmu->smrs)
1907                         return -ENOMEM;
1908
1909                 dev_notice(smmu->dev,
1910                            "\tstream matching with %u register groups", size);
1911         }
1912         /* s2cr->type == 0 means translation, so initialise explicitly */
1913         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1914                                          GFP_KERNEL);
1915         if (!smmu->s2crs)
1916                 return -ENOMEM;
1917         for (i = 0; i < size; i++)
1918                 smmu->s2crs[i] = s2cr_init_val;
1919
1920         smmu->num_mapping_groups = size;
1921         mutex_init(&smmu->stream_map_mutex);
1922         spin_lock_init(&smmu->global_sync_lock);
1923
1924         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1925                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1926                 if (!(id & ID0_PTFS_NO_AARCH32S))
1927                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1928         }
1929
1930         /* ID1 */
1931         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1932         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1933
1934         /* Check for size mismatch of SMMU address space from mapped region */
1935         size = 1 << (FIELD_GET(ID1_NUMPAGENDXB, id) + 1);
1936         if (smmu->numpage != 2 * size << smmu->pgshift)
1937                 dev_warn(smmu->dev,
1938                         "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1939                         2 * size << smmu->pgshift, smmu->numpage);
1940         /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1941         smmu->numpage = size;
1942
1943         smmu->num_s2_context_banks = FIELD_GET(ID1_NUMS2CB, id);
1944         smmu->num_context_banks = FIELD_GET(ID1_NUMCB, id);
1945         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1946                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1947                 return -ENODEV;
1948         }
1949         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1950                    smmu->num_context_banks, smmu->num_s2_context_banks);
1951         /*
1952          * Cavium CN88xx erratum #27704.
1953          * Ensure ASID and VMID allocation is unique across all SMMUs in
1954          * the system.
1955          */
1956         if (smmu->model == CAVIUM_SMMUV2) {
1957                 smmu->cavium_id_base =
1958                         atomic_add_return(smmu->num_context_banks,
1959                                           &cavium_smmu_context_count);
1960                 smmu->cavium_id_base -= smmu->num_context_banks;
1961                 dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
1962         }
1963         smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1964                                  sizeof(*smmu->cbs), GFP_KERNEL);
1965         if (!smmu->cbs)
1966                 return -ENOMEM;
1967
1968         /* ID2 */
1969         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1970         size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_IAS, id));
1971         smmu->ipa_size = size;
1972
1973         /* The output mask is also applied for bypass */
1974         size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_OAS, id));
1975         smmu->pa_size = size;
1976
1977         if (id & ID2_VMID16)
1978                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1979
1980         /*
1981          * What the page table walker can address actually depends on which
1982          * descriptor format is in use, but since a) we don't know that yet,
1983          * and b) it can vary per context bank, this will have to do...
1984          */
1985         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1986                 dev_warn(smmu->dev,
1987                          "failed to set DMA mask for table walker\n");
1988
1989         if (smmu->version < ARM_SMMU_V2) {
1990                 smmu->va_size = smmu->ipa_size;
1991                 if (smmu->version == ARM_SMMU_V1_64K)
1992                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1993         } else {
1994                 size = FIELD_GET(ID2_UBS, id);
1995                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1996                 if (id & ID2_PTFS_4K)
1997                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1998                 if (id & ID2_PTFS_16K)
1999                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
2000                 if (id & ID2_PTFS_64K)
2001                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
2002         }
2003
2004         /* Now we've corralled the various formats, what'll it do? */
2005         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
2006                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
2007         if (smmu->features &
2008             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
2009                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2010         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
2011                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
2012         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
2013                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
2014
2015         if (arm_smmu_ops.pgsize_bitmap == -1UL)
2016                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
2017         else
2018                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
2019         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
2020                    smmu->pgsize_bitmap);
2021
2022
2023         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
2024                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
2025                            smmu->va_size, smmu->ipa_size);
2026
2027         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
2028                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
2029                            smmu->ipa_size, smmu->pa_size);
2030
2031         return 0;
2032 }
2033
2034 struct arm_smmu_match_data {
2035         enum arm_smmu_arch_version version;
2036         enum arm_smmu_implementation model;
2037 };
2038
2039 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
2040 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
2041
2042 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
2043 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
2044 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
2045 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
2046 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
2047 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
2048
2049 static const struct of_device_id arm_smmu_of_match[] = {
2050         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
2051         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
2052         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
2053         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
2054         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
2055         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
2056         { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
2057         { },
2058 };
2059
2060 #ifdef CONFIG_ACPI
2061 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
2062 {
2063         int ret = 0;
2064
2065         switch (model) {
2066         case ACPI_IORT_SMMU_V1:
2067         case ACPI_IORT_SMMU_CORELINK_MMU400:
2068                 smmu->version = ARM_SMMU_V1;
2069                 smmu->model = GENERIC_SMMU;
2070                 break;
2071         case ACPI_IORT_SMMU_CORELINK_MMU401:
2072                 smmu->version = ARM_SMMU_V1_64K;
2073                 smmu->model = GENERIC_SMMU;
2074                 break;
2075         case ACPI_IORT_SMMU_V2:
2076                 smmu->version = ARM_SMMU_V2;
2077                 smmu->model = GENERIC_SMMU;
2078                 break;
2079         case ACPI_IORT_SMMU_CORELINK_MMU500:
2080                 smmu->version = ARM_SMMU_V2;
2081                 smmu->model = ARM_MMU500;
2082                 break;
2083         case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
2084                 smmu->version = ARM_SMMU_V2;
2085                 smmu->model = CAVIUM_SMMUV2;
2086                 break;
2087         default:
2088                 ret = -ENODEV;
2089         }
2090
2091         return ret;
2092 }
2093
2094 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2095                                       struct arm_smmu_device *smmu)
2096 {
2097         struct device *dev = smmu->dev;
2098         struct acpi_iort_node *node =
2099                 *(struct acpi_iort_node **)dev_get_platdata(dev);
2100         struct acpi_iort_smmu *iort_smmu;
2101         int ret;
2102
2103         /* Retrieve SMMU1/2 specific data */
2104         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2105
2106         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2107         if (ret < 0)
2108                 return ret;
2109
2110         /* Ignore the configuration access interrupt */
2111         smmu->num_global_irqs = 1;
2112
2113         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2114                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2115
2116         return 0;
2117 }
2118 #else
2119 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2120                                              struct arm_smmu_device *smmu)
2121 {
2122         return -ENODEV;
2123 }
2124 #endif
2125
2126 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2127                                     struct arm_smmu_device *smmu)
2128 {
2129         const struct arm_smmu_match_data *data;
2130         struct device *dev = &pdev->dev;
2131         bool legacy_binding;
2132
2133         if (of_property_read_u32(dev->of_node, "#global-interrupts",
2134                                  &smmu->num_global_irqs)) {
2135                 dev_err(dev, "missing #global-interrupts property\n");
2136                 return -ENODEV;
2137         }
2138
2139         data = of_device_get_match_data(dev);
2140         smmu->version = data->version;
2141         smmu->model = data->model;
2142
2143         parse_driver_options(smmu);
2144
2145         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2146         if (legacy_binding && !using_generic_binding) {
2147                 if (!using_legacy_binding)
2148                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2149                 using_legacy_binding = true;
2150         } else if (!legacy_binding && !using_legacy_binding) {
2151                 using_generic_binding = true;
2152         } else {
2153                 dev_err(dev, "not probing due to mismatched DT properties\n");
2154                 return -ENODEV;
2155         }
2156
2157         if (of_dma_is_coherent(dev->of_node))
2158                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2159
2160         return 0;
2161 }
2162
2163 static void arm_smmu_bus_init(void)
2164 {
2165         /* Oh, for a proper bus abstraction */
2166         if (!iommu_present(&platform_bus_type))
2167                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2168 #ifdef CONFIG_ARM_AMBA
2169         if (!iommu_present(&amba_bustype))
2170                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2171 #endif
2172 #ifdef CONFIG_PCI
2173         if (!iommu_present(&pci_bus_type)) {
2174                 pci_request_acs();
2175                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2176         }
2177 #endif
2178 #ifdef CONFIG_FSL_MC_BUS
2179         if (!iommu_present(&fsl_mc_bus_type))
2180                 bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
2181 #endif
2182 }
2183
2184 static int arm_smmu_device_probe(struct platform_device *pdev)
2185 {
2186         struct resource *res;
2187         resource_size_t ioaddr;
2188         struct arm_smmu_device *smmu;
2189         struct device *dev = &pdev->dev;
2190         int num_irqs, i, err;
2191
2192         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2193         if (!smmu) {
2194                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2195                 return -ENOMEM;
2196         }
2197         smmu->dev = dev;
2198
2199         if (dev->of_node)
2200                 err = arm_smmu_device_dt_probe(pdev, smmu);
2201         else
2202                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2203
2204         if (err)
2205                 return err;
2206
2207         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2208         ioaddr = res->start;
2209         smmu->base = devm_ioremap_resource(dev, res);
2210         if (IS_ERR(smmu->base))
2211                 return PTR_ERR(smmu->base);
2212         /*
2213          * The resource size should effectively match the value of SMMU_TOP;
2214          * stash that temporarily until we know PAGESIZE to validate it with.
2215          */
2216         smmu->numpage = resource_size(res);
2217
2218         num_irqs = 0;
2219         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2220                 num_irqs++;
2221                 if (num_irqs > smmu->num_global_irqs)
2222                         smmu->num_context_irqs++;
2223         }
2224
2225         if (!smmu->num_context_irqs) {
2226                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2227                         num_irqs, smmu->num_global_irqs + 1);
2228                 return -ENODEV;
2229         }
2230
2231         smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2232                                   GFP_KERNEL);
2233         if (!smmu->irqs) {
2234                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2235                 return -ENOMEM;
2236         }
2237
2238         for (i = 0; i < num_irqs; ++i) {
2239                 int irq = platform_get_irq(pdev, i);
2240
2241                 if (irq < 0) {
2242                         dev_err(dev, "failed to get irq index %d\n", i);
2243                         return -ENODEV;
2244                 }
2245                 smmu->irqs[i] = irq;
2246         }
2247
2248         err = devm_clk_bulk_get_all(dev, &smmu->clks);
2249         if (err < 0) {
2250                 dev_err(dev, "failed to get clocks %d\n", err);
2251                 return err;
2252         }
2253         smmu->num_clks = err;
2254
2255         err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2256         if (err)
2257                 return err;
2258
2259         err = arm_smmu_device_cfg_probe(smmu);
2260         if (err)
2261                 return err;
2262
2263         if (smmu->version == ARM_SMMU_V2) {
2264                 if (smmu->num_context_banks > smmu->num_context_irqs) {
2265                         dev_err(dev,
2266                               "found only %d context irq(s) but %d required\n",
2267                               smmu->num_context_irqs, smmu->num_context_banks);
2268                         return -ENODEV;
2269                 }
2270
2271                 /* Ignore superfluous interrupts */
2272                 smmu->num_context_irqs = smmu->num_context_banks;
2273         }
2274
2275         for (i = 0; i < smmu->num_global_irqs; ++i) {
2276                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2277                                        arm_smmu_global_fault,
2278                                        IRQF_SHARED,
2279                                        "arm-smmu global fault",
2280                                        smmu);
2281                 if (err) {
2282                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2283                                 i, smmu->irqs[i]);
2284                         return err;
2285                 }
2286         }
2287
2288         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2289                                      "smmu.%pa", &ioaddr);
2290         if (err) {
2291                 dev_err(dev, "Failed to register iommu in sysfs\n");
2292                 return err;
2293         }
2294
2295         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2296         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2297
2298         err = iommu_device_register(&smmu->iommu);
2299         if (err) {
2300                 dev_err(dev, "Failed to register iommu\n");
2301                 return err;
2302         }
2303
2304         platform_set_drvdata(pdev, smmu);
2305         arm_smmu_device_reset(smmu);
2306         arm_smmu_test_smr_masks(smmu);
2307
2308         /*
2309          * We want to avoid touching dev->power.lock in fastpaths unless
2310          * it's really going to do something useful - pm_runtime_enabled()
2311          * can serve as an ideal proxy for that decision. So, conditionally
2312          * enable pm_runtime.
2313          */
2314         if (dev->pm_domain) {
2315                 pm_runtime_set_active(dev);
2316                 pm_runtime_enable(dev);
2317         }
2318
2319         /*
2320          * For ACPI and generic DT bindings, an SMMU will be probed before
2321          * any device which might need it, so we want the bus ops in place
2322          * ready to handle default domain setup as soon as any SMMU exists.
2323          */
2324         if (!using_legacy_binding)
2325                 arm_smmu_bus_init();
2326
2327         return 0;
2328 }
2329
2330 /*
2331  * With the legacy DT binding in play, though, we have no guarantees about
2332  * probe order, but then we're also not doing default domains, so we can
2333  * delay setting bus ops until we're sure every possible SMMU is ready,
2334  * and that way ensure that no add_device() calls get missed.
2335  */
2336 static int arm_smmu_legacy_bus_init(void)
2337 {
2338         if (using_legacy_binding)
2339                 arm_smmu_bus_init();
2340         return 0;
2341 }
2342 device_initcall_sync(arm_smmu_legacy_bus_init);
2343
2344 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2345 {
2346         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2347
2348         if (!smmu)
2349                 return;
2350
2351         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2352                 dev_err(&pdev->dev, "removing device with active domains!\n");
2353
2354         arm_smmu_rpm_get(smmu);
2355         /* Turn the thing off */
2356         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2357         arm_smmu_rpm_put(smmu);
2358
2359         if (pm_runtime_enabled(smmu->dev))
2360                 pm_runtime_force_suspend(smmu->dev);
2361         else
2362                 clk_bulk_disable(smmu->num_clks, smmu->clks);
2363
2364         clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2365 }
2366
2367 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2368 {
2369         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2370         int ret;
2371
2372         ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2373         if (ret)
2374                 return ret;
2375
2376         arm_smmu_device_reset(smmu);
2377
2378         return 0;
2379 }
2380
2381 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2382 {
2383         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2384
2385         clk_bulk_disable(smmu->num_clks, smmu->clks);
2386
2387         return 0;
2388 }
2389
2390 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2391 {
2392         if (pm_runtime_suspended(dev))
2393                 return 0;
2394
2395         return arm_smmu_runtime_resume(dev);
2396 }
2397
2398 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2399 {
2400         if (pm_runtime_suspended(dev))
2401                 return 0;
2402
2403         return arm_smmu_runtime_suspend(dev);
2404 }
2405
2406 static const struct dev_pm_ops arm_smmu_pm_ops = {
2407         SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2408         SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2409                            arm_smmu_runtime_resume, NULL)
2410 };
2411
2412 static struct platform_driver arm_smmu_driver = {
2413         .driver = {
2414                 .name                   = "arm-smmu",
2415                 .of_match_table         = of_match_ptr(arm_smmu_of_match),
2416                 .pm                     = &arm_smmu_pm_ops,
2417                 .suppress_bind_attrs    = true,
2418         },
2419         .probe  = arm_smmu_device_probe,
2420         .shutdown = arm_smmu_device_shutdown,
2421 };
2422 builtin_platform_driver(arm_smmu_driver);