drivers/iommu/arm-smmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * IOMMU API for ARM architected SMMU implementations.
   4  *
   5  * Copyright (C) 2013 ARM Limited
   6  *
   7  * Author: Will Deacon <will.deacon@arm.com>
   8  *
   9  * This driver currently supports:
  10  *      - SMMUv1 and v2 implementations
  11  *      - Stream-matching and stream-indexing
  12  *      - v7/v8 long-descriptor format
  13  *      - Non-secure access to the SMMU
  14  *      - Context fault reporting
  15  *      - Extended Stream ID (16 bit)
  16  */
  17
  18 #define pr_fmt(fmt) "arm-smmu: " fmt
  19
  20 #include <linux/acpi.h>
  21 #include <linux/acpi_iort.h>
  22 #include <linux/atomic.h>
  23 #include <linux/bitfield.h>
  24 #include <linux/delay.h>
  25 #include <linux/dma-iommu.h>
  26 #include <linux/dma-mapping.h>
  27 #include <linux/err.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/io.h>
  30 #include <linux/io-64-nonatomic-hi-lo.h>
  31 #include <linux/io-pgtable.h>
  32 #include <linux/iommu.h>
  33 #include <linux/iopoll.h>
  34 #include <linux/init.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/of.h>
  37 #include <linux/of_address.h>
  38 #include <linux/of_device.h>
  39 #include <linux/of_iommu.h>
  40 #include <linux/pci.h>
  41 #include <linux/platform_device.h>
  42 #include <linux/pm_runtime.h>
  43 #include <linux/slab.h>
  44 #include <linux/spinlock.h>
  45
  46 #include <linux/amba/bus.h>
  47 #include <linux/fsl/mc.h>
  48
  49 #include "arm-smmu-regs.h"
  50
  51 /*
  52  * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
  53  * global register space are still, in fact, using a hypervisor to mediate it
  54  * by trapping and emulating register accesses. Sadly, some deployed versions
  55  * of said trapping code have bugs wherein they go horribly wrong for stores
  56  * using r31 (i.e. XZR/WZR) as the source register.
  57  */
  58 #define QCOM_DUMMY_VAL -1
  59
  60 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
  61
  62 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
  63 #define ARM_MMU500_ACR_S2CRB_TLBEN      (1 << 10)
  64 #define ARM_MMU500_ACR_SMTNMB_TLBEN     (1 << 8)
  65
  66 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
  67 #define TLB_SPIN_COUNT                  10
  68
  69 /* Maximum number of context banks per SMMU */
  70 #define ARM_SMMU_MAX_CBS                128
  71
  72 /* SMMU global address space */
  73 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  74
  75 /*
  76  * SMMU global address space with conditional offset to access secure
  77  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  78  * nsGFSYNR0: 0x450)
  79  */
  80 #define ARM_SMMU_GR0_NS(smmu)                                           \
  81         ((smmu)->base +                                                 \
  82                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  83                         ? 0x400 : 0))
  84
  85 /* Translation context bank */
  86 #define ARM_SMMU_CB(smmu, n)    ((smmu)->base + (((smmu)->numpage + (n)) << (smmu)->pgshift))
  87
  88 #define MSI_IOVA_BASE                   0x8000000
  89 #define MSI_IOVA_LENGTH                 0x100000
  90
  91 static int force_stage;
  92 /*
  93  * not really modular, but the easiest way to keep compat with existing
  94  * bootargs behaviour is to continue using module_param() here.
  95  */
  96 module_param(force_stage, int, S_IRUGO);
  97 MODULE_PARM_DESC(force_stage,
  98         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
  99 static bool disable_bypass =
 100         IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
 101 module_param(disable_bypass, bool, S_IRUGO);
 102 MODULE_PARM_DESC(disable_bypass,
 103         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 104
 105 enum arm_smmu_arch_version {
 106         ARM_SMMU_V1,
 107         ARM_SMMU_V1_64K,
 108         ARM_SMMU_V2,
 109 };
 110
 111 enum arm_smmu_implementation {
 112         GENERIC_SMMU,
 113         ARM_MMU500,
 114         CAVIUM_SMMUV2,
 115         QCOM_SMMUV2,
 116 };
 117
 118 struct arm_smmu_s2cr {
 119         struct iommu_group              *group;
 120         int                             count;
 121         enum arm_smmu_s2cr_type         type;
 122         enum arm_smmu_s2cr_privcfg      privcfg;
 123         u8                              cbndx;
 124 };
 125
 126 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 127         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 128 }
 129
 130 struct arm_smmu_smr {
 131         u16                             mask;
 132         u16                             id;
 133         bool                            valid;
 134 };
 135
 136 struct arm_smmu_cb {
 137         u64                             ttbr[2];
 138         u32                             tcr[2];
 139         u32                             mair[2];
 140         struct arm_smmu_cfg             *cfg;
 141 };
 142
 143 struct arm_smmu_master_cfg {
 144         struct arm_smmu_device          *smmu;
 145         s16                             smendx[];
 146 };
 147 #define INVALID_SMENDX                  -1
 148 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 149 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 150 #define fwspec_smendx(fw, i) \
 151         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 152 #define for_each_cfg_sme(fw, i, idx) \
 153         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 154
 155 struct arm_smmu_device {
 156         struct device                   *dev;
 157
 158         void __iomem                    *base;
 159         unsigned int                    numpage;
 160         unsigned int                    pgshift;
 161
 162 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 163 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 164 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 165 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 166 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 167 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 168 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 169 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 170 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 171 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 172 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 173 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 174 #define ARM_SMMU_FEAT_EXIDS             (1 << 12)
 175         u32                             features;
 176
 177 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 178         u32                             options;
 179         enum arm_smmu_arch_version      version;
 180         enum arm_smmu_implementation    model;
 181
 182         u32                             num_context_banks;
 183         u32                             num_s2_context_banks;
 184         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 185         struct arm_smmu_cb              *cbs;
 186         atomic_t                        irptndx;
 187
 188         u32                             num_mapping_groups;
 189         u16                             streamid_mask;
 190         u16                             smr_mask_mask;
 191         struct arm_smmu_smr             *smrs;
 192         struct arm_smmu_s2cr            *s2crs;
 193         struct mutex                    stream_map_mutex;
 194
 195         unsigned long                   va_size;
 196         unsigned long                   ipa_size;
 197         unsigned long                   pa_size;
 198         unsigned long                   pgsize_bitmap;
 199
 200         u32                             num_global_irqs;
 201         u32                             num_context_irqs;
 202         unsigned int                    *irqs;
 203         struct clk_bulk_data            *clks;
 204         int                             num_clks;
 205
 206         u32                             cavium_id_base; /* Specific to Cavium */
 207
 208         spinlock_t                      global_sync_lock;
 209
 210         /* IOMMU core code handle */
 211         struct iommu_device             iommu;
 212 };
 213
 214 enum arm_smmu_context_fmt {
 215         ARM_SMMU_CTX_FMT_NONE,
 216         ARM_SMMU_CTX_FMT_AARCH64,
 217         ARM_SMMU_CTX_FMT_AARCH32_L,
 218         ARM_SMMU_CTX_FMT_AARCH32_S,
 219 };
 220
 221 struct arm_smmu_cfg {
 222         u8                              cbndx;
 223         u8                              irptndx;
 224         union {
 225                 u16                     asid;
 226                 u16                     vmid;
 227         };
 228         enum arm_smmu_cbar_type         cbar;
 229         enum arm_smmu_context_fmt       fmt;
 230 };
 231 #define INVALID_IRPTNDX                 0xff
 232
 233 enum arm_smmu_domain_stage {
 234         ARM_SMMU_DOMAIN_S1 = 0,
 235         ARM_SMMU_DOMAIN_S2,
 236         ARM_SMMU_DOMAIN_NESTED,
 237         ARM_SMMU_DOMAIN_BYPASS,
 238 };
 239
 240 struct arm_smmu_domain {
 241         struct arm_smmu_device          *smmu;
 242         struct io_pgtable_ops           *pgtbl_ops;
 243         const struct iommu_gather_ops   *tlb_ops;
 244         struct arm_smmu_cfg             cfg;
 245         enum arm_smmu_domain_stage      stage;
 246         bool                            non_strict;
 247         struct mutex                    init_mutex; /* Protects smmu pointer */
 248         spinlock_t                      cb_lock; /* Serialises ATS1* ops and TLB syncs */
 249         struct iommu_domain             domain;
 250 };
 251
 252 static void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)
 253 {
 254         return smmu->base + (n << smmu->pgshift);
 255 }
 256
 257 static u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)
 258 {
 259         return readl_relaxed(arm_smmu_page(smmu, page) + offset);
 260 }
 261
 262 static void arm_smmu_writel(struct arm_smmu_device *smmu, int page, int offset,
 263                             u32 val)
 264 {
 265         writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
 266 }
 267
 268 #define ARM_SMMU_GR1            1
 269
 270 #define arm_smmu_gr1_read(s, o)         \
 271         arm_smmu_readl((s), ARM_SMMU_GR1, (o))
 272 #define arm_smmu_gr1_write(s, o, v)     \
 273         arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v))
 274
 275 struct arm_smmu_option_prop {
 276         u32 opt;
 277         const char *prop;
 278 };
 279
 280 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 281
 282 static bool using_legacy_binding, using_generic_binding;
 283
 284 static struct arm_smmu_option_prop arm_smmu_options[] = {
 285         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 286         { 0, NULL},
 287 };
 288
 289 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
 290 {
 291         if (pm_runtime_enabled(smmu->dev))
 292                 return pm_runtime_get_sync(smmu->dev);
 293
 294         return 0;
 295 }
 296
 297 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
 298 {
 299         if (pm_runtime_enabled(smmu->dev))
 300                 pm_runtime_put(smmu->dev);
 301 }
 302
 303 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 304 {
 305         return container_of(dom, struct arm_smmu_domain, domain);
 306 }
 307
 308 static void parse_driver_options(struct arm_smmu_device *smmu)
 309 {
 310         int i = 0;
 311
 312         do {
 313                 if (of_property_read_bool(smmu->dev->of_node,
 314                                                 arm_smmu_options[i].prop)) {
 315                         smmu->options |= arm_smmu_options[i].opt;
 316                         dev_notice(smmu->dev, "option %s\n",
 317                                 arm_smmu_options[i].prop);
 318                 }
 319         } while (arm_smmu_options[++i].opt);
 320 }
 321
 322 static struct device_node *dev_get_dev_node(struct device *dev)
 323 {
 324         if (dev_is_pci(dev)) {
 325                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 326
 327                 while (!pci_is_root_bus(bus))
 328                         bus = bus->parent;
 329                 return of_node_get(bus->bridge->parent->of_node);
 330         }
 331
 332         return of_node_get(dev->of_node);
 333 }
 334
 335 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 336 {
 337         *((__be32 *)data) = cpu_to_be32(alias);
 338         return 0; /* Continue walking */
 339 }
 340
 341 static int __find_legacy_master_phandle(struct device *dev, void *data)
 342 {
 343         struct of_phandle_iterator *it = *(void **)data;
 344         struct device_node *np = it->node;
 345         int err;
 346
 347         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 348                             "#stream-id-cells", 0)
 349                 if (it->node == np) {
 350                         *(void **)data = dev;
 351                         return 1;
 352                 }
 353         it->node = np;
 354         return err == -ENOENT ? 0 : err;
 355 }
 356
 357 static struct platform_driver arm_smmu_driver;
 358 static struct iommu_ops arm_smmu_ops;
 359
 360 static int arm_smmu_register_legacy_master(struct device *dev,
 361                                            struct arm_smmu_device **smmu)
 362 {
 363         struct device *smmu_dev;
 364         struct device_node *np;
 365         struct of_phandle_iterator it;
 366         void *data = &it;
 367         u32 *sids;
 368         __be32 pci_sid;
 369         int err;
 370
 371         np = dev_get_dev_node(dev);
 372         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 373                 of_node_put(np);
 374                 return -ENODEV;
 375         }
 376
 377         it.node = np;
 378         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 379                                      __find_legacy_master_phandle);
 380         smmu_dev = data;
 381         of_node_put(np);
 382         if (err == 0)
 383                 return -ENODEV;
 384         if (err < 0)
 385                 return err;
 386
 387         if (dev_is_pci(dev)) {
 388                 /* "mmu-masters" assumes Stream ID == Requester ID */
 389                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 390                                        &pci_sid);
 391                 it.cur = &pci_sid;
 392                 it.cur_count = 1;
 393         }
 394
 395         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 396                                 &arm_smmu_ops);
 397         if (err)
 398                 return err;
 399
 400         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 401         if (!sids)
 402                 return -ENOMEM;
 403
 404         *smmu = dev_get_drvdata(smmu_dev);
 405         of_phandle_iterator_args(&it, sids, it.cur_count);
 406         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 407         kfree(sids);
 408         return err;
 409 }
 410
 411 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 412 {
 413         int idx;
 414
 415         do {
 416                 idx = find_next_zero_bit(map, end, start);
 417                 if (idx == end)
 418                         return -ENOSPC;
 419         } while (test_and_set_bit(idx, map));
 420
 421         return idx;
 422 }
 423
 424 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 425 {
 426         clear_bit(idx, map);
 427 }
 428
 429 /* Wait for any pending TLB invalidations to complete */
 430 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu,
 431                                 void __iomem *sync, void __iomem *status)
 432 {
 433         unsigned int spin_cnt, delay;
 434
 435         writel_relaxed(QCOM_DUMMY_VAL, sync);
 436         for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 437                 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 438                         if (!(readl_relaxed(status) & sTLBGSTATUS_GSACTIVE))
 439                                 return;
 440                         cpu_relax();
 441                 }
 442                 udelay(delay);
 443         }
 444         dev_err_ratelimited(smmu->dev,
 445                             "TLB sync timed out -- SMMU may be deadlocked\n");
 446 }
 447
 448 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 449 {
 450         void __iomem *base = ARM_SMMU_GR0(smmu);
 451         unsigned long flags;
 452
 453         spin_lock_irqsave(&smmu->global_sync_lock, flags);
 454         __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_GR0_sTLBGSYNC,
 455                             base + ARM_SMMU_GR0_sTLBGSTATUS);
 456         spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
 457 }
 458
 459 static void arm_smmu_tlb_sync_context(void *cookie)
 460 {
 461         struct arm_smmu_domain *smmu_domain = cookie;
 462         struct arm_smmu_device *smmu = smmu_domain->smmu;
 463         void __iomem *base = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx);
 464         unsigned long flags;
 465
 466         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
 467         __arm_smmu_tlb_sync(smmu, base + ARM_SMMU_CB_TLBSYNC,
 468                             base + ARM_SMMU_CB_TLBSTATUS);
 469         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
 470 }
 471
 472 static void arm_smmu_tlb_sync_vmid(void *cookie)
 473 {
 474         struct arm_smmu_domain *smmu_domain = cookie;
 475
 476         arm_smmu_tlb_sync_global(smmu_domain->smmu);
 477 }
 478
 479 static void arm_smmu_tlb_inv_context_s1(void *cookie)
 480 {
 481         struct arm_smmu_domain *smmu_domain = cookie;
 482         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 483         void __iomem *base = ARM_SMMU_CB(smmu_domain->smmu, cfg->cbndx);
 484
 485         /*
 486          * NOTE: this is not a relaxed write; it needs to guarantee that PTEs
 487          * cleared by the current CPU are visible to the SMMU before the TLBI.
 488          */
 489         writel(cfg->asid, base + ARM_SMMU_CB_S1_TLBIASID);
 490         arm_smmu_tlb_sync_context(cookie);
 491 }
 492
 493 static void arm_smmu_tlb_inv_context_s2(void *cookie)
 494 {
 495         struct arm_smmu_domain *smmu_domain = cookie;
 496         struct arm_smmu_device *smmu = smmu_domain->smmu;
 497         void __iomem *base = ARM_SMMU_GR0(smmu);
 498
 499         /* NOTE: see above */
 500         writel(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
 501         arm_smmu_tlb_sync_global(smmu);
 502 }
 503
 504 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
 505                                       size_t granule, bool leaf, void *cookie)
 506 {
 507         struct arm_smmu_domain *smmu_domain = cookie;
 508         struct arm_smmu_device *smmu = smmu_domain->smmu;
 509         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 510         void __iomem *reg = ARM_SMMU_CB(smmu, cfg->cbndx);
 511
 512         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 513                 wmb();
 514
 515         reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 516
 517         if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 518                 iova = (iova >> 12) << 12;
 519                 iova |= cfg->asid;
 520                 do {
 521                         writel_relaxed(iova, reg);
 522                         iova += granule;
 523                 } while (size -= granule);
 524         } else {
 525                 iova >>= 12;
 526                 iova |= (u64)cfg->asid << 48;
 527                 do {
 528                         writeq_relaxed(iova, reg);
 529                         iova += granule >> 12;
 530                 } while (size -= granule);
 531         }
 532 }
 533
 534 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
 535                                       size_t granule, bool leaf, void *cookie)
 536 {
 537         struct arm_smmu_domain *smmu_domain = cookie;
 538         struct arm_smmu_device *smmu = smmu_domain->smmu;
 539         void __iomem *reg = ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx);
 540
 541         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 542                 wmb();
 543
 544         reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : ARM_SMMU_CB_S2_TLBIIPAS2;
 545         iova >>= 12;
 546         do {
 547                 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
 548                         writeq_relaxed(iova, reg);
 549                 else
 550                         writel_relaxed(iova, reg);
 551                 iova += granule >> 12;
 552         } while (size -= granule);
 553 }
 554
 555 /*
 556  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 557  * almost negligible, but the benefit of getting the first one in as far ahead
 558  * of the sync as possible is significant, hence we don't just make this a
 559  * no-op and set .tlb_sync to arm_smmu_inv_context_s2() as you might think.
 560  */
 561 static void arm_smmu_tlb_inv_vmid_nosync(unsigned long iova, size_t size,
 562                                          size_t granule, bool leaf, void *cookie)
 563 {
 564         struct arm_smmu_domain *smmu_domain = cookie;
 565         void __iomem *base = ARM_SMMU_GR0(smmu_domain->smmu);
 566
 567         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 568                 wmb();
 569
 570         writel_relaxed(smmu_domain->cfg.vmid, base + ARM_SMMU_GR0_TLBIVMID);
 571 }
 572
 573 static const struct iommu_gather_ops arm_smmu_s1_tlb_ops = {
 574         .tlb_flush_all  = arm_smmu_tlb_inv_context_s1,
 575         .tlb_add_flush  = arm_smmu_tlb_inv_range_s1,
 576         .tlb_sync       = arm_smmu_tlb_sync_context,
 577 };
 578
 579 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v2 = {
 580         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 581         .tlb_add_flush  = arm_smmu_tlb_inv_range_s2,
 582         .tlb_sync       = arm_smmu_tlb_sync_context,
 583 };
 584
 585 static const struct iommu_gather_ops arm_smmu_s2_tlb_ops_v1 = {
 586         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 587         .tlb_add_flush  = arm_smmu_tlb_inv_vmid_nosync,
 588         .tlb_sync       = arm_smmu_tlb_sync_vmid,
 589 };
 590
 591 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 592 {
 593         u32 fsr, fsynr, cbfrsynra;
 594         unsigned long iova;
 595         struct iommu_domain *domain = dev;
 596         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 597         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 598         struct arm_smmu_device *smmu = smmu_domain->smmu;
 599         void __iomem *cb_base;
 600
 601         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
 602         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 603
 604         if (!(fsr & FSR_FAULT))
 605                 return IRQ_NONE;
 606
 607         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 608         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 609         cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx));
 610
 611         dev_err_ratelimited(smmu->dev,
 612         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
 613                             fsr, iova, fsynr, cbfrsynra, cfg->cbndx);
 614
 615         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 616         return IRQ_HANDLED;
 617 }
 618
 619 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 620 {
 621         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 622         struct arm_smmu_device *smmu = dev;
 623         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 624
 625         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 626         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 627         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 628         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 629
 630         if (!gfsr)
 631                 return IRQ_NONE;
 632
 633         dev_err_ratelimited(smmu->dev,
 634                 "Unexpected global fault, this could be serious\n");
 635         dev_err_ratelimited(smmu->dev,
 636                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 637                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 638
 639         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 640         return IRQ_HANDLED;
 641 }
 642
 643 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 644                                        struct io_pgtable_cfg *pgtbl_cfg)
 645 {
 646         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 647         struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
 648         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 649
 650         cb->cfg = cfg;
 651
 652         /* TCR */
 653         if (stage1) {
 654                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 655                         cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
 656                 } else {
 657                         cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 658                         cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 659                         cb->tcr[1] |= FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM);
 660                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 661                                 cb->tcr[1] |= TCR2_AS;
 662                 }
 663         } else {
 664                 cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 665         }
 666
 667         /* TTBRs */
 668         if (stage1) {
 669                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 670                         cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 671                         cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 672                 } else {
 673                         cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 674                         cb->ttbr[0] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
 675                         cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 676                         cb->ttbr[1] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
 677                 }
 678         } else {
 679                 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 680         }
 681
 682         /* MAIRs (stage-1 only) */
 683         if (stage1) {
 684                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 685                         cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
 686                         cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
 687                 } else {
 688                         cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 689                         cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 690                 }
 691         }
 692 }
 693
 694 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
 695 {
 696         u32 reg;
 697         bool stage1;
 698         struct arm_smmu_cb *cb = &smmu->cbs[idx];
 699         struct arm_smmu_cfg *cfg = cb->cfg;
 700         void __iomem *cb_base;
 701
 702         cb_base = ARM_SMMU_CB(smmu, idx);
 703
 704         /* Unassigned context banks only need disabling */
 705         if (!cfg) {
 706                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
 707                 return;
 708         }
 709
 710         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 711
 712         /* CBA2R */
 713         if (smmu->version > ARM_SMMU_V1) {
 714                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 715                         reg = CBA2R_VA64;
 716                 else
 717                         reg = 0;
 718                 /* 16-bit VMIDs live in CBA2R */
 719                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 720                         reg |= FIELD_PREP(CBA2R_VMID16, cfg->vmid);
 721
 722                 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
 723         }
 724
 725         /* CBAR */
 726         reg = FIELD_PREP(CBAR_TYPE, cfg->cbar);
 727         if (smmu->version < ARM_SMMU_V2)
 728                 reg |= FIELD_PREP(CBAR_IRPTNDX, cfg->irptndx);
 729
 730         /*
 731          * Use the weakest shareability/memory types, so they are
 732          * overridden by the ttbcr/pte.
 733          */
 734         if (stage1) {
 735                 reg |= FIELD_PREP(CBAR_S1_BPSHCFG, CBAR_S1_BPSHCFG_NSH) |
 736                         FIELD_PREP(CBAR_S1_MEMATTR, CBAR_S1_MEMATTR_WB);
 737         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 738                 /* 8-bit VMIDs live in CBAR */
 739                 reg |= FIELD_PREP(CBAR_VMID, cfg->vmid);
 740         }
 741         arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
 742
 743         /*
 744          * TCR
 745          * We must write this before the TTBRs, since it determines the
 746          * access behaviour of some fields (in particular, ASID[15:8]).
 747          */
 748         if (stage1 && smmu->version > ARM_SMMU_V1)
 749                 writel_relaxed(cb->tcr[1], cb_base + ARM_SMMU_CB_TCR2);
 750         writel_relaxed(cb->tcr[0], cb_base + ARM_SMMU_CB_TCR);
 751
 752         /* TTBRs */
 753         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 754                 writel_relaxed(cfg->asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 755                 writel_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
 756                 writel_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
 757         } else {
 758                 writeq_relaxed(cb->ttbr[0], cb_base + ARM_SMMU_CB_TTBR0);
 759                 if (stage1)
 760                         writeq_relaxed(cb->ttbr[1], cb_base + ARM_SMMU_CB_TTBR1);
 761         }
 762
 763         /* MAIRs (stage-1 only) */
 764         if (stage1) {
 765                 writel_relaxed(cb->mair[0], cb_base + ARM_SMMU_CB_S1_MAIR0);
 766                 writel_relaxed(cb->mair[1], cb_base + ARM_SMMU_CB_S1_MAIR1);
 767         }
 768
 769         /* SCTLR */
 770         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 771         if (stage1)
 772                 reg |= SCTLR_S1_ASIDPNE;
 773         if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 774                 reg |= SCTLR_E;
 775
 776         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 777 }
 778
 779 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 780                                         struct arm_smmu_device *smmu)
 781 {
 782         int irq, start, ret = 0;
 783         unsigned long ias, oas;
 784         struct io_pgtable_ops *pgtbl_ops;
 785         struct io_pgtable_cfg pgtbl_cfg;
 786         enum io_pgtable_fmt fmt;
 787         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 788         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 789
 790         mutex_lock(&smmu_domain->init_mutex);
 791         if (smmu_domain->smmu)
 792                 goto out_unlock;
 793
 794         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
 795                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
 796                 smmu_domain->smmu = smmu;
 797                 goto out_unlock;
 798         }
 799
 800         /*
 801          * Mapping the requested stage onto what we support is surprisingly
 802          * complicated, mainly because the spec allows S1+S2 SMMUs without
 803          * support for nested translation. That means we end up with the
 804          * following table:
 805          *
 806          * Requested        Supported        Actual
 807          *     S1               N              S1
 808          *     S1             S1+S2            S1
 809          *     S1               S2             S2
 810          *     S1               S1             S1
 811          *     N                N              N
 812          *     N              S1+S2            S2
 813          *     N                S2             S2
 814          *     N                S1             S1
 815          *
 816          * Note that you can't actually request stage-2 mappings.
 817          */
 818         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 819                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 820         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 821                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 822
 823         /*
 824          * Choosing a suitable context format is even more fiddly. Until we
 825          * grow some way for the caller to express a preference, and/or move
 826          * the decision into the io-pgtable code where it arguably belongs,
 827          * just aim for the closest thing to the rest of the system, and hope
 828          * that the hardware isn't esoteric enough that we can't assume AArch64
 829          * support to be a superset of AArch32 support...
 830          */
 831         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 832                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 833         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 834             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 835             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 836             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 837                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 838         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 839             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 840                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 841                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 842                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 843
 844         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 845                 ret = -EINVAL;
 846                 goto out_unlock;
 847         }
 848
 849         switch (smmu_domain->stage) {
 850         case ARM_SMMU_DOMAIN_S1:
 851                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 852                 start = smmu->num_s2_context_banks;
 853                 ias = smmu->va_size;
 854                 oas = smmu->ipa_size;
 855                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 856                         fmt = ARM_64_LPAE_S1;
 857                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 858                         fmt = ARM_32_LPAE_S1;
 859                         ias = min(ias, 32UL);
 860                         oas = min(oas, 40UL);
 861                 } else {
 862                         fmt = ARM_V7S;
 863                         ias = min(ias, 32UL);
 864                         oas = min(oas, 32UL);
 865                 }
 866                 smmu_domain->tlb_ops = &arm_smmu_s1_tlb_ops;
 867                 break;
 868         case ARM_SMMU_DOMAIN_NESTED:
 869                 /*
 870                  * We will likely want to change this if/when KVM gets
 871                  * involved.
 872                  */
 873         case ARM_SMMU_DOMAIN_S2:
 874                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 875                 start = 0;
 876                 ias = smmu->ipa_size;
 877                 oas = smmu->pa_size;
 878                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 879                         fmt = ARM_64_LPAE_S2;
 880                 } else {
 881                         fmt = ARM_32_LPAE_S2;
 882                         ias = min(ias, 40UL);
 883                         oas = min(oas, 40UL);
 884                 }
 885                 if (smmu->version == ARM_SMMU_V2)
 886                         smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v2;
 887                 else
 888                         smmu_domain->tlb_ops = &arm_smmu_s2_tlb_ops_v1;
 889                 break;
 890         default:
 891                 ret = -EINVAL;
 892                 goto out_unlock;
 893         }
 894         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 895                                       smmu->num_context_banks);
 896         if (ret < 0)
 897                 goto out_unlock;
 898
 899         cfg->cbndx = ret;
 900         if (smmu->version < ARM_SMMU_V2) {
 901                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 902                 cfg->irptndx %= smmu->num_context_irqs;
 903         } else {
 904                 cfg->irptndx = cfg->cbndx;
 905         }
 906
 907         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
 908                 cfg->vmid = cfg->cbndx + 1 + smmu->cavium_id_base;
 909         else
 910                 cfg->asid = cfg->cbndx + smmu->cavium_id_base;
 911
 912         pgtbl_cfg = (struct io_pgtable_cfg) {
 913                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 914                 .ias            = ias,
 915                 .oas            = oas,
 916                 .coherent_walk  = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
 917                 .tlb            = smmu_domain->tlb_ops,
 918                 .iommu_dev      = smmu->dev,
 919         };
 920
 921         if (smmu_domain->non_strict)
 922                 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
 923
 924         smmu_domain->smmu = smmu;
 925         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 926         if (!pgtbl_ops) {
 927                 ret = -ENOMEM;
 928                 goto out_clear_smmu;
 929         }
 930
 931         /* Update the domain's page sizes to reflect the page table format */
 932         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 933         domain->geometry.aperture_end = (1UL << ias) - 1;
 934         domain->geometry.force_aperture = true;
 935
 936         /* Initialise the context bank with our page table cfg */
 937         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 938         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 939
 940         /*
 941          * Request context fault interrupt. Do this last to avoid the
 942          * handler seeing a half-initialised domain state.
 943          */
 944         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 945         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 946                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 947         if (ret < 0) {
 948                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 949                         cfg->irptndx, irq);
 950                 cfg->irptndx = INVALID_IRPTNDX;
 951         }
 952
 953         mutex_unlock(&smmu_domain->init_mutex);
 954
 955         /* Publish page table ops for map/unmap */
 956         smmu_domain->pgtbl_ops = pgtbl_ops;
 957         return 0;
 958
 959 out_clear_smmu:
 960         smmu_domain->smmu = NULL;
 961 out_unlock:
 962         mutex_unlock(&smmu_domain->init_mutex);
 963         return ret;
 964 }
 965
 966 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 967 {
 968         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 969         struct arm_smmu_device *smmu = smmu_domain->smmu;
 970         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 971         int ret, irq;
 972
 973         if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
 974                 return;
 975
 976         ret = arm_smmu_rpm_get(smmu);
 977         if (ret < 0)
 978                 return;
 979
 980         /*
 981          * Disable the context bank and free the page tables before freeing
 982          * it.
 983          */
 984         smmu->cbs[cfg->cbndx].cfg = NULL;
 985         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 986
 987         if (cfg->irptndx != INVALID_IRPTNDX) {
 988                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 989                 devm_free_irq(smmu->dev, irq, domain);
 990         }
 991
 992         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 993         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 994
 995         arm_smmu_rpm_put(smmu);
 996 }
 997
 998 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 999 {
1000         struct arm_smmu_domain *smmu_domain;
1001
1002         if (type != IOMMU_DOMAIN_UNMANAGED &&
1003             type != IOMMU_DOMAIN_DMA &&
1004             type != IOMMU_DOMAIN_IDENTITY)
1005                 return NULL;
1006         /*
1007          * Allocate the domain and initialise some of its data structures.
1008          * We can't really do anything meaningful until we've added a
1009          * master.
1010          */
1011         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1012         if (!smmu_domain)
1013                 return NULL;
1014
1015         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1016             iommu_get_dma_cookie(&smmu_domain->domain))) {
1017                 kfree(smmu_domain);
1018                 return NULL;
1019         }
1020
1021         mutex_init(&smmu_domain->init_mutex);
1022         spin_lock_init(&smmu_domain->cb_lock);
1023
1024         return &smmu_domain->domain;
1025 }
1026
1027 static void arm_smmu_domain_free(struct iommu_domain *domain)
1028 {
1029         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1030
1031         /*
1032          * Free the domain resources. We assume that all devices have
1033          * already been detached.
1034          */
1035         iommu_put_dma_cookie(domain);
1036         arm_smmu_destroy_domain_context(domain);
1037         kfree(smmu_domain);
1038 }
1039
1040 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1041 {
1042         struct arm_smmu_smr *smr = smmu->smrs + idx;
1043         u32 reg = FIELD_PREP(SMR_ID, smr->id) | FIELD_PREP(SMR_MASK, smr->mask);
1044
1045         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
1046                 reg |= SMR_VALID;
1047         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1048 }
1049
1050 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1051 {
1052         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1053         u32 reg = FIELD_PREP(S2CR_TYPE, s2cr->type) |
1054                   FIELD_PREP(S2CR_CBNDX, s2cr->cbndx) |
1055                   FIELD_PREP(S2CR_PRIVCFG, s2cr->privcfg);
1056
1057         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1058             smmu->smrs[idx].valid)
1059                 reg |= S2CR_EXIDVALID;
1060         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1061 }
1062
1063 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1064 {
1065         arm_smmu_write_s2cr(smmu, idx);
1066         if (smmu->smrs)
1067                 arm_smmu_write_smr(smmu, idx);
1068 }
1069
1070 /*
1071  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1072  * should be called after sCR0 is written.
1073  */
1074 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1075 {
1076         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1077         u32 smr;
1078
1079         if (!smmu->smrs)
1080                 return;
1081
1082         /*
1083          * SMR.ID bits may not be preserved if the corresponding MASK
1084          * bits are set, so check each one separately. We can reject
1085          * masters later if they try to claim IDs outside these masks.
1086          */
1087         smr = FIELD_PREP(SMR_ID, smmu->streamid_mask);
1088         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1089         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1090         smmu->streamid_mask = FIELD_GET(SMR_ID, smr);
1091
1092         smr = FIELD_PREP(SMR_MASK, smmu->streamid_mask);
1093         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1094         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1095         smmu->smr_mask_mask = FIELD_GET(SMR_MASK, smr);
1096 }
1097
1098 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1099 {
1100         struct arm_smmu_smr *smrs = smmu->smrs;
1101         int i, free_idx = -ENOSPC;
1102
1103         /* Stream indexing is blissfully easy */
1104         if (!smrs)
1105                 return id;
1106
1107         /* Validating SMRs is... less so */
1108         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1109                 if (!smrs[i].valid) {
1110                         /*
1111                          * Note the first free entry we come across, which
1112                          * we'll claim in the end if nothing else matches.
1113                          */
1114                         if (free_idx < 0)
1115                                 free_idx = i;
1116                         continue;
1117                 }
1118                 /*
1119                  * If the new entry is _entirely_ matched by an existing entry,
1120                  * then reuse that, with the guarantee that there also cannot
1121                  * be any subsequent conflicting entries. In normal use we'd
1122                  * expect simply identical entries for this case, but there's
1123                  * no harm in accommodating the generalisation.
1124                  */
1125                 if ((mask & smrs[i].mask) == mask &&
1126                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1127                         return i;
1128                 /*
1129                  * If the new entry has any other overlap with an existing one,
1130                  * though, then there always exists at least one stream ID
1131                  * which would cause a conflict, and we can't allow that risk.
1132                  */
1133                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1134                         return -EINVAL;
1135         }
1136
1137         return free_idx;
1138 }
1139
1140 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1141 {
1142         if (--smmu->s2crs[idx].count)
1143                 return false;
1144
1145         smmu->s2crs[idx] = s2cr_init_val;
1146         if (smmu->smrs)
1147                 smmu->smrs[idx].valid = false;
1148
1149         return true;
1150 }
1151
1152 static int arm_smmu_master_alloc_smes(struct device *dev)
1153 {
1154         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1155         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1156         struct arm_smmu_device *smmu = cfg->smmu;
1157         struct arm_smmu_smr *smrs = smmu->smrs;
1158         struct iommu_group *group;
1159         int i, idx, ret;
1160
1161         mutex_lock(&smmu->stream_map_mutex);
1162         /* Figure out a viable stream map entry allocation */
1163         for_each_cfg_sme(fwspec, i, idx) {
1164                 u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1165                 u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
1166
1167                 if (idx != INVALID_SMENDX) {
1168                         ret = -EEXIST;
1169                         goto out_err;
1170                 }
1171
1172                 ret = arm_smmu_find_sme(smmu, sid, mask);
1173                 if (ret < 0)
1174                         goto out_err;
1175
1176                 idx = ret;
1177                 if (smrs && smmu->s2crs[idx].count == 0) {
1178                         smrs[idx].id = sid;
1179                         smrs[idx].mask = mask;
1180                         smrs[idx].valid = true;
1181                 }
1182                 smmu->s2crs[idx].count++;
1183                 cfg->smendx[i] = (s16)idx;
1184         }
1185
1186         group = iommu_group_get_for_dev(dev);
1187         if (!group)
1188                 group = ERR_PTR(-ENOMEM);
1189         if (IS_ERR(group)) {
1190                 ret = PTR_ERR(group);
1191                 goto out_err;
1192         }
1193         iommu_group_put(group);
1194
1195         /* It worked! Now, poke the actual hardware */
1196         for_each_cfg_sme(fwspec, i, idx) {
1197                 arm_smmu_write_sme(smmu, idx);
1198                 smmu->s2crs[idx].group = group;
1199         }
1200
1201         mutex_unlock(&smmu->stream_map_mutex);
1202         return 0;
1203
1204 out_err:
1205         while (i--) {
1206                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1207                 cfg->smendx[i] = INVALID_SMENDX;
1208         }
1209         mutex_unlock(&smmu->stream_map_mutex);
1210         return ret;
1211 }
1212
1213 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1214 {
1215         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1216         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1217         int i, idx;
1218
1219         mutex_lock(&smmu->stream_map_mutex);
1220         for_each_cfg_sme(fwspec, i, idx) {
1221                 if (arm_smmu_free_sme(smmu, idx))
1222                         arm_smmu_write_sme(smmu, idx);
1223                 cfg->smendx[i] = INVALID_SMENDX;
1224         }
1225         mutex_unlock(&smmu->stream_map_mutex);
1226 }
1227
1228 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1229                                       struct iommu_fwspec *fwspec)
1230 {
1231         struct arm_smmu_device *smmu = smmu_domain->smmu;
1232         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1233         u8 cbndx = smmu_domain->cfg.cbndx;
1234         enum arm_smmu_s2cr_type type;
1235         int i, idx;
1236
1237         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1238                 type = S2CR_TYPE_BYPASS;
1239         else
1240                 type = S2CR_TYPE_TRANS;
1241
1242         for_each_cfg_sme(fwspec, i, idx) {
1243                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1244                         continue;
1245
1246                 s2cr[idx].type = type;
1247                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1248                 s2cr[idx].cbndx = cbndx;
1249                 arm_smmu_write_s2cr(smmu, idx);
1250         }
1251         return 0;
1252 }
1253
1254 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1255 {
1256         int ret;
1257         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1258         struct arm_smmu_device *smmu;
1259         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1260
1261         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1262                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1263                 return -ENXIO;
1264         }
1265
1266         /*
1267          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1268          * domains between of_xlate() and add_device() - we have no way to cope
1269          * with that, so until ARM gets converted to rely on groups and default
1270          * domains, just say no (but more politely than by dereferencing NULL).
1271          * This should be at least a WARN_ON once that's sorted.
1272          */
1273         if (!fwspec->iommu_priv)
1274                 return -ENODEV;
1275
1276         smmu = fwspec_smmu(fwspec);
1277
1278         ret = arm_smmu_rpm_get(smmu);
1279         if (ret < 0)
1280                 return ret;
1281
1282         /* Ensure that the domain is finalised */
1283         ret = arm_smmu_init_domain_context(domain, smmu);
1284         if (ret < 0)
1285                 goto rpm_put;
1286
1287         /*
1288          * Sanity check the domain. We don't support domains across
1289          * different SMMUs.
1290          */
1291         if (smmu_domain->smmu != smmu) {
1292                 dev_err(dev,
1293                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1294                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1295                 ret = -EINVAL;
1296                 goto rpm_put;
1297         }
1298
1299         /* Looks ok, so add the device to the domain */
1300         ret = arm_smmu_domain_add_master(smmu_domain, fwspec);
1301
1302 rpm_put:
1303         arm_smmu_rpm_put(smmu);
1304         return ret;
1305 }
1306
1307 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1308                         phys_addr_t paddr, size_t size, int prot)
1309 {
1310         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1311         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1312         int ret;
1313
1314         if (!ops)
1315                 return -ENODEV;
1316
1317         arm_smmu_rpm_get(smmu);
1318         ret = ops->map(ops, iova, paddr, size, prot);
1319         arm_smmu_rpm_put(smmu);
1320
1321         return ret;
1322 }
1323
1324 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1325                              size_t size)
1326 {
1327         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1328         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1329         size_t ret;
1330
1331         if (!ops)
1332                 return 0;
1333
1334         arm_smmu_rpm_get(smmu);
1335         ret = ops->unmap(ops, iova, size);
1336         arm_smmu_rpm_put(smmu);
1337
1338         return ret;
1339 }
1340
1341 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1342 {
1343         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1344         struct arm_smmu_device *smmu = smmu_domain->smmu;
1345
1346         if (smmu_domain->tlb_ops) {
1347                 arm_smmu_rpm_get(smmu);
1348                 smmu_domain->tlb_ops->tlb_flush_all(smmu_domain);
1349                 arm_smmu_rpm_put(smmu);
1350         }
1351 }
1352
1353 static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
1354 {
1355         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1356         struct arm_smmu_device *smmu = smmu_domain->smmu;
1357
1358         if (smmu_domain->tlb_ops) {
1359                 arm_smmu_rpm_get(smmu);
1360                 smmu_domain->tlb_ops->tlb_sync(smmu_domain);
1361                 arm_smmu_rpm_put(smmu);
1362         }
1363 }
1364
1365 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1366                                               dma_addr_t iova)
1367 {
1368         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1369         struct arm_smmu_device *smmu = smmu_domain->smmu;
1370         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1371         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1372         struct device *dev = smmu->dev;
1373         void __iomem *cb_base;
1374         u32 tmp;
1375         u64 phys;
1376         unsigned long va, flags;
1377         int ret;
1378
1379         ret = arm_smmu_rpm_get(smmu);
1380         if (ret < 0)
1381                 return 0;
1382
1383         cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
1384
1385         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1386         va = iova & ~0xfffUL;
1387         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1388                 writeq_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1389         else
1390                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1391
1392         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1393                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1394                 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1395                 dev_err(dev,
1396                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1397                         &iova);
1398                 return ops->iova_to_phys(ops, iova);
1399         }
1400
1401         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1402         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1403         if (phys & CB_PAR_F) {
1404                 dev_err(dev, "translation fault!\n");
1405                 dev_err(dev, "PAR = 0x%llx\n", phys);
1406                 return 0;
1407         }
1408
1409         arm_smmu_rpm_put(smmu);
1410
1411         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1412 }
1413
1414 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1415                                         dma_addr_t iova)
1416 {
1417         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1418         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1419
1420         if (domain->type == IOMMU_DOMAIN_IDENTITY)
1421                 return iova;
1422
1423         if (!ops)
1424                 return 0;
1425
1426         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1427                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1428                 return arm_smmu_iova_to_phys_hard(domain, iova);
1429
1430         return ops->iova_to_phys(ops, iova);
1431 }
1432
1433 static bool arm_smmu_capable(enum iommu_cap cap)
1434 {
1435         switch (cap) {
1436         case IOMMU_CAP_CACHE_COHERENCY:
1437                 /*
1438                  * Return true here as the SMMU can always send out coherent
1439                  * requests.
1440                  */
1441                 return true;
1442         case IOMMU_CAP_NOEXEC:
1443                 return true;
1444         default:
1445                 return false;
1446         }
1447 }
1448
1449 static int arm_smmu_match_node(struct device *dev, const void *data)
1450 {
1451         return dev->fwnode == data;
1452 }
1453
1454 static
1455 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1456 {
1457         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1458                                                 fwnode, arm_smmu_match_node);
1459         put_device(dev);
1460         return dev ? dev_get_drvdata(dev) : NULL;
1461 }
1462
1463 static int arm_smmu_add_device(struct device *dev)
1464 {
1465         struct arm_smmu_device *smmu;
1466         struct arm_smmu_master_cfg *cfg;
1467         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1468         int i, ret;
1469
1470         if (using_legacy_binding) {
1471                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1472
1473                 /*
1474                  * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1475                  * will allocate/initialise a new one. Thus we need to update fwspec for
1476                  * later use.
1477                  */
1478                 fwspec = dev_iommu_fwspec_get(dev);
1479                 if (ret)
1480                         goto out_free;
1481         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1482                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1483         } else {
1484                 return -ENODEV;
1485         }
1486
1487         ret = -EINVAL;
1488         for (i = 0; i < fwspec->num_ids; i++) {
1489                 u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1490                 u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
1491
1492                 if (sid & ~smmu->streamid_mask) {
1493                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1494                                 sid, smmu->streamid_mask);
1495                         goto out_free;
1496                 }
1497                 if (mask & ~smmu->smr_mask_mask) {
1498                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1499                                 mask, smmu->smr_mask_mask);
1500                         goto out_free;
1501                 }
1502         }
1503
1504         ret = -ENOMEM;
1505         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1506                       GFP_KERNEL);
1507         if (!cfg)
1508                 goto out_free;
1509
1510         cfg->smmu = smmu;
1511         fwspec->iommu_priv = cfg;
1512         while (i--)
1513                 cfg->smendx[i] = INVALID_SMENDX;
1514
1515         ret = arm_smmu_rpm_get(smmu);
1516         if (ret < 0)
1517                 goto out_cfg_free;
1518
1519         ret = arm_smmu_master_alloc_smes(dev);
1520         arm_smmu_rpm_put(smmu);
1521
1522         if (ret)
1523                 goto out_cfg_free;
1524
1525         iommu_device_link(&smmu->iommu, dev);
1526
1527         device_link_add(dev, smmu->dev,
1528                         DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1529
1530         return 0;
1531
1532 out_cfg_free:
1533         kfree(cfg);
1534 out_free:
1535         iommu_fwspec_free(dev);
1536         return ret;
1537 }
1538
1539 static void arm_smmu_remove_device(struct device *dev)
1540 {
1541         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1542         struct arm_smmu_master_cfg *cfg;
1543         struct arm_smmu_device *smmu;
1544         int ret;
1545
1546         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1547                 return;
1548
1549         cfg  = fwspec->iommu_priv;
1550         smmu = cfg->smmu;
1551
1552         ret = arm_smmu_rpm_get(smmu);
1553         if (ret < 0)
1554                 return;
1555
1556         iommu_device_unlink(&smmu->iommu, dev);
1557         arm_smmu_master_free_smes(fwspec);
1558
1559         arm_smmu_rpm_put(smmu);
1560
1561         iommu_group_remove_device(dev);
1562         kfree(fwspec->iommu_priv);
1563         iommu_fwspec_free(dev);
1564 }
1565
1566 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1567 {
1568         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1569         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1570         struct iommu_group *group = NULL;
1571         int i, idx;
1572
1573         for_each_cfg_sme(fwspec, i, idx) {
1574                 if (group && smmu->s2crs[idx].group &&
1575                     group != smmu->s2crs[idx].group)
1576                         return ERR_PTR(-EINVAL);
1577
1578                 group = smmu->s2crs[idx].group;
1579         }
1580
1581         if (group)
1582                 return iommu_group_ref_get(group);
1583
1584         if (dev_is_pci(dev))
1585                 group = pci_device_group(dev);
1586         else if (dev_is_fsl_mc(dev))
1587                 group = fsl_mc_device_group(dev);
1588         else
1589                 group = generic_device_group(dev);
1590
1591         return group;
1592 }
1593
1594 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1595                                     enum iommu_attr attr, void *data)
1596 {
1597         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1598
1599         switch(domain->type) {
1600         case IOMMU_DOMAIN_UNMANAGED:
1601                 switch (attr) {
1602                 case DOMAIN_ATTR_NESTING:
1603                         *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1604                         return 0;
1605                 default:
1606                         return -ENODEV;
1607                 }
1608                 break;
1609         case IOMMU_DOMAIN_DMA:
1610                 switch (attr) {
1611                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1612                         *(int *)data = smmu_domain->non_strict;
1613                         return 0;
1614                 default:
1615                         return -ENODEV;
1616                 }
1617                 break;
1618         default:
1619                 return -EINVAL;
1620         }
1621 }
1622
1623 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1624                                     enum iommu_attr attr, void *data)
1625 {
1626         int ret = 0;
1627         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1628
1629         mutex_lock(&smmu_domain->init_mutex);
1630
1631         switch(domain->type) {
1632         case IOMMU_DOMAIN_UNMANAGED:
1633                 switch (attr) {
1634                 case DOMAIN_ATTR_NESTING:
1635                         if (smmu_domain->smmu) {
1636                                 ret = -EPERM;
1637                                 goto out_unlock;
1638                         }
1639
1640                         if (*(int *)data)
1641                                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1642                         else
1643                                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1644                         break;
1645                 default:
1646                         ret = -ENODEV;
1647                 }
1648                 break;
1649         case IOMMU_DOMAIN_DMA:
1650                 switch (attr) {
1651                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1652                         smmu_domain->non_strict = *(int *)data;
1653                         break;
1654                 default:
1655                         ret = -ENODEV;
1656                 }
1657                 break;
1658         default:
1659                 ret = -EINVAL;
1660         }
1661 out_unlock:
1662         mutex_unlock(&smmu_domain->init_mutex);
1663         return ret;
1664 }
1665
1666 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1667 {
1668         u32 mask, fwid = 0;
1669
1670         if (args->args_count > 0)
1671                 fwid |= FIELD_PREP(SMR_ID, args->args[0]);
1672
1673         if (args->args_count > 1)
1674                 fwid |= FIELD_PREP(SMR_MASK, args->args[1]);
1675         else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1676                 fwid |= FIELD_PREP(SMR_MASK, mask);
1677
1678         return iommu_fwspec_add_ids(dev, &fwid, 1);
1679 }
1680
1681 static void arm_smmu_get_resv_regions(struct device *dev,
1682                                       struct list_head *head)
1683 {
1684         struct iommu_resv_region *region;
1685         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1686
1687         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1688                                          prot, IOMMU_RESV_SW_MSI);
1689         if (!region)
1690                 return;
1691
1692         list_add_tail(&region->list, head);
1693
1694         iommu_dma_get_resv_regions(dev, head);
1695 }
1696
1697 static void arm_smmu_put_resv_regions(struct device *dev,
1698                                       struct list_head *head)
1699 {
1700         struct iommu_resv_region *entry, *next;
1701
1702         list_for_each_entry_safe(entry, next, head, list)
1703                 kfree(entry);
1704 }
1705
1706 static struct iommu_ops arm_smmu_ops = {
1707         .capable                = arm_smmu_capable,
1708         .domain_alloc           = arm_smmu_domain_alloc,
1709         .domain_free            = arm_smmu_domain_free,
1710         .attach_dev             = arm_smmu_attach_dev,
1711         .map                    = arm_smmu_map,
1712         .unmap                  = arm_smmu_unmap,
1713         .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
1714         .iotlb_sync             = arm_smmu_iotlb_sync,
1715         .iova_to_phys           = arm_smmu_iova_to_phys,
1716         .add_device             = arm_smmu_add_device,
1717         .remove_device          = arm_smmu_remove_device,
1718         .device_group           = arm_smmu_device_group,
1719         .domain_get_attr        = arm_smmu_domain_get_attr,
1720         .domain_set_attr        = arm_smmu_domain_set_attr,
1721         .of_xlate               = arm_smmu_of_xlate,
1722         .get_resv_regions       = arm_smmu_get_resv_regions,
1723         .put_resv_regions       = arm_smmu_put_resv_regions,
1724         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1725 };
1726
1727 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1728 {
1729         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1730         int i;
1731         u32 reg, major;
1732
1733         /* clear global FSR */
1734         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1735         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1736
1737         /*
1738          * Reset stream mapping groups: Initial values mark all SMRn as
1739          * invalid and all S2CRn as bypass unless overridden.
1740          */
1741         for (i = 0; i < smmu->num_mapping_groups; ++i)
1742                 arm_smmu_write_sme(smmu, i);
1743
1744         if (smmu->model == ARM_MMU500) {
1745                 /*
1746                  * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1747                  * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1748                  * bit is only present in MMU-500r2 onwards.
1749                  */
1750                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1751                 major = FIELD_GET(ID7_MAJOR, reg);
1752                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1753                 if (major >= 2)
1754                         reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1755                 /*
1756                  * Allow unmatched Stream IDs to allocate bypass
1757                  * TLB entries for reduced latency.
1758                  */
1759                 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN;
1760                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1761         }
1762
1763         /* Make sure all context banks are disabled and clear CB_FSR  */
1764         for (i = 0; i < smmu->num_context_banks; ++i) {
1765                 void __iomem *cb_base = ARM_SMMU_CB(smmu, i);
1766
1767                 arm_smmu_write_context_bank(smmu, i);
1768                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1769                 /*
1770                  * Disable MMU-500's not-particularly-beneficial next-page
1771                  * prefetcher for the sake of errata #841119 and #826419.
1772                  */
1773                 if (smmu->model == ARM_MMU500) {
1774                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1775                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1776                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1777                 }
1778         }
1779
1780         /* Invalidate the TLB, just in case */
1781         writel_relaxed(QCOM_DUMMY_VAL, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1782         writel_relaxed(QCOM_DUMMY_VAL, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1783
1784         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1785
1786         /* Enable fault reporting */
1787         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1788
1789         /* Disable TLB broadcasting. */
1790         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1791
1792         /* Enable client access, handling unmatched streams as appropriate */
1793         reg &= ~sCR0_CLIENTPD;
1794         if (disable_bypass)
1795                 reg |= sCR0_USFCFG;
1796         else
1797                 reg &= ~sCR0_USFCFG;
1798
1799         /* Disable forced broadcasting */
1800         reg &= ~sCR0_FB;
1801
1802         /* Don't upgrade barriers */
1803         reg &= ~(sCR0_BSU);
1804
1805         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1806                 reg |= sCR0_VMID16EN;
1807
1808         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1809                 reg |= sCR0_EXIDENABLE;
1810
1811         /* Push the button */
1812         arm_smmu_tlb_sync_global(smmu);
1813         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1814 }
1815
1816 static int arm_smmu_id_size_to_bits(int size)
1817 {
1818         switch (size) {
1819         case 0:
1820                 return 32;
1821         case 1:
1822                 return 36;
1823         case 2:
1824                 return 40;
1825         case 3:
1826                 return 42;
1827         case 4:
1828                 return 44;
1829         case 5:
1830         default:
1831                 return 48;
1832         }
1833 }
1834
1835 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1836 {
1837         unsigned int size;
1838         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1839         u32 id;
1840         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1841         int i;
1842
1843         dev_notice(smmu->dev, "probing hardware configuration...\n");
1844         dev_notice(smmu->dev, "SMMUv%d with:\n",
1845                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1846
1847         /* ID0 */
1848         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1849
1850         /* Restrict available stages based on module parameter */
1851         if (force_stage == 1)
1852                 id &= ~(ID0_S2TS | ID0_NTS);
1853         else if (force_stage == 2)
1854                 id &= ~(ID0_S1TS | ID0_NTS);
1855
1856         if (id & ID0_S1TS) {
1857                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1858                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1859         }
1860
1861         if (id & ID0_S2TS) {
1862                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1863                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1864         }
1865
1866         if (id & ID0_NTS) {
1867                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1868                 dev_notice(smmu->dev, "\tnested translation\n");
1869         }
1870
1871         if (!(smmu->features &
1872                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1873                 dev_err(smmu->dev, "\tno translation support!\n");
1874                 return -ENODEV;
1875         }
1876
1877         if ((id & ID0_S1TS) &&
1878                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1879                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1880                 dev_notice(smmu->dev, "\taddress translation ops\n");
1881         }
1882
1883         /*
1884          * In order for DMA API calls to work properly, we must defer to what
1885          * the FW says about coherency, regardless of what the hardware claims.
1886          * Fortunately, this also opens up a workaround for systems where the
1887          * ID register value has ended up configured incorrectly.
1888          */
1889         cttw_reg = !!(id & ID0_CTTW);
1890         if (cttw_fw || cttw_reg)
1891                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1892                            cttw_fw ? "" : "non-");
1893         if (cttw_fw != cttw_reg)
1894                 dev_notice(smmu->dev,
1895                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1896
1897         /* Max. number of entries we have for stream matching/indexing */
1898         if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1899                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1900                 size = 1 << 16;
1901         } else {
1902                 size = 1 << FIELD_GET(ID0_NUMSIDB, id);
1903         }
1904         smmu->streamid_mask = size - 1;
1905         if (id & ID0_SMS) {
1906                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1907                 size = FIELD_GET(ID0_NUMSMRG, id);
1908                 if (size == 0) {
1909                         dev_err(smmu->dev,
1910                                 "stream-matching supported, but no SMRs present!\n");
1911                         return -ENODEV;
1912                 }
1913
1914                 /* Zero-initialised to mark as invalid */
1915                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1916                                           GFP_KERNEL);
1917                 if (!smmu->smrs)
1918                         return -ENOMEM;
1919
1920                 dev_notice(smmu->dev,
1921                            "\tstream matching with %u register groups", size);
1922         }
1923         /* s2cr->type == 0 means translation, so initialise explicitly */
1924         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1925                                          GFP_KERNEL);
1926         if (!smmu->s2crs)
1927                 return -ENOMEM;
1928         for (i = 0; i < size; i++)
1929                 smmu->s2crs[i] = s2cr_init_val;
1930
1931         smmu->num_mapping_groups = size;
1932         mutex_init(&smmu->stream_map_mutex);
1933         spin_lock_init(&smmu->global_sync_lock);
1934
1935         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1936                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1937                 if (!(id & ID0_PTFS_NO_AARCH32S))
1938                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1939         }
1940
1941         /* ID1 */
1942         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1943         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1944
1945         /* Check for size mismatch of SMMU address space from mapped region */
1946         size = 1 << (FIELD_GET(ID1_NUMPAGENDXB, id) + 1);
1947         if (smmu->numpage != 2 * size << smmu->pgshift)
1948                 dev_warn(smmu->dev,
1949                         "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1950                         2 * size << smmu->pgshift, smmu->numpage);
1951         /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1952         smmu->numpage = size;
1953
1954         smmu->num_s2_context_banks = FIELD_GET(ID1_NUMS2CB, id);
1955         smmu->num_context_banks = FIELD_GET(ID1_NUMCB, id);
1956         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1957                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1958                 return -ENODEV;
1959         }
1960         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1961                    smmu->num_context_banks, smmu->num_s2_context_banks);
1962         /*
1963          * Cavium CN88xx erratum #27704.
1964          * Ensure ASID and VMID allocation is unique across all SMMUs in
1965          * the system.
1966          */
1967         if (smmu->model == CAVIUM_SMMUV2) {
1968                 smmu->cavium_id_base =
1969                         atomic_add_return(smmu->num_context_banks,
1970                                           &cavium_smmu_context_count);
1971                 smmu->cavium_id_base -= smmu->num_context_banks;
1972                 dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
1973         }
1974         smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1975                                  sizeof(*smmu->cbs), GFP_KERNEL);
1976         if (!smmu->cbs)
1977                 return -ENOMEM;
1978
1979         /* ID2 */
1980         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1981         size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_IAS, id));
1982         smmu->ipa_size = size;
1983
1984         /* The output mask is also applied for bypass */
1985         size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_OAS, id));
1986         smmu->pa_size = size;
1987
1988         if (id & ID2_VMID16)
1989                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1990
1991         /*
1992          * What the page table walker can address actually depends on which
1993          * descriptor format is in use, but since a) we don't know that yet,
1994          * and b) it can vary per context bank, this will have to do...
1995          */
1996         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1997                 dev_warn(smmu->dev,
1998                          "failed to set DMA mask for table walker\n");
1999
2000         if (smmu->version < ARM_SMMU_V2) {
2001                 smmu->va_size = smmu->ipa_size;
2002                 if (smmu->version == ARM_SMMU_V1_64K)
2003                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
2004         } else {
2005                 size = FIELD_GET(ID2_UBS, id);
2006                 smmu->va_size = arm_smmu_id_size_to_bits(size);
2007                 if (id & ID2_PTFS_4K)
2008                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
2009                 if (id & ID2_PTFS_16K)
2010                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
2011                 if (id & ID2_PTFS_64K)
2012                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
2013         }
2014
2015         /* Now we've corralled the various formats, what'll it do? */
2016         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
2017                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
2018         if (smmu->features &
2019             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
2020                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
2021         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
2022                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
2023         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
2024                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
2025
2026         if (arm_smmu_ops.pgsize_bitmap == -1UL)
2027                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
2028         else
2029                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
2030         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
2031                    smmu->pgsize_bitmap);
2032
2033
2034         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
2035                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
2036                            smmu->va_size, smmu->ipa_size);
2037
2038         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
2039                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
2040                            smmu->ipa_size, smmu->pa_size);
2041
2042         return 0;
2043 }
2044
2045 struct arm_smmu_match_data {
2046         enum arm_smmu_arch_version version;
2047         enum arm_smmu_implementation model;
2048 };
2049
2050 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
2051 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
2052
2053 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
2054 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
2055 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
2056 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
2057 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
2058 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
2059
2060 static const struct of_device_id arm_smmu_of_match[] = {
2061         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
2062         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
2063         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
2064         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
2065         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
2066         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
2067         { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
2068         { },
2069 };
2070
2071 #ifdef CONFIG_ACPI
2072 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
2073 {
2074         int ret = 0;
2075
2076         switch (model) {
2077         case ACPI_IORT_SMMU_V1:
2078         case ACPI_IORT_SMMU_CORELINK_MMU400:
2079                 smmu->version = ARM_SMMU_V1;
2080                 smmu->model = GENERIC_SMMU;
2081                 break;
2082         case ACPI_IORT_SMMU_CORELINK_MMU401:
2083                 smmu->version = ARM_SMMU_V1_64K;
2084                 smmu->model = GENERIC_SMMU;
2085                 break;
2086         case ACPI_IORT_SMMU_V2:
2087                 smmu->version = ARM_SMMU_V2;
2088                 smmu->model = GENERIC_SMMU;
2089                 break;
2090         case ACPI_IORT_SMMU_CORELINK_MMU500:
2091                 smmu->version = ARM_SMMU_V2;
2092                 smmu->model = ARM_MMU500;
2093                 break;
2094         case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
2095                 smmu->version = ARM_SMMU_V2;
2096                 smmu->model = CAVIUM_SMMUV2;
2097                 break;
2098         default:
2099                 ret = -ENODEV;
2100         }
2101
2102         return ret;
2103 }
2104
2105 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2106                                       struct arm_smmu_device *smmu)
2107 {
2108         struct device *dev = smmu->dev;
2109         struct acpi_iort_node *node =
2110                 *(struct acpi_iort_node **)dev_get_platdata(dev);
2111         struct acpi_iort_smmu *iort_smmu;
2112         int ret;
2113
2114         /* Retrieve SMMU1/2 specific data */
2115         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2116
2117         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2118         if (ret < 0)
2119                 return ret;
2120
2121         /* Ignore the configuration access interrupt */
2122         smmu->num_global_irqs = 1;
2123
2124         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2125                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2126
2127         return 0;
2128 }
2129 #else
2130 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
2131                                              struct arm_smmu_device *smmu)
2132 {
2133         return -ENODEV;
2134 }
2135 #endif
2136
2137 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2138                                     struct arm_smmu_device *smmu)
2139 {
2140         const struct arm_smmu_match_data *data;
2141         struct device *dev = &pdev->dev;
2142         bool legacy_binding;
2143
2144         if (of_property_read_u32(dev->of_node, "#global-interrupts",
2145                                  &smmu->num_global_irqs)) {
2146                 dev_err(dev, "missing #global-interrupts property\n");
2147                 return -ENODEV;
2148         }
2149
2150         data = of_device_get_match_data(dev);
2151         smmu->version = data->version;
2152         smmu->model = data->model;
2153
2154         parse_driver_options(smmu);
2155
2156         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2157         if (legacy_binding && !using_generic_binding) {
2158                 if (!using_legacy_binding)
2159                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2160                 using_legacy_binding = true;
2161         } else if (!legacy_binding && !using_legacy_binding) {
2162                 using_generic_binding = true;
2163         } else {
2164                 dev_err(dev, "not probing due to mismatched DT properties\n");
2165                 return -ENODEV;
2166         }
2167
2168         if (of_dma_is_coherent(dev->of_node))
2169                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2170
2171         return 0;
2172 }
2173
2174 static void arm_smmu_bus_init(void)
2175 {
2176         /* Oh, for a proper bus abstraction */
2177         if (!iommu_present(&platform_bus_type))
2178                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2179 #ifdef CONFIG_ARM_AMBA
2180         if (!iommu_present(&amba_bustype))
2181                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2182 #endif
2183 #ifdef CONFIG_PCI
2184         if (!iommu_present(&pci_bus_type)) {
2185                 pci_request_acs();
2186                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2187         }
2188 #endif
2189 #ifdef CONFIG_FSL_MC_BUS
2190         if (!iommu_present(&fsl_mc_bus_type))
2191                 bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
2192 #endif
2193 }
2194
2195 static int arm_smmu_device_probe(struct platform_device *pdev)
2196 {
2197         struct resource *res;
2198         resource_size_t ioaddr;
2199         struct arm_smmu_device *smmu;
2200         struct device *dev = &pdev->dev;
2201         int num_irqs, i, err;
2202
2203         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2204         if (!smmu) {
2205                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2206                 return -ENOMEM;
2207         }
2208         smmu->dev = dev;
2209
2210         if (dev->of_node)
2211                 err = arm_smmu_device_dt_probe(pdev, smmu);
2212         else
2213                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2214
2215         if (err)
2216                 return err;
2217
2218         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2219         ioaddr = res->start;
2220         smmu->base = devm_ioremap_resource(dev, res);
2221         if (IS_ERR(smmu->base))
2222                 return PTR_ERR(smmu->base);
2223         /*
2224          * The resource size should effectively match the value of SMMU_TOP;
2225          * stash that temporarily until we know PAGESIZE to validate it with.
2226          */
2227         smmu->numpage = resource_size(res);
2228
2229         num_irqs = 0;
2230         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2231                 num_irqs++;
2232                 if (num_irqs > smmu->num_global_irqs)
2233                         smmu->num_context_irqs++;
2234         }
2235
2236         if (!smmu->num_context_irqs) {
2237                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2238                         num_irqs, smmu->num_global_irqs + 1);
2239                 return -ENODEV;
2240         }
2241
2242         smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2243                                   GFP_KERNEL);
2244         if (!smmu->irqs) {
2245                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2246                 return -ENOMEM;
2247         }
2248
2249         for (i = 0; i < num_irqs; ++i) {
2250                 int irq = platform_get_irq(pdev, i);
2251
2252                 if (irq < 0) {
2253                         dev_err(dev, "failed to get irq index %d\n", i);
2254                         return -ENODEV;
2255                 }
2256                 smmu->irqs[i] = irq;
2257         }
2258
2259         err = devm_clk_bulk_get_all(dev, &smmu->clks);
2260         if (err < 0) {
2261                 dev_err(dev, "failed to get clocks %d\n", err);
2262                 return err;
2263         }
2264         smmu->num_clks = err;
2265
2266         err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2267         if (err)
2268                 return err;
2269
2270         err = arm_smmu_device_cfg_probe(smmu);
2271         if (err)
2272                 return err;
2273
2274         if (smmu->version == ARM_SMMU_V2) {
2275                 if (smmu->num_context_banks > smmu->num_context_irqs) {
2276                         dev_err(dev,
2277                               "found only %d context irq(s) but %d required\n",
2278                               smmu->num_context_irqs, smmu->num_context_banks);
2279                         return -ENODEV;
2280                 }
2281
2282                 /* Ignore superfluous interrupts */
2283                 smmu->num_context_irqs = smmu->num_context_banks;
2284         }
2285
2286         for (i = 0; i < smmu->num_global_irqs; ++i) {
2287                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2288                                        arm_smmu_global_fault,
2289                                        IRQF_SHARED,
2290                                        "arm-smmu global fault",
2291                                        smmu);
2292                 if (err) {
2293                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2294                                 i, smmu->irqs[i]);
2295                         return err;
2296                 }
2297         }
2298
2299         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2300                                      "smmu.%pa", &ioaddr);
2301         if (err) {
2302                 dev_err(dev, "Failed to register iommu in sysfs\n");
2303                 return err;
2304         }
2305
2306         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2307         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2308
2309         err = iommu_device_register(&smmu->iommu);
2310         if (err) {
2311                 dev_err(dev, "Failed to register iommu\n");
2312                 return err;
2313         }
2314
2315         platform_set_drvdata(pdev, smmu);
2316         arm_smmu_device_reset(smmu);
2317         arm_smmu_test_smr_masks(smmu);
2318
2319         /*
2320          * We want to avoid touching dev->power.lock in fastpaths unless
2321          * it's really going to do something useful - pm_runtime_enabled()
2322          * can serve as an ideal proxy for that decision. So, conditionally
2323          * enable pm_runtime.
2324          */
2325         if (dev->pm_domain) {
2326                 pm_runtime_set_active(dev);
2327                 pm_runtime_enable(dev);
2328         }
2329
2330         /*
2331          * For ACPI and generic DT bindings, an SMMU will be probed before
2332          * any device which might need it, so we want the bus ops in place
2333          * ready to handle default domain setup as soon as any SMMU exists.
2334          */
2335         if (!using_legacy_binding)
2336                 arm_smmu_bus_init();
2337
2338         return 0;
2339 }
2340
2341 /*
2342  * With the legacy DT binding in play, though, we have no guarantees about
2343  * probe order, but then we're also not doing default domains, so we can
2344  * delay setting bus ops until we're sure every possible SMMU is ready,
2345  * and that way ensure that no add_device() calls get missed.
2346  */
2347 static int arm_smmu_legacy_bus_init(void)
2348 {
2349         if (using_legacy_binding)
2350                 arm_smmu_bus_init();
2351         return 0;
2352 }
2353 device_initcall_sync(arm_smmu_legacy_bus_init);
2354
2355 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2356 {
2357         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2358
2359         if (!smmu)
2360                 return;
2361
2362         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2363                 dev_err(&pdev->dev, "removing device with active domains!\n");
2364
2365         arm_smmu_rpm_get(smmu);
2366         /* Turn the thing off */
2367         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2368         arm_smmu_rpm_put(smmu);
2369
2370         if (pm_runtime_enabled(smmu->dev))
2371                 pm_runtime_force_suspend(smmu->dev);
2372         else
2373                 clk_bulk_disable(smmu->num_clks, smmu->clks);
2374
2375         clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2376 }
2377
2378 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2379 {
2380         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2381         int ret;
2382
2383         ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2384         if (ret)
2385                 return ret;
2386
2387         arm_smmu_device_reset(smmu);
2388
2389         return 0;
2390 }
2391
2392 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2393 {
2394         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2395
2396         clk_bulk_disable(smmu->num_clks, smmu->clks);
2397
2398         return 0;
2399 }
2400
2401 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2402 {
2403         if (pm_runtime_suspended(dev))
2404                 return 0;
2405
2406         return arm_smmu_runtime_resume(dev);
2407 }
2408
2409 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2410 {
2411         if (pm_runtime_suspended(dev))
2412                 return 0;
2413
2414         return arm_smmu_runtime_suspend(dev);
2415 }
2416
2417 static const struct dev_pm_ops arm_smmu_pm_ops = {
2418         SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2419         SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2420                            arm_smmu_runtime_resume, NULL)
2421 };
2422
2423 static struct platform_driver arm_smmu_driver = {
2424         .driver = {
2425                 .name                   = "arm-smmu",
2426                 .of_match_table         = of_match_ptr(arm_smmu_of_match),
2427                 .pm                     = &arm_smmu_pm_ops,
2428                 .suppress_bind_attrs    = true,
2429         },
2430         .probe  = arm_smmu_device_probe,
2431         .shutdown = arm_smmu_device_shutdown,
2432 };
2433 builtin_platform_driver(arm_smmu_driver);