drivers/iommu/arm-smmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * IOMMU API for ARM architected SMMU implementations.
   4  *
   5  * Copyright (C) 2013 ARM Limited
   6  *
   7  * Author: Will Deacon <will.deacon@arm.com>
   8  *
   9  * This driver currently supports:
  10  *      - SMMUv1 and v2 implementations
  11  *      - Stream-matching and stream-indexing
  12  *      - v7/v8 long-descriptor format
  13  *      - Non-secure access to the SMMU
  14  *      - Context fault reporting
  15  *      - Extended Stream ID (16 bit)
  16  */
  17
  18 #define pr_fmt(fmt) "arm-smmu: " fmt
  19
  20 #include <linux/acpi.h>
  21 #include <linux/acpi_iort.h>
  22 #include <linux/bitfield.h>
  23 #include <linux/delay.h>
  24 #include <linux/dma-iommu.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/err.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/io.h>
  29 #include <linux/iopoll.h>
  30 #include <linux/init.h>
  31 #include <linux/moduleparam.h>
  32 #include <linux/of.h>
  33 #include <linux/of_address.h>
  34 #include <linux/of_device.h>
  35 #include <linux/of_iommu.h>
  36 #include <linux/pci.h>
  37 #include <linux/platform_device.h>
  38 #include <linux/pm_runtime.h>
  39 #include <linux/ratelimit.h>
  40 #include <linux/slab.h>
  41
  42 #include <linux/amba/bus.h>
  43 #include <linux/fsl/mc.h>
  44
  45 #include "arm-smmu.h"
  46
  47 /*
  48  * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
  49  * global register space are still, in fact, using a hypervisor to mediate it
  50  * by trapping and emulating register accesses. Sadly, some deployed versions
  51  * of said trapping code have bugs wherein they go horribly wrong for stores
  52  * using r31 (i.e. XZR/WZR) as the source register.
  53  */
  54 #define QCOM_DUMMY_VAL -1
  55
  56 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
  57 #define TLB_SPIN_COUNT                  10
  58
  59 #define MSI_IOVA_BASE                   0x8000000
  60 #define MSI_IOVA_LENGTH                 0x100000
  61
  62 static int force_stage;
  63 /*
  64  * not really modular, but the easiest way to keep compat with existing
  65  * bootargs behaviour is to continue using module_param() here.
  66  */
  67 module_param(force_stage, int, S_IRUGO);
  68 MODULE_PARM_DESC(force_stage,
  69         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
  70 static bool disable_bypass =
  71         IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
  72 module_param(disable_bypass, bool, S_IRUGO);
  73 MODULE_PARM_DESC(disable_bypass,
  74         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
  75
  76 struct arm_smmu_s2cr {
  77         struct iommu_group              *group;
  78         int                             count;
  79         enum arm_smmu_s2cr_type         type;
  80         enum arm_smmu_s2cr_privcfg      privcfg;
  81         u8                              cbndx;
  82 };
  83
  84 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
  85         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
  86 }
  87
  88 struct arm_smmu_smr {
  89         u16                             mask;
  90         u16                             id;
  91         bool                            valid;
  92 };
  93
  94 struct arm_smmu_cb {
  95         u64                             ttbr[2];
  96         u32                             tcr[2];
  97         u32                             mair[2];
  98         struct arm_smmu_cfg             *cfg;
  99 };
 100
 101 struct arm_smmu_master_cfg {
 102         struct arm_smmu_device          *smmu;
 103         s16                             smendx[];
 104 };
 105 #define INVALID_SMENDX                  -1
 106 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 107 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 108 #define fwspec_smendx(fw, i) \
 109         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 110 #define for_each_cfg_sme(fw, i, idx) \
 111         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 112
 113 static bool using_legacy_binding, using_generic_binding;
 114
 115 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
 116 {
 117         if (pm_runtime_enabled(smmu->dev))
 118                 return pm_runtime_get_sync(smmu->dev);
 119
 120         return 0;
 121 }
 122
 123 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
 124 {
 125         if (pm_runtime_enabled(smmu->dev))
 126                 pm_runtime_put(smmu->dev);
 127 }
 128
 129 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 130 {
 131         return container_of(dom, struct arm_smmu_domain, domain);
 132 }
 133
 134 static struct device_node *dev_get_dev_node(struct device *dev)
 135 {
 136         if (dev_is_pci(dev)) {
 137                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 138
 139                 while (!pci_is_root_bus(bus))
 140                         bus = bus->parent;
 141                 return of_node_get(bus->bridge->parent->of_node);
 142         }
 143
 144         return of_node_get(dev->of_node);
 145 }
 146
 147 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 148 {
 149         *((__be32 *)data) = cpu_to_be32(alias);
 150         return 0; /* Continue walking */
 151 }
 152
 153 static int __find_legacy_master_phandle(struct device *dev, void *data)
 154 {
 155         struct of_phandle_iterator *it = *(void **)data;
 156         struct device_node *np = it->node;
 157         int err;
 158
 159         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 160                             "#stream-id-cells", -1)
 161                 if (it->node == np) {
 162                         *(void **)data = dev;
 163                         return 1;
 164                 }
 165         it->node = np;
 166         return err == -ENOENT ? 0 : err;
 167 }
 168
 169 static struct platform_driver arm_smmu_driver;
 170 static struct iommu_ops arm_smmu_ops;
 171
 172 static int arm_smmu_register_legacy_master(struct device *dev,
 173                                            struct arm_smmu_device **smmu)
 174 {
 175         struct device *smmu_dev;
 176         struct device_node *np;
 177         struct of_phandle_iterator it;
 178         void *data = &it;
 179         u32 *sids;
 180         __be32 pci_sid;
 181         int err;
 182
 183         np = dev_get_dev_node(dev);
 184         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 185                 of_node_put(np);
 186                 return -ENODEV;
 187         }
 188
 189         it.node = np;
 190         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 191                                      __find_legacy_master_phandle);
 192         smmu_dev = data;
 193         of_node_put(np);
 194         if (err == 0)
 195                 return -ENODEV;
 196         if (err < 0)
 197                 return err;
 198
 199         if (dev_is_pci(dev)) {
 200                 /* "mmu-masters" assumes Stream ID == Requester ID */
 201                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 202                                        &pci_sid);
 203                 it.cur = &pci_sid;
 204                 it.cur_count = 1;
 205         }
 206
 207         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 208                                 &arm_smmu_ops);
 209         if (err)
 210                 return err;
 211
 212         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 213         if (!sids)
 214                 return -ENOMEM;
 215
 216         *smmu = dev_get_drvdata(smmu_dev);
 217         of_phandle_iterator_args(&it, sids, it.cur_count);
 218         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 219         kfree(sids);
 220         return err;
 221 }
 222
 223 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 224 {
 225         int idx;
 226
 227         do {
 228                 idx = find_next_zero_bit(map, end, start);
 229                 if (idx == end)
 230                         return -ENOSPC;
 231         } while (test_and_set_bit(idx, map));
 232
 233         return idx;
 234 }
 235
 236 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 237 {
 238         clear_bit(idx, map);
 239 }
 240
 241 /* Wait for any pending TLB invalidations to complete */
 242 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
 243                                 int sync, int status)
 244 {
 245         unsigned int spin_cnt, delay;
 246         u32 reg;
 247
 248         if (smmu->impl && unlikely(smmu->impl->tlb_sync))
 249                 return smmu->impl->tlb_sync(smmu, page, sync, status);
 250
 251         arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
 252         for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
 253                 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
 254                         reg = arm_smmu_readl(smmu, page, status);
 255                         if (!(reg & sTLBGSTATUS_GSACTIVE))
 256                                 return;
 257                         cpu_relax();
 258                 }
 259                 udelay(delay);
 260         }
 261         dev_err_ratelimited(smmu->dev,
 262                             "TLB sync timed out -- SMMU may be deadlocked\n");
 263 }
 264
 265 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
 266 {
 267         unsigned long flags;
 268
 269         spin_lock_irqsave(&smmu->global_sync_lock, flags);
 270         __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
 271                             ARM_SMMU_GR0_sTLBGSTATUS);
 272         spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
 273 }
 274
 275 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
 276 {
 277         struct arm_smmu_device *smmu = smmu_domain->smmu;
 278         unsigned long flags;
 279
 280         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
 281         __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
 282                             ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
 283         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
 284 }
 285
 286 static void arm_smmu_tlb_inv_context_s1(void *cookie)
 287 {
 288         struct arm_smmu_domain *smmu_domain = cookie;
 289         /*
 290          * The TLBI write may be relaxed, so ensure that PTEs cleared by the
 291          * current CPU are visible beforehand.
 292          */
 293         wmb();
 294         arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
 295                           ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
 296         arm_smmu_tlb_sync_context(smmu_domain);
 297 }
 298
 299 static void arm_smmu_tlb_inv_context_s2(void *cookie)
 300 {
 301         struct arm_smmu_domain *smmu_domain = cookie;
 302         struct arm_smmu_device *smmu = smmu_domain->smmu;
 303
 304         /* See above */
 305         wmb();
 306         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 307         arm_smmu_tlb_sync_global(smmu);
 308 }
 309
 310 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
 311                                       size_t granule, void *cookie, int reg)
 312 {
 313         struct arm_smmu_domain *smmu_domain = cookie;
 314         struct arm_smmu_device *smmu = smmu_domain->smmu;
 315         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 316         int idx = cfg->cbndx;
 317
 318         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 319                 wmb();
 320
 321         if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 322                 iova = (iova >> 12) << 12;
 323                 iova |= cfg->asid;
 324                 do {
 325                         arm_smmu_cb_write(smmu, idx, reg, iova);
 326                         iova += granule;
 327                 } while (size -= granule);
 328         } else {
 329                 iova >>= 12;
 330                 iova |= (u64)cfg->asid << 48;
 331                 do {
 332                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 333                         iova += granule >> 12;
 334                 } while (size -= granule);
 335         }
 336 }
 337
 338 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
 339                                       size_t granule, void *cookie, int reg)
 340 {
 341         struct arm_smmu_domain *smmu_domain = cookie;
 342         struct arm_smmu_device *smmu = smmu_domain->smmu;
 343         int idx = smmu_domain->cfg.cbndx;
 344
 345         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 346                 wmb();
 347
 348         iova >>= 12;
 349         do {
 350                 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
 351                         arm_smmu_cb_writeq(smmu, idx, reg, iova);
 352                 else
 353                         arm_smmu_cb_write(smmu, idx, reg, iova);
 354                 iova += granule >> 12;
 355         } while (size -= granule);
 356 }
 357
 358 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
 359                                      size_t granule, void *cookie)
 360 {
 361         arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
 362                                   ARM_SMMU_CB_S1_TLBIVA);
 363         arm_smmu_tlb_sync_context(cookie);
 364 }
 365
 366 static void arm_smmu_tlb_inv_leaf_s1(unsigned long iova, size_t size,
 367                                      size_t granule, void *cookie)
 368 {
 369         arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
 370                                   ARM_SMMU_CB_S1_TLBIVAL);
 371         arm_smmu_tlb_sync_context(cookie);
 372 }
 373
 374 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
 375                                      unsigned long iova, size_t granule,
 376                                      void *cookie)
 377 {
 378         arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
 379                                   ARM_SMMU_CB_S1_TLBIVAL);
 380 }
 381
 382 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
 383                                      size_t granule, void *cookie)
 384 {
 385         arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
 386                                   ARM_SMMU_CB_S2_TLBIIPAS2);
 387         arm_smmu_tlb_sync_context(cookie);
 388 }
 389
 390 static void arm_smmu_tlb_inv_leaf_s2(unsigned long iova, size_t size,
 391                                      size_t granule, void *cookie)
 392 {
 393         arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
 394                                   ARM_SMMU_CB_S2_TLBIIPAS2L);
 395         arm_smmu_tlb_sync_context(cookie);
 396 }
 397
 398 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
 399                                      unsigned long iova, size_t granule,
 400                                      void *cookie)
 401 {
 402         arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
 403                                   ARM_SMMU_CB_S2_TLBIIPAS2L);
 404 }
 405
 406 static void arm_smmu_tlb_inv_any_s2_v1(unsigned long iova, size_t size,
 407                                        size_t granule, void *cookie)
 408 {
 409         arm_smmu_tlb_inv_context_s2(cookie);
 410 }
 411 /*
 412  * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
 413  * almost negligible, but the benefit of getting the first one in as far ahead
 414  * of the sync as possible is significant, hence we don't just make this a
 415  * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
 416  * think.
 417  */
 418 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
 419                                         unsigned long iova, size_t granule,
 420                                         void *cookie)
 421 {
 422         struct arm_smmu_domain *smmu_domain = cookie;
 423         struct arm_smmu_device *smmu = smmu_domain->smmu;
 424
 425         if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
 426                 wmb();
 427
 428         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
 429 }
 430
 431 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
 432         .tlb_flush_all  = arm_smmu_tlb_inv_context_s1,
 433         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
 434         .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s1,
 435         .tlb_add_page   = arm_smmu_tlb_add_page_s1,
 436 };
 437
 438 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
 439         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 440         .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
 441         .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s2,
 442         .tlb_add_page   = arm_smmu_tlb_add_page_s2,
 443 };
 444
 445 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
 446         .tlb_flush_all  = arm_smmu_tlb_inv_context_s2,
 447         .tlb_flush_walk = arm_smmu_tlb_inv_any_s2_v1,
 448         .tlb_flush_leaf = arm_smmu_tlb_inv_any_s2_v1,
 449         .tlb_add_page   = arm_smmu_tlb_add_page_s2_v1,
 450 };
 451
 452 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 453 {
 454         u32 fsr, fsynr, cbfrsynra;
 455         unsigned long iova;
 456         struct iommu_domain *domain = dev;
 457         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 458         struct arm_smmu_device *smmu = smmu_domain->smmu;
 459         int idx = smmu_domain->cfg.cbndx;
 460
 461         fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
 462         if (!(fsr & FSR_FAULT))
 463                 return IRQ_NONE;
 464
 465         fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
 466         iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
 467         cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
 468
 469         dev_err_ratelimited(smmu->dev,
 470         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
 471                             fsr, iova, fsynr, cbfrsynra, idx);
 472
 473         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
 474         return IRQ_HANDLED;
 475 }
 476
 477 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 478 {
 479         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 480         struct arm_smmu_device *smmu = dev;
 481         static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 482                                       DEFAULT_RATELIMIT_BURST);
 483
 484         gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
 485         gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
 486         gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
 487         gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
 488
 489         if (!gfsr)
 490                 return IRQ_NONE;
 491
 492         if (__ratelimit(&rs)) {
 493                 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
 494                     (gfsr & sGFSR_USF))
 495                         dev_err(smmu->dev,
 496                                 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
 497                                 (u16)gfsynr1);
 498                 else
 499                         dev_err(smmu->dev,
 500                                 "Unexpected global fault, this could be serious\n");
 501                 dev_err(smmu->dev,
 502                         "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 503                         gfsr, gfsynr0, gfsynr1, gfsynr2);
 504         }
 505
 506         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
 507         return IRQ_HANDLED;
 508 }
 509
 510 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 511                                        struct io_pgtable_cfg *pgtbl_cfg)
 512 {
 513         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 514         struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
 515         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 516
 517         cb->cfg = cfg;
 518
 519         /* TCR */
 520         if (stage1) {
 521                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 522                         cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
 523                 } else {
 524                         cb->tcr[0] = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 525                         cb->tcr[1] = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 526                         cb->tcr[1] |= FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM);
 527                         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 528                                 cb->tcr[1] |= TCR2_AS;
 529                 }
 530         } else {
 531                 cb->tcr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 532         }
 533
 534         /* TTBRs */
 535         if (stage1) {
 536                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 537                         cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 538                         cb->ttbr[1] = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 539                 } else {
 540                         cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 541                         cb->ttbr[0] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
 542                         cb->ttbr[1] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 543                         cb->ttbr[1] |= FIELD_PREP(TTBRn_ASID, cfg->asid);
 544                 }
 545         } else {
 546                 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 547         }
 548
 549         /* MAIRs (stage-1 only) */
 550         if (stage1) {
 551                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 552                         cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
 553                         cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
 554                 } else {
 555                         cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 556                         cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 557                 }
 558         }
 559 }
 560
 561 static void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
 562 {
 563         u32 reg;
 564         bool stage1;
 565         struct arm_smmu_cb *cb = &smmu->cbs[idx];
 566         struct arm_smmu_cfg *cfg = cb->cfg;
 567
 568         /* Unassigned context banks only need disabling */
 569         if (!cfg) {
 570                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
 571                 return;
 572         }
 573
 574         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 575
 576         /* CBA2R */
 577         if (smmu->version > ARM_SMMU_V1) {
 578                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 579                         reg = CBA2R_VA64;
 580                 else
 581                         reg = 0;
 582                 /* 16-bit VMIDs live in CBA2R */
 583                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 584                         reg |= FIELD_PREP(CBA2R_VMID16, cfg->vmid);
 585
 586                 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
 587         }
 588
 589         /* CBAR */
 590         reg = FIELD_PREP(CBAR_TYPE, cfg->cbar);
 591         if (smmu->version < ARM_SMMU_V2)
 592                 reg |= FIELD_PREP(CBAR_IRPTNDX, cfg->irptndx);
 593
 594         /*
 595          * Use the weakest shareability/memory types, so they are
 596          * overridden by the ttbcr/pte.
 597          */
 598         if (stage1) {
 599                 reg |= FIELD_PREP(CBAR_S1_BPSHCFG, CBAR_S1_BPSHCFG_NSH) |
 600                         FIELD_PREP(CBAR_S1_MEMATTR, CBAR_S1_MEMATTR_WB);
 601         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 602                 /* 8-bit VMIDs live in CBAR */
 603                 reg |= FIELD_PREP(CBAR_VMID, cfg->vmid);
 604         }
 605         arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
 606
 607         /*
 608          * TCR
 609          * We must write this before the TTBRs, since it determines the
 610          * access behaviour of some fields (in particular, ASID[15:8]).
 611          */
 612         if (stage1 && smmu->version > ARM_SMMU_V1)
 613                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
 614         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
 615
 616         /* TTBRs */
 617         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 618                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
 619                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 620                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
 621         } else {
 622                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
 623                 if (stage1)
 624                         arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
 625                                            cb->ttbr[1]);
 626         }
 627
 628         /* MAIRs (stage-1 only) */
 629         if (stage1) {
 630                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
 631                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
 632         }
 633
 634         /* SCTLR */
 635         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 636         if (stage1)
 637                 reg |= SCTLR_S1_ASIDPNE;
 638         if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
 639                 reg |= SCTLR_E;
 640
 641         arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
 642 }
 643
 644 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 645                                         struct arm_smmu_device *smmu)
 646 {
 647         int irq, start, ret = 0;
 648         unsigned long ias, oas;
 649         struct io_pgtable_ops *pgtbl_ops;
 650         struct io_pgtable_cfg pgtbl_cfg;
 651         enum io_pgtable_fmt fmt;
 652         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 653         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 654
 655         mutex_lock(&smmu_domain->init_mutex);
 656         if (smmu_domain->smmu)
 657                 goto out_unlock;
 658
 659         if (domain->type == IOMMU_DOMAIN_IDENTITY) {
 660                 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
 661                 smmu_domain->smmu = smmu;
 662                 goto out_unlock;
 663         }
 664
 665         /*
 666          * Mapping the requested stage onto what we support is surprisingly
 667          * complicated, mainly because the spec allows S1+S2 SMMUs without
 668          * support for nested translation. That means we end up with the
 669          * following table:
 670          *
 671          * Requested        Supported        Actual
 672          *     S1               N              S1
 673          *     S1             S1+S2            S1
 674          *     S1               S2             S2
 675          *     S1               S1             S1
 676          *     N                N              N
 677          *     N              S1+S2            S2
 678          *     N                S2             S2
 679          *     N                S1             S1
 680          *
 681          * Note that you can't actually request stage-2 mappings.
 682          */
 683         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 684                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 685         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 686                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 687
 688         /*
 689          * Choosing a suitable context format is even more fiddly. Until we
 690          * grow some way for the caller to express a preference, and/or move
 691          * the decision into the io-pgtable code where it arguably belongs,
 692          * just aim for the closest thing to the rest of the system, and hope
 693          * that the hardware isn't esoteric enough that we can't assume AArch64
 694          * support to be a superset of AArch32 support...
 695          */
 696         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 697                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 698         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 699             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 700             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 701             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 702                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 703         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 704             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 705                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 706                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 707                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 708
 709         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 710                 ret = -EINVAL;
 711                 goto out_unlock;
 712         }
 713
 714         switch (smmu_domain->stage) {
 715         case ARM_SMMU_DOMAIN_S1:
 716                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 717                 start = smmu->num_s2_context_banks;
 718                 ias = smmu->va_size;
 719                 oas = smmu->ipa_size;
 720                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 721                         fmt = ARM_64_LPAE_S1;
 722                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 723                         fmt = ARM_32_LPAE_S1;
 724                         ias = min(ias, 32UL);
 725                         oas = min(oas, 40UL);
 726                 } else {
 727                         fmt = ARM_V7S;
 728                         ias = min(ias, 32UL);
 729                         oas = min(oas, 32UL);
 730                 }
 731                 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
 732                 break;
 733         case ARM_SMMU_DOMAIN_NESTED:
 734                 /*
 735                  * We will likely want to change this if/when KVM gets
 736                  * involved.
 737                  */
 738         case ARM_SMMU_DOMAIN_S2:
 739                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 740                 start = 0;
 741                 ias = smmu->ipa_size;
 742                 oas = smmu->pa_size;
 743                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 744                         fmt = ARM_64_LPAE_S2;
 745                 } else {
 746                         fmt = ARM_32_LPAE_S2;
 747                         ias = min(ias, 40UL);
 748                         oas = min(oas, 40UL);
 749                 }
 750                 if (smmu->version == ARM_SMMU_V2)
 751                         smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
 752                 else
 753                         smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
 754                 break;
 755         default:
 756                 ret = -EINVAL;
 757                 goto out_unlock;
 758         }
 759         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 760                                       smmu->num_context_banks);
 761         if (ret < 0)
 762                 goto out_unlock;
 763
 764         cfg->cbndx = ret;
 765         if (smmu->version < ARM_SMMU_V2) {
 766                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 767                 cfg->irptndx %= smmu->num_context_irqs;
 768         } else {
 769                 cfg->irptndx = cfg->cbndx;
 770         }
 771
 772         if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
 773                 cfg->vmid = cfg->cbndx + 1;
 774         else
 775                 cfg->asid = cfg->cbndx;
 776
 777         smmu_domain->smmu = smmu;
 778         if (smmu->impl && smmu->impl->init_context) {
 779                 ret = smmu->impl->init_context(smmu_domain);
 780                 if (ret)
 781                         goto out_unlock;
 782         }
 783
 784         pgtbl_cfg = (struct io_pgtable_cfg) {
 785                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 786                 .ias            = ias,
 787                 .oas            = oas,
 788                 .coherent_walk  = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
 789                 .tlb            = smmu_domain->flush_ops,
 790                 .iommu_dev      = smmu->dev,
 791         };
 792
 793         if (smmu_domain->non_strict)
 794                 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
 795
 796         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 797         if (!pgtbl_ops) {
 798                 ret = -ENOMEM;
 799                 goto out_clear_smmu;
 800         }
 801
 802         /* Update the domain's page sizes to reflect the page table format */
 803         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 804         domain->geometry.aperture_end = (1UL << ias) - 1;
 805         domain->geometry.force_aperture = true;
 806
 807         /* Initialise the context bank with our page table cfg */
 808         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 809         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 810
 811         /*
 812          * Request context fault interrupt. Do this last to avoid the
 813          * handler seeing a half-initialised domain state.
 814          */
 815         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 816         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 817                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 818         if (ret < 0) {
 819                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 820                         cfg->irptndx, irq);
 821                 cfg->irptndx = INVALID_IRPTNDX;
 822         }
 823
 824         mutex_unlock(&smmu_domain->init_mutex);
 825
 826         /* Publish page table ops for map/unmap */
 827         smmu_domain->pgtbl_ops = pgtbl_ops;
 828         return 0;
 829
 830 out_clear_smmu:
 831         smmu_domain->smmu = NULL;
 832 out_unlock:
 833         mutex_unlock(&smmu_domain->init_mutex);
 834         return ret;
 835 }
 836
 837 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 838 {
 839         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 840         struct arm_smmu_device *smmu = smmu_domain->smmu;
 841         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 842         int ret, irq;
 843
 844         if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)
 845                 return;
 846
 847         ret = arm_smmu_rpm_get(smmu);
 848         if (ret < 0)
 849                 return;
 850
 851         /*
 852          * Disable the context bank and free the page tables before freeing
 853          * it.
 854          */
 855         smmu->cbs[cfg->cbndx].cfg = NULL;
 856         arm_smmu_write_context_bank(smmu, cfg->cbndx);
 857
 858         if (cfg->irptndx != INVALID_IRPTNDX) {
 859                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 860                 devm_free_irq(smmu->dev, irq, domain);
 861         }
 862
 863         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
 864         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
 865
 866         arm_smmu_rpm_put(smmu);
 867 }
 868
 869 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
 870 {
 871         struct arm_smmu_domain *smmu_domain;
 872
 873         if (type != IOMMU_DOMAIN_UNMANAGED &&
 874             type != IOMMU_DOMAIN_DMA &&
 875             type != IOMMU_DOMAIN_IDENTITY)
 876                 return NULL;
 877         /*
 878          * Allocate the domain and initialise some of its data structures.
 879          * We can't really do anything meaningful until we've added a
 880          * master.
 881          */
 882         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
 883         if (!smmu_domain)
 884                 return NULL;
 885
 886         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
 887             iommu_get_dma_cookie(&smmu_domain->domain))) {
 888                 kfree(smmu_domain);
 889                 return NULL;
 890         }
 891
 892         mutex_init(&smmu_domain->init_mutex);
 893         spin_lock_init(&smmu_domain->cb_lock);
 894
 895         return &smmu_domain->domain;
 896 }
 897
 898 static void arm_smmu_domain_free(struct iommu_domain *domain)
 899 {
 900         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 901
 902         /*
 903          * Free the domain resources. We assume that all devices have
 904          * already been detached.
 905          */
 906         iommu_put_dma_cookie(domain);
 907         arm_smmu_destroy_domain_context(domain);
 908         kfree(smmu_domain);
 909 }
 910
 911 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
 912 {
 913         struct arm_smmu_smr *smr = smmu->smrs + idx;
 914         u32 reg = FIELD_PREP(SMR_ID, smr->id) | FIELD_PREP(SMR_MASK, smr->mask);
 915
 916         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
 917                 reg |= SMR_VALID;
 918         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
 919 }
 920
 921 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
 922 {
 923         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
 924         u32 reg = FIELD_PREP(S2CR_TYPE, s2cr->type) |
 925                   FIELD_PREP(S2CR_CBNDX, s2cr->cbndx) |
 926                   FIELD_PREP(S2CR_PRIVCFG, s2cr->privcfg);
 927
 928         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
 929             smmu->smrs[idx].valid)
 930                 reg |= S2CR_EXIDVALID;
 931         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
 932 }
 933
 934 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
 935 {
 936         arm_smmu_write_s2cr(smmu, idx);
 937         if (smmu->smrs)
 938                 arm_smmu_write_smr(smmu, idx);
 939 }
 940
 941 /*
 942  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
 943  * should be called after sCR0 is written.
 944  */
 945 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
 946 {
 947         u32 smr;
 948
 949         if (!smmu->smrs)
 950                 return;
 951
 952         /*
 953          * SMR.ID bits may not be preserved if the corresponding MASK
 954          * bits are set, so check each one separately. We can reject
 955          * masters later if they try to claim IDs outside these masks.
 956          */
 957         smr = FIELD_PREP(SMR_ID, smmu->streamid_mask);
 958         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);
 959         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));
 960         smmu->streamid_mask = FIELD_GET(SMR_ID, smr);
 961
 962         smr = FIELD_PREP(SMR_MASK, smmu->streamid_mask);
 963         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(0), smr);
 964         smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(0));
 965         smmu->smr_mask_mask = FIELD_GET(SMR_MASK, smr);
 966 }
 967
 968 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
 969 {
 970         struct arm_smmu_smr *smrs = smmu->smrs;
 971         int i, free_idx = -ENOSPC;
 972
 973         /* Stream indexing is blissfully easy */
 974         if (!smrs)
 975                 return id;
 976
 977         /* Validating SMRs is... less so */
 978         for (i = 0; i < smmu->num_mapping_groups; ++i) {
 979                 if (!smrs[i].valid) {
 980                         /*
 981                          * Note the first free entry we come across, which
 982                          * we'll claim in the end if nothing else matches.
 983                          */
 984                         if (free_idx < 0)
 985                                 free_idx = i;
 986                         continue;
 987                 }
 988                 /*
 989                  * If the new entry is _entirely_ matched by an existing entry,
 990                  * then reuse that, with the guarantee that there also cannot
 991                  * be any subsequent conflicting entries. In normal use we'd
 992                  * expect simply identical entries for this case, but there's
 993                  * no harm in accommodating the generalisation.
 994                  */
 995                 if ((mask & smrs[i].mask) == mask &&
 996                     !((id ^ smrs[i].id) & ~smrs[i].mask))
 997                         return i;
 998                 /*
 999                  * If the new entry has any other overlap with an existing one,
1000                  * though, then there always exists at least one stream ID
1001                  * which would cause a conflict, and we can't allow that risk.
1002                  */
1003                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1004                         return -EINVAL;
1005         }
1006
1007         return free_idx;
1008 }
1009
1010 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1011 {
1012         if (--smmu->s2crs[idx].count)
1013                 return false;
1014
1015         smmu->s2crs[idx] = s2cr_init_val;
1016         if (smmu->smrs)
1017                 smmu->smrs[idx].valid = false;
1018
1019         return true;
1020 }
1021
1022 static int arm_smmu_master_alloc_smes(struct device *dev)
1023 {
1024         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1025         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1026         struct arm_smmu_device *smmu = cfg->smmu;
1027         struct arm_smmu_smr *smrs = smmu->smrs;
1028         struct iommu_group *group;
1029         int i, idx, ret;
1030
1031         mutex_lock(&smmu->stream_map_mutex);
1032         /* Figure out a viable stream map entry allocation */
1033         for_each_cfg_sme(fwspec, i, idx) {
1034                 u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1035                 u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
1036
1037                 if (idx != INVALID_SMENDX) {
1038                         ret = -EEXIST;
1039                         goto out_err;
1040                 }
1041
1042                 ret = arm_smmu_find_sme(smmu, sid, mask);
1043                 if (ret < 0)
1044                         goto out_err;
1045
1046                 idx = ret;
1047                 if (smrs && smmu->s2crs[idx].count == 0) {
1048                         smrs[idx].id = sid;
1049                         smrs[idx].mask = mask;
1050                         smrs[idx].valid = true;
1051                 }
1052                 smmu->s2crs[idx].count++;
1053                 cfg->smendx[i] = (s16)idx;
1054         }
1055
1056         group = iommu_group_get_for_dev(dev);
1057         if (!group)
1058                 group = ERR_PTR(-ENOMEM);
1059         if (IS_ERR(group)) {
1060                 ret = PTR_ERR(group);
1061                 goto out_err;
1062         }
1063         iommu_group_put(group);
1064
1065         /* It worked! Now, poke the actual hardware */
1066         for_each_cfg_sme(fwspec, i, idx) {
1067                 arm_smmu_write_sme(smmu, idx);
1068                 smmu->s2crs[idx].group = group;
1069         }
1070
1071         mutex_unlock(&smmu->stream_map_mutex);
1072         return 0;
1073
1074 out_err:
1075         while (i--) {
1076                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1077                 cfg->smendx[i] = INVALID_SMENDX;
1078         }
1079         mutex_unlock(&smmu->stream_map_mutex);
1080         return ret;
1081 }
1082
1083 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1084 {
1085         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1086         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1087         int i, idx;
1088
1089         mutex_lock(&smmu->stream_map_mutex);
1090         for_each_cfg_sme(fwspec, i, idx) {
1091                 if (arm_smmu_free_sme(smmu, idx))
1092                         arm_smmu_write_sme(smmu, idx);
1093                 cfg->smendx[i] = INVALID_SMENDX;
1094         }
1095         mutex_unlock(&smmu->stream_map_mutex);
1096 }
1097
1098 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1099                                       struct iommu_fwspec *fwspec)
1100 {
1101         struct arm_smmu_device *smmu = smmu_domain->smmu;
1102         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1103         u8 cbndx = smmu_domain->cfg.cbndx;
1104         enum arm_smmu_s2cr_type type;
1105         int i, idx;
1106
1107         if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS)
1108                 type = S2CR_TYPE_BYPASS;
1109         else
1110                 type = S2CR_TYPE_TRANS;
1111
1112         for_each_cfg_sme(fwspec, i, idx) {
1113                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1114                         continue;
1115
1116                 s2cr[idx].type = type;
1117                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1118                 s2cr[idx].cbndx = cbndx;
1119                 arm_smmu_write_s2cr(smmu, idx);
1120         }
1121         return 0;
1122 }
1123
1124 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1125 {
1126         int ret;
1127         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1128         struct arm_smmu_device *smmu;
1129         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1130
1131         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1132                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1133                 return -ENXIO;
1134         }
1135
1136         /*
1137          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1138          * domains between of_xlate() and add_device() - we have no way to cope
1139          * with that, so until ARM gets converted to rely on groups and default
1140          * domains, just say no (but more politely than by dereferencing NULL).
1141          * This should be at least a WARN_ON once that's sorted.
1142          */
1143         if (!fwspec->iommu_priv)
1144                 return -ENODEV;
1145
1146         smmu = fwspec_smmu(fwspec);
1147
1148         ret = arm_smmu_rpm_get(smmu);
1149         if (ret < 0)
1150                 return ret;
1151
1152         /* Ensure that the domain is finalised */
1153         ret = arm_smmu_init_domain_context(domain, smmu);
1154         if (ret < 0)
1155                 goto rpm_put;
1156
1157         /*
1158          * Sanity check the domain. We don't support domains across
1159          * different SMMUs.
1160          */
1161         if (smmu_domain->smmu != smmu) {
1162                 dev_err(dev,
1163                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1164                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1165                 ret = -EINVAL;
1166                 goto rpm_put;
1167         }
1168
1169         /* Looks ok, so add the device to the domain */
1170         ret = arm_smmu_domain_add_master(smmu_domain, fwspec);
1171
1172 rpm_put:
1173         arm_smmu_rpm_put(smmu);
1174         return ret;
1175 }
1176
1177 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1178                         phys_addr_t paddr, size_t size, int prot)
1179 {
1180         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1181         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1182         int ret;
1183
1184         if (!ops)
1185                 return -ENODEV;
1186
1187         arm_smmu_rpm_get(smmu);
1188         ret = ops->map(ops, iova, paddr, size, prot);
1189         arm_smmu_rpm_put(smmu);
1190
1191         return ret;
1192 }
1193
1194 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1195                              size_t size, struct iommu_iotlb_gather *gather)
1196 {
1197         struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1198         struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1199         size_t ret;
1200
1201         if (!ops)
1202                 return 0;
1203
1204         arm_smmu_rpm_get(smmu);
1205         ret = ops->unmap(ops, iova, size, gather);
1206         arm_smmu_rpm_put(smmu);
1207
1208         return ret;
1209 }
1210
1211 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1212 {
1213         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1214         struct arm_smmu_device *smmu = smmu_domain->smmu;
1215
1216         if (smmu_domain->flush_ops) {
1217                 arm_smmu_rpm_get(smmu);
1218                 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1219                 arm_smmu_rpm_put(smmu);
1220         }
1221 }
1222
1223 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1224                                 struct iommu_iotlb_gather *gather)
1225 {
1226         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1227         struct arm_smmu_device *smmu = smmu_domain->smmu;
1228
1229         if (!smmu)
1230                 return;
1231
1232         arm_smmu_rpm_get(smmu);
1233         if (smmu->version == ARM_SMMU_V2 ||
1234             smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1235                 arm_smmu_tlb_sync_context(smmu_domain);
1236         else
1237                 arm_smmu_tlb_sync_global(smmu);
1238         arm_smmu_rpm_put(smmu);
1239 }
1240
1241 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1242                                               dma_addr_t iova)
1243 {
1244         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1245         struct arm_smmu_device *smmu = smmu_domain->smmu;
1246         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1247         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1248         struct device *dev = smmu->dev;
1249         void __iomem *reg;
1250         u32 tmp;
1251         u64 phys;
1252         unsigned long va, flags;
1253         int ret, idx = cfg->cbndx;
1254
1255         ret = arm_smmu_rpm_get(smmu);
1256         if (ret < 0)
1257                 return 0;
1258
1259         spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1260         va = iova & ~0xfffUL;
1261         if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1262                 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1263         else
1264                 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1265
1266         reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1267         if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) {
1268                 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1269                 dev_err(dev,
1270                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1271                         &iova);
1272                 return ops->iova_to_phys(ops, iova);
1273         }
1274
1275         phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1276         spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1277         if (phys & CB_PAR_F) {
1278                 dev_err(dev, "translation fault!\n");
1279                 dev_err(dev, "PAR = 0x%llx\n", phys);
1280                 return 0;
1281         }
1282
1283         arm_smmu_rpm_put(smmu);
1284
1285         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1286 }
1287
1288 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1289                                         dma_addr_t iova)
1290 {
1291         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1292         struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1293
1294         if (domain->type == IOMMU_DOMAIN_IDENTITY)
1295                 return iova;
1296
1297         if (!ops)
1298                 return 0;
1299
1300         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1301                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1302                 return arm_smmu_iova_to_phys_hard(domain, iova);
1303
1304         return ops->iova_to_phys(ops, iova);
1305 }
1306
1307 static bool arm_smmu_capable(enum iommu_cap cap)
1308 {
1309         switch (cap) {
1310         case IOMMU_CAP_CACHE_COHERENCY:
1311                 /*
1312                  * Return true here as the SMMU can always send out coherent
1313                  * requests.
1314                  */
1315                 return true;
1316         case IOMMU_CAP_NOEXEC:
1317                 return true;
1318         default:
1319                 return false;
1320         }
1321 }
1322
1323 static
1324 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1325 {
1326         struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
1327                                                           fwnode);
1328         put_device(dev);
1329         return dev ? dev_get_drvdata(dev) : NULL;
1330 }
1331
1332 static int arm_smmu_add_device(struct device *dev)
1333 {
1334         struct arm_smmu_device *smmu;
1335         struct arm_smmu_master_cfg *cfg;
1336         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1337         int i, ret;
1338
1339         if (using_legacy_binding) {
1340                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1341
1342                 /*
1343                  * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1344                  * will allocate/initialise a new one. Thus we need to update fwspec for
1345                  * later use.
1346                  */
1347                 fwspec = dev_iommu_fwspec_get(dev);
1348                 if (ret)
1349                         goto out_free;
1350         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1351                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1352         } else {
1353                 return -ENODEV;
1354         }
1355
1356         ret = -EINVAL;
1357         for (i = 0; i < fwspec->num_ids; i++) {
1358                 u16 sid = FIELD_GET(SMR_ID, fwspec->ids[i]);
1359                 u16 mask = FIELD_GET(SMR_MASK, fwspec->ids[i]);
1360
1361                 if (sid & ~smmu->streamid_mask) {
1362                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1363                                 sid, smmu->streamid_mask);
1364                         goto out_free;
1365                 }
1366                 if (mask & ~smmu->smr_mask_mask) {
1367                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1368                                 mask, smmu->smr_mask_mask);
1369                         goto out_free;
1370                 }
1371         }
1372
1373         ret = -ENOMEM;
1374         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1375                       GFP_KERNEL);
1376         if (!cfg)
1377                 goto out_free;
1378
1379         cfg->smmu = smmu;
1380         fwspec->iommu_priv = cfg;
1381         while (i--)
1382                 cfg->smendx[i] = INVALID_SMENDX;
1383
1384         ret = arm_smmu_rpm_get(smmu);
1385         if (ret < 0)
1386                 goto out_cfg_free;
1387
1388         ret = arm_smmu_master_alloc_smes(dev);
1389         arm_smmu_rpm_put(smmu);
1390
1391         if (ret)
1392                 goto out_cfg_free;
1393
1394         iommu_device_link(&smmu->iommu, dev);
1395
1396         device_link_add(dev, smmu->dev,
1397                         DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1398
1399         return 0;
1400
1401 out_cfg_free:
1402         kfree(cfg);
1403 out_free:
1404         iommu_fwspec_free(dev);
1405         return ret;
1406 }
1407
1408 static void arm_smmu_remove_device(struct device *dev)
1409 {
1410         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1411         struct arm_smmu_master_cfg *cfg;
1412         struct arm_smmu_device *smmu;
1413         int ret;
1414
1415         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1416                 return;
1417
1418         cfg  = fwspec->iommu_priv;
1419         smmu = cfg->smmu;
1420
1421         ret = arm_smmu_rpm_get(smmu);
1422         if (ret < 0)
1423                 return;
1424
1425         iommu_device_unlink(&smmu->iommu, dev);
1426         arm_smmu_master_free_smes(fwspec);
1427
1428         arm_smmu_rpm_put(smmu);
1429
1430         iommu_group_remove_device(dev);
1431         kfree(fwspec->iommu_priv);
1432         iommu_fwspec_free(dev);
1433 }
1434
1435 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1436 {
1437         struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1438         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1439         struct iommu_group *group = NULL;
1440         int i, idx;
1441
1442         for_each_cfg_sme(fwspec, i, idx) {
1443                 if (group && smmu->s2crs[idx].group &&
1444                     group != smmu->s2crs[idx].group)
1445                         return ERR_PTR(-EINVAL);
1446
1447                 group = smmu->s2crs[idx].group;
1448         }
1449
1450         if (group)
1451                 return iommu_group_ref_get(group);
1452
1453         if (dev_is_pci(dev))
1454                 group = pci_device_group(dev);
1455         else if (dev_is_fsl_mc(dev))
1456                 group = fsl_mc_device_group(dev);
1457         else
1458                 group = generic_device_group(dev);
1459
1460         return group;
1461 }
1462
1463 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1464                                     enum iommu_attr attr, void *data)
1465 {
1466         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1467
1468         switch(domain->type) {
1469         case IOMMU_DOMAIN_UNMANAGED:
1470                 switch (attr) {
1471                 case DOMAIN_ATTR_NESTING:
1472                         *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1473                         return 0;
1474                 default:
1475                         return -ENODEV;
1476                 }
1477                 break;
1478         case IOMMU_DOMAIN_DMA:
1479                 switch (attr) {
1480                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1481                         *(int *)data = smmu_domain->non_strict;
1482                         return 0;
1483                 default:
1484                         return -ENODEV;
1485                 }
1486                 break;
1487         default:
1488                 return -EINVAL;
1489         }
1490 }
1491
1492 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1493                                     enum iommu_attr attr, void *data)
1494 {
1495         int ret = 0;
1496         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1497
1498         mutex_lock(&smmu_domain->init_mutex);
1499
1500         switch(domain->type) {
1501         case IOMMU_DOMAIN_UNMANAGED:
1502                 switch (attr) {
1503                 case DOMAIN_ATTR_NESTING:
1504                         if (smmu_domain->smmu) {
1505                                 ret = -EPERM;
1506                                 goto out_unlock;
1507                         }
1508
1509                         if (*(int *)data)
1510                                 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1511                         else
1512                                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1513                         break;
1514                 default:
1515                         ret = -ENODEV;
1516                 }
1517                 break;
1518         case IOMMU_DOMAIN_DMA:
1519                 switch (attr) {
1520                 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
1521                         smmu_domain->non_strict = *(int *)data;
1522                         break;
1523                 default:
1524                         ret = -ENODEV;
1525                 }
1526                 break;
1527         default:
1528                 ret = -EINVAL;
1529         }
1530 out_unlock:
1531         mutex_unlock(&smmu_domain->init_mutex);
1532         return ret;
1533 }
1534
1535 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1536 {
1537         u32 mask, fwid = 0;
1538
1539         if (args->args_count > 0)
1540                 fwid |= FIELD_PREP(SMR_ID, args->args[0]);
1541
1542         if (args->args_count > 1)
1543                 fwid |= FIELD_PREP(SMR_MASK, args->args[1]);
1544         else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1545                 fwid |= FIELD_PREP(SMR_MASK, mask);
1546
1547         return iommu_fwspec_add_ids(dev, &fwid, 1);
1548 }
1549
1550 static void arm_smmu_get_resv_regions(struct device *dev,
1551                                       struct list_head *head)
1552 {
1553         struct iommu_resv_region *region;
1554         int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1555
1556         region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1557                                          prot, IOMMU_RESV_SW_MSI);
1558         if (!region)
1559                 return;
1560
1561         list_add_tail(&region->list, head);
1562
1563         iommu_dma_get_resv_regions(dev, head);
1564 }
1565
1566 static void arm_smmu_put_resv_regions(struct device *dev,
1567                                       struct list_head *head)
1568 {
1569         struct iommu_resv_region *entry, *next;
1570
1571         list_for_each_entry_safe(entry, next, head, list)
1572                 kfree(entry);
1573 }
1574
1575 static struct iommu_ops arm_smmu_ops = {
1576         .capable                = arm_smmu_capable,
1577         .domain_alloc           = arm_smmu_domain_alloc,
1578         .domain_free            = arm_smmu_domain_free,
1579         .attach_dev             = arm_smmu_attach_dev,
1580         .map                    = arm_smmu_map,
1581         .unmap                  = arm_smmu_unmap,
1582         .flush_iotlb_all        = arm_smmu_flush_iotlb_all,
1583         .iotlb_sync             = arm_smmu_iotlb_sync,
1584         .iova_to_phys           = arm_smmu_iova_to_phys,
1585         .add_device             = arm_smmu_add_device,
1586         .remove_device          = arm_smmu_remove_device,
1587         .device_group           = arm_smmu_device_group,
1588         .domain_get_attr        = arm_smmu_domain_get_attr,
1589         .domain_set_attr        = arm_smmu_domain_set_attr,
1590         .of_xlate               = arm_smmu_of_xlate,
1591         .get_resv_regions       = arm_smmu_get_resv_regions,
1592         .put_resv_regions       = arm_smmu_put_resv_regions,
1593         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1594 };
1595
1596 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1597 {
1598         int i;
1599         u32 reg;
1600
1601         /* clear global FSR */
1602         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1603         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1604
1605         /*
1606          * Reset stream mapping groups: Initial values mark all SMRn as
1607          * invalid and all S2CRn as bypass unless overridden.
1608          */
1609         for (i = 0; i < smmu->num_mapping_groups; ++i)
1610                 arm_smmu_write_sme(smmu, i);
1611
1612         /* Make sure all context banks are disabled and clear CB_FSR  */
1613         for (i = 0; i < smmu->num_context_banks; ++i) {
1614                 arm_smmu_write_context_bank(smmu, i);
1615                 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, FSR_FAULT);
1616         }
1617
1618         /* Invalidate the TLB, just in case */
1619         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1620         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1621
1622         reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1623
1624         /* Enable fault reporting */
1625         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1626
1627         /* Disable TLB broadcasting. */
1628         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1629
1630         /* Enable client access, handling unmatched streams as appropriate */
1631         reg &= ~sCR0_CLIENTPD;
1632         if (disable_bypass)
1633                 reg |= sCR0_USFCFG;
1634         else
1635                 reg &= ~sCR0_USFCFG;
1636
1637         /* Disable forced broadcasting */
1638         reg &= ~sCR0_FB;
1639
1640         /* Don't upgrade barriers */
1641         reg &= ~(sCR0_BSU);
1642
1643         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1644                 reg |= sCR0_VMID16EN;
1645
1646         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1647                 reg |= sCR0_EXIDENABLE;
1648
1649         if (smmu->impl && smmu->impl->reset)
1650                 smmu->impl->reset(smmu);
1651
1652         /* Push the button */
1653         arm_smmu_tlb_sync_global(smmu);
1654         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1655 }
1656
1657 static int arm_smmu_id_size_to_bits(int size)
1658 {
1659         switch (size) {
1660         case 0:
1661                 return 32;
1662         case 1:
1663                 return 36;
1664         case 2:
1665                 return 40;
1666         case 3:
1667                 return 42;
1668         case 4:
1669                 return 44;
1670         case 5:
1671         default:
1672                 return 48;
1673         }
1674 }
1675
1676 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1677 {
1678         unsigned int size;
1679         u32 id;
1680         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1681         int i;
1682
1683         dev_notice(smmu->dev, "probing hardware configuration...\n");
1684         dev_notice(smmu->dev, "SMMUv%d with:\n",
1685                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1686
1687         /* ID0 */
1688         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1689
1690         /* Restrict available stages based on module parameter */
1691         if (force_stage == 1)
1692                 id &= ~(ID0_S2TS | ID0_NTS);
1693         else if (force_stage == 2)
1694                 id &= ~(ID0_S1TS | ID0_NTS);
1695
1696         if (id & ID0_S1TS) {
1697                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1698                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1699         }
1700
1701         if (id & ID0_S2TS) {
1702                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1703                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1704         }
1705
1706         if (id & ID0_NTS) {
1707                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1708                 dev_notice(smmu->dev, "\tnested translation\n");
1709         }
1710
1711         if (!(smmu->features &
1712                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1713                 dev_err(smmu->dev, "\tno translation support!\n");
1714                 return -ENODEV;
1715         }
1716
1717         if ((id & ID0_S1TS) &&
1718                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1719                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1720                 dev_notice(smmu->dev, "\taddress translation ops\n");
1721         }
1722
1723         /*
1724          * In order for DMA API calls to work properly, we must defer to what
1725          * the FW says about coherency, regardless of what the hardware claims.
1726          * Fortunately, this also opens up a workaround for systems where the
1727          * ID register value has ended up configured incorrectly.
1728          */
1729         cttw_reg = !!(id & ID0_CTTW);
1730         if (cttw_fw || cttw_reg)
1731                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1732                            cttw_fw ? "" : "non-");
1733         if (cttw_fw != cttw_reg)
1734                 dev_notice(smmu->dev,
1735                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1736
1737         /* Max. number of entries we have for stream matching/indexing */
1738         if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1739                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1740                 size = 1 << 16;
1741         } else {
1742                 size = 1 << FIELD_GET(ID0_NUMSIDB, id);
1743         }
1744         smmu->streamid_mask = size - 1;
1745         if (id & ID0_SMS) {
1746                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1747                 size = FIELD_GET(ID0_NUMSMRG, id);
1748                 if (size == 0) {
1749                         dev_err(smmu->dev,
1750                                 "stream-matching supported, but no SMRs present!\n");
1751                         return -ENODEV;
1752                 }
1753
1754                 /* Zero-initialised to mark as invalid */
1755                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1756                                           GFP_KERNEL);
1757                 if (!smmu->smrs)
1758                         return -ENOMEM;
1759
1760                 dev_notice(smmu->dev,
1761                            "\tstream matching with %u register groups", size);
1762         }
1763         /* s2cr->type == 0 means translation, so initialise explicitly */
1764         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1765                                          GFP_KERNEL);
1766         if (!smmu->s2crs)
1767                 return -ENOMEM;
1768         for (i = 0; i < size; i++)
1769                 smmu->s2crs[i] = s2cr_init_val;
1770
1771         smmu->num_mapping_groups = size;
1772         mutex_init(&smmu->stream_map_mutex);
1773         spin_lock_init(&smmu->global_sync_lock);
1774
1775         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1776                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1777                 if (!(id & ID0_PTFS_NO_AARCH32S))
1778                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1779         }
1780
1781         /* ID1 */
1782         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1783         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1784
1785         /* Check for size mismatch of SMMU address space from mapped region */
1786         size = 1 << (FIELD_GET(ID1_NUMPAGENDXB, id) + 1);
1787         if (smmu->numpage != 2 * size << smmu->pgshift)
1788                 dev_warn(smmu->dev,
1789                         "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1790                         2 * size << smmu->pgshift, smmu->numpage);
1791         /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1792         smmu->numpage = size;
1793
1794         smmu->num_s2_context_banks = FIELD_GET(ID1_NUMS2CB, id);
1795         smmu->num_context_banks = FIELD_GET(ID1_NUMCB, id);
1796         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1797                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1798                 return -ENODEV;
1799         }
1800         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1801                    smmu->num_context_banks, smmu->num_s2_context_banks);
1802         smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1803                                  sizeof(*smmu->cbs), GFP_KERNEL);
1804         if (!smmu->cbs)
1805                 return -ENOMEM;
1806
1807         /* ID2 */
1808         id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1809         size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_IAS, id));
1810         smmu->ipa_size = size;
1811
1812         /* The output mask is also applied for bypass */
1813         size = arm_smmu_id_size_to_bits(FIELD_GET(ID2_OAS, id));
1814         smmu->pa_size = size;
1815
1816         if (id & ID2_VMID16)
1817                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1818
1819         /*
1820          * What the page table walker can address actually depends on which
1821          * descriptor format is in use, but since a) we don't know that yet,
1822          * and b) it can vary per context bank, this will have to do...
1823          */
1824         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1825                 dev_warn(smmu->dev,
1826                          "failed to set DMA mask for table walker\n");
1827
1828         if (smmu->version < ARM_SMMU_V2) {
1829                 smmu->va_size = smmu->ipa_size;
1830                 if (smmu->version == ARM_SMMU_V1_64K)
1831                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1832         } else {
1833                 size = FIELD_GET(ID2_UBS, id);
1834                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1835                 if (id & ID2_PTFS_4K)
1836                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1837                 if (id & ID2_PTFS_16K)
1838                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1839                 if (id & ID2_PTFS_64K)
1840                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1841         }
1842
1843         /* Now we've corralled the various formats, what'll it do? */
1844         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1845                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1846         if (smmu->features &
1847             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1848                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1849         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1850                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1851         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1852                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1853
1854         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1855                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1856         else
1857                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1858         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1859                    smmu->pgsize_bitmap);
1860
1861
1862         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1863                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1864                            smmu->va_size, smmu->ipa_size);
1865
1866         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1867                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1868                            smmu->ipa_size, smmu->pa_size);
1869
1870         if (smmu->impl && smmu->impl->cfg_probe)
1871                 return smmu->impl->cfg_probe(smmu);
1872
1873         return 0;
1874 }
1875
1876 struct arm_smmu_match_data {
1877         enum arm_smmu_arch_version version;
1878         enum arm_smmu_implementation model;
1879 };
1880
1881 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1882 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1883
1884 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1885 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1886 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1887 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1888 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1889 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1890
1891 static const struct of_device_id arm_smmu_of_match[] = {
1892         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1893         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1894         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1895         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1896         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1897         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1898         { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1899         { },
1900 };
1901
1902 #ifdef CONFIG_ACPI
1903 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1904 {
1905         int ret = 0;
1906
1907         switch (model) {
1908         case ACPI_IORT_SMMU_V1:
1909         case ACPI_IORT_SMMU_CORELINK_MMU400:
1910                 smmu->version = ARM_SMMU_V1;
1911                 smmu->model = GENERIC_SMMU;
1912                 break;
1913         case ACPI_IORT_SMMU_CORELINK_MMU401:
1914                 smmu->version = ARM_SMMU_V1_64K;
1915                 smmu->model = GENERIC_SMMU;
1916                 break;
1917         case ACPI_IORT_SMMU_V2:
1918                 smmu->version = ARM_SMMU_V2;
1919                 smmu->model = GENERIC_SMMU;
1920                 break;
1921         case ACPI_IORT_SMMU_CORELINK_MMU500:
1922                 smmu->version = ARM_SMMU_V2;
1923                 smmu->model = ARM_MMU500;
1924                 break;
1925         case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1926                 smmu->version = ARM_SMMU_V2;
1927                 smmu->model = CAVIUM_SMMUV2;
1928                 break;
1929         default:
1930                 ret = -ENODEV;
1931         }
1932
1933         return ret;
1934 }
1935
1936 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1937                                       struct arm_smmu_device *smmu)
1938 {
1939         struct device *dev = smmu->dev;
1940         struct acpi_iort_node *node =
1941                 *(struct acpi_iort_node **)dev_get_platdata(dev);
1942         struct acpi_iort_smmu *iort_smmu;
1943         int ret;
1944
1945         /* Retrieve SMMU1/2 specific data */
1946         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1947
1948         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1949         if (ret < 0)
1950                 return ret;
1951
1952         /* Ignore the configuration access interrupt */
1953         smmu->num_global_irqs = 1;
1954
1955         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1956                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1957
1958         return 0;
1959 }
1960 #else
1961 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1962                                              struct arm_smmu_device *smmu)
1963 {
1964         return -ENODEV;
1965 }
1966 #endif
1967
1968 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
1969                                     struct arm_smmu_device *smmu)
1970 {
1971         const struct arm_smmu_match_data *data;
1972         struct device *dev = &pdev->dev;
1973         bool legacy_binding;
1974
1975         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1976                                  &smmu->num_global_irqs)) {
1977                 dev_err(dev, "missing #global-interrupts property\n");
1978                 return -ENODEV;
1979         }
1980
1981         data = of_device_get_match_data(dev);
1982         smmu->version = data->version;
1983         smmu->model = data->model;
1984
1985         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1986         if (legacy_binding && !using_generic_binding) {
1987                 if (!using_legacy_binding)
1988                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
1989                 using_legacy_binding = true;
1990         } else if (!legacy_binding && !using_legacy_binding) {
1991                 using_generic_binding = true;
1992         } else {
1993                 dev_err(dev, "not probing due to mismatched DT properties\n");
1994                 return -ENODEV;
1995         }
1996
1997         if (of_dma_is_coherent(dev->of_node))
1998                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1999
2000         return 0;
2001 }
2002
2003 static void arm_smmu_bus_init(void)
2004 {
2005         /* Oh, for a proper bus abstraction */
2006         if (!iommu_present(&platform_bus_type))
2007                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2008 #ifdef CONFIG_ARM_AMBA
2009         if (!iommu_present(&amba_bustype))
2010                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2011 #endif
2012 #ifdef CONFIG_PCI
2013         if (!iommu_present(&pci_bus_type)) {
2014                 pci_request_acs();
2015                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2016         }
2017 #endif
2018 #ifdef CONFIG_FSL_MC_BUS
2019         if (!iommu_present(&fsl_mc_bus_type))
2020                 bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
2021 #endif
2022 }
2023
2024 static int arm_smmu_device_probe(struct platform_device *pdev)
2025 {
2026         struct resource *res;
2027         resource_size_t ioaddr;
2028         struct arm_smmu_device *smmu;
2029         struct device *dev = &pdev->dev;
2030         int num_irqs, i, err;
2031
2032         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2033         if (!smmu) {
2034                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2035                 return -ENOMEM;
2036         }
2037         smmu->dev = dev;
2038
2039         if (dev->of_node)
2040                 err = arm_smmu_device_dt_probe(pdev, smmu);
2041         else
2042                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2043
2044         if (err)
2045                 return err;
2046
2047         smmu = arm_smmu_impl_init(smmu);
2048         if (IS_ERR(smmu))
2049                 return PTR_ERR(smmu);
2050
2051         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2052         ioaddr = res->start;
2053         smmu->base = devm_ioremap_resource(dev, res);
2054         if (IS_ERR(smmu->base))
2055                 return PTR_ERR(smmu->base);
2056         /*
2057          * The resource size should effectively match the value of SMMU_TOP;
2058          * stash that temporarily until we know PAGESIZE to validate it with.
2059          */
2060         smmu->numpage = resource_size(res);
2061
2062         num_irqs = 0;
2063         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2064                 num_irqs++;
2065                 if (num_irqs > smmu->num_global_irqs)
2066                         smmu->num_context_irqs++;
2067         }
2068
2069         if (!smmu->num_context_irqs) {
2070                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2071                         num_irqs, smmu->num_global_irqs + 1);
2072                 return -ENODEV;
2073         }
2074
2075         smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
2076                                   GFP_KERNEL);
2077         if (!smmu->irqs) {
2078                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2079                 return -ENOMEM;
2080         }
2081
2082         for (i = 0; i < num_irqs; ++i) {
2083                 int irq = platform_get_irq(pdev, i);
2084
2085                 if (irq < 0) {
2086                         dev_err(dev, "failed to get irq index %d\n", i);
2087                         return -ENODEV;
2088                 }
2089                 smmu->irqs[i] = irq;
2090         }
2091
2092         err = devm_clk_bulk_get_all(dev, &smmu->clks);
2093         if (err < 0) {
2094                 dev_err(dev, "failed to get clocks %d\n", err);
2095                 return err;
2096         }
2097         smmu->num_clks = err;
2098
2099         err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2100         if (err)
2101                 return err;
2102
2103         err = arm_smmu_device_cfg_probe(smmu);
2104         if (err)
2105                 return err;
2106
2107         if (smmu->version == ARM_SMMU_V2) {
2108                 if (smmu->num_context_banks > smmu->num_context_irqs) {
2109                         dev_err(dev,
2110                               "found only %d context irq(s) but %d required\n",
2111                               smmu->num_context_irqs, smmu->num_context_banks);
2112                         return -ENODEV;
2113                 }
2114
2115                 /* Ignore superfluous interrupts */
2116                 smmu->num_context_irqs = smmu->num_context_banks;
2117         }
2118
2119         for (i = 0; i < smmu->num_global_irqs; ++i) {
2120                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2121                                        arm_smmu_global_fault,
2122                                        IRQF_SHARED,
2123                                        "arm-smmu global fault",
2124                                        smmu);
2125                 if (err) {
2126                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2127                                 i, smmu->irqs[i]);
2128                         return err;
2129                 }
2130         }
2131
2132         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2133                                      "smmu.%pa", &ioaddr);
2134         if (err) {
2135                 dev_err(dev, "Failed to register iommu in sysfs\n");
2136                 return err;
2137         }
2138
2139         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2140         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2141
2142         err = iommu_device_register(&smmu->iommu);
2143         if (err) {
2144                 dev_err(dev, "Failed to register iommu\n");
2145                 return err;
2146         }
2147
2148         platform_set_drvdata(pdev, smmu);
2149         arm_smmu_device_reset(smmu);
2150         arm_smmu_test_smr_masks(smmu);
2151
2152         /*
2153          * We want to avoid touching dev->power.lock in fastpaths unless
2154          * it's really going to do something useful - pm_runtime_enabled()
2155          * can serve as an ideal proxy for that decision. So, conditionally
2156          * enable pm_runtime.
2157          */
2158         if (dev->pm_domain) {
2159                 pm_runtime_set_active(dev);
2160                 pm_runtime_enable(dev);
2161         }
2162
2163         /*
2164          * For ACPI and generic DT bindings, an SMMU will be probed before
2165          * any device which might need it, so we want the bus ops in place
2166          * ready to handle default domain setup as soon as any SMMU exists.
2167          */
2168         if (!using_legacy_binding)
2169                 arm_smmu_bus_init();
2170
2171         return 0;
2172 }
2173
2174 /*
2175  * With the legacy DT binding in play, though, we have no guarantees about
2176  * probe order, but then we're also not doing default domains, so we can
2177  * delay setting bus ops until we're sure every possible SMMU is ready,
2178  * and that way ensure that no add_device() calls get missed.
2179  */
2180 static int arm_smmu_legacy_bus_init(void)
2181 {
2182         if (using_legacy_binding)
2183                 arm_smmu_bus_init();
2184         return 0;
2185 }
2186 device_initcall_sync(arm_smmu_legacy_bus_init);
2187
2188 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2189 {
2190         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2191
2192         if (!smmu)
2193                 return;
2194
2195         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2196                 dev_err(&pdev->dev, "removing device with active domains!\n");
2197
2198         arm_smmu_rpm_get(smmu);
2199         /* Turn the thing off */
2200         arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, sCR0_CLIENTPD);
2201         arm_smmu_rpm_put(smmu);
2202
2203         if (pm_runtime_enabled(smmu->dev))
2204                 pm_runtime_force_suspend(smmu->dev);
2205         else
2206                 clk_bulk_disable(smmu->num_clks, smmu->clks);
2207
2208         clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2209 }
2210
2211 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2212 {
2213         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2214         int ret;
2215
2216         ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2217         if (ret)
2218                 return ret;
2219
2220         arm_smmu_device_reset(smmu);
2221
2222         return 0;
2223 }
2224
2225 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2226 {
2227         struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2228
2229         clk_bulk_disable(smmu->num_clks, smmu->clks);
2230
2231         return 0;
2232 }
2233
2234 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2235 {
2236         if (pm_runtime_suspended(dev))
2237                 return 0;
2238
2239         return arm_smmu_runtime_resume(dev);
2240 }
2241
2242 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2243 {
2244         if (pm_runtime_suspended(dev))
2245                 return 0;
2246
2247         return arm_smmu_runtime_suspend(dev);
2248 }
2249
2250 static const struct dev_pm_ops arm_smmu_pm_ops = {
2251         SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2252         SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2253                            arm_smmu_runtime_resume, NULL)
2254 };
2255
2256 static struct platform_driver arm_smmu_driver = {
2257         .driver = {
2258                 .name                   = "arm-smmu",
2259                 .of_match_table         = of_match_ptr(arm_smmu_of_match),
2260                 .pm                     = &arm_smmu_pm_ops,
2261                 .suppress_bind_attrs    = true,
2262         },
2263         .probe  = arm_smmu_device_probe,
2264         .shutdown = arm_smmu_device_shutdown,
2265 };
2266 builtin_platform_driver(arm_smmu_driver);