drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  */
  28
  29 #define pr_fmt(fmt) "arm-smmu: " fmt
  30
  31 #include <linux/acpi.h>
  32 #include <linux/acpi_iort.h>
  33 #include <linux/atomic.h>
  34 #include <linux/delay.h>
  35 #include <linux/dma-iommu.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/err.h>
  38 #include <linux/interrupt.h>
  39 #include <linux/io.h>
  40 #include <linux/io-64-nonatomic-hi-lo.h>
  41 #include <linux/iommu.h>
  42 #include <linux/iopoll.h>
  43 #include <linux/module.h>
  44 #include <linux/of.h>
  45 #include <linux/of_address.h>
  46 #include <linux/of_device.h>
  47 #include <linux/of_iommu.h>
  48 #include <linux/pci.h>
  49 #include <linux/platform_device.h>
  50 #include <linux/slab.h>
  51 #include <linux/spinlock.h>
  52
  53 #include <linux/amba/bus.h>
  54
  55 #include "io-pgtable.h"
  56
  57 /* Maximum number of context banks per SMMU */
  58 #define ARM_SMMU_MAX_CBS                128
  59
  60 /* SMMU global address space */
  61 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  62 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  63
  64 /*
  65  * SMMU global address space with conditional offset to access secure
  66  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  67  * nsGFSYNR0: 0x450)
  68  */
  69 #define ARM_SMMU_GR0_NS(smmu)                                           \
  70         ((smmu)->base +                                                 \
  71                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  72                         ? 0x400 : 0))
  73
  74 /*
  75  * Some 64-bit registers only make sense to write atomically, but in such
  76  * cases all the data relevant to AArch32 formats lies within the lower word,
  77  * therefore this actually makes more sense than it might first appear.
  78  */
  79 #ifdef CONFIG_64BIT
  80 #define smmu_write_atomic_lq            writeq_relaxed
  81 #else
  82 #define smmu_write_atomic_lq            writel_relaxed
  83 #endif
  84
  85 /* Configuration registers */
  86 #define ARM_SMMU_GR0_sCR0               0x0
  87 #define sCR0_CLIENTPD                   (1 << 0)
  88 #define sCR0_GFRE                       (1 << 1)
  89 #define sCR0_GFIE                       (1 << 2)
  90 #define sCR0_GCFGFRE                    (1 << 4)
  91 #define sCR0_GCFGFIE                    (1 << 5)
  92 #define sCR0_USFCFG                     (1 << 10)
  93 #define sCR0_VMIDPNE                    (1 << 11)
  94 #define sCR0_PTM                        (1 << 12)
  95 #define sCR0_FB                         (1 << 13)
  96 #define sCR0_VMID16EN                   (1 << 31)
  97 #define sCR0_BSU_SHIFT                  14
  98 #define sCR0_BSU_MASK                   0x3
  99
 100 /* Auxiliary Configuration register */
 101 #define ARM_SMMU_GR0_sACR               0x10
 102
 103 /* Identification registers */
 104 #define ARM_SMMU_GR0_ID0                0x20
 105 #define ARM_SMMU_GR0_ID1                0x24
 106 #define ARM_SMMU_GR0_ID2                0x28
 107 #define ARM_SMMU_GR0_ID3                0x2c
 108 #define ARM_SMMU_GR0_ID4                0x30
 109 #define ARM_SMMU_GR0_ID5                0x34
 110 #define ARM_SMMU_GR0_ID6                0x38
 111 #define ARM_SMMU_GR0_ID7                0x3c
 112 #define ARM_SMMU_GR0_sGFSR              0x48
 113 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 114 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 115 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 116
 117 #define ID0_S1TS                        (1 << 30)
 118 #define ID0_S2TS                        (1 << 29)
 119 #define ID0_NTS                         (1 << 28)
 120 #define ID0_SMS                         (1 << 27)
 121 #define ID0_ATOSNS                      (1 << 26)
 122 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 123 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 124 #define ID0_CTTW                        (1 << 14)
 125 #define ID0_NUMIRPT_SHIFT               16
 126 #define ID0_NUMIRPT_MASK                0xff
 127 #define ID0_NUMSIDB_SHIFT               9
 128 #define ID0_NUMSIDB_MASK                0xf
 129 #define ID0_NUMSMRG_SHIFT               0
 130 #define ID0_NUMSMRG_MASK                0xff
 131
 132 #define ID1_PAGESIZE                    (1 << 31)
 133 #define ID1_NUMPAGENDXB_SHIFT           28
 134 #define ID1_NUMPAGENDXB_MASK            7
 135 #define ID1_NUMS2CB_SHIFT               16
 136 #define ID1_NUMS2CB_MASK                0xff
 137 #define ID1_NUMCB_SHIFT                 0
 138 #define ID1_NUMCB_MASK                  0xff
 139
 140 #define ID2_OAS_SHIFT                   4
 141 #define ID2_OAS_MASK                    0xf
 142 #define ID2_IAS_SHIFT                   0
 143 #define ID2_IAS_MASK                    0xf
 144 #define ID2_UBS_SHIFT                   8
 145 #define ID2_UBS_MASK                    0xf
 146 #define ID2_PTFS_4K                     (1 << 12)
 147 #define ID2_PTFS_16K                    (1 << 13)
 148 #define ID2_PTFS_64K                    (1 << 14)
 149 #define ID2_VMID16                      (1 << 15)
 150
 151 #define ID7_MAJOR_SHIFT                 4
 152 #define ID7_MAJOR_MASK                  0xf
 153
 154 /* Global TLB invalidation */
 155 #define ARM_SMMU_GR0_TLBIVMID           0x64
 156 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 157 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 158 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 159 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 160 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 161 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 162
 163 /* Stream mapping registers */
 164 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 165 #define SMR_VALID                       (1 << 31)
 166 #define SMR_MASK_SHIFT                  16
 167 #define SMR_ID_SHIFT                    0
 168
 169 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 170 #define S2CR_CBNDX_SHIFT                0
 171 #define S2CR_CBNDX_MASK                 0xff
 172 #define S2CR_TYPE_SHIFT                 16
 173 #define S2CR_TYPE_MASK                  0x3
 174 enum arm_smmu_s2cr_type {
 175         S2CR_TYPE_TRANS,
 176         S2CR_TYPE_BYPASS,
 177         S2CR_TYPE_FAULT,
 178 };
 179
 180 #define S2CR_PRIVCFG_SHIFT              24
 181 #define S2CR_PRIVCFG_MASK               0x3
 182 enum arm_smmu_s2cr_privcfg {
 183         S2CR_PRIVCFG_DEFAULT,
 184         S2CR_PRIVCFG_DIPAN,
 185         S2CR_PRIVCFG_UNPRIV,
 186         S2CR_PRIVCFG_PRIV,
 187 };
 188
 189 /* Context bank attribute registers */
 190 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 191 #define CBAR_VMID_SHIFT                 0
 192 #define CBAR_VMID_MASK                  0xff
 193 #define CBAR_S1_BPSHCFG_SHIFT           8
 194 #define CBAR_S1_BPSHCFG_MASK            3
 195 #define CBAR_S1_BPSHCFG_NSH             3
 196 #define CBAR_S1_MEMATTR_SHIFT           12
 197 #define CBAR_S1_MEMATTR_MASK            0xf
 198 #define CBAR_S1_MEMATTR_WB              0xf
 199 #define CBAR_TYPE_SHIFT                 16
 200 #define CBAR_TYPE_MASK                  0x3
 201 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 202 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 203 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 204 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 205 #define CBAR_IRPTNDX_SHIFT              24
 206 #define CBAR_IRPTNDX_MASK               0xff
 207
 208 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 209 #define CBA2R_RW64_32BIT                (0 << 0)
 210 #define CBA2R_RW64_64BIT                (1 << 0)
 211 #define CBA2R_VMID_SHIFT                16
 212 #define CBA2R_VMID_MASK                 0xffff
 213
 214 /* Translation context bank */
 215 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
 216 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
 217
 218 #define ARM_SMMU_CB_SCTLR               0x0
 219 #define ARM_SMMU_CB_ACTLR               0x4
 220 #define ARM_SMMU_CB_RESUME              0x8
 221 #define ARM_SMMU_CB_TTBCR2              0x10
 222 #define ARM_SMMU_CB_TTBR0               0x20
 223 #define ARM_SMMU_CB_TTBR1               0x28
 224 #define ARM_SMMU_CB_TTBCR               0x30
 225 #define ARM_SMMU_CB_CONTEXTIDR          0x34
 226 #define ARM_SMMU_CB_S1_MAIR0            0x38
 227 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 228 #define ARM_SMMU_CB_PAR                 0x50
 229 #define ARM_SMMU_CB_FSR                 0x58
 230 #define ARM_SMMU_CB_FAR                 0x60
 231 #define ARM_SMMU_CB_FSYNR0              0x68
 232 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 233 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 234 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 235 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 236 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 237 #define ARM_SMMU_CB_ATS1PR              0x800
 238 #define ARM_SMMU_CB_ATSR                0x8f0
 239
 240 #define SCTLR_S1_ASIDPNE                (1 << 12)
 241 #define SCTLR_CFCFG                     (1 << 7)
 242 #define SCTLR_CFIE                      (1 << 6)
 243 #define SCTLR_CFRE                      (1 << 5)
 244 #define SCTLR_E                         (1 << 4)
 245 #define SCTLR_AFE                       (1 << 2)
 246 #define SCTLR_TRE                       (1 << 1)
 247 #define SCTLR_M                         (1 << 0)
 248
 249 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 250
 251 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 252 #define ARM_MMU500_ACR_SMTNMB_TLBEN     (1 << 8)
 253
 254 #define CB_PAR_F                        (1 << 0)
 255
 256 #define ATSR_ACTIVE                     (1 << 0)
 257
 258 #define RESUME_RETRY                    (0 << 0)
 259 #define RESUME_TERMINATE                (1 << 0)
 260
 261 #define TTBCR2_SEP_SHIFT                15
 262 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 263
 264 #define TTBRn_ASID_SHIFT                48
 265
 266 #define FSR_MULTI                       (1 << 31)
 267 #define FSR_SS                          (1 << 30)
 268 #define FSR_UUT                         (1 << 8)
 269 #define FSR_ASF                         (1 << 7)
 270 #define FSR_TLBLKF                      (1 << 6)
 271 #define FSR_TLBMCF                      (1 << 5)
 272 #define FSR_EF                          (1 << 4)
 273 #define FSR_PF                          (1 << 3)
 274 #define FSR_AFF                         (1 << 2)
 275 #define FSR_TF                          (1 << 1)
 276
 277 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 278                                          FSR_TLBMCF | FSR_TLBLKF)
 279 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 280                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 281
 282 #define FSYNR0_WNR                      (1 << 4)
 283
 284 static int force_stage;
 285 module_param(force_stage, int, S_IRUGO);
 286 MODULE_PARM_DESC(force_stage,
 287         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 288 static bool disable_bypass;
 289 module_param(disable_bypass, bool, S_IRUGO);
 290 MODULE_PARM_DESC(disable_bypass,
 291         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 292
 293 enum arm_smmu_arch_version {
 294         ARM_SMMU_V1,
 295         ARM_SMMU_V1_64K,
 296         ARM_SMMU_V2,
 297 };
 298
 299 enum arm_smmu_implementation {
 300         GENERIC_SMMU,
 301         ARM_MMU500,
 302         CAVIUM_SMMUV2,
 303 };
 304
 305 struct arm_smmu_s2cr {
 306         struct iommu_group              *group;
 307         int                             count;
 308         enum arm_smmu_s2cr_type         type;
 309         enum arm_smmu_s2cr_privcfg      privcfg;
 310         u8                              cbndx;
 311 };
 312
 313 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 314         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 315 }
 316
 317 struct arm_smmu_smr {
 318         u16                             mask;
 319         u16                             id;
 320         bool                            valid;
 321 };
 322
 323 struct arm_smmu_master_cfg {
 324         struct arm_smmu_device          *smmu;
 325         s16                             smendx[];
 326 };
 327 #define INVALID_SMENDX                  -1
 328 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 329 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 330 #define fwspec_smendx(fw, i) \
 331         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 332 #define for_each_cfg_sme(fw, i, idx) \
 333         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 334
 335 struct arm_smmu_device {
 336         struct device                   *dev;
 337
 338         void __iomem                    *base;
 339         unsigned long                   size;
 340         unsigned long                   pgshift;
 341
 342 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 343 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 344 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 345 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 346 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 347 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 348 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 349 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 350 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 351 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 352 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 353 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 354         u32                             features;
 355
 356 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 357         u32                             options;
 358         enum arm_smmu_arch_version      version;
 359         enum arm_smmu_implementation    model;
 360
 361         u32                             num_context_banks;
 362         u32                             num_s2_context_banks;
 363         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 364         atomic_t                        irptndx;
 365
 366         u32                             num_mapping_groups;
 367         u16                             streamid_mask;
 368         u16                             smr_mask_mask;
 369         struct arm_smmu_smr             *smrs;
 370         struct arm_smmu_s2cr            *s2crs;
 371         struct mutex                    stream_map_mutex;
 372
 373         unsigned long                   va_size;
 374         unsigned long                   ipa_size;
 375         unsigned long                   pa_size;
 376         unsigned long                   pgsize_bitmap;
 377
 378         u32                             num_global_irqs;
 379         u32                             num_context_irqs;
 380         unsigned int                    *irqs;
 381
 382         u32                             cavium_id_base; /* Specific to Cavium */
 383
 384         /* IOMMU core code handle */
 385         struct iommu_device             iommu;
 386 };
 387
 388 enum arm_smmu_context_fmt {
 389         ARM_SMMU_CTX_FMT_NONE,
 390         ARM_SMMU_CTX_FMT_AARCH64,
 391         ARM_SMMU_CTX_FMT_AARCH32_L,
 392         ARM_SMMU_CTX_FMT_AARCH32_S,
 393 };
 394
 395 struct arm_smmu_cfg {
 396         u8                              cbndx;
 397         u8                              irptndx;
 398         u32                             cbar;
 399         enum arm_smmu_context_fmt       fmt;
 400 };
 401 #define INVALID_IRPTNDX                 0xff
 402
 403 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
 404 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
 405
 406 enum arm_smmu_domain_stage {
 407         ARM_SMMU_DOMAIN_S1 = 0,
 408         ARM_SMMU_DOMAIN_S2,
 409         ARM_SMMU_DOMAIN_NESTED,
 410 };
 411
 412 struct arm_smmu_domain {
 413         struct arm_smmu_device          *smmu;
 414         struct io_pgtable_ops           *pgtbl_ops;
 415         spinlock_t                      pgtbl_lock;
 416         struct arm_smmu_cfg             cfg;
 417         enum arm_smmu_domain_stage      stage;
 418         struct mutex                    init_mutex; /* Protects smmu pointer */
 419         struct iommu_domain             domain;
 420 };
 421
 422 struct arm_smmu_option_prop {
 423         u32 opt;
 424         const char *prop;
 425 };
 426
 427 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 428
 429 static bool using_legacy_binding, using_generic_binding;
 430
 431 static struct arm_smmu_option_prop arm_smmu_options[] = {
 432         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 433         { 0, NULL},
 434 };
 435
 436 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 437 {
 438         return container_of(dom, struct arm_smmu_domain, domain);
 439 }
 440
 441 static void parse_driver_options(struct arm_smmu_device *smmu)
 442 {
 443         int i = 0;
 444
 445         do {
 446                 if (of_property_read_bool(smmu->dev->of_node,
 447                                                 arm_smmu_options[i].prop)) {
 448                         smmu->options |= arm_smmu_options[i].opt;
 449                         dev_notice(smmu->dev, "option %s\n",
 450                                 arm_smmu_options[i].prop);
 451                 }
 452         } while (arm_smmu_options[++i].opt);
 453 }
 454
 455 static struct device_node *dev_get_dev_node(struct device *dev)
 456 {
 457         if (dev_is_pci(dev)) {
 458                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 459
 460                 while (!pci_is_root_bus(bus))
 461                         bus = bus->parent;
 462                 return of_node_get(bus->bridge->parent->of_node);
 463         }
 464
 465         return of_node_get(dev->of_node);
 466 }
 467
 468 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 469 {
 470         *((__be32 *)data) = cpu_to_be32(alias);
 471         return 0; /* Continue walking */
 472 }
 473
 474 static int __find_legacy_master_phandle(struct device *dev, void *data)
 475 {
 476         struct of_phandle_iterator *it = *(void **)data;
 477         struct device_node *np = it->node;
 478         int err;
 479
 480         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 481                             "#stream-id-cells", 0)
 482                 if (it->node == np) {
 483                         *(void **)data = dev;
 484                         return 1;
 485                 }
 486         it->node = np;
 487         return err == -ENOENT ? 0 : err;
 488 }
 489
 490 static struct platform_driver arm_smmu_driver;
 491 static struct iommu_ops arm_smmu_ops;
 492
 493 static int arm_smmu_register_legacy_master(struct device *dev,
 494                                            struct arm_smmu_device **smmu)
 495 {
 496         struct device *smmu_dev;
 497         struct device_node *np;
 498         struct of_phandle_iterator it;
 499         void *data = &it;
 500         u32 *sids;
 501         __be32 pci_sid;
 502         int err;
 503
 504         np = dev_get_dev_node(dev);
 505         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 506                 of_node_put(np);
 507                 return -ENODEV;
 508         }
 509
 510         it.node = np;
 511         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 512                                      __find_legacy_master_phandle);
 513         smmu_dev = data;
 514         of_node_put(np);
 515         if (err == 0)
 516                 return -ENODEV;
 517         if (err < 0)
 518                 return err;
 519
 520         if (dev_is_pci(dev)) {
 521                 /* "mmu-masters" assumes Stream ID == Requester ID */
 522                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 523                                        &pci_sid);
 524                 it.cur = &pci_sid;
 525                 it.cur_count = 1;
 526         }
 527
 528         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 529                                 &arm_smmu_ops);
 530         if (err)
 531                 return err;
 532
 533         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 534         if (!sids)
 535                 return -ENOMEM;
 536
 537         *smmu = dev_get_drvdata(smmu_dev);
 538         of_phandle_iterator_args(&it, sids, it.cur_count);
 539         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 540         kfree(sids);
 541         return err;
 542 }
 543
 544 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 545 {
 546         int idx;
 547
 548         do {
 549                 idx = find_next_zero_bit(map, end, start);
 550                 if (idx == end)
 551                         return -ENOSPC;
 552         } while (test_and_set_bit(idx, map));
 553
 554         return idx;
 555 }
 556
 557 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 558 {
 559         clear_bit(idx, map);
 560 }
 561
 562 /* Wait for any pending TLB invalidations to complete */
 563 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
 564 {
 565         int count = 0;
 566         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 567
 568         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
 569         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
 570                & sTLBGSTATUS_GSACTIVE) {
 571                 cpu_relax();
 572                 if (++count == TLB_LOOP_TIMEOUT) {
 573                         dev_err_ratelimited(smmu->dev,
 574                         "TLB sync timed out -- SMMU may be deadlocked\n");
 575                         return;
 576                 }
 577                 udelay(1);
 578         }
 579 }
 580
 581 static void arm_smmu_tlb_sync(void *cookie)
 582 {
 583         struct arm_smmu_domain *smmu_domain = cookie;
 584         __arm_smmu_tlb_sync(smmu_domain->smmu);
 585 }
 586
 587 static void arm_smmu_tlb_inv_context(void *cookie)
 588 {
 589         struct arm_smmu_domain *smmu_domain = cookie;
 590         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 591         struct arm_smmu_device *smmu = smmu_domain->smmu;
 592         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 593         void __iomem *base;
 594
 595         if (stage1) {
 596                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 597                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
 598                                base + ARM_SMMU_CB_S1_TLBIASID);
 599         } else {
 600                 base = ARM_SMMU_GR0(smmu);
 601                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
 602                                base + ARM_SMMU_GR0_TLBIVMID);
 603         }
 604
 605         __arm_smmu_tlb_sync(smmu);
 606 }
 607
 608 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 609                                           size_t granule, bool leaf, void *cookie)
 610 {
 611         struct arm_smmu_domain *smmu_domain = cookie;
 612         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 613         struct arm_smmu_device *smmu = smmu_domain->smmu;
 614         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 615         void __iomem *reg;
 616
 617         if (stage1) {
 618                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 619                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 620
 621                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 622                         iova &= ~12UL;
 623                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
 624                         do {
 625                                 writel_relaxed(iova, reg);
 626                                 iova += granule;
 627                         } while (size -= granule);
 628                 } else {
 629                         iova >>= 12;
 630                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
 631                         do {
 632                                 writeq_relaxed(iova, reg);
 633                                 iova += granule >> 12;
 634                         } while (size -= granule);
 635                 }
 636         } else if (smmu->version == ARM_SMMU_V2) {
 637                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 638                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 639                               ARM_SMMU_CB_S2_TLBIIPAS2;
 640                 iova >>= 12;
 641                 do {
 642                         smmu_write_atomic_lq(iova, reg);
 643                         iova += granule >> 12;
 644                 } while (size -= granule);
 645         } else {
 646                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
 647                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
 648         }
 649 }
 650
 651 static const struct iommu_gather_ops arm_smmu_gather_ops = {
 652         .tlb_flush_all  = arm_smmu_tlb_inv_context,
 653         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 654         .tlb_sync       = arm_smmu_tlb_sync,
 655 };
 656
 657 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 658 {
 659         u32 fsr, fsynr;
 660         unsigned long iova;
 661         struct iommu_domain *domain = dev;
 662         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 663         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 664         struct arm_smmu_device *smmu = smmu_domain->smmu;
 665         void __iomem *cb_base;
 666
 667         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 668         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 669
 670         if (!(fsr & FSR_FAULT))
 671                 return IRQ_NONE;
 672
 673         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 674         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 675
 676         dev_err_ratelimited(smmu->dev,
 677         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 678                             fsr, iova, fsynr, cfg->cbndx);
 679
 680         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 681         return IRQ_HANDLED;
 682 }
 683
 684 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 685 {
 686         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 687         struct arm_smmu_device *smmu = dev;
 688         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 689
 690         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 691         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 692         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 693         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 694
 695         if (!gfsr)
 696                 return IRQ_NONE;
 697
 698         dev_err_ratelimited(smmu->dev,
 699                 "Unexpected global fault, this could be serious\n");
 700         dev_err_ratelimited(smmu->dev,
 701                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 702                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 703
 704         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 705         return IRQ_HANDLED;
 706 }
 707
 708 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 709                                        struct io_pgtable_cfg *pgtbl_cfg)
 710 {
 711         u32 reg, reg2;
 712         u64 reg64;
 713         bool stage1;
 714         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 715         struct arm_smmu_device *smmu = smmu_domain->smmu;
 716         void __iomem *cb_base, *gr1_base;
 717
 718         gr1_base = ARM_SMMU_GR1(smmu);
 719         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 720         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 721
 722         if (smmu->version > ARM_SMMU_V1) {
 723                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 724                         reg = CBA2R_RW64_64BIT;
 725                 else
 726                         reg = CBA2R_RW64_32BIT;
 727                 /* 16-bit VMIDs live in CBA2R */
 728                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 729                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
 730
 731                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 732         }
 733
 734         /* CBAR */
 735         reg = cfg->cbar;
 736         if (smmu->version < ARM_SMMU_V2)
 737                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 738
 739         /*
 740          * Use the weakest shareability/memory types, so they are
 741          * overridden by the ttbcr/pte.
 742          */
 743         if (stage1) {
 744                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 745                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 746         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 747                 /* 8-bit VMIDs live in CBAR */
 748                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
 749         }
 750         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 751
 752         /* TTBRs */
 753         if (stage1) {
 754                 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
 755
 756                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 757                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 758                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
 759                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 760                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
 761                         writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 762                 } else {
 763                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 764                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 765                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 766                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 767                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 768                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 769                 }
 770         } else {
 771                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 772                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 773         }
 774
 775         /* TTBCR */
 776         if (stage1) {
 777                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 778                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
 779                         reg2 = 0;
 780                 } else {
 781                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 782                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 783                         reg2 |= TTBCR2_SEP_UPSTREAM;
 784                 }
 785                 if (smmu->version > ARM_SMMU_V1)
 786                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
 787         } else {
 788                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 789         }
 790         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 791
 792         /* MAIRs (stage-1 only) */
 793         if (stage1) {
 794                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 795                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
 796                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
 797                 } else {
 798                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 799                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 800                 }
 801                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 802                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
 803         }
 804
 805         /* SCTLR */
 806         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 807         if (stage1)
 808                 reg |= SCTLR_S1_ASIDPNE;
 809 #ifdef __BIG_ENDIAN
 810         reg |= SCTLR_E;
 811 #endif
 812         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 813 }
 814
 815 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 816                                         struct arm_smmu_device *smmu)
 817 {
 818         int irq, start, ret = 0;
 819         unsigned long ias, oas;
 820         struct io_pgtable_ops *pgtbl_ops;
 821         struct io_pgtable_cfg pgtbl_cfg;
 822         enum io_pgtable_fmt fmt;
 823         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 824         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 825
 826         mutex_lock(&smmu_domain->init_mutex);
 827         if (smmu_domain->smmu)
 828                 goto out_unlock;
 829
 830         /*
 831          * Mapping the requested stage onto what we support is surprisingly
 832          * complicated, mainly because the spec allows S1+S2 SMMUs without
 833          * support for nested translation. That means we end up with the
 834          * following table:
 835          *
 836          * Requested        Supported        Actual
 837          *     S1               N              S1
 838          *     S1             S1+S2            S1
 839          *     S1               S2             S2
 840          *     S1               S1             S1
 841          *     N                N              N
 842          *     N              S1+S2            S2
 843          *     N                S2             S2
 844          *     N                S1             S1
 845          *
 846          * Note that you can't actually request stage-2 mappings.
 847          */
 848         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 849                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 850         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 851                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 852
 853         /*
 854          * Choosing a suitable context format is even more fiddly. Until we
 855          * grow some way for the caller to express a preference, and/or move
 856          * the decision into the io-pgtable code where it arguably belongs,
 857          * just aim for the closest thing to the rest of the system, and hope
 858          * that the hardware isn't esoteric enough that we can't assume AArch64
 859          * support to be a superset of AArch32 support...
 860          */
 861         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 862                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 863         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 864             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 865             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 866             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 867                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 868         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 869             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 870                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 871                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 872                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 873
 874         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 875                 ret = -EINVAL;
 876                 goto out_unlock;
 877         }
 878
 879         switch (smmu_domain->stage) {
 880         case ARM_SMMU_DOMAIN_S1:
 881                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 882                 start = smmu->num_s2_context_banks;
 883                 ias = smmu->va_size;
 884                 oas = smmu->ipa_size;
 885                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 886                         fmt = ARM_64_LPAE_S1;
 887                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 888                         fmt = ARM_32_LPAE_S1;
 889                         ias = min(ias, 32UL);
 890                         oas = min(oas, 40UL);
 891                 } else {
 892                         fmt = ARM_V7S;
 893                         ias = min(ias, 32UL);
 894                         oas = min(oas, 32UL);
 895                 }
 896                 break;
 897         case ARM_SMMU_DOMAIN_NESTED:
 898                 /*
 899                  * We will likely want to change this if/when KVM gets
 900                  * involved.
 901                  */
 902         case ARM_SMMU_DOMAIN_S2:
 903                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 904                 start = 0;
 905                 ias = smmu->ipa_size;
 906                 oas = smmu->pa_size;
 907                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 908                         fmt = ARM_64_LPAE_S2;
 909                 } else {
 910                         fmt = ARM_32_LPAE_S2;
 911                         ias = min(ias, 40UL);
 912                         oas = min(oas, 40UL);
 913                 }
 914                 break;
 915         default:
 916                 ret = -EINVAL;
 917                 goto out_unlock;
 918         }
 919
 920         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 921                                       smmu->num_context_banks);
 922         if (ret < 0)
 923                 goto out_unlock;
 924
 925         cfg->cbndx = ret;
 926         if (smmu->version < ARM_SMMU_V2) {
 927                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 928                 cfg->irptndx %= smmu->num_context_irqs;
 929         } else {
 930                 cfg->irptndx = cfg->cbndx;
 931         }
 932
 933         pgtbl_cfg = (struct io_pgtable_cfg) {
 934                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 935                 .ias            = ias,
 936                 .oas            = oas,
 937                 .tlb            = &arm_smmu_gather_ops,
 938                 .iommu_dev      = smmu->dev,
 939         };
 940
 941         smmu_domain->smmu = smmu;
 942         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 943         if (!pgtbl_ops) {
 944                 ret = -ENOMEM;
 945                 goto out_clear_smmu;
 946         }
 947
 948         /* Update the domain's page sizes to reflect the page table format */
 949         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 950         domain->geometry.aperture_end = (1UL << ias) - 1;
 951         domain->geometry.force_aperture = true;
 952
 953         /* Initialise the context bank with our page table cfg */
 954         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 955
 956         /*
 957          * Request context fault interrupt. Do this last to avoid the
 958          * handler seeing a half-initialised domain state.
 959          */
 960         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 961         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 962                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 963         if (ret < 0) {
 964                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 965                         cfg->irptndx, irq);
 966                 cfg->irptndx = INVALID_IRPTNDX;
 967         }
 968
 969         mutex_unlock(&smmu_domain->init_mutex);
 970
 971         /* Publish page table ops for map/unmap */
 972         smmu_domain->pgtbl_ops = pgtbl_ops;
 973         return 0;
 974
 975 out_clear_smmu:
 976         smmu_domain->smmu = NULL;
 977 out_unlock:
 978         mutex_unlock(&smmu_domain->init_mutex);
 979         return ret;
 980 }
 981
 982 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 983 {
 984         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 985         struct arm_smmu_device *smmu = smmu_domain->smmu;
 986         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 987         void __iomem *cb_base;
 988         int irq;
 989
 990         if (!smmu)
 991                 return;
 992
 993         /*
 994          * Disable the context bank and free the page tables before freeing
 995          * it.
 996          */
 997         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 998         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
 999
1000         if (cfg->irptndx != INVALID_IRPTNDX) {
1001                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1002                 devm_free_irq(smmu->dev, irq, domain);
1003         }
1004
1005         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1006         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1007 }
1008
1009 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1010 {
1011         struct arm_smmu_domain *smmu_domain;
1012
1013         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1014                 return NULL;
1015         /*
1016          * Allocate the domain and initialise some of its data structures.
1017          * We can't really do anything meaningful until we've added a
1018          * master.
1019          */
1020         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1021         if (!smmu_domain)
1022                 return NULL;
1023
1024         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1025             iommu_get_dma_cookie(&smmu_domain->domain))) {
1026                 kfree(smmu_domain);
1027                 return NULL;
1028         }
1029
1030         mutex_init(&smmu_domain->init_mutex);
1031         spin_lock_init(&smmu_domain->pgtbl_lock);
1032
1033         return &smmu_domain->domain;
1034 }
1035
1036 static void arm_smmu_domain_free(struct iommu_domain *domain)
1037 {
1038         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1039
1040         /*
1041          * Free the domain resources. We assume that all devices have
1042          * already been detached.
1043          */
1044         iommu_put_dma_cookie(domain);
1045         arm_smmu_destroy_domain_context(domain);
1046         kfree(smmu_domain);
1047 }
1048
1049 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1050 {
1051         struct arm_smmu_smr *smr = smmu->smrs + idx;
1052         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1053
1054         if (smr->valid)
1055                 reg |= SMR_VALID;
1056         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1057 }
1058
1059 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1060 {
1061         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1062         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1063                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1064                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1065
1066         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1067 }
1068
1069 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1070 {
1071         arm_smmu_write_s2cr(smmu, idx);
1072         if (smmu->smrs)
1073                 arm_smmu_write_smr(smmu, idx);
1074 }
1075
1076 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1077 {
1078         struct arm_smmu_smr *smrs = smmu->smrs;
1079         int i, free_idx = -ENOSPC;
1080
1081         /* Stream indexing is blissfully easy */
1082         if (!smrs)
1083                 return id;
1084
1085         /* Validating SMRs is... less so */
1086         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1087                 if (!smrs[i].valid) {
1088                         /*
1089                          * Note the first free entry we come across, which
1090                          * we'll claim in the end if nothing else matches.
1091                          */
1092                         if (free_idx < 0)
1093                                 free_idx = i;
1094                         continue;
1095                 }
1096                 /*
1097                  * If the new entry is _entirely_ matched by an existing entry,
1098                  * then reuse that, with the guarantee that there also cannot
1099                  * be any subsequent conflicting entries. In normal use we'd
1100                  * expect simply identical entries for this case, but there's
1101                  * no harm in accommodating the generalisation.
1102                  */
1103                 if ((mask & smrs[i].mask) == mask &&
1104                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1105                         return i;
1106                 /*
1107                  * If the new entry has any other overlap with an existing one,
1108                  * though, then there always exists at least one stream ID
1109                  * which would cause a conflict, and we can't allow that risk.
1110                  */
1111                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1112                         return -EINVAL;
1113         }
1114
1115         return free_idx;
1116 }
1117
1118 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1119 {
1120         if (--smmu->s2crs[idx].count)
1121                 return false;
1122
1123         smmu->s2crs[idx] = s2cr_init_val;
1124         if (smmu->smrs)
1125                 smmu->smrs[idx].valid = false;
1126
1127         return true;
1128 }
1129
1130 static int arm_smmu_master_alloc_smes(struct device *dev)
1131 {
1132         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1133         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1134         struct arm_smmu_device *smmu = cfg->smmu;
1135         struct arm_smmu_smr *smrs = smmu->smrs;
1136         struct iommu_group *group;
1137         int i, idx, ret;
1138
1139         mutex_lock(&smmu->stream_map_mutex);
1140         /* Figure out a viable stream map entry allocation */
1141         for_each_cfg_sme(fwspec, i, idx) {
1142                 u16 sid = fwspec->ids[i];
1143                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1144
1145                 if (idx != INVALID_SMENDX) {
1146                         ret = -EEXIST;
1147                         goto out_err;
1148                 }
1149
1150                 ret = arm_smmu_find_sme(smmu, sid, mask);
1151                 if (ret < 0)
1152                         goto out_err;
1153
1154                 idx = ret;
1155                 if (smrs && smmu->s2crs[idx].count == 0) {
1156                         smrs[idx].id = sid;
1157                         smrs[idx].mask = mask;
1158                         smrs[idx].valid = true;
1159                 }
1160                 smmu->s2crs[idx].count++;
1161                 cfg->smendx[i] = (s16)idx;
1162         }
1163
1164         group = iommu_group_get_for_dev(dev);
1165         if (!group)
1166                 group = ERR_PTR(-ENOMEM);
1167         if (IS_ERR(group)) {
1168                 ret = PTR_ERR(group);
1169                 goto out_err;
1170         }
1171         iommu_group_put(group);
1172
1173         /* It worked! Now, poke the actual hardware */
1174         for_each_cfg_sme(fwspec, i, idx) {
1175                 arm_smmu_write_sme(smmu, idx);
1176                 smmu->s2crs[idx].group = group;
1177         }
1178
1179         mutex_unlock(&smmu->stream_map_mutex);
1180         return 0;
1181
1182 out_err:
1183         while (i--) {
1184                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1185                 cfg->smendx[i] = INVALID_SMENDX;
1186         }
1187         mutex_unlock(&smmu->stream_map_mutex);
1188         return ret;
1189 }
1190
1191 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1192 {
1193         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1194         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1195         int i, idx;
1196
1197         mutex_lock(&smmu->stream_map_mutex);
1198         for_each_cfg_sme(fwspec, i, idx) {
1199                 if (arm_smmu_free_sme(smmu, idx))
1200                         arm_smmu_write_sme(smmu, idx);
1201                 cfg->smendx[i] = INVALID_SMENDX;
1202         }
1203         mutex_unlock(&smmu->stream_map_mutex);
1204 }
1205
1206 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1207                                       struct iommu_fwspec *fwspec)
1208 {
1209         struct arm_smmu_device *smmu = smmu_domain->smmu;
1210         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1211         enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1212         u8 cbndx = smmu_domain->cfg.cbndx;
1213         int i, idx;
1214
1215         for_each_cfg_sme(fwspec, i, idx) {
1216                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1217                         continue;
1218
1219                 s2cr[idx].type = type;
1220                 s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV;
1221                 s2cr[idx].cbndx = cbndx;
1222                 arm_smmu_write_s2cr(smmu, idx);
1223         }
1224         return 0;
1225 }
1226
1227 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1228 {
1229         int ret;
1230         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1231         struct arm_smmu_device *smmu;
1232         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1233
1234         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1235                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1236                 return -ENXIO;
1237         }
1238
1239         /*
1240          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1241          * domains between of_xlate() and add_device() - we have no way to cope
1242          * with that, so until ARM gets converted to rely on groups and default
1243          * domains, just say no (but more politely than by dereferencing NULL).
1244          * This should be at least a WARN_ON once that's sorted.
1245          */
1246         if (!fwspec->iommu_priv)
1247                 return -ENODEV;
1248
1249         smmu = fwspec_smmu(fwspec);
1250         /* Ensure that the domain is finalised */
1251         ret = arm_smmu_init_domain_context(domain, smmu);
1252         if (ret < 0)
1253                 return ret;
1254
1255         /*
1256          * Sanity check the domain. We don't support domains across
1257          * different SMMUs.
1258          */
1259         if (smmu_domain->smmu != smmu) {
1260                 dev_err(dev,
1261                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1262                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1263                 return -EINVAL;
1264         }
1265
1266         /* Looks ok, so add the device to the domain */
1267         return arm_smmu_domain_add_master(smmu_domain, fwspec);
1268 }
1269
1270 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1271                         phys_addr_t paddr, size_t size, int prot)
1272 {
1273         int ret;
1274         unsigned long flags;
1275         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1276         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1277
1278         if (!ops)
1279                 return -ENODEV;
1280
1281         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1282         ret = ops->map(ops, iova, paddr, size, prot);
1283         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1284         return ret;
1285 }
1286
1287 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1288                              size_t size)
1289 {
1290         size_t ret;
1291         unsigned long flags;
1292         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1293         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1294
1295         if (!ops)
1296                 return 0;
1297
1298         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1299         ret = ops->unmap(ops, iova, size);
1300         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1301         return ret;
1302 }
1303
1304 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1305                                               dma_addr_t iova)
1306 {
1307         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1308         struct arm_smmu_device *smmu = smmu_domain->smmu;
1309         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1310         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1311         struct device *dev = smmu->dev;
1312         void __iomem *cb_base;
1313         u32 tmp;
1314         u64 phys;
1315         unsigned long va;
1316
1317         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1318
1319         /* ATS1 registers can only be written atomically */
1320         va = iova & ~0xfffUL;
1321         if (smmu->version == ARM_SMMU_V2)
1322                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1323         else /* Register is only 32-bit in v1 */
1324                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1325
1326         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1327                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1328                 dev_err(dev,
1329                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1330                         &iova);
1331                 return ops->iova_to_phys(ops, iova);
1332         }
1333
1334         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1335         if (phys & CB_PAR_F) {
1336                 dev_err(dev, "translation fault!\n");
1337                 dev_err(dev, "PAR = 0x%llx\n", phys);
1338                 return 0;
1339         }
1340
1341         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1342 }
1343
1344 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1345                                         dma_addr_t iova)
1346 {
1347         phys_addr_t ret;
1348         unsigned long flags;
1349         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1350         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1351
1352         if (!ops)
1353                 return 0;
1354
1355         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1356         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1357                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1358                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1359         } else {
1360                 ret = ops->iova_to_phys(ops, iova);
1361         }
1362
1363         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1364
1365         return ret;
1366 }
1367
1368 static bool arm_smmu_capable(enum iommu_cap cap)
1369 {
1370         switch (cap) {
1371         case IOMMU_CAP_CACHE_COHERENCY:
1372                 /*
1373                  * Return true here as the SMMU can always send out coherent
1374                  * requests.
1375                  */
1376                 return true;
1377         case IOMMU_CAP_INTR_REMAP:
1378                 return true; /* MSIs are just memory writes */
1379         case IOMMU_CAP_NOEXEC:
1380                 return true;
1381         default:
1382                 return false;
1383         }
1384 }
1385
1386 static int arm_smmu_match_node(struct device *dev, void *data)
1387 {
1388         return dev->fwnode == data;
1389 }
1390
1391 static
1392 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1393 {
1394         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1395                                                 fwnode, arm_smmu_match_node);
1396         put_device(dev);
1397         return dev ? dev_get_drvdata(dev) : NULL;
1398 }
1399
1400 static int arm_smmu_add_device(struct device *dev)
1401 {
1402         struct arm_smmu_device *smmu;
1403         struct arm_smmu_master_cfg *cfg;
1404         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1405         int i, ret;
1406
1407         if (using_legacy_binding) {
1408                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1409                 fwspec = dev->iommu_fwspec;
1410                 if (ret)
1411                         goto out_free;
1412         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1413                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1414         } else {
1415                 return -ENODEV;
1416         }
1417
1418         ret = -EINVAL;
1419         for (i = 0; i < fwspec->num_ids; i++) {
1420                 u16 sid = fwspec->ids[i];
1421                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1422
1423                 if (sid & ~smmu->streamid_mask) {
1424                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1425                                 sid, smmu->streamid_mask);
1426                         goto out_free;
1427                 }
1428                 if (mask & ~smmu->smr_mask_mask) {
1429                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1430                                 sid, smmu->smr_mask_mask);
1431                         goto out_free;
1432                 }
1433         }
1434
1435         ret = -ENOMEM;
1436         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1437                       GFP_KERNEL);
1438         if (!cfg)
1439                 goto out_free;
1440
1441         cfg->smmu = smmu;
1442         fwspec->iommu_priv = cfg;
1443         while (i--)
1444                 cfg->smendx[i] = INVALID_SMENDX;
1445
1446         ret = arm_smmu_master_alloc_smes(dev);
1447         if (ret)
1448                 goto out_free;
1449
1450         iommu_device_link(&smmu->iommu, dev);
1451
1452         return 0;
1453
1454 out_free:
1455         if (fwspec)
1456                 kfree(fwspec->iommu_priv);
1457         iommu_fwspec_free(dev);
1458         return ret;
1459 }
1460
1461 static void arm_smmu_remove_device(struct device *dev)
1462 {
1463         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1464         struct arm_smmu_master_cfg *cfg;
1465         struct arm_smmu_device *smmu;
1466
1467
1468         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1469                 return;
1470
1471         cfg  = fwspec->iommu_priv;
1472         smmu = cfg->smmu;
1473
1474         iommu_device_unlink(&smmu->iommu, dev);
1475         arm_smmu_master_free_smes(fwspec);
1476         iommu_group_remove_device(dev);
1477         kfree(fwspec->iommu_priv);
1478         iommu_fwspec_free(dev);
1479 }
1480
1481 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1482 {
1483         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1484         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1485         struct iommu_group *group = NULL;
1486         int i, idx;
1487
1488         for_each_cfg_sme(fwspec, i, idx) {
1489                 if (group && smmu->s2crs[idx].group &&
1490                     group != smmu->s2crs[idx].group)
1491                         return ERR_PTR(-EINVAL);
1492
1493                 group = smmu->s2crs[idx].group;
1494         }
1495
1496         if (group)
1497                 return iommu_group_ref_get(group);
1498
1499         if (dev_is_pci(dev))
1500                 group = pci_device_group(dev);
1501         else
1502                 group = generic_device_group(dev);
1503
1504         return group;
1505 }
1506
1507 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1508                                     enum iommu_attr attr, void *data)
1509 {
1510         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1511
1512         switch (attr) {
1513         case DOMAIN_ATTR_NESTING:
1514                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1515                 return 0;
1516         default:
1517                 return -ENODEV;
1518         }
1519 }
1520
1521 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1522                                     enum iommu_attr attr, void *data)
1523 {
1524         int ret = 0;
1525         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1526
1527         mutex_lock(&smmu_domain->init_mutex);
1528
1529         switch (attr) {
1530         case DOMAIN_ATTR_NESTING:
1531                 if (smmu_domain->smmu) {
1532                         ret = -EPERM;
1533                         goto out_unlock;
1534                 }
1535
1536                 if (*(int *)data)
1537                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1538                 else
1539                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1540
1541                 break;
1542         default:
1543                 ret = -ENODEV;
1544         }
1545
1546 out_unlock:
1547         mutex_unlock(&smmu_domain->init_mutex);
1548         return ret;
1549 }
1550
1551 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1552 {
1553         u32 fwid = 0;
1554
1555         if (args->args_count > 0)
1556                 fwid |= (u16)args->args[0];
1557
1558         if (args->args_count > 1)
1559                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1560
1561         return iommu_fwspec_add_ids(dev, &fwid, 1);
1562 }
1563
1564 static struct iommu_ops arm_smmu_ops = {
1565         .capable                = arm_smmu_capable,
1566         .domain_alloc           = arm_smmu_domain_alloc,
1567         .domain_free            = arm_smmu_domain_free,
1568         .attach_dev             = arm_smmu_attach_dev,
1569         .map                    = arm_smmu_map,
1570         .unmap                  = arm_smmu_unmap,
1571         .map_sg                 = default_iommu_map_sg,
1572         .iova_to_phys           = arm_smmu_iova_to_phys,
1573         .add_device             = arm_smmu_add_device,
1574         .remove_device          = arm_smmu_remove_device,
1575         .device_group           = arm_smmu_device_group,
1576         .domain_get_attr        = arm_smmu_domain_get_attr,
1577         .domain_set_attr        = arm_smmu_domain_set_attr,
1578         .of_xlate               = arm_smmu_of_xlate,
1579         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1580 };
1581
1582 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1583 {
1584         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1585         void __iomem *cb_base;
1586         int i;
1587         u32 reg, major;
1588
1589         /* clear global FSR */
1590         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1591         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1592
1593         /*
1594          * Reset stream mapping groups: Initial values mark all SMRn as
1595          * invalid and all S2CRn as bypass unless overridden.
1596          */
1597         for (i = 0; i < smmu->num_mapping_groups; ++i)
1598                 arm_smmu_write_sme(smmu, i);
1599
1600         if (smmu->model == ARM_MMU500) {
1601                 /*
1602                  * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1603                  * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1604                  * bit is only present in MMU-500r2 onwards.
1605                  */
1606                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1607                 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1608                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1609                 if (major >= 2)
1610                         reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1611                 /*
1612                  * Allow unmatched Stream IDs to allocate bypass
1613                  * TLB entries for reduced latency.
1614                  */
1615                 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN;
1616                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1617         }
1618
1619         /* Make sure all context banks are disabled and clear CB_FSR  */
1620         for (i = 0; i < smmu->num_context_banks; ++i) {
1621                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1622                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1623                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1624                 /*
1625                  * Disable MMU-500's not-particularly-beneficial next-page
1626                  * prefetcher for the sake of errata #841119 and #826419.
1627                  */
1628                 if (smmu->model == ARM_MMU500) {
1629                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1630                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1631                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1632                 }
1633         }
1634
1635         /* Invalidate the TLB, just in case */
1636         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1637         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1638
1639         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1640
1641         /* Enable fault reporting */
1642         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1643
1644         /* Disable TLB broadcasting. */
1645         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1646
1647         /* Enable client access, handling unmatched streams as appropriate */
1648         reg &= ~sCR0_CLIENTPD;
1649         if (disable_bypass)
1650                 reg |= sCR0_USFCFG;
1651         else
1652                 reg &= ~sCR0_USFCFG;
1653
1654         /* Disable forced broadcasting */
1655         reg &= ~sCR0_FB;
1656
1657         /* Don't upgrade barriers */
1658         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1659
1660         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1661                 reg |= sCR0_VMID16EN;
1662
1663         /* Push the button */
1664         __arm_smmu_tlb_sync(smmu);
1665         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1666 }
1667
1668 static int arm_smmu_id_size_to_bits(int size)
1669 {
1670         switch (size) {
1671         case 0:
1672                 return 32;
1673         case 1:
1674                 return 36;
1675         case 2:
1676                 return 40;
1677         case 3:
1678                 return 42;
1679         case 4:
1680                 return 44;
1681         case 5:
1682         default:
1683                 return 48;
1684         }
1685 }
1686
1687 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1688 {
1689         unsigned long size;
1690         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1691         u32 id;
1692         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1693         int i;
1694
1695         dev_notice(smmu->dev, "probing hardware configuration...\n");
1696         dev_notice(smmu->dev, "SMMUv%d with:\n",
1697                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1698
1699         /* ID0 */
1700         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1701
1702         /* Restrict available stages based on module parameter */
1703         if (force_stage == 1)
1704                 id &= ~(ID0_S2TS | ID0_NTS);
1705         else if (force_stage == 2)
1706                 id &= ~(ID0_S1TS | ID0_NTS);
1707
1708         if (id & ID0_S1TS) {
1709                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1710                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1711         }
1712
1713         if (id & ID0_S2TS) {
1714                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1715                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1716         }
1717
1718         if (id & ID0_NTS) {
1719                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1720                 dev_notice(smmu->dev, "\tnested translation\n");
1721         }
1722
1723         if (!(smmu->features &
1724                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1725                 dev_err(smmu->dev, "\tno translation support!\n");
1726                 return -ENODEV;
1727         }
1728
1729         if ((id & ID0_S1TS) &&
1730                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1731                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1732                 dev_notice(smmu->dev, "\taddress translation ops\n");
1733         }
1734
1735         /*
1736          * In order for DMA API calls to work properly, we must defer to what
1737          * the FW says about coherency, regardless of what the hardware claims.
1738          * Fortunately, this also opens up a workaround for systems where the
1739          * ID register value has ended up configured incorrectly.
1740          */
1741         cttw_reg = !!(id & ID0_CTTW);
1742         if (cttw_fw || cttw_reg)
1743                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1744                            cttw_fw ? "" : "non-");
1745         if (cttw_fw != cttw_reg)
1746                 dev_notice(smmu->dev,
1747                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1748
1749         /* Max. number of entries we have for stream matching/indexing */
1750         size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1751         smmu->streamid_mask = size - 1;
1752         if (id & ID0_SMS) {
1753                 u32 smr;
1754
1755                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1756                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1757                 if (size == 0) {
1758                         dev_err(smmu->dev,
1759                                 "stream-matching supported, but no SMRs present!\n");
1760                         return -ENODEV;
1761                 }
1762
1763                 /*
1764                  * SMR.ID bits may not be preserved if the corresponding MASK
1765                  * bits are set, so check each one separately. We can reject
1766                  * masters later if they try to claim IDs outside these masks.
1767                  */
1768                 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1769                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1770                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1771                 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1772
1773                 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1774                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1775                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1776                 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1777
1778                 /* Zero-initialised to mark as invalid */
1779                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1780                                           GFP_KERNEL);
1781                 if (!smmu->smrs)
1782                         return -ENOMEM;
1783
1784                 dev_notice(smmu->dev,
1785                            "\tstream matching with %lu register groups, mask 0x%x",
1786                            size, smmu->smr_mask_mask);
1787         }
1788         /* s2cr->type == 0 means translation, so initialise explicitly */
1789         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1790                                          GFP_KERNEL);
1791         if (!smmu->s2crs)
1792                 return -ENOMEM;
1793         for (i = 0; i < size; i++)
1794                 smmu->s2crs[i] = s2cr_init_val;
1795
1796         smmu->num_mapping_groups = size;
1797         mutex_init(&smmu->stream_map_mutex);
1798
1799         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1800                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1801                 if (!(id & ID0_PTFS_NO_AARCH32S))
1802                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1803         }
1804
1805         /* ID1 */
1806         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1807         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1808
1809         /* Check for size mismatch of SMMU address space from mapped region */
1810         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1811         size *= 2 << smmu->pgshift;
1812         if (smmu->size != size)
1813                 dev_warn(smmu->dev,
1814                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1815                         size, smmu->size);
1816
1817         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1818         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1819         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1820                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1821                 return -ENODEV;
1822         }
1823         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1824                    smmu->num_context_banks, smmu->num_s2_context_banks);
1825         /*
1826          * Cavium CN88xx erratum #27704.
1827          * Ensure ASID and VMID allocation is unique across all SMMUs in
1828          * the system.
1829          */
1830         if (smmu->model == CAVIUM_SMMUV2) {
1831                 smmu->cavium_id_base =
1832                         atomic_add_return(smmu->num_context_banks,
1833                                           &cavium_smmu_context_count);
1834                 smmu->cavium_id_base -= smmu->num_context_banks;
1835         }
1836
1837         /* ID2 */
1838         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1839         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1840         smmu->ipa_size = size;
1841
1842         /* The output mask is also applied for bypass */
1843         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1844         smmu->pa_size = size;
1845
1846         if (id & ID2_VMID16)
1847                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1848
1849         /*
1850          * What the page table walker can address actually depends on which
1851          * descriptor format is in use, but since a) we don't know that yet,
1852          * and b) it can vary per context bank, this will have to do...
1853          */
1854         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1855                 dev_warn(smmu->dev,
1856                          "failed to set DMA mask for table walker\n");
1857
1858         if (smmu->version < ARM_SMMU_V2) {
1859                 smmu->va_size = smmu->ipa_size;
1860                 if (smmu->version == ARM_SMMU_V1_64K)
1861                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1862         } else {
1863                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1864                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1865                 if (id & ID2_PTFS_4K)
1866                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1867                 if (id & ID2_PTFS_16K)
1868                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1869                 if (id & ID2_PTFS_64K)
1870                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1871         }
1872
1873         /* Now we've corralled the various formats, what'll it do? */
1874         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1875                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1876         if (smmu->features &
1877             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1878                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1879         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1880                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1881         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1882                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1883
1884         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1885                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1886         else
1887                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1888         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1889                    smmu->pgsize_bitmap);
1890
1891
1892         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1893                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1894                            smmu->va_size, smmu->ipa_size);
1895
1896         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1897                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1898                            smmu->ipa_size, smmu->pa_size);
1899
1900         return 0;
1901 }
1902
1903 struct arm_smmu_match_data {
1904         enum arm_smmu_arch_version version;
1905         enum arm_smmu_implementation model;
1906 };
1907
1908 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1909 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1910
1911 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1912 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1913 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1914 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1915 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1916
1917 static const struct of_device_id arm_smmu_of_match[] = {
1918         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1919         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1920         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1921         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1922         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1923         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1924         { },
1925 };
1926 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1927
1928 #ifdef CONFIG_ACPI
1929 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1930 {
1931         int ret = 0;
1932
1933         switch (model) {
1934         case ACPI_IORT_SMMU_V1:
1935         case ACPI_IORT_SMMU_CORELINK_MMU400:
1936                 smmu->version = ARM_SMMU_V1;
1937                 smmu->model = GENERIC_SMMU;
1938                 break;
1939         case ACPI_IORT_SMMU_V2:
1940                 smmu->version = ARM_SMMU_V2;
1941                 smmu->model = GENERIC_SMMU;
1942                 break;
1943         case ACPI_IORT_SMMU_CORELINK_MMU500:
1944                 smmu->version = ARM_SMMU_V2;
1945                 smmu->model = ARM_MMU500;
1946                 break;
1947         default:
1948                 ret = -ENODEV;
1949         }
1950
1951         return ret;
1952 }
1953
1954 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1955                                       struct arm_smmu_device *smmu)
1956 {
1957         struct device *dev = smmu->dev;
1958         struct acpi_iort_node *node =
1959                 *(struct acpi_iort_node **)dev_get_platdata(dev);
1960         struct acpi_iort_smmu *iort_smmu;
1961         int ret;
1962
1963         /* Retrieve SMMU1/2 specific data */
1964         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1965
1966         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1967         if (ret < 0)
1968                 return ret;
1969
1970         /* Ignore the configuration access interrupt */
1971         smmu->num_global_irqs = 1;
1972
1973         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1974                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1975
1976         return 0;
1977 }
1978 #else
1979 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1980                                              struct arm_smmu_device *smmu)
1981 {
1982         return -ENODEV;
1983 }
1984 #endif
1985
1986 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
1987                                     struct arm_smmu_device *smmu)
1988 {
1989         const struct arm_smmu_match_data *data;
1990         struct device *dev = &pdev->dev;
1991         bool legacy_binding;
1992
1993         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1994                                  &smmu->num_global_irqs)) {
1995                 dev_err(dev, "missing #global-interrupts property\n");
1996                 return -ENODEV;
1997         }
1998
1999         data = of_device_get_match_data(dev);
2000         smmu->version = data->version;
2001         smmu->model = data->model;
2002
2003         parse_driver_options(smmu);
2004
2005         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2006         if (legacy_binding && !using_generic_binding) {
2007                 if (!using_legacy_binding)
2008                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2009                 using_legacy_binding = true;
2010         } else if (!legacy_binding && !using_legacy_binding) {
2011                 using_generic_binding = true;
2012         } else {
2013                 dev_err(dev, "not probing due to mismatched DT properties\n");
2014                 return -ENODEV;
2015         }
2016
2017         if (of_dma_is_coherent(dev->of_node))
2018                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2019
2020         return 0;
2021 }
2022
2023 static int arm_smmu_device_probe(struct platform_device *pdev)
2024 {
2025         struct resource *res;
2026         resource_size_t ioaddr;
2027         struct arm_smmu_device *smmu;
2028         struct device *dev = &pdev->dev;
2029         int num_irqs, i, err;
2030
2031         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2032         if (!smmu) {
2033                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2034                 return -ENOMEM;
2035         }
2036         smmu->dev = dev;
2037
2038         if (dev->of_node)
2039                 err = arm_smmu_device_dt_probe(pdev, smmu);
2040         else
2041                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2042
2043         if (err)
2044                 return err;
2045
2046         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2047         ioaddr = res->start;
2048         smmu->base = devm_ioremap_resource(dev, res);
2049         if (IS_ERR(smmu->base))
2050                 return PTR_ERR(smmu->base);
2051         smmu->size = resource_size(res);
2052
2053         num_irqs = 0;
2054         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2055                 num_irqs++;
2056                 if (num_irqs > smmu->num_global_irqs)
2057                         smmu->num_context_irqs++;
2058         }
2059
2060         if (!smmu->num_context_irqs) {
2061                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2062                         num_irqs, smmu->num_global_irqs + 1);
2063                 return -ENODEV;
2064         }
2065
2066         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
2067                                   GFP_KERNEL);
2068         if (!smmu->irqs) {
2069                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2070                 return -ENOMEM;
2071         }
2072
2073         for (i = 0; i < num_irqs; ++i) {
2074                 int irq = platform_get_irq(pdev, i);
2075
2076                 if (irq < 0) {
2077                         dev_err(dev, "failed to get irq index %d\n", i);
2078                         return -ENODEV;
2079                 }
2080                 smmu->irqs[i] = irq;
2081         }
2082
2083         err = arm_smmu_device_cfg_probe(smmu);
2084         if (err)
2085                 return err;
2086
2087         if (smmu->version == ARM_SMMU_V2 &&
2088             smmu->num_context_banks != smmu->num_context_irqs) {
2089                 dev_err(dev,
2090                         "found only %d context interrupt(s) but %d required\n",
2091                         smmu->num_context_irqs, smmu->num_context_banks);
2092                 return -ENODEV;
2093         }
2094
2095         for (i = 0; i < smmu->num_global_irqs; ++i) {
2096                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2097                                        arm_smmu_global_fault,
2098                                        IRQF_SHARED,
2099                                        "arm-smmu global fault",
2100                                        smmu);
2101                 if (err) {
2102                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2103                                 i, smmu->irqs[i]);
2104                         return err;
2105                 }
2106         }
2107
2108         err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2109                                      "smmu.%pa", &ioaddr);
2110         if (err) {
2111                 dev_err(dev, "Failed to register iommu in sysfs\n");
2112                 return err;
2113         }
2114
2115         iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
2116         iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
2117
2118         err = iommu_device_register(&smmu->iommu);
2119         if (err) {
2120                 dev_err(dev, "Failed to register iommu\n");
2121                 return err;
2122         }
2123
2124         iommu_register_instance(dev->fwnode, &arm_smmu_ops);
2125         platform_set_drvdata(pdev, smmu);
2126         arm_smmu_device_reset(smmu);
2127
2128         /* Oh, for a proper bus abstraction */
2129         if (!iommu_present(&platform_bus_type))
2130                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2131 #ifdef CONFIG_ARM_AMBA
2132         if (!iommu_present(&amba_bustype))
2133                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2134 #endif
2135 #ifdef CONFIG_PCI
2136         if (!iommu_present(&pci_bus_type)) {
2137                 pci_request_acs();
2138                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2139         }
2140 #endif
2141         return 0;
2142 }
2143
2144 static int arm_smmu_device_remove(struct platform_device *pdev)
2145 {
2146         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2147
2148         if (!smmu)
2149                 return -ENODEV;
2150
2151         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2152                 dev_err(&pdev->dev, "removing device with active domains!\n");
2153
2154         /* Turn the thing off */
2155         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2156         return 0;
2157 }
2158
2159 static struct platform_driver arm_smmu_driver = {
2160         .driver = {
2161                 .name           = "arm-smmu",
2162                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2163         },
2164         .probe  = arm_smmu_device_probe,
2165         .remove = arm_smmu_device_remove,
2166 };
2167
2168 static int __init arm_smmu_init(void)
2169 {
2170         static bool registered;
2171         int ret = 0;
2172
2173         if (!registered) {
2174                 ret = platform_driver_register(&arm_smmu_driver);
2175                 registered = !ret;
2176         }
2177         return ret;
2178 }
2179
2180 static void __exit arm_smmu_exit(void)
2181 {
2182         return platform_driver_unregister(&arm_smmu_driver);
2183 }
2184
2185 subsys_initcall(arm_smmu_init);
2186 module_exit(arm_smmu_exit);
2187
2188 static int __init arm_smmu_of_init(struct device_node *np)
2189 {
2190         int ret = arm_smmu_init();
2191
2192         if (ret)
2193                 return ret;
2194
2195         if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2196                 return -ENODEV;
2197
2198         return 0;
2199 }
2200 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
2201 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
2202 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
2203 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
2204 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
2205 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
2206
2207 #ifdef CONFIG_ACPI
2208 static int __init arm_smmu_acpi_init(struct acpi_table_header *table)
2209 {
2210         if (iort_node_match(ACPI_IORT_NODE_SMMU))
2211                 return arm_smmu_init();
2212
2213         return 0;
2214 }
2215 IORT_ACPI_DECLARE(arm_smmu, ACPI_SIG_IORT, arm_smmu_acpi_init);
2216 #endif
2217
2218 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2219 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2220 MODULE_LICENSE("GPL v2");