drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  *      - Extended Stream ID (16 bit)
  28  */
  29
  30 #define pr_fmt(fmt) "arm-smmu: " fmt
  31
  32 #include <linux/acpi.h>
  33 #include <linux/acpi_iort.h>
  34 #include <linux/atomic.h>
  35 #include <linux/delay.h>
  36 #include <linux/dma-iommu.h>
  37 #include <linux/dma-mapping.h>
  38 #include <linux/err.h>
  39 #include <linux/interrupt.h>
  40 #include <linux/io.h>
  41 #include <linux/io-64-nonatomic-hi-lo.h>
  42 #include <linux/iommu.h>
  43 #include <linux/iopoll.h>
  44 #include <linux/module.h>
  45 #include <linux/of.h>
  46 #include <linux/of_address.h>
  47 #include <linux/of_device.h>
  48 #include <linux/of_iommu.h>
  49 #include <linux/pci.h>
  50 #include <linux/platform_device.h>
  51 #include <linux/slab.h>
  52 #include <linux/spinlock.h>
  53
  54 #include <linux/amba/bus.h>
  55
  56 #include "io-pgtable.h"
  57
  58 /* Maximum number of context banks per SMMU */
  59 #define ARM_SMMU_MAX_CBS                128
  60
  61 /* SMMU global address space */
  62 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  63 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  64
  65 /*
  66  * SMMU global address space with conditional offset to access secure
  67  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  68  * nsGFSYNR0: 0x450)
  69  */
  70 #define ARM_SMMU_GR0_NS(smmu)                                           \
  71         ((smmu)->base +                                                 \
  72                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  73                         ? 0x400 : 0))
  74
  75 /*
  76  * Some 64-bit registers only make sense to write atomically, but in such
  77  * cases all the data relevant to AArch32 formats lies within the lower word,
  78  * therefore this actually makes more sense than it might first appear.
  79  */
  80 #ifdef CONFIG_64BIT
  81 #define smmu_write_atomic_lq            writeq_relaxed
  82 #else
  83 #define smmu_write_atomic_lq            writel_relaxed
  84 #endif
  85
  86 /* Configuration registers */
  87 #define ARM_SMMU_GR0_sCR0               0x0
  88 #define sCR0_CLIENTPD                   (1 << 0)
  89 #define sCR0_GFRE                       (1 << 1)
  90 #define sCR0_GFIE                       (1 << 2)
  91 #define sCR0_EXIDENABLE                 (1 << 3)
  92 #define sCR0_GCFGFRE                    (1 << 4)
  93 #define sCR0_GCFGFIE                    (1 << 5)
  94 #define sCR0_USFCFG                     (1 << 10)
  95 #define sCR0_VMIDPNE                    (1 << 11)
  96 #define sCR0_PTM                        (1 << 12)
  97 #define sCR0_FB                         (1 << 13)
  98 #define sCR0_VMID16EN                   (1 << 31)
  99 #define sCR0_BSU_SHIFT                  14
 100 #define sCR0_BSU_MASK                   0x3
 101
 102 /* Auxiliary Configuration register */
 103 #define ARM_SMMU_GR0_sACR               0x10
 104
 105 /* Identification registers */
 106 #define ARM_SMMU_GR0_ID0                0x20
 107 #define ARM_SMMU_GR0_ID1                0x24
 108 #define ARM_SMMU_GR0_ID2                0x28
 109 #define ARM_SMMU_GR0_ID3                0x2c
 110 #define ARM_SMMU_GR0_ID4                0x30
 111 #define ARM_SMMU_GR0_ID5                0x34
 112 #define ARM_SMMU_GR0_ID6                0x38
 113 #define ARM_SMMU_GR0_ID7                0x3c
 114 #define ARM_SMMU_GR0_sGFSR              0x48
 115 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 116 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 117 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 118
 119 #define ID0_S1TS                        (1 << 30)
 120 #define ID0_S2TS                        (1 << 29)
 121 #define ID0_NTS                         (1 << 28)
 122 #define ID0_SMS                         (1 << 27)
 123 #define ID0_ATOSNS                      (1 << 26)
 124 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 125 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 126 #define ID0_CTTW                        (1 << 14)
 127 #define ID0_NUMIRPT_SHIFT               16
 128 #define ID0_NUMIRPT_MASK                0xff
 129 #define ID0_NUMSIDB_SHIFT               9
 130 #define ID0_NUMSIDB_MASK                0xf
 131 #define ID0_EXIDS                       (1 << 8)
 132 #define ID0_NUMSMRG_SHIFT               0
 133 #define ID0_NUMSMRG_MASK                0xff
 134
 135 #define ID1_PAGESIZE                    (1 << 31)
 136 #define ID1_NUMPAGENDXB_SHIFT           28
 137 #define ID1_NUMPAGENDXB_MASK            7
 138 #define ID1_NUMS2CB_SHIFT               16
 139 #define ID1_NUMS2CB_MASK                0xff
 140 #define ID1_NUMCB_SHIFT                 0
 141 #define ID1_NUMCB_MASK                  0xff
 142
 143 #define ID2_OAS_SHIFT                   4
 144 #define ID2_OAS_MASK                    0xf
 145 #define ID2_IAS_SHIFT                   0
 146 #define ID2_IAS_MASK                    0xf
 147 #define ID2_UBS_SHIFT                   8
 148 #define ID2_UBS_MASK                    0xf
 149 #define ID2_PTFS_4K                     (1 << 12)
 150 #define ID2_PTFS_16K                    (1 << 13)
 151 #define ID2_PTFS_64K                    (1 << 14)
 152 #define ID2_VMID16                      (1 << 15)
 153
 154 #define ID7_MAJOR_SHIFT                 4
 155 #define ID7_MAJOR_MASK                  0xf
 156
 157 /* Global TLB invalidation */
 158 #define ARM_SMMU_GR0_TLBIVMID           0x64
 159 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 160 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 161 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 162 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 163 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 164 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 165
 166 /* Stream mapping registers */
 167 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 168 #define SMR_VALID                       (1 << 31)
 169 #define SMR_MASK_SHIFT                  16
 170 #define SMR_ID_SHIFT                    0
 171
 172 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 173 #define S2CR_CBNDX_SHIFT                0
 174 #define S2CR_CBNDX_MASK                 0xff
 175 #define S2CR_EXIDVALID                  (1 << 10)
 176 #define S2CR_TYPE_SHIFT                 16
 177 #define S2CR_TYPE_MASK                  0x3
 178 enum arm_smmu_s2cr_type {
 179         S2CR_TYPE_TRANS,
 180         S2CR_TYPE_BYPASS,
 181         S2CR_TYPE_FAULT,
 182 };
 183
 184 #define S2CR_PRIVCFG_SHIFT              24
 185 #define S2CR_PRIVCFG_MASK               0x3
 186 enum arm_smmu_s2cr_privcfg {
 187         S2CR_PRIVCFG_DEFAULT,
 188         S2CR_PRIVCFG_DIPAN,
 189         S2CR_PRIVCFG_UNPRIV,
 190         S2CR_PRIVCFG_PRIV,
 191 };
 192
 193 /* Context bank attribute registers */
 194 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 195 #define CBAR_VMID_SHIFT                 0
 196 #define CBAR_VMID_MASK                  0xff
 197 #define CBAR_S1_BPSHCFG_SHIFT           8
 198 #define CBAR_S1_BPSHCFG_MASK            3
 199 #define CBAR_S1_BPSHCFG_NSH             3
 200 #define CBAR_S1_MEMATTR_SHIFT           12
 201 #define CBAR_S1_MEMATTR_MASK            0xf
 202 #define CBAR_S1_MEMATTR_WB              0xf
 203 #define CBAR_TYPE_SHIFT                 16
 204 #define CBAR_TYPE_MASK                  0x3
 205 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 206 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 207 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 208 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 209 #define CBAR_IRPTNDX_SHIFT              24
 210 #define CBAR_IRPTNDX_MASK               0xff
 211
 212 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 213 #define CBA2R_RW64_32BIT                (0 << 0)
 214 #define CBA2R_RW64_64BIT                (1 << 0)
 215 #define CBA2R_VMID_SHIFT                16
 216 #define CBA2R_VMID_MASK                 0xffff
 217
 218 /* Translation context bank */
 219 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
 220 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
 221
 222 #define ARM_SMMU_CB_SCTLR               0x0
 223 #define ARM_SMMU_CB_ACTLR               0x4
 224 #define ARM_SMMU_CB_RESUME              0x8
 225 #define ARM_SMMU_CB_TTBCR2              0x10
 226 #define ARM_SMMU_CB_TTBR0               0x20
 227 #define ARM_SMMU_CB_TTBR1               0x28
 228 #define ARM_SMMU_CB_TTBCR               0x30
 229 #define ARM_SMMU_CB_CONTEXTIDR          0x34
 230 #define ARM_SMMU_CB_S1_MAIR0            0x38
 231 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 232 #define ARM_SMMU_CB_PAR                 0x50
 233 #define ARM_SMMU_CB_FSR                 0x58
 234 #define ARM_SMMU_CB_FAR                 0x60
 235 #define ARM_SMMU_CB_FSYNR0              0x68
 236 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 237 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 238 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 239 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 240 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 241 #define ARM_SMMU_CB_ATS1PR              0x800
 242 #define ARM_SMMU_CB_ATSR                0x8f0
 243
 244 #define SCTLR_S1_ASIDPNE                (1 << 12)
 245 #define SCTLR_CFCFG                     (1 << 7)
 246 #define SCTLR_CFIE                      (1 << 6)
 247 #define SCTLR_CFRE                      (1 << 5)
 248 #define SCTLR_E                         (1 << 4)
 249 #define SCTLR_AFE                       (1 << 2)
 250 #define SCTLR_TRE                       (1 << 1)
 251 #define SCTLR_M                         (1 << 0)
 252
 253 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 254
 255 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 256 #define ARM_MMU500_ACR_SMTNMB_TLBEN     (1 << 8)
 257
 258 #define CB_PAR_F                        (1 << 0)
 259
 260 #define ATSR_ACTIVE                     (1 << 0)
 261
 262 #define RESUME_RETRY                    (0 << 0)
 263 #define RESUME_TERMINATE                (1 << 0)
 264
 265 #define TTBCR2_SEP_SHIFT                15
 266 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 267
 268 #define TTBRn_ASID_SHIFT                48
 269
 270 #define FSR_MULTI                       (1 << 31)
 271 #define FSR_SS                          (1 << 30)
 272 #define FSR_UUT                         (1 << 8)
 273 #define FSR_ASF                         (1 << 7)
 274 #define FSR_TLBLKF                      (1 << 6)
 275 #define FSR_TLBMCF                      (1 << 5)
 276 #define FSR_EF                          (1 << 4)
 277 #define FSR_PF                          (1 << 3)
 278 #define FSR_AFF                         (1 << 2)
 279 #define FSR_TF                          (1 << 1)
 280
 281 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 282                                          FSR_TLBMCF | FSR_TLBLKF)
 283 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 284                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 285
 286 #define FSYNR0_WNR                      (1 << 4)
 287
 288 static int force_stage;
 289 module_param(force_stage, int, S_IRUGO);
 290 MODULE_PARM_DESC(force_stage,
 291         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 292 static bool disable_bypass;
 293 module_param(disable_bypass, bool, S_IRUGO);
 294 MODULE_PARM_DESC(disable_bypass,
 295         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 296
 297 enum arm_smmu_arch_version {
 298         ARM_SMMU_V1,
 299         ARM_SMMU_V1_64K,
 300         ARM_SMMU_V2,
 301 };
 302
 303 enum arm_smmu_implementation {
 304         GENERIC_SMMU,
 305         ARM_MMU500,
 306         CAVIUM_SMMUV2,
 307 };
 308
 309 struct arm_smmu_s2cr {
 310         struct iommu_group              *group;
 311         int                             count;
 312         enum arm_smmu_s2cr_type         type;
 313         enum arm_smmu_s2cr_privcfg      privcfg;
 314         u8                              cbndx;
 315 };
 316
 317 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 318         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 319 }
 320
 321 struct arm_smmu_smr {
 322         u16                             mask;
 323         u16                             id;
 324         bool                            valid;
 325 };
 326
 327 struct arm_smmu_master_cfg {
 328         struct arm_smmu_device          *smmu;
 329         s16                             smendx[];
 330 };
 331 #define INVALID_SMENDX                  -1
 332 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 333 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 334 #define fwspec_smendx(fw, i) \
 335         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 336 #define for_each_cfg_sme(fw, i, idx) \
 337         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 338
 339 struct arm_smmu_device {
 340         struct device                   *dev;
 341
 342         void __iomem                    *base;
 343         unsigned long                   size;
 344         unsigned long                   pgshift;
 345
 346 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 347 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 348 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 349 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 350 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 351 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 352 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 353 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 354 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 355 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 356 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 357 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 358 #define ARM_SMMU_FEAT_EXIDS             (1 << 12)
 359         u32                             features;
 360
 361 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 362         u32                             options;
 363         enum arm_smmu_arch_version      version;
 364         enum arm_smmu_implementation    model;
 365
 366         u32                             num_context_banks;
 367         u32                             num_s2_context_banks;
 368         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 369         atomic_t                        irptndx;
 370
 371         u32                             num_mapping_groups;
 372         u16                             streamid_mask;
 373         u16                             smr_mask_mask;
 374         struct arm_smmu_smr             *smrs;
 375         struct arm_smmu_s2cr            *s2crs;
 376         struct mutex                    stream_map_mutex;
 377
 378         unsigned long                   va_size;
 379         unsigned long                   ipa_size;
 380         unsigned long                   pa_size;
 381         unsigned long                   pgsize_bitmap;
 382
 383         u32                             num_global_irqs;
 384         u32                             num_context_irqs;
 385         unsigned int                    *irqs;
 386
 387         u32                             cavium_id_base; /* Specific to Cavium */
 388 };
 389
 390 enum arm_smmu_context_fmt {
 391         ARM_SMMU_CTX_FMT_NONE,
 392         ARM_SMMU_CTX_FMT_AARCH64,
 393         ARM_SMMU_CTX_FMT_AARCH32_L,
 394         ARM_SMMU_CTX_FMT_AARCH32_S,
 395 };
 396
 397 struct arm_smmu_cfg {
 398         u8                              cbndx;
 399         u8                              irptndx;
 400         u32                             cbar;
 401         enum arm_smmu_context_fmt       fmt;
 402 };
 403 #define INVALID_IRPTNDX                 0xff
 404
 405 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
 406 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
 407
 408 enum arm_smmu_domain_stage {
 409         ARM_SMMU_DOMAIN_S1 = 0,
 410         ARM_SMMU_DOMAIN_S2,
 411         ARM_SMMU_DOMAIN_NESTED,
 412 };
 413
 414 struct arm_smmu_domain {
 415         struct arm_smmu_device          *smmu;
 416         struct io_pgtable_ops           *pgtbl_ops;
 417         spinlock_t                      pgtbl_lock;
 418         struct arm_smmu_cfg             cfg;
 419         enum arm_smmu_domain_stage      stage;
 420         struct mutex                    init_mutex; /* Protects smmu pointer */
 421         struct iommu_domain             domain;
 422 };
 423
 424 struct arm_smmu_option_prop {
 425         u32 opt;
 426         const char *prop;
 427 };
 428
 429 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 430
 431 static bool using_legacy_binding, using_generic_binding;
 432
 433 static struct arm_smmu_option_prop arm_smmu_options[] = {
 434         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 435         { 0, NULL},
 436 };
 437
 438 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 439 {
 440         return container_of(dom, struct arm_smmu_domain, domain);
 441 }
 442
 443 static void parse_driver_options(struct arm_smmu_device *smmu)
 444 {
 445         int i = 0;
 446
 447         do {
 448                 if (of_property_read_bool(smmu->dev->of_node,
 449                                                 arm_smmu_options[i].prop)) {
 450                         smmu->options |= arm_smmu_options[i].opt;
 451                         dev_notice(smmu->dev, "option %s\n",
 452                                 arm_smmu_options[i].prop);
 453                 }
 454         } while (arm_smmu_options[++i].opt);
 455 }
 456
 457 static struct device_node *dev_get_dev_node(struct device *dev)
 458 {
 459         if (dev_is_pci(dev)) {
 460                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 461
 462                 while (!pci_is_root_bus(bus))
 463                         bus = bus->parent;
 464                 return of_node_get(bus->bridge->parent->of_node);
 465         }
 466
 467         return of_node_get(dev->of_node);
 468 }
 469
 470 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 471 {
 472         *((__be32 *)data) = cpu_to_be32(alias);
 473         return 0; /* Continue walking */
 474 }
 475
 476 static int __find_legacy_master_phandle(struct device *dev, void *data)
 477 {
 478         struct of_phandle_iterator *it = *(void **)data;
 479         struct device_node *np = it->node;
 480         int err;
 481
 482         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 483                             "#stream-id-cells", 0)
 484                 if (it->node == np) {
 485                         *(void **)data = dev;
 486                         return 1;
 487                 }
 488         it->node = np;
 489         return err == -ENOENT ? 0 : err;
 490 }
 491
 492 static struct platform_driver arm_smmu_driver;
 493 static struct iommu_ops arm_smmu_ops;
 494
 495 static int arm_smmu_register_legacy_master(struct device *dev,
 496                                            struct arm_smmu_device **smmu)
 497 {
 498         struct device *smmu_dev;
 499         struct device_node *np;
 500         struct of_phandle_iterator it;
 501         void *data = &it;
 502         u32 *sids;
 503         __be32 pci_sid;
 504         int err;
 505
 506         np = dev_get_dev_node(dev);
 507         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 508                 of_node_put(np);
 509                 return -ENODEV;
 510         }
 511
 512         it.node = np;
 513         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 514                                      __find_legacy_master_phandle);
 515         smmu_dev = data;
 516         of_node_put(np);
 517         if (err == 0)
 518                 return -ENODEV;
 519         if (err < 0)
 520                 return err;
 521
 522         if (dev_is_pci(dev)) {
 523                 /* "mmu-masters" assumes Stream ID == Requester ID */
 524                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 525                                        &pci_sid);
 526                 it.cur = &pci_sid;
 527                 it.cur_count = 1;
 528         }
 529
 530         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 531                                 &arm_smmu_ops);
 532         if (err)
 533                 return err;
 534
 535         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 536         if (!sids)
 537                 return -ENOMEM;
 538
 539         *smmu = dev_get_drvdata(smmu_dev);
 540         of_phandle_iterator_args(&it, sids, it.cur_count);
 541         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 542         kfree(sids);
 543         return err;
 544 }
 545
 546 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 547 {
 548         int idx;
 549
 550         do {
 551                 idx = find_next_zero_bit(map, end, start);
 552                 if (idx == end)
 553                         return -ENOSPC;
 554         } while (test_and_set_bit(idx, map));
 555
 556         return idx;
 557 }
 558
 559 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 560 {
 561         clear_bit(idx, map);
 562 }
 563
 564 /* Wait for any pending TLB invalidations to complete */
 565 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
 566 {
 567         int count = 0;
 568         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 569
 570         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
 571         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
 572                & sTLBGSTATUS_GSACTIVE) {
 573                 cpu_relax();
 574                 if (++count == TLB_LOOP_TIMEOUT) {
 575                         dev_err_ratelimited(smmu->dev,
 576                         "TLB sync timed out -- SMMU may be deadlocked\n");
 577                         return;
 578                 }
 579                 udelay(1);
 580         }
 581 }
 582
 583 static void arm_smmu_tlb_sync(void *cookie)
 584 {
 585         struct arm_smmu_domain *smmu_domain = cookie;
 586         __arm_smmu_tlb_sync(smmu_domain->smmu);
 587 }
 588
 589 static void arm_smmu_tlb_inv_context(void *cookie)
 590 {
 591         struct arm_smmu_domain *smmu_domain = cookie;
 592         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 593         struct arm_smmu_device *smmu = smmu_domain->smmu;
 594         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 595         void __iomem *base;
 596
 597         if (stage1) {
 598                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 599                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
 600                                base + ARM_SMMU_CB_S1_TLBIASID);
 601         } else {
 602                 base = ARM_SMMU_GR0(smmu);
 603                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
 604                                base + ARM_SMMU_GR0_TLBIVMID);
 605         }
 606
 607         __arm_smmu_tlb_sync(smmu);
 608 }
 609
 610 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 611                                           size_t granule, bool leaf, void *cookie)
 612 {
 613         struct arm_smmu_domain *smmu_domain = cookie;
 614         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 615         struct arm_smmu_device *smmu = smmu_domain->smmu;
 616         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 617         void __iomem *reg;
 618
 619         if (stage1) {
 620                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 621                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 622
 623                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 624                         iova &= ~12UL;
 625                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
 626                         do {
 627                                 writel_relaxed(iova, reg);
 628                                 iova += granule;
 629                         } while (size -= granule);
 630                 } else {
 631                         iova >>= 12;
 632                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
 633                         do {
 634                                 writeq_relaxed(iova, reg);
 635                                 iova += granule >> 12;
 636                         } while (size -= granule);
 637                 }
 638         } else if (smmu->version == ARM_SMMU_V2) {
 639                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 640                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 641                               ARM_SMMU_CB_S2_TLBIIPAS2;
 642                 iova >>= 12;
 643                 do {
 644                         smmu_write_atomic_lq(iova, reg);
 645                         iova += granule >> 12;
 646                 } while (size -= granule);
 647         } else {
 648                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
 649                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
 650         }
 651 }
 652
 653 static const struct iommu_gather_ops arm_smmu_gather_ops = {
 654         .tlb_flush_all  = arm_smmu_tlb_inv_context,
 655         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 656         .tlb_sync       = arm_smmu_tlb_sync,
 657 };
 658
 659 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 660 {
 661         u32 fsr, fsynr;
 662         unsigned long iova;
 663         struct iommu_domain *domain = dev;
 664         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 665         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 666         struct arm_smmu_device *smmu = smmu_domain->smmu;
 667         void __iomem *cb_base;
 668
 669         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 670         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 671
 672         if (!(fsr & FSR_FAULT))
 673                 return IRQ_NONE;
 674
 675         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 676         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 677
 678         dev_err_ratelimited(smmu->dev,
 679         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 680                             fsr, iova, fsynr, cfg->cbndx);
 681
 682         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 683         return IRQ_HANDLED;
 684 }
 685
 686 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 687 {
 688         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 689         struct arm_smmu_device *smmu = dev;
 690         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 691
 692         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 693         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 694         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 695         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 696
 697         if (!gfsr)
 698                 return IRQ_NONE;
 699
 700         dev_err_ratelimited(smmu->dev,
 701                 "Unexpected global fault, this could be serious\n");
 702         dev_err_ratelimited(smmu->dev,
 703                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 704                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 705
 706         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 707         return IRQ_HANDLED;
 708 }
 709
 710 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 711                                        struct io_pgtable_cfg *pgtbl_cfg)
 712 {
 713         u32 reg, reg2;
 714         u64 reg64;
 715         bool stage1;
 716         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 717         struct arm_smmu_device *smmu = smmu_domain->smmu;
 718         void __iomem *cb_base, *gr1_base;
 719
 720         gr1_base = ARM_SMMU_GR1(smmu);
 721         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 722         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 723
 724         if (smmu->version > ARM_SMMU_V1) {
 725                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 726                         reg = CBA2R_RW64_64BIT;
 727                 else
 728                         reg = CBA2R_RW64_32BIT;
 729                 /* 16-bit VMIDs live in CBA2R */
 730                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 731                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
 732
 733                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 734         }
 735
 736         /* CBAR */
 737         reg = cfg->cbar;
 738         if (smmu->version < ARM_SMMU_V2)
 739                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 740
 741         /*
 742          * Use the weakest shareability/memory types, so they are
 743          * overridden by the ttbcr/pte.
 744          */
 745         if (stage1) {
 746                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 747                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 748         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 749                 /* 8-bit VMIDs live in CBAR */
 750                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
 751         }
 752         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 753
 754         /* TTBRs */
 755         if (stage1) {
 756                 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
 757
 758                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 759                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 760                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
 761                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 762                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
 763                         writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 764                 } else {
 765                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 766                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 767                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 768                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 769                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 770                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 771                 }
 772         } else {
 773                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 774                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 775         }
 776
 777         /* TTBCR */
 778         if (stage1) {
 779                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 780                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
 781                         reg2 = 0;
 782                 } else {
 783                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 784                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 785                         reg2 |= TTBCR2_SEP_UPSTREAM;
 786                 }
 787                 if (smmu->version > ARM_SMMU_V1)
 788                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
 789         } else {
 790                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 791         }
 792         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 793
 794         /* MAIRs (stage-1 only) */
 795         if (stage1) {
 796                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 797                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
 798                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
 799                 } else {
 800                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 801                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 802                 }
 803                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 804                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
 805         }
 806
 807         /* SCTLR */
 808         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 809         if (stage1)
 810                 reg |= SCTLR_S1_ASIDPNE;
 811 #ifdef __BIG_ENDIAN
 812         reg |= SCTLR_E;
 813 #endif
 814         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 815 }
 816
 817 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 818                                         struct arm_smmu_device *smmu)
 819 {
 820         int irq, start, ret = 0;
 821         unsigned long ias, oas;
 822         struct io_pgtable_ops *pgtbl_ops;
 823         struct io_pgtable_cfg pgtbl_cfg;
 824         enum io_pgtable_fmt fmt;
 825         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 826         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 827
 828         mutex_lock(&smmu_domain->init_mutex);
 829         if (smmu_domain->smmu)
 830                 goto out_unlock;
 831
 832         /*
 833          * Mapping the requested stage onto what we support is surprisingly
 834          * complicated, mainly because the spec allows S1+S2 SMMUs without
 835          * support for nested translation. That means we end up with the
 836          * following table:
 837          *
 838          * Requested        Supported        Actual
 839          *     S1               N              S1
 840          *     S1             S1+S2            S1
 841          *     S1               S2             S2
 842          *     S1               S1             S1
 843          *     N                N              N
 844          *     N              S1+S2            S2
 845          *     N                S2             S2
 846          *     N                S1             S1
 847          *
 848          * Note that you can't actually request stage-2 mappings.
 849          */
 850         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 851                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 852         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 853                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 854
 855         /*
 856          * Choosing a suitable context format is even more fiddly. Until we
 857          * grow some way for the caller to express a preference, and/or move
 858          * the decision into the io-pgtable code where it arguably belongs,
 859          * just aim for the closest thing to the rest of the system, and hope
 860          * that the hardware isn't esoteric enough that we can't assume AArch64
 861          * support to be a superset of AArch32 support...
 862          */
 863         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 864                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 865         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 866             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 867             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 868             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 869                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 870         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 871             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 872                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 873                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 874                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 875
 876         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 877                 ret = -EINVAL;
 878                 goto out_unlock;
 879         }
 880
 881         switch (smmu_domain->stage) {
 882         case ARM_SMMU_DOMAIN_S1:
 883                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 884                 start = smmu->num_s2_context_banks;
 885                 ias = smmu->va_size;
 886                 oas = smmu->ipa_size;
 887                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 888                         fmt = ARM_64_LPAE_S1;
 889                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 890                         fmt = ARM_32_LPAE_S1;
 891                         ias = min(ias, 32UL);
 892                         oas = min(oas, 40UL);
 893                 } else {
 894                         fmt = ARM_V7S;
 895                         ias = min(ias, 32UL);
 896                         oas = min(oas, 32UL);
 897                 }
 898                 break;
 899         case ARM_SMMU_DOMAIN_NESTED:
 900                 /*
 901                  * We will likely want to change this if/when KVM gets
 902                  * involved.
 903                  */
 904         case ARM_SMMU_DOMAIN_S2:
 905                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 906                 start = 0;
 907                 ias = smmu->ipa_size;
 908                 oas = smmu->pa_size;
 909                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 910                         fmt = ARM_64_LPAE_S2;
 911                 } else {
 912                         fmt = ARM_32_LPAE_S2;
 913                         ias = min(ias, 40UL);
 914                         oas = min(oas, 40UL);
 915                 }
 916                 break;
 917         default:
 918                 ret = -EINVAL;
 919                 goto out_unlock;
 920         }
 921
 922         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 923                                       smmu->num_context_banks);
 924         if (ret < 0)
 925                 goto out_unlock;
 926
 927         cfg->cbndx = ret;
 928         if (smmu->version < ARM_SMMU_V2) {
 929                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 930                 cfg->irptndx %= smmu->num_context_irqs;
 931         } else {
 932                 cfg->irptndx = cfg->cbndx;
 933         }
 934
 935         pgtbl_cfg = (struct io_pgtable_cfg) {
 936                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 937                 .ias            = ias,
 938                 .oas            = oas,
 939                 .tlb            = &arm_smmu_gather_ops,
 940                 .iommu_dev      = smmu->dev,
 941         };
 942
 943         smmu_domain->smmu = smmu;
 944         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 945         if (!pgtbl_ops) {
 946                 ret = -ENOMEM;
 947                 goto out_clear_smmu;
 948         }
 949
 950         /* Update the domain's page sizes to reflect the page table format */
 951         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 952         domain->geometry.aperture_end = (1UL << ias) - 1;
 953         domain->geometry.force_aperture = true;
 954
 955         /* Initialise the context bank with our page table cfg */
 956         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 957
 958         /*
 959          * Request context fault interrupt. Do this last to avoid the
 960          * handler seeing a half-initialised domain state.
 961          */
 962         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 963         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 964                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 965         if (ret < 0) {
 966                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 967                         cfg->irptndx, irq);
 968                 cfg->irptndx = INVALID_IRPTNDX;
 969         }
 970
 971         mutex_unlock(&smmu_domain->init_mutex);
 972
 973         /* Publish page table ops for map/unmap */
 974         smmu_domain->pgtbl_ops = pgtbl_ops;
 975         return 0;
 976
 977 out_clear_smmu:
 978         smmu_domain->smmu = NULL;
 979 out_unlock:
 980         mutex_unlock(&smmu_domain->init_mutex);
 981         return ret;
 982 }
 983
 984 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 985 {
 986         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 987         struct arm_smmu_device *smmu = smmu_domain->smmu;
 988         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 989         void __iomem *cb_base;
 990         int irq;
 991
 992         if (!smmu)
 993                 return;
 994
 995         /*
 996          * Disable the context bank and free the page tables before freeing
 997          * it.
 998          */
 999         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1000         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1001
1002         if (cfg->irptndx != INVALID_IRPTNDX) {
1003                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
1004                 devm_free_irq(smmu->dev, irq, domain);
1005         }
1006
1007         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1008         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1009 }
1010
1011 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1012 {
1013         struct arm_smmu_domain *smmu_domain;
1014
1015         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1016                 return NULL;
1017         /*
1018          * Allocate the domain and initialise some of its data structures.
1019          * We can't really do anything meaningful until we've added a
1020          * master.
1021          */
1022         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1023         if (!smmu_domain)
1024                 return NULL;
1025
1026         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1027             iommu_get_dma_cookie(&smmu_domain->domain))) {
1028                 kfree(smmu_domain);
1029                 return NULL;
1030         }
1031
1032         mutex_init(&smmu_domain->init_mutex);
1033         spin_lock_init(&smmu_domain->pgtbl_lock);
1034
1035         return &smmu_domain->domain;
1036 }
1037
1038 static void arm_smmu_domain_free(struct iommu_domain *domain)
1039 {
1040         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1041
1042         /*
1043          * Free the domain resources. We assume that all devices have
1044          * already been detached.
1045          */
1046         iommu_put_dma_cookie(domain);
1047         arm_smmu_destroy_domain_context(domain);
1048         kfree(smmu_domain);
1049 }
1050
1051 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1052 {
1053         struct arm_smmu_smr *smr = smmu->smrs + idx;
1054         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1055
1056         if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
1057                 reg |= SMR_VALID;
1058         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1059 }
1060
1061 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1062 {
1063         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1064         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1065                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1066                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1067
1068         if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
1069             smmu->smrs[idx].valid)
1070                 reg |= S2CR_EXIDVALID;
1071         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1072 }
1073
1074 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1075 {
1076         arm_smmu_write_s2cr(smmu, idx);
1077         if (smmu->smrs)
1078                 arm_smmu_write_smr(smmu, idx);
1079 }
1080
1081 /*
1082  * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
1083  * should be called after sCR0 is written.
1084  */
1085 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
1086 {
1087         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1088         u32 smr;
1089
1090         if (!smmu->smrs)
1091                 return;
1092
1093         /*
1094          * SMR.ID bits may not be preserved if the corresponding MASK
1095          * bits are set, so check each one separately. We can reject
1096          * masters later if they try to claim IDs outside these masks.
1097          */
1098         smr = smmu->streamid_mask << SMR_ID_SHIFT;
1099         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1100         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1101         smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1102
1103         smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1104         writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1105         smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1106         smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1107 }
1108
1109 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1110 {
1111         struct arm_smmu_smr *smrs = smmu->smrs;
1112         int i, free_idx = -ENOSPC;
1113
1114         /* Stream indexing is blissfully easy */
1115         if (!smrs)
1116                 return id;
1117
1118         /* Validating SMRs is... less so */
1119         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1120                 if (!smrs[i].valid) {
1121                         /*
1122                          * Note the first free entry we come across, which
1123                          * we'll claim in the end if nothing else matches.
1124                          */
1125                         if (free_idx < 0)
1126                                 free_idx = i;
1127                         continue;
1128                 }
1129                 /*
1130                  * If the new entry is _entirely_ matched by an existing entry,
1131                  * then reuse that, with the guarantee that there also cannot
1132                  * be any subsequent conflicting entries. In normal use we'd
1133                  * expect simply identical entries for this case, but there's
1134                  * no harm in accommodating the generalisation.
1135                  */
1136                 if ((mask & smrs[i].mask) == mask &&
1137                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1138                         return i;
1139                 /*
1140                  * If the new entry has any other overlap with an existing one,
1141                  * though, then there always exists at least one stream ID
1142                  * which would cause a conflict, and we can't allow that risk.
1143                  */
1144                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1145                         return -EINVAL;
1146         }
1147
1148         return free_idx;
1149 }
1150
1151 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1152 {
1153         if (--smmu->s2crs[idx].count)
1154                 return false;
1155
1156         smmu->s2crs[idx] = s2cr_init_val;
1157         if (smmu->smrs)
1158                 smmu->smrs[idx].valid = false;
1159
1160         return true;
1161 }
1162
1163 static int arm_smmu_master_alloc_smes(struct device *dev)
1164 {
1165         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1166         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1167         struct arm_smmu_device *smmu = cfg->smmu;
1168         struct arm_smmu_smr *smrs = smmu->smrs;
1169         struct iommu_group *group;
1170         int i, idx, ret;
1171
1172         mutex_lock(&smmu->stream_map_mutex);
1173         /* Figure out a viable stream map entry allocation */
1174         for_each_cfg_sme(fwspec, i, idx) {
1175                 u16 sid = fwspec->ids[i];
1176                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1177
1178                 if (idx != INVALID_SMENDX) {
1179                         ret = -EEXIST;
1180                         goto out_err;
1181                 }
1182
1183                 ret = arm_smmu_find_sme(smmu, sid, mask);
1184                 if (ret < 0)
1185                         goto out_err;
1186
1187                 idx = ret;
1188                 if (smrs && smmu->s2crs[idx].count == 0) {
1189                         smrs[idx].id = sid;
1190                         smrs[idx].mask = mask;
1191                         smrs[idx].valid = true;
1192                 }
1193                 smmu->s2crs[idx].count++;
1194                 cfg->smendx[i] = (s16)idx;
1195         }
1196
1197         group = iommu_group_get_for_dev(dev);
1198         if (!group)
1199                 group = ERR_PTR(-ENOMEM);
1200         if (IS_ERR(group)) {
1201                 ret = PTR_ERR(group);
1202                 goto out_err;
1203         }
1204         iommu_group_put(group);
1205
1206         /* It worked! Now, poke the actual hardware */
1207         for_each_cfg_sme(fwspec, i, idx) {
1208                 arm_smmu_write_sme(smmu, idx);
1209                 smmu->s2crs[idx].group = group;
1210         }
1211
1212         mutex_unlock(&smmu->stream_map_mutex);
1213         return 0;
1214
1215 out_err:
1216         while (i--) {
1217                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1218                 cfg->smendx[i] = INVALID_SMENDX;
1219         }
1220         mutex_unlock(&smmu->stream_map_mutex);
1221         return ret;
1222 }
1223
1224 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1225 {
1226         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1227         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1228         int i, idx;
1229
1230         mutex_lock(&smmu->stream_map_mutex);
1231         for_each_cfg_sme(fwspec, i, idx) {
1232                 if (arm_smmu_free_sme(smmu, idx))
1233                         arm_smmu_write_sme(smmu, idx);
1234                 cfg->smendx[i] = INVALID_SMENDX;
1235         }
1236         mutex_unlock(&smmu->stream_map_mutex);
1237 }
1238
1239 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1240                                       struct iommu_fwspec *fwspec)
1241 {
1242         struct arm_smmu_device *smmu = smmu_domain->smmu;
1243         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1244         enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1245         u8 cbndx = smmu_domain->cfg.cbndx;
1246         int i, idx;
1247
1248         for_each_cfg_sme(fwspec, i, idx) {
1249                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1250                         continue;
1251
1252                 s2cr[idx].type = type;
1253                 s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV;
1254                 s2cr[idx].cbndx = cbndx;
1255                 arm_smmu_write_s2cr(smmu, idx);
1256         }
1257         return 0;
1258 }
1259
1260 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1261 {
1262         int ret;
1263         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1264         struct arm_smmu_device *smmu;
1265         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1266
1267         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1268                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1269                 return -ENXIO;
1270         }
1271
1272         /*
1273          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1274          * domains between of_xlate() and add_device() - we have no way to cope
1275          * with that, so until ARM gets converted to rely on groups and default
1276          * domains, just say no (but more politely than by dereferencing NULL).
1277          * This should be at least a WARN_ON once that's sorted.
1278          */
1279         if (!fwspec->iommu_priv)
1280                 return -ENODEV;
1281
1282         smmu = fwspec_smmu(fwspec);
1283         /* Ensure that the domain is finalised */
1284         ret = arm_smmu_init_domain_context(domain, smmu);
1285         if (ret < 0)
1286                 return ret;
1287
1288         /*
1289          * Sanity check the domain. We don't support domains across
1290          * different SMMUs.
1291          */
1292         if (smmu_domain->smmu != smmu) {
1293                 dev_err(dev,
1294                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1295                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1296                 return -EINVAL;
1297         }
1298
1299         /* Looks ok, so add the device to the domain */
1300         return arm_smmu_domain_add_master(smmu_domain, fwspec);
1301 }
1302
1303 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1304                         phys_addr_t paddr, size_t size, int prot)
1305 {
1306         int ret;
1307         unsigned long flags;
1308         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1309         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1310
1311         if (!ops)
1312                 return -ENODEV;
1313
1314         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1315         ret = ops->map(ops, iova, paddr, size, prot);
1316         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1317         return ret;
1318 }
1319
1320 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1321                              size_t size)
1322 {
1323         size_t ret;
1324         unsigned long flags;
1325         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1326         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1327
1328         if (!ops)
1329                 return 0;
1330
1331         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1332         ret = ops->unmap(ops, iova, size);
1333         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1334         return ret;
1335 }
1336
1337 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1338                                               dma_addr_t iova)
1339 {
1340         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1341         struct arm_smmu_device *smmu = smmu_domain->smmu;
1342         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1343         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1344         struct device *dev = smmu->dev;
1345         void __iomem *cb_base;
1346         u32 tmp;
1347         u64 phys;
1348         unsigned long va;
1349
1350         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1351
1352         /* ATS1 registers can only be written atomically */
1353         va = iova & ~0xfffUL;
1354         if (smmu->version == ARM_SMMU_V2)
1355                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1356         else /* Register is only 32-bit in v1 */
1357                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1358
1359         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1360                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1361                 dev_err(dev,
1362                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1363                         &iova);
1364                 return ops->iova_to_phys(ops, iova);
1365         }
1366
1367         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1368         if (phys & CB_PAR_F) {
1369                 dev_err(dev, "translation fault!\n");
1370                 dev_err(dev, "PAR = 0x%llx\n", phys);
1371                 return 0;
1372         }
1373
1374         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1375 }
1376
1377 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1378                                         dma_addr_t iova)
1379 {
1380         phys_addr_t ret;
1381         unsigned long flags;
1382         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1383         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1384
1385         if (!ops)
1386                 return 0;
1387
1388         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1389         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1390                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1391                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1392         } else {
1393                 ret = ops->iova_to_phys(ops, iova);
1394         }
1395
1396         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1397
1398         return ret;
1399 }
1400
1401 static bool arm_smmu_capable(enum iommu_cap cap)
1402 {
1403         switch (cap) {
1404         case IOMMU_CAP_CACHE_COHERENCY:
1405                 /*
1406                  * Return true here as the SMMU can always send out coherent
1407                  * requests.
1408                  */
1409                 return true;
1410         case IOMMU_CAP_INTR_REMAP:
1411                 return true; /* MSIs are just memory writes */
1412         case IOMMU_CAP_NOEXEC:
1413                 return true;
1414         default:
1415                 return false;
1416         }
1417 }
1418
1419 static int arm_smmu_match_node(struct device *dev, void *data)
1420 {
1421         return dev->fwnode == data;
1422 }
1423
1424 static
1425 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1426 {
1427         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1428                                                 fwnode, arm_smmu_match_node);
1429         put_device(dev);
1430         return dev ? dev_get_drvdata(dev) : NULL;
1431 }
1432
1433 static int arm_smmu_add_device(struct device *dev)
1434 {
1435         struct arm_smmu_device *smmu;
1436         struct arm_smmu_master_cfg *cfg;
1437         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1438         int i, ret;
1439
1440         if (using_legacy_binding) {
1441                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1442                 fwspec = dev->iommu_fwspec;
1443                 if (ret)
1444                         goto out_free;
1445         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1446                 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1447         } else {
1448                 return -ENODEV;
1449         }
1450
1451         ret = -EINVAL;
1452         for (i = 0; i < fwspec->num_ids; i++) {
1453                 u16 sid = fwspec->ids[i];
1454                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1455
1456                 if (sid & ~smmu->streamid_mask) {
1457                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1458                                 sid, smmu->streamid_mask);
1459                         goto out_free;
1460                 }
1461                 if (mask & ~smmu->smr_mask_mask) {
1462                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1463                                 sid, smmu->smr_mask_mask);
1464                         goto out_free;
1465                 }
1466         }
1467
1468         ret = -ENOMEM;
1469         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1470                       GFP_KERNEL);
1471         if (!cfg)
1472                 goto out_free;
1473
1474         cfg->smmu = smmu;
1475         fwspec->iommu_priv = cfg;
1476         while (i--)
1477                 cfg->smendx[i] = INVALID_SMENDX;
1478
1479         ret = arm_smmu_master_alloc_smes(dev);
1480         if (ret)
1481                 goto out_free;
1482
1483         return 0;
1484
1485 out_free:
1486         if (fwspec)
1487                 kfree(fwspec->iommu_priv);
1488         iommu_fwspec_free(dev);
1489         return ret;
1490 }
1491
1492 static void arm_smmu_remove_device(struct device *dev)
1493 {
1494         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1495
1496         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1497                 return;
1498
1499         arm_smmu_master_free_smes(fwspec);
1500         iommu_group_remove_device(dev);
1501         kfree(fwspec->iommu_priv);
1502         iommu_fwspec_free(dev);
1503 }
1504
1505 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1506 {
1507         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1508         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1509         struct iommu_group *group = NULL;
1510         int i, idx;
1511
1512         for_each_cfg_sme(fwspec, i, idx) {
1513                 if (group && smmu->s2crs[idx].group &&
1514                     group != smmu->s2crs[idx].group)
1515                         return ERR_PTR(-EINVAL);
1516
1517                 group = smmu->s2crs[idx].group;
1518         }
1519
1520         if (group)
1521                 return iommu_group_ref_get(group);
1522
1523         if (dev_is_pci(dev))
1524                 group = pci_device_group(dev);
1525         else
1526                 group = generic_device_group(dev);
1527
1528         return group;
1529 }
1530
1531 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1532                                     enum iommu_attr attr, void *data)
1533 {
1534         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1535
1536         switch (attr) {
1537         case DOMAIN_ATTR_NESTING:
1538                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1539                 return 0;
1540         default:
1541                 return -ENODEV;
1542         }
1543 }
1544
1545 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1546                                     enum iommu_attr attr, void *data)
1547 {
1548         int ret = 0;
1549         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1550
1551         mutex_lock(&smmu_domain->init_mutex);
1552
1553         switch (attr) {
1554         case DOMAIN_ATTR_NESTING:
1555                 if (smmu_domain->smmu) {
1556                         ret = -EPERM;
1557                         goto out_unlock;
1558                 }
1559
1560                 if (*(int *)data)
1561                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1562                 else
1563                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1564
1565                 break;
1566         default:
1567                 ret = -ENODEV;
1568         }
1569
1570 out_unlock:
1571         mutex_unlock(&smmu_domain->init_mutex);
1572         return ret;
1573 }
1574
1575 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1576 {
1577         u32 fwid = 0;
1578
1579         if (args->args_count > 0)
1580                 fwid |= (u16)args->args[0];
1581
1582         if (args->args_count > 1)
1583                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1584
1585         return iommu_fwspec_add_ids(dev, &fwid, 1);
1586 }
1587
1588 static struct iommu_ops arm_smmu_ops = {
1589         .capable                = arm_smmu_capable,
1590         .domain_alloc           = arm_smmu_domain_alloc,
1591         .domain_free            = arm_smmu_domain_free,
1592         .attach_dev             = arm_smmu_attach_dev,
1593         .map                    = arm_smmu_map,
1594         .unmap                  = arm_smmu_unmap,
1595         .map_sg                 = default_iommu_map_sg,
1596         .iova_to_phys           = arm_smmu_iova_to_phys,
1597         .add_device             = arm_smmu_add_device,
1598         .remove_device          = arm_smmu_remove_device,
1599         .device_group           = arm_smmu_device_group,
1600         .domain_get_attr        = arm_smmu_domain_get_attr,
1601         .domain_set_attr        = arm_smmu_domain_set_attr,
1602         .of_xlate               = arm_smmu_of_xlate,
1603         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1604 };
1605
1606 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1607 {
1608         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1609         void __iomem *cb_base;
1610         int i;
1611         u32 reg, major;
1612
1613         /* clear global FSR */
1614         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1615         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1616
1617         /*
1618          * Reset stream mapping groups: Initial values mark all SMRn as
1619          * invalid and all S2CRn as bypass unless overridden.
1620          */
1621         for (i = 0; i < smmu->num_mapping_groups; ++i)
1622                 arm_smmu_write_sme(smmu, i);
1623
1624         if (smmu->model == ARM_MMU500) {
1625                 /*
1626                  * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1627                  * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1628                  * bit is only present in MMU-500r2 onwards.
1629                  */
1630                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1631                 major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1632                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1633                 if (major >= 2)
1634                         reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1635                 /*
1636                  * Allow unmatched Stream IDs to allocate bypass
1637                  * TLB entries for reduced latency.
1638                  */
1639                 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN;
1640                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1641         }
1642
1643         /* Make sure all context banks are disabled and clear CB_FSR  */
1644         for (i = 0; i < smmu->num_context_banks; ++i) {
1645                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1646                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1647                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1648                 /*
1649                  * Disable MMU-500's not-particularly-beneficial next-page
1650                  * prefetcher for the sake of errata #841119 and #826419.
1651                  */
1652                 if (smmu->model == ARM_MMU500) {
1653                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1654                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1655                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1656                 }
1657         }
1658
1659         /* Invalidate the TLB, just in case */
1660         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1661         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1662
1663         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1664
1665         /* Enable fault reporting */
1666         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1667
1668         /* Disable TLB broadcasting. */
1669         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1670
1671         /* Enable client access, handling unmatched streams as appropriate */
1672         reg &= ~sCR0_CLIENTPD;
1673         if (disable_bypass)
1674                 reg |= sCR0_USFCFG;
1675         else
1676                 reg &= ~sCR0_USFCFG;
1677
1678         /* Disable forced broadcasting */
1679         reg &= ~sCR0_FB;
1680
1681         /* Don't upgrade barriers */
1682         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1683
1684         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1685                 reg |= sCR0_VMID16EN;
1686
1687         if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1688                 reg |= sCR0_EXIDENABLE;
1689
1690         /* Push the button */
1691         __arm_smmu_tlb_sync(smmu);
1692         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1693 }
1694
1695 static int arm_smmu_id_size_to_bits(int size)
1696 {
1697         switch (size) {
1698         case 0:
1699                 return 32;
1700         case 1:
1701                 return 36;
1702         case 2:
1703                 return 40;
1704         case 3:
1705                 return 42;
1706         case 4:
1707                 return 44;
1708         case 5:
1709         default:
1710                 return 48;
1711         }
1712 }
1713
1714 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1715 {
1716         unsigned long size;
1717         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1718         u32 id;
1719         bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1720         int i;
1721
1722         dev_notice(smmu->dev, "probing hardware configuration...\n");
1723         dev_notice(smmu->dev, "SMMUv%d with:\n",
1724                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1725
1726         /* ID0 */
1727         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1728
1729         /* Restrict available stages based on module parameter */
1730         if (force_stage == 1)
1731                 id &= ~(ID0_S2TS | ID0_NTS);
1732         else if (force_stage == 2)
1733                 id &= ~(ID0_S1TS | ID0_NTS);
1734
1735         if (id & ID0_S1TS) {
1736                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1737                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1738         }
1739
1740         if (id & ID0_S2TS) {
1741                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1742                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1743         }
1744
1745         if (id & ID0_NTS) {
1746                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1747                 dev_notice(smmu->dev, "\tnested translation\n");
1748         }
1749
1750         if (!(smmu->features &
1751                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1752                 dev_err(smmu->dev, "\tno translation support!\n");
1753                 return -ENODEV;
1754         }
1755
1756         if ((id & ID0_S1TS) &&
1757                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1758                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1759                 dev_notice(smmu->dev, "\taddress translation ops\n");
1760         }
1761
1762         /*
1763          * In order for DMA API calls to work properly, we must defer to what
1764          * the FW says about coherency, regardless of what the hardware claims.
1765          * Fortunately, this also opens up a workaround for systems where the
1766          * ID register value has ended up configured incorrectly.
1767          */
1768         cttw_reg = !!(id & ID0_CTTW);
1769         if (cttw_fw || cttw_reg)
1770                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1771                            cttw_fw ? "" : "non-");
1772         if (cttw_fw != cttw_reg)
1773                 dev_notice(smmu->dev,
1774                            "\t(IDR0.CTTW overridden by FW configuration)\n");
1775
1776         /* Max. number of entries we have for stream matching/indexing */
1777         if (smmu->version == ARM_SMMU_V2 && id & ID0_EXIDS) {
1778                 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1779                 size = 1 << 16;
1780         } else {
1781                 size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1782         }
1783         smmu->streamid_mask = size - 1;
1784         if (id & ID0_SMS) {
1785                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1786                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1787                 if (size == 0) {
1788                         dev_err(smmu->dev,
1789                                 "stream-matching supported, but no SMRs present!\n");
1790                         return -ENODEV;
1791                 }
1792
1793                 /* Zero-initialised to mark as invalid */
1794                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1795                                           GFP_KERNEL);
1796                 if (!smmu->smrs)
1797                         return -ENOMEM;
1798
1799                 dev_notice(smmu->dev,
1800                            "\tstream matching with %lu register groups", size);
1801         }
1802         /* s2cr->type == 0 means translation, so initialise explicitly */
1803         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1804                                          GFP_KERNEL);
1805         if (!smmu->s2crs)
1806                 return -ENOMEM;
1807         for (i = 0; i < size; i++)
1808                 smmu->s2crs[i] = s2cr_init_val;
1809
1810         smmu->num_mapping_groups = size;
1811         mutex_init(&smmu->stream_map_mutex);
1812
1813         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1814                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1815                 if (!(id & ID0_PTFS_NO_AARCH32S))
1816                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1817         }
1818
1819         /* ID1 */
1820         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1821         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1822
1823         /* Check for size mismatch of SMMU address space from mapped region */
1824         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1825         size *= 2 << smmu->pgshift;
1826         if (smmu->size != size)
1827                 dev_warn(smmu->dev,
1828                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1829                         size, smmu->size);
1830
1831         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1832         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1833         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1834                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1835                 return -ENODEV;
1836         }
1837         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1838                    smmu->num_context_banks, smmu->num_s2_context_banks);
1839         /*
1840          * Cavium CN88xx erratum #27704.
1841          * Ensure ASID and VMID allocation is unique across all SMMUs in
1842          * the system.
1843          */
1844         if (smmu->model == CAVIUM_SMMUV2) {
1845                 smmu->cavium_id_base =
1846                         atomic_add_return(smmu->num_context_banks,
1847                                           &cavium_smmu_context_count);
1848                 smmu->cavium_id_base -= smmu->num_context_banks;
1849         }
1850
1851         /* ID2 */
1852         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1853         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1854         smmu->ipa_size = size;
1855
1856         /* The output mask is also applied for bypass */
1857         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1858         smmu->pa_size = size;
1859
1860         if (id & ID2_VMID16)
1861                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1862
1863         /*
1864          * What the page table walker can address actually depends on which
1865          * descriptor format is in use, but since a) we don't know that yet,
1866          * and b) it can vary per context bank, this will have to do...
1867          */
1868         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1869                 dev_warn(smmu->dev,
1870                          "failed to set DMA mask for table walker\n");
1871
1872         if (smmu->version < ARM_SMMU_V2) {
1873                 smmu->va_size = smmu->ipa_size;
1874                 if (smmu->version == ARM_SMMU_V1_64K)
1875                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1876         } else {
1877                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1878                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1879                 if (id & ID2_PTFS_4K)
1880                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1881                 if (id & ID2_PTFS_16K)
1882                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1883                 if (id & ID2_PTFS_64K)
1884                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1885         }
1886
1887         /* Now we've corralled the various formats, what'll it do? */
1888         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1889                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1890         if (smmu->features &
1891             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1892                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1893         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1894                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1895         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1896                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1897
1898         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1899                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1900         else
1901                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1902         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1903                    smmu->pgsize_bitmap);
1904
1905
1906         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1907                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1908                            smmu->va_size, smmu->ipa_size);
1909
1910         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1911                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1912                            smmu->ipa_size, smmu->pa_size);
1913
1914         return 0;
1915 }
1916
1917 struct arm_smmu_match_data {
1918         enum arm_smmu_arch_version version;
1919         enum arm_smmu_implementation model;
1920 };
1921
1922 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1923 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1924
1925 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1926 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1927 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1928 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1929 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1930
1931 static const struct of_device_id arm_smmu_of_match[] = {
1932         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1933         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1934         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1935         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1936         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1937         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1938         { },
1939 };
1940 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1941
1942 #ifdef CONFIG_ACPI
1943 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1944 {
1945         int ret = 0;
1946
1947         switch (model) {
1948         case ACPI_IORT_SMMU_V1:
1949         case ACPI_IORT_SMMU_CORELINK_MMU400:
1950                 smmu->version = ARM_SMMU_V1;
1951                 smmu->model = GENERIC_SMMU;
1952                 break;
1953         case ACPI_IORT_SMMU_V2:
1954                 smmu->version = ARM_SMMU_V2;
1955                 smmu->model = GENERIC_SMMU;
1956                 break;
1957         case ACPI_IORT_SMMU_CORELINK_MMU500:
1958                 smmu->version = ARM_SMMU_V2;
1959                 smmu->model = ARM_MMU500;
1960                 break;
1961         default:
1962                 ret = -ENODEV;
1963         }
1964
1965         return ret;
1966 }
1967
1968 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1969                                       struct arm_smmu_device *smmu)
1970 {
1971         struct device *dev = smmu->dev;
1972         struct acpi_iort_node *node =
1973                 *(struct acpi_iort_node **)dev_get_platdata(dev);
1974         struct acpi_iort_smmu *iort_smmu;
1975         int ret;
1976
1977         /* Retrieve SMMU1/2 specific data */
1978         iort_smmu = (struct acpi_iort_smmu *)node->node_data;
1979
1980         ret = acpi_smmu_get_data(iort_smmu->model, smmu);
1981         if (ret < 0)
1982                 return ret;
1983
1984         /* Ignore the configuration access interrupt */
1985         smmu->num_global_irqs = 1;
1986
1987         if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
1988                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1989
1990         return 0;
1991 }
1992 #else
1993 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
1994                                              struct arm_smmu_device *smmu)
1995 {
1996         return -ENODEV;
1997 }
1998 #endif
1999
2000 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
2001                                     struct arm_smmu_device *smmu)
2002 {
2003         const struct arm_smmu_match_data *data;
2004         struct device *dev = &pdev->dev;
2005         bool legacy_binding;
2006
2007         if (of_property_read_u32(dev->of_node, "#global-interrupts",
2008                                  &smmu->num_global_irqs)) {
2009                 dev_err(dev, "missing #global-interrupts property\n");
2010                 return -ENODEV;
2011         }
2012
2013         data = of_device_get_match_data(dev);
2014         smmu->version = data->version;
2015         smmu->model = data->model;
2016
2017         parse_driver_options(smmu);
2018
2019         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2020         if (legacy_binding && !using_generic_binding) {
2021                 if (!using_legacy_binding)
2022                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
2023                 using_legacy_binding = true;
2024         } else if (!legacy_binding && !using_legacy_binding) {
2025                 using_generic_binding = true;
2026         } else {
2027                 dev_err(dev, "not probing due to mismatched DT properties\n");
2028                 return -ENODEV;
2029         }
2030
2031         if (of_dma_is_coherent(dev->of_node))
2032                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2033
2034         return 0;
2035 }
2036
2037 static int arm_smmu_device_probe(struct platform_device *pdev)
2038 {
2039         struct resource *res;
2040         struct arm_smmu_device *smmu;
2041         struct device *dev = &pdev->dev;
2042         int num_irqs, i, err;
2043
2044         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2045         if (!smmu) {
2046                 dev_err(dev, "failed to allocate arm_smmu_device\n");
2047                 return -ENOMEM;
2048         }
2049         smmu->dev = dev;
2050
2051         if (dev->of_node)
2052                 err = arm_smmu_device_dt_probe(pdev, smmu);
2053         else
2054                 err = arm_smmu_device_acpi_probe(pdev, smmu);
2055
2056         if (err)
2057                 return err;
2058
2059         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2060         smmu->base = devm_ioremap_resource(dev, res);
2061         if (IS_ERR(smmu->base))
2062                 return PTR_ERR(smmu->base);
2063         smmu->size = resource_size(res);
2064
2065         num_irqs = 0;
2066         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
2067                 num_irqs++;
2068                 if (num_irqs > smmu->num_global_irqs)
2069                         smmu->num_context_irqs++;
2070         }
2071
2072         if (!smmu->num_context_irqs) {
2073                 dev_err(dev, "found %d interrupts but expected at least %d\n",
2074                         num_irqs, smmu->num_global_irqs + 1);
2075                 return -ENODEV;
2076         }
2077
2078         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
2079                                   GFP_KERNEL);
2080         if (!smmu->irqs) {
2081                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
2082                 return -ENOMEM;
2083         }
2084
2085         for (i = 0; i < num_irqs; ++i) {
2086                 int irq = platform_get_irq(pdev, i);
2087
2088                 if (irq < 0) {
2089                         dev_err(dev, "failed to get irq index %d\n", i);
2090                         return -ENODEV;
2091                 }
2092                 smmu->irqs[i] = irq;
2093         }
2094
2095         err = arm_smmu_device_cfg_probe(smmu);
2096         if (err)
2097                 return err;
2098
2099         if (smmu->version == ARM_SMMU_V2 &&
2100             smmu->num_context_banks != smmu->num_context_irqs) {
2101                 dev_err(dev,
2102                         "found only %d context interrupt(s) but %d required\n",
2103                         smmu->num_context_irqs, smmu->num_context_banks);
2104                 return -ENODEV;
2105         }
2106
2107         for (i = 0; i < smmu->num_global_irqs; ++i) {
2108                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2109                                        arm_smmu_global_fault,
2110                                        IRQF_SHARED,
2111                                        "arm-smmu global fault",
2112                                        smmu);
2113                 if (err) {
2114                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2115                                 i, smmu->irqs[i]);
2116                         return err;
2117                 }
2118         }
2119
2120         iommu_register_instance(dev->fwnode, &arm_smmu_ops);
2121         platform_set_drvdata(pdev, smmu);
2122         arm_smmu_device_reset(smmu);
2123         arm_smmu_test_smr_masks(smmu);
2124
2125         /* Oh, for a proper bus abstraction */
2126         if (!iommu_present(&platform_bus_type))
2127                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2128 #ifdef CONFIG_ARM_AMBA
2129         if (!iommu_present(&amba_bustype))
2130                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2131 #endif
2132 #ifdef CONFIG_PCI
2133         if (!iommu_present(&pci_bus_type)) {
2134                 pci_request_acs();
2135                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2136         }
2137 #endif
2138         return 0;
2139 }
2140
2141 static int arm_smmu_device_remove(struct platform_device *pdev)
2142 {
2143         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2144
2145         if (!smmu)
2146                 return -ENODEV;
2147
2148         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2149                 dev_err(&pdev->dev, "removing device with active domains!\n");
2150
2151         /* Turn the thing off */
2152         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2153         return 0;
2154 }
2155
2156 static struct platform_driver arm_smmu_driver = {
2157         .driver = {
2158                 .name           = "arm-smmu",
2159                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2160         },
2161         .probe  = arm_smmu_device_probe,
2162         .remove = arm_smmu_device_remove,
2163 };
2164
2165 static int __init arm_smmu_init(void)
2166 {
2167         static bool registered;
2168         int ret = 0;
2169
2170         if (!registered) {
2171                 ret = platform_driver_register(&arm_smmu_driver);
2172                 registered = !ret;
2173         }
2174         return ret;
2175 }
2176
2177 static void __exit arm_smmu_exit(void)
2178 {
2179         return platform_driver_unregister(&arm_smmu_driver);
2180 }
2181
2182 subsys_initcall(arm_smmu_init);
2183 module_exit(arm_smmu_exit);
2184
2185 static int __init arm_smmu_of_init(struct device_node *np)
2186 {
2187         int ret = arm_smmu_init();
2188
2189         if (ret)
2190                 return ret;
2191
2192         if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2193                 return -ENODEV;
2194
2195         return 0;
2196 }
2197 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
2198 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
2199 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
2200 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
2201 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
2202 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
2203
2204 #ifdef CONFIG_ACPI
2205 static int __init arm_smmu_acpi_init(struct acpi_table_header *table)
2206 {
2207         if (iort_node_match(ACPI_IORT_NODE_SMMU))
2208                 return arm_smmu_init();
2209
2210         return 0;
2211 }
2212 IORT_ACPI_DECLARE(arm_smmu, ACPI_SIG_IORT, arm_smmu_acpi_init);
2213 #endif
2214
2215 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2216 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2217 MODULE_LICENSE("GPL v2");