drivers/staging/media/ipu3/ipu3-mmu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2018 Intel Corporation.
   4  * Copyright 2018 Google LLC.
   5  *
   6  * Author: Tuukka Toivonen <tuukka.toivonen@intel.com>
   7  * Author: Sakari Ailus <sakari.ailus@linux.intel.com>
   8  * Author: Samu Onkalo <samu.onkalo@intel.com>
   9  * Author: Tomasz Figa <tfiga@chromium.org>
  10  *
  11  */
  12
  13 #include <linux/dma-mapping.h>
  14 #include <linux/iopoll.h>
  15 #include <linux/pm_runtime.h>
  16 #include <linux/slab.h>
  17 #include <linux/vmalloc.h>
  18
  19 #include <asm/set_memory.h>
  20
  21 #include "ipu3-mmu.h"
  22
  23 #define IPU3_PAGE_SHIFT         12
  24 #define IPU3_PAGE_SIZE          (1UL << IPU3_PAGE_SHIFT)
  25
  26 #define IPU3_PT_BITS            10
  27 #define IPU3_PT_PTES            (1UL << IPU3_PT_BITS)
  28 #define IPU3_PT_SIZE            (IPU3_PT_PTES << 2)
  29 #define IPU3_PT_ORDER           (IPU3_PT_SIZE >> PAGE_SHIFT)
  30
  31 #define IPU3_ADDR2PTE(addr)     ((addr) >> IPU3_PAGE_SHIFT)
  32 #define IPU3_PTE2ADDR(pte)      ((phys_addr_t)(pte) << IPU3_PAGE_SHIFT)
  33
  34 #define IPU3_L2PT_SHIFT         IPU3_PT_BITS
  35 #define IPU3_L2PT_MASK          ((1UL << IPU3_L2PT_SHIFT) - 1)
  36
  37 #define IPU3_L1PT_SHIFT         IPU3_PT_BITS
  38 #define IPU3_L1PT_MASK          ((1UL << IPU3_L1PT_SHIFT) - 1)
  39
  40 #define IPU3_MMU_ADDRESS_BITS   (IPU3_PAGE_SHIFT + \
  41                                  IPU3_L2PT_SHIFT + \
  42                                  IPU3_L1PT_SHIFT)
  43
  44 #define IMGU_REG_BASE           0x4000
  45 #define REG_TLB_INVALIDATE      (IMGU_REG_BASE + 0x300)
  46 #define TLB_INVALIDATE          1
  47 #define REG_L1_PHYS             (IMGU_REG_BASE + 0x304) /* 27-bit pfn */
  48 #define REG_GP_HALT             (IMGU_REG_BASE + 0x5dc)
  49 #define REG_GP_HALTED           (IMGU_REG_BASE + 0x5e0)
  50
  51 struct ipu3_mmu {
  52         struct device *dev;
  53         void __iomem *base;
  54         /* protect access to l2pts, l1pt */
  55         spinlock_t lock;
  56
  57         void *dummy_page;
  58         u32 dummy_page_pteval;
  59
  60         u32 *dummy_l2pt;
  61         u32 dummy_l2pt_pteval;
  62
  63         u32 **l2pts;
  64         u32 *l1pt;
  65
  66         struct ipu3_mmu_info geometry;
  67 };
  68
  69 static inline struct ipu3_mmu *to_ipu3_mmu(struct ipu3_mmu_info *info)
  70 {
  71         return container_of(info, struct ipu3_mmu, geometry);
  72 }
  73
  74 /**
  75  * ipu3_mmu_tlb_invalidate - invalidate translation look-aside buffer
  76  * @mmu: MMU to perform the invalidate operation on
  77  *
  78  * This function invalidates the whole TLB. Must be called when the hardware
  79  * is powered on.
  80  */
  81 static void ipu3_mmu_tlb_invalidate(struct ipu3_mmu *mmu)
  82 {
  83         writel(TLB_INVALIDATE, mmu->base + REG_TLB_INVALIDATE);
  84 }
  85
  86 static void call_if_ipu3_is_powered(struct ipu3_mmu *mmu,
  87                                     void (*func)(struct ipu3_mmu *mmu))
  88 {
  89         if (!pm_runtime_get_if_in_use(mmu->dev))
  90                 return;
  91
  92         func(mmu);
  93         pm_runtime_put(mmu->dev);
  94 }
  95
  96 /**
  97  * ipu3_mmu_set_halt - set CIO gate halt bit
  98  * @mmu: MMU to set the CIO gate bit in.
  99  * @halt: Desired state of the gate bit.
 100  *
 101  * This function sets the CIO gate bit that controls whether external memory
 102  * accesses are allowed. Must be called when the hardware is powered on.
 103  */
 104 static void ipu3_mmu_set_halt(struct ipu3_mmu *mmu, bool halt)
 105 {
 106         int ret;
 107         u32 val;
 108
 109         writel(halt, mmu->base + REG_GP_HALT);
 110         ret = readl_poll_timeout(mmu->base + REG_GP_HALTED,
 111                                  val, (val & 1) == halt, 1000, 100000);
 112
 113         if (ret)
 114                 dev_err(mmu->dev, "failed to %s CIO gate halt\n",
 115                         halt ? "set" : "clear");
 116 }
 117
 118 /**
 119  * ipu3_mmu_alloc_page_table - allocate a pre-filled page table
 120  * @pteval: Value to initialize for page table entries with.
 121  *
 122  * Return: Pointer to allocated page table or NULL on failure.
 123  */
 124 static u32 *ipu3_mmu_alloc_page_table(u32 pteval)
 125 {
 126         u32 *pt;
 127         int pte;
 128
 129         pt = (u32 *)__get_free_page(GFP_KERNEL);
 130         if (!pt)
 131                 return NULL;
 132
 133         for (pte = 0; pte < IPU3_PT_PTES; pte++)
 134                 pt[pte] = pteval;
 135
 136         set_memory_uc((unsigned long int)pt, IPU3_PT_ORDER);
 137
 138         return pt;
 139 }
 140
 141 /**
 142  * ipu3_mmu_free_page_table - free page table
 143  * @pt: Page table to free.
 144  */
 145 static void ipu3_mmu_free_page_table(u32 *pt)
 146 {
 147         set_memory_wb((unsigned long int)pt, IPU3_PT_ORDER);
 148         free_page((unsigned long)pt);
 149 }
 150
 151 /**
 152  * address_to_pte_idx - split IOVA into L1 and L2 page table indices
 153  * @iova: IOVA to split.
 154  * @l1pt_idx: Output for the L1 page table index.
 155  * @l2pt_idx: Output for the L2 page index.
 156  */
 157 static inline void address_to_pte_idx(unsigned long iova, u32 *l1pt_idx,
 158                                       u32 *l2pt_idx)
 159 {
 160         iova >>= IPU3_PAGE_SHIFT;
 161
 162         if (l2pt_idx)
 163                 *l2pt_idx = iova & IPU3_L2PT_MASK;
 164
 165         iova >>= IPU3_L2PT_SHIFT;
 166
 167         if (l1pt_idx)
 168                 *l1pt_idx = iova & IPU3_L1PT_MASK;
 169 }
 170
 171 static u32 *ipu3_mmu_get_l2pt(struct ipu3_mmu *mmu, u32 l1pt_idx)
 172 {
 173         unsigned long flags;
 174         u32 *l2pt, *new_l2pt;
 175         u32 pteval;
 176
 177         spin_lock_irqsave(&mmu->lock, flags);
 178
 179         l2pt = mmu->l2pts[l1pt_idx];
 180         if (l2pt)
 181                 goto done;
 182
 183         spin_unlock_irqrestore(&mmu->lock, flags);
 184
 185         new_l2pt = ipu3_mmu_alloc_page_table(mmu->dummy_page_pteval);
 186         if (!new_l2pt)
 187                 return NULL;
 188
 189         spin_lock_irqsave(&mmu->lock, flags);
 190
 191         dev_dbg(mmu->dev, "allocated page table %p for l1pt_idx %u\n",
 192                 new_l2pt, l1pt_idx);
 193
 194         l2pt = mmu->l2pts[l1pt_idx];
 195         if (l2pt) {
 196                 ipu3_mmu_free_page_table(new_l2pt);
 197                 goto done;
 198         }
 199
 200         l2pt = new_l2pt;
 201         mmu->l2pts[l1pt_idx] = new_l2pt;
 202
 203         pteval = IPU3_ADDR2PTE(virt_to_phys(new_l2pt));
 204         mmu->l1pt[l1pt_idx] = pteval;
 205
 206 done:
 207         spin_unlock_irqrestore(&mmu->lock, flags);
 208         return l2pt;
 209 }
 210
 211 static int __ipu3_mmu_map(struct ipu3_mmu *mmu, unsigned long iova,
 212                           phys_addr_t paddr)
 213 {
 214         u32 l1pt_idx, l2pt_idx;
 215         unsigned long flags;
 216         u32 *l2pt;
 217
 218         if (!mmu)
 219                 return -ENODEV;
 220
 221         address_to_pte_idx(iova, &l1pt_idx, &l2pt_idx);
 222
 223         l2pt = ipu3_mmu_get_l2pt(mmu, l1pt_idx);
 224         if (!l2pt)
 225                 return -ENOMEM;
 226
 227         spin_lock_irqsave(&mmu->lock, flags);
 228
 229         if (l2pt[l2pt_idx] != mmu->dummy_page_pteval) {
 230                 spin_unlock_irqrestore(&mmu->lock, flags);
 231                 return -EBUSY;
 232         }
 233
 234         l2pt[l2pt_idx] = IPU3_ADDR2PTE(paddr);
 235
 236         spin_unlock_irqrestore(&mmu->lock, flags);
 237
 238         return 0;
 239 }
 240
 241 /**
 242  * The following four functions are implemented based on iommu.c
 243  * drivers/iommu/iommu.c/iommu_pgsize().
 244  */
 245 static size_t ipu3_mmu_pgsize(unsigned long pgsize_bitmap,
 246                               unsigned long addr_merge, size_t size)
 247 {
 248         unsigned int pgsize_idx;
 249         size_t pgsize;
 250
 251         /* Max page size that still fits into 'size' */
 252         pgsize_idx = __fls(size);
 253
 254         /* need to consider alignment requirements ? */
 255         if (likely(addr_merge)) {
 256                 /* Max page size allowed by address */
 257                 unsigned int align_pgsize_idx = __ffs(addr_merge);
 258
 259                 pgsize_idx = min(pgsize_idx, align_pgsize_idx);
 260         }
 261
 262         /* build a mask of acceptable page sizes */
 263         pgsize = (1UL << (pgsize_idx + 1)) - 1;
 264
 265         /* throw away page sizes not supported by the hardware */
 266         pgsize &= pgsize_bitmap;
 267
 268         /* make sure we're still sane */
 269         WARN_ON(!pgsize);
 270
 271         /* pick the biggest page */
 272         pgsize_idx = __fls(pgsize);
 273         pgsize = 1UL << pgsize_idx;
 274
 275         return pgsize;
 276 }
 277
 278 /* drivers/iommu/iommu.c/iommu_map() */
 279 int ipu3_mmu_map(struct ipu3_mmu_info *info, unsigned long iova,
 280                  phys_addr_t paddr, size_t size)
 281 {
 282         struct ipu3_mmu *mmu = to_ipu3_mmu(info);
 283         unsigned int min_pagesz;
 284         int ret = 0;
 285
 286         /* find out the minimum page size supported */
 287         min_pagesz = 1 << __ffs(mmu->geometry.pgsize_bitmap);
 288
 289         /*
 290          * both the virtual address and the physical one, as well as
 291          * the size of the mapping, must be aligned (at least) to the
 292          * size of the smallest page supported by the hardware
 293          */
 294         if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
 295                 dev_err(mmu->dev, "unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
 296                         iova, &paddr, size, min_pagesz);
 297                 return -EINVAL;
 298         }
 299
 300         dev_dbg(mmu->dev, "map: iova 0x%lx pa %pa size 0x%zx\n",
 301                 iova, &paddr, size);
 302
 303         while (size) {
 304                 size_t pgsize = ipu3_mmu_pgsize(mmu->geometry.pgsize_bitmap,
 305                                                 iova | paddr, size);
 306
 307                 dev_dbg(mmu->dev, "mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
 308                         iova, &paddr, pgsize);
 309
 310                 ret = __ipu3_mmu_map(mmu, iova, paddr);
 311                 if (ret)
 312                         break;
 313
 314                 iova += pgsize;
 315                 paddr += pgsize;
 316                 size -= pgsize;
 317         }
 318
 319         call_if_ipu3_is_powered(mmu, ipu3_mmu_tlb_invalidate);
 320
 321         return ret;
 322 }
 323
 324 /* drivers/iommu/iommu.c/default_iommu_map_sg() */
 325 size_t ipu3_mmu_map_sg(struct ipu3_mmu_info *info, unsigned long iova,
 326                        struct scatterlist *sg, unsigned int nents)
 327 {
 328         struct ipu3_mmu *mmu = to_ipu3_mmu(info);
 329         struct scatterlist *s;
 330         size_t s_length, mapped = 0;
 331         unsigned int i, min_pagesz;
 332         int ret;
 333
 334         min_pagesz = 1 << __ffs(mmu->geometry.pgsize_bitmap);
 335
 336         for_each_sg(sg, s, nents, i) {
 337                 phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset;
 338
 339                 s_length = s->length;
 340
 341                 if (!IS_ALIGNED(s->offset, min_pagesz))
 342                         goto out_err;
 343
 344                 /* must be min_pagesz aligned to be mapped singlely */
 345                 if (i == nents - 1 && !IS_ALIGNED(s->length, min_pagesz))
 346                         s_length = PAGE_ALIGN(s->length);
 347
 348                 ret = ipu3_mmu_map(info, iova + mapped, phys, s_length);
 349                 if (ret)
 350                         goto out_err;
 351
 352                 mapped += s_length;
 353         }
 354
 355         call_if_ipu3_is_powered(mmu, ipu3_mmu_tlb_invalidate);
 356
 357         return mapped;
 358
 359 out_err:
 360         /* undo mappings already done */
 361         ipu3_mmu_unmap(info, iova, mapped);
 362
 363         return 0;
 364 }
 365
 366 static size_t __ipu3_mmu_unmap(struct ipu3_mmu *mmu,
 367                                unsigned long iova, size_t size)
 368 {
 369         u32 l1pt_idx, l2pt_idx;
 370         unsigned long flags;
 371         size_t unmap = size;
 372         u32 *l2pt;
 373
 374         if (!mmu)
 375                 return 0;
 376
 377         address_to_pte_idx(iova, &l1pt_idx, &l2pt_idx);
 378
 379         spin_lock_irqsave(&mmu->lock, flags);
 380
 381         l2pt = mmu->l2pts[l1pt_idx];
 382         if (!l2pt) {
 383                 spin_unlock_irqrestore(&mmu->lock, flags);
 384                 return 0;
 385         }
 386
 387         if (l2pt[l2pt_idx] == mmu->dummy_page_pteval)
 388                 unmap = 0;
 389
 390         l2pt[l2pt_idx] = mmu->dummy_page_pteval;
 391
 392         spin_unlock_irqrestore(&mmu->lock, flags);
 393
 394         return unmap;
 395 }
 396
 397 /* drivers/iommu/iommu.c/iommu_unmap() */
 398 size_t ipu3_mmu_unmap(struct ipu3_mmu_info *info, unsigned long iova,
 399                       size_t size)
 400 {
 401         struct ipu3_mmu *mmu = to_ipu3_mmu(info);
 402         size_t unmapped_page, unmapped = 0;
 403         unsigned int min_pagesz;
 404
 405         /* find out the minimum page size supported */
 406         min_pagesz = 1 << __ffs(mmu->geometry.pgsize_bitmap);
 407
 408         /*
 409          * The virtual address, as well as the size of the mapping, must be
 410          * aligned (at least) to the size of the smallest page supported
 411          * by the hardware
 412          */
 413         if (!IS_ALIGNED(iova | size, min_pagesz)) {
 414                 dev_err(mmu->dev, "unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n",
 415                         iova, size, min_pagesz);
 416                 return -EINVAL;
 417         }
 418
 419         dev_dbg(mmu->dev, "unmap this: iova 0x%lx size 0x%zx\n", iova, size);
 420
 421         /*
 422          * Keep iterating until we either unmap 'size' bytes (or more)
 423          * or we hit an area that isn't mapped.
 424          */
 425         while (unmapped < size) {
 426                 size_t pgsize = ipu3_mmu_pgsize(mmu->geometry.pgsize_bitmap,
 427                                                 iova, size - unmapped);
 428
 429                 unmapped_page = __ipu3_mmu_unmap(mmu, iova, pgsize);
 430                 if (!unmapped_page)
 431                         break;
 432
 433                 dev_dbg(mmu->dev, "unmapped: iova 0x%lx size 0x%zx\n",
 434                         iova, unmapped_page);
 435
 436                 iova += unmapped_page;
 437                 unmapped += unmapped_page;
 438         }
 439
 440         call_if_ipu3_is_powered(mmu, ipu3_mmu_tlb_invalidate);
 441
 442         return unmapped;
 443 }
 444
 445 /**
 446  * ipu3_mmu_init() - initialize IPU3 MMU block
 447  * @base:       IOMEM base of hardware registers.
 448  *
 449  * Return: Pointer to IPU3 MMU private data pointer or ERR_PTR() on error.
 450  */
 451 struct ipu3_mmu_info *ipu3_mmu_init(struct device *parent, void __iomem *base)
 452 {
 453         struct ipu3_mmu *mmu;
 454         u32 pteval;
 455
 456         mmu = kzalloc(sizeof(*mmu), GFP_KERNEL);
 457         if (!mmu)
 458                 return ERR_PTR(-ENOMEM);
 459
 460         mmu->dev = parent;
 461         mmu->base = base;
 462         spin_lock_init(&mmu->lock);
 463
 464         /* Disallow external memory access when having no valid page tables. */
 465         ipu3_mmu_set_halt(mmu, true);
 466
 467         /*
 468          * The MMU does not have a "valid" bit, so we have to use a dummy
 469          * page for invalid entries.
 470          */
 471         mmu->dummy_page = (void *)__get_free_page(GFP_KERNEL);
 472         if (!mmu->dummy_page)
 473                 goto fail_group;
 474         pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->dummy_page));
 475         mmu->dummy_page_pteval = pteval;
 476
 477         /*
 478          * Allocate a dummy L2 page table with all entries pointing to
 479          * the dummy page.
 480          */
 481         mmu->dummy_l2pt = ipu3_mmu_alloc_page_table(pteval);
 482         if (!mmu->dummy_l2pt)
 483                 goto fail_dummy_page;
 484         pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->dummy_l2pt));
 485         mmu->dummy_l2pt_pteval = pteval;
 486
 487         /*
 488          * Allocate the array of L2PT CPU pointers, initialized to zero,
 489          * which means the dummy L2PT allocated above.
 490          */
 491         mmu->l2pts = vzalloc(IPU3_PT_PTES * sizeof(*mmu->l2pts));
 492         if (!mmu->l2pts)
 493                 goto fail_l2pt;
 494
 495         /* Allocate the L1 page table. */
 496         mmu->l1pt = ipu3_mmu_alloc_page_table(mmu->dummy_l2pt_pteval);
 497         if (!mmu->l1pt)
 498                 goto fail_l2pts;
 499
 500         pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->l1pt));
 501         writel(pteval, mmu->base + REG_L1_PHYS);
 502         ipu3_mmu_tlb_invalidate(mmu);
 503         ipu3_mmu_set_halt(mmu, false);
 504
 505         mmu->geometry.aperture_start = 0;
 506         mmu->geometry.aperture_end = DMA_BIT_MASK(IPU3_MMU_ADDRESS_BITS);
 507         mmu->geometry.pgsize_bitmap = IPU3_PAGE_SIZE;
 508
 509         return &mmu->geometry;
 510
 511 fail_l2pts:
 512         vfree(mmu->l2pts);
 513 fail_l2pt:
 514         ipu3_mmu_free_page_table(mmu->dummy_l2pt);
 515 fail_dummy_page:
 516         free_page((unsigned long)mmu->dummy_page);
 517 fail_group:
 518         kfree(mmu);
 519
 520         return ERR_PTR(-ENOMEM);
 521 }
 522
 523 /**
 524  * ipu3_mmu_exit() - clean up IPU3 MMU block
 525  * @mmu: IPU3 MMU private data
 526  */
 527 void ipu3_mmu_exit(struct ipu3_mmu_info *info)
 528 {
 529         struct ipu3_mmu *mmu = to_ipu3_mmu(info);
 530
 531         /* We are going to free our page tables, no more memory access. */
 532         ipu3_mmu_set_halt(mmu, true);
 533         ipu3_mmu_tlb_invalidate(mmu);
 534
 535         ipu3_mmu_free_page_table(mmu->l1pt);
 536         vfree(mmu->l2pts);
 537         ipu3_mmu_free_page_table(mmu->dummy_l2pt);
 538         free_page((unsigned long)mmu->dummy_page);
 539         kfree(mmu);
 540 }
 541
 542 void ipu3_mmu_suspend(struct ipu3_mmu_info *info)
 543 {
 544         struct ipu3_mmu *mmu = to_ipu3_mmu(info);
 545
 546         ipu3_mmu_set_halt(mmu, true);
 547 }
 548
 549 void ipu3_mmu_resume(struct ipu3_mmu_info *info)
 550 {
 551         struct ipu3_mmu *mmu = to_ipu3_mmu(info);
 552         u32 pteval;
 553
 554         ipu3_mmu_set_halt(mmu, true);
 555
 556         pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->l1pt));
 557         writel(pteval, mmu->base + REG_L1_PHYS);
 558
 559         ipu3_mmu_tlb_invalidate(mmu);
 560         ipu3_mmu_set_halt(mmu, false);
 561 }