drivers/gpu/drm/panfrost/panfrost_mmu.c

   1 // SPDX-License-Identifier:     GPL-2.0
   2 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
   3 #include <linux/bitfield.h>
   4 #include <linux/delay.h>
   5 #include <linux/dma-mapping.h>
   6 #include <linux/interrupt.h>
   7 #include <linux/io.h>
   8 #include <linux/iopoll.h>
   9 #include <linux/io-pgtable.h>
  10 #include <linux/iommu.h>
  11 #include <linux/platform_device.h>
  12 #include <linux/pm_runtime.h>
  13 #include <linux/shmem_fs.h>
  14 #include <linux/sizes.h>
  15
  16 #include "panfrost_device.h"
  17 #include "panfrost_mmu.h"
  18 #include "panfrost_gem.h"
  19 #include "panfrost_features.h"
  20 #include "panfrost_regs.h"
  21
  22 #define mmu_write(dev, reg, data) writel(data, dev->iomem + reg)
  23 #define mmu_read(dev, reg) readl(dev->iomem + reg)
  24
  25 struct panfrost_mmu {
  26         struct io_pgtable_cfg pgtbl_cfg;
  27         struct io_pgtable_ops *pgtbl_ops;
  28         struct mutex lock;
  29 };
  30
  31 static int wait_ready(struct panfrost_device *pfdev, u32 as_nr)
  32 {
  33         int ret;
  34         u32 val;
  35
  36         /* Wait for the MMU status to indicate there is no active command, in
  37          * case one is pending. */
  38         ret = readl_relaxed_poll_timeout_atomic(pfdev->iomem + AS_STATUS(as_nr),
  39                 val, !(val & AS_STATUS_AS_ACTIVE), 10, 1000);
  40
  41         if (ret)
  42                 dev_err(pfdev->dev, "AS_ACTIVE bit stuck\n");
  43
  44         return ret;
  45 }
  46
  47 static int write_cmd(struct panfrost_device *pfdev, u32 as_nr, u32 cmd)
  48 {
  49         int status;
  50
  51         /* write AS_COMMAND when MMU is ready to accept another command */
  52         status = wait_ready(pfdev, as_nr);
  53         if (!status)
  54                 mmu_write(pfdev, AS_COMMAND(as_nr), cmd);
  55
  56         return status;
  57 }
  58
  59 static void lock_region(struct panfrost_device *pfdev, u32 as_nr,
  60                         u64 iova, size_t size)
  61 {
  62         u8 region_width;
  63         u64 region = iova & PAGE_MASK;
  64         /*
  65          * fls returns:
  66          * 1 .. 32
  67          *
  68          * 10 + fls(num_pages)
  69          * results in the range (11 .. 42)
  70          */
  71
  72         size = round_up(size, PAGE_SIZE);
  73
  74         region_width = 10 + fls(size >> PAGE_SHIFT);
  75         if ((size >> PAGE_SHIFT) != (1ul << (region_width - 11))) {
  76                 /* not pow2, so must go up to the next pow2 */
  77                 region_width += 1;
  78         }
  79         region |= region_width;
  80
  81         /* Lock the region that needs to be updated */
  82         mmu_write(pfdev, AS_LOCKADDR_LO(as_nr), region & 0xFFFFFFFFUL);
  83         mmu_write(pfdev, AS_LOCKADDR_HI(as_nr), (region >> 32) & 0xFFFFFFFFUL);
  84         write_cmd(pfdev, as_nr, AS_COMMAND_LOCK);
  85 }
  86
  87
  88 static int mmu_hw_do_operation(struct panfrost_device *pfdev, u32 as_nr,
  89                 u64 iova, size_t size, u32 op)
  90 {
  91         unsigned long flags;
  92         int ret;
  93
  94         spin_lock_irqsave(&pfdev->hwaccess_lock, flags);
  95
  96         if (op != AS_COMMAND_UNLOCK)
  97                 lock_region(pfdev, as_nr, iova, size);
  98
  99         /* Run the MMU operation */
 100         write_cmd(pfdev, as_nr, op);
 101
 102         /* Wait for the flush to complete */
 103         ret = wait_ready(pfdev, as_nr);
 104
 105         spin_unlock_irqrestore(&pfdev->hwaccess_lock, flags);
 106
 107         return ret;
 108 }
 109
 110 static void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr)
 111 {
 112         struct io_pgtable_cfg *cfg = &pfdev->mmu->pgtbl_cfg;
 113         u64 transtab = cfg->arm_mali_lpae_cfg.transtab;
 114         u64 memattr = cfg->arm_mali_lpae_cfg.memattr;
 115
 116         mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), transtab & 0xffffffffUL);
 117         mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), transtab >> 32);
 118
 119         /* Need to revisit mem attrs.
 120          * NC is the default, Mali driver is inner WT.
 121          */
 122         mmu_write(pfdev, AS_MEMATTR_LO(as_nr), memattr & 0xffffffffUL);
 123         mmu_write(pfdev, AS_MEMATTR_HI(as_nr), memattr >> 32);
 124
 125         write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
 126 }
 127
 128 static void mmu_disable(struct panfrost_device *pfdev, u32 as_nr)
 129 {
 130         mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), 0);
 131         mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), 0);
 132
 133         mmu_write(pfdev, AS_MEMATTR_LO(as_nr), 0);
 134         mmu_write(pfdev, AS_MEMATTR_HI(as_nr), 0);
 135
 136         write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
 137 }
 138
 139 void panfrost_mmu_reset(struct panfrost_device *pfdev)
 140 {
 141         panfrost_mmu_enable(pfdev, 0);
 142
 143         mmu_write(pfdev, MMU_INT_CLEAR, ~0);
 144         mmu_write(pfdev, MMU_INT_MASK, ~0);
 145 }
 146
 147 static size_t get_pgsize(u64 addr, size_t size)
 148 {
 149         if (addr & (SZ_2M - 1) || size < SZ_2M)
 150                 return SZ_4K;
 151
 152         return SZ_2M;
 153 }
 154
 155 static int mmu_map_sg(struct panfrost_device *pfdev, u64 iova,
 156                       int prot, struct sg_table *sgt)
 157 {
 158         unsigned int count;
 159         struct scatterlist *sgl;
 160         struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops;
 161         u64 start_iova = iova;
 162
 163         mutex_lock(&pfdev->mmu->lock);
 164
 165         for_each_sg(sgt->sgl, sgl, sgt->nents, count) {
 166                 unsigned long paddr = sg_dma_address(sgl);
 167                 size_t len = sg_dma_len(sgl);
 168
 169                 dev_dbg(pfdev->dev, "map: iova=%llx, paddr=%lx, len=%zx", iova, paddr, len);
 170
 171                 while (len) {
 172                         size_t pgsize = get_pgsize(iova | paddr, len);
 173
 174                         ops->map(ops, iova, paddr, pgsize, prot);
 175                         iova += pgsize;
 176                         paddr += pgsize;
 177                         len -= pgsize;
 178                 }
 179         }
 180
 181         mmu_hw_do_operation(pfdev, 0, start_iova, iova - start_iova,
 182                             AS_COMMAND_FLUSH_PT);
 183
 184         mutex_unlock(&pfdev->mmu->lock);
 185
 186         return 0;
 187 }
 188
 189 int panfrost_mmu_map(struct panfrost_gem_object *bo)
 190 {
 191         struct drm_gem_object *obj = &bo->base.base;
 192         struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
 193         struct sg_table *sgt;
 194         int ret;
 195         int prot = IOMMU_READ | IOMMU_WRITE;
 196
 197         if (WARN_ON(bo->is_mapped))
 198                 return 0;
 199
 200         if (bo->noexec)
 201                 prot |= IOMMU_NOEXEC;
 202
 203         sgt = drm_gem_shmem_get_pages_sgt(obj);
 204         if (WARN_ON(IS_ERR(sgt)))
 205                 return PTR_ERR(sgt);
 206
 207         ret = pm_runtime_get_sync(pfdev->dev);
 208         if (ret < 0)
 209                 return ret;
 210
 211         mmu_map_sg(pfdev, bo->node.start << PAGE_SHIFT, prot, sgt);
 212
 213         pm_runtime_mark_last_busy(pfdev->dev);
 214         pm_runtime_put_autosuspend(pfdev->dev);
 215         bo->is_mapped = true;
 216
 217         return 0;
 218 }
 219
 220 void panfrost_mmu_unmap(struct panfrost_gem_object *bo)
 221 {
 222         struct drm_gem_object *obj = &bo->base.base;
 223         struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
 224         struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops;
 225         u64 iova = bo->node.start << PAGE_SHIFT;
 226         size_t len = bo->node.size << PAGE_SHIFT;
 227         size_t unmapped_len = 0;
 228         int ret;
 229
 230         if (WARN_ON(!bo->is_mapped))
 231                 return;
 232
 233         dev_dbg(pfdev->dev, "unmap: iova=%llx, len=%zx", iova, len);
 234
 235         ret = pm_runtime_get_sync(pfdev->dev);
 236         if (ret < 0)
 237                 return;
 238
 239         mutex_lock(&pfdev->mmu->lock);
 240
 241         while (unmapped_len < len) {
 242                 size_t unmapped_page;
 243                 size_t pgsize = get_pgsize(iova, len - unmapped_len);
 244
 245                 if (ops->iova_to_phys(ops, iova)) {
 246                         unmapped_page = ops->unmap(ops, iova, pgsize);
 247                         WARN_ON(unmapped_page != pgsize);
 248                 }
 249                 iova += pgsize;
 250                 unmapped_len += pgsize;
 251         }
 252
 253         mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT,
 254                             bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT);
 255
 256         mutex_unlock(&pfdev->mmu->lock);
 257
 258         pm_runtime_mark_last_busy(pfdev->dev);
 259         pm_runtime_put_autosuspend(pfdev->dev);
 260         bo->is_mapped = false;
 261 }
 262
 263 static void mmu_tlb_inv_context_s1(void *cookie)
 264 {
 265         struct panfrost_device *pfdev = cookie;
 266
 267         mmu_hw_do_operation(pfdev, 0, 0, ~0UL, AS_COMMAND_FLUSH_MEM);
 268 }
 269
 270 static void mmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 271                                      size_t granule, bool leaf, void *cookie)
 272 {}
 273
 274 static void mmu_tlb_sync_context(void *cookie)
 275 {
 276         //struct panfrost_device *pfdev = cookie;
 277         // TODO: Wait 1000 GPU cycles for HW_ISSUE_6367/T60X
 278 }
 279
 280 static const struct iommu_gather_ops mmu_tlb_ops = {
 281         .tlb_flush_all  = mmu_tlb_inv_context_s1,
 282         .tlb_add_flush  = mmu_tlb_inv_range_nosync,
 283         .tlb_sync       = mmu_tlb_sync_context,
 284 };
 285
 286 static struct drm_mm_node *addr_to_drm_mm_node(struct panfrost_device *pfdev, int as, u64 addr)
 287 {
 288         struct drm_mm_node *node;
 289         u64 offset = addr >> PAGE_SHIFT;
 290
 291         drm_mm_for_each_node(node, &pfdev->mm) {
 292                 if (offset >= node->start && offset < (node->start + node->size))
 293                         return node;
 294         }
 295         return NULL;
 296 }
 297
 298 #define NUM_FAULT_PAGES (SZ_2M / PAGE_SIZE)
 299
 300 int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, u64 addr)
 301 {
 302         int ret, i;
 303         struct drm_mm_node *node;
 304         struct panfrost_gem_object *bo;
 305         struct address_space *mapping;
 306         pgoff_t page_offset;
 307         struct sg_table *sgt;
 308         struct page **pages;
 309
 310         node = addr_to_drm_mm_node(pfdev, as, addr);
 311         if (!node)
 312                 return -ENOENT;
 313
 314         bo = drm_mm_node_to_panfrost_bo(node);
 315         if (!bo->is_heap) {
 316                 dev_WARN(pfdev->dev, "matching BO is not heap type (GPU VA = %llx)",
 317                          node->start << PAGE_SHIFT);
 318                 return -EINVAL;
 319         }
 320         /* Assume 2MB alignment and size multiple */
 321         addr &= ~((u64)SZ_2M - 1);
 322         page_offset = addr >> PAGE_SHIFT;
 323         page_offset -= node->start;
 324
 325         mutex_lock(&bo->base.pages_lock);
 326
 327         if (!bo->base.pages) {
 328                 bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M,
 329                                      sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO);
 330                 if (!bo->sgts)
 331                         return -ENOMEM;
 332
 333                 pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT,
 334                                        sizeof(struct page *), GFP_KERNEL | __GFP_ZERO);
 335                 if (!pages) {
 336                         kfree(bo->sgts);
 337                         bo->sgts = NULL;
 338                         return -ENOMEM;
 339                 }
 340                 bo->base.pages = pages;
 341                 bo->base.pages_use_count = 1;
 342         } else
 343                 pages = bo->base.pages;
 344
 345         mapping = bo->base.base.filp->f_mapping;
 346         mapping_set_unevictable(mapping);
 347
 348         for (i = page_offset; i < page_offset + NUM_FAULT_PAGES; i++) {
 349                 pages[i] = shmem_read_mapping_page(mapping, i);
 350                 if (IS_ERR(pages[i])) {
 351                         mutex_unlock(&bo->base.pages_lock);
 352                         ret = PTR_ERR(pages[i]);
 353                         goto err_pages;
 354                 }
 355         }
 356
 357         mutex_unlock(&bo->base.pages_lock);
 358
 359         sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)];
 360         ret = sg_alloc_table_from_pages(sgt, pages + page_offset,
 361                                         NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL);
 362         if (ret)
 363                 goto err_pages;
 364
 365         if (!dma_map_sg(pfdev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL)) {
 366                 ret = -EINVAL;
 367                 goto err_map;
 368         }
 369
 370         mmu_map_sg(pfdev, addr, IOMMU_WRITE | IOMMU_READ | IOMMU_NOEXEC, sgt);
 371
 372         bo->is_mapped = true;
 373
 374         dev_dbg(pfdev->dev, "mapped page fault @ %llx", addr);
 375
 376         return 0;
 377
 378 err_map:
 379         sg_free_table(sgt);
 380 err_pages:
 381         drm_gem_shmem_put_pages(&bo->base);
 382         return ret;
 383 }
 384
 385 static const char *access_type_name(struct panfrost_device *pfdev,
 386                 u32 fault_status)
 387 {
 388         switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
 389         case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
 390                 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_AARCH64_MMU))
 391                         return "ATOMIC";
 392                 else
 393                         return "UNKNOWN";
 394         case AS_FAULTSTATUS_ACCESS_TYPE_READ:
 395                 return "READ";
 396         case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
 397                 return "WRITE";
 398         case AS_FAULTSTATUS_ACCESS_TYPE_EX:
 399                 return "EXECUTE";
 400         default:
 401                 WARN_ON(1);
 402                 return NULL;
 403         }
 404 }
 405
 406 static irqreturn_t panfrost_mmu_irq_handler(int irq, void *data)
 407 {
 408         struct panfrost_device *pfdev = data;
 409
 410         if (!mmu_read(pfdev, MMU_INT_STAT))
 411                 return IRQ_NONE;
 412
 413         mmu_write(pfdev, MMU_INT_MASK, 0);
 414         return IRQ_WAKE_THREAD;
 415 }
 416
 417 static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data)
 418 {
 419         struct panfrost_device *pfdev = data;
 420         u32 status = mmu_read(pfdev, MMU_INT_RAWSTAT);
 421         int i, ret;
 422
 423         for (i = 0; status; i++) {
 424                 u32 mask = BIT(i) | BIT(i + 16);
 425                 u64 addr;
 426                 u32 fault_status;
 427                 u32 exception_type;
 428                 u32 access_type;
 429                 u32 source_id;
 430
 431                 if (!(status & mask))
 432                         continue;
 433
 434                 fault_status = mmu_read(pfdev, AS_FAULTSTATUS(i));
 435                 addr = mmu_read(pfdev, AS_FAULTADDRESS_LO(i));
 436                 addr |= (u64)mmu_read(pfdev, AS_FAULTADDRESS_HI(i)) << 32;
 437
 438                 /* decode the fault status */
 439                 exception_type = fault_status & 0xFF;
 440                 access_type = (fault_status >> 8) & 0x3;
 441                 source_id = (fault_status >> 16);
 442
 443                 /* Page fault only */
 444                 if ((status & mask) == BIT(i)) {
 445                         WARN_ON(exception_type < 0xC1 || exception_type > 0xC4);
 446
 447                         ret = panfrost_mmu_map_fault_addr(pfdev, i, addr);
 448                         if (!ret) {
 449                                 mmu_write(pfdev, MMU_INT_CLEAR, BIT(i));
 450                                 status &= ~mask;
 451                                 continue;
 452                         }
 453                 }
 454
 455                 /* terminal fault, print info about the fault */
 456                 dev_err(pfdev->dev,
 457                         "Unhandled Page fault in AS%d at VA 0x%016llX\n"
 458                         "Reason: %s\n"
 459                         "raw fault status: 0x%X\n"
 460                         "decoded fault status: %s\n"
 461                         "exception type 0x%X: %s\n"
 462                         "access type 0x%X: %s\n"
 463                         "source id 0x%X\n",
 464                         i, addr,
 465                         "TODO",
 466                         fault_status,
 467                         (fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
 468                         exception_type, panfrost_exception_name(pfdev, exception_type),
 469                         access_type, access_type_name(pfdev, fault_status),
 470                         source_id);
 471
 472                 mmu_write(pfdev, MMU_INT_CLEAR, mask);
 473
 474                 status &= ~mask;
 475         }
 476
 477         mmu_write(pfdev, MMU_INT_MASK, ~0);
 478         return IRQ_HANDLED;
 479 };
 480
 481 int panfrost_mmu_init(struct panfrost_device *pfdev)
 482 {
 483         struct io_pgtable_ops *pgtbl_ops;
 484         int err, irq;
 485
 486         pfdev->mmu = devm_kzalloc(pfdev->dev, sizeof(*pfdev->mmu), GFP_KERNEL);
 487         if (!pfdev->mmu)
 488                 return -ENOMEM;
 489
 490         mutex_init(&pfdev->mmu->lock);
 491
 492         irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "mmu");
 493         if (irq <= 0)
 494                 return -ENODEV;
 495
 496         err = devm_request_threaded_irq(pfdev->dev, irq, panfrost_mmu_irq_handler,
 497                                         panfrost_mmu_irq_handler_thread,
 498                                         IRQF_SHARED, "mmu", pfdev);
 499
 500         if (err) {
 501                 dev_err(pfdev->dev, "failed to request mmu irq");
 502                 return err;
 503         }
 504         pfdev->mmu->pgtbl_cfg = (struct io_pgtable_cfg) {
 505                 .pgsize_bitmap  = SZ_4K | SZ_2M,
 506                 .ias            = FIELD_GET(0xff, pfdev->features.mmu_features),
 507                 .oas            = FIELD_GET(0xff00, pfdev->features.mmu_features),
 508                 .tlb            = &mmu_tlb_ops,
 509                 .iommu_dev      = pfdev->dev,
 510         };
 511
 512         pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &pfdev->mmu->pgtbl_cfg,
 513                                          pfdev);
 514         if (!pgtbl_ops)
 515                 return -ENOMEM;
 516
 517         pfdev->mmu->pgtbl_ops = pgtbl_ops;
 518
 519         panfrost_mmu_enable(pfdev, 0);
 520
 521         return 0;
 522 }
 523
 524 void panfrost_mmu_fini(struct panfrost_device *pfdev)
 525 {
 526         mmu_write(pfdev, MMU_INT_MASK, 0);
 527         mmu_disable(pfdev, 0);
 528
 529         free_io_pgtable_ops(pfdev->mmu->pgtbl_ops);
 530 }