drivers/gpu/drm/nouveau/nouveau_dmem.c

   1 /*
   2  * Copyright 2018 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22 #include "nouveau_dmem.h"
  23 #include "nouveau_drv.h"
  24 #include "nouveau_chan.h"
  25 #include "nouveau_dma.h"
  26 #include "nouveau_mem.h"
  27 #include "nouveau_bo.h"
  28
  29 #include <nvif/class.h>
  30 #include <nvif/object.h>
  31 #include <nvif/if500b.h>
  32 #include <nvif/if900b.h>
  33
  34 #include <linux/sched/mm.h>
  35 #include <linux/hmm.h>
  36
  37 /*
  38  * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
  39  * it in vram while in use. We likely want to overhaul memory management for
  40  * nouveau to be more page like (not necessarily with system page size but a
  41  * bigger page size) at lowest level and have some shim layer on top that would
  42  * provide the same functionality as TTM.
  43  */
  44 #define DMEM_CHUNK_SIZE (2UL << 20)
  45 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
  46
  47 struct nouveau_migrate;
  48
  49 enum nouveau_aper {
  50         NOUVEAU_APER_VIRT,
  51         NOUVEAU_APER_VRAM,
  52         NOUVEAU_APER_HOST,
  53 };
  54
  55 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
  56                                       enum nouveau_aper, u64 dst_addr,
  57                                       enum nouveau_aper, u64 src_addr);
  58
  59 struct nouveau_dmem_chunk {
  60         struct list_head list;
  61         struct nouveau_bo *bo;
  62         struct nouveau_drm *drm;
  63         unsigned long pfn_first;
  64         unsigned long callocated;
  65         unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)];
  66         struct nvif_vma vma;
  67         spinlock_t lock;
  68 };
  69
  70 struct nouveau_dmem_migrate {
  71         nouveau_migrate_copy_t copy_func;
  72         struct nouveau_channel *chan;
  73 };
  74
  75 struct nouveau_dmem {
  76         struct hmm_devmem *devmem;
  77         struct nouveau_dmem_migrate migrate;
  78         struct list_head chunk_free;
  79         struct list_head chunk_full;
  80         struct list_head chunk_empty;
  81         struct mutex mutex;
  82 };
  83
  84 struct nouveau_migrate_hmem {
  85         struct scatterlist *sg;
  86         struct nouveau_mem mem;
  87         unsigned long npages;
  88         struct nvif_vma vma;
  89 };
  90
  91 struct nouveau_dmem_fault {
  92         struct nouveau_drm *drm;
  93         struct nouveau_fence *fence;
  94         struct nouveau_migrate_hmem hmem;
  95 };
  96
  97 struct nouveau_migrate {
  98         struct vm_area_struct *vma;
  99         struct nouveau_drm *drm;
 100         struct nouveau_fence *fence;
 101         unsigned long npages;
 102         struct nouveau_migrate_hmem hmem;
 103 };
 104
 105 static void
 106 nouveau_migrate_hmem_fini(struct nouveau_drm *drm,
 107                           struct nouveau_migrate_hmem *hmem)
 108 {
 109         struct nvif_vmm *vmm = &drm->client.vmm.vmm;
 110
 111         nouveau_mem_fini(&hmem->mem);
 112         nvif_vmm_put(vmm, &hmem->vma);
 113
 114         if (hmem->sg) {
 115                 dma_unmap_sg_attrs(drm->dev->dev, hmem->sg,
 116                                    hmem->npages, DMA_BIDIRECTIONAL,
 117                                    DMA_ATTR_SKIP_CPU_SYNC);
 118                 kfree(hmem->sg);
 119                 hmem->sg = NULL;
 120         }
 121 }
 122
 123 static int
 124 nouveau_migrate_hmem_init(struct nouveau_drm *drm,
 125                           struct nouveau_migrate_hmem *hmem,
 126                           unsigned long npages,
 127                           const unsigned long *pfns)
 128 {
 129         struct nvif_vmm *vmm = &drm->client.vmm.vmm;
 130         unsigned long i;
 131         int ret;
 132
 133         hmem->sg = kzalloc(npages * sizeof(*hmem->sg), GFP_KERNEL);
 134         if (hmem->sg == NULL)
 135                 return -ENOMEM;
 136
 137         for (i = 0, hmem->npages = 0; hmem->npages < npages; ++i) {
 138                 struct page *page;
 139
 140                 if (!pfns[i] || pfns[i] == MIGRATE_PFN_ERROR)
 141                         continue;
 142
 143                 page = migrate_pfn_to_page(pfns[i]);
 144                 if (page == NULL) {
 145                         ret = -EINVAL;
 146                         goto error;
 147                 }
 148
 149                 sg_set_page(&hmem->sg[hmem->npages], page, PAGE_SIZE, 0);
 150                 hmem->npages++;
 151         }
 152         sg_mark_end(&hmem->sg[hmem->npages - 1]);
 153
 154         i = dma_map_sg_attrs(drm->dev->dev, hmem->sg, hmem->npages,
 155                              DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
 156         if (i != hmem->npages) {
 157                 ret = -ENOMEM;
 158                 goto error;
 159         }
 160
 161         ret = nouveau_mem_sgl(&hmem->mem, &drm->client,
 162                               hmem->npages, hmem->sg);
 163         if (ret)
 164                 goto error;
 165
 166         ret = nvif_vmm_get(vmm, LAZY, false, hmem->mem.mem.page,
 167                            0, hmem->mem.mem.size, &hmem->vma);
 168         if (ret)
 169                 goto error;
 170
 171         ret = nouveau_mem_map(&hmem->mem, vmm, &hmem->vma);
 172         if (ret)
 173                 goto error;
 174
 175         return 0;
 176
 177 error:
 178         nouveau_migrate_hmem_fini(drm, hmem);
 179         return ret;
 180 }
 181
 182
 183 static void
 184 nouveau_dmem_free(struct hmm_devmem *devmem, struct page *page)
 185 {
 186         struct nouveau_dmem_chunk *chunk;
 187         struct nouveau_drm *drm;
 188         unsigned long idx;
 189
 190         chunk = (void *)hmm_devmem_page_get_drvdata(page);
 191         idx = page_to_pfn(page) - chunk->pfn_first;
 192         drm = chunk->drm;
 193
 194         /*
 195          * FIXME:
 196          *
 197          * This is really a bad example, we need to overhaul nouveau memory
 198          * management to be more page focus and allow lighter locking scheme
 199          * to be use in the process.
 200          */
 201         spin_lock(&chunk->lock);
 202         clear_bit(idx, chunk->bitmap);
 203         WARN_ON(!chunk->callocated);
 204         chunk->callocated--;
 205         /*
 206          * FIXME when chunk->callocated reach 0 we should add the chunk to
 207          * a reclaim list so that it can be freed in case of memory pressure.
 208          */
 209         spin_unlock(&chunk->lock);
 210 }
 211
 212 static void
 213 nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma,
 214                                   const unsigned long *src_pfns,
 215                                   unsigned long *dst_pfns,
 216                                   unsigned long start,
 217                                   unsigned long end,
 218                                   void *private)
 219 {
 220         struct nouveau_dmem_fault *fault = private;
 221         struct nouveau_drm *drm = fault->drm;
 222         unsigned long addr, i, c, npages = 0;
 223         nouveau_migrate_copy_t copy;
 224         int ret;
 225
 226
 227         /* First allocate new memory */
 228         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 229                 struct page *dpage, *spage;
 230
 231                 dst_pfns[i] = 0;
 232                 spage = migrate_pfn_to_page(src_pfns[i]);
 233                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
 234                         continue;
 235
 236                 dpage = hmm_vma_alloc_locked_page(vma, addr);
 237                 if (!dpage) {
 238                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 239                         continue;
 240                 }
 241
 242                 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
 243                               MIGRATE_PFN_LOCKED;
 244                 npages++;
 245         }
 246
 247         /* Create scatter list FIXME: get rid of scatter list */
 248         ret = nouveau_migrate_hmem_init(drm, &fault->hmem, npages, dst_pfns);
 249         if (ret)
 250                 goto error;
 251
 252         /* Copy things over */
 253         copy = drm->dmem->migrate.copy_func;
 254         for (addr = start, i = c = 0; addr < end; addr += PAGE_SIZE, i++) {
 255                 struct nouveau_dmem_chunk *chunk;
 256                 struct page *spage, *dpage;
 257                 u64 src_addr, dst_addr;
 258
 259                 dpage = migrate_pfn_to_page(dst_pfns[i]);
 260                 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
 261                         continue;
 262
 263                 dst_addr = fault->hmem.vma.addr + (c << PAGE_SHIFT);
 264                 c++;
 265
 266                 spage = migrate_pfn_to_page(src_pfns[i]);
 267                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
 268                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 269                         __free_page(dpage);
 270                         continue;
 271                 }
 272
 273                 chunk = (void *)hmm_devmem_page_get_drvdata(spage);
 274                 src_addr = page_to_pfn(spage) - chunk->pfn_first;
 275                 src_addr = (src_addr << PAGE_SHIFT) + chunk->vma.addr;
 276
 277                 ret = copy(drm, 1, NOUVEAU_APER_VIRT, dst_addr,
 278                                    NOUVEAU_APER_VIRT, src_addr);
 279                 if (ret) {
 280                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 281                         __free_page(dpage);
 282                         continue;
 283                 }
 284         }
 285
 286         nouveau_fence_new(drm->dmem->migrate.chan, false, &fault->fence);
 287
 288         return;
 289
 290 error:
 291         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
 292                 struct page *page;
 293
 294                 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
 295                         continue;
 296
 297                 page = migrate_pfn_to_page(dst_pfns[i]);
 298                 dst_pfns[i] = MIGRATE_PFN_ERROR;
 299                 if (page == NULL)
 300                         continue;
 301
 302                 __free_page(page);
 303         }
 304 }
 305
 306 void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma,
 307                                          const unsigned long *src_pfns,
 308                                          const unsigned long *dst_pfns,
 309                                          unsigned long start,
 310                                          unsigned long end,
 311                                          void *private)
 312 {
 313         struct nouveau_dmem_fault *fault = private;
 314         struct nouveau_drm *drm = fault->drm;
 315
 316         if (fault->fence) {
 317                 nouveau_fence_wait(fault->fence, true, false);
 318                 nouveau_fence_unref(&fault->fence);
 319         } else {
 320                 /*
 321                  * FIXME wait for channel to be IDLE before calling finalizing
 322                  * the hmem object below (nouveau_migrate_hmem_fini()).
 323                  */
 324         }
 325         nouveau_migrate_hmem_fini(drm, &fault->hmem);
 326 }
 327
 328 static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = {
 329         .alloc_and_copy         = nouveau_dmem_fault_alloc_and_copy,
 330         .finalize_and_map       = nouveau_dmem_fault_finalize_and_map,
 331 };
 332
 333 static int
 334 nouveau_dmem_fault(struct hmm_devmem *devmem,
 335                    struct vm_area_struct *vma,
 336                    unsigned long addr,
 337                    const struct page *page,
 338                    unsigned int flags,
 339                    pmd_t *pmdp)
 340 {
 341         struct drm_device *drm_dev = dev_get_drvdata(devmem->device);
 342         unsigned long src[1] = {0}, dst[1] = {0};
 343         struct nouveau_dmem_fault fault = {0};
 344         int ret;
 345
 346
 347
 348         /*
 349          * FIXME what we really want is to find some heuristic to migrate more
 350          * than just one page on CPU fault. When such fault happens it is very
 351          * likely that more surrounding page will CPU fault too.
 352          */
 353         fault.drm = nouveau_drm(drm_dev);
 354         ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vma, addr,
 355                           addr + PAGE_SIZE, src, dst, &fault);
 356         if (ret)
 357                 return VM_FAULT_SIGBUS;
 358
 359         if (dst[0] == MIGRATE_PFN_ERROR)
 360                 return VM_FAULT_SIGBUS;
 361
 362         return 0;
 363 }
 364
 365 static const struct hmm_devmem_ops
 366 nouveau_dmem_devmem_ops = {
 367         .free = nouveau_dmem_free,
 368         .fault = nouveau_dmem_fault,
 369 };
 370
 371 static int
 372 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm)
 373 {
 374         struct nvif_vmm *vmm = &drm->client.vmm.vmm;
 375         struct nouveau_dmem_chunk *chunk;
 376         int ret;
 377
 378         if (drm->dmem == NULL)
 379                 return -EINVAL;
 380
 381         mutex_lock(&drm->dmem->mutex);
 382         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
 383                                          struct nouveau_dmem_chunk,
 384                                          list);
 385         if (chunk == NULL) {
 386                 mutex_unlock(&drm->dmem->mutex);
 387                 return -ENOMEM;
 388         }
 389
 390         list_del(&chunk->list);
 391         mutex_unlock(&drm->dmem->mutex);
 392
 393         ret = nvif_vmm_get(vmm, LAZY, false, 12, 0,
 394                            DMEM_CHUNK_SIZE, &chunk->vma);
 395         if (ret)
 396                 goto out;
 397
 398         ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0,
 399                              TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL,
 400                              &chunk->bo);
 401         if (ret)
 402                 goto out;
 403
 404         ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 405         if (ret) {
 406                 nouveau_bo_ref(NULL, &chunk->bo);
 407                 goto out;
 408         }
 409
 410         ret = nouveau_mem_map(nouveau_mem(&chunk->bo->bo.mem), vmm, &chunk->vma);
 411         if (ret) {
 412                 nouveau_bo_unpin(chunk->bo);
 413                 nouveau_bo_ref(NULL, &chunk->bo);
 414                 goto out;
 415         }
 416
 417         bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES);
 418         spin_lock_init(&chunk->lock);
 419
 420 out:
 421         mutex_lock(&drm->dmem->mutex);
 422         if (chunk->bo)
 423                 list_add(&chunk->list, &drm->dmem->chunk_empty);
 424         else
 425                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
 426         mutex_unlock(&drm->dmem->mutex);
 427
 428         return ret;
 429 }
 430
 431 static struct nouveau_dmem_chunk *
 432 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm)
 433 {
 434         struct nouveau_dmem_chunk *chunk;
 435
 436         chunk = list_first_entry_or_null(&drm->dmem->chunk_free,
 437                                          struct nouveau_dmem_chunk,
 438                                          list);
 439         if (chunk)
 440                 return chunk;
 441
 442         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
 443                                          struct nouveau_dmem_chunk,
 444                                          list);
 445         if (chunk->bo)
 446                 return chunk;
 447
 448         return NULL;
 449 }
 450
 451 static int
 452 nouveau_dmem_pages_alloc(struct nouveau_drm *drm,
 453                          unsigned long npages,
 454                          unsigned long *pages)
 455 {
 456         struct nouveau_dmem_chunk *chunk;
 457         unsigned long c;
 458         int ret;
 459
 460         memset(pages, 0xff, npages * sizeof(*pages));
 461
 462         mutex_lock(&drm->dmem->mutex);
 463         for (c = 0; c < npages;) {
 464                 unsigned long i;
 465
 466                 chunk = nouveau_dmem_chunk_first_free_locked(drm);
 467                 if (chunk == NULL) {
 468                         mutex_unlock(&drm->dmem->mutex);
 469                         ret = nouveau_dmem_chunk_alloc(drm);
 470                         if (ret) {
 471                                 if (c)
 472                                         break;
 473                                 return ret;
 474                         }
 475                         continue;
 476                 }
 477
 478                 spin_lock(&chunk->lock);
 479                 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES);
 480                 while (i < DMEM_CHUNK_NPAGES && c < npages) {
 481                         pages[c] = chunk->pfn_first + i;
 482                         set_bit(i, chunk->bitmap);
 483                         chunk->callocated++;
 484                         c++;
 485
 486                         i = find_next_zero_bit(chunk->bitmap,
 487                                         DMEM_CHUNK_NPAGES, i);
 488                 }
 489                 spin_unlock(&chunk->lock);
 490         }
 491         mutex_unlock(&drm->dmem->mutex);
 492
 493         return 0;
 494 }
 495
 496 static struct page *
 497 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
 498 {
 499         unsigned long pfns[1];
 500         struct page *page;
 501         int ret;
 502
 503         /* FIXME stop all the miss-match API ... */
 504         ret = nouveau_dmem_pages_alloc(drm, 1, pfns);
 505         if (ret)
 506                 return NULL;
 507
 508         page = pfn_to_page(pfns[0]);
 509         get_page(page);
 510         lock_page(page);
 511         return page;
 512 }
 513
 514 static void
 515 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
 516 {
 517         unlock_page(page);
 518         put_page(page);
 519 }
 520
 521 void
 522 nouveau_dmem_resume(struct nouveau_drm *drm)
 523 {
 524         struct nouveau_dmem_chunk *chunk;
 525         int ret;
 526
 527         if (drm->dmem == NULL)
 528                 return;
 529
 530         mutex_lock(&drm->dmem->mutex);
 531         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
 532                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 533                 /* FIXME handle pin failure */
 534                 WARN_ON(ret);
 535         }
 536         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
 537                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 538                 /* FIXME handle pin failure */
 539                 WARN_ON(ret);
 540         }
 541         list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
 542                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 543                 /* FIXME handle pin failure */
 544                 WARN_ON(ret);
 545         }
 546         mutex_unlock(&drm->dmem->mutex);
 547 }
 548
 549 void
 550 nouveau_dmem_suspend(struct nouveau_drm *drm)
 551 {
 552         struct nouveau_dmem_chunk *chunk;
 553
 554         if (drm->dmem == NULL)
 555                 return;
 556
 557         mutex_lock(&drm->dmem->mutex);
 558         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
 559                 nouveau_bo_unpin(chunk->bo);
 560         }
 561         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
 562                 nouveau_bo_unpin(chunk->bo);
 563         }
 564         list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
 565                 nouveau_bo_unpin(chunk->bo);
 566         }
 567         mutex_unlock(&drm->dmem->mutex);
 568 }
 569
 570 void
 571 nouveau_dmem_fini(struct nouveau_drm *drm)
 572 {
 573         struct nvif_vmm *vmm = &drm->client.vmm.vmm;
 574         struct nouveau_dmem_chunk *chunk, *tmp;
 575
 576         if (drm->dmem == NULL)
 577                 return;
 578
 579         mutex_lock(&drm->dmem->mutex);
 580
 581         WARN_ON(!list_empty(&drm->dmem->chunk_free));
 582         WARN_ON(!list_empty(&drm->dmem->chunk_full));
 583
 584         list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) {
 585                 if (chunk->bo) {
 586                         nouveau_bo_unpin(chunk->bo);
 587                         nouveau_bo_ref(NULL, &chunk->bo);
 588                 }
 589                 nvif_vmm_put(vmm, &chunk->vma);
 590                 list_del(&chunk->list);
 591                 kfree(chunk);
 592         }
 593
 594         mutex_unlock(&drm->dmem->mutex);
 595 }
 596
 597 static int
 598 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
 599                     enum nouveau_aper dst_aper, u64 dst_addr,
 600                     enum nouveau_aper src_aper, u64 src_addr)
 601 {
 602         struct nouveau_channel *chan = drm->dmem->migrate.chan;
 603         u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ |
 604                          (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ |
 605                          (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ |
 606                          (1 << 2) /* FLUSH_ENABLE_TRUE. */ |
 607                          (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */;
 608         int ret;
 609
 610         ret = RING_SPACE(chan, 13);
 611         if (ret)
 612                 return ret;
 613
 614         if (src_aper != NOUVEAU_APER_VIRT) {
 615                 switch (src_aper) {
 616                 case NOUVEAU_APER_VRAM:
 617                         BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0);
 618                         break;
 619                 case NOUVEAU_APER_HOST:
 620                         BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1);
 621                         break;
 622                 default:
 623                         return -EINVAL;
 624                 }
 625                 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */
 626         }
 627
 628         if (dst_aper != NOUVEAU_APER_VIRT) {
 629                 switch (dst_aper) {
 630                 case NOUVEAU_APER_VRAM:
 631                         BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0);
 632                         break;
 633                 case NOUVEAU_APER_HOST:
 634                         BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1);
 635                         break;
 636                 default:
 637                         return -EINVAL;
 638                 }
 639                 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */
 640         }
 641
 642         BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
 643         OUT_RING  (chan, upper_32_bits(src_addr));
 644         OUT_RING  (chan, lower_32_bits(src_addr));
 645         OUT_RING  (chan, upper_32_bits(dst_addr));
 646         OUT_RING  (chan, lower_32_bits(dst_addr));
 647         OUT_RING  (chan, PAGE_SIZE);
 648         OUT_RING  (chan, PAGE_SIZE);
 649         OUT_RING  (chan, PAGE_SIZE);
 650         OUT_RING  (chan, npages);
 651         BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
 652         OUT_RING  (chan, launch_dma);
 653         return 0;
 654 }
 655
 656 static int
 657 nouveau_dmem_migrate_init(struct nouveau_drm *drm)
 658 {
 659         switch (drm->ttm.copy.oclass) {
 660         case PASCAL_DMA_COPY_A:
 661         case PASCAL_DMA_COPY_B:
 662         case  VOLTA_DMA_COPY_A:
 663         case TURING_DMA_COPY_A:
 664                 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
 665                 drm->dmem->migrate.chan = drm->ttm.chan;
 666                 return 0;
 667         default:
 668                 break;
 669         }
 670         return -ENODEV;
 671 }
 672
 673 void
 674 nouveau_dmem_init(struct nouveau_drm *drm)
 675 {
 676         struct device *device = drm->dev->dev;
 677         unsigned long i, size;
 678         int ret;
 679
 680         /* This only make sense on PASCAL or newer */
 681         if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
 682                 return;
 683
 684         if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
 685                 return;
 686
 687         mutex_init(&drm->dmem->mutex);
 688         INIT_LIST_HEAD(&drm->dmem->chunk_free);
 689         INIT_LIST_HEAD(&drm->dmem->chunk_full);
 690         INIT_LIST_HEAD(&drm->dmem->chunk_empty);
 691
 692         size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE);
 693
 694         /* Initialize migration dma helpers before registering memory */
 695         ret = nouveau_dmem_migrate_init(drm);
 696         if (ret) {
 697                 kfree(drm->dmem);
 698                 drm->dmem = NULL;
 699                 return;
 700         }
 701
 702         /*
 703          * FIXME we need some kind of policy to decide how much VRAM we
 704          * want to register with HMM. For now just register everything
 705          * and latter if we want to do thing like over commit then we
 706          * could revisit this.
 707          */
 708         drm->dmem->devmem = hmm_devmem_add(&nouveau_dmem_devmem_ops,
 709                                            device, size);
 710         if (drm->dmem->devmem == NULL) {
 711                 kfree(drm->dmem);
 712                 drm->dmem = NULL;
 713                 return;
 714         }
 715
 716         for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
 717                 struct nouveau_dmem_chunk *chunk;
 718                 struct page *page;
 719                 unsigned long j;
 720
 721                 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
 722                 if (chunk == NULL) {
 723                         nouveau_dmem_fini(drm);
 724                         return;
 725                 }
 726
 727                 chunk->drm = drm;
 728                 chunk->pfn_first = drm->dmem->devmem->pfn_first;
 729                 chunk->pfn_first += (i * DMEM_CHUNK_NPAGES);
 730                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
 731
 732                 page = pfn_to_page(chunk->pfn_first);
 733                 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) {
 734                         hmm_devmem_page_set_drvdata(page, (long)chunk);
 735                 }
 736         }
 737
 738         NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
 739 }
 740
 741 static void
 742 nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma,
 743                                     const unsigned long *src_pfns,
 744                                     unsigned long *dst_pfns,
 745                                     unsigned long start,
 746                                     unsigned long end,
 747                                     void *private)
 748 {
 749         struct nouveau_migrate *migrate = private;
 750         struct nouveau_drm *drm = migrate->drm;
 751         unsigned long addr, i, c, npages = 0;
 752         nouveau_migrate_copy_t copy;
 753         int ret;
 754
 755         /* First allocate new memory */
 756         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 757                 struct page *dpage, *spage;
 758
 759                 dst_pfns[i] = 0;
 760                 spage = migrate_pfn_to_page(src_pfns[i]);
 761                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
 762                         continue;
 763
 764                 dpage = nouveau_dmem_page_alloc_locked(drm);
 765                 if (!dpage)
 766                         continue;
 767
 768                 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
 769                               MIGRATE_PFN_LOCKED |
 770                               MIGRATE_PFN_DEVICE;
 771                 npages++;
 772         }
 773
 774         if (!npages)
 775                 return;
 776
 777         /* Create scatter list FIXME: get rid of scatter list */
 778         ret = nouveau_migrate_hmem_init(drm, &migrate->hmem, npages, src_pfns);
 779         if (ret)
 780                 goto error;
 781
 782         /* Copy things over */
 783         copy = drm->dmem->migrate.copy_func;
 784         for (addr = start, i = c = 0; addr < end; addr += PAGE_SIZE, i++) {
 785                 struct nouveau_dmem_chunk *chunk;
 786                 struct page *spage, *dpage;
 787                 u64 src_addr, dst_addr;
 788
 789                 dpage = migrate_pfn_to_page(dst_pfns[i]);
 790                 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
 791                         continue;
 792
 793                 chunk = (void *)hmm_devmem_page_get_drvdata(dpage);
 794                 dst_addr = page_to_pfn(dpage) - chunk->pfn_first;
 795                 dst_addr = (dst_addr << PAGE_SHIFT) + chunk->vma.addr;
 796
 797                 spage = migrate_pfn_to_page(src_pfns[i]);
 798                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
 799                         nouveau_dmem_page_free_locked(drm, dpage);
 800                         dst_pfns[i] = 0;
 801                         continue;
 802                 }
 803
 804                 src_addr = migrate->hmem.vma.addr + (c << PAGE_SHIFT);
 805                 c++;
 806
 807                 ret = copy(drm, 1, NOUVEAU_APER_VIRT, dst_addr,
 808                                    NOUVEAU_APER_VIRT, src_addr);
 809                 if (ret) {
 810                         nouveau_dmem_page_free_locked(drm, dpage);
 811                         dst_pfns[i] = 0;
 812                         continue;
 813                 }
 814         }
 815
 816         nouveau_fence_new(drm->dmem->migrate.chan, false, &migrate->fence);
 817
 818         return;
 819
 820 error:
 821         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
 822                 struct page *page;
 823
 824                 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
 825                         continue;
 826
 827                 page = migrate_pfn_to_page(dst_pfns[i]);
 828                 dst_pfns[i] = MIGRATE_PFN_ERROR;
 829                 if (page == NULL)
 830                         continue;
 831
 832                 __free_page(page);
 833         }
 834 }
 835
 836 void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma,
 837                                            const unsigned long *src_pfns,
 838                                            const unsigned long *dst_pfns,
 839                                            unsigned long start,
 840                                            unsigned long end,
 841                                            void *private)
 842 {
 843         struct nouveau_migrate *migrate = private;
 844         struct nouveau_drm *drm = migrate->drm;
 845
 846         if (migrate->fence) {
 847                 nouveau_fence_wait(migrate->fence, true, false);
 848                 nouveau_fence_unref(&migrate->fence);
 849         } else {
 850                 /*
 851                  * FIXME wait for channel to be IDLE before finalizing
 852                  * the hmem object below (nouveau_migrate_hmem_fini()) ?
 853                  */
 854         }
 855         nouveau_migrate_hmem_fini(drm, &migrate->hmem);
 856
 857         /*
 858          * FIXME optimization: update GPU page table to point to newly
 859          * migrated memory.
 860          */
 861 }
 862
 863 static const struct migrate_vma_ops nouveau_dmem_migrate_ops = {
 864         .alloc_and_copy         = nouveau_dmem_migrate_alloc_and_copy,
 865         .finalize_and_map       = nouveau_dmem_migrate_finalize_and_map,
 866 };
 867
 868 int
 869 nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
 870                          struct vm_area_struct *vma,
 871                          unsigned long start,
 872                          unsigned long end)
 873 {
 874         unsigned long *src_pfns, *dst_pfns, npages;
 875         struct nouveau_migrate migrate = {0};
 876         unsigned long i, c, max;
 877         int ret = 0;
 878
 879         npages = (end - start) >> PAGE_SHIFT;
 880         max = min(SG_MAX_SINGLE_ALLOC, npages);
 881         src_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
 882         if (src_pfns == NULL)
 883                 return -ENOMEM;
 884         dst_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
 885         if (dst_pfns == NULL) {
 886                 kfree(src_pfns);
 887                 return -ENOMEM;
 888         }
 889
 890         migrate.drm = drm;
 891         migrate.vma = vma;
 892         migrate.npages = npages;
 893         for (i = 0; i < npages; i += c) {
 894                 unsigned long next;
 895
 896                 c = min(SG_MAX_SINGLE_ALLOC, npages);
 897                 next = start + (c << PAGE_SHIFT);
 898                 ret = migrate_vma(&nouveau_dmem_migrate_ops, vma, start,
 899                                   next, src_pfns, dst_pfns, &migrate);
 900                 if (ret)
 901                         goto out;
 902                 start = next;
 903         }
 904
 905 out:
 906         kfree(dst_pfns);
 907         kfree(src_pfns);
 908         return ret;
 909 }
 910
 911 static inline bool
 912 nouveau_dmem_page(struct nouveau_drm *drm, struct page *page)
 913 {
 914         if (!is_device_private_page(page))
 915                 return false;
 916
 917         if (drm->dmem->devmem != page->pgmap->data)
 918                 return false;
 919
 920         return true;
 921 }
 922
 923 void
 924 nouveau_dmem_convert_pfn(struct nouveau_drm *drm,
 925                          struct hmm_range *range)
 926 {
 927         unsigned long i, npages;
 928
 929         npages = (range->end - range->start) >> PAGE_SHIFT;
 930         for (i = 0; i < npages; ++i) {
 931                 struct nouveau_dmem_chunk *chunk;
 932                 struct page *page;
 933                 uint64_t addr;
 934
 935                 page = hmm_pfn_to_page(range, range->pfns[i]);
 936                 if (page == NULL)
 937                         continue;
 938
 939                 if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
 940                         continue;
 941                 }
 942
 943                 if (!nouveau_dmem_page(drm, page)) {
 944                         WARN(1, "Some unknown device memory !\n");
 945                         range->pfns[i] = 0;
 946                         continue;
 947                 }
 948
 949                 chunk = (void *)hmm_devmem_page_get_drvdata(page);
 950                 addr = page_to_pfn(page) - chunk->pfn_first;
 951                 addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT;
 952
 953                 range->pfns[i] &= ((1UL << range->pfn_shift) - 1);
 954                 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift;
 955         }
 956 }