drivers/gpu/drm/nouveau/nouveau_dmem.c

   1 /*
   2  * Copyright 2018 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22 #include "nouveau_dmem.h"
  23 #include "nouveau_drv.h"
  24 #include "nouveau_chan.h"
  25 #include "nouveau_dma.h"
  26 #include "nouveau_mem.h"
  27 #include "nouveau_bo.h"
  28
  29 #include <nvif/class.h>
  30 #include <nvif/object.h>
  31 #include <nvif/if500b.h>
  32 #include <nvif/if900b.h>
  33
  34 #include <linux/sched/mm.h>
  35 #include <linux/hmm.h>
  36
  37 /*
  38  * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
  39  * it in vram while in use. We likely want to overhaul memory management for
  40  * nouveau to be more page like (not necessarily with system page size but a
  41  * bigger page size) at lowest level and have some shim layer on top that would
  42  * provide the same functionality as TTM.
  43  */
  44 #define DMEM_CHUNK_SIZE (2UL << 20)
  45 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
  46
  47 struct nouveau_migrate;
  48
  49 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
  50                                       u64 dst_addr, u64 src_addr);
  51
  52 struct nouveau_dmem_chunk {
  53         struct list_head list;
  54         struct nouveau_bo *bo;
  55         struct nouveau_drm *drm;
  56         unsigned long pfn_first;
  57         unsigned long callocated;
  58         unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)];
  59         struct nvif_vma vma;
  60         spinlock_t lock;
  61 };
  62
  63 struct nouveau_dmem_migrate {
  64         nouveau_migrate_copy_t copy_func;
  65         struct nouveau_channel *chan;
  66 };
  67
  68 struct nouveau_dmem {
  69         struct hmm_devmem *devmem;
  70         struct nouveau_dmem_migrate migrate;
  71         struct list_head chunk_free;
  72         struct list_head chunk_full;
  73         struct list_head chunk_empty;
  74         struct mutex mutex;
  75 };
  76
  77 struct nouveau_migrate_hmem {
  78         struct scatterlist *sg;
  79         struct nouveau_mem mem;
  80         unsigned long npages;
  81         struct nvif_vma vma;
  82 };
  83
  84 struct nouveau_dmem_fault {
  85         struct nouveau_drm *drm;
  86         struct nouveau_fence *fence;
  87         struct nouveau_migrate_hmem hmem;
  88 };
  89
  90 struct nouveau_migrate {
  91         struct vm_area_struct *vma;
  92         struct nouveau_drm *drm;
  93         struct nouveau_fence *fence;
  94         unsigned long npages;
  95         struct nouveau_migrate_hmem hmem;
  96 };
  97
  98 static void
  99 nouveau_migrate_hmem_fini(struct nouveau_drm *drm,
 100                           struct nouveau_migrate_hmem *hmem)
 101 {
 102         struct nvif_vmm *vmm = &drm->client.vmm.vmm;
 103
 104         nouveau_mem_fini(&hmem->mem);
 105         nvif_vmm_put(vmm, &hmem->vma);
 106
 107         if (hmem->sg) {
 108                 dma_unmap_sg_attrs(drm->dev->dev, hmem->sg,
 109                                    hmem->npages, DMA_BIDIRECTIONAL,
 110                                    DMA_ATTR_SKIP_CPU_SYNC);
 111                 kfree(hmem->sg);
 112                 hmem->sg = NULL;
 113         }
 114 }
 115
 116 static int
 117 nouveau_migrate_hmem_init(struct nouveau_drm *drm,
 118                           struct nouveau_migrate_hmem *hmem,
 119                           unsigned long npages,
 120                           const unsigned long *pfns)
 121 {
 122         struct nvif_vmm *vmm = &drm->client.vmm.vmm;
 123         unsigned long i;
 124         int ret;
 125
 126         hmem->sg = kzalloc(npages * sizeof(*hmem->sg), GFP_KERNEL);
 127         if (hmem->sg == NULL)
 128                 return -ENOMEM;
 129
 130         for (i = 0, hmem->npages = 0; hmem->npages < npages; ++i) {
 131                 struct page *page;
 132
 133                 if (!pfns[i] || pfns[i] == MIGRATE_PFN_ERROR)
 134                         continue;
 135
 136                 page = migrate_pfn_to_page(pfns[i]);
 137                 if (page == NULL) {
 138                         ret = -EINVAL;
 139                         goto error;
 140                 }
 141
 142                 sg_set_page(&hmem->sg[hmem->npages], page, PAGE_SIZE, 0);
 143                 hmem->npages++;
 144         }
 145         sg_mark_end(&hmem->sg[hmem->npages - 1]);
 146
 147         i = dma_map_sg_attrs(drm->dev->dev, hmem->sg, hmem->npages,
 148                              DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
 149         if (i != hmem->npages) {
 150                 ret = -ENOMEM;
 151                 goto error;
 152         }
 153
 154         ret = nouveau_mem_sgl(&hmem->mem, &drm->client,
 155                               hmem->npages, hmem->sg);
 156         if (ret)
 157                 goto error;
 158
 159         ret = nvif_vmm_get(vmm, LAZY, false, hmem->mem.mem.page,
 160                            0, hmem->mem.mem.size, &hmem->vma);
 161         if (ret)
 162                 goto error;
 163
 164         ret = nouveau_mem_map(&hmem->mem, vmm, &hmem->vma);
 165         if (ret)
 166                 goto error;
 167
 168         return 0;
 169
 170 error:
 171         nouveau_migrate_hmem_fini(drm, hmem);
 172         return ret;
 173 }
 174
 175
 176 static void
 177 nouveau_dmem_free(struct hmm_devmem *devmem, struct page *page)
 178 {
 179         struct nouveau_dmem_chunk *chunk;
 180         struct nouveau_drm *drm;
 181         unsigned long idx;
 182
 183         chunk = (void *)hmm_devmem_page_get_drvdata(page);
 184         idx = page_to_pfn(page) - chunk->pfn_first;
 185         drm = chunk->drm;
 186
 187         /*
 188          * FIXME:
 189          *
 190          * This is really a bad example, we need to overhaul nouveau memory
 191          * management to be more page focus and allow lighter locking scheme
 192          * to be use in the process.
 193          */
 194         spin_lock(&chunk->lock);
 195         clear_bit(idx, chunk->bitmap);
 196         WARN_ON(!chunk->callocated);
 197         chunk->callocated--;
 198         /*
 199          * FIXME when chunk->callocated reach 0 we should add the chunk to
 200          * a reclaim list so that it can be freed in case of memory pressure.
 201          */
 202         spin_unlock(&chunk->lock);
 203 }
 204
 205 static void
 206 nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma,
 207                                   const unsigned long *src_pfns,
 208                                   unsigned long *dst_pfns,
 209                                   unsigned long start,
 210                                   unsigned long end,
 211                                   void *private)
 212 {
 213         struct nouveau_dmem_fault *fault = private;
 214         struct nouveau_drm *drm = fault->drm;
 215         unsigned long addr, i, c, npages = 0;
 216         nouveau_migrate_copy_t copy;
 217         int ret;
 218
 219
 220         /* First allocate new memory */
 221         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 222                 struct page *dpage, *spage;
 223
 224                 dst_pfns[i] = 0;
 225                 spage = migrate_pfn_to_page(src_pfns[i]);
 226                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
 227                         continue;
 228
 229                 dpage = hmm_vma_alloc_locked_page(vma, addr);
 230                 if (!dpage) {
 231                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 232                         continue;
 233                 }
 234
 235                 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
 236                               MIGRATE_PFN_LOCKED;
 237                 npages++;
 238         }
 239
 240         /* Create scatter list FIXME: get rid of scatter list */
 241         ret = nouveau_migrate_hmem_init(drm, &fault->hmem, npages, dst_pfns);
 242         if (ret)
 243                 goto error;
 244
 245         /* Copy things over */
 246         copy = drm->dmem->migrate.copy_func;
 247         for (addr = start, i = c = 0; addr < end; addr += PAGE_SIZE, i++) {
 248                 struct nouveau_dmem_chunk *chunk;
 249                 struct page *spage, *dpage;
 250                 u64 src_addr, dst_addr;
 251
 252                 dpage = migrate_pfn_to_page(dst_pfns[i]);
 253                 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
 254                         continue;
 255
 256                 dst_addr = fault->hmem.vma.addr + (c << PAGE_SHIFT);
 257                 c++;
 258
 259                 spage = migrate_pfn_to_page(src_pfns[i]);
 260                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
 261                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 262                         __free_page(dpage);
 263                         continue;
 264                 }
 265
 266                 chunk = (void *)hmm_devmem_page_get_drvdata(spage);
 267                 src_addr = page_to_pfn(spage) - chunk->pfn_first;
 268                 src_addr = (src_addr << PAGE_SHIFT) + chunk->vma.addr;
 269
 270                 ret = copy(drm, 1, dst_addr, src_addr);
 271                 if (ret) {
 272                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 273                         __free_page(dpage);
 274                         continue;
 275                 }
 276         }
 277
 278         nouveau_fence_new(drm->dmem->migrate.chan, false, &fault->fence);
 279
 280         return;
 281
 282 error:
 283         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
 284                 struct page *page;
 285
 286                 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
 287                         continue;
 288
 289                 page = migrate_pfn_to_page(dst_pfns[i]);
 290                 dst_pfns[i] = MIGRATE_PFN_ERROR;
 291                 if (page == NULL)
 292                         continue;
 293
 294                 __free_page(page);
 295         }
 296 }
 297
 298 void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma,
 299                                          const unsigned long *src_pfns,
 300                                          const unsigned long *dst_pfns,
 301                                          unsigned long start,
 302                                          unsigned long end,
 303                                          void *private)
 304 {
 305         struct nouveau_dmem_fault *fault = private;
 306         struct nouveau_drm *drm = fault->drm;
 307
 308         if (fault->fence) {
 309                 nouveau_fence_wait(fault->fence, true, false);
 310                 nouveau_fence_unref(&fault->fence);
 311         } else {
 312                 /*
 313                  * FIXME wait for channel to be IDLE before calling finalizing
 314                  * the hmem object below (nouveau_migrate_hmem_fini()).
 315                  */
 316         }
 317         nouveau_migrate_hmem_fini(drm, &fault->hmem);
 318 }
 319
 320 static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = {
 321         .alloc_and_copy         = nouveau_dmem_fault_alloc_and_copy,
 322         .finalize_and_map       = nouveau_dmem_fault_finalize_and_map,
 323 };
 324
 325 static int
 326 nouveau_dmem_fault(struct hmm_devmem *devmem,
 327                    struct vm_area_struct *vma,
 328                    unsigned long addr,
 329                    const struct page *page,
 330                    unsigned int flags,
 331                    pmd_t *pmdp)
 332 {
 333         struct drm_device *drm_dev = dev_get_drvdata(devmem->device);
 334         unsigned long src[1] = {0}, dst[1] = {0};
 335         struct nouveau_dmem_fault fault = {0};
 336         int ret;
 337
 338
 339
 340         /*
 341          * FIXME what we really want is to find some heuristic to migrate more
 342          * than just one page on CPU fault. When such fault happens it is very
 343          * likely that more surrounding page will CPU fault too.
 344          */
 345         fault.drm = nouveau_drm(drm_dev);
 346         ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vma, addr,
 347                           addr + PAGE_SIZE, src, dst, &fault);
 348         if (ret)
 349                 return VM_FAULT_SIGBUS;
 350
 351         if (dst[0] == MIGRATE_PFN_ERROR)
 352                 return VM_FAULT_SIGBUS;
 353
 354         return 0;
 355 }
 356
 357 static const struct hmm_devmem_ops
 358 nouveau_dmem_devmem_ops = {
 359         .free = nouveau_dmem_free,
 360         .fault = nouveau_dmem_fault,
 361 };
 362
 363 static int
 364 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm)
 365 {
 366         struct nvif_vmm *vmm = &drm->client.vmm.vmm;
 367         struct nouveau_dmem_chunk *chunk;
 368         int ret;
 369
 370         if (drm->dmem == NULL)
 371                 return -EINVAL;
 372
 373         mutex_lock(&drm->dmem->mutex);
 374         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
 375                                          struct nouveau_dmem_chunk,
 376                                          list);
 377         if (chunk == NULL) {
 378                 mutex_unlock(&drm->dmem->mutex);
 379                 return -ENOMEM;
 380         }
 381
 382         list_del(&chunk->list);
 383         mutex_unlock(&drm->dmem->mutex);
 384
 385         ret = nvif_vmm_get(vmm, LAZY, false, 12, 0,
 386                            DMEM_CHUNK_SIZE, &chunk->vma);
 387         if (ret)
 388                 goto out;
 389
 390         ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0,
 391                              TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL,
 392                              &chunk->bo);
 393         if (ret)
 394                 goto out;
 395
 396         ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 397         if (ret) {
 398                 nouveau_bo_ref(NULL, &chunk->bo);
 399                 goto out;
 400         }
 401
 402         ret = nouveau_mem_map(nouveau_mem(&chunk->bo->bo.mem), vmm, &chunk->vma);
 403         if (ret) {
 404                 nouveau_bo_unpin(chunk->bo);
 405                 nouveau_bo_ref(NULL, &chunk->bo);
 406                 goto out;
 407         }
 408
 409         bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES);
 410         spin_lock_init(&chunk->lock);
 411
 412 out:
 413         mutex_lock(&drm->dmem->mutex);
 414         if (chunk->bo)
 415                 list_add(&chunk->list, &drm->dmem->chunk_empty);
 416         else
 417                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
 418         mutex_unlock(&drm->dmem->mutex);
 419
 420         return ret;
 421 }
 422
 423 static struct nouveau_dmem_chunk *
 424 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm)
 425 {
 426         struct nouveau_dmem_chunk *chunk;
 427
 428         chunk = list_first_entry_or_null(&drm->dmem->chunk_free,
 429                                          struct nouveau_dmem_chunk,
 430                                          list);
 431         if (chunk)
 432                 return chunk;
 433
 434         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
 435                                          struct nouveau_dmem_chunk,
 436                                          list);
 437         if (chunk->bo)
 438                 return chunk;
 439
 440         return NULL;
 441 }
 442
 443 static int
 444 nouveau_dmem_pages_alloc(struct nouveau_drm *drm,
 445                          unsigned long npages,
 446                          unsigned long *pages)
 447 {
 448         struct nouveau_dmem_chunk *chunk;
 449         unsigned long c;
 450         int ret;
 451
 452         memset(pages, 0xff, npages * sizeof(*pages));
 453
 454         mutex_lock(&drm->dmem->mutex);
 455         for (c = 0; c < npages;) {
 456                 unsigned long i;
 457
 458                 chunk = nouveau_dmem_chunk_first_free_locked(drm);
 459                 if (chunk == NULL) {
 460                         mutex_unlock(&drm->dmem->mutex);
 461                         ret = nouveau_dmem_chunk_alloc(drm);
 462                         if (ret) {
 463                                 if (c)
 464                                         break;
 465                                 return ret;
 466                         }
 467                         continue;
 468                 }
 469
 470                 spin_lock(&chunk->lock);
 471                 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES);
 472                 while (i < DMEM_CHUNK_NPAGES && c < npages) {
 473                         pages[c] = chunk->pfn_first + i;
 474                         set_bit(i, chunk->bitmap);
 475                         chunk->callocated++;
 476                         c++;
 477
 478                         i = find_next_zero_bit(chunk->bitmap,
 479                                         DMEM_CHUNK_NPAGES, i);
 480                 }
 481                 spin_unlock(&chunk->lock);
 482         }
 483         mutex_unlock(&drm->dmem->mutex);
 484
 485         return 0;
 486 }
 487
 488 static struct page *
 489 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
 490 {
 491         unsigned long pfns[1];
 492         struct page *page;
 493         int ret;
 494
 495         /* FIXME stop all the miss-match API ... */
 496         ret = nouveau_dmem_pages_alloc(drm, 1, pfns);
 497         if (ret)
 498                 return NULL;
 499
 500         page = pfn_to_page(pfns[0]);
 501         get_page(page);
 502         lock_page(page);
 503         return page;
 504 }
 505
 506 static void
 507 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
 508 {
 509         unlock_page(page);
 510         put_page(page);
 511 }
 512
 513 void
 514 nouveau_dmem_resume(struct nouveau_drm *drm)
 515 {
 516         struct nouveau_dmem_chunk *chunk;
 517         int ret;
 518
 519         if (drm->dmem == NULL)
 520                 return;
 521
 522         mutex_lock(&drm->dmem->mutex);
 523         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
 524                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 525                 /* FIXME handle pin failure */
 526                 WARN_ON(ret);
 527         }
 528         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
 529                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 530                 /* FIXME handle pin failure */
 531                 WARN_ON(ret);
 532         }
 533         list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
 534                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 535                 /* FIXME handle pin failure */
 536                 WARN_ON(ret);
 537         }
 538         mutex_unlock(&drm->dmem->mutex);
 539 }
 540
 541 void
 542 nouveau_dmem_suspend(struct nouveau_drm *drm)
 543 {
 544         struct nouveau_dmem_chunk *chunk;
 545
 546         if (drm->dmem == NULL)
 547                 return;
 548
 549         mutex_lock(&drm->dmem->mutex);
 550         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
 551                 nouveau_bo_unpin(chunk->bo);
 552         }
 553         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
 554                 nouveau_bo_unpin(chunk->bo);
 555         }
 556         list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
 557                 nouveau_bo_unpin(chunk->bo);
 558         }
 559         mutex_unlock(&drm->dmem->mutex);
 560 }
 561
 562 void
 563 nouveau_dmem_fini(struct nouveau_drm *drm)
 564 {
 565         struct nvif_vmm *vmm = &drm->client.vmm.vmm;
 566         struct nouveau_dmem_chunk *chunk, *tmp;
 567
 568         if (drm->dmem == NULL)
 569                 return;
 570
 571         mutex_lock(&drm->dmem->mutex);
 572
 573         WARN_ON(!list_empty(&drm->dmem->chunk_free));
 574         WARN_ON(!list_empty(&drm->dmem->chunk_full));
 575
 576         list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) {
 577                 if (chunk->bo) {
 578                         nouveau_bo_unpin(chunk->bo);
 579                         nouveau_bo_ref(NULL, &chunk->bo);
 580                 }
 581                 nvif_vmm_put(vmm, &chunk->vma);
 582                 list_del(&chunk->list);
 583                 kfree(chunk);
 584         }
 585
 586         mutex_unlock(&drm->dmem->mutex);
 587 }
 588
 589 static int
 590 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
 591                     u64 dst_addr, u64 src_addr)
 592 {
 593         struct nouveau_channel *chan = drm->dmem->migrate.chan;
 594         int ret;
 595
 596         ret = RING_SPACE(chan, 10);
 597         if (ret)
 598                 return ret;
 599
 600         BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
 601         OUT_RING  (chan, upper_32_bits(src_addr));
 602         OUT_RING  (chan, lower_32_bits(src_addr));
 603         OUT_RING  (chan, upper_32_bits(dst_addr));
 604         OUT_RING  (chan, lower_32_bits(dst_addr));
 605         OUT_RING  (chan, PAGE_SIZE);
 606         OUT_RING  (chan, PAGE_SIZE);
 607         OUT_RING  (chan, PAGE_SIZE);
 608         OUT_RING  (chan, npages);
 609         BEGIN_IMC0(chan, NvSubCopy, 0x0300, 0x0386);
 610         return 0;
 611 }
 612
 613 static int
 614 nouveau_dmem_migrate_init(struct nouveau_drm *drm)
 615 {
 616         switch (drm->ttm.copy.oclass) {
 617         case PASCAL_DMA_COPY_A:
 618         case PASCAL_DMA_COPY_B:
 619         case  VOLTA_DMA_COPY_A:
 620         case TURING_DMA_COPY_A:
 621                 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
 622                 drm->dmem->migrate.chan = drm->ttm.chan;
 623                 return 0;
 624         default:
 625                 break;
 626         }
 627         return -ENODEV;
 628 }
 629
 630 void
 631 nouveau_dmem_init(struct nouveau_drm *drm)
 632 {
 633         struct device *device = drm->dev->dev;
 634         unsigned long i, size;
 635         int ret;
 636
 637         /* This only make sense on PASCAL or newer */
 638         if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
 639                 return;
 640
 641         if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
 642                 return;
 643
 644         mutex_init(&drm->dmem->mutex);
 645         INIT_LIST_HEAD(&drm->dmem->chunk_free);
 646         INIT_LIST_HEAD(&drm->dmem->chunk_full);
 647         INIT_LIST_HEAD(&drm->dmem->chunk_empty);
 648
 649         size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE);
 650
 651         /* Initialize migration dma helpers before registering memory */
 652         ret = nouveau_dmem_migrate_init(drm);
 653         if (ret) {
 654                 kfree(drm->dmem);
 655                 drm->dmem = NULL;
 656                 return;
 657         }
 658
 659         /*
 660          * FIXME we need some kind of policy to decide how much VRAM we
 661          * want to register with HMM. For now just register everything
 662          * and latter if we want to do thing like over commit then we
 663          * could revisit this.
 664          */
 665         drm->dmem->devmem = hmm_devmem_add(&nouveau_dmem_devmem_ops,
 666                                            device, size);
 667         if (drm->dmem->devmem == NULL) {
 668                 kfree(drm->dmem);
 669                 drm->dmem = NULL;
 670                 return;
 671         }
 672
 673         for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
 674                 struct nouveau_dmem_chunk *chunk;
 675                 struct page *page;
 676                 unsigned long j;
 677
 678                 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
 679                 if (chunk == NULL) {
 680                         nouveau_dmem_fini(drm);
 681                         return;
 682                 }
 683
 684                 chunk->drm = drm;
 685                 chunk->pfn_first = drm->dmem->devmem->pfn_first;
 686                 chunk->pfn_first += (i * DMEM_CHUNK_NPAGES);
 687                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
 688
 689                 page = pfn_to_page(chunk->pfn_first);
 690                 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) {
 691                         hmm_devmem_page_set_drvdata(page, (long)chunk);
 692                 }
 693         }
 694
 695         NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
 696 }
 697
 698 static void
 699 nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma,
 700                                     const unsigned long *src_pfns,
 701                                     unsigned long *dst_pfns,
 702                                     unsigned long start,
 703                                     unsigned long end,
 704                                     void *private)
 705 {
 706         struct nouveau_migrate *migrate = private;
 707         struct nouveau_drm *drm = migrate->drm;
 708         unsigned long addr, i, c, npages = 0;
 709         nouveau_migrate_copy_t copy;
 710         int ret;
 711
 712         /* First allocate new memory */
 713         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 714                 struct page *dpage, *spage;
 715
 716                 dst_pfns[i] = 0;
 717                 spage = migrate_pfn_to_page(src_pfns[i]);
 718                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
 719                         continue;
 720
 721                 dpage = nouveau_dmem_page_alloc_locked(drm);
 722                 if (!dpage)
 723                         continue;
 724
 725                 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
 726                               MIGRATE_PFN_LOCKED |
 727                               MIGRATE_PFN_DEVICE;
 728                 npages++;
 729         }
 730
 731         if (!npages)
 732                 return;
 733
 734         /* Create scatter list FIXME: get rid of scatter list */
 735         ret = nouveau_migrate_hmem_init(drm, &migrate->hmem, npages, src_pfns);
 736         if (ret)
 737                 goto error;
 738
 739         /* Copy things over */
 740         copy = drm->dmem->migrate.copy_func;
 741         for (addr = start, i = c = 0; addr < end; addr += PAGE_SIZE, i++) {
 742                 struct nouveau_dmem_chunk *chunk;
 743                 struct page *spage, *dpage;
 744                 u64 src_addr, dst_addr;
 745
 746                 dpage = migrate_pfn_to_page(dst_pfns[i]);
 747                 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
 748                         continue;
 749
 750                 chunk = (void *)hmm_devmem_page_get_drvdata(dpage);
 751                 dst_addr = page_to_pfn(dpage) - chunk->pfn_first;
 752                 dst_addr = (dst_addr << PAGE_SHIFT) + chunk->vma.addr;
 753
 754                 spage = migrate_pfn_to_page(src_pfns[i]);
 755                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
 756                         nouveau_dmem_page_free_locked(drm, dpage);
 757                         dst_pfns[i] = 0;
 758                         continue;
 759                 }
 760
 761                 src_addr = migrate->hmem.vma.addr + (c << PAGE_SHIFT);
 762                 c++;
 763
 764                 ret = copy(drm, 1, dst_addr, src_addr);
 765                 if (ret) {
 766                         nouveau_dmem_page_free_locked(drm, dpage);
 767                         dst_pfns[i] = 0;
 768                         continue;
 769                 }
 770         }
 771
 772         nouveau_fence_new(drm->dmem->migrate.chan, false, &migrate->fence);
 773
 774         return;
 775
 776 error:
 777         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
 778                 struct page *page;
 779
 780                 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
 781                         continue;
 782
 783                 page = migrate_pfn_to_page(dst_pfns[i]);
 784                 dst_pfns[i] = MIGRATE_PFN_ERROR;
 785                 if (page == NULL)
 786                         continue;
 787
 788                 __free_page(page);
 789         }
 790 }
 791
 792 void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma,
 793                                            const unsigned long *src_pfns,
 794                                            const unsigned long *dst_pfns,
 795                                            unsigned long start,
 796                                            unsigned long end,
 797                                            void *private)
 798 {
 799         struct nouveau_migrate *migrate = private;
 800         struct nouveau_drm *drm = migrate->drm;
 801
 802         if (migrate->fence) {
 803                 nouveau_fence_wait(migrate->fence, true, false);
 804                 nouveau_fence_unref(&migrate->fence);
 805         } else {
 806                 /*
 807                  * FIXME wait for channel to be IDLE before finalizing
 808                  * the hmem object below (nouveau_migrate_hmem_fini()) ?
 809                  */
 810         }
 811         nouveau_migrate_hmem_fini(drm, &migrate->hmem);
 812
 813         /*
 814          * FIXME optimization: update GPU page table to point to newly
 815          * migrated memory.
 816          */
 817 }
 818
 819 static const struct migrate_vma_ops nouveau_dmem_migrate_ops = {
 820         .alloc_and_copy         = nouveau_dmem_migrate_alloc_and_copy,
 821         .finalize_and_map       = nouveau_dmem_migrate_finalize_and_map,
 822 };
 823
 824 int
 825 nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
 826                          struct vm_area_struct *vma,
 827                          unsigned long start,
 828                          unsigned long end)
 829 {
 830         unsigned long *src_pfns, *dst_pfns, npages;
 831         struct nouveau_migrate migrate = {0};
 832         unsigned long i, c, max;
 833         int ret = 0;
 834
 835         npages = (end - start) >> PAGE_SHIFT;
 836         max = min(SG_MAX_SINGLE_ALLOC, npages);
 837         src_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
 838         if (src_pfns == NULL)
 839                 return -ENOMEM;
 840         dst_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
 841         if (dst_pfns == NULL) {
 842                 kfree(src_pfns);
 843                 return -ENOMEM;
 844         }
 845
 846         migrate.drm = drm;
 847         migrate.vma = vma;
 848         migrate.npages = npages;
 849         for (i = 0; i < npages; i += c) {
 850                 unsigned long next;
 851
 852                 c = min(SG_MAX_SINGLE_ALLOC, npages);
 853                 next = start + (c << PAGE_SHIFT);
 854                 ret = migrate_vma(&nouveau_dmem_migrate_ops, vma, start,
 855                                   next, src_pfns, dst_pfns, &migrate);
 856                 if (ret)
 857                         goto out;
 858                 start = next;
 859         }
 860
 861 out:
 862         kfree(dst_pfns);
 863         kfree(src_pfns);
 864         return ret;
 865 }
 866
 867 static inline bool
 868 nouveau_dmem_page(struct nouveau_drm *drm, struct page *page)
 869 {
 870         if (!is_device_private_page(page))
 871                 return false;
 872
 873         if (drm->dmem->devmem != page->pgmap->data)
 874                 return false;
 875
 876         return true;
 877 }
 878
 879 void
 880 nouveau_dmem_convert_pfn(struct nouveau_drm *drm,
 881                          struct hmm_range *range)
 882 {
 883         unsigned long i, npages;
 884
 885         npages = (range->end - range->start) >> PAGE_SHIFT;
 886         for (i = 0; i < npages; ++i) {
 887                 struct nouveau_dmem_chunk *chunk;
 888                 struct page *page;
 889                 uint64_t addr;
 890
 891                 page = hmm_pfn_to_page(range, range->pfns[i]);
 892                 if (page == NULL)
 893                         continue;
 894
 895                 if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
 896                         continue;
 897                 }
 898
 899                 if (!nouveau_dmem_page(drm, page)) {
 900                         WARN(1, "Some unknown device memory !\n");
 901                         range->pfns[i] = 0;
 902                         continue;
 903                 }
 904
 905                 chunk = (void *)hmm_devmem_page_get_drvdata(page);
 906                 addr = page_to_pfn(page) - chunk->pfn_first;
 907                 addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT;
 908
 909                 range->pfns[i] &= ((1UL << range->pfn_shift) - 1);
 910                 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift;
 911         }
 912 }