drivers/gpu/drm/nouveau/nouveau_dmem.c

   1 /*
   2  * Copyright 2018 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22 #include "nouveau_dmem.h"
  23 #include "nouveau_drv.h"
  24 #include "nouveau_chan.h"
  25 #include "nouveau_dma.h"
  26 #include "nouveau_mem.h"
  27 #include "nouveau_bo.h"
  28
  29 #include <nvif/class.h>
  30 #include <nvif/object.h>
  31 #include <nvif/if500b.h>
  32 #include <nvif/if900b.h>
  33
  34 #include <linux/sched/mm.h>
  35 #include <linux/hmm.h>
  36
  37 /*
  38  * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
  39  * it in vram while in use. We likely want to overhaul memory management for
  40  * nouveau to be more page like (not necessarily with system page size but a
  41  * bigger page size) at lowest level and have some shim layer on top that would
  42  * provide the same functionality as TTM.
  43  */
  44 #define DMEM_CHUNK_SIZE (2UL << 20)
  45 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
  46
  47 struct nouveau_migrate;
  48
  49 enum nouveau_aper {
  50         NOUVEAU_APER_VIRT,
  51         NOUVEAU_APER_VRAM,
  52         NOUVEAU_APER_HOST,
  53 };
  54
  55 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
  56                                       enum nouveau_aper, u64 dst_addr,
  57                                       enum nouveau_aper, u64 src_addr);
  58
  59 struct nouveau_dmem_chunk {
  60         struct list_head list;
  61         struct nouveau_bo *bo;
  62         struct nouveau_drm *drm;
  63         unsigned long pfn_first;
  64         unsigned long callocated;
  65         unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)];
  66         spinlock_t lock;
  67 };
  68
  69 struct nouveau_dmem_migrate {
  70         nouveau_migrate_copy_t copy_func;
  71         struct nouveau_channel *chan;
  72 };
  73
  74 struct nouveau_dmem {
  75         struct hmm_devmem *devmem;
  76         struct nouveau_dmem_migrate migrate;
  77         struct list_head chunk_free;
  78         struct list_head chunk_full;
  79         struct list_head chunk_empty;
  80         struct mutex mutex;
  81 };
  82
  83 struct nouveau_dmem_fault {
  84         struct nouveau_drm *drm;
  85         struct nouveau_fence *fence;
  86         dma_addr_t *dma;
  87         unsigned long npages;
  88 };
  89
  90 struct nouveau_migrate {
  91         struct vm_area_struct *vma;
  92         struct nouveau_drm *drm;
  93         struct nouveau_fence *fence;
  94         unsigned long npages;
  95         dma_addr_t *dma;
  96         unsigned long dma_nr;
  97 };
  98
  99 static void
 100 nouveau_dmem_free(struct hmm_devmem *devmem, struct page *page)
 101 {
 102         struct nouveau_dmem_chunk *chunk;
 103         struct nouveau_drm *drm;
 104         unsigned long idx;
 105
 106         chunk = (void *)hmm_devmem_page_get_drvdata(page);
 107         idx = page_to_pfn(page) - chunk->pfn_first;
 108         drm = chunk->drm;
 109
 110         /*
 111          * FIXME:
 112          *
 113          * This is really a bad example, we need to overhaul nouveau memory
 114          * management to be more page focus and allow lighter locking scheme
 115          * to be use in the process.
 116          */
 117         spin_lock(&chunk->lock);
 118         clear_bit(idx, chunk->bitmap);
 119         WARN_ON(!chunk->callocated);
 120         chunk->callocated--;
 121         /*
 122          * FIXME when chunk->callocated reach 0 we should add the chunk to
 123          * a reclaim list so that it can be freed in case of memory pressure.
 124          */
 125         spin_unlock(&chunk->lock);
 126 }
 127
 128 static void
 129 nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma,
 130                                   const unsigned long *src_pfns,
 131                                   unsigned long *dst_pfns,
 132                                   unsigned long start,
 133                                   unsigned long end,
 134                                   void *private)
 135 {
 136         struct nouveau_dmem_fault *fault = private;
 137         struct nouveau_drm *drm = fault->drm;
 138         struct device *dev = drm->dev->dev;
 139         unsigned long addr, i, npages = 0;
 140         nouveau_migrate_copy_t copy;
 141         int ret;
 142
 143
 144         /* First allocate new memory */
 145         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 146                 struct page *dpage, *spage;
 147
 148                 dst_pfns[i] = 0;
 149                 spage = migrate_pfn_to_page(src_pfns[i]);
 150                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
 151                         continue;
 152
 153                 dpage = hmm_vma_alloc_locked_page(vma, addr);
 154                 if (!dpage) {
 155                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 156                         continue;
 157                 }
 158
 159                 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
 160                               MIGRATE_PFN_LOCKED;
 161                 npages++;
 162         }
 163
 164         /* Allocate storage for DMA addresses, so we can unmap later. */
 165         fault->dma = kmalloc(sizeof(*fault->dma) * npages, GFP_KERNEL);
 166         if (!fault->dma)
 167                 goto error;
 168
 169         /* Copy things over */
 170         copy = drm->dmem->migrate.copy_func;
 171         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 172                 struct nouveau_dmem_chunk *chunk;
 173                 struct page *spage, *dpage;
 174                 u64 src_addr, dst_addr;
 175
 176                 dpage = migrate_pfn_to_page(dst_pfns[i]);
 177                 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
 178                         continue;
 179
 180                 spage = migrate_pfn_to_page(src_pfns[i]);
 181                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
 182                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 183                         __free_page(dpage);
 184                         continue;
 185                 }
 186
 187                 fault->dma[fault->npages] =
 188                         dma_map_page_attrs(dev, dpage, 0, PAGE_SIZE,
 189                                            PCI_DMA_BIDIRECTIONAL,
 190                                            DMA_ATTR_SKIP_CPU_SYNC);
 191                 if (dma_mapping_error(dev, fault->dma[fault->npages])) {
 192                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 193                         __free_page(dpage);
 194                         continue;
 195                 }
 196
 197                 dst_addr = fault->dma[fault->npages++];
 198
 199                 chunk = (void *)hmm_devmem_page_get_drvdata(spage);
 200                 src_addr = page_to_pfn(spage) - chunk->pfn_first;
 201                 src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
 202
 203                 ret = copy(drm, 1, NOUVEAU_APER_HOST, dst_addr,
 204                                    NOUVEAU_APER_VRAM, src_addr);
 205                 if (ret) {
 206                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 207                         __free_page(dpage);
 208                         continue;
 209                 }
 210         }
 211
 212         nouveau_fence_new(drm->dmem->migrate.chan, false, &fault->fence);
 213
 214         return;
 215
 216 error:
 217         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
 218                 struct page *page;
 219
 220                 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
 221                         continue;
 222
 223                 page = migrate_pfn_to_page(dst_pfns[i]);
 224                 dst_pfns[i] = MIGRATE_PFN_ERROR;
 225                 if (page == NULL)
 226                         continue;
 227
 228                 __free_page(page);
 229         }
 230 }
 231
 232 void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma,
 233                                          const unsigned long *src_pfns,
 234                                          const unsigned long *dst_pfns,
 235                                          unsigned long start,
 236                                          unsigned long end,
 237                                          void *private)
 238 {
 239         struct nouveau_dmem_fault *fault = private;
 240         struct nouveau_drm *drm = fault->drm;
 241
 242         if (fault->fence) {
 243                 nouveau_fence_wait(fault->fence, true, false);
 244                 nouveau_fence_unref(&fault->fence);
 245         } else {
 246                 /*
 247                  * FIXME wait for channel to be IDLE before calling finalizing
 248                  * the hmem object below (nouveau_migrate_hmem_fini()).
 249                  */
 250         }
 251
 252         while (fault->npages--) {
 253                 dma_unmap_page(drm->dev->dev, fault->dma[fault->npages],
 254                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 255         }
 256         kfree(fault->dma);
 257 }
 258
 259 static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = {
 260         .alloc_and_copy         = nouveau_dmem_fault_alloc_and_copy,
 261         .finalize_and_map       = nouveau_dmem_fault_finalize_and_map,
 262 };
 263
 264 static int
 265 nouveau_dmem_fault(struct hmm_devmem *devmem,
 266                    struct vm_area_struct *vma,
 267                    unsigned long addr,
 268                    const struct page *page,
 269                    unsigned int flags,
 270                    pmd_t *pmdp)
 271 {
 272         struct drm_device *drm_dev = dev_get_drvdata(devmem->device);
 273         unsigned long src[1] = {0}, dst[1] = {0};
 274         struct nouveau_dmem_fault fault = {0};
 275         int ret;
 276
 277
 278
 279         /*
 280          * FIXME what we really want is to find some heuristic to migrate more
 281          * than just one page on CPU fault. When such fault happens it is very
 282          * likely that more surrounding page will CPU fault too.
 283          */
 284         fault.drm = nouveau_drm(drm_dev);
 285         ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vma, addr,
 286                           addr + PAGE_SIZE, src, dst, &fault);
 287         if (ret)
 288                 return VM_FAULT_SIGBUS;
 289
 290         if (dst[0] == MIGRATE_PFN_ERROR)
 291                 return VM_FAULT_SIGBUS;
 292
 293         return 0;
 294 }
 295
 296 static const struct hmm_devmem_ops
 297 nouveau_dmem_devmem_ops = {
 298         .free = nouveau_dmem_free,
 299         .fault = nouveau_dmem_fault,
 300 };
 301
 302 static int
 303 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm)
 304 {
 305         struct nouveau_dmem_chunk *chunk;
 306         int ret;
 307
 308         if (drm->dmem == NULL)
 309                 return -EINVAL;
 310
 311         mutex_lock(&drm->dmem->mutex);
 312         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
 313                                          struct nouveau_dmem_chunk,
 314                                          list);
 315         if (chunk == NULL) {
 316                 mutex_unlock(&drm->dmem->mutex);
 317                 return -ENOMEM;
 318         }
 319
 320         list_del(&chunk->list);
 321         mutex_unlock(&drm->dmem->mutex);
 322
 323         ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0,
 324                              TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL,
 325                              &chunk->bo);
 326         if (ret)
 327                 goto out;
 328
 329         ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 330         if (ret) {
 331                 nouveau_bo_ref(NULL, &chunk->bo);
 332                 goto out;
 333         }
 334
 335         bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES);
 336         spin_lock_init(&chunk->lock);
 337
 338 out:
 339         mutex_lock(&drm->dmem->mutex);
 340         if (chunk->bo)
 341                 list_add(&chunk->list, &drm->dmem->chunk_empty);
 342         else
 343                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
 344         mutex_unlock(&drm->dmem->mutex);
 345
 346         return ret;
 347 }
 348
 349 static struct nouveau_dmem_chunk *
 350 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm)
 351 {
 352         struct nouveau_dmem_chunk *chunk;
 353
 354         chunk = list_first_entry_or_null(&drm->dmem->chunk_free,
 355                                          struct nouveau_dmem_chunk,
 356                                          list);
 357         if (chunk)
 358                 return chunk;
 359
 360         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
 361                                          struct nouveau_dmem_chunk,
 362                                          list);
 363         if (chunk->bo)
 364                 return chunk;
 365
 366         return NULL;
 367 }
 368
 369 static int
 370 nouveau_dmem_pages_alloc(struct nouveau_drm *drm,
 371                          unsigned long npages,
 372                          unsigned long *pages)
 373 {
 374         struct nouveau_dmem_chunk *chunk;
 375         unsigned long c;
 376         int ret;
 377
 378         memset(pages, 0xff, npages * sizeof(*pages));
 379
 380         mutex_lock(&drm->dmem->mutex);
 381         for (c = 0; c < npages;) {
 382                 unsigned long i;
 383
 384                 chunk = nouveau_dmem_chunk_first_free_locked(drm);
 385                 if (chunk == NULL) {
 386                         mutex_unlock(&drm->dmem->mutex);
 387                         ret = nouveau_dmem_chunk_alloc(drm);
 388                         if (ret) {
 389                                 if (c)
 390                                         break;
 391                                 return ret;
 392                         }
 393                         continue;
 394                 }
 395
 396                 spin_lock(&chunk->lock);
 397                 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES);
 398                 while (i < DMEM_CHUNK_NPAGES && c < npages) {
 399                         pages[c] = chunk->pfn_first + i;
 400                         set_bit(i, chunk->bitmap);
 401                         chunk->callocated++;
 402                         c++;
 403
 404                         i = find_next_zero_bit(chunk->bitmap,
 405                                         DMEM_CHUNK_NPAGES, i);
 406                 }
 407                 spin_unlock(&chunk->lock);
 408         }
 409         mutex_unlock(&drm->dmem->mutex);
 410
 411         return 0;
 412 }
 413
 414 static struct page *
 415 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
 416 {
 417         unsigned long pfns[1];
 418         struct page *page;
 419         int ret;
 420
 421         /* FIXME stop all the miss-match API ... */
 422         ret = nouveau_dmem_pages_alloc(drm, 1, pfns);
 423         if (ret)
 424                 return NULL;
 425
 426         page = pfn_to_page(pfns[0]);
 427         get_page(page);
 428         lock_page(page);
 429         return page;
 430 }
 431
 432 static void
 433 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
 434 {
 435         unlock_page(page);
 436         put_page(page);
 437 }
 438
 439 void
 440 nouveau_dmem_resume(struct nouveau_drm *drm)
 441 {
 442         struct nouveau_dmem_chunk *chunk;
 443         int ret;
 444
 445         if (drm->dmem == NULL)
 446                 return;
 447
 448         mutex_lock(&drm->dmem->mutex);
 449         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
 450                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 451                 /* FIXME handle pin failure */
 452                 WARN_ON(ret);
 453         }
 454         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
 455                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 456                 /* FIXME handle pin failure */
 457                 WARN_ON(ret);
 458         }
 459         list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
 460                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 461                 /* FIXME handle pin failure */
 462                 WARN_ON(ret);
 463         }
 464         mutex_unlock(&drm->dmem->mutex);
 465 }
 466
 467 void
 468 nouveau_dmem_suspend(struct nouveau_drm *drm)
 469 {
 470         struct nouveau_dmem_chunk *chunk;
 471
 472         if (drm->dmem == NULL)
 473                 return;
 474
 475         mutex_lock(&drm->dmem->mutex);
 476         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
 477                 nouveau_bo_unpin(chunk->bo);
 478         }
 479         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
 480                 nouveau_bo_unpin(chunk->bo);
 481         }
 482         list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
 483                 nouveau_bo_unpin(chunk->bo);
 484         }
 485         mutex_unlock(&drm->dmem->mutex);
 486 }
 487
 488 void
 489 nouveau_dmem_fini(struct nouveau_drm *drm)
 490 {
 491         struct nouveau_dmem_chunk *chunk, *tmp;
 492
 493         if (drm->dmem == NULL)
 494                 return;
 495
 496         mutex_lock(&drm->dmem->mutex);
 497
 498         WARN_ON(!list_empty(&drm->dmem->chunk_free));
 499         WARN_ON(!list_empty(&drm->dmem->chunk_full));
 500
 501         list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) {
 502                 if (chunk->bo) {
 503                         nouveau_bo_unpin(chunk->bo);
 504                         nouveau_bo_ref(NULL, &chunk->bo);
 505                 }
 506                 list_del(&chunk->list);
 507                 kfree(chunk);
 508         }
 509
 510         mutex_unlock(&drm->dmem->mutex);
 511 }
 512
 513 static int
 514 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
 515                     enum nouveau_aper dst_aper, u64 dst_addr,
 516                     enum nouveau_aper src_aper, u64 src_addr)
 517 {
 518         struct nouveau_channel *chan = drm->dmem->migrate.chan;
 519         u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ |
 520                          (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ |
 521                          (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ |
 522                          (1 << 2) /* FLUSH_ENABLE_TRUE. */ |
 523                          (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */;
 524         int ret;
 525
 526         ret = RING_SPACE(chan, 13);
 527         if (ret)
 528                 return ret;
 529
 530         if (src_aper != NOUVEAU_APER_VIRT) {
 531                 switch (src_aper) {
 532                 case NOUVEAU_APER_VRAM:
 533                         BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0);
 534                         break;
 535                 case NOUVEAU_APER_HOST:
 536                         BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1);
 537                         break;
 538                 default:
 539                         return -EINVAL;
 540                 }
 541                 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */
 542         }
 543
 544         if (dst_aper != NOUVEAU_APER_VIRT) {
 545                 switch (dst_aper) {
 546                 case NOUVEAU_APER_VRAM:
 547                         BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0);
 548                         break;
 549                 case NOUVEAU_APER_HOST:
 550                         BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1);
 551                         break;
 552                 default:
 553                         return -EINVAL;
 554                 }
 555                 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */
 556         }
 557
 558         BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
 559         OUT_RING  (chan, upper_32_bits(src_addr));
 560         OUT_RING  (chan, lower_32_bits(src_addr));
 561         OUT_RING  (chan, upper_32_bits(dst_addr));
 562         OUT_RING  (chan, lower_32_bits(dst_addr));
 563         OUT_RING  (chan, PAGE_SIZE);
 564         OUT_RING  (chan, PAGE_SIZE);
 565         OUT_RING  (chan, PAGE_SIZE);
 566         OUT_RING  (chan, npages);
 567         BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
 568         OUT_RING  (chan, launch_dma);
 569         return 0;
 570 }
 571
 572 static int
 573 nouveau_dmem_migrate_init(struct nouveau_drm *drm)
 574 {
 575         switch (drm->ttm.copy.oclass) {
 576         case PASCAL_DMA_COPY_A:
 577         case PASCAL_DMA_COPY_B:
 578         case  VOLTA_DMA_COPY_A:
 579         case TURING_DMA_COPY_A:
 580                 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
 581                 drm->dmem->migrate.chan = drm->ttm.chan;
 582                 return 0;
 583         default:
 584                 break;
 585         }
 586         return -ENODEV;
 587 }
 588
 589 void
 590 nouveau_dmem_init(struct nouveau_drm *drm)
 591 {
 592         struct device *device = drm->dev->dev;
 593         unsigned long i, size;
 594         int ret;
 595
 596         /* This only make sense on PASCAL or newer */
 597         if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
 598                 return;
 599
 600         if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
 601                 return;
 602
 603         mutex_init(&drm->dmem->mutex);
 604         INIT_LIST_HEAD(&drm->dmem->chunk_free);
 605         INIT_LIST_HEAD(&drm->dmem->chunk_full);
 606         INIT_LIST_HEAD(&drm->dmem->chunk_empty);
 607
 608         size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE);
 609
 610         /* Initialize migration dma helpers before registering memory */
 611         ret = nouveau_dmem_migrate_init(drm);
 612         if (ret) {
 613                 kfree(drm->dmem);
 614                 drm->dmem = NULL;
 615                 return;
 616         }
 617
 618         /*
 619          * FIXME we need some kind of policy to decide how much VRAM we
 620          * want to register with HMM. For now just register everything
 621          * and latter if we want to do thing like over commit then we
 622          * could revisit this.
 623          */
 624         drm->dmem->devmem = hmm_devmem_add(&nouveau_dmem_devmem_ops,
 625                                            device, size);
 626         if (drm->dmem->devmem == NULL) {
 627                 kfree(drm->dmem);
 628                 drm->dmem = NULL;
 629                 return;
 630         }
 631
 632         for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
 633                 struct nouveau_dmem_chunk *chunk;
 634                 struct page *page;
 635                 unsigned long j;
 636
 637                 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
 638                 if (chunk == NULL) {
 639                         nouveau_dmem_fini(drm);
 640                         return;
 641                 }
 642
 643                 chunk->drm = drm;
 644                 chunk->pfn_first = drm->dmem->devmem->pfn_first;
 645                 chunk->pfn_first += (i * DMEM_CHUNK_NPAGES);
 646                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
 647
 648                 page = pfn_to_page(chunk->pfn_first);
 649                 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) {
 650                         hmm_devmem_page_set_drvdata(page, (long)chunk);
 651                 }
 652         }
 653
 654         NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
 655 }
 656
 657 static void
 658 nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma,
 659                                     const unsigned long *src_pfns,
 660                                     unsigned long *dst_pfns,
 661                                     unsigned long start,
 662                                     unsigned long end,
 663                                     void *private)
 664 {
 665         struct nouveau_migrate *migrate = private;
 666         struct nouveau_drm *drm = migrate->drm;
 667         struct device *dev = drm->dev->dev;
 668         unsigned long addr, i, npages = 0;
 669         nouveau_migrate_copy_t copy;
 670         int ret;
 671
 672         /* First allocate new memory */
 673         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 674                 struct page *dpage, *spage;
 675
 676                 dst_pfns[i] = 0;
 677                 spage = migrate_pfn_to_page(src_pfns[i]);
 678                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
 679                         continue;
 680
 681                 dpage = nouveau_dmem_page_alloc_locked(drm);
 682                 if (!dpage)
 683                         continue;
 684
 685                 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
 686                               MIGRATE_PFN_LOCKED |
 687                               MIGRATE_PFN_DEVICE;
 688                 npages++;
 689         }
 690
 691         if (!npages)
 692                 return;
 693
 694         /* Allocate storage for DMA addresses, so we can unmap later. */
 695         migrate->dma = kmalloc(sizeof(*migrate->dma) * npages, GFP_KERNEL);
 696         if (!migrate->dma)
 697                 goto error;
 698
 699         /* Copy things over */
 700         copy = drm->dmem->migrate.copy_func;
 701         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 702                 struct nouveau_dmem_chunk *chunk;
 703                 struct page *spage, *dpage;
 704                 u64 src_addr, dst_addr;
 705
 706                 dpage = migrate_pfn_to_page(dst_pfns[i]);
 707                 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
 708                         continue;
 709
 710                 chunk = (void *)hmm_devmem_page_get_drvdata(dpage);
 711                 dst_addr = page_to_pfn(dpage) - chunk->pfn_first;
 712                 dst_addr = (dst_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
 713
 714                 spage = migrate_pfn_to_page(src_pfns[i]);
 715                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
 716                         nouveau_dmem_page_free_locked(drm, dpage);
 717                         dst_pfns[i] = 0;
 718                         continue;
 719                 }
 720
 721                 migrate->dma[migrate->dma_nr] =
 722                         dma_map_page_attrs(dev, spage, 0, PAGE_SIZE,
 723                                            PCI_DMA_BIDIRECTIONAL,
 724                                            DMA_ATTR_SKIP_CPU_SYNC);
 725                 if (dma_mapping_error(dev, migrate->dma[migrate->dma_nr])) {
 726                         nouveau_dmem_page_free_locked(drm, dpage);
 727                         dst_pfns[i] = 0;
 728                         continue;
 729                 }
 730
 731                 src_addr = migrate->dma[migrate->dma_nr++];
 732
 733                 ret = copy(drm, 1, NOUVEAU_APER_VRAM, dst_addr,
 734                                    NOUVEAU_APER_HOST, src_addr);
 735                 if (ret) {
 736                         nouveau_dmem_page_free_locked(drm, dpage);
 737                         dst_pfns[i] = 0;
 738                         continue;
 739                 }
 740         }
 741
 742         nouveau_fence_new(drm->dmem->migrate.chan, false, &migrate->fence);
 743
 744         return;
 745
 746 error:
 747         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
 748                 struct page *page;
 749
 750                 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
 751                         continue;
 752
 753                 page = migrate_pfn_to_page(dst_pfns[i]);
 754                 dst_pfns[i] = MIGRATE_PFN_ERROR;
 755                 if (page == NULL)
 756                         continue;
 757
 758                 __free_page(page);
 759         }
 760 }
 761
 762 void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma,
 763                                            const unsigned long *src_pfns,
 764                                            const unsigned long *dst_pfns,
 765                                            unsigned long start,
 766                                            unsigned long end,
 767                                            void *private)
 768 {
 769         struct nouveau_migrate *migrate = private;
 770         struct nouveau_drm *drm = migrate->drm;
 771
 772         if (migrate->fence) {
 773                 nouveau_fence_wait(migrate->fence, true, false);
 774                 nouveau_fence_unref(&migrate->fence);
 775         } else {
 776                 /*
 777                  * FIXME wait for channel to be IDLE before finalizing
 778                  * the hmem object below (nouveau_migrate_hmem_fini()) ?
 779                  */
 780         }
 781
 782         while (migrate->dma_nr--) {
 783                 dma_unmap_page(drm->dev->dev, migrate->dma[migrate->dma_nr],
 784                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 785         }
 786         kfree(migrate->dma);
 787
 788         /*
 789          * FIXME optimization: update GPU page table to point to newly
 790          * migrated memory.
 791          */
 792 }
 793
 794 static const struct migrate_vma_ops nouveau_dmem_migrate_ops = {
 795         .alloc_and_copy         = nouveau_dmem_migrate_alloc_and_copy,
 796         .finalize_and_map       = nouveau_dmem_migrate_finalize_and_map,
 797 };
 798
 799 int
 800 nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
 801                          struct vm_area_struct *vma,
 802                          unsigned long start,
 803                          unsigned long end)
 804 {
 805         unsigned long *src_pfns, *dst_pfns, npages;
 806         struct nouveau_migrate migrate = {0};
 807         unsigned long i, c, max;
 808         int ret = 0;
 809
 810         npages = (end - start) >> PAGE_SHIFT;
 811         max = min(SG_MAX_SINGLE_ALLOC, npages);
 812         src_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
 813         if (src_pfns == NULL)
 814                 return -ENOMEM;
 815         dst_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
 816         if (dst_pfns == NULL) {
 817                 kfree(src_pfns);
 818                 return -ENOMEM;
 819         }
 820
 821         migrate.drm = drm;
 822         migrate.vma = vma;
 823         migrate.npages = npages;
 824         for (i = 0; i < npages; i += c) {
 825                 unsigned long next;
 826
 827                 c = min(SG_MAX_SINGLE_ALLOC, npages);
 828                 next = start + (c << PAGE_SHIFT);
 829                 ret = migrate_vma(&nouveau_dmem_migrate_ops, vma, start,
 830                                   next, src_pfns, dst_pfns, &migrate);
 831                 if (ret)
 832                         goto out;
 833                 start = next;
 834         }
 835
 836 out:
 837         kfree(dst_pfns);
 838         kfree(src_pfns);
 839         return ret;
 840 }
 841
 842 static inline bool
 843 nouveau_dmem_page(struct nouveau_drm *drm, struct page *page)
 844 {
 845         if (!is_device_private_page(page))
 846                 return false;
 847
 848         if (drm->dmem->devmem != page->pgmap->data)
 849                 return false;
 850
 851         return true;
 852 }
 853
 854 void
 855 nouveau_dmem_convert_pfn(struct nouveau_drm *drm,
 856                          struct hmm_range *range)
 857 {
 858         unsigned long i, npages;
 859
 860         npages = (range->end - range->start) >> PAGE_SHIFT;
 861         for (i = 0; i < npages; ++i) {
 862                 struct nouveau_dmem_chunk *chunk;
 863                 struct page *page;
 864                 uint64_t addr;
 865
 866                 page = hmm_pfn_to_page(range, range->pfns[i]);
 867                 if (page == NULL)
 868                         continue;
 869
 870                 if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
 871                         continue;
 872                 }
 873
 874                 if (!nouveau_dmem_page(drm, page)) {
 875                         WARN(1, "Some unknown device memory !\n");
 876                         range->pfns[i] = 0;
 877                         continue;
 878                 }
 879
 880                 chunk = (void *)hmm_devmem_page_get_drvdata(page);
 881                 addr = page_to_pfn(page) - chunk->pfn_first;
 882                 addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT;
 883
 884                 range->pfns[i] &= ((1UL << range->pfn_shift) - 1);
 885                 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift;
 886         }
 887 }