drivers/gpu/drm/nouveau/nouveau_dmem.c

   1 /*
   2  * Copyright 2018 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22 #include "nouveau_dmem.h"
  23 #include "nouveau_drv.h"
  24 #include "nouveau_chan.h"
  25 #include "nouveau_dma.h"
  26 #include "nouveau_mem.h"
  27 #include "nouveau_bo.h"
  28
  29 #include <nvif/class.h>
  30 #include <nvif/object.h>
  31 #include <nvif/if500b.h>
  32 #include <nvif/if900b.h>
  33
  34 #include <linux/sched/mm.h>
  35 #include <linux/hmm.h>
  36
  37 /*
  38  * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
  39  * it in vram while in use. We likely want to overhaul memory management for
  40  * nouveau to be more page like (not necessarily with system page size but a
  41  * bigger page size) at lowest level and have some shim layer on top that would
  42  * provide the same functionality as TTM.
  43  */
  44 #define DMEM_CHUNK_SIZE (2UL << 20)
  45 #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
  46
  47 struct nouveau_migrate;
  48
  49 enum nouveau_aper {
  50         NOUVEAU_APER_VIRT,
  51         NOUVEAU_APER_VRAM,
  52         NOUVEAU_APER_HOST,
  53 };
  54
  55 typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
  56                                       enum nouveau_aper, u64 dst_addr,
  57                                       enum nouveau_aper, u64 src_addr);
  58
  59 struct nouveau_dmem_chunk {
  60         struct list_head list;
  61         struct nouveau_bo *bo;
  62         struct nouveau_drm *drm;
  63         unsigned long pfn_first;
  64         unsigned long callocated;
  65         unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)];
  66         spinlock_t lock;
  67 };
  68
  69 struct nouveau_dmem_migrate {
  70         nouveau_migrate_copy_t copy_func;
  71         struct nouveau_channel *chan;
  72 };
  73
  74 struct nouveau_dmem {
  75         struct hmm_devmem *devmem;
  76         struct nouveau_dmem_migrate migrate;
  77         struct list_head chunk_free;
  78         struct list_head chunk_full;
  79         struct list_head chunk_empty;
  80         struct mutex mutex;
  81 };
  82
  83 struct nouveau_dmem_fault {
  84         struct nouveau_drm *drm;
  85         struct nouveau_fence *fence;
  86         dma_addr_t *dma;
  87         unsigned long npages;
  88 };
  89
  90 struct nouveau_migrate {
  91         struct vm_area_struct *vma;
  92         struct nouveau_drm *drm;
  93         struct nouveau_fence *fence;
  94         unsigned long npages;
  95         dma_addr_t *dma;
  96         unsigned long dma_nr;
  97 };
  98
  99 static void
 100 nouveau_dmem_free(struct hmm_devmem *devmem, struct page *page)
 101 {
 102         struct nouveau_dmem_chunk *chunk;
 103         unsigned long idx;
 104
 105         chunk = (void *)hmm_devmem_page_get_drvdata(page);
 106         idx = page_to_pfn(page) - chunk->pfn_first;
 107
 108         /*
 109          * FIXME:
 110          *
 111          * This is really a bad example, we need to overhaul nouveau memory
 112          * management to be more page focus and allow lighter locking scheme
 113          * to be use in the process.
 114          */
 115         spin_lock(&chunk->lock);
 116         clear_bit(idx, chunk->bitmap);
 117         WARN_ON(!chunk->callocated);
 118         chunk->callocated--;
 119         /*
 120          * FIXME when chunk->callocated reach 0 we should add the chunk to
 121          * a reclaim list so that it can be freed in case of memory pressure.
 122          */
 123         spin_unlock(&chunk->lock);
 124 }
 125
 126 static void
 127 nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma,
 128                                   const unsigned long *src_pfns,
 129                                   unsigned long *dst_pfns,
 130                                   unsigned long start,
 131                                   unsigned long end,
 132                                   void *private)
 133 {
 134         struct nouveau_dmem_fault *fault = private;
 135         struct nouveau_drm *drm = fault->drm;
 136         struct device *dev = drm->dev->dev;
 137         unsigned long addr, i, npages = 0;
 138         nouveau_migrate_copy_t copy;
 139         int ret;
 140
 141
 142         /* First allocate new memory */
 143         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 144                 struct page *dpage, *spage;
 145
 146                 dst_pfns[i] = 0;
 147                 spage = migrate_pfn_to_page(src_pfns[i]);
 148                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
 149                         continue;
 150
 151                 dpage = hmm_vma_alloc_locked_page(vma, addr);
 152                 if (!dpage) {
 153                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 154                         continue;
 155                 }
 156
 157                 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
 158                               MIGRATE_PFN_LOCKED;
 159                 npages++;
 160         }
 161
 162         /* Allocate storage for DMA addresses, so we can unmap later. */
 163         fault->dma = kmalloc(sizeof(*fault->dma) * npages, GFP_KERNEL);
 164         if (!fault->dma)
 165                 goto error;
 166
 167         /* Copy things over */
 168         copy = drm->dmem->migrate.copy_func;
 169         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 170                 struct nouveau_dmem_chunk *chunk;
 171                 struct page *spage, *dpage;
 172                 u64 src_addr, dst_addr;
 173
 174                 dpage = migrate_pfn_to_page(dst_pfns[i]);
 175                 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
 176                         continue;
 177
 178                 spage = migrate_pfn_to_page(src_pfns[i]);
 179                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
 180                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 181                         __free_page(dpage);
 182                         continue;
 183                 }
 184
 185                 fault->dma[fault->npages] =
 186                         dma_map_page_attrs(dev, dpage, 0, PAGE_SIZE,
 187                                            PCI_DMA_BIDIRECTIONAL,
 188                                            DMA_ATTR_SKIP_CPU_SYNC);
 189                 if (dma_mapping_error(dev, fault->dma[fault->npages])) {
 190                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 191                         __free_page(dpage);
 192                         continue;
 193                 }
 194
 195                 dst_addr = fault->dma[fault->npages++];
 196
 197                 chunk = (void *)hmm_devmem_page_get_drvdata(spage);
 198                 src_addr = page_to_pfn(spage) - chunk->pfn_first;
 199                 src_addr = (src_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
 200
 201                 ret = copy(drm, 1, NOUVEAU_APER_HOST, dst_addr,
 202                                    NOUVEAU_APER_VRAM, src_addr);
 203                 if (ret) {
 204                         dst_pfns[i] = MIGRATE_PFN_ERROR;
 205                         __free_page(dpage);
 206                         continue;
 207                 }
 208         }
 209
 210         nouveau_fence_new(drm->dmem->migrate.chan, false, &fault->fence);
 211
 212         return;
 213
 214 error:
 215         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
 216                 struct page *page;
 217
 218                 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
 219                         continue;
 220
 221                 page = migrate_pfn_to_page(dst_pfns[i]);
 222                 dst_pfns[i] = MIGRATE_PFN_ERROR;
 223                 if (page == NULL)
 224                         continue;
 225
 226                 __free_page(page);
 227         }
 228 }
 229
 230 void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma,
 231                                          const unsigned long *src_pfns,
 232                                          const unsigned long *dst_pfns,
 233                                          unsigned long start,
 234                                          unsigned long end,
 235                                          void *private)
 236 {
 237         struct nouveau_dmem_fault *fault = private;
 238         struct nouveau_drm *drm = fault->drm;
 239
 240         if (fault->fence) {
 241                 nouveau_fence_wait(fault->fence, true, false);
 242                 nouveau_fence_unref(&fault->fence);
 243         } else {
 244                 /*
 245                  * FIXME wait for channel to be IDLE before calling finalizing
 246                  * the hmem object below (nouveau_migrate_hmem_fini()).
 247                  */
 248         }
 249
 250         while (fault->npages--) {
 251                 dma_unmap_page(drm->dev->dev, fault->dma[fault->npages],
 252                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 253         }
 254         kfree(fault->dma);
 255 }
 256
 257 static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = {
 258         .alloc_and_copy         = nouveau_dmem_fault_alloc_and_copy,
 259         .finalize_and_map       = nouveau_dmem_fault_finalize_and_map,
 260 };
 261
 262 static int
 263 nouveau_dmem_fault(struct hmm_devmem *devmem,
 264                    struct vm_area_struct *vma,
 265                    unsigned long addr,
 266                    const struct page *page,
 267                    unsigned int flags,
 268                    pmd_t *pmdp)
 269 {
 270         struct drm_device *drm_dev = dev_get_drvdata(devmem->device);
 271         unsigned long src[1] = {0}, dst[1] = {0};
 272         struct nouveau_dmem_fault fault = {0};
 273         int ret;
 274
 275
 276
 277         /*
 278          * FIXME what we really want is to find some heuristic to migrate more
 279          * than just one page on CPU fault. When such fault happens it is very
 280          * likely that more surrounding page will CPU fault too.
 281          */
 282         fault.drm = nouveau_drm(drm_dev);
 283         ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vma, addr,
 284                           addr + PAGE_SIZE, src, dst, &fault);
 285         if (ret)
 286                 return VM_FAULT_SIGBUS;
 287
 288         if (dst[0] == MIGRATE_PFN_ERROR)
 289                 return VM_FAULT_SIGBUS;
 290
 291         return 0;
 292 }
 293
 294 static const struct hmm_devmem_ops
 295 nouveau_dmem_devmem_ops = {
 296         .free = nouveau_dmem_free,
 297         .fault = nouveau_dmem_fault,
 298 };
 299
 300 static int
 301 nouveau_dmem_chunk_alloc(struct nouveau_drm *drm)
 302 {
 303         struct nouveau_dmem_chunk *chunk;
 304         int ret;
 305
 306         if (drm->dmem == NULL)
 307                 return -EINVAL;
 308
 309         mutex_lock(&drm->dmem->mutex);
 310         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
 311                                          struct nouveau_dmem_chunk,
 312                                          list);
 313         if (chunk == NULL) {
 314                 mutex_unlock(&drm->dmem->mutex);
 315                 return -ENOMEM;
 316         }
 317
 318         list_del(&chunk->list);
 319         mutex_unlock(&drm->dmem->mutex);
 320
 321         ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0,
 322                              TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL,
 323                              &chunk->bo);
 324         if (ret)
 325                 goto out;
 326
 327         ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 328         if (ret) {
 329                 nouveau_bo_ref(NULL, &chunk->bo);
 330                 goto out;
 331         }
 332
 333         bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES);
 334         spin_lock_init(&chunk->lock);
 335
 336 out:
 337         mutex_lock(&drm->dmem->mutex);
 338         if (chunk->bo)
 339                 list_add(&chunk->list, &drm->dmem->chunk_empty);
 340         else
 341                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
 342         mutex_unlock(&drm->dmem->mutex);
 343
 344         return ret;
 345 }
 346
 347 static struct nouveau_dmem_chunk *
 348 nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm)
 349 {
 350         struct nouveau_dmem_chunk *chunk;
 351
 352         chunk = list_first_entry_or_null(&drm->dmem->chunk_free,
 353                                          struct nouveau_dmem_chunk,
 354                                          list);
 355         if (chunk)
 356                 return chunk;
 357
 358         chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
 359                                          struct nouveau_dmem_chunk,
 360                                          list);
 361         if (chunk->bo)
 362                 return chunk;
 363
 364         return NULL;
 365 }
 366
 367 static int
 368 nouveau_dmem_pages_alloc(struct nouveau_drm *drm,
 369                          unsigned long npages,
 370                          unsigned long *pages)
 371 {
 372         struct nouveau_dmem_chunk *chunk;
 373         unsigned long c;
 374         int ret;
 375
 376         memset(pages, 0xff, npages * sizeof(*pages));
 377
 378         mutex_lock(&drm->dmem->mutex);
 379         for (c = 0; c < npages;) {
 380                 unsigned long i;
 381
 382                 chunk = nouveau_dmem_chunk_first_free_locked(drm);
 383                 if (chunk == NULL) {
 384                         mutex_unlock(&drm->dmem->mutex);
 385                         ret = nouveau_dmem_chunk_alloc(drm);
 386                         if (ret) {
 387                                 if (c)
 388                                         break;
 389                                 return ret;
 390                         }
 391                         continue;
 392                 }
 393
 394                 spin_lock(&chunk->lock);
 395                 i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES);
 396                 while (i < DMEM_CHUNK_NPAGES && c < npages) {
 397                         pages[c] = chunk->pfn_first + i;
 398                         set_bit(i, chunk->bitmap);
 399                         chunk->callocated++;
 400                         c++;
 401
 402                         i = find_next_zero_bit(chunk->bitmap,
 403                                         DMEM_CHUNK_NPAGES, i);
 404                 }
 405                 spin_unlock(&chunk->lock);
 406         }
 407         mutex_unlock(&drm->dmem->mutex);
 408
 409         return 0;
 410 }
 411
 412 static struct page *
 413 nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
 414 {
 415         unsigned long pfns[1];
 416         struct page *page;
 417         int ret;
 418
 419         /* FIXME stop all the miss-match API ... */
 420         ret = nouveau_dmem_pages_alloc(drm, 1, pfns);
 421         if (ret)
 422                 return NULL;
 423
 424         page = pfn_to_page(pfns[0]);
 425         get_page(page);
 426         lock_page(page);
 427         return page;
 428 }
 429
 430 static void
 431 nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
 432 {
 433         unlock_page(page);
 434         put_page(page);
 435 }
 436
 437 void
 438 nouveau_dmem_resume(struct nouveau_drm *drm)
 439 {
 440         struct nouveau_dmem_chunk *chunk;
 441         int ret;
 442
 443         if (drm->dmem == NULL)
 444                 return;
 445
 446         mutex_lock(&drm->dmem->mutex);
 447         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
 448                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 449                 /* FIXME handle pin failure */
 450                 WARN_ON(ret);
 451         }
 452         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
 453                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 454                 /* FIXME handle pin failure */
 455                 WARN_ON(ret);
 456         }
 457         list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
 458                 ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
 459                 /* FIXME handle pin failure */
 460                 WARN_ON(ret);
 461         }
 462         mutex_unlock(&drm->dmem->mutex);
 463 }
 464
 465 void
 466 nouveau_dmem_suspend(struct nouveau_drm *drm)
 467 {
 468         struct nouveau_dmem_chunk *chunk;
 469
 470         if (drm->dmem == NULL)
 471                 return;
 472
 473         mutex_lock(&drm->dmem->mutex);
 474         list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
 475                 nouveau_bo_unpin(chunk->bo);
 476         }
 477         list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
 478                 nouveau_bo_unpin(chunk->bo);
 479         }
 480         list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
 481                 nouveau_bo_unpin(chunk->bo);
 482         }
 483         mutex_unlock(&drm->dmem->mutex);
 484 }
 485
 486 void
 487 nouveau_dmem_fini(struct nouveau_drm *drm)
 488 {
 489         struct nouveau_dmem_chunk *chunk, *tmp;
 490
 491         if (drm->dmem == NULL)
 492                 return;
 493
 494         mutex_lock(&drm->dmem->mutex);
 495
 496         WARN_ON(!list_empty(&drm->dmem->chunk_free));
 497         WARN_ON(!list_empty(&drm->dmem->chunk_full));
 498
 499         list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) {
 500                 if (chunk->bo) {
 501                         nouveau_bo_unpin(chunk->bo);
 502                         nouveau_bo_ref(NULL, &chunk->bo);
 503                 }
 504                 list_del(&chunk->list);
 505                 kfree(chunk);
 506         }
 507
 508         mutex_unlock(&drm->dmem->mutex);
 509 }
 510
 511 static int
 512 nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
 513                     enum nouveau_aper dst_aper, u64 dst_addr,
 514                     enum nouveau_aper src_aper, u64 src_addr)
 515 {
 516         struct nouveau_channel *chan = drm->dmem->migrate.chan;
 517         u32 launch_dma = (1 << 9) /* MULTI_LINE_ENABLE. */ |
 518                          (1 << 8) /* DST_MEMORY_LAYOUT_PITCH. */ |
 519                          (1 << 7) /* SRC_MEMORY_LAYOUT_PITCH. */ |
 520                          (1 << 2) /* FLUSH_ENABLE_TRUE. */ |
 521                          (2 << 0) /* DATA_TRANSFER_TYPE_NON_PIPELINED. */;
 522         int ret;
 523
 524         ret = RING_SPACE(chan, 13);
 525         if (ret)
 526                 return ret;
 527
 528         if (src_aper != NOUVEAU_APER_VIRT) {
 529                 switch (src_aper) {
 530                 case NOUVEAU_APER_VRAM:
 531                         BEGIN_IMC0(chan, NvSubCopy, 0x0260, 0);
 532                         break;
 533                 case NOUVEAU_APER_HOST:
 534                         BEGIN_IMC0(chan, NvSubCopy, 0x0260, 1);
 535                         break;
 536                 default:
 537                         return -EINVAL;
 538                 }
 539                 launch_dma |= 0x00001000; /* SRC_TYPE_PHYSICAL. */
 540         }
 541
 542         if (dst_aper != NOUVEAU_APER_VIRT) {
 543                 switch (dst_aper) {
 544                 case NOUVEAU_APER_VRAM:
 545                         BEGIN_IMC0(chan, NvSubCopy, 0x0264, 0);
 546                         break;
 547                 case NOUVEAU_APER_HOST:
 548                         BEGIN_IMC0(chan, NvSubCopy, 0x0264, 1);
 549                         break;
 550                 default:
 551                         return -EINVAL;
 552                 }
 553                 launch_dma |= 0x00002000; /* DST_TYPE_PHYSICAL. */
 554         }
 555
 556         BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
 557         OUT_RING  (chan, upper_32_bits(src_addr));
 558         OUT_RING  (chan, lower_32_bits(src_addr));
 559         OUT_RING  (chan, upper_32_bits(dst_addr));
 560         OUT_RING  (chan, lower_32_bits(dst_addr));
 561         OUT_RING  (chan, PAGE_SIZE);
 562         OUT_RING  (chan, PAGE_SIZE);
 563         OUT_RING  (chan, PAGE_SIZE);
 564         OUT_RING  (chan, npages);
 565         BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
 566         OUT_RING  (chan, launch_dma);
 567         return 0;
 568 }
 569
 570 static int
 571 nouveau_dmem_migrate_init(struct nouveau_drm *drm)
 572 {
 573         switch (drm->ttm.copy.oclass) {
 574         case PASCAL_DMA_COPY_A:
 575         case PASCAL_DMA_COPY_B:
 576         case  VOLTA_DMA_COPY_A:
 577         case TURING_DMA_COPY_A:
 578                 drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
 579                 drm->dmem->migrate.chan = drm->ttm.chan;
 580                 return 0;
 581         default:
 582                 break;
 583         }
 584         return -ENODEV;
 585 }
 586
 587 void
 588 nouveau_dmem_init(struct nouveau_drm *drm)
 589 {
 590         struct device *device = drm->dev->dev;
 591         unsigned long i, size;
 592         int ret;
 593
 594         /* This only make sense on PASCAL or newer */
 595         if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
 596                 return;
 597
 598         if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
 599                 return;
 600
 601         mutex_init(&drm->dmem->mutex);
 602         INIT_LIST_HEAD(&drm->dmem->chunk_free);
 603         INIT_LIST_HEAD(&drm->dmem->chunk_full);
 604         INIT_LIST_HEAD(&drm->dmem->chunk_empty);
 605
 606         size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE);
 607
 608         /* Initialize migration dma helpers before registering memory */
 609         ret = nouveau_dmem_migrate_init(drm);
 610         if (ret) {
 611                 kfree(drm->dmem);
 612                 drm->dmem = NULL;
 613                 return;
 614         }
 615
 616         /*
 617          * FIXME we need some kind of policy to decide how much VRAM we
 618          * want to register with HMM. For now just register everything
 619          * and latter if we want to do thing like over commit then we
 620          * could revisit this.
 621          */
 622         drm->dmem->devmem = hmm_devmem_add(&nouveau_dmem_devmem_ops,
 623                                            device, size);
 624         if (drm->dmem->devmem == NULL) {
 625                 kfree(drm->dmem);
 626                 drm->dmem = NULL;
 627                 return;
 628         }
 629
 630         for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
 631                 struct nouveau_dmem_chunk *chunk;
 632                 struct page *page;
 633                 unsigned long j;
 634
 635                 chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
 636                 if (chunk == NULL) {
 637                         nouveau_dmem_fini(drm);
 638                         return;
 639                 }
 640
 641                 chunk->drm = drm;
 642                 chunk->pfn_first = drm->dmem->devmem->pfn_first;
 643                 chunk->pfn_first += (i * DMEM_CHUNK_NPAGES);
 644                 list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
 645
 646                 page = pfn_to_page(chunk->pfn_first);
 647                 for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) {
 648                         hmm_devmem_page_set_drvdata(page, (long)chunk);
 649                 }
 650         }
 651
 652         NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
 653 }
 654
 655 static void
 656 nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma,
 657                                     const unsigned long *src_pfns,
 658                                     unsigned long *dst_pfns,
 659                                     unsigned long start,
 660                                     unsigned long end,
 661                                     void *private)
 662 {
 663         struct nouveau_migrate *migrate = private;
 664         struct nouveau_drm *drm = migrate->drm;
 665         struct device *dev = drm->dev->dev;
 666         unsigned long addr, i, npages = 0;
 667         nouveau_migrate_copy_t copy;
 668         int ret;
 669
 670         /* First allocate new memory */
 671         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 672                 struct page *dpage, *spage;
 673
 674                 dst_pfns[i] = 0;
 675                 spage = migrate_pfn_to_page(src_pfns[i]);
 676                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
 677                         continue;
 678
 679                 dpage = nouveau_dmem_page_alloc_locked(drm);
 680                 if (!dpage)
 681                         continue;
 682
 683                 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
 684                               MIGRATE_PFN_LOCKED |
 685                               MIGRATE_PFN_DEVICE;
 686                 npages++;
 687         }
 688
 689         if (!npages)
 690                 return;
 691
 692         /* Allocate storage for DMA addresses, so we can unmap later. */
 693         migrate->dma = kmalloc(sizeof(*migrate->dma) * npages, GFP_KERNEL);
 694         if (!migrate->dma)
 695                 goto error;
 696
 697         /* Copy things over */
 698         copy = drm->dmem->migrate.copy_func;
 699         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
 700                 struct nouveau_dmem_chunk *chunk;
 701                 struct page *spage, *dpage;
 702                 u64 src_addr, dst_addr;
 703
 704                 dpage = migrate_pfn_to_page(dst_pfns[i]);
 705                 if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
 706                         continue;
 707
 708                 chunk = (void *)hmm_devmem_page_get_drvdata(dpage);
 709                 dst_addr = page_to_pfn(dpage) - chunk->pfn_first;
 710                 dst_addr = (dst_addr << PAGE_SHIFT) + chunk->bo->bo.offset;
 711
 712                 spage = migrate_pfn_to_page(src_pfns[i]);
 713                 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
 714                         nouveau_dmem_page_free_locked(drm, dpage);
 715                         dst_pfns[i] = 0;
 716                         continue;
 717                 }
 718
 719                 migrate->dma[migrate->dma_nr] =
 720                         dma_map_page_attrs(dev, spage, 0, PAGE_SIZE,
 721                                            PCI_DMA_BIDIRECTIONAL,
 722                                            DMA_ATTR_SKIP_CPU_SYNC);
 723                 if (dma_mapping_error(dev, migrate->dma[migrate->dma_nr])) {
 724                         nouveau_dmem_page_free_locked(drm, dpage);
 725                         dst_pfns[i] = 0;
 726                         continue;
 727                 }
 728
 729                 src_addr = migrate->dma[migrate->dma_nr++];
 730
 731                 ret = copy(drm, 1, NOUVEAU_APER_VRAM, dst_addr,
 732                                    NOUVEAU_APER_HOST, src_addr);
 733                 if (ret) {
 734                         nouveau_dmem_page_free_locked(drm, dpage);
 735                         dst_pfns[i] = 0;
 736                         continue;
 737                 }
 738         }
 739
 740         nouveau_fence_new(drm->dmem->migrate.chan, false, &migrate->fence);
 741
 742         return;
 743
 744 error:
 745         for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
 746                 struct page *page;
 747
 748                 if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
 749                         continue;
 750
 751                 page = migrate_pfn_to_page(dst_pfns[i]);
 752                 dst_pfns[i] = MIGRATE_PFN_ERROR;
 753                 if (page == NULL)
 754                         continue;
 755
 756                 __free_page(page);
 757         }
 758 }
 759
 760 void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma,
 761                                            const unsigned long *src_pfns,
 762                                            const unsigned long *dst_pfns,
 763                                            unsigned long start,
 764                                            unsigned long end,
 765                                            void *private)
 766 {
 767         struct nouveau_migrate *migrate = private;
 768         struct nouveau_drm *drm = migrate->drm;
 769
 770         if (migrate->fence) {
 771                 nouveau_fence_wait(migrate->fence, true, false);
 772                 nouveau_fence_unref(&migrate->fence);
 773         } else {
 774                 /*
 775                  * FIXME wait for channel to be IDLE before finalizing
 776                  * the hmem object below (nouveau_migrate_hmem_fini()) ?
 777                  */
 778         }
 779
 780         while (migrate->dma_nr--) {
 781                 dma_unmap_page(drm->dev->dev, migrate->dma[migrate->dma_nr],
 782                                PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 783         }
 784         kfree(migrate->dma);
 785
 786         /*
 787          * FIXME optimization: update GPU page table to point to newly
 788          * migrated memory.
 789          */
 790 }
 791
 792 static const struct migrate_vma_ops nouveau_dmem_migrate_ops = {
 793         .alloc_and_copy         = nouveau_dmem_migrate_alloc_and_copy,
 794         .finalize_and_map       = nouveau_dmem_migrate_finalize_and_map,
 795 };
 796
 797 int
 798 nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
 799                          struct vm_area_struct *vma,
 800                          unsigned long start,
 801                          unsigned long end)
 802 {
 803         unsigned long *src_pfns, *dst_pfns, npages;
 804         struct nouveau_migrate migrate = {0};
 805         unsigned long i, c, max;
 806         int ret = 0;
 807
 808         npages = (end - start) >> PAGE_SHIFT;
 809         max = min(SG_MAX_SINGLE_ALLOC, npages);
 810         src_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
 811         if (src_pfns == NULL)
 812                 return -ENOMEM;
 813         dst_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
 814         if (dst_pfns == NULL) {
 815                 kfree(src_pfns);
 816                 return -ENOMEM;
 817         }
 818
 819         migrate.drm = drm;
 820         migrate.vma = vma;
 821         migrate.npages = npages;
 822         for (i = 0; i < npages; i += c) {
 823                 unsigned long next;
 824
 825                 c = min(SG_MAX_SINGLE_ALLOC, npages);
 826                 next = start + (c << PAGE_SHIFT);
 827                 ret = migrate_vma(&nouveau_dmem_migrate_ops, vma, start,
 828                                   next, src_pfns, dst_pfns, &migrate);
 829                 if (ret)
 830                         goto out;
 831                 start = next;
 832         }
 833
 834 out:
 835         kfree(dst_pfns);
 836         kfree(src_pfns);
 837         return ret;
 838 }
 839
 840 static inline bool
 841 nouveau_dmem_page(struct nouveau_drm *drm, struct page *page)
 842 {
 843         if (!is_device_private_page(page))
 844                 return false;
 845
 846         if (drm->dmem->devmem != page->pgmap->data)
 847                 return false;
 848
 849         return true;
 850 }
 851
 852 void
 853 nouveau_dmem_convert_pfn(struct nouveau_drm *drm,
 854                          struct hmm_range *range)
 855 {
 856         unsigned long i, npages;
 857
 858         npages = (range->end - range->start) >> PAGE_SHIFT;
 859         for (i = 0; i < npages; ++i) {
 860                 struct nouveau_dmem_chunk *chunk;
 861                 struct page *page;
 862                 uint64_t addr;
 863
 864                 page = hmm_pfn_to_page(range, range->pfns[i]);
 865                 if (page == NULL)
 866                         continue;
 867
 868                 if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
 869                         continue;
 870                 }
 871
 872                 if (!nouveau_dmem_page(drm, page)) {
 873                         WARN(1, "Some unknown device memory !\n");
 874                         range->pfns[i] = 0;
 875                         continue;
 876                 }
 877
 878                 chunk = (void *)hmm_devmem_page_get_drvdata(page);
 879                 addr = page_to_pfn(page) - chunk->pfn_first;
 880                 addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT;
 881
 882                 range->pfns[i] &= ((1UL << range->pfn_shift) - 1);
 883                 range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift;
 884         }
 885 }