drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c

   1 /*
   2  * Copyright 2017 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22 #define NVKM_VMM_LEVELS_MAX 5
  23 #include "vmm.h"
  24
  25 #include <subdev/fb.h>
  26
  27 static void
  28 nvkm_vmm_pt_del(struct nvkm_vmm_pt **ppgt)
  29 {
  30         struct nvkm_vmm_pt *pgt = *ppgt;
  31         if (pgt) {
  32                 kvfree(pgt->pde);
  33                 kfree(pgt);
  34                 *ppgt = NULL;
  35         }
  36 }
  37
  38
  39 static struct nvkm_vmm_pt *
  40 nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse,
  41                 const struct nvkm_vmm_page *page)
  42 {
  43         const u32 pten = 1 << desc->bits;
  44         struct nvkm_vmm_pt *pgt;
  45         u32 lpte = 0;
  46
  47         if (desc->type > PGT) {
  48                 if (desc->type == SPT) {
  49                         const struct nvkm_vmm_desc *pair = page[-1].desc;
  50                         lpte = pten >> (desc->bits - pair->bits);
  51                 } else {
  52                         lpte = pten;
  53                 }
  54         }
  55
  56         if (!(pgt = kzalloc(sizeof(*pgt) + lpte, GFP_KERNEL)))
  57                 return NULL;
  58         pgt->page = page ? page->shift : 0;
  59         pgt->sparse = sparse;
  60
  61         if (desc->type == PGD) {
  62                 pgt->pde = kvcalloc(pten, sizeof(*pgt->pde), GFP_KERNEL);
  63                 if (!pgt->pde) {
  64                         kfree(pgt);
  65                         return NULL;
  66                 }
  67         }
  68
  69         return pgt;
  70 }
  71
  72 struct nvkm_vmm_iter {
  73         const struct nvkm_vmm_page *page;
  74         const struct nvkm_vmm_desc *desc;
  75         struct nvkm_vmm *vmm;
  76         u64 cnt;
  77         u16 max, lvl;
  78         u32 pte[NVKM_VMM_LEVELS_MAX];
  79         struct nvkm_vmm_pt *pt[NVKM_VMM_LEVELS_MAX];
  80         int flush;
  81 };
  82
  83 #ifdef CONFIG_NOUVEAU_DEBUG_MMU
  84 static const char *
  85 nvkm_vmm_desc_type(const struct nvkm_vmm_desc *desc)
  86 {
  87         switch (desc->type) {
  88         case PGD: return "PGD";
  89         case PGT: return "PGT";
  90         case SPT: return "SPT";
  91         case LPT: return "LPT";
  92         default:
  93                 return "UNKNOWN";
  94         }
  95 }
  96
  97 static void
  98 nvkm_vmm_trace(struct nvkm_vmm_iter *it, char *buf)
  99 {
 100         int lvl;
 101         for (lvl = it->max; lvl >= 0; lvl--) {
 102                 if (lvl >= it->lvl)
 103                         buf += sprintf(buf,  "%05x:", it->pte[lvl]);
 104                 else
 105                         buf += sprintf(buf, "xxxxx:");
 106         }
 107 }
 108
 109 #define TRA(i,f,a...) do {                                                     \
 110         char _buf[NVKM_VMM_LEVELS_MAX * 7];                                    \
 111         struct nvkm_vmm_iter *_it = (i);                                       \
 112         nvkm_vmm_trace(_it, _buf);                                             \
 113         VMM_TRACE(_it->vmm, "%s "f, _buf, ##a);                                \
 114 } while(0)
 115 #else
 116 #define TRA(i,f,a...)
 117 #endif
 118
 119 static inline void
 120 nvkm_vmm_flush_mark(struct nvkm_vmm_iter *it)
 121 {
 122         it->flush = min(it->flush, it->max - it->lvl);
 123 }
 124
 125 static inline void
 126 nvkm_vmm_flush(struct nvkm_vmm_iter *it)
 127 {
 128         if (it->flush != NVKM_VMM_LEVELS_MAX) {
 129                 if (it->vmm->func->flush) {
 130                         TRA(it, "flush: %d", it->flush);
 131                         it->vmm->func->flush(it->vmm, it->flush);
 132                 }
 133                 it->flush = NVKM_VMM_LEVELS_MAX;
 134         }
 135 }
 136
 137 static void
 138 nvkm_vmm_unref_pdes(struct nvkm_vmm_iter *it)
 139 {
 140         const struct nvkm_vmm_desc *desc = it->desc;
 141         const int type = desc[it->lvl].type == SPT;
 142         struct nvkm_vmm_pt *pgd = it->pt[it->lvl + 1];
 143         struct nvkm_vmm_pt *pgt = it->pt[it->lvl];
 144         struct nvkm_mmu_pt *pt = pgt->pt[type];
 145         struct nvkm_vmm *vmm = it->vmm;
 146         u32 pdei = it->pte[it->lvl + 1];
 147
 148         /* Recurse up the tree, unreferencing/destroying unneeded PDs. */
 149         it->lvl++;
 150         if (--pgd->refs[0]) {
 151                 const struct nvkm_vmm_desc_func *func = desc[it->lvl].func;
 152                 /* PD has other valid PDEs, so we need a proper update. */
 153                 TRA(it, "PDE unmap %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
 154                 pgt->pt[type] = NULL;
 155                 if (!pgt->refs[!type]) {
 156                         /* PDE no longer required. */
 157                         if (pgd->pt[0]) {
 158                                 if (pgt->sparse) {
 159                                         func->sparse(vmm, pgd->pt[0], pdei, 1);
 160                                         pgd->pde[pdei] = NVKM_VMM_PDE_SPARSE;
 161                                 } else {
 162                                         func->unmap(vmm, pgd->pt[0], pdei, 1);
 163                                         pgd->pde[pdei] = NULL;
 164                                 }
 165                         } else {
 166                                 /* Special handling for Tesla-class GPUs,
 167                                  * where there's no central PD, but each
 168                                  * instance has its own embedded PD.
 169                                  */
 170                                 func->pde(vmm, pgd, pdei);
 171                                 pgd->pde[pdei] = NULL;
 172                         }
 173                 } else {
 174                         /* PDE was pointing at dual-PTs and we're removing
 175                          * one of them, leaving the other in place.
 176                          */
 177                         func->pde(vmm, pgd, pdei);
 178                 }
 179
 180                 /* GPU may have cached the PTs, flush before freeing. */
 181                 nvkm_vmm_flush_mark(it);
 182                 nvkm_vmm_flush(it);
 183         } else {
 184                 /* PD has no valid PDEs left, so we can just destroy it. */
 185                 nvkm_vmm_unref_pdes(it);
 186         }
 187
 188         /* Destroy PD/PT. */
 189         TRA(it, "PDE free %s", nvkm_vmm_desc_type(&desc[it->lvl - 1]));
 190         nvkm_mmu_ptc_put(vmm->mmu, vmm->bootstrapped, &pt);
 191         if (!pgt->refs[!type])
 192                 nvkm_vmm_pt_del(&pgt);
 193         it->lvl--;
 194 }
 195
 196 static void
 197 nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
 198                      const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
 199 {
 200         const struct nvkm_vmm_desc *pair = it->page[-1].desc;
 201         const u32 sptb = desc->bits - pair->bits;
 202         const u32 sptn = 1 << sptb;
 203         struct nvkm_vmm *vmm = it->vmm;
 204         u32 spti = ptei & (sptn - 1), lpti, pteb;
 205
 206         /* Determine how many SPTEs are being touched under each LPTE,
 207          * and drop reference counts.
 208          */
 209         for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
 210                 const u32 pten = min(sptn - spti, ptes);
 211                 pgt->pte[lpti] -= pten;
 212                 ptes -= pten;
 213         }
 214
 215         /* We're done here if there's no corresponding LPT. */
 216         if (!pgt->refs[0])
 217                 return;
 218
 219         for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
 220                 /* Skip over any LPTEs that still have valid SPTEs. */
 221                 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPTES) {
 222                         for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
 223                                 if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES))
 224                                         break;
 225                         }
 226                         continue;
 227                 }
 228
 229                 /* As there's no more non-UNMAPPED SPTEs left in the range
 230                  * covered by a number of LPTEs, the LPTEs once again take
 231                  * control over their address range.
 232                  *
 233                  * Determine how many LPTEs need to transition state.
 234                  */
 235                 pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
 236                 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
 237                         if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)
 238                                 break;
 239                         pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;
 240                 }
 241
 242                 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
 243                         TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes);
 244                         pair->func->sparse(vmm, pgt->pt[0], pteb, ptes);
 245                 } else
 246                 if (pair->func->invalid) {
 247                         /* If the MMU supports it, restore the LPTE to the
 248                          * INVALID state to tell the MMU there is no point
 249                          * trying to fetch the corresponding SPTEs.
 250                          */
 251                         TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes);
 252                         pair->func->invalid(vmm, pgt->pt[0], pteb, ptes);
 253                 }
 254         }
 255 }
 256
 257 static bool
 258 nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 259 {
 260         const struct nvkm_vmm_desc *desc = it->desc;
 261         const int type = desc->type == SPT;
 262         struct nvkm_vmm_pt *pgt = it->pt[0];
 263         bool dma;
 264
 265         if (pfn) {
 266                 /* Need to clear PTE valid bits before we dma_unmap_page(). */
 267                 dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);
 268                 if (dma) {
 269                         /* GPU may have cached the PT, flush before unmap. */
 270                         nvkm_vmm_flush_mark(it);
 271                         nvkm_vmm_flush(it);
 272                         desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);
 273                 }
 274         }
 275
 276         /* Drop PTE references. */
 277         pgt->refs[type] -= ptes;
 278
 279         /* Dual-PTs need special handling, unless PDE becoming invalid. */
 280         if (desc->type == SPT && (pgt->refs[0] || pgt->refs[1]))
 281                 nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);
 282
 283         /* PT no longer neeed?  Destroy it. */
 284         if (!pgt->refs[type]) {
 285                 it->lvl++;
 286                 TRA(it, "%s empty", nvkm_vmm_desc_type(desc));
 287                 it->lvl--;
 288                 nvkm_vmm_unref_pdes(it);
 289                 return false; /* PTE writes for unmap() not necessary. */
 290         }
 291
 292         return true;
 293 }
 294
 295 static void
 296 nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt,
 297                    const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)
 298 {
 299         const struct nvkm_vmm_desc *pair = it->page[-1].desc;
 300         const u32 sptb = desc->bits - pair->bits;
 301         const u32 sptn = 1 << sptb;
 302         struct nvkm_vmm *vmm = it->vmm;
 303         u32 spti = ptei & (sptn - 1), lpti, pteb;
 304
 305         /* Determine how many SPTEs are being touched under each LPTE,
 306          * and increase reference counts.
 307          */
 308         for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {
 309                 const u32 pten = min(sptn - spti, ptes);
 310                 pgt->pte[lpti] += pten;
 311                 ptes -= pten;
 312         }
 313
 314         /* We're done here if there's no corresponding LPT. */
 315         if (!pgt->refs[0])
 316                 return;
 317
 318         for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {
 319                 /* Skip over any LPTEs that already have valid SPTEs. */
 320                 if (pgt->pte[pteb] & NVKM_VMM_PTE_VALID) {
 321                         for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
 322                                 if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID))
 323                                         break;
 324                         }
 325                         continue;
 326                 }
 327
 328                 /* As there are now non-UNMAPPED SPTEs in the range covered
 329                  * by a number of LPTEs, we need to transfer control of the
 330                  * address range to the SPTEs.
 331                  *
 332                  * Determine how many LPTEs need to transition state.
 333                  */
 334                 pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
 335                 for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {
 336                         if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID)
 337                                 break;
 338                         pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;
 339                 }
 340
 341                 if (pgt->pte[pteb] & NVKM_VMM_PTE_SPARSE) {
 342                         const u32 spti = pteb * sptn;
 343                         const u32 sptc = ptes * sptn;
 344                         /* The entire LPTE is marked as sparse, we need
 345                          * to make sure that the SPTEs are too.
 346                          */
 347                         TRA(it, "SPTE %05x: U -> S %d PTEs", spti, sptc);
 348                         desc->func->sparse(vmm, pgt->pt[1], spti, sptc);
 349                         /* Sparse LPTEs prevent SPTEs from being accessed. */
 350                         TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes);
 351                         pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
 352                 } else
 353                 if (pair->func->invalid) {
 354                         /* MMU supports blocking SPTEs by marking an LPTE
 355                          * as INVALID.  We need to reverse that here.
 356                          */
 357                         TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes);
 358                         pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);
 359                 }
 360         }
 361 }
 362
 363 static bool
 364 nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 365 {
 366         const struct nvkm_vmm_desc *desc = it->desc;
 367         const int type = desc->type == SPT;
 368         struct nvkm_vmm_pt *pgt = it->pt[0];
 369
 370         /* Take PTE references. */
 371         pgt->refs[type] += ptes;
 372
 373         /* Dual-PTs need special handling. */
 374         if (desc->type == SPT)
 375                 nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);
 376
 377         return true;
 378 }
 379
 380 static void
 381 nvkm_vmm_sparse_ptes(const struct nvkm_vmm_desc *desc,
 382                      struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)
 383 {
 384         if (desc->type == PGD) {
 385                 while (ptes--)
 386                         pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;
 387         } else
 388         if (desc->type == LPT) {
 389                 memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes);
 390         }
 391 }
 392
 393 static bool
 394 nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 395 {
 396         struct nvkm_vmm_pt *pt = it->pt[0];
 397         if (it->desc->type == PGD)
 398                 memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes);
 399         else
 400         if (it->desc->type == LPT)
 401                 memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);
 402         return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes);
 403 }
 404
 405 static bool
 406 nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
 407 {
 408         nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);
 409         return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes);
 410 }
 411
 412 static bool
 413 nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
 414 {
 415         const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
 416         const int type = desc->type == SPT;
 417         struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
 418         const bool zero = !pgt->sparse && !desc->func->invalid;
 419         struct nvkm_vmm *vmm = it->vmm;
 420         struct nvkm_mmu *mmu = vmm->mmu;
 421         struct nvkm_mmu_pt *pt;
 422         u32 pten = 1 << desc->bits;
 423         u32 pteb, ptei, ptes;
 424         u32 size = desc->size * pten;
 425
 426         pgd->refs[0]++;
 427
 428         pgt->pt[type] = nvkm_mmu_ptc_get(mmu, size, desc->align, zero);
 429         if (!pgt->pt[type]) {
 430                 it->lvl--;
 431                 nvkm_vmm_unref_pdes(it);
 432                 return false;
 433         }
 434
 435         if (zero)
 436                 goto done;
 437
 438         pt = pgt->pt[type];
 439
 440         if (desc->type == LPT && pgt->refs[1]) {
 441                 /* SPT already exists covering the same range as this LPT,
 442                  * which means we need to be careful that any LPTEs which
 443                  * overlap valid SPTEs are unmapped as opposed to invalid
 444                  * or sparse, which would prevent the MMU from looking at
 445                  * the SPTEs on some GPUs.
 446                  */
 447                 for (ptei = pteb = 0; ptei < pten; pteb = ptei) {
 448                         bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
 449                         for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) {
 450                                 bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;
 451                                 if (spte != next)
 452                                         break;
 453                         }
 454
 455                         if (!spte) {
 456                                 if (pgt->sparse)
 457                                         desc->func->sparse(vmm, pt, pteb, ptes);
 458                                 else
 459                                         desc->func->invalid(vmm, pt, pteb, ptes);
 460                                 memset(&pgt->pte[pteb], 0x00, ptes);
 461                         } else {
 462                                 desc->func->unmap(vmm, pt, pteb, ptes);
 463                                 while (ptes--)
 464                                         pgt->pte[pteb++] |= NVKM_VMM_PTE_VALID;
 465                         }
 466                 }
 467         } else {
 468                 if (pgt->sparse) {
 469                         nvkm_vmm_sparse_ptes(desc, pgt, 0, pten);
 470                         desc->func->sparse(vmm, pt, 0, pten);
 471                 } else {
 472                         desc->func->invalid(vmm, pt, 0, pten);
 473                 }
 474         }
 475
 476 done:
 477         TRA(it, "PDE write %s", nvkm_vmm_desc_type(desc));
 478         it->desc[it->lvl].func->pde(it->vmm, pgd, pdei);
 479         nvkm_vmm_flush_mark(it);
 480         return true;
 481 }
 482
 483 static bool
 484 nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei)
 485 {
 486         const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1];
 487         struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
 488
 489         pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page);
 490         if (!pgt) {
 491                 if (!pgd->refs[0])
 492                         nvkm_vmm_unref_pdes(it);
 493                 return false;
 494         }
 495
 496         pgd->pde[pdei] = pgt;
 497         return true;
 498 }
 499
 500 static inline u64
 501 nvkm_vmm_iter(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 502               u64 addr, u64 size, const char *name, bool ref, bool pfn,
 503               bool (*REF_PTES)(struct nvkm_vmm_iter *, bool pfn, u32, u32),
 504               nvkm_vmm_pte_func MAP_PTES, struct nvkm_vmm_map *map,
 505               nvkm_vmm_pxe_func CLR_PTES)
 506 {
 507         const struct nvkm_vmm_desc *desc = page->desc;
 508         struct nvkm_vmm_iter it;
 509         u64 bits = addr >> page->shift;
 510
 511         it.page = page;
 512         it.desc = desc;
 513         it.vmm = vmm;
 514         it.cnt = size >> page->shift;
 515         it.flush = NVKM_VMM_LEVELS_MAX;
 516
 517         /* Deconstruct address into PTE indices for each mapping level. */
 518         for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) {
 519                 it.pte[it.lvl] = bits & ((1 << desc[it.lvl].bits) - 1);
 520                 bits >>= desc[it.lvl].bits;
 521         }
 522         it.max = --it.lvl;
 523         it.pt[it.max] = vmm->pd;
 524
 525         it.lvl = 0;
 526         TRA(&it, "%s: %016llx %016llx %d %lld PTEs", name,
 527                  addr, size, page->shift, it.cnt);
 528         it.lvl = it.max;
 529
 530         /* Depth-first traversal of page tables. */
 531         while (it.cnt) {
 532                 struct nvkm_vmm_pt *pgt = it.pt[it.lvl];
 533                 const int type = desc->type == SPT;
 534                 const u32 pten = 1 << desc->bits;
 535                 const u32 ptei = it.pte[0];
 536                 const u32 ptes = min_t(u64, it.cnt, pten - ptei);
 537
 538                 /* Walk down the tree, finding page tables for each level. */
 539                 for (; it.lvl; it.lvl--) {
 540                         const u32 pdei = it.pte[it.lvl];
 541                         struct nvkm_vmm_pt *pgd = pgt;
 542
 543                         /* Software PT. */
 544                         if (ref && NVKM_VMM_PDE_INVALID(pgd->pde[pdei])) {
 545                                 if (!nvkm_vmm_ref_swpt(&it, pgd, pdei))
 546                                         goto fail;
 547                         }
 548                         it.pt[it.lvl - 1] = pgt = pgd->pde[pdei];
 549
 550                         /* Hardware PT.
 551                          *
 552                          * This is a separate step from above due to GF100 and
 553                          * newer having dual page tables at some levels, which
 554                          * are refcounted independently.
 555                          */
 556                         if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) {
 557                                 if (!nvkm_vmm_ref_hwpt(&it, pgd, pdei))
 558                                         goto fail;
 559                         }
 560                 }
 561
 562                 /* Handle PTE updates. */
 563                 if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) {
 564                         struct nvkm_mmu_pt *pt = pgt->pt[type];
 565                         if (MAP_PTES || CLR_PTES) {
 566                                 if (MAP_PTES)
 567                                         MAP_PTES(vmm, pt, ptei, ptes, map);
 568                                 else
 569                                         CLR_PTES(vmm, pt, ptei, ptes);
 570                                 nvkm_vmm_flush_mark(&it);
 571                         }
 572                 }
 573
 574                 /* Walk back up the tree to the next position. */
 575                 it.pte[it.lvl] += ptes;
 576                 it.cnt -= ptes;
 577                 if (it.cnt) {
 578                         while (it.pte[it.lvl] == (1 << desc[it.lvl].bits)) {
 579                                 it.pte[it.lvl++] = 0;
 580                                 it.pte[it.lvl]++;
 581                         }
 582                 }
 583         };
 584
 585         nvkm_vmm_flush(&it);
 586         return ~0ULL;
 587
 588 fail:
 589         /* Reconstruct the failure address so the caller is able to
 590          * reverse any partially completed operations.
 591          */
 592         addr = it.pte[it.max--];
 593         do {
 594                 addr  = addr << desc[it.max].bits;
 595                 addr |= it.pte[it.max];
 596         } while (it.max--);
 597
 598         return addr << page->shift;
 599 }
 600
 601 static void
 602 nvkm_vmm_ptes_sparse_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 603                          u64 addr, u64 size)
 604 {
 605         nvkm_vmm_iter(vmm, page, addr, size, "sparse unref", false, false,
 606                       nvkm_vmm_sparse_unref_ptes, NULL, NULL,
 607                       page->desc->func->invalid ?
 608                       page->desc->func->invalid : page->desc->func->unmap);
 609 }
 610
 611 static int
 612 nvkm_vmm_ptes_sparse_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 613                          u64 addr, u64 size)
 614 {
 615         if ((page->type & NVKM_VMM_PAGE_SPARSE)) {
 616                 u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "sparse ref",
 617                                          true, false, nvkm_vmm_sparse_ref_ptes,
 618                                          NULL, NULL, page->desc->func->sparse);
 619                 if (fail != ~0ULL) {
 620                         if ((size = fail - addr))
 621                                 nvkm_vmm_ptes_sparse_put(vmm, page, addr, size);
 622                         return -ENOMEM;
 623                 }
 624                 return 0;
 625         }
 626         return -EINVAL;
 627 }
 628
 629 static int
 630 nvkm_vmm_ptes_sparse(struct nvkm_vmm *vmm, u64 addr, u64 size, bool ref)
 631 {
 632         const struct nvkm_vmm_page *page = vmm->func->page;
 633         int m = 0, i;
 634         u64 start = addr;
 635         u64 block;
 636
 637         while (size) {
 638                 /* Limit maximum page size based on remaining size. */
 639                 while (size < (1ULL << page[m].shift))
 640                         m++;
 641                 i = m;
 642
 643                 /* Find largest page size suitable for alignment. */
 644                 while (!IS_ALIGNED(addr, 1ULL << page[i].shift))
 645                         i++;
 646
 647                 /* Determine number of PTEs at this page size. */
 648                 if (i != m) {
 649                         /* Limited to alignment boundary of next page size. */
 650                         u64 next = 1ULL << page[i - 1].shift;
 651                         u64 part = ALIGN(addr, next) - addr;
 652                         if (size - part >= next)
 653                                 block = (part >> page[i].shift) << page[i].shift;
 654                         else
 655                                 block = (size >> page[i].shift) << page[i].shift;
 656                 } else {
 657                         block = (size >> page[i].shift) << page[i].shift;
 658                 }
 659
 660                 /* Perform operation. */
 661                 if (ref) {
 662                         int ret = nvkm_vmm_ptes_sparse_get(vmm, &page[i], addr, block);
 663                         if (ret) {
 664                                 if ((size = addr - start))
 665                                         nvkm_vmm_ptes_sparse(vmm, start, size, false);
 666                                 return ret;
 667                         }
 668                 } else {
 669                         nvkm_vmm_ptes_sparse_put(vmm, &page[i], addr, block);
 670                 }
 671
 672                 size -= block;
 673                 addr += block;
 674         }
 675
 676         return 0;
 677 }
 678
 679 static void
 680 nvkm_vmm_ptes_unmap_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 681                         u64 addr, u64 size, bool sparse, bool pfn)
 682 {
 683         const struct nvkm_vmm_desc_func *func = page->desc->func;
 684         nvkm_vmm_iter(vmm, page, addr, size, "unmap + unref",
 685                       false, pfn, nvkm_vmm_unref_ptes, NULL, NULL,
 686                       sparse ? func->sparse : func->invalid ? func->invalid :
 687                                                               func->unmap);
 688 }
 689
 690 static int
 691 nvkm_vmm_ptes_get_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 692                       u64 addr, u64 size, struct nvkm_vmm_map *map,
 693                       nvkm_vmm_pte_func func)
 694 {
 695         u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref + map", true,
 696                                  false, nvkm_vmm_ref_ptes, func, map, NULL);
 697         if (fail != ~0ULL) {
 698                 if ((size = fail - addr))
 699                         nvkm_vmm_ptes_unmap_put(vmm, page, addr, size, false, false);
 700                 return -ENOMEM;
 701         }
 702         return 0;
 703 }
 704
 705 static void
 706 nvkm_vmm_ptes_unmap(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 707                     u64 addr, u64 size, bool sparse, bool pfn)
 708 {
 709         const struct nvkm_vmm_desc_func *func = page->desc->func;
 710         nvkm_vmm_iter(vmm, page, addr, size, "unmap", false, pfn,
 711                       NULL, NULL, NULL,
 712                       sparse ? func->sparse : func->invalid ? func->invalid :
 713                                                               func->unmap);
 714 }
 715
 716 static void
 717 nvkm_vmm_ptes_map(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 718                   u64 addr, u64 size, struct nvkm_vmm_map *map,
 719                   nvkm_vmm_pte_func func)
 720 {
 721         nvkm_vmm_iter(vmm, page, addr, size, "map", false, false,
 722                       NULL, func, map, NULL);
 723 }
 724
 725 static void
 726 nvkm_vmm_ptes_put(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 727                   u64 addr, u64 size)
 728 {
 729         nvkm_vmm_iter(vmm, page, addr, size, "unref", false, false,
 730                       nvkm_vmm_unref_ptes, NULL, NULL, NULL);
 731 }
 732
 733 static int
 734 nvkm_vmm_ptes_get(struct nvkm_vmm *vmm, const struct nvkm_vmm_page *page,
 735                   u64 addr, u64 size)
 736 {
 737         u64 fail = nvkm_vmm_iter(vmm, page, addr, size, "ref", true, false,
 738                                  nvkm_vmm_ref_ptes, NULL, NULL, NULL);
 739         if (fail != ~0ULL) {
 740                 if (fail != addr)
 741                         nvkm_vmm_ptes_put(vmm, page, addr, fail - addr);
 742                 return -ENOMEM;
 743         }
 744         return 0;
 745 }
 746
 747 static inline struct nvkm_vma *
 748 nvkm_vma_new(u64 addr, u64 size)
 749 {
 750         struct nvkm_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
 751         if (vma) {
 752                 vma->addr = addr;
 753                 vma->size = size;
 754                 vma->page = NVKM_VMA_PAGE_NONE;
 755                 vma->refd = NVKM_VMA_PAGE_NONE;
 756         }
 757         return vma;
 758 }
 759
 760 struct nvkm_vma *
 761 nvkm_vma_tail(struct nvkm_vma *vma, u64 tail)
 762 {
 763         struct nvkm_vma *new;
 764
 765         BUG_ON(vma->size == tail);
 766
 767         if (!(new = nvkm_vma_new(vma->addr + (vma->size - tail), tail)))
 768                 return NULL;
 769         vma->size -= tail;
 770
 771         new->mapref = vma->mapref;
 772         new->sparse = vma->sparse;
 773         new->page = vma->page;
 774         new->refd = vma->refd;
 775         new->used = vma->used;
 776         new->part = vma->part;
 777         new->user = vma->user;
 778         new->busy = vma->busy;
 779         new->mapped = vma->mapped;
 780         list_add(&new->head, &vma->head);
 781         return new;
 782 }
 783
 784 static inline void
 785 nvkm_vmm_free_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 786 {
 787         rb_erase(&vma->tree, &vmm->free);
 788 }
 789
 790 static inline void
 791 nvkm_vmm_free_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 792 {
 793         nvkm_vmm_free_remove(vmm, vma);
 794         list_del(&vma->head);
 795         kfree(vma);
 796 }
 797
 798 static void
 799 nvkm_vmm_free_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 800 {
 801         struct rb_node **ptr = &vmm->free.rb_node;
 802         struct rb_node *parent = NULL;
 803
 804         while (*ptr) {
 805                 struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
 806                 parent = *ptr;
 807                 if (vma->size < this->size)
 808                         ptr = &parent->rb_left;
 809                 else
 810                 if (vma->size > this->size)
 811                         ptr = &parent->rb_right;
 812                 else
 813                 if (vma->addr < this->addr)
 814                         ptr = &parent->rb_left;
 815                 else
 816                 if (vma->addr > this->addr)
 817                         ptr = &parent->rb_right;
 818                 else
 819                         BUG();
 820         }
 821
 822         rb_link_node(&vma->tree, parent, ptr);
 823         rb_insert_color(&vma->tree, &vmm->free);
 824 }
 825
 826 static inline void
 827 nvkm_vmm_node_remove(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 828 {
 829         rb_erase(&vma->tree, &vmm->root);
 830 }
 831
 832 static inline void
 833 nvkm_vmm_node_delete(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 834 {
 835         nvkm_vmm_node_remove(vmm, vma);
 836         list_del(&vma->head);
 837         kfree(vma);
 838 }
 839
 840 static void
 841 nvkm_vmm_node_insert(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
 842 {
 843         struct rb_node **ptr = &vmm->root.rb_node;
 844         struct rb_node *parent = NULL;
 845
 846         while (*ptr) {
 847                 struct nvkm_vma *this = rb_entry(*ptr, typeof(*this), tree);
 848                 parent = *ptr;
 849                 if (vma->addr < this->addr)
 850                         ptr = &parent->rb_left;
 851                 else
 852                 if (vma->addr > this->addr)
 853                         ptr = &parent->rb_right;
 854                 else
 855                         BUG();
 856         }
 857
 858         rb_link_node(&vma->tree, parent, ptr);
 859         rb_insert_color(&vma->tree, &vmm->root);
 860 }
 861
 862 struct nvkm_vma *
 863 nvkm_vmm_node_search(struct nvkm_vmm *vmm, u64 addr)
 864 {
 865         struct rb_node *node = vmm->root.rb_node;
 866         while (node) {
 867                 struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
 868                 if (addr < vma->addr)
 869                         node = node->rb_left;
 870                 else
 871                 if (addr >= vma->addr + vma->size)
 872                         node = node->rb_right;
 873                 else
 874                         return vma;
 875         }
 876         return NULL;
 877 }
 878
 879 #define node(root, dir) (((root)->head.dir == &vmm->list) ? NULL :             \
 880         list_entry((root)->head.dir, struct nvkm_vma, head))
 881
 882 static struct nvkm_vma *
 883 nvkm_vmm_node_merge(struct nvkm_vmm *vmm, struct nvkm_vma *prev,
 884                     struct nvkm_vma *vma, struct nvkm_vma *next, u64 size)
 885 {
 886         if (next) {
 887                 if (vma->size == size) {
 888                         vma->size += next->size;
 889                         nvkm_vmm_node_delete(vmm, next);
 890                         if (prev) {
 891                                 prev->size += vma->size;
 892                                 nvkm_vmm_node_delete(vmm, vma);
 893                                 return prev;
 894                         }
 895                         return vma;
 896                 }
 897                 BUG_ON(prev);
 898
 899                 nvkm_vmm_node_remove(vmm, next);
 900                 vma->size -= size;
 901                 next->addr -= size;
 902                 next->size += size;
 903                 nvkm_vmm_node_insert(vmm, next);
 904                 return next;
 905         }
 906
 907         if (prev) {
 908                 if (vma->size != size) {
 909                         nvkm_vmm_node_remove(vmm, vma);
 910                         prev->size += size;
 911                         vma->addr += size;
 912                         vma->size -= size;
 913                         nvkm_vmm_node_insert(vmm, vma);
 914                 } else {
 915                         prev->size += vma->size;
 916                         nvkm_vmm_node_delete(vmm, vma);
 917                 }
 918                 return prev;
 919         }
 920
 921         return vma;
 922 }
 923
 924 struct nvkm_vma *
 925 nvkm_vmm_node_split(struct nvkm_vmm *vmm,
 926                     struct nvkm_vma *vma, u64 addr, u64 size)
 927 {
 928         struct nvkm_vma *prev = NULL;
 929
 930         if (vma->addr != addr) {
 931                 prev = vma;
 932                 if (!(vma = nvkm_vma_tail(vma, vma->size + vma->addr - addr)))
 933                         return NULL;
 934                 vma->part = true;
 935                 nvkm_vmm_node_insert(vmm, vma);
 936         }
 937
 938         if (vma->size != size) {
 939                 struct nvkm_vma *tmp;
 940                 if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
 941                         nvkm_vmm_node_merge(vmm, prev, vma, NULL, vma->size);
 942                         return NULL;
 943                 }
 944                 tmp->part = true;
 945                 nvkm_vmm_node_insert(vmm, tmp);
 946         }
 947
 948         return vma;
 949 }
 950
 951 static void
 952 nvkm_vma_dump(struct nvkm_vma *vma)
 953 {
 954         printk(KERN_ERR "%016llx %016llx %c%c%c%c%c%c%c%c%c %p\n",
 955                vma->addr, (u64)vma->size,
 956                vma->used ? '-' : 'F',
 957                vma->mapref ? 'R' : '-',
 958                vma->sparse ? 'S' : '-',
 959                vma->page != NVKM_VMA_PAGE_NONE ? '0' + vma->page : '-',
 960                vma->refd != NVKM_VMA_PAGE_NONE ? '0' + vma->refd : '-',
 961                vma->part ? 'P' : '-',
 962                vma->user ? 'U' : '-',
 963                vma->busy ? 'B' : '-',
 964                vma->mapped ? 'M' : '-',
 965                vma->memory);
 966 }
 967
 968 static void
 969 nvkm_vmm_dump(struct nvkm_vmm *vmm)
 970 {
 971         struct nvkm_vma *vma;
 972         list_for_each_entry(vma, &vmm->list, head) {
 973                 nvkm_vma_dump(vma);
 974         }
 975 }
 976
 977 static void
 978 nvkm_vmm_dtor(struct nvkm_vmm *vmm)
 979 {
 980         struct nvkm_vma *vma;
 981         struct rb_node *node;
 982
 983         if (0)
 984                 nvkm_vmm_dump(vmm);
 985
 986         while ((node = rb_first(&vmm->root))) {
 987                 struct nvkm_vma *vma = rb_entry(node, typeof(*vma), tree);
 988                 nvkm_vmm_put(vmm, &vma);
 989         }
 990
 991         if (vmm->bootstrapped) {
 992                 const struct nvkm_vmm_page *page = vmm->func->page;
 993                 const u64 limit = vmm->limit - vmm->start;
 994
 995                 while (page[1].shift)
 996                         page++;
 997
 998                 nvkm_mmu_ptc_dump(vmm->mmu);
 999                 nvkm_vmm_ptes_put(vmm, page, vmm->start, limit);
1000         }
1001
1002         vma = list_first_entry(&vmm->list, typeof(*vma), head);
1003         list_del(&vma->head);
1004         kfree(vma);
1005         WARN_ON(!list_empty(&vmm->list));
1006
1007         if (vmm->nullp) {
1008                 dma_free_coherent(vmm->mmu->subdev.device->dev, 16 * 1024,
1009                                   vmm->nullp, vmm->null);
1010         }
1011
1012         if (vmm->pd) {
1013                 nvkm_mmu_ptc_put(vmm->mmu, true, &vmm->pd->pt[0]);
1014                 nvkm_vmm_pt_del(&vmm->pd);
1015         }
1016 }
1017
1018 static int
1019 nvkm_vmm_ctor_managed(struct nvkm_vmm *vmm, u64 addr, u64 size)
1020 {
1021         struct nvkm_vma *vma;
1022         if (!(vma = nvkm_vma_new(addr, size)))
1023                 return -ENOMEM;
1024         vma->mapref = true;
1025         vma->sparse = false;
1026         vma->used = true;
1027         vma->user = true;
1028         nvkm_vmm_node_insert(vmm, vma);
1029         list_add_tail(&vma->head, &vmm->list);
1030         return 0;
1031 }
1032
1033 int
1034 nvkm_vmm_ctor(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
1035               u32 pd_header, bool managed, u64 addr, u64 size,
1036               struct lock_class_key *key, const char *name,
1037               struct nvkm_vmm *vmm)
1038 {
1039         static struct lock_class_key _key;
1040         const struct nvkm_vmm_page *page = func->page;
1041         const struct nvkm_vmm_desc *desc;
1042         struct nvkm_vma *vma;
1043         int levels, bits = 0, ret;
1044
1045         vmm->func = func;
1046         vmm->mmu = mmu;
1047         vmm->name = name;
1048         vmm->debug = mmu->subdev.debug;
1049         kref_init(&vmm->kref);
1050
1051         __mutex_init(&vmm->mutex, "&vmm->mutex", key ? key : &_key);
1052
1053         /* Locate the smallest page size supported by the backend, it will
1054          * have the the deepest nesting of page tables.
1055          */
1056         while (page[1].shift)
1057                 page++;
1058
1059         /* Locate the structure that describes the layout of the top-level
1060          * page table, and determine the number of valid bits in a virtual
1061          * address.
1062          */
1063         for (levels = 0, desc = page->desc; desc->bits; desc++, levels++)
1064                 bits += desc->bits;
1065         bits += page->shift;
1066         desc--;
1067
1068         if (WARN_ON(levels > NVKM_VMM_LEVELS_MAX))
1069                 return -EINVAL;
1070
1071         /* Allocate top-level page table. */
1072         vmm->pd = nvkm_vmm_pt_new(desc, false, NULL);
1073         if (!vmm->pd)
1074                 return -ENOMEM;
1075         vmm->pd->refs[0] = 1;
1076         INIT_LIST_HEAD(&vmm->join);
1077
1078         /* ... and the GPU storage for it, except on Tesla-class GPUs that
1079          * have the PD embedded in the instance structure.
1080          */
1081         if (desc->size) {
1082                 const u32 size = pd_header + desc->size * (1 << desc->bits);
1083                 vmm->pd->pt[0] = nvkm_mmu_ptc_get(mmu, size, desc->align, true);
1084                 if (!vmm->pd->pt[0])
1085                         return -ENOMEM;
1086         }
1087
1088         /* Initialise address-space MM. */
1089         INIT_LIST_HEAD(&vmm->list);
1090         vmm->free = RB_ROOT;
1091         vmm->root = RB_ROOT;
1092
1093         if (managed) {
1094                 /* Address-space will be managed by the client for the most
1095                  * part, except for a specified area where NVKM allocations
1096                  * are allowed to be placed.
1097                  */
1098                 vmm->start = 0;
1099                 vmm->limit = 1ULL << bits;
1100                 if (addr + size < addr || addr + size > vmm->limit)
1101                         return -EINVAL;
1102
1103                 /* Client-managed area before the NVKM-managed area. */
1104                 if (addr && (ret = nvkm_vmm_ctor_managed(vmm, 0, addr)))
1105                         return ret;
1106
1107                 /* NVKM-managed area. */
1108                 if (size) {
1109                         if (!(vma = nvkm_vma_new(addr, size)))
1110                                 return -ENOMEM;
1111                         nvkm_vmm_free_insert(vmm, vma);
1112                         list_add_tail(&vma->head, &vmm->list);
1113                 }
1114
1115                 /* Client-managed area after the NVKM-managed area. */
1116                 addr = addr + size;
1117                 size = vmm->limit - addr;
1118                 if (size && (ret = nvkm_vmm_ctor_managed(vmm, addr, size)))
1119                         return ret;
1120         } else {
1121                 /* Address-space fully managed by NVKM, requiring calls to
1122                  * nvkm_vmm_get()/nvkm_vmm_put() to allocate address-space.
1123                  */
1124                 vmm->start = addr;
1125                 vmm->limit = size ? (addr + size) : (1ULL << bits);
1126                 if (vmm->start > vmm->limit || vmm->limit > (1ULL << bits))
1127                         return -EINVAL;
1128
1129                 if (!(vma = nvkm_vma_new(vmm->start, vmm->limit - vmm->start)))
1130                         return -ENOMEM;
1131
1132                 nvkm_vmm_free_insert(vmm, vma);
1133                 list_add(&vma->head, &vmm->list);
1134         }
1135
1136         return 0;
1137 }
1138
1139 int
1140 nvkm_vmm_new_(const struct nvkm_vmm_func *func, struct nvkm_mmu *mmu,
1141               u32 hdr, bool managed, u64 addr, u64 size,
1142               struct lock_class_key *key, const char *name,
1143               struct nvkm_vmm **pvmm)
1144 {
1145         if (!(*pvmm = kzalloc(sizeof(**pvmm), GFP_KERNEL)))
1146                 return -ENOMEM;
1147         return nvkm_vmm_ctor(func, mmu, hdr, managed, addr, size, key, name, *pvmm);
1148 }
1149
1150 static struct nvkm_vma *
1151 nvkm_vmm_pfn_split_merge(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1152                          u64 addr, u64 size, u8 page, bool map)
1153 {
1154         struct nvkm_vma *prev = NULL;
1155         struct nvkm_vma *next = NULL;
1156
1157         if (vma->addr == addr && vma->part && (prev = node(vma, prev))) {
1158                 if (prev->memory || prev->mapped != map)
1159                         prev = NULL;
1160         }
1161
1162         if (vma->addr + vma->size == addr + size && (next = node(vma, next))) {
1163                 if (!next->part ||
1164                     next->memory || next->mapped != map)
1165                         next = NULL;
1166         }
1167
1168         if (prev || next)
1169                 return nvkm_vmm_node_merge(vmm, prev, vma, next, size);
1170         return nvkm_vmm_node_split(vmm, vma, addr, size);
1171 }
1172
1173 int
1174 nvkm_vmm_pfn_unmap(struct nvkm_vmm *vmm, u64 addr, u64 size)
1175 {
1176         struct nvkm_vma *vma = nvkm_vmm_node_search(vmm, addr);
1177         struct nvkm_vma *next;
1178         u64 limit = addr + size;
1179         u64 start = addr;
1180
1181         if (!vma)
1182                 return -EINVAL;
1183
1184         do {
1185                 if (!vma->mapped || vma->memory)
1186                         continue;
1187
1188                 size = min(limit - start, vma->size - (start - vma->addr));
1189
1190                 nvkm_vmm_ptes_unmap_put(vmm, &vmm->func->page[vma->refd],
1191                                         start, size, false, true);
1192
1193                 next = nvkm_vmm_pfn_split_merge(vmm, vma, start, size, 0, false);
1194                 if (!WARN_ON(!next)) {
1195                         vma = next;
1196                         vma->refd = NVKM_VMA_PAGE_NONE;
1197                         vma->mapped = false;
1198                 }
1199         } while ((vma = node(vma, next)) && (start = vma->addr) < limit);
1200
1201         return 0;
1202 }
1203
1204 /*TODO:
1205  * - Avoid PT readback (for dma_unmap etc), this might end up being dealt
1206  *   with inside HMM, which would be a lot nicer for us to deal with.
1207  * - Multiple page sizes (particularly for huge page support).
1208  * - Support for systems without a 4KiB page size.
1209  */
1210 int
1211 nvkm_vmm_pfn_map(struct nvkm_vmm *vmm, u8 shift, u64 addr, u64 size, u64 *pfn)
1212 {
1213         const struct nvkm_vmm_page *page = vmm->func->page;
1214         struct nvkm_vma *vma, *tmp;
1215         u64 limit = addr + size;
1216         u64 start = addr;
1217         int pm = size >> shift;
1218         int pi = 0;
1219
1220         /* Only support mapping where the page size of the incoming page
1221          * array matches a page size available for direct mapping.
1222          */
1223         while (page->shift && page->shift != shift &&
1224                page->desc->func->pfn == NULL)
1225                 page++;
1226
1227         if (!page->shift || !IS_ALIGNED(addr, 1ULL << shift) ||
1228                             !IS_ALIGNED(size, 1ULL << shift) ||
1229             addr + size < addr || addr + size > vmm->limit) {
1230                 VMM_DEBUG(vmm, "paged map %d %d %016llx %016llx\n",
1231                           shift, page->shift, addr, size);
1232                 return -EINVAL;
1233         }
1234
1235         if (!(vma = nvkm_vmm_node_search(vmm, addr)))
1236                 return -ENOENT;
1237
1238         do {
1239                 bool map = !!(pfn[pi] & NVKM_VMM_PFN_V);
1240                 bool mapped = vma->mapped;
1241                 u64 size = limit - start;
1242                 u64 addr = start;
1243                 int pn, ret = 0;
1244
1245                 /* Narrow the operation window to cover a single action (page
1246                  * should be mapped or not) within a single VMA.
1247                  */
1248                 for (pn = 0; pi + pn < pm; pn++) {
1249                         if (map != !!(pfn[pi + pn] & NVKM_VMM_PFN_V))
1250                                 break;
1251                 }
1252                 size = min_t(u64, size, pn << page->shift);
1253                 size = min_t(u64, size, vma->size + vma->addr - addr);
1254
1255                 /* Reject any operation to unmanaged regions, and areas that
1256                  * have nvkm_memory objects mapped in them already.
1257                  */
1258                 if (!vma->mapref || vma->memory) {
1259                         ret = -EINVAL;
1260                         goto next;
1261                 }
1262
1263                 /* In order to both properly refcount GPU page tables, and
1264                  * prevent "normal" mappings and these direct mappings from
1265                  * interfering with each other, we need to track contiguous
1266                  * ranges that have been mapped with this interface.
1267                  *
1268                  * Here we attempt to either split an existing VMA so we're
1269                  * able to flag the region as either unmapped/mapped, or to
1270                  * merge with adjacent VMAs that are already compatible.
1271                  *
1272                  * If the region is already compatible, nothing is required.
1273                  */
1274                 if (map != mapped) {
1275                         tmp = nvkm_vmm_pfn_split_merge(vmm, vma, addr, size,
1276                                                        page -
1277                                                        vmm->func->page, map);
1278                         if (WARN_ON(!tmp)) {
1279                                 ret = -ENOMEM;
1280                                 goto next;
1281                         }
1282
1283                         if ((tmp->mapped = map))
1284                                 tmp->refd = page - vmm->func->page;
1285                         else
1286                                 tmp->refd = NVKM_VMA_PAGE_NONE;
1287                         vma = tmp;
1288                 }
1289
1290                 /* Update HW page tables. */
1291                 if (map) {
1292                         struct nvkm_vmm_map args;
1293                         args.page = page;
1294                         args.pfn = &pfn[pi];
1295
1296                         if (!mapped) {
1297                                 ret = nvkm_vmm_ptes_get_map(vmm, page, addr,
1298                                                             size, &args, page->
1299                                                             desc->func->pfn);
1300                         } else {
1301                                 nvkm_vmm_ptes_map(vmm, page, addr, size, &args,
1302                                                   page->desc->func->pfn);
1303                         }
1304                 } else {
1305                         if (mapped) {
1306                                 nvkm_vmm_ptes_unmap_put(vmm, page, addr, size,
1307                                                         false, true);
1308                         }
1309                 }
1310
1311 next:
1312                 /* Iterate to next operation. */
1313                 if (vma->addr + vma->size == addr + size)
1314                         vma = node(vma, next);
1315                 start += size;
1316
1317                 if (ret) {
1318                         /* Failure is signalled by clearing the valid bit on
1319                          * any PFN that couldn't be modified as requested.
1320                          */
1321                         while (size) {
1322                                 pfn[pi++] = NVKM_VMM_PFN_NONE;
1323                                 size -= 1 << page->shift;
1324                         }
1325                 } else {
1326                         pi += size >> page->shift;
1327                 }
1328         } while (vma && start < limit);
1329
1330         return 0;
1331 }
1332
1333 void
1334 nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1335 {
1336         struct nvkm_vma *prev = NULL;
1337         struct nvkm_vma *next;
1338
1339         nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1340         nvkm_memory_unref(&vma->memory);
1341         vma->mapped = false;
1342
1343         if (vma->part && (prev = node(vma, prev)) && prev->mapped)
1344                 prev = NULL;
1345         if ((next = node(vma, next)) && (!next->part || next->mapped))
1346                 next = NULL;
1347         nvkm_vmm_node_merge(vmm, prev, vma, next, vma->size);
1348 }
1349
1350 void
1351 nvkm_vmm_unmap_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma, bool pfn)
1352 {
1353         const struct nvkm_vmm_page *page = &vmm->func->page[vma->refd];
1354
1355         if (vma->mapref) {
1356                 nvkm_vmm_ptes_unmap_put(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1357                 vma->refd = NVKM_VMA_PAGE_NONE;
1358         } else {
1359                 nvkm_vmm_ptes_unmap(vmm, page, vma->addr, vma->size, vma->sparse, pfn);
1360         }
1361
1362         nvkm_vmm_unmap_region(vmm, vma);
1363 }
1364
1365 void
1366 nvkm_vmm_unmap(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1367 {
1368         if (vma->memory) {
1369                 mutex_lock(&vmm->mutex);
1370                 nvkm_vmm_unmap_locked(vmm, vma, false);
1371                 mutex_unlock(&vmm->mutex);
1372         }
1373 }
1374
1375 static int
1376 nvkm_vmm_map_valid(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1377                    void *argv, u32 argc, struct nvkm_vmm_map *map)
1378 {
1379         switch (nvkm_memory_target(map->memory)) {
1380         case NVKM_MEM_TARGET_VRAM:
1381                 if (!(map->page->type & NVKM_VMM_PAGE_VRAM)) {
1382                         VMM_DEBUG(vmm, "%d !VRAM", map->page->shift);
1383                         return -EINVAL;
1384                 }
1385                 break;
1386         case NVKM_MEM_TARGET_HOST:
1387         case NVKM_MEM_TARGET_NCOH:
1388                 if (!(map->page->type & NVKM_VMM_PAGE_HOST)) {
1389                         VMM_DEBUG(vmm, "%d !HOST", map->page->shift);
1390                         return -EINVAL;
1391                 }
1392                 break;
1393         default:
1394                 WARN_ON(1);
1395                 return -ENOSYS;
1396         }
1397
1398         if (!IS_ALIGNED(     vma->addr, 1ULL << map->page->shift) ||
1399             !IS_ALIGNED((u64)vma->size, 1ULL << map->page->shift) ||
1400             !IS_ALIGNED(   map->offset, 1ULL << map->page->shift) ||
1401             nvkm_memory_page(map->memory) < map->page->shift) {
1402                 VMM_DEBUG(vmm, "alignment %016llx %016llx %016llx %d %d",
1403                     vma->addr, (u64)vma->size, map->offset, map->page->shift,
1404                     nvkm_memory_page(map->memory));
1405                 return -EINVAL;
1406         }
1407
1408         return vmm->func->valid(vmm, argv, argc, map);
1409 }
1410
1411 static int
1412 nvkm_vmm_map_choose(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1413                     void *argv, u32 argc, struct nvkm_vmm_map *map)
1414 {
1415         for (map->page = vmm->func->page; map->page->shift; map->page++) {
1416                 VMM_DEBUG(vmm, "trying %d", map->page->shift);
1417                 if (!nvkm_vmm_map_valid(vmm, vma, argv, argc, map))
1418                         return 0;
1419         }
1420         return -EINVAL;
1421 }
1422
1423 static int
1424 nvkm_vmm_map_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma,
1425                     void *argv, u32 argc, struct nvkm_vmm_map *map)
1426 {
1427         nvkm_vmm_pte_func func;
1428         int ret;
1429
1430         /* Make sure we won't overrun the end of the memory object. */
1431         if (unlikely(nvkm_memory_size(map->memory) < map->offset + vma->size)) {
1432                 VMM_DEBUG(vmm, "overrun %016llx %016llx %016llx",
1433                           nvkm_memory_size(map->memory),
1434                           map->offset, (u64)vma->size);
1435                 return -EINVAL;
1436         }
1437
1438         /* Check remaining arguments for validity. */
1439         if (vma->page == NVKM_VMA_PAGE_NONE &&
1440             vma->refd == NVKM_VMA_PAGE_NONE) {
1441                 /* Find the largest page size we can perform the mapping at. */
1442                 const u32 debug = vmm->debug;
1443                 vmm->debug = 0;
1444                 ret = nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1445                 vmm->debug = debug;
1446                 if (ret) {
1447                         VMM_DEBUG(vmm, "invalid at any page size");
1448                         nvkm_vmm_map_choose(vmm, vma, argv, argc, map);
1449                         return -EINVAL;
1450                 }
1451         } else {
1452                 /* Page size of the VMA is already pre-determined. */
1453                 if (vma->refd != NVKM_VMA_PAGE_NONE)
1454                         map->page = &vmm->func->page[vma->refd];
1455                 else
1456                         map->page = &vmm->func->page[vma->page];
1457
1458                 ret = nvkm_vmm_map_valid(vmm, vma, argv, argc, map);
1459                 if (ret) {
1460                         VMM_DEBUG(vmm, "invalid %d\n", ret);
1461                         return ret;
1462                 }
1463         }
1464
1465         /* Deal with the 'offset' argument, and fetch the backend function. */
1466         map->off = map->offset;
1467         if (map->mem) {
1468                 for (; map->off; map->mem = map->mem->next) {
1469                         u64 size = (u64)map->mem->length << NVKM_RAM_MM_SHIFT;
1470                         if (size > map->off)
1471                                 break;
1472                         map->off -= size;
1473                 }
1474                 func = map->page->desc->func->mem;
1475         } else
1476         if (map->sgl) {
1477                 for (; map->off; map->sgl = sg_next(map->sgl)) {
1478                         u64 size = sg_dma_len(map->sgl);
1479                         if (size > map->off)
1480                                 break;
1481                         map->off -= size;
1482                 }
1483                 func = map->page->desc->func->sgl;
1484         } else {
1485                 map->dma += map->offset >> PAGE_SHIFT;
1486                 map->off  = map->offset & PAGE_MASK;
1487                 func = map->page->desc->func->dma;
1488         }
1489
1490         /* Perform the map. */
1491         if (vma->refd == NVKM_VMA_PAGE_NONE) {
1492                 ret = nvkm_vmm_ptes_get_map(vmm, map->page, vma->addr, vma->size, map, func);
1493                 if (ret)
1494                         return ret;
1495
1496                 vma->refd = map->page - vmm->func->page;
1497         } else {
1498                 nvkm_vmm_ptes_map(vmm, map->page, vma->addr, vma->size, map, func);
1499         }
1500
1501         nvkm_memory_tags_put(vma->memory, vmm->mmu->subdev.device, &vma->tags);
1502         nvkm_memory_unref(&vma->memory);
1503         vma->memory = nvkm_memory_ref(map->memory);
1504         vma->mapped = true;
1505         vma->tags = map->tags;
1506         return 0;
1507 }
1508
1509 int
1510 nvkm_vmm_map(struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc,
1511              struct nvkm_vmm_map *map)
1512 {
1513         int ret;
1514         mutex_lock(&vmm->mutex);
1515         ret = nvkm_vmm_map_locked(vmm, vma, argv, argc, map);
1516         vma->busy = false;
1517         mutex_unlock(&vmm->mutex);
1518         return ret;
1519 }
1520
1521 static void
1522 nvkm_vmm_put_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1523 {
1524         struct nvkm_vma *prev, *next;
1525
1526         if ((prev = node(vma, prev)) && !prev->used) {
1527                 vma->addr  = prev->addr;
1528                 vma->size += prev->size;
1529                 nvkm_vmm_free_delete(vmm, prev);
1530         }
1531
1532         if ((next = node(vma, next)) && !next->used) {
1533                 vma->size += next->size;
1534                 nvkm_vmm_free_delete(vmm, next);
1535         }
1536
1537         nvkm_vmm_free_insert(vmm, vma);
1538 }
1539
1540 void
1541 nvkm_vmm_put_locked(struct nvkm_vmm *vmm, struct nvkm_vma *vma)
1542 {
1543         const struct nvkm_vmm_page *page = vmm->func->page;
1544         struct nvkm_vma *next = vma;
1545
1546         BUG_ON(vma->part);
1547
1548         if (vma->mapref || !vma->sparse) {
1549                 do {
1550                         const bool mem = next->memory != NULL;
1551                         const bool map = next->mapped;
1552                         const u8  refd = next->refd;
1553                         const u64 addr = next->addr;
1554                         u64 size = next->size;
1555
1556                         /* Merge regions that are in the same state. */
1557                         while ((next = node(next, next)) && next->part &&
1558                                (next->mapped == map) &&
1559                                (next->memory != NULL) == mem &&
1560                                (next->refd == refd))
1561                                 size += next->size;
1562
1563                         if (map) {
1564                                 /* Region(s) are mapped, merge the unmap
1565                                  * and dereference into a single walk of
1566                                  * the page tree.
1567                                  */
1568                                 nvkm_vmm_ptes_unmap_put(vmm, &page[refd], addr,
1569                                                         size, vma->sparse,
1570                                                         !mem);
1571                         } else
1572                         if (refd != NVKM_VMA_PAGE_NONE) {
1573                                 /* Drop allocation-time PTE references. */
1574                                 nvkm_vmm_ptes_put(vmm, &page[refd], addr, size);
1575                         }
1576                 } while (next && next->part);
1577         }
1578
1579         /* Merge any mapped regions that were split from the initial
1580          * address-space allocation back into the allocated VMA, and
1581          * release memory/compression resources.
1582          */
1583         next = vma;
1584         do {
1585                 if (next->mapped)
1586                         nvkm_vmm_unmap_region(vmm, next);
1587         } while ((next = node(vma, next)) && next->part);
1588
1589         if (vma->sparse && !vma->mapref) {
1590                 /* Sparse region that was allocated with a fixed page size,
1591                  * meaning all relevant PTEs were referenced once when the
1592                  * region was allocated, and remained that way, regardless
1593                  * of whether memory was mapped into it afterwards.
1594                  *
1595                  * The process of unmapping, unsparsing, and dereferencing
1596                  * PTEs can be done in a single page tree walk.
1597                  */
1598                 nvkm_vmm_ptes_sparse_put(vmm, &page[vma->refd], vma->addr, vma->size);
1599         } else
1600         if (vma->sparse) {
1601                 /* Sparse region that wasn't allocated with a fixed page size,
1602                  * PTE references were taken both at allocation time (to make
1603                  * the GPU see the region as sparse), and when mapping memory
1604                  * into the region.
1605                  *
1606                  * The latter was handled above, and the remaining references
1607                  * are dealt with here.
1608                  */
1609                 nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, false);
1610         }
1611
1612         /* Remove VMA from the list of allocated nodes. */
1613         nvkm_vmm_node_remove(vmm, vma);
1614
1615         /* Merge VMA back into the free list. */
1616         vma->page = NVKM_VMA_PAGE_NONE;
1617         vma->refd = NVKM_VMA_PAGE_NONE;
1618         vma->used = false;
1619         vma->user = false;
1620         nvkm_vmm_put_region(vmm, vma);
1621 }
1622
1623 void
1624 nvkm_vmm_put(struct nvkm_vmm *vmm, struct nvkm_vma **pvma)
1625 {
1626         struct nvkm_vma *vma = *pvma;
1627         if (vma) {
1628                 mutex_lock(&vmm->mutex);
1629                 nvkm_vmm_put_locked(vmm, vma);
1630                 mutex_unlock(&vmm->mutex);
1631                 *pvma = NULL;
1632         }
1633 }
1634
1635 int
1636 nvkm_vmm_get_locked(struct nvkm_vmm *vmm, bool getref, bool mapref, bool sparse,
1637                     u8 shift, u8 align, u64 size, struct nvkm_vma **pvma)
1638 {
1639         const struct nvkm_vmm_page *page = &vmm->func->page[NVKM_VMA_PAGE_NONE];
1640         struct rb_node *node = NULL, *temp;
1641         struct nvkm_vma *vma = NULL, *tmp;
1642         u64 addr, tail;
1643         int ret;
1644
1645         VMM_TRACE(vmm, "getref %d mapref %d sparse %d "
1646                        "shift: %d align: %d size: %016llx",
1647                   getref, mapref, sparse, shift, align, size);
1648
1649         /* Zero-sized, or lazily-allocated sparse VMAs, make no sense. */
1650         if (unlikely(!size || (!getref && !mapref && sparse))) {
1651                 VMM_DEBUG(vmm, "args %016llx %d %d %d",
1652                           size, getref, mapref, sparse);
1653                 return -EINVAL;
1654         }
1655
1656         /* Tesla-class GPUs can only select page size per-PDE, which means
1657          * we're required to know the mapping granularity up-front to find
1658          * a suitable region of address-space.
1659          *
1660          * The same goes if we're requesting up-front allocation of PTES.
1661          */
1662         if (unlikely((getref || vmm->func->page_block) && !shift)) {
1663                 VMM_DEBUG(vmm, "page size required: %d %016llx",
1664                           getref, vmm->func->page_block);
1665                 return -EINVAL;
1666         }
1667
1668         /* If a specific page size was requested, determine its index and
1669          * make sure the requested size is a multiple of the page size.
1670          */
1671         if (shift) {
1672                 for (page = vmm->func->page; page->shift; page++) {
1673                         if (shift == page->shift)
1674                                 break;
1675                 }
1676
1677                 if (!page->shift || !IS_ALIGNED(size, 1ULL << page->shift)) {
1678                         VMM_DEBUG(vmm, "page %d %016llx", shift, size);
1679                         return -EINVAL;
1680                 }
1681                 align = max_t(u8, align, shift);
1682         } else {
1683                 align = max_t(u8, align, 12);
1684         }
1685
1686         /* Locate smallest block that can possibly satisfy the allocation. */
1687         temp = vmm->free.rb_node;
1688         while (temp) {
1689                 struct nvkm_vma *this = rb_entry(temp, typeof(*this), tree);
1690                 if (this->size < size) {
1691                         temp = temp->rb_right;
1692                 } else {
1693                         node = temp;
1694                         temp = temp->rb_left;
1695                 }
1696         }
1697
1698         if (unlikely(!node))
1699                 return -ENOSPC;
1700
1701         /* Take into account alignment restrictions, trying larger blocks
1702          * in turn until we find a suitable free block.
1703          */
1704         do {
1705                 struct nvkm_vma *this = rb_entry(node, typeof(*this), tree);
1706                 struct nvkm_vma *prev = node(this, prev);
1707                 struct nvkm_vma *next = node(this, next);
1708                 const int p = page - vmm->func->page;
1709
1710                 addr = this->addr;
1711                 if (vmm->func->page_block && prev && prev->page != p)
1712                         addr = ALIGN(addr, vmm->func->page_block);
1713                 addr = ALIGN(addr, 1ULL << align);
1714
1715                 tail = this->addr + this->size;
1716                 if (vmm->func->page_block && next && next->page != p)
1717                         tail = ALIGN_DOWN(tail, vmm->func->page_block);
1718
1719                 if (addr <= tail && tail - addr >= size) {
1720                         nvkm_vmm_free_remove(vmm, this);
1721                         vma = this;
1722                         break;
1723                 }
1724         } while ((node = rb_next(node)));
1725
1726         if (unlikely(!vma))
1727                 return -ENOSPC;
1728
1729         /* If the VMA we found isn't already exactly the requested size,
1730          * it needs to be split, and the remaining free blocks returned.
1731          */
1732         if (addr != vma->addr) {
1733                 if (!(tmp = nvkm_vma_tail(vma, vma->size + vma->addr - addr))) {
1734                         nvkm_vmm_put_region(vmm, vma);
1735                         return -ENOMEM;
1736                 }
1737                 nvkm_vmm_free_insert(vmm, vma);
1738                 vma = tmp;
1739         }
1740
1741         if (size != vma->size) {
1742                 if (!(tmp = nvkm_vma_tail(vma, vma->size - size))) {
1743                         nvkm_vmm_put_region(vmm, vma);
1744                         return -ENOMEM;
1745                 }
1746                 nvkm_vmm_free_insert(vmm, tmp);
1747         }
1748
1749         /* Pre-allocate page tables and/or setup sparse mappings. */
1750         if (sparse && getref)
1751                 ret = nvkm_vmm_ptes_sparse_get(vmm, page, vma->addr, vma->size);
1752         else if (sparse)
1753                 ret = nvkm_vmm_ptes_sparse(vmm, vma->addr, vma->size, true);
1754         else if (getref)
1755                 ret = nvkm_vmm_ptes_get(vmm, page, vma->addr, vma->size);
1756         else
1757                 ret = 0;
1758         if (ret) {
1759                 nvkm_vmm_put_region(vmm, vma);
1760                 return ret;
1761         }
1762
1763         vma->mapref = mapref && !getref;
1764         vma->sparse = sparse;
1765         vma->page = page - vmm->func->page;
1766         vma->refd = getref ? vma->page : NVKM_VMA_PAGE_NONE;
1767         vma->used = true;
1768         nvkm_vmm_node_insert(vmm, vma);
1769         *pvma = vma;
1770         return 0;
1771 }
1772
1773 int
1774 nvkm_vmm_get(struct nvkm_vmm *vmm, u8 page, u64 size, struct nvkm_vma **pvma)
1775 {
1776         int ret;
1777         mutex_lock(&vmm->mutex);
1778         ret = nvkm_vmm_get_locked(vmm, false, true, false, page, 0, size, pvma);
1779         mutex_unlock(&vmm->mutex);
1780         return ret;
1781 }
1782
1783 void
1784 nvkm_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1785 {
1786         if (inst && vmm && vmm->func->part) {
1787                 mutex_lock(&vmm->mutex);
1788                 vmm->func->part(vmm, inst);
1789                 mutex_unlock(&vmm->mutex);
1790         }
1791 }
1792
1793 int
1794 nvkm_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
1795 {
1796         int ret = 0;
1797         if (vmm->func->join) {
1798                 mutex_lock(&vmm->mutex);
1799                 ret = vmm->func->join(vmm, inst);
1800                 mutex_unlock(&vmm->mutex);
1801         }
1802         return ret;
1803 }
1804
1805 static bool
1806 nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)
1807 {
1808         const struct nvkm_vmm_desc *desc = it->desc;
1809         const int type = desc->type == SPT;
1810         nvkm_memory_boot(it->pt[0]->pt[type]->memory, it->vmm);
1811         return false;
1812 }
1813
1814 int
1815 nvkm_vmm_boot(struct nvkm_vmm *vmm)
1816 {
1817         const struct nvkm_vmm_page *page = vmm->func->page;
1818         const u64 limit = vmm->limit - vmm->start;
1819         int ret;
1820
1821         while (page[1].shift)
1822                 page++;
1823
1824         ret = nvkm_vmm_ptes_get(vmm, page, vmm->start, limit);
1825         if (ret)
1826                 return ret;
1827
1828         nvkm_vmm_iter(vmm, page, vmm->start, limit, "bootstrap", false, false,
1829                       nvkm_vmm_boot_ptes, NULL, NULL, NULL);
1830         vmm->bootstrapped = true;
1831         return 0;
1832 }
1833
1834 static void
1835 nvkm_vmm_del(struct kref *kref)
1836 {
1837         struct nvkm_vmm *vmm = container_of(kref, typeof(*vmm), kref);
1838         nvkm_vmm_dtor(vmm);
1839         kfree(vmm);
1840 }
1841
1842 void
1843 nvkm_vmm_unref(struct nvkm_vmm **pvmm)
1844 {
1845         struct nvkm_vmm *vmm = *pvmm;
1846         if (vmm) {
1847                 kref_put(&vmm->kref, nvkm_vmm_del);
1848                 *pvmm = NULL;
1849         }
1850 }
1851
1852 struct nvkm_vmm *
1853 nvkm_vmm_ref(struct nvkm_vmm *vmm)
1854 {
1855         if (vmm)
1856                 kref_get(&vmm->kref);
1857         return vmm;
1858 }
1859
1860 int
1861 nvkm_vmm_new(struct nvkm_device *device, u64 addr, u64 size, void *argv,
1862              u32 argc, struct lock_class_key *key, const char *name,
1863              struct nvkm_vmm **pvmm)
1864 {
1865         struct nvkm_mmu *mmu = device->mmu;
1866         struct nvkm_vmm *vmm = NULL;
1867         int ret;
1868         ret = mmu->func->vmm.ctor(mmu, false, addr, size, argv, argc,
1869                                   key, name, &vmm);
1870         if (ret)
1871                 nvkm_vmm_unref(&vmm);
1872         *pvmm = vmm;
1873         return ret;
1874 }