arch/x86/mm/pageattr.c

   1 /*
   2  * Copyright 2002 Andi Kleen, SuSE Labs.
   3  * Thanks to Ben LaHaise for precious feedback.
   4  */
   5 #include <linux/highmem.h>
   6 #include <linux/bootmem.h>
   7 #include <linux/sched.h>
   8 #include <linux/mm.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/seq_file.h>
  11 #include <linux/debugfs.h>
  12 #include <linux/pfn.h>
  13 #include <linux/percpu.h>
  14 #include <linux/gfp.h>
  15 #include <linux/pci.h>
  16 #include <linux/vmalloc.h>
  17
  18 #include <asm/e820/api.h>
  19 #include <asm/processor.h>
  20 #include <asm/tlbflush.h>
  21 #include <asm/sections.h>
  22 #include <asm/setup.h>
  23 #include <linux/uaccess.h>
  24 #include <asm/pgalloc.h>
  25 #include <asm/proto.h>
  26 #include <asm/pat.h>
  27 #include <asm/set_memory.h>
  28
  29 /*
  30  * The current flushing context - we pass it instead of 5 arguments:
  31  */
  32 struct cpa_data {
  33         unsigned long   *vaddr;
  34         pgd_t           *pgd;
  35         pgprot_t        mask_set;
  36         pgprot_t        mask_clr;
  37         unsigned long   numpages;
  38         int             flags;
  39         unsigned long   pfn;
  40         unsigned        force_split             : 1,
  41                         force_static_prot       : 1;
  42         int             curpage;
  43         struct page     **pages;
  44 };
  45
  46 enum cpa_warn {
  47         CPA_CONFLICT,
  48         CPA_PROTECT,
  49         CPA_DETECT,
  50 };
  51
  52 static const int cpa_warn_level = CPA_PROTECT;
  53
  54 /*
  55  * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings)
  56  * using cpa_lock. So that we don't allow any other cpu, with stale large tlb
  57  * entries change the page attribute in parallel to some other cpu
  58  * splitting a large page entry along with changing the attribute.
  59  */
  60 static DEFINE_SPINLOCK(cpa_lock);
  61
  62 #define CPA_FLUSHTLB 1
  63 #define CPA_ARRAY 2
  64 #define CPA_PAGES_ARRAY 4
  65 #define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */
  66
  67 #ifdef CONFIG_PROC_FS
  68 static unsigned long direct_pages_count[PG_LEVEL_NUM];
  69
  70 void update_page_count(int level, unsigned long pages)
  71 {
  72         /* Protect against CPA */
  73         spin_lock(&pgd_lock);
  74         direct_pages_count[level] += pages;
  75         spin_unlock(&pgd_lock);
  76 }
  77
  78 static void split_page_count(int level)
  79 {
  80         if (direct_pages_count[level] == 0)
  81                 return;
  82
  83         direct_pages_count[level]--;
  84         direct_pages_count[level - 1] += PTRS_PER_PTE;
  85 }
  86
  87 void arch_report_meminfo(struct seq_file *m)
  88 {
  89         seq_printf(m, "DirectMap4k:    %8lu kB\n",
  90                         direct_pages_count[PG_LEVEL_4K] << 2);
  91 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
  92         seq_printf(m, "DirectMap2M:    %8lu kB\n",
  93                         direct_pages_count[PG_LEVEL_2M] << 11);
  94 #else
  95         seq_printf(m, "DirectMap4M:    %8lu kB\n",
  96                         direct_pages_count[PG_LEVEL_2M] << 12);
  97 #endif
  98         if (direct_gbpages)
  99                 seq_printf(m, "DirectMap1G:    %8lu kB\n",
 100                         direct_pages_count[PG_LEVEL_1G] << 20);
 101 }
 102 #else
 103 static inline void split_page_count(int level) { }
 104 #endif
 105
 106 #ifdef CONFIG_X86_CPA_STATISTICS
 107
 108 static unsigned long cpa_1g_checked;
 109 static unsigned long cpa_1g_sameprot;
 110 static unsigned long cpa_1g_preserved;
 111 static unsigned long cpa_2m_checked;
 112 static unsigned long cpa_2m_sameprot;
 113 static unsigned long cpa_2m_preserved;
 114 static unsigned long cpa_4k_checked;
 115 static unsigned long cpa_4k_install;
 116
 117 static inline void cpa_inc_1g_checked(void)
 118 {
 119         cpa_1g_checked++;
 120 }
 121
 122 static inline void cpa_inc_2m_checked(void)
 123 {
 124         cpa_2m_checked++;
 125 }
 126
 127 static inline void cpa_inc_4k_checked(void)
 128 {
 129         cpa_4k_checked++;
 130 }
 131
 132 static inline void cpa_inc_4k_install(void)
 133 {
 134         cpa_4k_install++;
 135 }
 136
 137 static inline void cpa_inc_lp_sameprot(int level)
 138 {
 139         if (level == PG_LEVEL_1G)
 140                 cpa_1g_sameprot++;
 141         else
 142                 cpa_2m_sameprot++;
 143 }
 144
 145 static inline void cpa_inc_lp_preserved(int level)
 146 {
 147         if (level == PG_LEVEL_1G)
 148                 cpa_1g_preserved++;
 149         else
 150                 cpa_2m_preserved++;
 151 }
 152
 153 static int cpastats_show(struct seq_file *m, void *p)
 154 {
 155         seq_printf(m, "1G pages checked:     %16lu\n", cpa_1g_checked);
 156         seq_printf(m, "1G pages sameprot:    %16lu\n", cpa_1g_sameprot);
 157         seq_printf(m, "1G pages preserved:   %16lu\n", cpa_1g_preserved);
 158         seq_printf(m, "2M pages checked:     %16lu\n", cpa_2m_checked);
 159         seq_printf(m, "2M pages sameprot:    %16lu\n", cpa_2m_sameprot);
 160         seq_printf(m, "2M pages preserved:   %16lu\n", cpa_2m_preserved);
 161         seq_printf(m, "4K pages checked:     %16lu\n", cpa_4k_checked);
 162         seq_printf(m, "4K pages set-checked: %16lu\n", cpa_4k_install);
 163         return 0;
 164 }
 165
 166 static int cpastats_open(struct inode *inode, struct file *file)
 167 {
 168         return single_open(file, cpastats_show, NULL);
 169 }
 170
 171 static const struct file_operations cpastats_fops = {
 172         .open           = cpastats_open,
 173         .read           = seq_read,
 174         .llseek         = seq_lseek,
 175         .release        = single_release,
 176 };
 177
 178 static int __init cpa_stats_init(void)
 179 {
 180         debugfs_create_file("cpa_stats", S_IRUSR, arch_debugfs_dir, NULL,
 181                             &cpastats_fops);
 182         return 0;
 183 }
 184 late_initcall(cpa_stats_init);
 185 #else
 186 static inline void cpa_inc_1g_checked(void) { }
 187 static inline void cpa_inc_2m_checked(void) { }
 188 static inline void cpa_inc_4k_checked(void) { }
 189 static inline void cpa_inc_4k_install(void) { }
 190 static inline void cpa_inc_lp_sameprot(int level) { }
 191 static inline void cpa_inc_lp_preserved(int level) { }
 192 #endif
 193
 194
 195 static inline int
 196 within(unsigned long addr, unsigned long start, unsigned long end)
 197 {
 198         return addr >= start && addr < end;
 199 }
 200
 201 static inline int
 202 within_inclusive(unsigned long addr, unsigned long start, unsigned long end)
 203 {
 204         return addr >= start && addr <= end;
 205 }
 206
 207 #ifdef CONFIG_X86_64
 208
 209 static inline unsigned long highmap_start_pfn(void)
 210 {
 211         return __pa_symbol(_text) >> PAGE_SHIFT;
 212 }
 213
 214 static inline unsigned long highmap_end_pfn(void)
 215 {
 216         /* Do not reference physical address outside the kernel. */
 217         return __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT;
 218 }
 219
 220 static bool __cpa_pfn_in_highmap(unsigned long pfn)
 221 {
 222         /*
 223          * Kernel text has an alias mapping at a high address, known
 224          * here as "highmap".
 225          */
 226         return within_inclusive(pfn, highmap_start_pfn(), highmap_end_pfn());
 227 }
 228
 229 #else
 230
 231 static bool __cpa_pfn_in_highmap(unsigned long pfn)
 232 {
 233         /* There is no highmap on 32-bit */
 234         return false;
 235 }
 236
 237 #endif
 238
 239 /*
 240  * Flushing functions
 241  */
 242
 243 /**
 244  * clflush_cache_range - flush a cache range with clflush
 245  * @vaddr:      virtual start address
 246  * @size:       number of bytes to flush
 247  *
 248  * clflushopt is an unordered instruction which needs fencing with mfence or
 249  * sfence to avoid ordering issues.
 250  */
 251 void clflush_cache_range(void *vaddr, unsigned int size)
 252 {
 253         const unsigned long clflush_size = boot_cpu_data.x86_clflush_size;
 254         void *p = (void *)((unsigned long)vaddr & ~(clflush_size - 1));
 255         void *vend = vaddr + size;
 256
 257         if (p >= vend)
 258                 return;
 259
 260         mb();
 261
 262         for (; p < vend; p += clflush_size)
 263                 clflushopt(p);
 264
 265         mb();
 266 }
 267 EXPORT_SYMBOL_GPL(clflush_cache_range);
 268
 269 void arch_invalidate_pmem(void *addr, size_t size)
 270 {
 271         clflush_cache_range(addr, size);
 272 }
 273 EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
 274
 275 static void __cpa_flush_all(void *arg)
 276 {
 277         unsigned long cache = (unsigned long)arg;
 278
 279         /*
 280          * Flush all to work around Errata in early athlons regarding
 281          * large page flushing.
 282          */
 283         __flush_tlb_all();
 284
 285         if (cache && boot_cpu_data.x86 >= 4)
 286                 wbinvd();
 287 }
 288
 289 static void cpa_flush_all(unsigned long cache)
 290 {
 291         BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
 292
 293         on_each_cpu(__cpa_flush_all, (void *) cache, 1);
 294 }
 295
 296 static void __cpa_flush_range(void *arg)
 297 {
 298         /*
 299          * We could optimize that further and do individual per page
 300          * tlb invalidates for a low number of pages. Caveat: we must
 301          * flush the high aliases on 64bit as well.
 302          */
 303         __flush_tlb_all();
 304 }
 305
 306 static void cpa_flush_range(unsigned long start, int numpages, int cache)
 307 {
 308         unsigned int i, level;
 309         unsigned long addr;
 310
 311         BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
 312         WARN_ON(PAGE_ALIGN(start) != start);
 313
 314         on_each_cpu(__cpa_flush_range, NULL, 1);
 315
 316         if (!cache)
 317                 return;
 318
 319         /*
 320          * We only need to flush on one CPU,
 321          * clflush is a MESI-coherent instruction that
 322          * will cause all other CPUs to flush the same
 323          * cachelines:
 324          */
 325         for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
 326                 pte_t *pte = lookup_address(addr, &level);
 327
 328                 /*
 329                  * Only flush present addresses:
 330                  */
 331                 if (pte && (pte_val(*pte) & _PAGE_PRESENT))
 332                         clflush_cache_range((void *) addr, PAGE_SIZE);
 333         }
 334 }
 335
 336 static void cpa_flush_array(unsigned long *start, int numpages, int cache,
 337                             int in_flags, struct page **pages)
 338 {
 339         unsigned int i, level;
 340 #ifdef CONFIG_PREEMPT
 341         /*
 342          * Avoid wbinvd() because it causes latencies on all CPUs,
 343          * regardless of any CPU isolation that may be in effect.
 344          *
 345          * This should be extended for CAT enabled systems independent of
 346          * PREEMPT because wbinvd() does not respect the CAT partitions and
 347          * this is exposed to unpriviledged users through the graphics
 348          * subsystem.
 349          */
 350         unsigned long do_wbinvd = 0;
 351 #else
 352         unsigned long do_wbinvd = cache && numpages >= 1024; /* 4M threshold */
 353 #endif
 354
 355         BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
 356
 357         on_each_cpu(__cpa_flush_all, (void *) do_wbinvd, 1);
 358
 359         if (!cache || do_wbinvd)
 360                 return;
 361
 362         /*
 363          * We only need to flush on one CPU,
 364          * clflush is a MESI-coherent instruction that
 365          * will cause all other CPUs to flush the same
 366          * cachelines:
 367          */
 368         for (i = 0; i < numpages; i++) {
 369                 unsigned long addr;
 370                 pte_t *pte;
 371
 372                 if (in_flags & CPA_PAGES_ARRAY)
 373                         addr = (unsigned long)page_address(pages[i]);
 374                 else
 375                         addr = start[i];
 376
 377                 pte = lookup_address(addr, &level);
 378
 379                 /*
 380                  * Only flush present addresses:
 381                  */
 382                 if (pte && (pte_val(*pte) & _PAGE_PRESENT))
 383                         clflush_cache_range((void *)addr, PAGE_SIZE);
 384         }
 385 }
 386
 387 static bool overlaps(unsigned long r1_start, unsigned long r1_end,
 388                      unsigned long r2_start, unsigned long r2_end)
 389 {
 390         return (r1_start <= r2_end && r1_end >= r2_start) ||
 391                 (r2_start <= r1_end && r2_end >= r1_start);
 392 }
 393
 394 #ifdef CONFIG_PCI_BIOS
 395 /*
 396  * The BIOS area between 640k and 1Mb needs to be executable for PCI BIOS
 397  * based config access (CONFIG_PCI_GOBIOS) support.
 398  */
 399 #define BIOS_PFN        PFN_DOWN(BIOS_BEGIN)
 400 #define BIOS_PFN_END    PFN_DOWN(BIOS_END - 1)
 401
 402 static pgprotval_t protect_pci_bios(unsigned long spfn, unsigned long epfn)
 403 {
 404         if (pcibios_enabled && overlaps(spfn, epfn, BIOS_PFN, BIOS_PFN_END))
 405                 return _PAGE_NX;
 406         return 0;
 407 }
 408 #else
 409 static pgprotval_t protect_pci_bios(unsigned long spfn, unsigned long epfn)
 410 {
 411         return 0;
 412 }
 413 #endif
 414
 415 /*
 416  * The .rodata section needs to be read-only. Using the pfn catches all
 417  * aliases.  This also includes __ro_after_init, so do not enforce until
 418  * kernel_set_to_readonly is true.
 419  */
 420 static pgprotval_t protect_rodata(unsigned long spfn, unsigned long epfn)
 421 {
 422         unsigned long epfn_ro, spfn_ro = PFN_DOWN(__pa_symbol(__start_rodata));
 423
 424         /*
 425          * Note: __end_rodata is at page aligned and not inclusive, so
 426          * subtract 1 to get the last enforced PFN in the rodata area.
 427          */
 428         epfn_ro = PFN_DOWN(__pa_symbol(__end_rodata)) - 1;
 429
 430         if (kernel_set_to_readonly && overlaps(spfn, epfn, spfn_ro, epfn_ro))
 431                 return _PAGE_RW;
 432         return 0;
 433 }
 434
 435 /*
 436  * Protect kernel text against becoming non executable by forbidding
 437  * _PAGE_NX.  This protects only the high kernel mapping (_text -> _etext)
 438  * out of which the kernel actually executes.  Do not protect the low
 439  * mapping.
 440  *
 441  * This does not cover __inittext since that is gone after boot.
 442  */
 443 static pgprotval_t protect_kernel_text(unsigned long start, unsigned long end)
 444 {
 445         unsigned long t_end = (unsigned long)_etext - 1;
 446         unsigned long t_start = (unsigned long)_text;
 447
 448         if (overlaps(start, end, t_start, t_end))
 449                 return _PAGE_NX;
 450         return 0;
 451 }
 452
 453 #if defined(CONFIG_X86_64)
 454 /*
 455  * Once the kernel maps the text as RO (kernel_set_to_readonly is set),
 456  * kernel text mappings for the large page aligned text, rodata sections
 457  * will be always read-only. For the kernel identity mappings covering the
 458  * holes caused by this alignment can be anything that user asks.
 459  *
 460  * This will preserve the large page mappings for kernel text/data at no
 461  * extra cost.
 462  */
 463 static pgprotval_t protect_kernel_text_ro(unsigned long start,
 464                                           unsigned long end)
 465 {
 466         unsigned long t_end = (unsigned long)__end_rodata_hpage_align - 1;
 467         unsigned long t_start = (unsigned long)_text;
 468         unsigned int level;
 469
 470         if (!kernel_set_to_readonly || !overlaps(start, end, t_start, t_end))
 471                 return 0;
 472         /*
 473          * Don't enforce the !RW mapping for the kernel text mapping, if
 474          * the current mapping is already using small page mapping.  No
 475          * need to work hard to preserve large page mappings in this case.
 476          *
 477          * This also fixes the Linux Xen paravirt guest boot failure caused
 478          * by unexpected read-only mappings for kernel identity
 479          * mappings. In this paravirt guest case, the kernel text mapping
 480          * and the kernel identity mapping share the same page-table pages,
 481          * so the protections for kernel text and identity mappings have to
 482          * be the same.
 483          */
 484         if (lookup_address(start, &level) && (level != PG_LEVEL_4K))
 485                 return _PAGE_RW;
 486         return 0;
 487 }
 488 #else
 489 static pgprotval_t protect_kernel_text_ro(unsigned long start,
 490                                           unsigned long end)
 491 {
 492         return 0;
 493 }
 494 #endif
 495
 496 static inline bool conflicts(pgprot_t prot, pgprotval_t val)
 497 {
 498         return (pgprot_val(prot) & ~val) != pgprot_val(prot);
 499 }
 500
 501 static inline void check_conflict(int warnlvl, pgprot_t prot, pgprotval_t val,
 502                                   unsigned long start, unsigned long end,
 503                                   unsigned long pfn, const char *txt)
 504 {
 505         static const char *lvltxt[] = {
 506                 [CPA_CONFLICT]  = "conflict",
 507                 [CPA_PROTECT]   = "protect",
 508                 [CPA_DETECT]    = "detect",
 509         };
 510
 511         if (warnlvl > cpa_warn_level || !conflicts(prot, val))
 512                 return;
 513
 514         pr_warn("CPA %8s %10s: 0x%016lx - 0x%016lx PFN %lx req %016llx prevent %016llx\n",
 515                 lvltxt[warnlvl], txt, start, end, pfn, (unsigned long long)pgprot_val(prot),
 516                 (unsigned long long)val);
 517 }
 518
 519 /*
 520  * Certain areas of memory on x86 require very specific protection flags,
 521  * for example the BIOS area or kernel text. Callers don't always get this
 522  * right (again, ioremap() on BIOS memory is not uncommon) so this function
 523  * checks and fixes these known static required protection bits.
 524  */
 525 static inline pgprot_t static_protections(pgprot_t prot, unsigned long start,
 526                                           unsigned long pfn, unsigned long npg,
 527                                           int warnlvl)
 528 {
 529         pgprotval_t forbidden, res;
 530         unsigned long end;
 531
 532         /*
 533          * There is no point in checking RW/NX conflicts when the requested
 534          * mapping is setting the page !PRESENT.
 535          */
 536         if (!(pgprot_val(prot) & _PAGE_PRESENT))
 537                 return prot;
 538
 539         /* Operate on the virtual address */
 540         end = start + npg * PAGE_SIZE - 1;
 541
 542         res = protect_kernel_text(start, end);
 543         check_conflict(warnlvl, prot, res, start, end, pfn, "Text NX");
 544         forbidden = res;
 545
 546         res = protect_kernel_text_ro(start, end);
 547         check_conflict(warnlvl, prot, res, start, end, pfn, "Text RO");
 548         forbidden |= res;
 549
 550         /* Check the PFN directly */
 551         res = protect_pci_bios(pfn, pfn + npg - 1);
 552         check_conflict(warnlvl, prot, res, start, end, pfn, "PCIBIOS NX");
 553         forbidden |= res;
 554
 555         res = protect_rodata(pfn, pfn + npg - 1);
 556         check_conflict(warnlvl, prot, res, start, end, pfn, "Rodata RO");
 557         forbidden |= res;
 558
 559         return __pgprot(pgprot_val(prot) & ~forbidden);
 560 }
 561
 562 /*
 563  * Lookup the page table entry for a virtual address in a specific pgd.
 564  * Return a pointer to the entry and the level of the mapping.
 565  */
 566 pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
 567                              unsigned int *level)
 568 {
 569         p4d_t *p4d;
 570         pud_t *pud;
 571         pmd_t *pmd;
 572
 573         *level = PG_LEVEL_NONE;
 574
 575         if (pgd_none(*pgd))
 576                 return NULL;
 577
 578         p4d = p4d_offset(pgd, address);
 579         if (p4d_none(*p4d))
 580                 return NULL;
 581
 582         *level = PG_LEVEL_512G;
 583         if (p4d_large(*p4d) || !p4d_present(*p4d))
 584                 return (pte_t *)p4d;
 585
 586         pud = pud_offset(p4d, address);
 587         if (pud_none(*pud))
 588                 return NULL;
 589
 590         *level = PG_LEVEL_1G;
 591         if (pud_large(*pud) || !pud_present(*pud))
 592                 return (pte_t *)pud;
 593
 594         pmd = pmd_offset(pud, address);
 595         if (pmd_none(*pmd))
 596                 return NULL;
 597
 598         *level = PG_LEVEL_2M;
 599         if (pmd_large(*pmd) || !pmd_present(*pmd))
 600                 return (pte_t *)pmd;
 601
 602         *level = PG_LEVEL_4K;
 603
 604         return pte_offset_kernel(pmd, address);
 605 }
 606
 607 /*
 608  * Lookup the page table entry for a virtual address. Return a pointer
 609  * to the entry and the level of the mapping.
 610  *
 611  * Note: We return pud and pmd either when the entry is marked large
 612  * or when the present bit is not set. Otherwise we would return a
 613  * pointer to a nonexisting mapping.
 614  */
 615 pte_t *lookup_address(unsigned long address, unsigned int *level)
 616 {
 617         return lookup_address_in_pgd(pgd_offset_k(address), address, level);
 618 }
 619 EXPORT_SYMBOL_GPL(lookup_address);
 620
 621 static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address,
 622                                   unsigned int *level)
 623 {
 624         if (cpa->pgd)
 625                 return lookup_address_in_pgd(cpa->pgd + pgd_index(address),
 626                                                address, level);
 627
 628         return lookup_address(address, level);
 629 }
 630
 631 /*
 632  * Lookup the PMD entry for a virtual address. Return a pointer to the entry
 633  * or NULL if not present.
 634  */
 635 pmd_t *lookup_pmd_address(unsigned long address)
 636 {
 637         pgd_t *pgd;
 638         p4d_t *p4d;
 639         pud_t *pud;
 640
 641         pgd = pgd_offset_k(address);
 642         if (pgd_none(*pgd))
 643                 return NULL;
 644
 645         p4d = p4d_offset(pgd, address);
 646         if (p4d_none(*p4d) || p4d_large(*p4d) || !p4d_present(*p4d))
 647                 return NULL;
 648
 649         pud = pud_offset(p4d, address);
 650         if (pud_none(*pud) || pud_large(*pud) || !pud_present(*pud))
 651                 return NULL;
 652
 653         return pmd_offset(pud, address);
 654 }
 655
 656 /*
 657  * This is necessary because __pa() does not work on some
 658  * kinds of memory, like vmalloc() or the alloc_remap()
 659  * areas on 32-bit NUMA systems.  The percpu areas can
 660  * end up in this kind of memory, for instance.
 661  *
 662  * This could be optimized, but it is only intended to be
 663  * used at inititalization time, and keeping it
 664  * unoptimized should increase the testing coverage for
 665  * the more obscure platforms.
 666  */
 667 phys_addr_t slow_virt_to_phys(void *__virt_addr)
 668 {
 669         unsigned long virt_addr = (unsigned long)__virt_addr;
 670         phys_addr_t phys_addr;
 671         unsigned long offset;
 672         enum pg_level level;
 673         pte_t *pte;
 674
 675         pte = lookup_address(virt_addr, &level);
 676         BUG_ON(!pte);
 677
 678         /*
 679          * pXX_pfn() returns unsigned long, which must be cast to phys_addr_t
 680          * before being left-shifted PAGE_SHIFT bits -- this trick is to
 681          * make 32-PAE kernel work correctly.
 682          */
 683         switch (level) {
 684         case PG_LEVEL_1G:
 685                 phys_addr = (phys_addr_t)pud_pfn(*(pud_t *)pte) << PAGE_SHIFT;
 686                 offset = virt_addr & ~PUD_PAGE_MASK;
 687                 break;
 688         case PG_LEVEL_2M:
 689                 phys_addr = (phys_addr_t)pmd_pfn(*(pmd_t *)pte) << PAGE_SHIFT;
 690                 offset = virt_addr & ~PMD_PAGE_MASK;
 691                 break;
 692         default:
 693                 phys_addr = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT;
 694                 offset = virt_addr & ~PAGE_MASK;
 695         }
 696
 697         return (phys_addr_t)(phys_addr | offset);
 698 }
 699 EXPORT_SYMBOL_GPL(slow_virt_to_phys);
 700
 701 /*
 702  * Set the new pmd in all the pgds we know about:
 703  */
 704 static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte)
 705 {
 706         /* change init_mm */
 707         set_pte_atomic(kpte, pte);
 708 #ifdef CONFIG_X86_32
 709         if (!SHARED_KERNEL_PMD) {
 710                 struct page *page;
 711
 712                 list_for_each_entry(page, &pgd_list, lru) {
 713                         pgd_t *pgd;
 714                         p4d_t *p4d;
 715                         pud_t *pud;
 716                         pmd_t *pmd;
 717
 718                         pgd = (pgd_t *)page_address(page) + pgd_index(address);
 719                         p4d = p4d_offset(pgd, address);
 720                         pud = pud_offset(p4d, address);
 721                         pmd = pmd_offset(pud, address);
 722                         set_pte_atomic((pte_t *)pmd, pte);
 723                 }
 724         }
 725 #endif
 726 }
 727
 728 static pgprot_t pgprot_clear_protnone_bits(pgprot_t prot)
 729 {
 730         /*
 731          * _PAGE_GLOBAL means "global page" for present PTEs.
 732          * But, it is also used to indicate _PAGE_PROTNONE
 733          * for non-present PTEs.
 734          *
 735          * This ensures that a _PAGE_GLOBAL PTE going from
 736          * present to non-present is not confused as
 737          * _PAGE_PROTNONE.
 738          */
 739         if (!(pgprot_val(prot) & _PAGE_PRESENT))
 740                 pgprot_val(prot) &= ~_PAGE_GLOBAL;
 741
 742         return prot;
 743 }
 744
 745 static int __should_split_large_page(pte_t *kpte, unsigned long address,
 746                                      struct cpa_data *cpa)
 747 {
 748         unsigned long numpages, pmask, psize, lpaddr, addr, pfn, old_pfn;
 749         pgprot_t old_prot, new_prot, req_prot, chk_prot;
 750         pte_t new_pte, old_pte, *tmp;
 751         enum pg_level level;
 752         int i;
 753
 754         /*
 755          * Check for races, another CPU might have split this page
 756          * up already:
 757          */
 758         tmp = _lookup_address_cpa(cpa, address, &level);
 759         if (tmp != kpte)
 760                 return 1;
 761
 762         switch (level) {
 763         case PG_LEVEL_2M:
 764                 old_prot = pmd_pgprot(*(pmd_t *)kpte);
 765                 old_pfn = pmd_pfn(*(pmd_t *)kpte);
 766                 cpa_inc_2m_checked();
 767                 break;
 768         case PG_LEVEL_1G:
 769                 old_prot = pud_pgprot(*(pud_t *)kpte);
 770                 old_pfn = pud_pfn(*(pud_t *)kpte);
 771                 cpa_inc_1g_checked();
 772                 break;
 773         default:
 774                 return -EINVAL;
 775         }
 776
 777         psize = page_level_size(level);
 778         pmask = page_level_mask(level);
 779
 780         /*
 781          * Calculate the number of pages, which fit into this large
 782          * page starting at address:
 783          */
 784         lpaddr = (address + psize) & pmask;
 785         numpages = (lpaddr - address) >> PAGE_SHIFT;
 786         if (numpages < cpa->numpages)
 787                 cpa->numpages = numpages;
 788
 789         /*
 790          * We are safe now. Check whether the new pgprot is the same:
 791          * Convert protection attributes to 4k-format, as cpa->mask* are set
 792          * up accordingly.
 793          */
 794         old_pte = *kpte;
 795         /* Clear PSE (aka _PAGE_PAT) and move PAT bit to correct position */
 796         req_prot = pgprot_large_2_4k(old_prot);
 797
 798         pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr);
 799         pgprot_val(req_prot) |= pgprot_val(cpa->mask_set);
 800
 801         /*
 802          * req_prot is in format of 4k pages. It must be converted to large
 803          * page format: the caching mode includes the PAT bit located at
 804          * different bit positions in the two formats.
 805          */
 806         req_prot = pgprot_4k_2_large(req_prot);
 807         req_prot = pgprot_clear_protnone_bits(req_prot);
 808         if (pgprot_val(req_prot) & _PAGE_PRESENT)
 809                 pgprot_val(req_prot) |= _PAGE_PSE;
 810
 811         /*
 812          * old_pfn points to the large page base pfn. So we need to add the
 813          * offset of the virtual address:
 814          */
 815         pfn = old_pfn + ((address & (psize - 1)) >> PAGE_SHIFT);
 816         cpa->pfn = pfn;
 817
 818         /*
 819          * Calculate the large page base address and the number of 4K pages
 820          * in the large page
 821          */
 822         lpaddr = address & pmask;
 823         numpages = psize >> PAGE_SHIFT;
 824
 825         /*
 826          * Sanity check that the existing mapping is correct versus the static
 827          * protections. static_protections() guards against !PRESENT, so no
 828          * extra conditional required here.
 829          */
 830         chk_prot = static_protections(old_prot, lpaddr, old_pfn, numpages,
 831                                       CPA_CONFLICT);
 832
 833         if (WARN_ON_ONCE(pgprot_val(chk_prot) != pgprot_val(old_prot))) {
 834                 /*
 835                  * Split the large page and tell the split code to
 836                  * enforce static protections.
 837                  */
 838                 cpa->force_static_prot = 1;
 839                 return 1;
 840         }
 841
 842         /*
 843          * Optimization: If the requested pgprot is the same as the current
 844          * pgprot, then the large page can be preserved and no updates are
 845          * required independent of alignment and length of the requested
 846          * range. The above already established that the current pgprot is
 847          * correct, which in consequence makes the requested pgprot correct
 848          * as well if it is the same. The static protection scan below will
 849          * not come to a different conclusion.
 850          */
 851         if (pgprot_val(req_prot) == pgprot_val(old_prot)) {
 852                 cpa_inc_lp_sameprot(level);
 853                 return 0;
 854         }
 855
 856         /*
 857          * Make sure that the requested pgprot does not violate the static
 858          * protections. Check the full large page whether one of the pages
 859          * in it results in a different pgprot than the first one of the
 860          * requested range. If yes, then the page needs to be split.
 861          */
 862         new_prot = static_protections(req_prot, address, pfn, 1, CPA_DETECT);
 863         pfn = old_pfn;
 864         for (i = 0, addr = lpaddr; i < numpages; i++, addr += PAGE_SIZE, pfn++) {
 865                 chk_prot = static_protections(req_prot, addr, pfn, 1,
 866                                               CPA_DETECT);
 867                 cpa_inc_4k_checked();
 868                 if (pgprot_val(chk_prot) != pgprot_val(new_prot))
 869                         return 1;
 870         }
 871
 872         /* If there are no changes, return. */
 873         if (pgprot_val(new_prot) == pgprot_val(old_prot)) {
 874                 cpa_inc_lp_sameprot(level);
 875                 return 0;
 876         }
 877
 878         /*
 879          * Verify that the address is aligned and the number of pages
 880          * covers the full page.
 881          */
 882         if (address != lpaddr || cpa->numpages != numpages)
 883                 return 1;
 884
 885         /* All checks passed. Update the large page mapping. */
 886         new_pte = pfn_pte(old_pfn, new_prot);
 887         __set_pmd_pte(kpte, address, new_pte);
 888         cpa->flags |= CPA_FLUSHTLB;
 889         cpa_inc_lp_preserved(level);
 890         return 0;
 891 }
 892
 893 static int should_split_large_page(pte_t *kpte, unsigned long address,
 894                                    struct cpa_data *cpa)
 895 {
 896         int do_split;
 897
 898         if (cpa->force_split)
 899                 return 1;
 900
 901         spin_lock(&pgd_lock);
 902         do_split = __should_split_large_page(kpte, address, cpa);
 903         spin_unlock(&pgd_lock);
 904
 905         return do_split;
 906 }
 907
 908 static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn,
 909                           pgprot_t ref_prot, unsigned long address,
 910                           unsigned long size)
 911 {
 912         unsigned int npg = PFN_DOWN(size);
 913         pgprot_t prot;
 914
 915         /*
 916          * If should_split_large_page() discovered an inconsistent mapping,
 917          * remove the invalid protection in the split mapping.
 918          */
 919         if (!cpa->force_static_prot)
 920                 goto set;
 921
 922         prot = static_protections(ref_prot, address, pfn, npg, CPA_PROTECT);
 923
 924         if (pgprot_val(prot) == pgprot_val(ref_prot))
 925                 goto set;
 926
 927         /*
 928          * If this is splitting a PMD, fix it up. PUD splits cannot be
 929          * fixed trivially as that would require to rescan the newly
 930          * installed PMD mappings after returning from split_large_page()
 931          * so an eventual further split can allocate the necessary PTE
 932          * pages. Warn for now and revisit it in case this actually
 933          * happens.
 934          */
 935         if (size == PAGE_SIZE)
 936                 ref_prot = prot;
 937         else
 938                 pr_warn_once("CPA: Cannot fixup static protections for PUD split\n");
 939 set:
 940         set_pte(pte, pfn_pte(pfn, ref_prot));
 941 }
 942
 943 static int
 944 __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
 945                    struct page *base)
 946 {
 947         unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1;
 948         pte_t *pbase = (pte_t *)page_address(base);
 949         unsigned int i, level;
 950         pgprot_t ref_prot;
 951         pte_t *tmp;
 952
 953         spin_lock(&pgd_lock);
 954         /*
 955          * Check for races, another CPU might have split this page
 956          * up for us already:
 957          */
 958         tmp = _lookup_address_cpa(cpa, address, &level);
 959         if (tmp != kpte) {
 960                 spin_unlock(&pgd_lock);
 961                 return 1;
 962         }
 963
 964         paravirt_alloc_pte(&init_mm, page_to_pfn(base));
 965
 966         switch (level) {
 967         case PG_LEVEL_2M:
 968                 ref_prot = pmd_pgprot(*(pmd_t *)kpte);
 969                 /*
 970                  * Clear PSE (aka _PAGE_PAT) and move
 971                  * PAT bit to correct position.
 972                  */
 973                 ref_prot = pgprot_large_2_4k(ref_prot);
 974                 ref_pfn = pmd_pfn(*(pmd_t *)kpte);
 975                 lpaddr = address & PMD_MASK;
 976                 lpinc = PAGE_SIZE;
 977                 break;
 978
 979         case PG_LEVEL_1G:
 980                 ref_prot = pud_pgprot(*(pud_t *)kpte);
 981                 ref_pfn = pud_pfn(*(pud_t *)kpte);
 982                 pfninc = PMD_PAGE_SIZE >> PAGE_SHIFT;
 983                 lpaddr = address & PUD_MASK;
 984                 lpinc = PMD_SIZE;
 985                 /*
 986                  * Clear the PSE flags if the PRESENT flag is not set
 987                  * otherwise pmd_present/pmd_huge will return true
 988                  * even on a non present pmd.
 989                  */
 990                 if (!(pgprot_val(ref_prot) & _PAGE_PRESENT))
 991                         pgprot_val(ref_prot) &= ~_PAGE_PSE;
 992                 break;
 993
 994         default:
 995                 spin_unlock(&pgd_lock);
 996                 return 1;
 997         }
 998
 999         ref_prot = pgprot_clear_protnone_bits(ref_prot);
1000
1001         /*
1002          * Get the target pfn from the original entry:
1003          */
1004         pfn = ref_pfn;
1005         for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc, lpaddr += lpinc)
1006                 split_set_pte(cpa, pbase + i, pfn, ref_prot, lpaddr, lpinc);
1007
1008         if (virt_addr_valid(address)) {
1009                 unsigned long pfn = PFN_DOWN(__pa(address));
1010
1011                 if (pfn_range_is_mapped(pfn, pfn + 1))
1012                         split_page_count(level);
1013         }
1014
1015         /*
1016          * Install the new, split up pagetable.
1017          *
1018          * We use the standard kernel pagetable protections for the new
1019          * pagetable protections, the actual ptes set above control the
1020          * primary protection behavior:
1021          */
1022         __set_pmd_pte(kpte, address, mk_pte(base, __pgprot(_KERNPG_TABLE)));
1023
1024         /*
1025          * Intel Atom errata AAH41 workaround.
1026          *
1027          * The real fix should be in hw or in a microcode update, but
1028          * we also probabilistically try to reduce the window of having
1029          * a large TLB mixed with 4K TLBs while instruction fetches are
1030          * going on.
1031          */
1032         __flush_tlb_all();
1033         spin_unlock(&pgd_lock);
1034
1035         return 0;
1036 }
1037
1038 static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
1039                             unsigned long address)
1040 {
1041         struct page *base;
1042
1043         if (!debug_pagealloc_enabled())
1044                 spin_unlock(&cpa_lock);
1045         base = alloc_pages(GFP_KERNEL, 0);
1046         if (!debug_pagealloc_enabled())
1047                 spin_lock(&cpa_lock);
1048         if (!base)
1049                 return -ENOMEM;
1050
1051         if (__split_large_page(cpa, kpte, address, base))
1052                 __free_page(base);
1053
1054         return 0;
1055 }
1056
1057 static bool try_to_free_pte_page(pte_t *pte)
1058 {
1059         int i;
1060
1061         for (i = 0; i < PTRS_PER_PTE; i++)
1062                 if (!pte_none(pte[i]))
1063                         return false;
1064
1065         free_page((unsigned long)pte);
1066         return true;
1067 }
1068
1069 static bool try_to_free_pmd_page(pmd_t *pmd)
1070 {
1071         int i;
1072
1073         for (i = 0; i < PTRS_PER_PMD; i++)
1074                 if (!pmd_none(pmd[i]))
1075                         return false;
1076
1077         free_page((unsigned long)pmd);
1078         return true;
1079 }
1080
1081 static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end)
1082 {
1083         pte_t *pte = pte_offset_kernel(pmd, start);
1084
1085         while (start < end) {
1086                 set_pte(pte, __pte(0));
1087
1088                 start += PAGE_SIZE;
1089                 pte++;
1090         }
1091
1092         if (try_to_free_pte_page((pte_t *)pmd_page_vaddr(*pmd))) {
1093                 pmd_clear(pmd);
1094                 return true;
1095         }
1096         return false;
1097 }
1098
1099 static void __unmap_pmd_range(pud_t *pud, pmd_t *pmd,
1100                               unsigned long start, unsigned long end)
1101 {
1102         if (unmap_pte_range(pmd, start, end))
1103                 if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
1104                         pud_clear(pud);
1105 }
1106
1107 static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end)
1108 {
1109         pmd_t *pmd = pmd_offset(pud, start);
1110
1111         /*
1112          * Not on a 2MB page boundary?
1113          */
1114         if (start & (PMD_SIZE - 1)) {
1115                 unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
1116                 unsigned long pre_end = min_t(unsigned long, end, next_page);
1117
1118                 __unmap_pmd_range(pud, pmd, start, pre_end);
1119
1120                 start = pre_end;
1121                 pmd++;
1122         }
1123
1124         /*
1125          * Try to unmap in 2M chunks.
1126          */
1127         while (end - start >= PMD_SIZE) {
1128                 if (pmd_large(*pmd))
1129                         pmd_clear(pmd);
1130                 else
1131                         __unmap_pmd_range(pud, pmd, start, start + PMD_SIZE);
1132
1133                 start += PMD_SIZE;
1134                 pmd++;
1135         }
1136
1137         /*
1138          * 4K leftovers?
1139          */
1140         if (start < end)
1141                 return __unmap_pmd_range(pud, pmd, start, end);
1142
1143         /*
1144          * Try again to free the PMD page if haven't succeeded above.
1145          */
1146         if (!pud_none(*pud))
1147                 if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud)))
1148                         pud_clear(pud);
1149 }
1150
1151 static void unmap_pud_range(p4d_t *p4d, unsigned long start, unsigned long end)
1152 {
1153         pud_t *pud = pud_offset(p4d, start);
1154
1155         /*
1156          * Not on a GB page boundary?
1157          */
1158         if (start & (PUD_SIZE - 1)) {
1159                 unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
1160                 unsigned long pre_end   = min_t(unsigned long, end, next_page);
1161
1162                 unmap_pmd_range(pud, start, pre_end);
1163
1164                 start = pre_end;
1165                 pud++;
1166         }
1167
1168         /*
1169          * Try to unmap in 1G chunks?
1170          */
1171         while (end - start >= PUD_SIZE) {
1172
1173                 if (pud_large(*pud))
1174                         pud_clear(pud);
1175                 else
1176                         unmap_pmd_range(pud, start, start + PUD_SIZE);
1177
1178                 start += PUD_SIZE;
1179                 pud++;
1180         }
1181
1182         /*
1183          * 2M leftovers?
1184          */
1185         if (start < end)
1186                 unmap_pmd_range(pud, start, end);
1187
1188         /*
1189          * No need to try to free the PUD page because we'll free it in
1190          * populate_pgd's error path
1191          */
1192 }
1193
1194 static int alloc_pte_page(pmd_t *pmd)
1195 {
1196         pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL);
1197         if (!pte)
1198                 return -1;
1199
1200         set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE));
1201         return 0;
1202 }
1203
1204 static int alloc_pmd_page(pud_t *pud)
1205 {
1206         pmd_t *pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL);
1207         if (!pmd)
1208                 return -1;
1209
1210         set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE));
1211         return 0;
1212 }
1213
1214 static void populate_pte(struct cpa_data *cpa,
1215                          unsigned long start, unsigned long end,
1216                          unsigned num_pages, pmd_t *pmd, pgprot_t pgprot)
1217 {
1218         pte_t *pte;
1219
1220         pte = pte_offset_kernel(pmd, start);
1221
1222         pgprot = pgprot_clear_protnone_bits(pgprot);
1223
1224         while (num_pages-- && start < end) {
1225                 set_pte(pte, pfn_pte(cpa->pfn, pgprot));
1226
1227                 start    += PAGE_SIZE;
1228                 cpa->pfn++;
1229                 pte++;
1230         }
1231 }
1232
1233 static long populate_pmd(struct cpa_data *cpa,
1234                          unsigned long start, unsigned long end,
1235                          unsigned num_pages, pud_t *pud, pgprot_t pgprot)
1236 {
1237         long cur_pages = 0;
1238         pmd_t *pmd;
1239         pgprot_t pmd_pgprot;
1240
1241         /*
1242          * Not on a 2M boundary?
1243          */
1244         if (start & (PMD_SIZE - 1)) {
1245                 unsigned long pre_end = start + (num_pages << PAGE_SHIFT);
1246                 unsigned long next_page = (start + PMD_SIZE) & PMD_MASK;
1247
1248                 pre_end   = min_t(unsigned long, pre_end, next_page);
1249                 cur_pages = (pre_end - start) >> PAGE_SHIFT;
1250                 cur_pages = min_t(unsigned int, num_pages, cur_pages);
1251
1252                 /*
1253                  * Need a PTE page?
1254                  */
1255                 pmd = pmd_offset(pud, start);
1256                 if (pmd_none(*pmd))
1257                         if (alloc_pte_page(pmd))
1258                                 return -1;
1259
1260                 populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot);
1261
1262                 start = pre_end;
1263         }
1264
1265         /*
1266          * We mapped them all?
1267          */
1268         if (num_pages == cur_pages)
1269                 return cur_pages;
1270
1271         pmd_pgprot = pgprot_4k_2_large(pgprot);
1272
1273         while (end - start >= PMD_SIZE) {
1274
1275                 /*
1276                  * We cannot use a 1G page so allocate a PMD page if needed.
1277                  */
1278                 if (pud_none(*pud))
1279                         if (alloc_pmd_page(pud))
1280                                 return -1;
1281
1282                 pmd = pmd_offset(pud, start);
1283
1284                 set_pmd(pmd, pmd_mkhuge(pfn_pmd(cpa->pfn,
1285                                         canon_pgprot(pmd_pgprot))));
1286
1287                 start     += PMD_SIZE;
1288                 cpa->pfn  += PMD_SIZE >> PAGE_SHIFT;
1289                 cur_pages += PMD_SIZE >> PAGE_SHIFT;
1290         }
1291
1292         /*
1293          * Map trailing 4K pages.
1294          */
1295         if (start < end) {
1296                 pmd = pmd_offset(pud, start);
1297                 if (pmd_none(*pmd))
1298                         if (alloc_pte_page(pmd))
1299                                 return -1;
1300
1301                 populate_pte(cpa, start, end, num_pages - cur_pages,
1302                              pmd, pgprot);
1303         }
1304         return num_pages;
1305 }
1306
1307 static int populate_pud(struct cpa_data *cpa, unsigned long start, p4d_t *p4d,
1308                         pgprot_t pgprot)
1309 {
1310         pud_t *pud;
1311         unsigned long end;
1312         long cur_pages = 0;
1313         pgprot_t pud_pgprot;
1314
1315         end = start + (cpa->numpages << PAGE_SHIFT);
1316
1317         /*
1318          * Not on a Gb page boundary? => map everything up to it with
1319          * smaller pages.
1320          */
1321         if (start & (PUD_SIZE - 1)) {
1322                 unsigned long pre_end;
1323                 unsigned long next_page = (start + PUD_SIZE) & PUD_MASK;
1324
1325                 pre_end   = min_t(unsigned long, end, next_page);
1326                 cur_pages = (pre_end - start) >> PAGE_SHIFT;
1327                 cur_pages = min_t(int, (int)cpa->numpages, cur_pages);
1328
1329                 pud = pud_offset(p4d, start);
1330
1331                 /*
1332                  * Need a PMD page?
1333                  */
1334                 if (pud_none(*pud))
1335                         if (alloc_pmd_page(pud))
1336                                 return -1;
1337
1338                 cur_pages = populate_pmd(cpa, start, pre_end, cur_pages,
1339                                          pud, pgprot);
1340                 if (cur_pages < 0)
1341                         return cur_pages;
1342
1343                 start = pre_end;
1344         }
1345
1346         /* We mapped them all? */
1347         if (cpa->numpages == cur_pages)
1348                 return cur_pages;
1349
1350         pud = pud_offset(p4d, start);
1351         pud_pgprot = pgprot_4k_2_large(pgprot);
1352
1353         /*
1354          * Map everything starting from the Gb boundary, possibly with 1G pages
1355          */
1356         while (boot_cpu_has(X86_FEATURE_GBPAGES) && end - start >= PUD_SIZE) {
1357                 set_pud(pud, pud_mkhuge(pfn_pud(cpa->pfn,
1358                                    canon_pgprot(pud_pgprot))));
1359
1360                 start     += PUD_SIZE;
1361                 cpa->pfn  += PUD_SIZE >> PAGE_SHIFT;
1362                 cur_pages += PUD_SIZE >> PAGE_SHIFT;
1363                 pud++;
1364         }
1365
1366         /* Map trailing leftover */
1367         if (start < end) {
1368                 long tmp;
1369
1370                 pud = pud_offset(p4d, start);
1371                 if (pud_none(*pud))
1372                         if (alloc_pmd_page(pud))
1373                                 return -1;
1374
1375                 tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages,
1376                                    pud, pgprot);
1377                 if (tmp < 0)
1378                         return cur_pages;
1379
1380                 cur_pages += tmp;
1381         }
1382         return cur_pages;
1383 }
1384
1385 /*
1386  * Restrictions for kernel page table do not necessarily apply when mapping in
1387  * an alternate PGD.
1388  */
1389 static int populate_pgd(struct cpa_data *cpa, unsigned long addr)
1390 {
1391         pgprot_t pgprot = __pgprot(_KERNPG_TABLE);
1392         pud_t *pud = NULL;      /* shut up gcc */
1393         p4d_t *p4d;
1394         pgd_t *pgd_entry;
1395         long ret;
1396
1397         pgd_entry = cpa->pgd + pgd_index(addr);
1398
1399         if (pgd_none(*pgd_entry)) {
1400                 p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL);
1401                 if (!p4d)
1402                         return -1;
1403
1404                 set_pgd(pgd_entry, __pgd(__pa(p4d) | _KERNPG_TABLE));
1405         }
1406
1407         /*
1408          * Allocate a PUD page and hand it down for mapping.
1409          */
1410         p4d = p4d_offset(pgd_entry, addr);
1411         if (p4d_none(*p4d)) {
1412                 pud = (pud_t *)get_zeroed_page(GFP_KERNEL);
1413                 if (!pud)
1414                         return -1;
1415
1416                 set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE));
1417         }
1418
1419         pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr);
1420         pgprot_val(pgprot) |=  pgprot_val(cpa->mask_set);
1421
1422         ret = populate_pud(cpa, addr, p4d, pgprot);
1423         if (ret < 0) {
1424                 /*
1425                  * Leave the PUD page in place in case some other CPU or thread
1426                  * already found it, but remove any useless entries we just
1427                  * added to it.
1428                  */
1429                 unmap_pud_range(p4d, addr,
1430                                 addr + (cpa->numpages << PAGE_SHIFT));
1431                 return ret;
1432         }
1433
1434         cpa->numpages = ret;
1435         return 0;
1436 }
1437
1438 static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr,
1439                                int primary)
1440 {
1441         if (cpa->pgd) {
1442                 /*
1443                  * Right now, we only execute this code path when mapping
1444                  * the EFI virtual memory map regions, no other users
1445                  * provide a ->pgd value. This may change in the future.
1446                  */
1447                 return populate_pgd(cpa, vaddr);
1448         }
1449
1450         /*
1451          * Ignore all non primary paths.
1452          */
1453         if (!primary) {
1454                 cpa->numpages = 1;
1455                 return 0;
1456         }
1457
1458         /*
1459          * Ignore the NULL PTE for kernel identity mapping, as it is expected
1460          * to have holes.
1461          * Also set numpages to '1' indicating that we processed cpa req for
1462          * one virtual address page and its pfn. TBD: numpages can be set based
1463          * on the initial value and the level returned by lookup_address().
1464          */
1465         if (within(vaddr, PAGE_OFFSET,
1466                    PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) {
1467                 cpa->numpages = 1;
1468                 cpa->pfn = __pa(vaddr) >> PAGE_SHIFT;
1469                 return 0;
1470
1471         } else if (__cpa_pfn_in_highmap(cpa->pfn)) {
1472                 /* Faults in the highmap are OK, so do not warn: */
1473                 return -EFAULT;
1474         } else {
1475                 WARN(1, KERN_WARNING "CPA: called for zero pte. "
1476                         "vaddr = %lx cpa->vaddr = %lx\n", vaddr,
1477                         *cpa->vaddr);
1478
1479                 return -EFAULT;
1480         }
1481 }
1482
1483 static int __change_page_attr(struct cpa_data *cpa, int primary)
1484 {
1485         unsigned long address;
1486         int do_split, err;
1487         unsigned int level;
1488         pte_t *kpte, old_pte;
1489
1490         if (cpa->flags & CPA_PAGES_ARRAY) {
1491                 struct page *page = cpa->pages[cpa->curpage];
1492                 if (unlikely(PageHighMem(page)))
1493                         return 0;
1494                 address = (unsigned long)page_address(page);
1495         } else if (cpa->flags & CPA_ARRAY)
1496                 address = cpa->vaddr[cpa->curpage];
1497         else
1498                 address = *cpa->vaddr;
1499 repeat:
1500         kpte = _lookup_address_cpa(cpa, address, &level);
1501         if (!kpte)
1502                 return __cpa_process_fault(cpa, address, primary);
1503
1504         old_pte = *kpte;
1505         if (pte_none(old_pte))
1506                 return __cpa_process_fault(cpa, address, primary);
1507
1508         if (level == PG_LEVEL_4K) {
1509                 pte_t new_pte;
1510                 pgprot_t new_prot = pte_pgprot(old_pte);
1511                 unsigned long pfn = pte_pfn(old_pte);
1512
1513                 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr);
1514                 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set);
1515
1516                 cpa_inc_4k_install();
1517                 new_prot = static_protections(new_prot, address, pfn, 1,
1518                                               CPA_PROTECT);
1519
1520                 new_prot = pgprot_clear_protnone_bits(new_prot);
1521
1522                 /*
1523                  * We need to keep the pfn from the existing PTE,
1524                  * after all we're only going to change it's attributes
1525                  * not the memory it points to
1526                  */
1527                 new_pte = pfn_pte(pfn, new_prot);
1528                 cpa->pfn = pfn;
1529                 /*
1530                  * Do we really change anything ?
1531                  */
1532                 if (pte_val(old_pte) != pte_val(new_pte)) {
1533                         set_pte_atomic(kpte, new_pte);
1534                         cpa->flags |= CPA_FLUSHTLB;
1535                 }
1536                 cpa->numpages = 1;
1537                 return 0;
1538         }
1539
1540         /*
1541          * Check, whether we can keep the large page intact
1542          * and just change the pte:
1543          */
1544         do_split = should_split_large_page(kpte, address, cpa);
1545         /*
1546          * When the range fits into the existing large page,
1547          * return. cp->numpages and cpa->tlbflush have been updated in
1548          * try_large_page:
1549          */
1550         if (do_split <= 0)
1551                 return do_split;
1552
1553         /*
1554          * We have to split the large page:
1555          */
1556         err = split_large_page(cpa, kpte, address);
1557         if (!err) {
1558                 /*
1559                  * Do a global flush tlb after splitting the large page
1560                  * and before we do the actual change page attribute in the PTE.
1561                  *
1562                  * With out this, we violate the TLB application note, that says
1563                  * "The TLBs may contain both ordinary and large-page
1564                  *  translations for a 4-KByte range of linear addresses. This
1565                  *  may occur if software modifies the paging structures so that
1566                  *  the page size used for the address range changes. If the two
1567                  *  translations differ with respect to page frame or attributes
1568                  *  (e.g., permissions), processor behavior is undefined and may
1569                  *  be implementation-specific."
1570                  *
1571                  * We do this global tlb flush inside the cpa_lock, so that we
1572                  * don't allow any other cpu, with stale tlb entries change the
1573                  * page attribute in parallel, that also falls into the
1574                  * just split large page entry.
1575                  */
1576                 flush_tlb_all();
1577                 goto repeat;
1578         }
1579
1580         return err;
1581 }
1582
1583 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);
1584
1585 static int cpa_process_alias(struct cpa_data *cpa)
1586 {
1587         struct cpa_data alias_cpa;
1588         unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT);
1589         unsigned long vaddr;
1590         int ret;
1591
1592         if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1))
1593                 return 0;
1594
1595         /*
1596          * No need to redo, when the primary call touched the direct
1597          * mapping already:
1598          */
1599         if (cpa->flags & CPA_PAGES_ARRAY) {
1600                 struct page *page = cpa->pages[cpa->curpage];
1601                 if (unlikely(PageHighMem(page)))
1602                         return 0;
1603                 vaddr = (unsigned long)page_address(page);
1604         } else if (cpa->flags & CPA_ARRAY)
1605                 vaddr = cpa->vaddr[cpa->curpage];
1606         else
1607                 vaddr = *cpa->vaddr;
1608
1609         if (!(within(vaddr, PAGE_OFFSET,
1610                     PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) {
1611
1612                 alias_cpa = *cpa;
1613                 alias_cpa.vaddr = &laddr;
1614                 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
1615
1616                 ret = __change_page_attr_set_clr(&alias_cpa, 0);
1617                 if (ret)
1618                         return ret;
1619         }
1620
1621 #ifdef CONFIG_X86_64
1622         /*
1623          * If the primary call didn't touch the high mapping already
1624          * and the physical address is inside the kernel map, we need
1625          * to touch the high mapped kernel as well:
1626          */
1627         if (!within(vaddr, (unsigned long)_text, _brk_end) &&
1628             __cpa_pfn_in_highmap(cpa->pfn)) {
1629                 unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) +
1630                                                __START_KERNEL_map - phys_base;
1631                 alias_cpa = *cpa;
1632                 alias_cpa.vaddr = &temp_cpa_vaddr;
1633                 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY);
1634
1635                 /*
1636                  * The high mapping range is imprecise, so ignore the
1637                  * return value.
1638                  */
1639                 __change_page_attr_set_clr(&alias_cpa, 0);
1640         }
1641 #endif
1642
1643         return 0;
1644 }
1645
1646 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias)
1647 {
1648         unsigned long numpages = cpa->numpages;
1649         int ret;
1650
1651         while (numpages) {
1652                 /*
1653                  * Store the remaining nr of pages for the large page
1654                  * preservation check.
1655                  */
1656                 cpa->numpages = numpages;
1657                 /* for array changes, we can't use large page */
1658                 if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY))
1659                         cpa->numpages = 1;
1660
1661                 if (!debug_pagealloc_enabled())
1662                         spin_lock(&cpa_lock);
1663                 ret = __change_page_attr(cpa, checkalias);
1664                 if (!debug_pagealloc_enabled())
1665                         spin_unlock(&cpa_lock);
1666                 if (ret)
1667                         return ret;
1668
1669                 if (checkalias) {
1670                         ret = cpa_process_alias(cpa);
1671                         if (ret)
1672                                 return ret;
1673                 }
1674
1675                 /*
1676                  * Adjust the number of pages with the result of the
1677                  * CPA operation. Either a large page has been
1678                  * preserved or a single page update happened.
1679                  */
1680                 BUG_ON(cpa->numpages > numpages || !cpa->numpages);
1681                 numpages -= cpa->numpages;
1682                 if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY))
1683                         cpa->curpage++;
1684                 else
1685                         *cpa->vaddr += cpa->numpages * PAGE_SIZE;
1686
1687         }
1688         return 0;
1689 }
1690
1691 /*
1692  * Machine check recovery code needs to change cache mode of poisoned
1693  * pages to UC to avoid speculative access logging another error. But
1694  * passing the address of the 1:1 mapping to set_memory_uc() is a fine
1695  * way to encourage a speculative access. So we cheat and flip the top
1696  * bit of the address. This works fine for the code that updates the
1697  * page tables. But at the end of the process we need to flush the cache
1698  * and the non-canonical address causes a #GP fault when used by the
1699  * CLFLUSH instruction.
1700  *
1701  * But in the common case we already have a canonical address. This code
1702  * will fix the top bit if needed and is a no-op otherwise.
1703  */
1704 static inline unsigned long make_addr_canonical_again(unsigned long addr)
1705 {
1706 #ifdef CONFIG_X86_64
1707         return (long)(addr << 1) >> 1;
1708 #else
1709         return addr;
1710 #endif
1711 }
1712
1713
1714 static int change_page_attr_set_clr(unsigned long *addr, int numpages,
1715                                     pgprot_t mask_set, pgprot_t mask_clr,
1716                                     int force_split, int in_flag,
1717                                     struct page **pages)
1718 {
1719         struct cpa_data cpa;
1720         int ret, cache, checkalias;
1721         unsigned long baddr = 0;
1722
1723         memset(&cpa, 0, sizeof(cpa));
1724
1725         /*
1726          * Check, if we are requested to set a not supported
1727          * feature.  Clearing non-supported features is OK.
1728          */
1729         mask_set = canon_pgprot(mask_set);
1730
1731         if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split)
1732                 return 0;
1733
1734         /* Ensure we are PAGE_SIZE aligned */
1735         if (in_flag & CPA_ARRAY) {
1736                 int i;
1737                 for (i = 0; i < numpages; i++) {
1738                         if (addr[i] & ~PAGE_MASK) {
1739                                 addr[i] &= PAGE_MASK;
1740                                 WARN_ON_ONCE(1);
1741                         }
1742                 }
1743         } else if (!(in_flag & CPA_PAGES_ARRAY)) {
1744                 /*
1745                  * in_flag of CPA_PAGES_ARRAY implies it is aligned.
1746                  * No need to cehck in that case
1747                  */
1748                 if (*addr & ~PAGE_MASK) {
1749                         *addr &= PAGE_MASK;
1750                         /*
1751                          * People should not be passing in unaligned addresses:
1752                          */
1753                         WARN_ON_ONCE(1);
1754                 }
1755                 /*
1756                  * Save address for cache flush. *addr is modified in the call
1757                  * to __change_page_attr_set_clr() below.
1758                  */
1759                 baddr = make_addr_canonical_again(*addr);
1760         }
1761
1762         /* Must avoid aliasing mappings in the highmem code */
1763         kmap_flush_unused();
1764
1765         vm_unmap_aliases();
1766
1767         cpa.vaddr = addr;
1768         cpa.pages = pages;
1769         cpa.numpages = numpages;
1770         cpa.mask_set = mask_set;
1771         cpa.mask_clr = mask_clr;
1772         cpa.flags = 0;
1773         cpa.curpage = 0;
1774         cpa.force_split = force_split;
1775
1776         if (in_flag & (CPA_ARRAY | CPA_PAGES_ARRAY))
1777                 cpa.flags |= in_flag;
1778
1779         /* No alias checking for _NX bit modifications */
1780         checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX;
1781         /* Has caller explicitly disabled alias checking? */
1782         if (in_flag & CPA_NO_CHECK_ALIAS)
1783                 checkalias = 0;
1784
1785         ret = __change_page_attr_set_clr(&cpa, checkalias);
1786
1787         /*
1788          * Check whether we really changed something:
1789          */
1790         if (!(cpa.flags & CPA_FLUSHTLB))
1791                 goto out;
1792
1793         /*
1794          * No need to flush, when we did not set any of the caching
1795          * attributes:
1796          */
1797         cache = !!pgprot2cachemode(mask_set);
1798
1799         /*
1800          * On success we use CLFLUSH, when the CPU supports it to
1801          * avoid the WBINVD. If the CPU does not support it and in the
1802          * error case we fall back to cpa_flush_all (which uses
1803          * WBINVD):
1804          */
1805         if (!ret && boot_cpu_has(X86_FEATURE_CLFLUSH)) {
1806                 if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
1807                         cpa_flush_array(addr, numpages, cache,
1808                                         cpa.flags, pages);
1809                 } else
1810                         cpa_flush_range(baddr, numpages, cache);
1811         } else
1812                 cpa_flush_all(cache);
1813
1814 out:
1815         return ret;
1816 }
1817
1818 static inline int change_page_attr_set(unsigned long *addr, int numpages,
1819                                        pgprot_t mask, int array)
1820 {
1821         return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0,
1822                 (array ? CPA_ARRAY : 0), NULL);
1823 }
1824
1825 static inline int change_page_attr_clear(unsigned long *addr, int numpages,
1826                                          pgprot_t mask, int array)
1827 {
1828         return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0,
1829                 (array ? CPA_ARRAY : 0), NULL);
1830 }
1831
1832 static inline int cpa_set_pages_array(struct page **pages, int numpages,
1833                                        pgprot_t mask)
1834 {
1835         return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0,
1836                 CPA_PAGES_ARRAY, pages);
1837 }
1838
1839 static inline int cpa_clear_pages_array(struct page **pages, int numpages,
1840                                          pgprot_t mask)
1841 {
1842         return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0,
1843                 CPA_PAGES_ARRAY, pages);
1844 }
1845
1846 int _set_memory_uc(unsigned long addr, int numpages)
1847 {
1848         /*
1849          * for now UC MINUS. see comments in ioremap_nocache()
1850          * If you really need strong UC use ioremap_uc(), but note
1851          * that you cannot override IO areas with set_memory_*() as
1852          * these helpers cannot work with IO memory.
1853          */
1854         return change_page_attr_set(&addr, numpages,
1855                                     cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
1856                                     0);
1857 }
1858
1859 int set_memory_uc(unsigned long addr, int numpages)
1860 {
1861         int ret;
1862
1863         /*
1864          * for now UC MINUS. see comments in ioremap_nocache()
1865          */
1866         ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1867                               _PAGE_CACHE_MODE_UC_MINUS, NULL);
1868         if (ret)
1869                 goto out_err;
1870
1871         ret = _set_memory_uc(addr, numpages);
1872         if (ret)
1873                 goto out_free;
1874
1875         return 0;
1876
1877 out_free:
1878         free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1879 out_err:
1880         return ret;
1881 }
1882 EXPORT_SYMBOL(set_memory_uc);
1883
1884 static int _set_memory_array(unsigned long *addr, int addrinarray,
1885                 enum page_cache_mode new_type)
1886 {
1887         enum page_cache_mode set_type;
1888         int i, j;
1889         int ret;
1890
1891         for (i = 0; i < addrinarray; i++) {
1892                 ret = reserve_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE,
1893                                         new_type, NULL);
1894                 if (ret)
1895                         goto out_free;
1896         }
1897
1898         /* If WC, set to UC- first and then WC */
1899         set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
1900                                 _PAGE_CACHE_MODE_UC_MINUS : new_type;
1901
1902         ret = change_page_attr_set(addr, addrinarray,
1903                                    cachemode2pgprot(set_type), 1);
1904
1905         if (!ret && new_type == _PAGE_CACHE_MODE_WC)
1906                 ret = change_page_attr_set_clr(addr, addrinarray,
1907                                                cachemode2pgprot(
1908                                                 _PAGE_CACHE_MODE_WC),
1909                                                __pgprot(_PAGE_CACHE_MASK),
1910                                                0, CPA_ARRAY, NULL);
1911         if (ret)
1912                 goto out_free;
1913
1914         return 0;
1915
1916 out_free:
1917         for (j = 0; j < i; j++)
1918                 free_memtype(__pa(addr[j]), __pa(addr[j]) + PAGE_SIZE);
1919
1920         return ret;
1921 }
1922
1923 int set_memory_array_uc(unsigned long *addr, int addrinarray)
1924 {
1925         return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_UC_MINUS);
1926 }
1927 EXPORT_SYMBOL(set_memory_array_uc);
1928
1929 int set_memory_array_wc(unsigned long *addr, int addrinarray)
1930 {
1931         return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_WC);
1932 }
1933 EXPORT_SYMBOL(set_memory_array_wc);
1934
1935 int set_memory_array_wt(unsigned long *addr, int addrinarray)
1936 {
1937         return _set_memory_array(addr, addrinarray, _PAGE_CACHE_MODE_WT);
1938 }
1939 EXPORT_SYMBOL_GPL(set_memory_array_wt);
1940
1941 int _set_memory_wc(unsigned long addr, int numpages)
1942 {
1943         int ret;
1944         unsigned long addr_copy = addr;
1945
1946         ret = change_page_attr_set(&addr, numpages,
1947                                    cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS),
1948                                    0);
1949         if (!ret) {
1950                 ret = change_page_attr_set_clr(&addr_copy, numpages,
1951                                                cachemode2pgprot(
1952                                                 _PAGE_CACHE_MODE_WC),
1953                                                __pgprot(_PAGE_CACHE_MASK),
1954                                                0, 0, NULL);
1955         }
1956         return ret;
1957 }
1958
1959 int set_memory_wc(unsigned long addr, int numpages)
1960 {
1961         int ret;
1962
1963         ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1964                 _PAGE_CACHE_MODE_WC, NULL);
1965         if (ret)
1966                 return ret;
1967
1968         ret = _set_memory_wc(addr, numpages);
1969         if (ret)
1970                 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1971
1972         return ret;
1973 }
1974 EXPORT_SYMBOL(set_memory_wc);
1975
1976 int _set_memory_wt(unsigned long addr, int numpages)
1977 {
1978         return change_page_attr_set(&addr, numpages,
1979                                     cachemode2pgprot(_PAGE_CACHE_MODE_WT), 0);
1980 }
1981
1982 int set_memory_wt(unsigned long addr, int numpages)
1983 {
1984         int ret;
1985
1986         ret = reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
1987                               _PAGE_CACHE_MODE_WT, NULL);
1988         if (ret)
1989                 return ret;
1990
1991         ret = _set_memory_wt(addr, numpages);
1992         if (ret)
1993                 free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
1994
1995         return ret;
1996 }
1997 EXPORT_SYMBOL_GPL(set_memory_wt);
1998
1999 int _set_memory_wb(unsigned long addr, int numpages)
2000 {
2001         /* WB cache mode is hard wired to all cache attribute bits being 0 */
2002         return change_page_attr_clear(&addr, numpages,
2003                                       __pgprot(_PAGE_CACHE_MASK), 0);
2004 }
2005
2006 int set_memory_wb(unsigned long addr, int numpages)
2007 {
2008         int ret;
2009
2010         ret = _set_memory_wb(addr, numpages);
2011         if (ret)
2012                 return ret;
2013
2014         free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
2015         return 0;
2016 }
2017 EXPORT_SYMBOL(set_memory_wb);
2018
2019 int set_memory_array_wb(unsigned long *addr, int addrinarray)
2020 {
2021         int i;
2022         int ret;
2023
2024         /* WB cache mode is hard wired to all cache attribute bits being 0 */
2025         ret = change_page_attr_clear(addr, addrinarray,
2026                                       __pgprot(_PAGE_CACHE_MASK), 1);
2027         if (ret)
2028                 return ret;
2029
2030         for (i = 0; i < addrinarray; i++)
2031                 free_memtype(__pa(addr[i]), __pa(addr[i]) + PAGE_SIZE);
2032
2033         return 0;
2034 }
2035 EXPORT_SYMBOL(set_memory_array_wb);
2036
2037 int set_memory_x(unsigned long addr, int numpages)
2038 {
2039         if (!(__supported_pte_mask & _PAGE_NX))
2040                 return 0;
2041
2042         return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NX), 0);
2043 }
2044 EXPORT_SYMBOL(set_memory_x);
2045
2046 int set_memory_nx(unsigned long addr, int numpages)
2047 {
2048         if (!(__supported_pte_mask & _PAGE_NX))
2049                 return 0;
2050
2051         return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NX), 0);
2052 }
2053 EXPORT_SYMBOL(set_memory_nx);
2054
2055 int set_memory_ro(unsigned long addr, int numpages)
2056 {
2057         return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
2058 }
2059
2060 int set_memory_rw(unsigned long addr, int numpages)
2061 {
2062         return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0);
2063 }
2064
2065 int set_memory_np(unsigned long addr, int numpages)
2066 {
2067         return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0);
2068 }
2069
2070 int set_memory_np_noalias(unsigned long addr, int numpages)
2071 {
2072         int cpa_flags = CPA_NO_CHECK_ALIAS;
2073
2074         return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
2075                                         __pgprot(_PAGE_PRESENT), 0,
2076                                         cpa_flags, NULL);
2077 }
2078
2079 int set_memory_4k(unsigned long addr, int numpages)
2080 {
2081         return change_page_attr_set_clr(&addr, numpages, __pgprot(0),
2082                                         __pgprot(0), 1, 0, NULL);
2083 }
2084
2085 int set_memory_nonglobal(unsigned long addr, int numpages)
2086 {
2087         return change_page_attr_clear(&addr, numpages,
2088                                       __pgprot(_PAGE_GLOBAL), 0);
2089 }
2090
2091 int set_memory_global(unsigned long addr, int numpages)
2092 {
2093         return change_page_attr_set(&addr, numpages,
2094                                     __pgprot(_PAGE_GLOBAL), 0);
2095 }
2096
2097 static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)
2098 {
2099         struct cpa_data cpa;
2100         unsigned long start;
2101         int ret;
2102
2103         /* Nothing to do if memory encryption is not active */
2104         if (!mem_encrypt_active())
2105                 return 0;
2106
2107         /* Should not be working on unaligned addresses */
2108         if (WARN_ONCE(addr & ~PAGE_MASK, "misaligned address: %#lx\n", addr))
2109                 addr &= PAGE_MASK;
2110
2111         start = addr;
2112
2113         memset(&cpa, 0, sizeof(cpa));
2114         cpa.vaddr = &addr;
2115         cpa.numpages = numpages;
2116         cpa.mask_set = enc ? __pgprot(_PAGE_ENC) : __pgprot(0);
2117         cpa.mask_clr = enc ? __pgprot(0) : __pgprot(_PAGE_ENC);
2118         cpa.pgd = init_mm.pgd;
2119
2120         /* Must avoid aliasing mappings in the highmem code */
2121         kmap_flush_unused();
2122         vm_unmap_aliases();
2123
2124         /*
2125          * Before changing the encryption attribute, we need to flush caches.
2126          */
2127         if (static_cpu_has(X86_FEATURE_CLFLUSH))
2128                 cpa_flush_range(start, numpages, 1);
2129         else
2130                 cpa_flush_all(1);
2131
2132         ret = __change_page_attr_set_clr(&cpa, 1);
2133
2134         /*
2135          * After changing the encryption attribute, we need to flush TLBs
2136          * again in case any speculative TLB caching occurred (but no need
2137          * to flush caches again).  We could just use cpa_flush_all(), but
2138          * in case TLB flushing gets optimized in the cpa_flush_range()
2139          * path use the same logic as above.
2140          */
2141         if (static_cpu_has(X86_FEATURE_CLFLUSH))
2142                 cpa_flush_range(start, numpages, 0);
2143         else
2144                 cpa_flush_all(0);
2145
2146         return ret;
2147 }
2148
2149 int set_memory_encrypted(unsigned long addr, int numpages)
2150 {
2151         return __set_memory_enc_dec(addr, numpages, true);
2152 }
2153 EXPORT_SYMBOL_GPL(set_memory_encrypted);
2154
2155 int set_memory_decrypted(unsigned long addr, int numpages)
2156 {
2157         return __set_memory_enc_dec(addr, numpages, false);
2158 }
2159 EXPORT_SYMBOL_GPL(set_memory_decrypted);
2160
2161 int set_pages_uc(struct page *page, int numpages)
2162 {
2163         unsigned long addr = (unsigned long)page_address(page);
2164
2165         return set_memory_uc(addr, numpages);
2166 }
2167 EXPORT_SYMBOL(set_pages_uc);
2168
2169 static int _set_pages_array(struct page **pages, int addrinarray,
2170                 enum page_cache_mode new_type)
2171 {
2172         unsigned long start;
2173         unsigned long end;
2174         enum page_cache_mode set_type;
2175         int i;
2176         int free_idx;
2177         int ret;
2178
2179         for (i = 0; i < addrinarray; i++) {
2180                 if (PageHighMem(pages[i]))
2181                         continue;
2182                 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
2183                 end = start + PAGE_SIZE;
2184                 if (reserve_memtype(start, end, new_type, NULL))
2185                         goto err_out;
2186         }
2187
2188         /* If WC, set to UC- first and then WC */
2189         set_type = (new_type == _PAGE_CACHE_MODE_WC) ?
2190                                 _PAGE_CACHE_MODE_UC_MINUS : new_type;
2191
2192         ret = cpa_set_pages_array(pages, addrinarray,
2193                                   cachemode2pgprot(set_type));
2194         if (!ret && new_type == _PAGE_CACHE_MODE_WC)
2195                 ret = change_page_attr_set_clr(NULL, addrinarray,
2196                                                cachemode2pgprot(
2197                                                 _PAGE_CACHE_MODE_WC),
2198                                                __pgprot(_PAGE_CACHE_MASK),
2199                                                0, CPA_PAGES_ARRAY, pages);
2200         if (ret)
2201                 goto err_out;
2202         return 0; /* Success */
2203 err_out:
2204         free_idx = i;
2205         for (i = 0; i < free_idx; i++) {
2206                 if (PageHighMem(pages[i]))
2207                         continue;
2208                 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
2209                 end = start + PAGE_SIZE;
2210                 free_memtype(start, end);
2211         }
2212         return -EINVAL;
2213 }
2214
2215 int set_pages_array_uc(struct page **pages, int addrinarray)
2216 {
2217         return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_UC_MINUS);
2218 }
2219 EXPORT_SYMBOL(set_pages_array_uc);
2220
2221 int set_pages_array_wc(struct page **pages, int addrinarray)
2222 {
2223         return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_WC);
2224 }
2225 EXPORT_SYMBOL(set_pages_array_wc);
2226
2227 int set_pages_array_wt(struct page **pages, int addrinarray)
2228 {
2229         return _set_pages_array(pages, addrinarray, _PAGE_CACHE_MODE_WT);
2230 }
2231 EXPORT_SYMBOL_GPL(set_pages_array_wt);
2232
2233 int set_pages_wb(struct page *page, int numpages)
2234 {
2235         unsigned long addr = (unsigned long)page_address(page);
2236
2237         return set_memory_wb(addr, numpages);
2238 }
2239 EXPORT_SYMBOL(set_pages_wb);
2240
2241 int set_pages_array_wb(struct page **pages, int addrinarray)
2242 {
2243         int retval;
2244         unsigned long start;
2245         unsigned long end;
2246         int i;
2247
2248         /* WB cache mode is hard wired to all cache attribute bits being 0 */
2249         retval = cpa_clear_pages_array(pages, addrinarray,
2250                         __pgprot(_PAGE_CACHE_MASK));
2251         if (retval)
2252                 return retval;
2253
2254         for (i = 0; i < addrinarray; i++) {
2255                 if (PageHighMem(pages[i]))
2256                         continue;
2257                 start = page_to_pfn(pages[i]) << PAGE_SHIFT;
2258                 end = start + PAGE_SIZE;
2259                 free_memtype(start, end);
2260         }
2261
2262         return 0;
2263 }
2264 EXPORT_SYMBOL(set_pages_array_wb);
2265
2266 int set_pages_x(struct page *page, int numpages)
2267 {
2268         unsigned long addr = (unsigned long)page_address(page);
2269
2270         return set_memory_x(addr, numpages);
2271 }
2272 EXPORT_SYMBOL(set_pages_x);
2273
2274 int set_pages_nx(struct page *page, int numpages)
2275 {
2276         unsigned long addr = (unsigned long)page_address(page);
2277
2278         return set_memory_nx(addr, numpages);
2279 }
2280 EXPORT_SYMBOL(set_pages_nx);
2281
2282 int set_pages_ro(struct page *page, int numpages)
2283 {
2284         unsigned long addr = (unsigned long)page_address(page);
2285
2286         return set_memory_ro(addr, numpages);
2287 }
2288
2289 int set_pages_rw(struct page *page, int numpages)
2290 {
2291         unsigned long addr = (unsigned long)page_address(page);
2292
2293         return set_memory_rw(addr, numpages);
2294 }
2295
2296 #ifdef CONFIG_DEBUG_PAGEALLOC
2297
2298 static int __set_pages_p(struct page *page, int numpages)
2299 {
2300         unsigned long tempaddr = (unsigned long) page_address(page);
2301         struct cpa_data cpa = { .vaddr = &tempaddr,
2302                                 .pgd = NULL,
2303                                 .numpages = numpages,
2304                                 .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW),
2305                                 .mask_clr = __pgprot(0),
2306                                 .flags = 0};
2307
2308         /*
2309          * No alias checking needed for setting present flag. otherwise,
2310          * we may need to break large pages for 64-bit kernel text
2311          * mappings (this adds to complexity if we want to do this from
2312          * atomic context especially). Let's keep it simple!
2313          */
2314         return __change_page_attr_set_clr(&cpa, 0);
2315 }
2316
2317 static int __set_pages_np(struct page *page, int numpages)
2318 {
2319         unsigned long tempaddr = (unsigned long) page_address(page);
2320         struct cpa_data cpa = { .vaddr = &tempaddr,
2321                                 .pgd = NULL,
2322                                 .numpages = numpages,
2323                                 .mask_set = __pgprot(0),
2324                                 .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW),
2325                                 .flags = 0};
2326
2327         /*
2328          * No alias checking needed for setting not present flag. otherwise,
2329          * we may need to break large pages for 64-bit kernel text
2330          * mappings (this adds to complexity if we want to do this from
2331          * atomic context especially). Let's keep it simple!
2332          */
2333         return __change_page_attr_set_clr(&cpa, 0);
2334 }
2335
2336 void __kernel_map_pages(struct page *page, int numpages, int enable)
2337 {
2338         if (PageHighMem(page))
2339                 return;
2340         if (!enable) {
2341                 debug_check_no_locks_freed(page_address(page),
2342                                            numpages * PAGE_SIZE);
2343         }
2344
2345         /*
2346          * The return value is ignored as the calls cannot fail.
2347          * Large pages for identity mappings are not used at boot time
2348          * and hence no memory allocations during large page split.
2349          */
2350         if (enable)
2351                 __set_pages_p(page, numpages);
2352         else
2353                 __set_pages_np(page, numpages);
2354
2355         /*
2356          * We should perform an IPI and flush all tlbs,
2357          * but that can deadlock->flush only current cpu:
2358          */
2359         __flush_tlb_all();
2360
2361         arch_flush_lazy_mmu_mode();
2362 }
2363
2364 #ifdef CONFIG_HIBERNATION
2365
2366 bool kernel_page_present(struct page *page)
2367 {
2368         unsigned int level;
2369         pte_t *pte;
2370
2371         if (PageHighMem(page))
2372                 return false;
2373
2374         pte = lookup_address((unsigned long)page_address(page), &level);
2375         return (pte_val(*pte) & _PAGE_PRESENT);
2376 }
2377
2378 #endif /* CONFIG_HIBERNATION */
2379
2380 #endif /* CONFIG_DEBUG_PAGEALLOC */
2381
2382 int kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, unsigned long address,
2383                             unsigned numpages, unsigned long page_flags)
2384 {
2385         int retval = -EINVAL;
2386
2387         struct cpa_data cpa = {
2388                 .vaddr = &address,
2389                 .pfn = pfn,
2390                 .pgd = pgd,
2391                 .numpages = numpages,
2392                 .mask_set = __pgprot(0),
2393                 .mask_clr = __pgprot(0),
2394                 .flags = 0,
2395         };
2396
2397         if (!(__supported_pte_mask & _PAGE_NX))
2398                 goto out;
2399
2400         if (!(page_flags & _PAGE_NX))
2401                 cpa.mask_clr = __pgprot(_PAGE_NX);
2402
2403         if (!(page_flags & _PAGE_RW))
2404                 cpa.mask_clr = __pgprot(_PAGE_RW);
2405
2406         if (!(page_flags & _PAGE_ENC))
2407                 cpa.mask_clr = pgprot_encrypted(cpa.mask_clr);
2408
2409         cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags);
2410
2411         retval = __change_page_attr_set_clr(&cpa, 0);
2412         __flush_tlb_all();
2413
2414 out:
2415         return retval;
2416 }
2417
2418 /*
2419  * The testcases use internal knowledge of the implementation that shouldn't
2420  * be exposed to the rest of the kernel. Include these directly here.
2421  */
2422 #ifdef CONFIG_CPA_DEBUG
2423 #include "pageattr-test.c"
2424 #endif