KVM: PPC: Book3S HV: Implement dirty page logging for radix guests

This adds code to keep track of dirty pages when requested (that is,
when memslot->dirty_bitmap is non-NULL) for radix guests.  We use the
dirty bits in the PTEs in the second-level (partition-scoped) page
tables, together with a bitmap of pages that were dirty when their
PTE was invalidated (e.g., when the page was paged out).  This bitmap
is stored in the first half of the memslot->dirty_bitmap area, and
kvm_vm_ioctl_get_dirty_log_hv() now uses the second half for the
bitmap that gets returned to userspace.

Signed-off-by: Paul Mackerras <paulus@ozlabs.org>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>

diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h
index 952cc4b954a1b9e5eed0bbd3a7b205eb4e0312a1..57dc407cec4aafbfd325a2725432fe4c4dac5a93 100644 (file)
--- a/arch/powerpc/include/asm/kvm_book3s.h
+++ b/arch/powerpc/include/asm/kvm_book3s.h
@@ -198,6 +198,8 @@ extern int kvm_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
                        unsigned long gfn);
 extern int kvm_test_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
                        unsigned long gfn);
+extern long kvmppc_hv_get_dirty_log_radix(struct kvm *kvm,
+                       struct kvm_memory_slot *memslot, unsigned long *map);
 
 /* XXX remove this export when load_last_inst() is generic */
 extern int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, bool data);
@@ -228,8 +230,11 @@ extern long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
 extern long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags,
                        unsigned long pte_index, unsigned long avpn,
                        unsigned long *hpret);
-extern long kvmppc_hv_get_dirty_log(struct kvm *kvm,
+extern long kvmppc_hv_get_dirty_log_hpt(struct kvm *kvm,
                        struct kvm_memory_slot *memslot, unsigned long *map);
+extern void kvmppc_harvest_vpa_dirty(struct kvmppc_vpa *vpa,
+                       struct kvm_memory_slot *memslot,
+                       unsigned long *map);
 extern void kvmppc_update_lpcr(struct kvm *kvm, unsigned long lpcr,
                        unsigned long mask);
 extern void kvmppc_set_fscr(struct kvm_vcpu *vcpu, u64 fscr);

diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c
index 088c82bb7ba4ba09b5fd5cc787538064b578195c..c9e587a2849dac3e58a975cd15d05900a15ffffd 100644 (file)
--- a/arch/powerpc/kvm/book3s_64_mmu_hv.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c
@@ -1068,7 +1068,7 @@ static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
        return npages_dirty;
 }
 
-static void harvest_vpa_dirty(struct kvmppc_vpa *vpa,
+void kvmppc_harvest_vpa_dirty(struct kvmppc_vpa *vpa,
                              struct kvm_memory_slot *memslot,
                              unsigned long *map)
 {
@@ -1086,12 +1086,11 @@ static void harvest_vpa_dirty(struct kvmppc_vpa *vpa,
                __set_bit_le(gfn - memslot->base_gfn, map);
 }
 
-long kvmppc_hv_get_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot,
-                            unsigned long *map)
+long kvmppc_hv_get_dirty_log_hpt(struct kvm *kvm,
+                       struct kvm_memory_slot *memslot, unsigned long *map)
 {
        unsigned long i, j;
        unsigned long *rmapp;
-       struct kvm_vcpu *vcpu;
 
        preempt_disable();
        rmapp = memslot->arch.rmap;
@@ -1107,15 +1106,6 @@ long kvmppc_hv_get_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot,
                                __set_bit_le(j, map);
                ++rmapp;
        }
-
-       /* Harvest dirty bits from VPA and DTL updates */
-       /* Note: we never modify the SLB shadow buffer areas */
-       kvm_for_each_vcpu(i, vcpu, kvm) {
-               spin_lock(&vcpu->arch.vpa_update_lock);
-               harvest_vpa_dirty(&vcpu->arch.vpa, memslot, map);
-               harvest_vpa_dirty(&vcpu->arch.dtl, memslot, map);
-               spin_unlock(&vcpu->arch.vpa_update_lock);
-       }
        preempt_enable();
        return 0;
 }
@@ -1170,10 +1160,14 @@ void kvmppc_unpin_guest_page(struct kvm *kvm, void *va, unsigned long gpa,
        srcu_idx = srcu_read_lock(&kvm->srcu);
        memslot = gfn_to_memslot(kvm, gfn);
        if (memslot) {
-               rmap = &memslot->arch.rmap[gfn - memslot->base_gfn];
-               lock_rmap(rmap);
-               *rmap |= KVMPPC_RMAP_CHANGED;
-               unlock_rmap(rmap);
+               if (!kvm_is_radix(kvm)) {
+                       rmap = &memslot->arch.rmap[gfn - memslot->base_gfn];
+                       lock_rmap(rmap);
+                       *rmap |= KVMPPC_RMAP_CHANGED;
+                       unlock_rmap(rmap);
+               } else if (memslot->dirty_bitmap) {
+                       mark_page_dirty(kvm, gfn);
+               }
        }
        srcu_read_unlock(&kvm->srcu, srcu_idx);
 }

diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c
index 69cabadc121a53ce26da2a51284ab3df1a29d1bb..125cc7ce1525014e468fb2adf2e03a3a82bd0cf1 100644 (file)
--- a/arch/powerpc/kvm/book3s_64_mmu_radix.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c
@@ -158,18 +158,21 @@ static void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr,
        asm volatile("ptesync": : :"memory");
 }
 
-void kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep, unsigned long clr,
-                            unsigned long set, unsigned long addr,
-                            unsigned int shift)
+unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep,
+                                     unsigned long clr, unsigned long set,
+                                     unsigned long addr, unsigned int shift)
 {
+       unsigned long old = 0;
+
        if (!(clr & _PAGE_PRESENT) && cpu_has_feature(CPU_FTR_POWER9_DD1) &&
            pte_present(*ptep)) {
                /* have to invalidate it first */
-               __radix_pte_update(ptep, _PAGE_PRESENT, 0);
+               old = __radix_pte_update(ptep, _PAGE_PRESENT, 0);
                kvmppc_radix_tlbie_page(kvm, addr, shift);
                set |= _PAGE_PRESENT;
+               old &= _PAGE_PRESENT;
        }
-       __radix_pte_update(ptep, clr, set);
+       return __radix_pte_update(ptep, clr, set) | old;
 }
 
 void kvmppc_radix_set_pte_at(struct kvm *kvm, unsigned long addr,
@@ -197,6 +200,7 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
        pud_t *pud, *new_pud = NULL;
        pmd_t *pmd, *new_pmd = NULL;
        pte_t *ptep, *new_ptep = NULL;
+       unsigned long old;
        int ret;
 
        /* Traverse the guest's 2nd-level tree, allocate new levels needed */
@@ -262,9 +266,11 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
                ptep = pte_offset_kernel(pmd, gpa);
                if (pte_present(*ptep)) {
                        /* PTE was previously valid, so invalidate it */
-                       kvmppc_radix_update_pte(kvm, ptep, _PAGE_PRESENT,
-                                               0, gpa, 0);
+                       old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_PRESENT,
+                                                     0, gpa, 0);
                        kvmppc_radix_tlbie_page(kvm, gpa, 0);
+                       if (old & _PAGE_DIRTY)
+                               mark_page_dirty(kvm, gpa >> PAGE_SHIFT);
                }
                kvmppc_radix_set_pte_at(kvm, gpa, ptep, pte);
        } else {
@@ -463,6 +469,26 @@ int kvmppc_book3s_radix_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
        return ret;
 }
 
+static void mark_pages_dirty(struct kvm *kvm, struct kvm_memory_slot *memslot,
+                            unsigned long gfn, unsigned int order)
+{
+       unsigned long i, limit;
+       unsigned long *dp;
+
+       if (!memslot->dirty_bitmap)
+               return;
+       limit = 1ul << order;
+       if (limit < BITS_PER_LONG) {
+               for (i = 0; i < limit; ++i)
+                       mark_page_dirty(kvm, gfn + i);
+               return;
+       }
+       dp = memslot->dirty_bitmap + (gfn - memslot->base_gfn);
+       limit /= BITS_PER_LONG;
+       for (i = 0; i < limit; ++i)
+               *dp++ = ~0ul;
+}
+
 /* Called with kvm->lock held */
 int kvm_unmap_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
                    unsigned long gfn)
@@ -470,13 +496,21 @@ int kvm_unmap_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
        pte_t *ptep;
        unsigned long gpa = gfn << PAGE_SHIFT;
        unsigned int shift;
+       unsigned long old;
 
        ptep = __find_linux_pte_or_hugepte(kvm->arch.pgtable, gpa,
                                           NULL, &shift);
        if (ptep && pte_present(*ptep)) {
-               kvmppc_radix_update_pte(kvm, ptep, _PAGE_PRESENT, 0,
-                                       gpa, shift);
+               old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_PRESENT, 0,
+                                             gpa, shift);
                kvmppc_radix_tlbie_page(kvm, gpa, shift);
+               if (old & _PAGE_DIRTY) {
+                       if (!shift)
+                               mark_page_dirty(kvm, gfn);
+                       else
+                               mark_pages_dirty(kvm, memslot,
+                                                gfn, shift - PAGE_SHIFT);
+               }
        }
        return 0;                               
 }
@@ -517,6 +551,65 @@ int kvm_test_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
        return ref;
 }
 
+/* Returns the number of PAGE_SIZE pages that are dirty */
+static int kvm_radix_test_clear_dirty(struct kvm *kvm,
+                               struct kvm_memory_slot *memslot, int pagenum)
+{
+       unsigned long gfn = memslot->base_gfn + pagenum;
+       unsigned long gpa = gfn << PAGE_SHIFT;
+       pte_t *ptep;
+       unsigned int shift;
+       int ret = 0;
+
+       ptep = __find_linux_pte_or_hugepte(kvm->arch.pgtable, gpa,
+                                          NULL, &shift);
+       if (ptep && pte_present(*ptep) && pte_dirty(*ptep)) {
+               ret = 1;
+               if (shift)
+                       ret = 1 << (shift - PAGE_SHIFT);
+               kvmppc_radix_update_pte(kvm, ptep, _PAGE_DIRTY, 0,
+                                       gpa, shift);
+               kvmppc_radix_tlbie_page(kvm, gpa, shift);
+       }
+       return ret;
+}
+
+long kvmppc_hv_get_dirty_log_radix(struct kvm *kvm,
+                       struct kvm_memory_slot *memslot, unsigned long *map)
+{
+       unsigned long i, j;
+       unsigned long n, *p;
+       int npages;
+
+       /*
+        * Radix accumulates dirty bits in the first half of the
+        * memslot's dirty_bitmap area, for when pages are paged
+        * out or modified by the host directly.  Pick up these
+        * bits and add them to the map.
+        */
+       n = kvm_dirty_bitmap_bytes(memslot) / sizeof(long);
+       p = memslot->dirty_bitmap;
+       for (i = 0; i < n; ++i)
+               map[i] |= xchg(&p[i], 0);
+
+       for (i = 0; i < memslot->npages; i = j) {
+               npages = kvm_radix_test_clear_dirty(kvm, memslot, i);
+
+               /*
+                * Note that if npages > 0 then i must be a multiple of npages,
+                * since huge pages are only used to back the guest at guest
+                * real addresses that are a multiple of their size.
+                * Since we have at most one PTE covering any given guest
+                * real address, if npages > 1 we can skip to i + npages.
+                */
+               j = i + 1;
+               if (npages)
+                       for (j = i; npages; ++j, --npages)
+                               __set_bit_le(j, map);
+       }
+       return 0;
+}
+
 void kvmppc_free_radix(struct kvm *kvm)
 {
        unsigned long ig, iu, im;

diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index d50251f9a3c9919951618eb8d4a77d15383fa0cd..401e4cc8a91f9744bf36389b5c2d55a56a43129a 100644 (file)
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -2961,8 +2961,10 @@ static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
 {
        struct kvm_memslots *slots;
        struct kvm_memory_slot *memslot;
-       int r;
+       int i, r;
        unsigned long n;
+       unsigned long *buf;
+       struct kvm_vcpu *vcpu;
 
        mutex_lock(&kvm->slots_lock);
 
@@ -2976,15 +2978,32 @@ static int kvm_vm_ioctl_get_dirty_log_hv(struct kvm *kvm,
        if (!memslot->dirty_bitmap)
                goto out;
 
+       /*
+        * Use second half of bitmap area because radix accumulates
+        * bits in the first half.
+        */
        n = kvm_dirty_bitmap_bytes(memslot);
-       memset(memslot->dirty_bitmap, 0, n);
+       buf = memslot->dirty_bitmap + n / sizeof(long);
+       memset(buf, 0, n);
 
-       r = kvmppc_hv_get_dirty_log(kvm, memslot, memslot->dirty_bitmap);
+       if (kvm_is_radix(kvm))
+               r = kvmppc_hv_get_dirty_log_radix(kvm, memslot, buf);
+       else
+               r = kvmppc_hv_get_dirty_log_hpt(kvm, memslot, buf);
        if (r)
                goto out;
 
+       /* Harvest dirty bits from VPA and DTL updates */
+       /* Note: we never modify the SLB shadow buffer areas */
+       kvm_for_each_vcpu(i, vcpu, kvm) {
+               spin_lock(&vcpu->arch.vpa_update_lock);
+               kvmppc_harvest_vpa_dirty(&vcpu->arch.vpa, memslot, buf);
+               kvmppc_harvest_vpa_dirty(&vcpu->arch.dtl, memslot, buf);
+               spin_unlock(&vcpu->arch.vpa_update_lock);
+       }
+
        r = -EFAULT;
-       if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
+       if (copy_to_user(log->dirty_bitmap, buf, n))
                goto out;
 
        r = 0;
@@ -3037,7 +3056,7 @@ static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
        if (npages)
                atomic64_inc(&kvm->arch.mmio_update);
 
-       if (npages && old->npages) {
+       if (npages && old->npages && !kvm_is_radix(kvm)) {
                /*
                 * If modifying a memslot, reset all the rmap dirty bits.
                 * If this is a new memslot, we don't need to do anything
@@ -3046,7 +3065,7 @@ static void kvmppc_core_commit_memory_region_hv(struct kvm *kvm,
                 */
                slots = kvm_memslots(kvm);
                memslot = id_to_memslot(slots, mem->slot);
-               kvmppc_hv_get_dirty_log(kvm, memslot, NULL);
+               kvmppc_hv_get_dirty_log_hpt(kvm, memslot, NULL);
        }
 }