mm/mmu_notifier: use correct mmu_notifier events for each invalidation

This updates each existing invalidation to use the correct mmu notifier
event that represent what is happening to the CPU page table.  See the
patch which introduced the events to see the rational behind this.

Link: http://lkml.kernel.org/r/20190326164747.24405-7-jglisse@redhat.com
Signed-off-by: Jérôme Glisse <redacted>
Reviewed-by: Ralph Campbell <redacted>
Reviewed-by: Ira Weiny <redacted>
Cc: Christian König <redacted>
Cc: Joonas Lahtinen <redacted>
Cc: Jani Nikula <redacted>
Cc: Rodrigo Vivi <redacted>
Cc: Jan Kara <redacted>
Cc: Andrea Arcangeli <redacted>
Cc: Peter Xu <redacted>
Cc: Felix Kuehling <redacted>
Cc: Jason Gunthorpe <redacted>
Cc: Ross Zwisler <redacted>
Cc: Dan Williams <redacted>
Cc: Paolo Bonzini <redacted>
Cc: Radim Krcmar <redacted>
Cc: Michal Hocko <redacted>
Cc: Christian Koenig <redacted>
Cc: John Hubbard <redacted>
Cc: Arnd Bergmann <redacted>
Signed-off-by: Andrew Morton <redacted>
Signed-off-by: Linus Torvalds <redacted>

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index ea464f2b986797e92fd05df56767d4e67a67dca2..01d4eb0e6bd1155f38df153915bfaf20b96406c8 100644 (file)
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -1169,8 +1169,8 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
                                break;
                        }
 
                                break;
                        }
 

-                       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0,
-                                               NULL, mm, 0, -1UL);
+                       mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY,
+                                               0, NULL, mm, 0, -1UL);

                        mmu_notifier_invalidate_range_start(&range);
                }
                walk_page_range(0, mm->highest_vm_end, &clear_refs_walk);
                        mmu_notifier_invalidate_range_start(&range);
                }
                walk_page_range(0, mm->highest_vm_end, &clear_refs_walk);

diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index e34b699f3865cb5eca63075896ee3595d57cf6ad..78f61bfc6b7976128d623a7915c5a394620b0bbd 100644 (file)
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -161,7 +161,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
        struct mmu_notifier_range range;
        struct mem_cgroup *memcg;
 
        struct mmu_notifier_range range;
        struct mem_cgroup *memcg;
 

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, addr,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr,

                                addr + PAGE_SIZE);
 
        VM_BUG_ON_PAGE(PageTransHuge(old_page), old_page);
                                addr + PAGE_SIZE);
 
        VM_BUG_ON_PAGE(PageTransHuge(old_page), old_page);

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 428b5794f4b87f8e7f5c97015721e9d8d6a7e1bf..61b1e05e86ee24df965b38fbc0db4717f71ba909 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1224,9 +1224,8 @@ static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf,
                cond_resched();
        }
 
                cond_resched();
        }
 

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
-                               haddr,
-                               haddr + HPAGE_PMD_SIZE);
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
+                               haddr, haddr + HPAGE_PMD_SIZE);

        mmu_notifier_invalidate_range_start(&range);
 
        vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
        mmu_notifier_invalidate_range_start(&range);
 
        vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);


@@ -1389,9 +1388,8 @@ alloc:
                                    vma, HPAGE_PMD_NR);
        __SetPageUptodate(new_page);
 
                                    vma, HPAGE_PMD_NR);
        __SetPageUptodate(new_page);
 

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
-                               haddr,
-                               haddr + HPAGE_PMD_SIZE);
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
+                               haddr, haddr + HPAGE_PMD_SIZE);

        mmu_notifier_invalidate_range_start(&range);
 
        spin_lock(vmf->ptl);
        mmu_notifier_invalidate_range_start(&range);
 
        spin_lock(vmf->ptl);


@@ -2066,7 +2064,7 @@ void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
        spinlock_t *ptl;
        struct mmu_notifier_range range;
 
        spinlock_t *ptl;
        struct mmu_notifier_range range;
 

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,

                                address & HPAGE_PUD_MASK,
                                (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE);
        mmu_notifier_invalidate_range_start(&range);
                                address & HPAGE_PUD_MASK,
                                (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE);
        mmu_notifier_invalidate_range_start(&range);


@@ -2285,7 +2283,7 @@ void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
        spinlock_t *ptl;
        struct mmu_notifier_range range;
 
        spinlock_t *ptl;
        struct mmu_notifier_range range;
 

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,

                                address & HPAGE_PMD_MASK,
                                (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE);
        mmu_notifier_invalidate_range_start(&range);
                                address & HPAGE_PMD_MASK,
                                (address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE);
        mmu_notifier_invalidate_range_start(&range);

diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 89d206d6ecf39f33c108ad9adec9db983ca90bee..cab38ef30238ce2f15c583045b1e1c641b0ca32e 100644 (file)
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3294,7 +3294,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
        cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
 
        if (cow) {
        cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
 
        if (cow) {

-               mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, src,
+               mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, src,

                                        vma->vm_start,
                                        vma->vm_end);
                mmu_notifier_invalidate_range_start(&range);
                                        vma->vm_start,
                                        vma->vm_end);
                mmu_notifier_invalidate_range_start(&range);


@@ -3675,7 +3675,7 @@ retry_avoidcopy:
                            pages_per_huge_page(h));
        __SetPageUptodate(new_page);
 
                            pages_per_huge_page(h));
        __SetPageUptodate(new_page);
 

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, haddr,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, haddr,

                                haddr + huge_page_size(h));
        mmu_notifier_invalidate_range_start(&range);
 
                                haddr + huge_page_size(h));
        mmu_notifier_invalidate_range_start(&range);
 


@@ -4411,8 +4411,8 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
         * start/end.  Set range.start/range.end to cover the maximum possible
         * range if PMD sharing is possible.
         */
         * start/end.  Set range.start/range.end to cover the maximum possible
         * range if PMD sharing is possible.
         */

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, start,
-                               end);
+       mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
+                               0, vma, mm, start, end);

        adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
 
        BUG_ON(address >= end);
        adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
 
        BUG_ON(address >= end);

diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 14581dbf62a5a5eeab084130fe5f9151d9a0b3e5..a335f7c1fac428b30d7e352ebaf717e033724f74 100644 (file)
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1016,7 +1016,7 @@ static void collapse_huge_page(struct mm_struct *mm,
        pte = pte_offset_map(pmd, address);
        pte_ptl = pte_lockptr(mm, pmd);
 
        pte = pte_offset_map(pmd, address);
        pte_ptl = pte_lockptr(mm, pmd);
 

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, NULL, mm,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm,

                                address, address + HPAGE_PMD_SIZE);
        mmu_notifier_invalidate_range_start(&range);
        pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
                                address, address + HPAGE_PMD_SIZE);
        mmu_notifier_invalidate_range_start(&range);
        pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */

diff --git a/mm/ksm.c b/mm/ksm.c
index 01f5fe2c90cfb96b014faadd5c96acc06b403800..81c20ed57bf68077041f195e2e2d8283ce8c1eb7 100644 (file)
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -1066,7 +1066,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page,
 
        BUG_ON(PageTransCompound(page));
 
 
        BUG_ON(PageTransCompound(page));
 

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,

                                pvmw.address,
                                pvmw.address + PAGE_SIZE);
        mmu_notifier_invalidate_range_start(&range);
                                pvmw.address,
                                pvmw.address + PAGE_SIZE);
        mmu_notifier_invalidate_range_start(&range);


@@ -1155,7 +1155,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page,
        if (!pmd)
                goto out;
 
        if (!pmd)
                goto out;
 

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, addr,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr,

                                addr + PAGE_SIZE);
        mmu_notifier_invalidate_range_start(&range);
 
                                addr + PAGE_SIZE);
        mmu_notifier_invalidate_range_start(&range);
 

diff --git a/mm/madvise.c b/mm/madvise.c
index 1c52bdf1b6962633f4989bb2895c97501bbedfcc..628022e674a7f52bc8c600106113ac0d8080dbb4 100644 (file)
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -472,7 +472,7 @@ static int madvise_free_single_vma(struct vm_area_struct *vma,
        range.end = min(vma->vm_end, end_addr);
        if (range.end <= vma->vm_start)
                return -EINVAL;
        range.end = min(vma->vm_end, end_addr);
        if (range.end <= vma->vm_start)
                return -EINVAL;

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,

                                range.start, range.end);
 
        lru_add_drain();
                                range.start, range.end);
 
        lru_add_drain();

diff --git a/mm/memory.c b/mm/memory.c
index 90672674c5828b58ee91b8279cd209e60236f52c..9b68a72f8c17593423c52089862b75fb89954941 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1010,8 +1010,8 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
        is_cow = is_cow_mapping(vma->vm_flags);
 
        if (is_cow) {
        is_cow = is_cow_mapping(vma->vm_flags);
 
        if (is_cow) {

-               mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma,
-                                       src_mm, addr, end);
+               mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
+                                       0, vma, src_mm, addr, end);

                mmu_notifier_invalidate_range_start(&range);
        }
 
                mmu_notifier_invalidate_range_start(&range);
        }
 


@@ -1358,7 +1358,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start,
        struct mmu_gather tlb;
 
        lru_add_drain();
        struct mmu_gather tlb;
 
        lru_add_drain();

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,

                                start, start + size);
        tlb_gather_mmu(&tlb, vma->vm_mm, start, range.end);
        update_hiwater_rss(vma->vm_mm);
                                start, start + size);
        tlb_gather_mmu(&tlb, vma->vm_mm, start, range.end);
        update_hiwater_rss(vma->vm_mm);


@@ -1385,7 +1385,7 @@ static void zap_page_range_single(struct vm_area_struct *vma, unsigned long addr
        struct mmu_gather tlb;
 
        lru_add_drain();
        struct mmu_gather tlb;
 
        lru_add_drain();

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,

                                address, address + size);
        tlb_gather_mmu(&tlb, vma->vm_mm, address, range.end);
        update_hiwater_rss(vma->vm_mm);
                                address, address + size);
        tlb_gather_mmu(&tlb, vma->vm_mm, address, range.end);
        update_hiwater_rss(vma->vm_mm);


@@ -2283,7 +2283,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
 
        __SetPageUptodate(new_page);
 
 
        __SetPageUptodate(new_page);
 

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,

                                vmf->address & PAGE_MASK,
                                (vmf->address & PAGE_MASK) + PAGE_SIZE);
        mmu_notifier_invalidate_range_start(&range);
                                vmf->address & PAGE_MASK,
                                (vmf->address & PAGE_MASK) + PAGE_SIZE);
        mmu_notifier_invalidate_range_start(&range);


@@ -4109,7 +4109,7 @@ static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address,
                        goto out;
 
                if (range) {
                        goto out;
 
                if (range) {

-                       mmu_notifier_range_init(range, MMU_NOTIFY_UNMAP, 0,
+                       mmu_notifier_range_init(range, MMU_NOTIFY_CLEAR, 0,

                                                NULL, mm, address & PMD_MASK,
                                                (address & PMD_MASK) + PMD_SIZE);
                        mmu_notifier_invalidate_range_start(range);
                                                NULL, mm, address & PMD_MASK,
                                                (address & PMD_MASK) + PMD_SIZE);
                        mmu_notifier_invalidate_range_start(range);


@@ -4128,7 +4128,7 @@ static int __follow_pte_pmd(struct mm_struct *mm, unsigned long address,
                goto out;
 
        if (range) {
                goto out;
 
        if (range) {

-               mmu_notifier_range_init(range, MMU_NOTIFY_UNMAP, 0, NULL, mm,
+               mmu_notifier_range_init(range, MMU_NOTIFY_CLEAR, 0, NULL, mm,

                                        address & PAGE_MASK,
                                        (address & PAGE_MASK) + PAGE_SIZE);
                mmu_notifier_invalidate_range_start(range);
                                        address & PAGE_MASK,
                                        (address & PAGE_MASK) + PAGE_SIZE);
                mmu_notifier_invalidate_range_start(range);

diff --git a/mm/migrate.c b/mm/migrate.c
index 855bdb3b3333b2ab804faefd2fd226a24aea11cc..f2ecc2855a12d72b43dc73e7914400ee334a06bf 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -2356,7 +2356,7 @@ static void migrate_vma_collect(struct migrate_vma *migrate)
        mm_walk.mm = migrate->vma->vm_mm;
        mm_walk.private = migrate;
 
        mm_walk.mm = migrate->vma->vm_mm;
        mm_walk.private = migrate;
 

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, NULL, mm_walk.mm,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm_walk.mm,

                                migrate->start,
                                migrate->end);
        mmu_notifier_invalidate_range_start(&range);
                                migrate->start,
                                migrate->end);
        mmu_notifier_invalidate_range_start(&range);


@@ -2765,7 +2765,7 @@ static void migrate_vma_pages(struct migrate_vma *migrate)
                                notified = true;
 
                                mmu_notifier_range_init(&range,
                                notified = true;
 
                                mmu_notifier_range_init(&range,

-                                                       MMU_NOTIFY_UNMAP, 0,
+                                                       MMU_NOTIFY_CLEAR, 0,

                                                        NULL,
                                                        migrate->vma->vm_mm,
                                                        addr, migrate->end);
                                                        NULL,
                                                        migrate->vma->vm_mm,
                                                        addr, migrate->end);

diff --git a/mm/mprotect.c b/mm/mprotect.c
index b10984052ae9f39987c0ebb1f63fc55d0560702e..65242f1e445779c19c0ed327ff76aac3afe2bdad 100644 (file)
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -185,8 +185,9 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
 
                /* invoke the mmu notifier if the pmd is populated */
                if (!range.start) {
 
                /* invoke the mmu notifier if the pmd is populated */
                if (!range.start) {

-                       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0,
-                                               vma, vma->vm_mm, addr, end);
+                       mmu_notifier_range_init(&range,
+                               MMU_NOTIFY_PROTECTION_VMA, 0,
+                               vma, vma->vm_mm, addr, end);

                        mmu_notifier_invalidate_range_start(&range);
                }
 
                        mmu_notifier_invalidate_range_start(&range);
                }
 

diff --git a/mm/rmap.c b/mm/rmap.c
index 288e636b78132c430d0bbe6dfacf8c474064a3a7..0cbed70700ed333dbac1493684a928268dbee1d1 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -896,8 +896,8 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
         * We have to assume the worse case ie pmd for invalidation. Note that
         * the page can not be free from this function.
         */
         * We have to assume the worse case ie pmd for invalidation. Note that
         * the page can not be free from this function.
         */

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
-                               address,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
+                               0, vma, vma->vm_mm, address,

                                min(vma->vm_end, address +
                                    (PAGE_SIZE << compound_order(page))));
        mmu_notifier_invalidate_range_start(&range);
                                min(vma->vm_end, address +
                                    (PAGE_SIZE << compound_order(page))));
        mmu_notifier_invalidate_range_start(&range);


@@ -1372,7 +1372,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
         * Note that the page can not be free in this function as call of
         * try_to_unmap() must hold a reference on the page.
         */
         * Note that the page can not be free in this function as call of
         * try_to_unmap() must hold a reference on the page.
         */

-       mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm,
+       mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,

                                address,
                                min(vma->vm_end, address +
                                    (PAGE_SIZE << compound_order(page))));
                                address,
                                min(vma->vm_end, address +
                                    (PAGE_SIZE << compound_order(page))));