From ca5999fde0a1761665a38e4c9a72dbcd7d190a81 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 8 Jun 2020 21:32:38 -0700 Subject: mm: introduce include/linux/pgtable.h The include/linux/pgtable.h is going to be the home of generic page table manipulation functions. Start with moving asm-generic/pgtable.h to include/linux/pgtable.h and make the latter include asm/pgtable.h. Signed-off-by: Mike Rapoport Signed-off-by: Andrew Morton Cc: Arnd Bergmann Cc: Borislav Petkov Cc: Brian Cain Cc: Catalin Marinas Cc: Chris Zankel Cc: "David S. Miller" Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Ungerer Cc: Guan Xuetao Cc: Guo Ren Cc: Heiko Carstens Cc: Helge Deller Cc: Ingo Molnar Cc: Ley Foon Tan Cc: Mark Salter Cc: Matthew Wilcox Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Nick Hu Cc: Paul Walmsley Cc: Richard Weinberger Cc: Rich Felker Cc: Russell King Cc: Stafford Horne Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Tony Luck Cc: Vincent Chen Cc: Vineet Gupta Cc: Will Deacon Cc: Yoshinori Sato Link: http://lkml.kernel.org/r/20200514170327.31389-3-rppt@kernel.org Signed-off-by: Linus Torvalds --- include/linux/pgtable.h | 1323 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1323 insertions(+) create mode 100644 include/linux/pgtable.h (limited to 'include/linux/pgtable.h') diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h new file mode 100644 index 000000000000..6e274da637fd --- /dev/null +++ b/include/linux/pgtable.h @@ -0,0 +1,1323 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_PGTABLE_H +#define _LINUX_PGTABLE_H + +#include +#include + +#ifndef __ASSEMBLY__ +#ifdef CONFIG_MMU + +#include +#include +#include +#include + +#if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ + defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS +#error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED +#endif + +/* + * On almost all architectures and configurations, 0 can be used as the + * upper ceiling to free_pgtables(): on many architectures it has the same + * effect as using TASK_SIZE. However, there is one configuration which + * must impose a more careful limit, to avoid freeing kernel pgtables. + */ +#ifndef USER_PGTABLES_CEILING +#define USER_PGTABLES_CEILING 0UL +#endif + +#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +extern int ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep, + pte_t entry, int dirty); +#endif + +#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +extern int pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty); +extern int pudp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pud_t *pudp, + pud_t entry, int dirty); +#else +static inline int pmdp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + pmd_t entry, int dirty) +{ + BUILD_BUG(); + return 0; +} +static inline int pudp_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pud_t *pudp, + pud_t entry, int dirty) +{ + BUILD_BUG(); + return 0; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif + +#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG +static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pte_t *ptep) +{ + pte_t pte = *ptep; + int r = 1; + if (!pte_young(pte)) + r = 0; + else + set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte)); + return r; +} +#endif + +#ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp) +{ + pmd_t pmd = *pmdp; + int r = 1; + if (!pmd_young(pmd)) + r = 0; + else + set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); + return r; +} +#else +static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp) +{ + BUILD_BUG(); + return 0; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif + +#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +int ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep); +#endif + +#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +extern int pmdp_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp); +#else +/* + * Despite relevant to THP only, this API is called from generic rmap code + * under PageTransHuge(), hence needs a dummy implementation for !THP + */ +static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) +{ + BUILD_BUG(); + return 0; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif + +#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, + pte_t *ptep) +{ + pte_t pte = *ptep; + pte_clear(mm, address, ptep); + return pte; +} +#endif + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR +static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long address, + pmd_t *pmdp) +{ + pmd_t pmd = *pmdp; + pmd_clear(pmdp); + return pmd; +} +#endif /* __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR */ +#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR +static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, + unsigned long address, + pud_t *pudp) +{ + pud_t pud = *pudp; + + pud_clear(pudp); + return pud; +} +#endif /* __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR */ +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL +static inline pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, + int full) +{ + return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); +} +#endif + +#ifndef __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR_FULL +static inline pud_t pudp_huge_get_and_clear_full(struct mm_struct *mm, + unsigned long address, pud_t *pudp, + int full) +{ + return pudp_huge_get_and_clear(mm, address, pudp); +} +#endif +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL +static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, + unsigned long address, pte_t *ptep, + int full) +{ + pte_t pte; + pte = ptep_get_and_clear(mm, address, ptep); + return pte; +} +#endif + + +/* + * If two threads concurrently fault at the same page, the thread that + * won the race updates the PTE and its local TLB/Cache. The other thread + * gives up, simply does nothing, and continues; on architectures where + * software can update TLB, local TLB can be updated here to avoid next page + * fault. This function updates TLB only, do nothing with cache or others. + * It is the difference with function update_mmu_cache. + */ +#ifndef __HAVE_ARCH_UPDATE_MMU_TLB +static inline void update_mmu_tlb(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) +{ +} +#define __HAVE_ARCH_UPDATE_MMU_TLB +#endif + +/* + * Some architectures may be able to avoid expensive synchronization + * primitives when modifications are made to PTE's which are already + * not present, or in the process of an address space destruction. + */ +#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL +static inline void pte_clear_not_present_full(struct mm_struct *mm, + unsigned long address, + pte_t *ptep, + int full) +{ + pte_clear(mm, address, ptep); +} +#endif + +#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH +extern pte_t ptep_clear_flush(struct vm_area_struct *vma, + unsigned long address, + pte_t *ptep); +#endif + +#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH +extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp); +extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, + unsigned long address, + pud_t *pudp); +#endif + +#ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT +struct mm_struct; +static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) +{ + pte_t old_pte = *ptep; + set_pte_at(mm, address, ptep, pte_wrprotect(old_pte)); +} +#endif + +/* + * On some architectures hardware does not set page access bit when accessing + * memory page, it is responsibilty of software setting this bit. It brings + * out extra page fault penalty to track page access bit. For optimization page + * access bit can be set during all page fault flow on these arches. + * To be differentiate with macro pte_mkyoung, this macro is used on platforms + * where software maintains page access bit. + */ +#ifndef pte_sw_mkyoung +static inline pte_t pte_sw_mkyoung(pte_t pte) +{ + return pte; +} +#define pte_sw_mkyoung pte_sw_mkyoung +#endif + +#ifndef pte_savedwrite +#define pte_savedwrite pte_write +#endif + +#ifndef pte_mk_savedwrite +#define pte_mk_savedwrite pte_mkwrite +#endif + +#ifndef pte_clear_savedwrite +#define pte_clear_savedwrite pte_wrprotect +#endif + +#ifndef pmd_savedwrite +#define pmd_savedwrite pmd_write +#endif + +#ifndef pmd_mk_savedwrite +#define pmd_mk_savedwrite pmd_mkwrite +#endif + +#ifndef pmd_clear_savedwrite +#define pmd_clear_savedwrite pmd_wrprotect +#endif + +#ifndef __HAVE_ARCH_PMDP_SET_WRPROTECT +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline void pmdp_set_wrprotect(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) +{ + pmd_t old_pmd = *pmdp; + set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd)); +} +#else +static inline void pmdp_set_wrprotect(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) +{ + BUILD_BUG(); +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif +#ifndef __HAVE_ARCH_PUDP_SET_WRPROTECT +#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD +static inline void pudp_set_wrprotect(struct mm_struct *mm, + unsigned long address, pud_t *pudp) +{ + pud_t old_pud = *pudp; + + set_pud_at(mm, address, pudp, pud_wrprotect(old_pud)); +} +#else +static inline void pudp_set_wrprotect(struct mm_struct *mm, + unsigned long address, pud_t *pudp) +{ + BUILD_BUG(); +} +#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ +#endif + +#ifndef pmdp_collapse_flush +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +extern pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp); +#else +static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp) +{ + BUILD_BUG(); + return *pmdp; +} +#define pmdp_collapse_flush pmdp_collapse_flush +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif + +#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT +extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pgtable); +#endif + +#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW +extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); +#endif + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +/* + * This is an implementation of pmdp_establish() that is only suitable for an + * architecture that doesn't have hardware dirty/accessed bits. In this case we + * can't race with CPU which sets these bits and non-atomic aproach is fine. + */ +static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, pmd_t pmd) +{ + pmd_t old_pmd = *pmdp; + set_pmd_at(vma->vm_mm, address, pmdp, pmd); + return old_pmd; +} +#endif + +#ifndef __HAVE_ARCH_PMDP_INVALIDATE +extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmdp); +#endif + +#ifndef __HAVE_ARCH_PTE_SAME +static inline int pte_same(pte_t pte_a, pte_t pte_b) +{ + return pte_val(pte_a) == pte_val(pte_b); +} +#endif + +#ifndef __HAVE_ARCH_PTE_UNUSED +/* + * Some architectures provide facilities to virtualization guests + * so that they can flag allocated pages as unused. This allows the + * host to transparently reclaim unused pages. This function returns + * whether the pte's page is unused. + */ +static inline int pte_unused(pte_t pte) +{ + return 0; +} +#endif + +#ifndef pte_access_permitted +#define pte_access_permitted(pte, write) \ + (pte_present(pte) && (!(write) || pte_write(pte))) +#endif + +#ifndef pmd_access_permitted +#define pmd_access_permitted(pmd, write) \ + (pmd_present(pmd) && (!(write) || pmd_write(pmd))) +#endif + +#ifndef pud_access_permitted +#define pud_access_permitted(pud, write) \ + (pud_present(pud) && (!(write) || pud_write(pud))) +#endif + +#ifndef p4d_access_permitted +#define p4d_access_permitted(p4d, write) \ + (p4d_present(p4d) && (!(write) || p4d_write(p4d))) +#endif + +#ifndef pgd_access_permitted +#define pgd_access_permitted(pgd, write) \ + (pgd_present(pgd) && (!(write) || pgd_write(pgd))) +#endif + +#ifndef __HAVE_ARCH_PMD_SAME +static inline int pmd_same(pmd_t pmd_a, pmd_t pmd_b) +{ + return pmd_val(pmd_a) == pmd_val(pmd_b); +} + +static inline int pud_same(pud_t pud_a, pud_t pud_b) +{ + return pud_val(pud_a) == pud_val(pud_b); +} +#endif + +#ifndef __HAVE_ARCH_P4D_SAME +static inline int p4d_same(p4d_t p4d_a, p4d_t p4d_b) +{ + return p4d_val(p4d_a) == p4d_val(p4d_b); +} +#endif + +#ifndef __HAVE_ARCH_PGD_SAME +static inline int pgd_same(pgd_t pgd_a, pgd_t pgd_b) +{ + return pgd_val(pgd_a) == pgd_val(pgd_b); +} +#endif + +/* + * Use set_p*_safe(), and elide TLB flushing, when confident that *no* + * TLB flush will be required as a result of the "set". For example, use + * in scenarios where it is known ahead of time that the routine is + * setting non-present entries, or re-setting an existing entry to the + * same value. Otherwise, use the typical "set" helpers and flush the + * TLB. + */ +#define set_pte_safe(ptep, pte) \ +({ \ + WARN_ON_ONCE(pte_present(*ptep) && !pte_same(*ptep, pte)); \ + set_pte(ptep, pte); \ +}) + +#define set_pmd_safe(pmdp, pmd) \ +({ \ + WARN_ON_ONCE(pmd_present(*pmdp) && !pmd_same(*pmdp, pmd)); \ + set_pmd(pmdp, pmd); \ +}) + +#define set_pud_safe(pudp, pud) \ +({ \ + WARN_ON_ONCE(pud_present(*pudp) && !pud_same(*pudp, pud)); \ + set_pud(pudp, pud); \ +}) + +#define set_p4d_safe(p4dp, p4d) \ +({ \ + WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \ + set_p4d(p4dp, p4d); \ +}) + +#define set_pgd_safe(pgdp, pgd) \ +({ \ + WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \ + set_pgd(pgdp, pgd); \ +}) + +#ifndef __HAVE_ARCH_DO_SWAP_PAGE +/* + * Some architectures support metadata associated with a page. When a + * page is being swapped out, this metadata must be saved so it can be + * restored when the page is swapped back in. SPARC M7 and newer + * processors support an ADI (Application Data Integrity) tag for the + * page as metadata for the page. arch_do_swap_page() can restore this + * metadata when a page is swapped back in. + */ +static inline void arch_do_swap_page(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long addr, + pte_t pte, pte_t oldpte) +{ + +} +#endif + +#ifndef __HAVE_ARCH_UNMAP_ONE +/* + * Some architectures support metadata associated with a page. When a + * page is being swapped out, this metadata must be saved so it can be + * restored when the page is swapped back in. SPARC M7 and newer + * processors support an ADI (Application Data Integrity) tag for the + * page as metadata for the page. arch_unmap_one() can save this + * metadata on a swap-out of a page. + */ +static inline int arch_unmap_one(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long addr, + pte_t orig_pte) +{ + return 0; +} +#endif + +#ifndef __HAVE_ARCH_PGD_OFFSET_GATE +#define pgd_offset_gate(mm, addr) pgd_offset(mm, addr) +#endif + +#ifndef __HAVE_ARCH_MOVE_PTE +#define move_pte(pte, prot, old_addr, new_addr) (pte) +#endif + +#ifndef pte_accessible +# define pte_accessible(mm, pte) ((void)(pte), 1) +#endif + +#ifndef flush_tlb_fix_spurious_fault +#define flush_tlb_fix_spurious_fault(vma, address) flush_tlb_page(vma, address) +#endif + +#ifndef pgprot_nx +#define pgprot_nx(prot) (prot) +#endif + +#ifndef pgprot_noncached +#define pgprot_noncached(prot) (prot) +#endif + +#ifndef pgprot_writecombine +#define pgprot_writecombine pgprot_noncached +#endif + +#ifndef pgprot_writethrough +#define pgprot_writethrough pgprot_noncached +#endif + +#ifndef pgprot_device +#define pgprot_device pgprot_noncached +#endif + +#ifndef pgprot_modify +#define pgprot_modify pgprot_modify +static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) +{ + if (pgprot_val(oldprot) == pgprot_val(pgprot_noncached(oldprot))) + newprot = pgprot_noncached(newprot); + if (pgprot_val(oldprot) == pgprot_val(pgprot_writecombine(oldprot))) + newprot = pgprot_writecombine(newprot); + if (pgprot_val(oldprot) == pgprot_val(pgprot_device(oldprot))) + newprot = pgprot_device(newprot); + return newprot; +} +#endif + +/* + * When walking page tables, get the address of the next boundary, + * or the end address of the range if that comes earlier. Although no + * vma end wraps to 0, rounded up __boundary may wrap to 0 throughout. + */ + +#define pgd_addr_end(addr, end) \ +({ unsigned long __boundary = ((addr) + PGDIR_SIZE) & PGDIR_MASK; \ + (__boundary - 1 < (end) - 1)? __boundary: (end); \ +}) + +#ifndef p4d_addr_end +#define p4d_addr_end(addr, end) \ +({ unsigned long __boundary = ((addr) + P4D_SIZE) & P4D_MASK; \ + (__boundary - 1 < (end) - 1)? __boundary: (end); \ +}) +#endif + +#ifndef pud_addr_end +#define pud_addr_end(addr, end) \ +({ unsigned long __boundary = ((addr) + PUD_SIZE) & PUD_MASK; \ + (__boundary - 1 < (end) - 1)? __boundary: (end); \ +}) +#endif + +#ifndef pmd_addr_end +#define pmd_addr_end(addr, end) \ +({ unsigned long __boundary = ((addr) + PMD_SIZE) & PMD_MASK; \ + (__boundary - 1 < (end) - 1)? __boundary: (end); \ +}) +#endif + +/* + * When walking page tables, we usually want to skip any p?d_none entries; + * and any p?d_bad entries - reporting the error before resetting to none. + * Do the tests inline, but report and clear the bad entry in mm/memory.c. + */ +void pgd_clear_bad(pgd_t *); + +#ifndef __PAGETABLE_P4D_FOLDED +void p4d_clear_bad(p4d_t *); +#else +#define p4d_clear_bad(p4d) do { } while (0) +#endif + +#ifndef __PAGETABLE_PUD_FOLDED +void pud_clear_bad(pud_t *); +#else +#define pud_clear_bad(p4d) do { } while (0) +#endif + +void pmd_clear_bad(pmd_t *); + +static inline int pgd_none_or_clear_bad(pgd_t *pgd) +{ + if (pgd_none(*pgd)) + return 1; + if (unlikely(pgd_bad(*pgd))) { + pgd_clear_bad(pgd); + return 1; + } + return 0; +} + +static inline int p4d_none_or_clear_bad(p4d_t *p4d) +{ + if (p4d_none(*p4d)) + return 1; + if (unlikely(p4d_bad(*p4d))) { + p4d_clear_bad(p4d); + return 1; + } + return 0; +} + +static inline int pud_none_or_clear_bad(pud_t *pud) +{ + if (pud_none(*pud)) + return 1; + if (unlikely(pud_bad(*pud))) { + pud_clear_bad(pud); + return 1; + } + return 0; +} + +static inline int pmd_none_or_clear_bad(pmd_t *pmd) +{ + if (pmd_none(*pmd)) + return 1; + if (unlikely(pmd_bad(*pmd))) { + pmd_clear_bad(pmd); + return 1; + } + return 0; +} + +static inline pte_t __ptep_modify_prot_start(struct vm_area_struct *vma, + unsigned long addr, + pte_t *ptep) +{ + /* + * Get the current pte state, but zero it out to make it + * non-present, preventing the hardware from asynchronously + * updating it. + */ + return ptep_get_and_clear(vma->vm_mm, addr, ptep); +} + +static inline void __ptep_modify_prot_commit(struct vm_area_struct *vma, + unsigned long addr, + pte_t *ptep, pte_t pte) +{ + /* + * The pte is non-present, so there's no hardware state to + * preserve. + */ + set_pte_at(vma->vm_mm, addr, ptep, pte); +} + +#ifndef __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION +/* + * Start a pte protection read-modify-write transaction, which + * protects against asynchronous hardware modifications to the pte. + * The intention is not to prevent the hardware from making pte + * updates, but to prevent any updates it may make from being lost. + * + * This does not protect against other software modifications of the + * pte; the appropriate pte lock must be held over the transation. + * + * Note that this interface is intended to be batchable, meaning that + * ptep_modify_prot_commit may not actually update the pte, but merely + * queue the update to be done at some later time. The update must be + * actually committed before the pte lock is released, however. + */ +static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, + unsigned long addr, + pte_t *ptep) +{ + return __ptep_modify_prot_start(vma, addr, ptep); +} + +/* + * Commit an update to a pte, leaving any hardware-controlled bits in + * the PTE unmodified. + */ +static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, + unsigned long addr, + pte_t *ptep, pte_t old_pte, pte_t pte) +{ + __ptep_modify_prot_commit(vma, addr, ptep, pte); +} +#endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */ +#endif /* CONFIG_MMU */ + +/* + * No-op macros that just return the current protection value. Defined here + * because these macros can be used used even if CONFIG_MMU is not defined. + */ +#ifndef pgprot_encrypted +#define pgprot_encrypted(prot) (prot) +#endif + +#ifndef pgprot_decrypted +#define pgprot_decrypted(prot) (prot) +#endif + +/* + * A facility to provide lazy MMU batching. This allows PTE updates and + * page invalidations to be delayed until a call to leave lazy MMU mode + * is issued. Some architectures may benefit from doing this, and it is + * beneficial for both shadow and direct mode hypervisors, which may batch + * the PTE updates which happen during this window. Note that using this + * interface requires that read hazards be removed from the code. A read + * hazard could result in the direct mode hypervisor case, since the actual + * write to the page tables may not yet have taken place, so reads though + * a raw PTE pointer after it has been modified are not guaranteed to be + * up to date. This mode can only be entered and left under the protection of + * the page table locks for all page tables which may be modified. In the UP + * case, this is required so that preemption is disabled, and in the SMP case, + * it must synchronize the delayed page table writes properly on other CPUs. + */ +#ifndef __HAVE_ARCH_ENTER_LAZY_MMU_MODE +#define arch_enter_lazy_mmu_mode() do {} while (0) +#define arch_leave_lazy_mmu_mode() do {} while (0) +#define arch_flush_lazy_mmu_mode() do {} while (0) +#endif + +/* + * A facility to provide batching of the reload of page tables and + * other process state with the actual context switch code for + * paravirtualized guests. By convention, only one of the batched + * update (lazy) modes (CPU, MMU) should be active at any given time, + * entry should never be nested, and entry and exits should always be + * paired. This is for sanity of maintaining and reasoning about the + * kernel code. In this case, the exit (end of the context switch) is + * in architecture-specific code, and so doesn't need a generic + * definition. + */ +#ifndef __HAVE_ARCH_START_CONTEXT_SWITCH +#define arch_start_context_switch(prev) do {} while (0) +#endif + +#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY +#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION +static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) +{ + return pmd; +} + +static inline int pmd_swp_soft_dirty(pmd_t pmd) +{ + return 0; +} + +static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) +{ + return pmd; +} +#endif +#else /* !CONFIG_HAVE_ARCH_SOFT_DIRTY */ +static inline int pte_soft_dirty(pte_t pte) +{ + return 0; +} + +static inline int pmd_soft_dirty(pmd_t pmd) +{ + return 0; +} + +static inline pte_t pte_mksoft_dirty(pte_t pte) +{ + return pte; +} + +static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) +{ + return pmd; +} + +static inline pte_t pte_clear_soft_dirty(pte_t pte) +{ + return pte; +} + +static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) +{ + return pmd; +} + +static inline pte_t pte_swp_mksoft_dirty(pte_t pte) +{ + return pte; +} + +static inline int pte_swp_soft_dirty(pte_t pte) +{ + return 0; +} + +static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) +{ + return pte; +} + +static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) +{ + return pmd; +} + +static inline int pmd_swp_soft_dirty(pmd_t pmd) +{ + return 0; +} + +static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd) +{ + return pmd; +} +#endif + +#ifndef __HAVE_PFNMAP_TRACKING +/* + * Interfaces that can be used by architecture code to keep track of + * memory type of pfn mappings specified by the remap_pfn_range, + * vmf_insert_pfn. + */ + +/* + * track_pfn_remap is called when a _new_ pfn mapping is being established + * by remap_pfn_range() for physical range indicated by pfn and size. + */ +static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot, + unsigned long pfn, unsigned long addr, + unsigned long size) +{ + return 0; +} + +/* + * track_pfn_insert is called when a _new_ single pfn is established + * by vmf_insert_pfn(). + */ +static inline void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, + pfn_t pfn) +{ +} + +/* + * track_pfn_copy is called when vma that is covering the pfnmap gets + * copied through copy_page_range(). + */ +static inline int track_pfn_copy(struct vm_area_struct *vma) +{ + return 0; +} + +/* + * untrack_pfn is called while unmapping a pfnmap for a region. + * untrack can be called for a specific region indicated by pfn and size or + * can be for the entire vma (in which case pfn, size are zero). + */ +static inline void untrack_pfn(struct vm_area_struct *vma, + unsigned long pfn, unsigned long size) +{ +} + +/* + * untrack_pfn_moved is called while mremapping a pfnmap for a new region. + */ +static inline void untrack_pfn_moved(struct vm_area_struct *vma) +{ +} +#else +extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot, + unsigned long pfn, unsigned long addr, + unsigned long size); +extern void track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot, + pfn_t pfn); +extern int track_pfn_copy(struct vm_area_struct *vma); +extern void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn, + unsigned long size); +extern void untrack_pfn_moved(struct vm_area_struct *vma); +#endif + +#ifdef __HAVE_COLOR_ZERO_PAGE +static inline int is_zero_pfn(unsigned long pfn) +{ + extern unsigned long zero_pfn; + unsigned long offset_from_zero_pfn = pfn - zero_pfn; + return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT); +} + +#define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) + +#else +static inline int is_zero_pfn(unsigned long pfn) +{ + extern unsigned long zero_pfn; + return pfn == zero_pfn; +} + +static inline unsigned long my_zero_pfn(unsigned long addr) +{ + extern unsigned long zero_pfn; + return zero_pfn; +} +#endif + +#ifdef CONFIG_MMU + +#ifndef CONFIG_TRANSPARENT_HUGEPAGE +static inline int pmd_trans_huge(pmd_t pmd) +{ + return 0; +} +#ifndef pmd_write +static inline int pmd_write(pmd_t pmd) +{ + BUG(); + return 0; +} +#endif /* pmd_write */ +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#ifndef pud_write +static inline int pud_write(pud_t pud) +{ + BUG(); + return 0; +} +#endif /* pud_write */ + +#if !defined(CONFIG_ARCH_HAS_PTE_DEVMAP) || !defined(CONFIG_TRANSPARENT_HUGEPAGE) +static inline int pmd_devmap(pmd_t pmd) +{ + return 0; +} +static inline int pud_devmap(pud_t pud) +{ + return 0; +} +static inline int pgd_devmap(pgd_t pgd) +{ + return 0; +} +#endif + +#if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \ + (defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ + !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) +static inline int pud_trans_huge(pud_t pud) +{ + return 0; +} +#endif + +/* See pmd_none_or_trans_huge_or_clear_bad for discussion. */ +static inline int pud_none_or_trans_huge_or_dev_or_clear_bad(pud_t *pud) +{ + pud_t pudval = READ_ONCE(*pud); + + if (pud_none(pudval) || pud_trans_huge(pudval) || pud_devmap(pudval)) + return 1; + if (unlikely(pud_bad(pudval))) { + pud_clear_bad(pud); + return 1; + } + return 0; +} + +/* See pmd_trans_unstable for discussion. */ +static inline int pud_trans_unstable(pud_t *pud) +{ +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ + defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) + return pud_none_or_trans_huge_or_dev_or_clear_bad(pud); +#else + return 0; +#endif +} + +#ifndef pmd_read_atomic +static inline pmd_t pmd_read_atomic(pmd_t *pmdp) +{ + /* + * Depend on compiler for an atomic pmd read. NOTE: this is + * only going to work, if the pmdval_t isn't larger than + * an unsigned long. + */ + return *pmdp; +} +#endif + +#ifndef arch_needs_pgtable_deposit +#define arch_needs_pgtable_deposit() (false) +#endif +/* + * This function is meant to be used by sites walking pagetables with + * the mmap_sem hold in read mode to protect against MADV_DONTNEED and + * transhuge page faults. MADV_DONTNEED can convert a transhuge pmd + * into a null pmd and the transhuge page fault can convert a null pmd + * into an hugepmd or into a regular pmd (if the hugepage allocation + * fails). While holding the mmap_sem in read mode the pmd becomes + * stable and stops changing under us only if it's not null and not a + * transhuge pmd. When those races occurs and this function makes a + * difference vs the standard pmd_none_or_clear_bad, the result is + * undefined so behaving like if the pmd was none is safe (because it + * can return none anyway). The compiler level barrier() is critically + * important to compute the two checks atomically on the same pmdval. + * + * For 32bit kernels with a 64bit large pmd_t this automatically takes + * care of reading the pmd atomically to avoid SMP race conditions + * against pmd_populate() when the mmap_sem is hold for reading by the + * caller (a special atomic read not done by "gcc" as in the generic + * version above, is also needed when THP is disabled because the page + * fault can populate the pmd from under us). + */ +static inline int pmd_none_or_trans_huge_or_clear_bad(pmd_t *pmd) +{ + pmd_t pmdval = pmd_read_atomic(pmd); + /* + * The barrier will stabilize the pmdval in a register or on + * the stack so that it will stop changing under the code. + * + * When CONFIG_TRANSPARENT_HUGEPAGE=y on x86 32bit PAE, + * pmd_read_atomic is allowed to return a not atomic pmdval + * (for example pointing to an hugepage that has never been + * mapped in the pmd). The below checks will only care about + * the low part of the pmd with 32bit PAE x86 anyway, with the + * exception of pmd_none(). So the important thing is that if + * the low part of the pmd is found null, the high part will + * be also null or the pmd_none() check below would be + * confused. + */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + barrier(); +#endif + /* + * !pmd_present() checks for pmd migration entries + * + * The complete check uses is_pmd_migration_entry() in linux/swapops.h + * But using that requires moving current function and pmd_trans_unstable() + * to linux/swapops.h to resovle dependency, which is too much code move. + * + * !pmd_present() is equivalent to is_pmd_migration_entry() currently, + * because !pmd_present() pages can only be under migration not swapped + * out. + * + * pmd_none() is preseved for future condition checks on pmd migration + * entries and not confusing with this function name, although it is + * redundant with !pmd_present(). + */ + if (pmd_none(pmdval) || pmd_trans_huge(pmdval) || + (IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION) && !pmd_present(pmdval))) + return 1; + if (unlikely(pmd_bad(pmdval))) { + pmd_clear_bad(pmd); + return 1; + } + return 0; +} + +/* + * This is a noop if Transparent Hugepage Support is not built into + * the kernel. Otherwise it is equivalent to + * pmd_none_or_trans_huge_or_clear_bad(), and shall only be called in + * places that already verified the pmd is not none and they want to + * walk ptes while holding the mmap sem in read mode (write mode don't + * need this). If THP is not enabled, the pmd can't go away under the + * code even if MADV_DONTNEED runs, but if THP is enabled we need to + * run a pmd_trans_unstable before walking the ptes after + * split_huge_pmd returns (because it may have run when the pmd become + * null, but then a page fault can map in a THP and not a regular page). + */ +static inline int pmd_trans_unstable(pmd_t *pmd) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + return pmd_none_or_trans_huge_or_clear_bad(pmd); +#else + return 0; +#endif +} + +#ifndef CONFIG_NUMA_BALANCING +/* + * Technically a PTE can be PROTNONE even when not doing NUMA balancing but + * the only case the kernel cares is for NUMA balancing and is only ever set + * when the VMA is accessible. For PROT_NONE VMAs, the PTEs are not marked + * _PAGE_PROTNONE so by by default, implement the helper as "always no". It + * is the responsibility of the caller to distinguish between PROT_NONE + * protections and NUMA hinting fault protections. + */ +static inline int pte_protnone(pte_t pte) +{ + return 0; +} + +static inline int pmd_protnone(pmd_t pmd) +{ + return 0; +} +#endif /* CONFIG_NUMA_BALANCING */ + +#endif /* CONFIG_MMU */ + +#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP + +#ifndef __PAGETABLE_P4D_FOLDED +int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot); +int p4d_clear_huge(p4d_t *p4d); +#else +static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) +{ + return 0; +} +static inline int p4d_clear_huge(p4d_t *p4d) +{ + return 0; +} +#endif /* !__PAGETABLE_P4D_FOLDED */ + +int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot); +int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot); +int pud_clear_huge(pud_t *pud); +int pmd_clear_huge(pmd_t *pmd); +int p4d_free_pud_page(p4d_t *p4d, unsigned long addr); +int pud_free_pmd_page(pud_t *pud, unsigned long addr); +int pmd_free_pte_page(pmd_t *pmd, unsigned long addr); +#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ +static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) +{ + return 0; +} +static inline int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) +{ + return 0; +} +static inline int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) +{ + return 0; +} +static inline int p4d_clear_huge(p4d_t *p4d) +{ + return 0; +} +static inline int pud_clear_huge(pud_t *pud) +{ + return 0; +} +static inline int pmd_clear_huge(pmd_t *pmd) +{ + return 0; +} +static inline int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) +{ + return 0; +} +static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr) +{ + return 0; +} +static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) +{ + return 0; +} +#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ + +#ifndef __HAVE_ARCH_FLUSH_PMD_TLB_RANGE +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +/* + * ARCHes with special requirements for evicting THP backing TLB entries can + * implement this. Otherwise also, it can help optimize normal TLB flush in + * THP regime. stock flush_tlb_range() typically has optimization to nuke the + * entire TLB TLB if flush span is greater than a threshold, which will + * likely be true for a single huge page. Thus a single thp flush will + * invalidate the entire TLB which is not desitable. + * e.g. see arch/arc: flush_pmd_tlb_range + */ +#define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) +#define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end) +#else +#define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG() +#define flush_pud_tlb_range(vma, addr, end) BUILD_BUG() +#endif +#endif + +struct file; +int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, + unsigned long size, pgprot_t *vma_prot); + +#ifndef CONFIG_X86_ESPFIX64 +static inline void init_espfix_bsp(void) { } +#endif + +extern void __init pgtable_cache_init(void); + +#ifndef __HAVE_ARCH_PFN_MODIFY_ALLOWED +static inline bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot) +{ + return true; +} + +static inline bool arch_has_pfn_modify_check(void) +{ + return false; +} +#endif /* !_HAVE_ARCH_PFN_MODIFY_ALLOWED */ + +/* + * Architecture PAGE_KERNEL_* fallbacks + * + * Some architectures don't define certain PAGE_KERNEL_* flags. This is either + * because they really don't support them, or the port needs to be updated to + * reflect the required functionality. Below are a set of relatively safe + * fallbacks, as best effort, which we can count on in lieu of the architectures + * not defining them on their own yet. + */ + +#ifndef PAGE_KERNEL_RO +# define PAGE_KERNEL_RO PAGE_KERNEL +#endif + +#ifndef PAGE_KERNEL_EXEC +# define PAGE_KERNEL_EXEC PAGE_KERNEL +#endif + +/* + * Page Table Modification bits for pgtbl_mod_mask. + * + * These are used by the p?d_alloc_track*() set of functions an in the generic + * vmalloc/ioremap code to track at which page-table levels entries have been + * modified. Based on that the code can better decide when vmalloc and ioremap + * mapping changes need to be synchronized to other page-tables in the system. + */ +#define __PGTBL_PGD_MODIFIED 0 +#define __PGTBL_P4D_MODIFIED 1 +#define __PGTBL_PUD_MODIFIED 2 +#define __PGTBL_PMD_MODIFIED 3 +#define __PGTBL_PTE_MODIFIED 4 + +#define PGTBL_PGD_MODIFIED BIT(__PGTBL_PGD_MODIFIED) +#define PGTBL_P4D_MODIFIED BIT(__PGTBL_P4D_MODIFIED) +#define PGTBL_PUD_MODIFIED BIT(__PGTBL_PUD_MODIFIED) +#define PGTBL_PMD_MODIFIED BIT(__PGTBL_PMD_MODIFIED) +#define PGTBL_PTE_MODIFIED BIT(__PGTBL_PTE_MODIFIED) + +/* Page-Table Modification Mask */ +typedef unsigned int pgtbl_mod_mask; + +#endif /* !__ASSEMBLY__ */ + +#ifndef io_remap_pfn_range +#define io_remap_pfn_range remap_pfn_range +#endif + +#ifndef has_transparent_hugepage +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define has_transparent_hugepage() 1 +#else +#define has_transparent_hugepage() 0 +#endif +#endif + +/* + * On some architectures it depends on the mm if the p4d/pud or pmd + * layer of the page table hierarchy is folded or not. + */ +#ifndef mm_p4d_folded +#define mm_p4d_folded(mm) __is_defined(__PAGETABLE_P4D_FOLDED) +#endif + +#ifndef mm_pud_folded +#define mm_pud_folded(mm) __is_defined(__PAGETABLE_PUD_FOLDED) +#endif + +#ifndef mm_pmd_folded +#define mm_pmd_folded(mm) __is_defined(__PAGETABLE_PMD_FOLDED) +#endif + +/* + * p?d_leaf() - true if this entry is a final mapping to a physical address. + * This differs from p?d_huge() by the fact that they are always available (if + * the architecture supports large pages at the appropriate level) even + * if CONFIG_HUGETLB_PAGE is not defined. + * Only meaningful when called on a valid entry. + */ +#ifndef pgd_leaf +#define pgd_leaf(x) 0 +#endif +#ifndef p4d_leaf +#define p4d_leaf(x) 0 +#endif +#ifndef pud_leaf +#define pud_leaf(x) 0 +#endif +#ifndef pmd_leaf +#define pmd_leaf(x) 0 +#endif + +#endif /* _LINUX_PGTABLE_H */ -- cgit v1.2.3-70-g09d2 From e05c7b1f2bc4b7b28199b9a7572f73436d97317e Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 8 Jun 2020 21:33:05 -0700 Subject: mm: pgtable: add shortcuts for accessing kernel PMD and PTE The powerpc 32-bit implementation of pgtable has nice shortcuts for accessing kernel PMD and PTE for a given virtual address. Make these helpers available for all architectures. [rppt@linux.ibm.com: microblaze: fix page table traversal in setup_rt_frame()] Link: http://lkml.kernel.org/r/20200518191511.GD1118872@kernel.org [akpm@linux-foundation.org: s/pmd_ptr_k/pmd_off_k/ in various powerpc places] Signed-off-by: Mike Rapoport Signed-off-by: Andrew Morton Cc: Arnd Bergmann Cc: Borislav Petkov Cc: Brian Cain Cc: Catalin Marinas Cc: Chris Zankel Cc: "David S. Miller" Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Ungerer Cc: Guan Xuetao Cc: Guo Ren Cc: Heiko Carstens Cc: Helge Deller Cc: Ingo Molnar Cc: Ley Foon Tan Cc: Mark Salter Cc: Matthew Wilcox Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Nick Hu Cc: Paul Walmsley Cc: Richard Weinberger Cc: Rich Felker Cc: Russell King Cc: Stafford Horne Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Tony Luck Cc: Vincent Chen Cc: Vineet Gupta Cc: Will Deacon Cc: Yoshinori Sato Link: http://lkml.kernel.org/r/20200514170327.31389-9-rppt@kernel.org Signed-off-by: Linus Torvalds --- arch/arc/mm/highmem.c | 10 +--------- arch/arm/mach-sa1100/assabet.c | 2 +- arch/arm/mm/highmem.c | 4 ++-- arch/arm/mm/ioremap.c | 31 ++++--------------------------- arch/arm/mm/mm.h | 5 ----- arch/arm/mm/mmu.c | 7 +------ arch/hexagon/include/asm/fixmap.h | 4 ---- arch/m68k/mm/motorola.c | 26 ++++---------------------- arch/microblaze/kernel/signal.c | 8 +------- arch/microblaze/mm/init.c | 9 --------- arch/mips/include/asm/fixmap.h | 3 --- arch/mips/mm/c-r3k.c | 10 ++-------- arch/mips/mm/c-r4k.c | 10 ++-------- arch/mips/mm/c-tx39.c | 10 ++-------- arch/mips/mm/highmem.c | 2 +- arch/nds32/include/asm/pgtable.h | 2 -- arch/nds32/mm/init.c | 13 ++----------- arch/nds32/mm/proc.c | 6 +----- arch/parisc/mm/fixmap.c | 6 +----- arch/powerpc/include/asm/pgtable.h | 19 ------------------- arch/powerpc/mm/book3s32/mmu.c | 2 +- arch/powerpc/mm/book3s32/tlb.c | 4 ++-- arch/powerpc/mm/kasan/8xx.c | 4 ++-- arch/powerpc/mm/kasan/book3s_32.c | 2 +- arch/powerpc/mm/kasan/kasan_init_32.c | 8 ++++---- arch/powerpc/mm/nohash/40x.c | 4 ++-- arch/powerpc/mm/nohash/8xx.c | 2 +- arch/powerpc/mm/pgtable_32.c | 4 ++-- arch/s390/mm/pageattr.c | 10 +--------- arch/sh/mm/cache-sh4.c | 8 +------- arch/sh/mm/kmap.c | 5 +---- arch/sparc/mm/highmem.c | 12 ++---------- arch/sparc/mm/init_64.c | 6 +----- arch/sparc/mm/io-unit.c | 10 ++-------- arch/sparc/mm/iommu.c | 8 +------- arch/um/kernel/mem.c | 10 +--------- arch/um/kernel/trap.c | 8 +------- arch/unicore32/mm/mm.h | 10 ---------- arch/x86/mm/init_32.c | 26 +++----------------------- arch/xtensa/include/asm/fixmap.h | 8 -------- arch/xtensa/mm/highmem.c | 2 +- arch/xtensa/mm/kasan_init.c | 10 ++-------- arch/xtensa/mm/mmu.c | 5 +---- include/linux/pgtable.h | 24 ++++++++++++++++++++++++ 44 files changed, 82 insertions(+), 297 deletions(-) (limited to 'include/linux/pgtable.h') diff --git a/arch/arc/mm/highmem.c b/arch/arc/mm/highmem.c index d6b74883fd1f..1b9f473c6369 100644 --- a/arch/arc/mm/highmem.c +++ b/arch/arc/mm/highmem.c @@ -92,17 +92,9 @@ EXPORT_SYMBOL(kunmap_atomic_high); static noinline pte_t * __init alloc_kmap_pgtable(unsigned long kvaddr) { - pgd_t *pgd_k; - p4d_t *p4d_k; - pud_t *pud_k; - pmd_t *pmd_k; + pmd_t *pmd_k = pmd_off_k(kvaddr); pte_t *pte_k; - pgd_k = pgd_offset_k(kvaddr); - p4d_k = p4d_offset(pgd_k, kvaddr); - pud_k = pud_offset(p4d_k, kvaddr); - pmd_k = pmd_offset(pud_k, kvaddr); - pte_k = (pte_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); if (!pte_k) panic("%s: Failed to allocate %lu bytes align=0x%lx\n", diff --git a/arch/arm/mach-sa1100/assabet.c b/arch/arm/mach-sa1100/assabet.c index 8e3f5fdb4883..aa265ede5730 100644 --- a/arch/arm/mach-sa1100/assabet.c +++ b/arch/arm/mach-sa1100/assabet.c @@ -632,7 +632,7 @@ static void __init map_sa1100_gpio_regs( void ) int prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_DOMAIN(DOMAIN_IO); pmd_t *pmd; - pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset_k(virt), virt), virt), virt); + pmd = pmd_off_k(virt); *pmd = __pmd(phys | prot); flush_pmd_entry(pmd); } diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c index e013f6b81328..187fab227b50 100644 --- a/arch/arm/mm/highmem.c +++ b/arch/arm/mm/highmem.c @@ -18,7 +18,7 @@ static inline void set_fixmap_pte(int idx, pte_t pte) { unsigned long vaddr = __fix_to_virt(idx); - pte_t *ptep = pte_offset_kernel(pmd_off_k(vaddr), vaddr); + pte_t *ptep = virt_to_kpte(vaddr); set_pte_ext(ptep, pte, 0); local_flush_tlb_kernel_page(vaddr); @@ -26,7 +26,7 @@ static inline void set_fixmap_pte(int idx, pte_t pte) static inline pte_t get_fixmap_pte(unsigned long vaddr) { - pte_t *ptep = pte_offset_kernel(pmd_off_k(vaddr), vaddr); + pte_t *ptep = virt_to_kpte(vaddr); return *ptep; } diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index 75529d76d28c..000e8210000b 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -141,16 +141,8 @@ void __check_vmalloc_seq(struct mm_struct *mm) static void unmap_area_sections(unsigned long virt, unsigned long size) { unsigned long addr = virt, end = virt + (size & ~(SZ_1M - 1)); - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmdp; - - flush_cache_vunmap(addr, end); - pgd = pgd_offset_k(addr); - p4d = p4d_offset(pgd, addr); - pud = pud_offset(p4d, addr); - pmdp = pmd_offset(pud, addr); + pmd_t *pmdp = pmd_off_k(addr); + do { pmd_t pmd = *pmdp; @@ -191,10 +183,7 @@ remap_area_sections(unsigned long virt, unsigned long pfn, size_t size, const struct mem_type *type) { unsigned long addr = virt, end = virt + size; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; + pmd_t *pmd = pmd_off_k(addr); /* * Remove and free any PTE-based mapping, and @@ -202,10 +191,6 @@ remap_area_sections(unsigned long virt, unsigned long pfn, */ unmap_area_sections(virt, size); - pgd = pgd_offset_k(addr); - p4d = p4d_offset(pgd, addr); - pud = pud_offset(p4d, addr); - pmd = pmd_offset(pud, addr); do { pmd[0] = __pmd(__pfn_to_phys(pfn) | type->prot_sect); pfn += SZ_1M >> PAGE_SHIFT; @@ -225,21 +210,13 @@ remap_area_supersections(unsigned long virt, unsigned long pfn, size_t size, const struct mem_type *type) { unsigned long addr = virt, end = virt + size; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; + pmd_t *pmd = pmd_off_k(addr); /* * Remove and free any PTE-based mapping, and * sync the current kernel mapping. */ unmap_area_sections(virt, size); - - pgd = pgd_offset_k(virt); - p4d = p4d_offset(pgd, addr); - pud = pud_offset(p4d, addr); - pmd = pmd_offset(pud, addr); do { unsigned long super_pmd_val, i; diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h index f9587d5fa8a4..9ff683612f2a 100644 --- a/arch/arm/mm/mm.h +++ b/arch/arm/mm/mm.h @@ -35,11 +35,6 @@ static inline pte_t get_top_pte(unsigned long va) return *ptep; } -static inline pmd_t *pmd_off_k(unsigned long virt) -{ - return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(virt), virt), virt), virt); -} - struct mem_type { pteval_t prot_pte; pteval_t prot_pte_s2; diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index c425288f1a86..628028bfbb92 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -356,12 +356,7 @@ static pte_t *pte_offset_late_fixmap(pmd_t *dir, unsigned long addr) static inline pmd_t * __init fixmap_pmd(unsigned long addr) { - pgd_t *pgd = pgd_offset_k(addr); - p4d_t *p4d = p4d_offset(pgd, addr); - pud_t *pud = pud_offset(p4d, addr); - pmd_t *pmd = pmd_offset(pud, addr); - - return pmd; + return pmd_off_k(addr); } void __init early_fixmap_init(void) diff --git a/arch/hexagon/include/asm/fixmap.h b/arch/hexagon/include/asm/fixmap.h index 97b1b062e750..920660a04aa4 100644 --- a/arch/hexagon/include/asm/fixmap.h +++ b/arch/hexagon/include/asm/fixmap.h @@ -15,8 +15,4 @@ #include -#define kmap_get_fixmap_pte(vaddr) \ - pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(vaddr), \ - (vaddr)), (vaddr)), (vaddr)), (vaddr)) - #endif diff --git a/arch/m68k/mm/motorola.c b/arch/m68k/mm/motorola.c index 8e5e74121a78..2bb006bdc31c 100644 --- a/arch/m68k/mm/motorola.c +++ b/arch/m68k/mm/motorola.c @@ -54,17 +54,8 @@ static inline void nocache_page(void *vaddr) unsigned long addr = (unsigned long)vaddr; if (CPU_IS_040_OR_060) { - pgd_t *dir; - p4d_t *p4dp; - pud_t *pudp; - pmd_t *pmdp; - pte_t *ptep; - - dir = pgd_offset_k(addr); - p4dp = p4d_offset(dir, addr); - pudp = pud_offset(p4dp, addr); - pmdp = pmd_offset(pudp, addr); - ptep = pte_offset_kernel(pmdp, addr); + pte_t *ptep = virt_to_kpte(addr); + *ptep = pte_mknocache(*ptep); } } @@ -74,17 +65,8 @@ static inline void cache_page(void *vaddr) unsigned long addr = (unsigned long)vaddr; if (CPU_IS_040_OR_060) { - pgd_t *dir; - p4d_t *p4dp; - pud_t *pudp; - pmd_t *pmdp; - pte_t *ptep; - - dir = pgd_offset_k(addr); - p4dp = p4d_offset(dir, addr); - pudp = pud_offset(p4dp, addr); - pmdp = pmd_offset(pudp, addr); - ptep = pte_offset_kernel(pmdp, addr); + pte_t *ptep = virt_to_kpte(addr); + *ptep = pte_mkcache(*ptep); } } diff --git a/arch/microblaze/kernel/signal.c b/arch/microblaze/kernel/signal.c index 7dcd2e4d430e..bdd6d0c86e16 100644 --- a/arch/microblaze/kernel/signal.c +++ b/arch/microblaze/kernel/signal.c @@ -159,9 +159,6 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, int err = 0, sig = ksig->sig; unsigned long address = 0; #ifdef CONFIG_MMU - pgd_t *pgdp; - p4d_t *p4dp; - pud_t *pudp; pmd_t *pmdp; pte_t *ptep; #endif @@ -197,10 +194,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, address = ((unsigned long)frame->tramp); #ifdef CONFIG_MMU - pgdp = pgd_offset(current->mm, address); - p4dp = p4d_offset(pgdp, address); - pudp = pud_offset(p4dp, address); - pmdp = pmd_offset(pudp, address); + pmdp = pmd_off(current->mm, address); preempt_disable(); ptep = pte_offset_map(pmdp, address); diff --git a/arch/microblaze/mm/init.c b/arch/microblaze/mm/init.c index d943f69784b1..521b59ba716c 100644 --- a/arch/microblaze/mm/init.c +++ b/arch/microblaze/mm/init.c @@ -50,15 +50,6 @@ unsigned long lowmem_size; pte_t *kmap_pte; EXPORT_SYMBOL(kmap_pte); -static inline pte_t *virt_to_kpte(unsigned long vaddr) -{ - pgd_t *pgd = pgd_offset_k(vaddr); - p4d_t *p4d = p4d_offset(pgd, vaddr); - pud_t *pud = pud_offset(p4d, vaddr); - - return pte_offset_kernel(pmd_offset(pud, vaddr), vaddr); -} - static void __init highmem_init(void) { pr_debug("%x\n", (u32)PKMAP_BASE); diff --git a/arch/mips/include/asm/fixmap.h b/arch/mips/include/asm/fixmap.h index 1784d4348c36..743535be7528 100644 --- a/arch/mips/include/asm/fixmap.h +++ b/arch/mips/include/asm/fixmap.h @@ -69,9 +69,6 @@ enum fixed_addresses { #include -#define kmap_get_fixmap_pte(vaddr) \ - pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr)), (vaddr)) - /* * Called from pgtable_init() */ diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c index 9f4a93f8b996..df6755ca1892 100644 --- a/arch/mips/mm/c-r3k.c +++ b/arch/mips/mm/c-r3k.c @@ -239,9 +239,6 @@ static void r3k_flush_cache_page(struct vm_area_struct *vma, unsigned long kaddr = KSEG0ADDR(pfn << PAGE_SHIFT); int exec = vma->vm_flags & VM_EXEC; struct mm_struct *mm = vma->vm_mm; - pgd_t *pgdp; - p4d_t *p4dp; - pud_t *pudp; pmd_t *pmdp; pte_t *ptep; @@ -252,11 +249,8 @@ static void r3k_flush_cache_page(struct vm_area_struct *vma, if (cpu_context(smp_processor_id(), mm) == 0) return; - pgdp = pgd_offset(mm, addr); - p4dp = p4d_offset(pgdp, addr); - pudp = pud_offset(p4dp, addr); - pmdp = pmd_offset(pudp, addr); - ptep = pte_offset(pmdp, addr); + pmdp = pmd_off(mm, addr); + ptep = pte_offset_kernel(pmdp, addr); /* Invalid => no such page in the cache. */ if (!(pte_val(*ptep) & _PAGE_PRESENT)) diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index eeb63aab9d44..49569e5666d7 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -652,9 +652,6 @@ static inline void local_r4k_flush_cache_page(void *args) int exec = vma->vm_flags & VM_EXEC; struct mm_struct *mm = vma->vm_mm; int map_coherent = 0; - pgd_t *pgdp; - p4d_t *p4dp; - pud_t *pudp; pmd_t *pmdp; pte_t *ptep; void *vaddr; @@ -667,11 +664,8 @@ static inline void local_r4k_flush_cache_page(void *args) return; addr &= PAGE_MASK; - pgdp = pgd_offset(mm, addr); - p4dp = p4d_offset(pgdp, addr); - pudp = pud_offset(p4dp, addr); - pmdp = pmd_offset(pudp, addr); - ptep = pte_offset(pmdp, addr); + pmdp = pmd_off(mm, addr); + ptep = pte_offset_kernel(pmdp, addr); /* * If the page isn't marked valid, the page cannot possibly be diff --git a/arch/mips/mm/c-tx39.c b/arch/mips/mm/c-tx39.c index 204059d28f21..03dfbb40ec73 100644 --- a/arch/mips/mm/c-tx39.c +++ b/arch/mips/mm/c-tx39.c @@ -168,9 +168,6 @@ static void tx39_flush_cache_page(struct vm_area_struct *vma, unsigned long page { int exec = vma->vm_flags & VM_EXEC; struct mm_struct *mm = vma->vm_mm; - pgd_t *pgdp; - p4d_t *p4dp; - pud_t *pudp; pmd_t *pmdp; pte_t *ptep; @@ -182,11 +179,8 @@ static void tx39_flush_cache_page(struct vm_area_struct *vma, unsigned long page return; page &= PAGE_MASK; - pgdp = pgd_offset(mm, page); - p4dp = p4d_offset(pgdp, page); - pudp = pud_offset(p4dp, page); - pmdp = pmd_offset(pudp, page); - ptep = pte_offset(pmdp, page); + pmdp = pmd_off(mm, page); + ptep = pte_offset_kernel(pmdp, page); /* * If the page isn't marked valid, the page cannot possibly be diff --git a/arch/mips/mm/highmem.c b/arch/mips/mm/highmem.c index 8e8726992720..5fec7f45d79a 100644 --- a/arch/mips/mm/highmem.c +++ b/arch/mips/mm/highmem.c @@ -90,5 +90,5 @@ void __init kmap_init(void) /* cache the first kmap pte */ kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); - kmap_pte = kmap_get_fixmap_pte(kmap_vstart); + kmap_pte = virt_to_kpte(kmap_vstart); } diff --git a/arch/nds32/include/asm/pgtable.h b/arch/nds32/include/asm/pgtable.h index 51fe1c567702..95c75282cb47 100644 --- a/arch/nds32/include/asm/pgtable.h +++ b/arch/nds32/include/asm/pgtable.h @@ -195,8 +195,6 @@ extern void paging_init(void); #define pte_unmap(pte) do { } while (0) #define pte_unmap_nested(pte) do { } while (0) -#define pmd_off_k(address) pmd_offset(pud_offset(p4d_offset(pgd_offset_k(address), (address)), (address)), (address)) - #define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) /* * Set a level 1 translation table entry, and clean it out of diff --git a/arch/nds32/mm/init.c b/arch/nds32/mm/init.c index 91147cca4b64..fa86f7b2f416 100644 --- a/arch/nds32/mm/init.c +++ b/arch/nds32/mm/init.c @@ -98,9 +98,6 @@ static pmd_t *fixmap_pmd_p; static void __init fixedrange_init(void) { unsigned long vaddr; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; pmd_t *pmd; #ifdef CONFIG_HIGHMEM pte_t *pte; @@ -110,10 +107,7 @@ static void __init fixedrange_init(void) * Fixed mappings: */ vaddr = __fix_to_virt(__end_of_fixed_addresses - 1); - pgd = swapper_pg_dir + pgd_index(vaddr); - p4d = p4d_offset(pgd, vaddr); - pud = pud_offset(p4d, vaddr); - pmd = pmd_offset(pud, vaddr); + pmd = pmd_off_k(vaddr); fixmap_pmd_p = memblock_alloc(PAGE_SIZE, PAGE_SIZE); if (!fixmap_pmd_p) panic("%s: Failed to allocate %lu bytes align=0x%lx\n", @@ -126,10 +120,7 @@ static void __init fixedrange_init(void) */ vaddr = PKMAP_BASE; - pgd = swapper_pg_dir + pgd_index(vaddr); - p4d = p4d_offset(pgd, vaddr); - pud = pud_offset(p4d, vaddr); - pmd = pmd_offset(pud, vaddr); + pmd = pmd_off_k(vaddr); pte = memblock_alloc(PAGE_SIZE, PAGE_SIZE); if (!pte) panic("%s: Failed to allocate %lu bytes align=0x%lx\n", diff --git a/arch/nds32/mm/proc.c b/arch/nds32/mm/proc.c index a2b9aacc5f89..848c845f5f33 100644 --- a/arch/nds32/mm/proc.c +++ b/arch/nds32/mm/proc.c @@ -15,14 +15,10 @@ extern struct cache_info L1_cache_info[2]; int va_kernel_present(unsigned long addr) { - p4d_t *p4d; - pud_t *pud; pmd_t *pmd; pte_t *ptep, pte; - p4d = p4d_offset(pgd_offset_k(addr), addr); - pud = pud_offset(p4d, addr); - pmd = pmd_offset(pud, addr); + pmd = pmd_off_k(addr); if (!pmd_none(*pmd)) { ptep = pte_offset_map(pmd, addr); pte = *ptep; diff --git a/arch/parisc/mm/fixmap.c b/arch/parisc/mm/fixmap.c index e2d8b0a857ee..24426a7e1a5e 100644 --- a/arch/parisc/mm/fixmap.c +++ b/arch/parisc/mm/fixmap.c @@ -33,11 +33,7 @@ void notrace set_fixmap(enum fixed_addresses idx, phys_addr_t phys) void notrace clear_fixmap(enum fixed_addresses idx) { unsigned long vaddr = __fix_to_virt(idx); - pgd_t *pgd = pgd_offset_k(vaddr); - p4d_t *p4d = p4d_offset(pgd, vaddr); - pud_t *pud = pud_offset(p4d, vaddr); - pmd_t *pmd = pmd_offset(pud, vaddr); - pte_t *pte = pte_offset_kernel(pmd, vaddr); + pte_t *pte = virt_to_kpte(vaddr); if (WARN_ON(pte_none(*pte))) return; diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index f74ef41920b7..3f50dc4c2d80 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -41,25 +41,6 @@ struct mm_struct; #ifndef __ASSEMBLY__ -#ifdef CONFIG_PPC32 -static inline pmd_t *pmd_ptr(struct mm_struct *mm, unsigned long va) -{ - return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va); -} - -static inline pmd_t *pmd_ptr_k(unsigned long va) -{ - return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va); -} - -static inline pte_t *virt_to_kpte(unsigned long vaddr) -{ - pmd_t *pmd = pmd_ptr_k(vaddr); - - return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr); -} -#endif - #include /* Keep these as a macros to avoid include dependency mess */ diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c index a6dcc708eee3..03b6ba54460e 100644 --- a/arch/powerpc/mm/book3s32/mmu.c +++ b/arch/powerpc/mm/book3s32/mmu.c @@ -320,7 +320,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea) if (!Hash) return; - pmd = pmd_ptr(mm, ea); + pmd = pmd_off(mm, ea); if (!pmd_none(*pmd)) add_hash_page(mm->context.id, ea, pmd_val(*pmd)); } diff --git a/arch/powerpc/mm/book3s32/tlb.c b/arch/powerpc/mm/book3s32/tlb.c index dc9039a170aa..157f98f6aea9 100644 --- a/arch/powerpc/mm/book3s32/tlb.c +++ b/arch/powerpc/mm/book3s32/tlb.c @@ -90,7 +90,7 @@ static void flush_range(struct mm_struct *mm, unsigned long start, if (start >= end) return; end = (end - 1) | ~PAGE_MASK; - pmd = pmd_ptr(mm, start); + pmd = pmd_off(mm, start); for (;;) { pmd_end = ((start + PGDIR_SIZE) & PGDIR_MASK) - 1; if (pmd_end > end) @@ -148,7 +148,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) return; } mm = (vmaddr < TASK_SIZE)? vma->vm_mm: &init_mm; - pmd = pmd_ptr(mm, vmaddr); + pmd = pmd_off(mm, vmaddr); if (!pmd_none(*pmd)) flush_hash_pages(mm->context.id, vmaddr, pmd_val(*pmd), 1); } diff --git a/arch/powerpc/mm/kasan/8xx.c b/arch/powerpc/mm/kasan/8xx.c index db4ef44af22f..569d98a41881 100644 --- a/arch/powerpc/mm/kasan/8xx.c +++ b/arch/powerpc/mm/kasan/8xx.c @@ -10,7 +10,7 @@ static int __init kasan_init_shadow_8M(unsigned long k_start, unsigned long k_end, void *block) { - pmd_t *pmd = pmd_ptr_k(k_start); + pmd_t *pmd = pmd_off_k(k_start); unsigned long k_cur, k_next; for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd += 2, block += SZ_8M) { @@ -59,7 +59,7 @@ int __init kasan_init_region(void *start, size_t size) return ret; for (; k_cur < k_end; k_cur += PAGE_SIZE) { - pmd_t *pmd = pmd_ptr_k(k_cur); + pmd_t *pmd = pmd_off_k(k_cur); void *va = block + k_cur - k_start; pte_t pte = pfn_pte(PHYS_PFN(__pa(va)), PAGE_KERNEL); diff --git a/arch/powerpc/mm/kasan/book3s_32.c b/arch/powerpc/mm/kasan/book3s_32.c index 4bc491a4a1fd..a32b4640b9de 100644 --- a/arch/powerpc/mm/kasan/book3s_32.c +++ b/arch/powerpc/mm/kasan/book3s_32.c @@ -46,7 +46,7 @@ int __init kasan_init_region(void *start, size_t size) kasan_update_early_region(k_start, k_cur, __pte(0)); for (; k_cur < k_end; k_cur += PAGE_SIZE) { - pmd_t *pmd = pmd_ptr_k(k_cur); + pmd_t *pmd = pmd_off_k(k_cur); void *va = block + k_cur - k_start; pte_t pte = pfn_pte(PHYS_PFN(__pa(va)), PAGE_KERNEL); diff --git a/arch/powerpc/mm/kasan/kasan_init_32.c b/arch/powerpc/mm/kasan/kasan_init_32.c index c42085801c04..0760e1e754e4 100644 --- a/arch/powerpc/mm/kasan/kasan_init_32.c +++ b/arch/powerpc/mm/kasan/kasan_init_32.c @@ -33,7 +33,7 @@ int __init kasan_init_shadow_page_tables(unsigned long k_start, unsigned long k_ pmd_t *pmd; unsigned long k_cur, k_next; - pmd = pmd_ptr_k(k_start); + pmd = pmd_off_k(k_start); for (k_cur = k_start; k_cur != k_end; k_cur = k_next, pmd++) { pte_t *new; @@ -69,7 +69,7 @@ int __init __weak kasan_init_region(void *start, size_t size) return -ENOMEM; for (k_cur = k_start & PAGE_MASK; k_cur < k_end; k_cur += PAGE_SIZE) { - pmd_t *pmd = pmd_ptr_k(k_cur); + pmd_t *pmd = pmd_off_k(k_cur); void *va = block + k_cur - k_start; pte_t pte = pfn_pte(PHYS_PFN(__pa(va)), PAGE_KERNEL); @@ -86,7 +86,7 @@ kasan_update_early_region(unsigned long k_start, unsigned long k_end, pte_t pte) phys_addr_t pa = __pa(kasan_early_shadow_page); for (k_cur = k_start; k_cur != k_end; k_cur += PAGE_SIZE) { - pmd_t *pmd = pmd_ptr_k(k_cur); + pmd_t *pmd = pmd_off_k(k_cur); pte_t *ptep = pte_offset_kernel(pmd, k_cur); if ((pte_val(*ptep) & PTE_RPN_MASK) != pa) @@ -184,7 +184,7 @@ void __init kasan_early_init(void) unsigned long addr = KASAN_SHADOW_START; unsigned long end = KASAN_SHADOW_END; unsigned long next; - pmd_t *pmd = pmd_ptr_k(addr); + pmd_t *pmd = pmd_off_k(addr); BUILD_BUG_ON(KASAN_SHADOW_START & ~PGDIR_MASK); diff --git a/arch/powerpc/mm/nohash/40x.c b/arch/powerpc/mm/nohash/40x.c index 7c1aedd4c867..13e74bc39ba5 100644 --- a/arch/powerpc/mm/nohash/40x.c +++ b/arch/powerpc/mm/nohash/40x.c @@ -103,7 +103,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top) pmd_t *pmdp; unsigned long val = p | _PMD_SIZE_16M | _PAGE_EXEC | _PAGE_RW; - pmdp = pmd_ptr_k(v); + pmdp = pmd_off_k(v); *pmdp++ = __pmd(val); *pmdp++ = __pmd(val); *pmdp++ = __pmd(val); @@ -118,7 +118,7 @@ unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top) pmd_t *pmdp; unsigned long val = p | _PMD_SIZE_4M | _PAGE_EXEC | _PAGE_RW; - pmdp = pmd_ptr_k(v); + pmdp = pmd_off_k(v); *pmdp = __pmd(val); v += LARGE_PAGE_SIZE_4M; diff --git a/arch/powerpc/mm/nohash/8xx.c b/arch/powerpc/mm/nohash/8xx.c index 286441bbbe49..92e8929cbe3e 100644 --- a/arch/powerpc/mm/nohash/8xx.c +++ b/arch/powerpc/mm/nohash/8xx.c @@ -74,7 +74,7 @@ static pte_t __init *early_hugepd_alloc_kernel(hugepd_t *pmdp, unsigned long va) static int __ref __early_map_kernel_hugepage(unsigned long va, phys_addr_t pa, pgprot_t prot, int psize, bool new) { - pmd_t *pmdp = pmd_ptr_k(va); + pmd_t *pmdp = pmd_off_k(va); pte_t *ptep; if (WARN_ON(psize != MMU_PAGE_512K && psize != MMU_PAGE_8M)) diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c index 9c0063b851ff..6eb4eab79385 100644 --- a/arch/powerpc/mm/pgtable_32.c +++ b/arch/powerpc/mm/pgtable_32.c @@ -40,7 +40,7 @@ notrace void __init early_ioremap_init(void) { unsigned long addr = ALIGN_DOWN(FIXADDR_START, PGDIR_SIZE); pte_t *ptep = (pte_t *)early_fixmap_pagetable; - pmd_t *pmdp = pmd_ptr_k(addr); + pmd_t *pmdp = pmd_off_k(addr); for (; (s32)(FIXADDR_TOP - addr) > 0; addr += PGDIR_SIZE, ptep += PTRS_PER_PTE, pmdp++) @@ -78,7 +78,7 @@ int __ref map_kernel_page(unsigned long va, phys_addr_t pa, pgprot_t prot) int err = -ENOMEM; /* Use upper 10 bits of VA to index the first level map */ - pd = pmd_ptr_k(va); + pd = pmd_off_k(va); /* Use middle 10 bits of VA to index the second-level map */ if (likely(slab_is_available())) pg = pte_alloc_kernel(pd, va); diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 732c12145d04..397944b9906b 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -337,19 +337,11 @@ void __kernel_map_pages(struct page *page, int numpages, int enable) { unsigned long address; int nr, i, j; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; pte_t *pte; for (i = 0; i < numpages;) { address = page_to_phys(page + i); - pgd = pgd_offset_k(address); - p4d = p4d_offset(pgd, address); - pud = pud_offset(p4d, address); - pmd = pmd_offset(pud, address); - pte = pte_offset_kernel(pmd, address); + pte = virt_to_kpte(address); nr = (unsigned long)pte >> ilog2(sizeof(long)); nr = PTRS_PER_PTE - (nr & (PTRS_PER_PTE - 1)); nr = min(numpages - i, nr); diff --git a/arch/sh/mm/cache-sh4.c b/arch/sh/mm/cache-sh4.c index ed599dadd698..de9e0a60e119 100644 --- a/arch/sh/mm/cache-sh4.c +++ b/arch/sh/mm/cache-sh4.c @@ -207,9 +207,6 @@ static void sh4_flush_cache_page(void *args) struct page *page; unsigned long address, pfn, phys; int map_coherent = 0; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; pmd_t *pmd; pte_t *pte; void *vaddr; @@ -223,10 +220,7 @@ static void sh4_flush_cache_page(void *args) if (cpu_context(smp_processor_id(), vma->vm_mm) == NO_CONTEXT) return; - pgd = pgd_offset(vma->vm_mm, address); - p4d = p4d_offset(pgd, address); - pud = pud_offset(p4d, address); - pmd = pmd_offset(pud, address); + pmd = pmd_off(vma->vm_mm, address); pte = pte_offset_kernel(pmd, address); /* If the page isn't present, there is nothing to do here. */ diff --git a/arch/sh/mm/kmap.c b/arch/sh/mm/kmap.c index 0e7039137f5a..73fd7cc99430 100644 --- a/arch/sh/mm/kmap.c +++ b/arch/sh/mm/kmap.c @@ -14,9 +14,6 @@ #include #include -#define kmap_get_fixmap_pte(vaddr) \ - pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr)), vaddr) - static pte_t *kmap_coherent_pte; void __init kmap_coherent_init(void) @@ -25,7 +22,7 @@ void __init kmap_coherent_init(void) /* cache the first coherent kmap pte */ vaddr = __fix_to_virt(FIX_CMAP_BEGIN); - kmap_coherent_pte = kmap_get_fixmap_pte(vaddr); + kmap_coherent_pte = virt_to_kpte(vaddr); } void *kmap_coherent(struct page *page, unsigned long addr) diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c index 6ff6e2a9f9b3..d1fc9a7b7d78 100644 --- a/arch/sparc/mm/highmem.c +++ b/arch/sparc/mm/highmem.c @@ -36,18 +36,10 @@ static pte_t *kmap_pte; void __init kmap_init(void) { - unsigned long address; - p4d_t *p4d; - pud_t *pud; - pmd_t *dir; - - address = __fix_to_virt(FIX_KMAP_BEGIN); - p4d = p4d_offset(pgd_offset_k(address), address); - pud = pud_offset(p4d, address); - dir = pmd_offset(pud, address); + unsigned long address = __fix_to_virt(FIX_KMAP_BEGIN); /* cache the first kmap pte */ - kmap_pte = pte_offset_kernel(dir, address); + kmap_pte = virt_to_kpte(address); } void *kmap_atomic_high_prot(struct page *page, pgprot_t prot) diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 141614f19084..02e6e5e0f106 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c @@ -503,11 +503,7 @@ void __kprobes flush_icache_range(unsigned long start, unsigned long end) if (kaddr >= PAGE_OFFSET) paddr = kaddr & mask; else { - pgd_t *pgdp = pgd_offset_k(kaddr); - p4d_t *p4dp = p4d_offset(pgdp, kaddr); - pud_t *pudp = pud_offset(p4dp, kaddr); - pmd_t *pmdp = pmd_offset(pudp, kaddr); - pte_t *ptep = pte_offset_kernel(pmdp, kaddr); + pte_t *ptep = virt_to_kpte(kaddr); paddr = pte_val(*ptep) & mask; } diff --git a/arch/sparc/mm/io-unit.c b/arch/sparc/mm/io-unit.c index 4aeed580b4d4..bfcc04bfce54 100644 --- a/arch/sparc/mm/io-unit.c +++ b/arch/sparc/mm/io-unit.c @@ -240,21 +240,15 @@ static void *iounit_alloc(struct device *dev, size_t len, while(addr < end) { page = va; { - pgd_t *pgdp; - p4d_t *p4dp; - pud_t *pudp; pmd_t *pmdp; pte_t *ptep; long i; - pgdp = pgd_offset(&init_mm, addr); - p4dp = p4d_offset(pgdp, addr); - pudp = pud_offset(p4dp, addr); - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_off_k(addr); ptep = pte_offset_map(pmdp, addr); set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); - + i = ((addr - IOUNIT_DMA_BASE) >> PAGE_SHIFT); iopte = iounit->page_table + i; diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c index fa9e298fd985..35b002eb312e 100644 --- a/arch/sparc/mm/iommu.c +++ b/arch/sparc/mm/iommu.c @@ -348,9 +348,6 @@ static void *sbus_iommu_alloc(struct device *dev, size_t len, while(addr < end) { page = va; { - pgd_t *pgdp; - p4d_t *p4dp; - pud_t *pudp; pmd_t *pmdp; pte_t *ptep; @@ -361,10 +358,7 @@ static void *sbus_iommu_alloc(struct device *dev, size_t len, else __flush_page_to_ram(page); - pgdp = pgd_offset(&init_mm, addr); - p4dp = p4d_offset(pgdp, addr); - pudp = pud_offset(p4dp, addr); - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_off_k(addr); ptep = pte_offset_map(pmdp, addr); set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c index 401b22f14743..c2ff76c8981e 100644 --- a/arch/um/kernel/mem.c +++ b/arch/um/kernel/mem.c @@ -125,10 +125,6 @@ static void __init fixaddr_user_init( void) { #ifdef CONFIG_ARCH_REUSE_HOST_VSYSCALL_AREA long size = FIXADDR_USER_END - FIXADDR_USER_START; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; pte_t *pte; phys_t p; unsigned long v, vaddr = FIXADDR_USER_START; @@ -146,11 +142,7 @@ static void __init fixaddr_user_init( void) p = __pa(v); for ( ; size > 0; size -= PAGE_SIZE, vaddr += PAGE_SIZE, p += PAGE_SIZE) { - pgd = swapper_pg_dir + pgd_index(vaddr); - p4d = p4d_offset(pgd, vaddr); - pud = pud_offset(p4d, vaddr); - pmd = pmd_offset(pud, vaddr); - pte = pte_offset_kernel(pmd, vaddr); + pte = virt_to_kpte(vaddr); pte_set_val(*pte, p, PAGE_READONLY); } #endif diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index b6ba694923e4..9c4b7e73ab8d 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -26,9 +26,6 @@ int handle_page_fault(unsigned long address, unsigned long ip, { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; pmd_t *pmd; pte_t *pte; int err = -EFAULT; @@ -102,10 +99,7 @@ good_area: } } - pgd = pgd_offset(mm, address); - p4d = p4d_offset(pgd, address); - pud = pud_offset(p4d, address); - pmd = pmd_offset(pud, address); + pmd = pmd_off(mm, address); pte = pte_offset_kernel(pmd, address); } while (!pte_present(*pte)); err = 0; diff --git a/arch/unicore32/mm/mm.h b/arch/unicore32/mm/mm.h index 27127abc95fb..f157f5d249ab 100644 --- a/arch/unicore32/mm/mm.h +++ b/arch/unicore32/mm/mm.h @@ -14,16 +14,6 @@ extern int sysctl_overcommit_memory; #define TOP_PTE(x) pte_offset_kernel(top_pmd, x) -static inline pmd_t *pmd_off(pgd_t *pgd, unsigned long virt) -{ - return pmd_offset((pud_t *)pgd, virt); -} - -static inline pmd_t *pmd_off_k(unsigned long virt) -{ - return pmd_off(pgd_offset_k(virt), virt); -} - struct mem_type { unsigned int prot_pte; unsigned int prot_l1; diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index 6a821727ceaf..bda909e3e37e 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c @@ -395,15 +395,6 @@ repeat: pte_t *kmap_pte; -static inline pte_t *kmap_get_fixmap_pte(unsigned long vaddr) -{ - pgd_t *pgd = pgd_offset_k(vaddr); - p4d_t *p4d = p4d_offset(pgd, vaddr); - pud_t *pud = pud_offset(p4d, vaddr); - pmd_t *pmd = pmd_offset(pud, vaddr); - return pte_offset_kernel(pmd, vaddr); -} - static void __init kmap_init(void) { unsigned long kmap_vstart; @@ -412,28 +403,17 @@ static void __init kmap_init(void) * Cache the first kmap pte: */ kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); - kmap_pte = kmap_get_fixmap_pte(kmap_vstart); + kmap_pte = virt_to_kpte(kmap_vstart); } #ifdef CONFIG_HIGHMEM static void __init permanent_kmaps_init(pgd_t *pgd_base) { - unsigned long vaddr; - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; + unsigned long vaddr = PKMAP_BASE; - vaddr = PKMAP_BASE; page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base); - pgd = swapper_pg_dir + pgd_index(vaddr); - p4d = p4d_offset(pgd, vaddr); - pud = pud_offset(p4d, vaddr); - pmd = pmd_offset(pud, vaddr); - pte = pte_offset_kernel(pmd, vaddr); - pkmap_page_table = pte; + pkmap_page_table = virt_to_kpte(vaddr); } void __init add_highpages_with_active_regions(int nid, diff --git a/arch/xtensa/include/asm/fixmap.h b/arch/xtensa/include/asm/fixmap.h index 89a6d1b7994d..a06ffb0c61c7 100644 --- a/arch/xtensa/include/asm/fixmap.h +++ b/arch/xtensa/include/asm/fixmap.h @@ -76,12 +76,4 @@ static inline unsigned long virt_to_fix(const unsigned long vaddr) #endif -#define kmap_get_fixmap_pte(vaddr) \ - pte_offset_kernel( \ - pmd_offset(pud_offset(p4d_offset(pgd_offset_k(vaddr), \ - (vaddr)), \ - (vaddr)), \ - (vaddr)), \ - (vaddr)) - #endif diff --git a/arch/xtensa/mm/highmem.c b/arch/xtensa/mm/highmem.c index 99b5ad137ab5..673196fe862e 100644 --- a/arch/xtensa/mm/highmem.c +++ b/arch/xtensa/mm/highmem.c @@ -86,6 +86,6 @@ void __init kmap_init(void) BUILD_BUG_ON(PKMAP_BASE < TLBTEMP_BASE_1 + TLBTEMP_SIZE); /* cache the first kmap pte */ kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); - kmap_pte = kmap_get_fixmap_pte(kmap_vstart); + kmap_pte = virt_to_kpte(kmap_vstart); kmap_waitqueues_init(); } diff --git a/arch/xtensa/mm/kasan_init.c b/arch/xtensa/mm/kasan_init.c index e3baa21ff24c..1fef24db2ff6 100644 --- a/arch/xtensa/mm/kasan_init.c +++ b/arch/xtensa/mm/kasan_init.c @@ -19,10 +19,7 @@ void __init kasan_early_init(void) { unsigned long vaddr = KASAN_SHADOW_START; - pgd_t *pgd = pgd_offset_k(vaddr); - p4d_t *p4d = p4d_offset(pgd, vaddr); - pud_t *pud = pud_offset(p4d, vaddr); - pmd_t *pmd = pmd_offset(pud, vaddr); + pmd_t *pmd = pmd_off_k(vaddr); int i; for (i = 0; i < PTRS_PER_PTE; ++i) @@ -43,10 +40,7 @@ static void __init populate(void *start, void *end) unsigned long n_pmds = n_pages / PTRS_PER_PTE; unsigned long i, j; unsigned long vaddr = (unsigned long)start; - pgd_t *pgd = pgd_offset_k(vaddr); - p4d_t *p4d = p4d_offset(pgd, vaddr); - pud_t *pud = pud_offset(p4d, vaddr); - pmd_t *pmd = pmd_offset(pud, vaddr); + pmd_t *pmd = pmd_off_k(vaddr); pte_t *pte = memblock_alloc(n_pages * sizeof(pte_t), PAGE_SIZE); if (!pte) diff --git a/arch/xtensa/mm/mmu.c b/arch/xtensa/mm/mmu.c index 37e478a27877..fd2193df8a14 100644 --- a/arch/xtensa/mm/mmu.c +++ b/arch/xtensa/mm/mmu.c @@ -21,10 +21,7 @@ #if defined(CONFIG_HIGHMEM) static void * __init init_pmd(unsigned long vaddr, unsigned long n_pages) { - pgd_t *pgd = pgd_offset_k(vaddr); - p4d_t *p4d = p4d_offset(pgd, vaddr); - pud_t *pud = pud_offset(p4d, vaddr); - pmd_t *pmd = pmd_offset(pud, vaddr); + pmd_t *pmd = pmd_off_k(vaddr); pte_t *pte; unsigned long i; diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 6e274da637fd..ee85b222d098 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -28,6 +28,30 @@ #define USER_PGTABLES_CEILING 0UL #endif +/* + * In many cases it is known that a virtual address is mapped at PMD or PTE + * level, so instead of traversing all the page table levels, we can get a + * pointer to the PMD entry in user or kernel page table or translate a virtual + * address to the pointer in the PTE in the kernel page tables with simple + * helpers. + */ +static inline pmd_t *pmd_off(struct mm_struct *mm, unsigned long va) +{ + return pmd_offset(pud_offset(p4d_offset(pgd_offset(mm, va), va), va), va); +} + +static inline pmd_t *pmd_off_k(unsigned long va) +{ + return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(va), va), va), va); +} + +static inline pte_t *virt_to_kpte(unsigned long vaddr) +{ + pmd_t *pmd = pmd_off_k(vaddr); + + return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr); +} + #ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, -- cgit v1.2.3-70-g09d2 From 974b9b2c68f3d35a65e80af9657fe378d2439b60 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Mon, 8 Jun 2020 21:33:10 -0700 Subject: mm: consolidate pte_index() and pte_offset_*() definitions All architectures define pte_index() as (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1) and all architectures define pte_offset_kernel() as an entry in the array of PTEs indexed by the pte_index(). For the most architectures the pte_offset_kernel() implementation relies on the availability of pmd_page_vaddr() that converts a PMD entry value to the virtual address of the page containing PTEs array. Let's move x86 definitions of the PTE accessors to the generic place in and then simply drop the respective definitions from the other architectures. The architectures that didn't provide pmd_page_vaddr() are updated to have that defined. The generic implementation of pte_offset_kernel() can be overridden by an architecture and alpha makes use of this because it has special ordering requirements for its version of pte_offset_kernel(). [rppt@linux.ibm.com: v2] Link: http://lkml.kernel.org/r/20200514170327.31389-11-rppt@kernel.org [rppt@linux.ibm.com: update] Link: http://lkml.kernel.org/r/20200514170327.31389-12-rppt@kernel.org [rppt@linux.ibm.com: update] Link: http://lkml.kernel.org/r/20200514170327.31389-13-rppt@kernel.org [akpm@linux-foundation.org: fix x86 warning] [sfr@canb.auug.org.au: fix powerpc build] Link: http://lkml.kernel.org/r/20200607153443.GB738695@linux.ibm.com Signed-off-by: Mike Rapoport Signed-off-by: Stephen Rothwell Signed-off-by: Andrew Morton Cc: Arnd Bergmann Cc: Borislav Petkov Cc: Brian Cain Cc: Catalin Marinas Cc: Chris Zankel Cc: "David S. Miller" Cc: Geert Uytterhoeven Cc: Greentime Hu Cc: Greg Ungerer Cc: Guan Xuetao Cc: Guo Ren Cc: Heiko Carstens Cc: Helge Deller Cc: Ingo Molnar Cc: Ley Foon Tan Cc: Mark Salter Cc: Matthew Wilcox Cc: Matt Turner Cc: Max Filippov Cc: Michael Ellerman Cc: Michal Simek Cc: Nick Hu Cc: Paul Walmsley Cc: Richard Weinberger Cc: Rich Felker Cc: Russell King Cc: Stafford Horne Cc: Thomas Bogendoerfer Cc: Thomas Gleixner Cc: Tony Luck Cc: Vincent Chen Cc: Vineet Gupta Cc: Will Deacon Cc: Yoshinori Sato Link: http://lkml.kernel.org/r/20200514170327.31389-10-rppt@kernel.org Signed-off-by: Linus Torvalds --- arch/alpha/include/asm/pgtable.h | 14 +--- arch/arc/include/asm/pgtable.h | 22 ------ arch/arm/include/asm/pgtable-2level.h | 1 + arch/arm/include/asm/pgtable-3level.h | 7 -- arch/arm/include/asm/pgtable-nommu.h | 1 - arch/arm/include/asm/pgtable.h | 23 ------- arch/arm64/include/asm/pgtable.h | 36 ++++------ arch/arm64/kernel/hibernate.c | 4 +- arch/arm64/mm/kasan_init.c | 2 +- arch/arm64/mm/mmu.c | 8 +-- arch/c6x/include/asm/pgtable.h | 1 - arch/csky/include/asm/pgtable.h | 30 -------- arch/hexagon/include/asm/pgtable.h | 52 ++------------ arch/ia64/include/asm/pgtable.h | 33 +-------- arch/m68k/include/asm/mcf_pgtable.h | 23 +------ arch/m68k/include/asm/motorola_pgalloc.h | 2 +- arch/m68k/include/asm/motorola_pgtable.h | 34 +-------- arch/m68k/include/asm/sun3_pgtable.h | 24 ++----- arch/m68k/mm/init.c | 2 +- arch/microblaze/include/asm/pgtable.h | 21 ++---- arch/mips/include/asm/pgtable-32.h | 22 ------ arch/mips/include/asm/pgtable-64.h | 32 --------- arch/mips/kvm/mmu.c | 20 +++--- arch/mips/kvm/trap_emul.c | 2 +- arch/nds32/include/asm/pgtable.h | 18 ++--- arch/nios2/include/asm/pgtable.h | 22 ++---- arch/openrisc/include/asm/pgtable.h | 31 ++------- arch/parisc/include/asm/pgtable.h | 32 ++------- arch/parisc/kernel/pci-dma.c | 2 +- arch/powerpc/include/asm/book3s/32/pgtable.h | 20 +----- arch/powerpc/include/asm/book3s/64/pgtable.h | 43 ------------ arch/powerpc/include/asm/nohash/32/pgtable.h | 17 +---- arch/powerpc/include/asm/nohash/64/pgtable-4k.h | 4 -- arch/powerpc/include/asm/nohash/64/pgtable.h | 22 ------ arch/powerpc/include/asm/pgtable.h | 7 ++ arch/riscv/include/asm/pgtable-64.h | 7 -- arch/riscv/include/asm/pgtable.h | 20 ------ arch/riscv/mm/init.c | 30 ++++---- arch/s390/include/asm/pgtable.h | 13 ++-- arch/s390/mm/pageattr.c | 2 +- arch/sh/include/asm/pgtable-3level.h | 7 -- arch/sh/include/asm/pgtable_32.h | 25 ++----- arch/sparc/include/asm/pgalloc_64.h | 2 +- arch/sparc/include/asm/pgtable_32.h | 32 ++------- arch/sparc/include/asm/pgtable_64.h | 31 +-------- arch/um/include/asm/pgtable-3level.h | 4 -- arch/um/include/asm/pgtable.h | 67 ++++-------------- arch/unicore32/include/asm/pgtable.h | 17 ----- arch/x86/include/asm/pgtable.h | 71 ------------------- arch/x86/include/asm/pgtable_32.h | 11 --- arch/x86/include/asm/pgtable_64.h | 4 -- arch/xtensa/include/asm/pgtable.h | 18 +---- include/asm-generic/pgtable-nopmd.h | 1 + include/asm-generic/pgtable-nopud.h | 1 + include/linux/pgtable.h | 91 +++++++++++++++++++++++++ 55 files changed, 228 insertions(+), 860 deletions(-) (limited to 'include/linux/pgtable.h') diff --git a/arch/alpha/include/asm/pgtable.h b/arch/alpha/include/asm/pgtable.h index 1c263922beb3..162c17b2631f 100644 --- a/arch/alpha/include/asm/pgtable.h +++ b/arch/alpha/include/asm/pgtable.h @@ -276,15 +276,6 @@ extern inline pte_t pte_mkwrite(pte_t pte) { pte_val(pte) &= ~_PAGE_FOW; return extern inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= __DIRTY_BITS; return pte; } extern inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= __ACCESS_BITS; return pte; } -#define PAGE_DIR_OFFSET(tsk,address) pgd_offset((tsk),(address)) - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, (address)) - -/* to find an entry in a page-table-directory. */ -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) -#define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address)) - /* * The smp_read_barrier_depends() in the following functions are required to * order the load of *dir (the pointer in the top level page table) with any @@ -305,6 +296,7 @@ extern inline pmd_t * pmd_offset(pud_t * dir, unsigned long address) smp_read_barrier_depends(); /* see above */ return ret; } +#define pmd_offset pmd_offset /* Find an entry in the third-level page table.. */ extern inline pte_t * pte_offset_kernel(pmd_t * dir, unsigned long address) @@ -314,9 +306,7 @@ extern inline pte_t * pte_offset_kernel(pmd_t * dir, unsigned long address) smp_read_barrier_depends(); /* see above */ return ret; } - -#define pte_offset_map(dir,addr) pte_offset_kernel((dir),(addr)) -#define pte_unmap(pte) do { } while (0) +#define pte_offset_kernel pte_offset_kernel extern pgd_t swapper_pg_dir[1024]; diff --git a/arch/arc/include/asm/pgtable.h b/arch/arc/include/asm/pgtable.h index 29137bcedd93..f1ed17edb085 100644 --- a/arch/arc/include/asm/pgtable.h +++ b/arch/arc/include/asm/pgtable.h @@ -248,9 +248,6 @@ extern char empty_zero_page[PAGE_SIZE]; #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) -#define pte_unmap(pte) do { } while (0) -#define pte_unmap_nested(pte) do { } while (0) - #define set_pte(pteptr, pteval) ((*(pteptr)) = (pteval)) #define set_pmd(pmdptr, pmdval) (*(pmdptr) = pmdval) @@ -282,18 +279,6 @@ static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) /* Don't use virt_to_pfn for macros below: could cause truncations for PAE40*/ #define pte_pfn(pte) (pte_val(pte) >> PAGE_SHIFT) -#define __pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) - -/* - * pte_offset gets a @ptr to PMD entry (PGD in our 2-tier paging system) - * and returns ptr to PTE entry corresponding to @addr - */ -#define pte_offset(dir, addr) ((pte_t *)(pmd_page_vaddr(*dir)) +\ - __pte_index(addr)) - -/* No mapping of Page Tables in high mem etc, so following same as above */ -#define pte_offset_kernel(dir, addr) pte_offset(dir, addr) -#define pte_offset_map(dir, addr) pte_offset(dir, addr) /* Zoo of pte_xxx function */ #define pte_read(pte) (pte_val(pte) & _PAGE_READ) @@ -331,13 +316,6 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, set_pte(ptep, pteval); } -/* - * All kernel related VM pages are in init's mm. - */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) -#define pgd_index(addr) ((addr) >> PGDIR_SHIFT) -#define pgd_offset(mm, addr) (((mm)->pgd)+pgd_index(addr)) - /* * Macro to quickly access the PGD entry, utlising the fact that some * arch may cache the pointer to Page Directory of "current" task diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h index 9e084a464a97..3502c2f746ca 100644 --- a/arch/arm/include/asm/pgtable-2level.h +++ b/arch/arm/include/asm/pgtable-2level.h @@ -187,6 +187,7 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) { return (pmd_t *)pud; } +#define pmd_offset pmd_offset #define pmd_large(pmd) (pmd_val(pmd) & 2) #define pmd_leaf(pmd) (pmd_val(pmd) & 2) diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h index 1933aed9f68d..fbb6693c3352 100644 --- a/arch/arm/include/asm/pgtable-3level.h +++ b/arch/arm/include/asm/pgtable-3level.h @@ -133,13 +133,6 @@ static inline pmd_t *pud_page_vaddr(pud_t pud) return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK); } -/* Find an entry in the second-level page table.. */ -#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) -static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) -{ - return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr); -} - #define pmd_bad(pmd) (!(pmd_val(pmd) & 2)) #define copy_pmd(pmdpd,pmdps) \ diff --git a/arch/arm/include/asm/pgtable-nommu.h b/arch/arm/include/asm/pgtable-nommu.h index 1a758f14e0c3..d16aba48fa0a 100644 --- a/arch/arm/include/asm/pgtable-nommu.h +++ b/arch/arm/include/asm/pgtable-nommu.h @@ -22,7 +22,6 @@ #define pgd_bad(pgd) (0) #define pgd_clear(pgdp) #define kern_addr_valid(addr) (1) -#define pmd_offset(a, b) ((void *)0) /* FIXME */ /* * PMD_SHIFT determines the size of the area a second-level page table can map diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index 38c7cd8b014b..c02f24400369 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -166,14 +166,6 @@ extern struct page *empty_zero_page; extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; -/* to find an entry in a page-table-directory */ -#define pgd_index(addr) ((addr) >> PGDIR_SHIFT) - -#define pgd_offset(mm, addr) ((mm)->pgd + pgd_index(addr)) - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(addr) pgd_offset(&init_mm, addr) - #define pmd_none(pmd) (!pmd_val(pmd)) static inline pte_t *pmd_page_vaddr(pmd_t pmd) @@ -183,21 +175,6 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd) #define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK)) -#ifndef CONFIG_HIGHPTE -#define __pte_map(pmd) pmd_page_vaddr(*(pmd)) -#define __pte_unmap(pte) do { } while (0) -#else -#define __pte_map(pmd) (pte_t *)kmap_atomic(pmd_page(*(pmd))) -#define __pte_unmap(pte) kunmap_atomic(pte) -#endif - -#define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) - -#define pte_offset_kernel(pmd,addr) (pmd_page_vaddr(*(pmd)) + pte_index(addr)) - -#define pte_offset_map(pmd,addr) (__pte_map(pmd) + pte_index(addr)) -#define pte_unmap(pte) __pte_unmap(pte) - #define pte_pfn(pte) ((pte_val(pte) & PHYS_MASK) >> PAGE_SHIFT) #define pfn_pte(pfn,prot) __pte(__pfn_to_phys(pfn) | pgprot_val(prot)) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 437c92ab2081..6dbd267ab931 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -506,15 +506,13 @@ static inline phys_addr_t pmd_page_paddr(pmd_t pmd) return __pmd_to_phys(pmd); } -static inline void pte_unmap(pte_t *pte) { } +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return (unsigned long)__va(pmd_page_paddr(pmd)); +} /* Find an entry in the third-level page table. */ -#define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) - #define pte_offset_phys(dir,addr) (pmd_page_paddr(READ_ONCE(*(dir))) + pte_index(addr) * sizeof(pte_t)) -#define pte_offset_kernel(dir,addr) ((pte_t *)__va(pte_offset_phys((dir), (addr)))) - -#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr)) #define pte_set_fixmap(addr) ((pte_t *)set_fixmap_offset(FIX_PTE, addr)) #define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr)) @@ -568,11 +566,13 @@ static inline phys_addr_t pud_page_paddr(pud_t pud) return __pud_to_phys(pud); } -/* Find an entry in the second-level page table. */ -#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) +static inline unsigned long pud_page_vaddr(pud_t pud) +{ + return (unsigned long)__va(pud_page_paddr(pud)); +} +/* Find an entry in the second-level page table. */ #define pmd_offset_phys(dir, addr) (pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t)) -#define pmd_offset(dir, addr) ((pmd_t *)__va(pmd_offset_phys((dir), (addr)))) #define pmd_set_fixmap(addr) ((pmd_t *)set_fixmap_offset(FIX_PMD, addr)) #define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr)) @@ -626,11 +626,13 @@ static inline phys_addr_t p4d_page_paddr(p4d_t p4d) return __p4d_to_phys(p4d); } -/* Find an entry in the frst-level page table. */ -#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) +static inline unsigned long p4d_page_vaddr(p4d_t p4d) +{ + return (unsigned long)__va(p4d_page_paddr(p4d)); +} +/* Find an entry in the frst-level page table. */ #define pud_offset_phys(dir, addr) (p4d_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t)) -#define pud_offset(dir, addr) ((pud_t *)__va(pud_offset_phys((dir), (addr)))) #define pud_set_fixmap(addr) ((pud_t *)set_fixmap_offset(FIX_PUD, addr)) #define pud_set_fixmap_offset(p4d, addr) pud_set_fixmap(pud_offset_phys(p4d, addr)) @@ -657,16 +659,6 @@ static inline phys_addr_t p4d_page_paddr(p4d_t p4d) #define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd)) -/* to find an entry in a page-table-directory */ -#define pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) - -#define pgd_offset_raw(pgd, addr) ((pgd) + pgd_index(addr)) - -#define pgd_offset(mm, addr) (pgd_offset_raw((mm)->pgd, (addr))) - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(addr) pgd_offset(&init_mm, addr) - #define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr)) #define pgd_clear_fixmap() clear_fixmap(FIX_PGD) diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 668903c94a2b..68e14152d6e9 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -188,7 +188,7 @@ static int trans_pgd_map_page(pgd_t *trans_pgd, void *page, pmd_t *pmdp; pte_t *ptep; - pgdp = pgd_offset_raw(trans_pgd, dst_addr); + pgdp = pgd_offset_pgd(trans_pgd, dst_addr); if (pgd_none(READ_ONCE(*pgdp))) { pudp = (void *)get_safe_page(GFP_ATOMIC); if (!pudp) @@ -490,7 +490,7 @@ static int copy_page_tables(pgd_t *dst_pgdp, unsigned long start, unsigned long addr = start; pgd_t *src_pgdp = pgd_offset_k(start); - dst_pgdp = pgd_offset_raw(dst_pgdp, start); + dst_pgdp = pgd_offset_pgd(dst_pgdp, start); do { next = pgd_addr_end(addr, end); if (pgd_none(READ_ONCE(*src_pgdp))) diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c index ca77eb8a0ccd..7291b26ce788 100644 --- a/arch/arm64/mm/kasan_init.c +++ b/arch/arm64/mm/kasan_init.c @@ -190,7 +190,7 @@ void __init kasan_copy_shadow(pgd_t *pgdir) pgdp = pgd_offset_k(KASAN_SHADOW_START); pgdp_end = pgd_offset_k(KASAN_SHADOW_END); - pgdp_new = pgd_offset_raw(pgdir, KASAN_SHADOW_START); + pgdp_new = pgd_offset_pgd(pgdir, KASAN_SHADOW_START); do { set_pgd(pgdp_new, READ_ONCE(*pgdp)); } while (pgdp++, pgdp_new++, pgdp != pgdp_end); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index e7fbc6275329..990929c8837e 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -341,7 +341,7 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, int flags) { unsigned long addr, end, next; - pgd_t *pgdp = pgd_offset_raw(pgdir, virt); + pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); /* * If the virtual and physical address don't have the same offset @@ -663,13 +663,13 @@ static void __init map_kernel(pgd_t *pgdp) &vmlinux_initdata, 0, VM_NO_GUARD); map_kernel_segment(pgdp, _data, _end, PAGE_KERNEL, &vmlinux_data, 0, 0); - if (!READ_ONCE(pgd_val(*pgd_offset_raw(pgdp, FIXADDR_START)))) { + if (!READ_ONCE(pgd_val(*pgd_offset_pgd(pgdp, FIXADDR_START)))) { /* * The fixmap falls in a separate pgd to the kernel, and doesn't * live in the carveout for the swapper_pg_dir. We can simply * re-use the existing dir for the fixmap. */ - set_pgd(pgd_offset_raw(pgdp, FIXADDR_START), + set_pgd(pgd_offset_pgd(pgdp, FIXADDR_START), READ_ONCE(*pgd_offset_k(FIXADDR_START))); } else if (CONFIG_PGTABLE_LEVELS > 3) { pgd_t *bm_pgdp; @@ -682,7 +682,7 @@ static void __init map_kernel(pgd_t *pgdp) * entry instead. */ BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES)); - bm_pgdp = pgd_offset_raw(pgdp, FIXADDR_START); + bm_pgdp = pgd_offset_pgd(pgdp, FIXADDR_START); bm_p4dp = p4d_offset(bm_pgdp, FIXADDR_START); bm_pudp = pud_set_fixmap_offset(bm_p4dp, FIXADDR_START); pud_populate(&init_mm, bm_pudp, lm_alias(bm_pmd)); diff --git a/arch/c6x/include/asm/pgtable.h b/arch/c6x/include/asm/pgtable.h index 60a67d833989..8a91ceda39fa 100644 --- a/arch/c6x/include/asm/pgtable.h +++ b/arch/c6x/include/asm/pgtable.h @@ -26,7 +26,6 @@ #define pgd_clear(pgdp) #define kern_addr_valid(addr) (1) -#define pmd_offset(a, b) ((void *)0) #define pmd_none(x) (!pmd_val(x)) #define pmd_present(x) (pmd_val(x)) #define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0) diff --git a/arch/csky/include/asm/pgtable.h b/arch/csky/include/asm/pgtable.h index 2eff4aea51b3..2002cb7f1053 100644 --- a/arch/csky/include/asm/pgtable.h +++ b/arch/csky/include/asm/pgtable.h @@ -32,13 +32,6 @@ #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) -/* Find an entry in the third-level page table.. */ -#define __pte_offset_t(address) \ - (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir, address) \ - (pmd_page_vaddr(*(dir)) + __pte_offset_t(address)) -#define pte_offset_map(dir, address) \ - ((pte_t *)page_address(pmd_page(*(dir))) + __pte_offset_t(address)) #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) #define pte_clear(mm, addr, ptep) set_pte((ptep), \ (((unsigned int) addr & PAGE_OFFSET) ? __pte(_PAGE_GLOBAL) : __pte(0))) @@ -54,8 +47,6 @@ #define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_MODIFIED | \ _CACHE_MASK) -#define pte_unmap(pte) ((void)(pte)) - #define __swp_type(x) (((x).val >> 4) & 0xff) #define __swp_offset(x) ((x).val >> 12) #define __swp_entry(type, offset) ((swp_entry_t) {((type) << 4) | \ @@ -229,14 +220,6 @@ static inline pte_t pte_mkyoung(pte_t pte) return pte; } -#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) -#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -#define pgd_index(address) ((address) >> PGDIR_SHIFT) - #define __HAVE_PHYS_MEM_ACCESS_PROT struct file; extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, @@ -280,19 +263,6 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) (pgprot_val(newprot))); } -/* to find an entry in a page-table-directory */ -static inline pgd_t *pgd_offset(struct mm_struct *mm, unsigned long address) -{ - return mm->pgd + pgd_index(address); -} - -/* Find an entry in the third-level page table.. */ -static inline pte_t *pte_offset(pmd_t *dir, unsigned long address) -{ - return (pte_t *) (pmd_page_vaddr(*dir)) + - ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); -} - extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; extern void paging_init(void); diff --git a/arch/hexagon/include/asm/pgtable.h b/arch/hexagon/include/asm/pgtable.h index 8df6e2702b53..dbb22b80b8c4 100644 --- a/arch/hexagon/include/asm/pgtable.h +++ b/arch/hexagon/include/asm/pgtable.h @@ -206,33 +206,6 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_val(*ptep) = _NULL_PTE; } -#ifdef NEED_PMD_INDEX_DESPITE_BEING_2_LEVEL -/** - * pmd_index - returns the index of the entry in the PMD page - * which would control the given virtual address - */ -#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) - -#endif - -/** - * pgd_index - returns the index of the entry in the PGD page - * which would control the given virtual address - * - * This returns the *index* for the address in the pgd_t - */ -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) - -/* - * pgd_offset - find an offset in a page-table-directory - */ -#define pgd_offset(mm, addr) ((mm)->pgd + pgd_index(addr)) - -/* - * pgd_offset_k - get kernel (init_mm) pgd entry pointer for addr - */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - /** * pmd_none - check if pmd_entry is mapped * @pmd_entry: pmd entry @@ -403,31 +376,14 @@ static inline int pte_exec(pte_t pte) */ #define set_pte_at(mm, addr, ptep, pte) set_pte(ptep, pte) -/* - * May need to invoke the virtual machine as well... - */ -#define pte_unmap(pte) do { } while (0) -#define pte_unmap_nested(pte) do { } while (0) - -/* - * pte_offset_map - returns the linear address of the page table entry - * corresponding to an address - */ -#define pte_offset_map(dir, address) \ - ((pte_t *)page_address(pmd_page(*(dir))) + __pte_offset(address)) - -#define pte_offset_map_nested(pmd, addr) pte_offset_map(pmd, addr) - -/* pte_offset_kernel - kernel version of pte_offset */ -#define pte_offset_kernel(dir, address) \ - ((pte_t *) (unsigned long) __va(pmd_val(*dir) & PAGE_MASK) \ - + __pte_offset(address)) +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return (unsigned long)__va(pmd_val(pmd) & PAGE_MASK); +} /* ZERO_PAGE - returns the globally shared zero page */ #define ZERO_PAGE(vaddr) (virt_to_page(&empty_zero_page)) -#define __pte_offset(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) - /* * Swap/file PTE definitions. If _PAGE_PRESENT is zero, the rest of the PTE is * interpreted as swap information. The remaining free bits are interpreted as diff --git a/arch/ia64/include/asm/pgtable.h b/arch/ia64/include/asm/pgtable.h index 3399824ad8a5..10850897a91c 100644 --- a/arch/ia64/include/asm/pgtable.h +++ b/arch/ia64/include/asm/pgtable.h @@ -364,44 +364,13 @@ pgd_index (unsigned long address) return (region << (PAGE_SHIFT - 6)) | l1index; } - -/* The offset in the 1-level directory is given by the 3 region bits - (61..63) and the level-1 bits. */ -static inline pgd_t* -pgd_offset (const struct mm_struct *mm, unsigned long address) -{ - return mm->pgd + pgd_index(address); -} - -/* In the kernel's mapped region we completely ignore the region number - (since we know it's in region number 5). */ -#define pgd_offset_k(addr) \ - (init_mm.pgd + (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))) +#define pgd_index pgd_index /* Look up a pgd entry in the gate area. On IA-64, the gate-area resides in the kernel-mapped segment, hence we use pgd_offset_k() here. */ #define pgd_offset_gate(mm, addr) pgd_offset_k(addr) -#if CONFIG_PGTABLE_LEVELS == 4 -/* Find an entry in the second-level page table.. */ -#define pud_offset(dir,addr) \ - ((pud_t *) p4d_page_vaddr(*(dir)) + (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))) -#endif - -/* Find an entry in the third-level page table.. */ -#define pmd_offset(dir,addr) \ - ((pmd_t *) pud_page_vaddr(*(dir)) + (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))) - -/* - * Find an entry in the third-level page table. This looks more complicated than it - * should be because some platforms place page tables in high memory. - */ -#define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir,addr) ((pte_t *) pmd_page_vaddr(*(dir)) + pte_index(addr)) -#define pte_offset_map(dir,addr) pte_offset_kernel(dir, addr) -#define pte_unmap(pte) do { } while (0) - /* atomic versions of the some PTE manipulations: */ static inline int diff --git a/arch/m68k/include/asm/mcf_pgtable.h b/arch/m68k/include/asm/mcf_pgtable.h index 737e826294f3..8d4ec05996c5 100644 --- a/arch/m68k/include/asm/mcf_pgtable.h +++ b/arch/m68k/include/asm/mcf_pgtable.h @@ -170,7 +170,7 @@ static inline void pgd_set(pgd_t *pgdp, pmd_t *pmdp) } #define __pte_page(pte) ((unsigned long) (pte_val(pte) & PAGE_MASK)) -#define __pmd_page(pmd) ((unsigned long) (pmd_val(pmd))) +#define pmd_page_vaddr(pmd) ((unsigned long) (pmd_val(pmd))) static inline int pte_none(pte_t pte) { @@ -310,24 +310,6 @@ static inline pte_t pte_mkcache(pte_t pte) #define swapper_pg_dir kernel_pg_dir extern pgd_t kernel_pg_dir[PTRS_PER_PGD]; -/* - * Find an entry in a pagetable directory. - */ -#define pgd_index(address) ((address) >> PGDIR_SHIFT) -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) - -/* - * Find an entry in a kernel pagetable directory. - */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -/* - * Find an entry in the third-level pagetable. - */ -#define __pte_offset(address) ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir, address) \ - ((pte_t *) __pmd_page(*(dir)) + __pte_offset(address)) - /* * Encode and de-code a swap entry (must be !pte_none(e) && !pte_present(e)) */ @@ -340,9 +322,6 @@ extern pgd_t kernel_pg_dir[PTRS_PER_PGD]; #define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)) -#define pte_offset_map(pmdp, addr) ((pte_t *)__pmd_page(*pmdp) + \ - __pte_offset(addr)) -#define pte_unmap(pte) ((void) 0) #define pfn_pte(pfn, prot) __pte(((pfn) << PAGE_SHIFT) | pgprot_val(prot)) #define pte_pfn(pte) (pte_val(pte) >> PAGE_SHIFT) diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h index f3cb453a07b7..b4fc3b4f6bb3 100644 --- a/arch/m68k/include/asm/motorola_pgalloc.h +++ b/arch/m68k/include/asm/motorola_pgalloc.h @@ -88,7 +88,7 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page { pmd_set(pmd, page); } -#define pmd_pgtable(pmd) ((pgtable_t)__pmd_page(pmd)) +#define pmd_pgtable(pmd) ((pgtable_t)pmd_page_vaddr(pmd)) static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) { diff --git a/arch/m68k/include/asm/motorola_pgtable.h b/arch/m68k/include/asm/motorola_pgtable.h index e1594acf7c7e..8076467eff4b 100644 --- a/arch/m68k/include/asm/motorola_pgtable.h +++ b/arch/m68k/include/asm/motorola_pgtable.h @@ -128,7 +128,7 @@ static inline void pud_set(pud_t *pudp, pmd_t *pmdp) } #define __pte_page(pte) ((unsigned long)__va(pte_val(pte) & PAGE_MASK)) -#define __pmd_page(pmd) ((unsigned long)__va(pmd_val(pmd) & _TABLE_MASK)) +#define pmd_page_vaddr(pmd) ((unsigned long)__va(pmd_val(pmd) & _TABLE_MASK)) #define pud_page_vaddr(pud) ((unsigned long)__va(pud_val(pud) & _TABLE_MASK)) @@ -192,41 +192,9 @@ static inline pte_t pte_mkcache(pte_t pte) return pte; } -#define PAGE_DIR_OFFSET(tsk,address) pgd_offset((tsk),(address)) - -#define pgd_index(address) ((address) >> PGDIR_SHIFT) - -/* to find an entry in a page-table-directory */ -static inline pgd_t *pgd_offset(const struct mm_struct *mm, - unsigned long address) -{ - return mm->pgd + pgd_index(address); -} - #define swapper_pg_dir kernel_pg_dir extern pgd_t kernel_pg_dir[128]; -static inline pgd_t *pgd_offset_k(unsigned long address) -{ - return kernel_pg_dir + (address >> PGDIR_SHIFT); -} - - -/* Find an entry in the second-level page table.. */ -static inline pmd_t *pmd_offset(pud_t *dir, unsigned long address) -{ - return (pmd_t *)pud_page_vaddr(*dir) + ((address >> PMD_SHIFT) & (PTRS_PER_PMD-1)); -} - -/* Find an entry in the third-level page table.. */ -static inline pte_t *pte_offset_kernel(pmd_t *pmdp, unsigned long address) -{ - return (pte_t *)__pmd_page(*pmdp) + ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); -} - -#define pte_offset_map(pmdp,address) ((pte_t *)__pmd_page(*pmdp) + (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))) -#define pte_unmap(pte) ((void)0) - /* Encode and de-code a swap entry (must be !pte_none(e) && !pte_present(e)) */ #define __swp_type(x) (((x).val >> 4) & 0xff) #define __swp_offset(x) ((x).val >> 12) diff --git a/arch/m68k/include/asm/sun3_pgtable.h b/arch/m68k/include/asm/sun3_pgtable.h index 0caa18a08437..5b24283a0a42 100644 --- a/arch/m68k/include/asm/sun3_pgtable.h +++ b/arch/m68k/include/asm/sun3_pgtable.h @@ -112,8 +112,11 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) #define __pte_page(pte) \ ((unsigned long) __va ((pte_val (pte) & SUN3_PAGE_PGNUM_MASK) << PAGE_SHIFT)) -#define __pmd_page(pmd) \ -((unsigned long) __va (pmd_val (pmd) & PAGE_MASK)) + +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return (unsigned long)__va(pmd_val(pmd) & PAGE_MASK); +} static inline int pte_none (pte_t pte) { return !pte_val (pte); } static inline int pte_present (pte_t pte) { return pte_val (pte) & SUN3_PAGE_VALID; } @@ -127,7 +130,7 @@ static inline void pte_clear (struct mm_struct *mm, unsigned long addr, pte_t *p ({ pte_t __pte; pte_val(__pte) = pfn | pgprot_val(pgprot); __pte; }) #define pte_page(pte) virt_to_page(__pte_page(pte)) -#define pmd_page(pmd) virt_to_page(__pmd_page(pmd)) +#define pmd_page(pmd) virt_to_page(pmd_page_vaddr(pmd)) static inline int pmd_none2 (pmd_t *pmd) { return !pmd_val (*pmd); } @@ -171,21 +174,6 @@ static inline pte_t pte_mkcache(pte_t pte) { return pte; } extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; extern pgd_t kernel_pg_dir[PTRS_PER_PGD]; -/* Find an entry in a pagetable directory. */ -#define pgd_index(address) ((address) >> PGDIR_SHIFT) - -#define pgd_offset(mm, address) \ -((mm)->pgd + pgd_index(address)) - -/* Find an entry in a kernel pagetable directory. */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -/* Find an entry in the third-level pagetable. */ -#define pte_index(address) ((address >> PAGE_SHIFT) & (PTRS_PER_PTE-1)) -#define pte_offset_kernel(pmd, address) ((pte_t *) __pmd_page(*pmd) + pte_index(address)) -#define pte_offset_map(pmd, address) ((pte_t *)page_address(pmd_page(*pmd)) + pte_index(address)) -#define pte_unmap(pte) do { } while (0) - /* Macros to (de)construct the fake PTEs representing swap pages. */ #define __swp_type(x) ((x).val & 0x7F) #define __swp_offset(x) (((x).val) >> 7) diff --git a/arch/m68k/mm/init.c b/arch/m68k/mm/init.c index 6d3147662ff2..53040857a9ed 100644 --- a/arch/m68k/mm/init.c +++ b/arch/m68k/mm/init.c @@ -141,7 +141,7 @@ static inline void init_pointer_tables(void) if (!pmd_present(*pmd)) continue; - pte_dir = (pte_t *)__pmd_page(*pmd); + pte_dir = (pte_t *)pmd_page_vaddr(*pmd); init_pointer_table(pte_dir, TABLE_PTE); } } diff --git a/arch/microblaze/include/asm/pgtable.h b/arch/microblaze/include/asm/pgtable.h index ee2115abf405..3fa1df90925e 100644 --- a/arch/microblaze/include/asm/pgtable.h +++ b/arch/microblaze/include/asm/pgtable.h @@ -21,7 +21,6 @@ extern int mem_init_done; #define pgd_bad(pgd) (0) #define pgd_clear(pgdp) #define kern_addr_valid(addr) (1) -#define pmd_offset(a, b) ((void *) 0) #define PAGE_NONE __pgprot(0) /* these mean nothing to non MMU */ #define PAGE_SHARED __pgprot(0) /* these mean nothing to non MMU */ @@ -438,27 +437,15 @@ static inline void ptep_mkdirty(struct mm_struct *mm, /* Convert pmd entry to page */ /* our pmd entry is an effective address of pte table*/ /* returns effective address of the pmd entry*/ -#define pmd_page_kernel(pmd) ((unsigned long) (pmd_val(pmd) & PAGE_MASK)) +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return ((unsigned long) (pmd_val(pmd) & PAGE_MASK)); +} /* returns struct *page of the pmd entry*/ #define pmd_page(pmd) (pfn_to_page(__pa(pmd_val(pmd)) >> PAGE_SHIFT)) -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -/* to find an entry in a page-table-directory */ -#define pgd_index(address) ((address) >> PGDIR_SHIFT) -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) - /* Find an entry in the third-level page table.. */ -#define pte_index(address) \ - (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir, addr) \ - ((pte_t *) pmd_page_kernel(*(dir)) + pte_index(addr)) -#define pte_offset_map(dir, addr) \ - ((pte_t *) kmap_atomic(pmd_page(*(dir))) + pte_index(addr)) - -#define pte_unmap(pte) kunmap_atomic(pte) extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; diff --git a/arch/mips/include/asm/pgtable-32.h b/arch/mips/include/asm/pgtable-32.h index 1945c8970141..a950fc1ddb4d 100644 --- a/arch/mips/include/asm/pgtable-32.h +++ b/arch/mips/include/asm/pgtable-32.h @@ -195,28 +195,6 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) #define pte_page(x) pfn_to_page(pte_pfn(x)) -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) -#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) -#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) - -/* to find an entry in a page-table-directory */ -#define pgd_offset(mm, addr) ((mm)->pgd + pgd_index(addr)) - -/* Find an entry in the third-level page table.. */ -#define __pte_offset(address) \ - (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset(dir, address) \ - ((pte_t *) pmd_page_vaddr(*(dir)) + __pte_offset(address)) -#define pte_offset_kernel(dir, address) \ - ((pte_t *) pmd_page_vaddr(*(dir)) + __pte_offset(address)) - -#define pte_offset_map(dir, address) \ - ((pte_t *)page_address(pmd_page(*(dir))) + __pte_offset(address)) -#define pte_unmap(pte) ((void)(pte)) - #if defined(CONFIG_CPU_R3K_TLB) /* Swap entries must have VALID bit cleared. */ diff --git a/arch/mips/include/asm/pgtable-64.h b/arch/mips/include/asm/pgtable-64.h index ee5dc0c145b9..1e7d6ce9d8d6 100644 --- a/arch/mips/include/asm/pgtable-64.h +++ b/arch/mips/include/asm/pgtable-64.h @@ -172,8 +172,6 @@ extern pte_t invalid_pte_table[PTRS_PER_PTE]; -#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) - #ifndef __PAGETABLE_PUD_FOLDED /* * For 4-level pagetables we defines these ourselves, for 3-level the @@ -222,11 +220,6 @@ static inline unsigned long p4d_page_vaddr(p4d_t p4d) #define p4d_index(address) (((address) >> P4D_SHIFT) & (PTRS_PER_P4D - 1)) -static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) -{ - return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address); -} - static inline void set_p4d(p4d_t *p4d, p4d_t p4dval) { *p4d = p4dval; @@ -320,15 +313,6 @@ static inline void pud_clear(pud_t *pudp) #define pfn_pmd(pfn, prot) __pmd(((pfn) << _PFN_SHIFT) | pgprot_val(prot)) #endif -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) -#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) - -/* to find an entry in a page-table-directory */ -#define pgd_offset(mm, addr) ((mm)->pgd + pgd_index(addr)) - #ifndef __PAGETABLE_PMD_FOLDED static inline unsigned long pud_page_vaddr(pud_t pud) { @@ -337,24 +321,8 @@ static inline unsigned long pud_page_vaddr(pud_t pud) #define pud_phys(pud) virt_to_phys((void *)pud_val(pud)) #define pud_page(pud) (pfn_to_page(pud_phys(pud) >> PAGE_SHIFT)) -/* Find an entry in the second-level page table.. */ -static inline pmd_t *pmd_offset(pud_t * pud, unsigned long address) -{ - return (pmd_t *) pud_page_vaddr(*pud) + pmd_index(address); -} #endif -/* Find an entry in the third-level page table.. */ -#define __pte_offset(address) \ - (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset(dir, address) \ - ((pte_t *) pmd_page_vaddr(*(dir)) + __pte_offset(address)) -#define pte_offset_kernel(dir, address) \ - ((pte_t *) pmd_page_vaddr(*(dir)) + __pte_offset(address)) -#define pte_offset_map(dir, address) \ - ((pte_t *)page_address(pmd_page(*(dir))) + __pte_offset(address)) -#define pte_unmap(pte) ((void)(pte)) - /* * Initialize a new pgd / pmd table with invalid pointers. */ diff --git a/arch/mips/kvm/mmu.c b/arch/mips/kvm/mmu.c index 7dad7a293eae..49bd160f4d85 100644 --- a/arch/mips/kvm/mmu.c +++ b/arch/mips/kvm/mmu.c @@ -168,7 +168,7 @@ static pte_t *kvm_mips_walk_pgd(pgd_t *pgd, struct kvm_mmu_memory_cache *cache, clear_page(new_pte); pmd_populate_kernel(NULL, pmd, new_pte); } - return pte_offset(pmd, addr); + return pte_offset_kernel(pmd, addr); } /* Caller must hold kvm->mm_lock */ @@ -187,8 +187,8 @@ static pte_t *kvm_mips_pte_for_gpa(struct kvm *kvm, static bool kvm_mips_flush_gpa_pte(pte_t *pte, unsigned long start_gpa, unsigned long end_gpa) { - int i_min = __pte_offset(start_gpa); - int i_max = __pte_offset(end_gpa); + int i_min = pte_index(start_gpa); + int i_max = pte_index(end_gpa); bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PTE - 1); int i; @@ -215,7 +215,7 @@ static bool kvm_mips_flush_gpa_pmd(pmd_t *pmd, unsigned long start_gpa, if (!pmd_present(pmd[i])) continue; - pte = pte_offset(pmd + i, 0); + pte = pte_offset_kernel(pmd + i, 0); if (i == i_max) end = end_gpa; @@ -312,8 +312,8 @@ static int kvm_mips_##name##_pte(pte_t *pte, unsigned long start, \ unsigned long end) \ { \ int ret = 0; \ - int i_min = __pte_offset(start); \ - int i_max = __pte_offset(end); \ + int i_min = pte_index(start); \ + int i_max = pte_index(end); \ int i; \ pte_t old, new; \ \ @@ -346,7 +346,7 @@ static int kvm_mips_##name##_pmd(pmd_t *pmd, unsigned long start, \ if (!pmd_present(pmd[i])) \ continue; \ \ - pte = pte_offset(pmd + i, 0); \ + pte = pte_offset_kernel(pmd + i, 0); \ if (i == i_max) \ cur_end = end; \ \ @@ -842,8 +842,8 @@ void kvm_trap_emul_invalidate_gva(struct kvm_vcpu *vcpu, unsigned long addr, static bool kvm_mips_flush_gva_pte(pte_t *pte, unsigned long start_gva, unsigned long end_gva) { - int i_min = __pte_offset(start_gva); - int i_max = __pte_offset(end_gva); + int i_min = pte_index(start_gva); + int i_max = pte_index(end_gva); bool safe_to_remove = (i_min == 0 && i_max == PTRS_PER_PTE - 1); int i; @@ -877,7 +877,7 @@ static bool kvm_mips_flush_gva_pmd(pmd_t *pmd, unsigned long start_gva, if (!pmd_present(pmd[i])) continue; - pte = pte_offset(pmd + i, 0); + pte = pte_offset_kernel(pmd + i, 0); if (i == i_max) end = end_gva; diff --git a/arch/mips/kvm/trap_emul.c b/arch/mips/kvm/trap_emul.c index 5a11e83dffe6..60763efa723e 100644 --- a/arch/mips/kvm/trap_emul.c +++ b/arch/mips/kvm/trap_emul.c @@ -594,7 +594,7 @@ static void kvm_mips_emul_free_gva_pt(pgd_t *pgd) pmd_va = pud_va | (k << PMD_SHIFT); if (pmd_va >= end) break; - pte = pte_offset(pmd + k, 0); + pte = pte_offset_kernel(pmd + k, 0); pte_free_kernel(NULL, pte); } pmd_free(NULL, pmd); diff --git a/arch/nds32/include/asm/pgtable.h b/arch/nds32/include/asm/pgtable.h index 95c75282cb47..419f984eef70 100644 --- a/arch/nds32/include/asm/pgtable.h +++ b/arch/nds32/include/asm/pgtable.h @@ -186,14 +186,10 @@ extern void paging_init(void); #define pte_clear(mm,addr,ptep) set_pte_at((mm),(addr),(ptep), __pte(0)) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) -#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir, address) ((pte_t *)pmd_page_kernel(*(dir)) + pte_index(address)) -#define pte_offset_map(dir, address) ((pte_t *)page_address(pmd_page(*(dir))) + pte_index(address)) -#define pte_offset_map_nested(dir, address) pte_offset_map(dir, address) -#define pmd_page_kernel(pmd) ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK)) - -#define pte_unmap(pte) do { } while (0) -#define pte_unmap_nested(pte) do { } while (0) +static unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK)); +} #define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) /* @@ -344,12 +340,6 @@ static inline pmd_t __mk_pmd(pte_t * ptep, unsigned long prot) * */ -/* to find an entry in a page-table-directory */ -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(addr) pgd_offset(&init_mm, addr) - static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { const unsigned long mask = 0xfff; diff --git a/arch/nios2/include/asm/pgtable.h b/arch/nios2/include/asm/pgtable.h index f0d4921d05ad..2600d76c310c 100644 --- a/arch/nios2/include/asm/pgtable.h +++ b/arch/nios2/include/asm/pgtable.h @@ -102,10 +102,6 @@ static inline void set_pmd(pmd_t *pmdptr, pmd_t pmdval) *pmdptr = pmdval; } -/* to find an entry in a page-table-directory */ -#define pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) -#define pgd_offset(mm, addr) ((mm)->pgd + pgd_index(addr)) - static inline int pte_write(pte_t pte) \ { return pte_val(pte) & _PAGE_WRITE; } static inline int pte_dirty(pte_t pte) \ @@ -236,27 +232,17 @@ static inline void pte_clear(struct mm_struct *mm, */ #define mk_pte(page, prot) (pfn_pte(page_to_pfn(page), prot)) -#define pte_unmap(pte) do { } while (0) - /* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. */ #define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd)) #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) -#define pmd_page_vaddr(pmd) pmd_val(pmd) - -#define pte_offset_map(dir, addr) \ - ((pte_t *) page_address(pmd_page(*dir)) + \ - (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))) -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(addr) pgd_offset(&init_mm, addr) - -/* Get the address to the PTE for a vaddr in specific directory */ -#define pte_offset_kernel(dir, addr) \ - ((pte_t *) pmd_page_vaddr(*(dir)) + \ - (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))) +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return pmd_val(pmd); +} #define pte_ERROR(e) \ pr_err("%s:%d: bad pte %08lx.\n", \ diff --git a/arch/openrisc/include/asm/pgtable.h b/arch/openrisc/include/asm/pgtable.h index f4273d387243..9425bedab4fc 100644 --- a/arch/openrisc/include/asm/pgtable.h +++ b/arch/openrisc/include/asm/pgtable.h @@ -363,38 +363,15 @@ static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) } #define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)) -#define pmd_page_kernel(pmd) ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK)) -/* to find an entry in a page-table-directory. */ -#define pgd_index(address) ((address >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) - -#define __pgd_offset(address) pgd_index(address) - -#define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address)) - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK)); +} #define __pmd_offset(address) \ (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) -/* - * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE] - * - * this macro returns the index of the entry in the pte page which would - * control the given virtual address - */ -#define __pte_offset(address) \ - (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir, address) \ - ((pte_t *) pmd_page_kernel(*(dir)) + __pte_offset(address)) -#define pte_offset_map(dir, address) \ - ((pte_t *)page_address(pmd_page(*(dir))) + __pte_offset(address)) -#define pte_offset_map_nested(dir, address) \ - pte_offset_map(dir, address) - -#define pte_unmap(pte) do { } while (0) -#define pte_unmap_nested(pte) do { } while (0) #define pte_pfn(x) ((unsigned long)(((x).pte)) >> PAGE_SHIFT) #define pfn_pte(pfn, prot) __pte((((pfn) << PAGE_SHIFT)) | pgprot_val(prot)) diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h index 0b98af5fbebe..75cf84070fc9 100644 --- a/arch/parisc/include/asm/pgtable.h +++ b/arch/parisc/include/asm/pgtable.h @@ -427,40 +427,16 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) -#define pmd_page_vaddr(pmd) ((unsigned long) __va(pmd_address(pmd))) +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return ((unsigned long) __va(pmd_address(pmd))); +} #define __pmd_page(pmd) ((unsigned long) __va(pmd_address(pmd))) #define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd)) -#define pgd_index(address) ((address) >> PGDIR_SHIFT) - -/* to find an entry in a page-table-directory */ -#define pgd_offset(mm, address) \ -((mm)->pgd + ((address) >> PGDIR_SHIFT)) - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - /* Find an entry in the second-level page table.. */ -#if CONFIG_PGTABLE_LEVELS == 3 -#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) -#define pmd_offset(dir,address) \ -((pmd_t *) pud_page_vaddr(*(dir)) + pmd_index(address)) -#else -#define pmd_offset(dir,addr) ((pmd_t *) dir) -#endif - -/* Find an entry in the third-level page table.. */ -#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1)) -#define pte_offset_kernel(pmd, address) \ - ((pte_t *) pmd_page_vaddr(*(pmd)) + pte_index(address)) -#define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address) -#define pte_unmap(pte) do { } while (0) - -#define pte_unmap(pte) do { } while (0) -#define pte_unmap_nested(pte) do { } while (0) - extern void paging_init (void); /* Used for deferring calls to flush_dcache_page() */ diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c index 0f1b460ee715..70cd24bdcfec 100644 --- a/arch/parisc/kernel/pci-dma.c +++ b/arch/parisc/kernel/pci-dma.c @@ -201,7 +201,7 @@ static inline void unmap_uncached_pmd(pgd_t * dir, unsigned long vaddr, pgd_clear(dir); return; } - pmd = pmd_offset(dir, vaddr); + pmd = pmd_offset(pud_offset(p4d_offset(dir, vaddr), vaddr), vaddr); vaddr &= ~PGDIR_MASK; end = vaddr + size; if (end > PGDIR_SIZE) diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index d7978a5a79c3..224912432821 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -112,6 +112,9 @@ static inline bool pte_user(pte_t pte) #define PMD_TABLE_SIZE 0 #define PUD_TABLE_SIZE 0 #define PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE) + +/* Bits to mask out from a PMD to get to the PTE page */ +#define PMD_MASKED_BITS (PTE_TABLE_SIZE - 1) #endif /* __ASSEMBLY__ */ #define PTRS_PER_PTE (1 << PTE_INDEX_SIZE) @@ -332,26 +335,9 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, #define __HAVE_ARCH_PTE_SAME #define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0) -#define pmd_page_vaddr(pmd) \ - ((unsigned long)__va(pmd_val(pmd) & ~(PTE_TABLE_SIZE - 1))) #define pmd_page(pmd) \ pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT) -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -/* to find an entry in a page-table-directory */ -#define pgd_index(address) ((address) >> PGDIR_SHIFT) -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) - -/* Find an entry in the third-level page table.. */ -#define pte_index(address) \ - (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir, addr) \ - ((pte_t *) pmd_page_vaddr(*(dir)) + pte_index(addr)) -#define pte_offset_map(dir, addr) pte_offset_kernel((dir), (addr)) -static inline void pte_unmap(pte_t *pte) { } - /* * Encode and decode a swap entry. * Note that the bits we use in a PTE for representing a swap entry diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index f17442c3a092..25c3cb8272c0 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -1005,52 +1005,9 @@ extern struct page *p4d_page(p4d_t p4d); /* Pointers in the page table tree are physical addresses */ #define __pgtable_ptr_val(ptr) __pa(ptr) -#define pmd_page_vaddr(pmd) __va(pmd_val(pmd) & ~PMD_MASKED_BITS) #define pud_page_vaddr(pud) __va(pud_val(pud) & ~PUD_MASKED_BITS) #define p4d_page_vaddr(p4d) __va(p4d_val(p4d) & ~P4D_MASKED_BITS) -static inline unsigned long pgd_index(unsigned long address) -{ - return (address >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1); -} - -static inline unsigned long pud_index(unsigned long address) -{ - return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1); -} - -static inline unsigned long pmd_index(unsigned long address) -{ - return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1); -} - -static inline unsigned long pte_index(unsigned long address) -{ - return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); -} - -/* - * Find an entry in a page-table-directory. We combine the address region - * (the high order N bits) and the pgd portion of the address. - */ - -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) - -#define pud_offset(p4dp, addr) \ - (((pud_t *) p4d_page_vaddr(*(p4dp))) + pud_index(addr)) -#define pmd_offset(pudp,addr) \ - (((pmd_t *) pud_page_vaddr(*(pudp))) + pmd_index(addr)) -#define pte_offset_kernel(dir,addr) \ - (((pte_t *) pmd_page_vaddr(*(dir))) + pte_index(addr)) - -#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr)) - -static inline void pte_unmap(pte_t *pte) { } - -/* to find an entry in a kernel page-table-directory */ -/* This now only contains the vmalloc pages */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - #define pte_ERROR(e) \ pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) #define pmd_ERROR(e) \ diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index af7f13cf90cf..b56f14160ae5 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -28,6 +28,8 @@ extern int icache_44x_need_flush; #define PMD_TABLE_SIZE 0 #define PUD_TABLE_SIZE 0 #define PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE) + +#define PMD_MASKED_BITS (PTE_TABLE_SIZE - 1) #endif /* __ASSEMBLY__ */ #define PTRS_PER_PTE (1 << PTE_INDEX_SIZE) @@ -203,10 +205,6 @@ static inline void pmd_clear(pmd_t *pmdp) *pmdp = __pmd(0); } - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - /* to find an entry in a page-table-directory */ #define pgd_index(address) ((address) >> PGDIR_SHIFT) #define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) @@ -330,8 +328,6 @@ static inline int pte_young(pte_t pte) * of the pte page. -- paulus */ #ifndef CONFIG_BOOKE -#define pmd_page_vaddr(pmd) \ - ((unsigned long)__va(pmd_val(pmd) & ~(PTE_TABLE_SIZE - 1))) #define pmd_page(pmd) \ pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT) #else @@ -341,15 +337,6 @@ static inline int pte_young(pte_t pte) pfn_to_page((__pa(pmd_val(pmd)) >> PAGE_SHIFT)) #endif -/* Find an entry in the third-level page table.. */ -#define pte_index(address) \ - (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir, addr) \ - (pmd_bad(*(dir)) ? NULL : (pte_t *)pmd_page_vaddr(*(dir)) + \ - pte_index(addr)) -#define pte_offset_map(dir, addr) pte_offset_kernel((dir), (addr)) -static inline void pte_unmap(pte_t *pte) { } - /* * Encode and decode a swap entry. * Note that the bits we use in a PTE for representing a swap entry diff --git a/arch/powerpc/include/asm/nohash/64/pgtable-4k.h b/arch/powerpc/include/asm/nohash/64/pgtable-4k.h index 81b1c54e3cf1..fe2f4c9acd9e 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable-4k.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable-4k.h @@ -78,10 +78,6 @@ extern struct page *p4d_page(p4d_t p4d); #endif /* !__ASSEMBLY__ */ -#define pud_offset(p4dp, addr) \ - (((pud_t *) p4d_page_vaddr(*(p4dp))) + \ - (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))) - #define pud_ERROR(e) \ pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index 3424381b81da..6cb8aa357191 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -182,28 +182,6 @@ static inline void p4d_set(p4d_t *p4dp, unsigned long val) *p4dp = __p4d(val); } -/* - * Find an entry in a page-table-directory. We combine the address region - * (the high order N bits) and the pgd portion of the address. - */ -#define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1)) - -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) - -#define pmd_offset(pudp,addr) \ - (((pmd_t *) pud_page_vaddr(*(pudp))) + (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))) - -#define pte_offset_kernel(dir,addr) \ - (((pte_t *) pmd_page_vaddr(*(dir))) + (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))) - -#define pte_offset_map(dir,addr) pte_offset_kernel((dir), (addr)) - -static inline void pte_unmap(pte_t *pte) { } - -/* to find an entry in a kernel page-table-directory */ -/* This now only contains the vmalloc pages */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - /* Atomic PTE updates */ static inline unsigned long pte_update(struct mm_struct *mm, unsigned long addr, diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index 3f50dc4c2d80..f7613f43c9cf 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -57,6 +57,13 @@ static inline pgprot_t pte_pgprot(pte_t pte) return __pgprot(pte_flags); } +#ifndef pmd_page_vaddr +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return ((unsigned long)__va(pmd_val(pmd) & ~PMD_MASKED_BITS)); +} +#define pmd_page_vaddr pmd_page_vaddr +#endif /* * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index b15f70a1fdfa..f3b0da64c6c8 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -70,13 +70,6 @@ static inline struct page *pud_page(pud_t pud) return pfn_to_page(pud_val(pud) >> _PAGE_PFN_SHIFT); } -#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) - -static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) -{ - return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr); -} - static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) { return __pmd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 8e3606fd580e..eaea1f717010 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -173,16 +173,6 @@ static inline unsigned long _pgd_pfn(pgd_t pgd) return pgd_val(pgd) >> _PAGE_PFN_SHIFT; } -#define pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) - -/* Locate an entry in the page global directory */ -static inline pgd_t *pgd_offset(const struct mm_struct *mm, unsigned long addr) -{ - return mm->pgd + pgd_index(addr); -} -/* Locate an entry in the kernel page global directory */ -#define pgd_offset_k(addr) pgd_offset(&init_mm, (addr)) - static inline struct page *pmd_page(pmd_t pmd) { return pfn_to_page(pmd_val(pmd) >> _PAGE_PFN_SHIFT); @@ -209,16 +199,6 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) #define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) -#define pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) - -static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long addr) -{ - return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(addr); -} - -#define pte_offset_map(dir, addr) pte_offset_kernel((dir), (addr)) -#define pte_unmap(pte) ((void)(pte)) - static inline int pte_present(pte_t pte) { return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)); diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 57797862f363..9996f49959bd 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -235,12 +235,12 @@ static void __init create_pte_mapping(pte_t *ptep, uintptr_t va, phys_addr_t pa, phys_addr_t sz, pgprot_t prot) { - uintptr_t pte_index = pte_index(va); + uintptr_t pte_idx = pte_index(va); BUG_ON(sz != PAGE_SIZE); - if (pte_none(ptep[pte_index])) - ptep[pte_index] = pfn_pte(PFN_DOWN(pa), prot); + if (pte_none(ptep[pte_idx])) + ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot); } #ifndef __PAGETABLE_PMD_FOLDED @@ -283,21 +283,21 @@ static void __init create_pmd_mapping(pmd_t *pmdp, { pte_t *ptep; phys_addr_t pte_phys; - uintptr_t pmd_index = pmd_index(va); + uintptr_t pmd_idx = pmd_index(va); if (sz == PMD_SIZE) { - if (pmd_none(pmdp[pmd_index])) - pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pa), prot); + if (pmd_none(pmdp[pmd_idx])) + pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot); return; } - if (pmd_none(pmdp[pmd_index])) { + if (pmd_none(pmdp[pmd_idx])) { pte_phys = alloc_pte(va); - pmdp[pmd_index] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); + pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); ptep = get_pte_virt(pte_phys); memset(ptep, 0, PAGE_SIZE); } else { - pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_index])); + pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx])); ptep = get_pte_virt(pte_phys); } @@ -325,21 +325,21 @@ static void __init create_pgd_mapping(pgd_t *pgdp, { pgd_next_t *nextp; phys_addr_t next_phys; - uintptr_t pgd_index = pgd_index(va); + uintptr_t pgd_idx = pgd_index(va); if (sz == PGDIR_SIZE) { - if (pgd_val(pgdp[pgd_index]) == 0) - pgdp[pgd_index] = pfn_pgd(PFN_DOWN(pa), prot); + if (pgd_val(pgdp[pgd_idx]) == 0) + pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot); return; } - if (pgd_val(pgdp[pgd_index]) == 0) { + if (pgd_val(pgdp[pgd_idx]) == 0) { next_phys = alloc_pgd_next(va); - pgdp[pgd_index] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); + pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); nextp = get_pgd_next_virt(next_phys); memset(nextp, 0, PAGE_SIZE); } else { - next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_index])); + next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx])); nextp = get_pgd_next_virt(next_phys); } diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 22757203828d..19d603bd1f36 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1229,7 +1229,6 @@ static inline pte_t mk_pte(struct page *page, pgprot_t pgprot) #define p4d_index(address) (((address) >> P4D_SHIFT) & (PTRS_PER_P4D-1)) #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) -#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1)) #define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN) #define pud_deref(pud) (pud_val(pud) & _REGION_ENTRY_ORIGIN) @@ -1260,7 +1259,6 @@ static inline pgd_t *pgd_offset_raw(pgd_t *pgd, unsigned long address) } #define pgd_offset(mm, address) pgd_offset_raw(READ_ONCE((mm)->pgd), address) -#define pgd_offset_k(address) pgd_offset(&init_mm, address) static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) { @@ -1275,6 +1273,7 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) return (pud_t *) p4d_deref(*p4d) + pud_index(address); return (pud_t *) p4d; } +#define pud_offset pud_offset static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) { @@ -1282,17 +1281,13 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) return (pmd_t *) pud_deref(*pud) + pmd_index(address); return (pmd_t *) pud; } +#define pmd_offset pmd_offset -static inline pte_t *pte_offset(pmd_t *pmd, unsigned long address) +static inline unsigned long pmd_page_vaddr(pmd_t pmd) { - return (pte_t *) pmd_deref(*pmd) + pte_index(address); + return (unsigned long) pmd_deref(pmd); } -#define pte_offset_kernel(pmd, address) pte_offset(pmd, address) -#define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address) - -static inline void pte_unmap(pte_t *pte) { } - static inline bool gup_fast_permitted(unsigned long start, unsigned long end) { return end <= current->mm->context.asce_limit; diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 397944b9906b..c5c52ec2b46f 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -85,7 +85,7 @@ static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, { pte_t *ptep, new; - ptep = pte_offset(pmdp, addr); + ptep = pte_offset_kernel(pmdp, addr); do { new = *ptep; if (pte_none(new)) diff --git a/arch/sh/include/asm/pgtable-3level.h b/arch/sh/include/asm/pgtable-3level.h index 0f80097e5c9c..82d74472dfcd 100644 --- a/arch/sh/include/asm/pgtable-3level.h +++ b/arch/sh/include/asm/pgtable-3level.h @@ -39,13 +39,6 @@ static inline unsigned long pud_page_vaddr(pud_t pud) /* only used by the stubbed out hugetlb gup code, should never be called */ #define pud_page(pud) NULL - -#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) -static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) -{ - return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address); -} - #define pud_none(x) (!pud_val(x)) #define pud_present(x) (pud_val(x)) #define pud_clear(xp) do { set_pud(xp, __pud(0)); } while (0) diff --git a/arch/sh/include/asm/pgtable_32.h b/arch/sh/include/asm/pgtable_32.h index 4acce5f2cbf9..41be43e99cff 100644 --- a/arch/sh/include/asm/pgtable_32.h +++ b/arch/sh/include/asm/pgtable_32.h @@ -401,27 +401,12 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) return pte; } -#define pmd_page_vaddr(pmd) ((unsigned long)pmd_val(pmd)) -#define pmd_page(pmd) (virt_to_page(pmd_val(pmd))) - -/* to find an entry in a page-table-directory. */ -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) -#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) - -/* Find an entry in the third-level page table.. */ -#define pte_index(address) ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define __pte_offset(address) pte_index(address) +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return (unsigned long)pmd_val(pmd); +} -#define pte_offset_kernel(dir, address) \ - ((pte_t *) pmd_page_vaddr(*(dir)) + pte_index(address)) -#define pte_offset_map(dir, address) pte_offset_kernel(dir, address) -#define pte_unmap(pte) do { } while (0) +#define pmd_page(pmd) (virt_to_page(pmd_val(pmd))) #ifdef CONFIG_X2TLB #define pte_ERROR(e) \ diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h index 264e76ceccf6..a8dafc550985 100644 --- a/arch/sparc/include/asm/pgalloc_64.h +++ b/arch/sparc/include/asm/pgalloc_64.h @@ -67,7 +67,7 @@ void pte_free(struct mm_struct *mm, pgtable_t ptepage); #define pmd_populate_kernel(MM, PMD, PTE) pmd_set(MM, PMD, PTE) #define pmd_populate(MM, PMD, PTE) pmd_set(MM, PMD, PTE) -#define pmd_pgtable(PMD) ((pte_t *)__pmd_page(PMD)) +#define pmd_pgtable(PMD) ((pte_t *)pmd_page_vaddr(PMD)) void pgtable_free(void *table, bool is_page); diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h index 484e3ce75881..632cdb959542 100644 --- a/arch/sparc/include/asm/pgtable_32.h +++ b/arch/sparc/include/asm/pgtable_32.h @@ -146,6 +146,12 @@ static inline unsigned long __pmd_page(pmd_t pmd) return (unsigned long)__nocache_va(v << 4); } +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + unsigned long v = pmd_val(pmd) & SRMMU_PTD_PMASK; + return (unsigned long)__nocache_va(v << 4); +} + static inline unsigned long pud_page_vaddr(pud_t pud) { if (srmmu_device_memory(pud_val(pud))) { @@ -315,30 +321,6 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) pgprot_val(newprot)); } -#define pgd_index(address) ((address) >> PGDIR_SHIFT) - -/* to find an entry in a page-table-directory */ -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -/* Find an entry in the second-level page table.. */ -static inline pmd_t *pmd_offset(pud_t * dir, unsigned long address) -{ - return (pmd_t *) pud_page_vaddr(*dir) + - ((address >> PMD_SHIFT) & (PTRS_PER_PMD - 1)); -} - -/* Find an entry in the third-level page table.. */ -pte_t *pte_offset_kernel(pmd_t * dir, unsigned long address); - -/* - * This shortcut works on sun4m (and sun4d) because the nocache area is static. - */ -#define pte_offset_map(d, a) pte_offset_kernel(d,a) -#define pte_unmap(pte) do{}while(0) - struct seq_file; void mmu_info(struct seq_file *m); @@ -427,7 +409,7 @@ static inline int io_remap_pfn_range(struct vm_area_struct *vma, return remap_pfn_range(vma, from, phys_base >> PAGE_SHIFT, size, prot); } -#define io_remap_pfn_range io_remap_pfn_range +#define io_remap_pfn_range io_remap_pfn_range #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS #define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \ diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index 6754363b261d..7ef6affa105e 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -835,7 +835,7 @@ static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) #define pud_set(pudp, pmdp) \ (pud_val(*(pudp)) = (__pa((unsigned long) (pmdp)))) -static inline unsigned long __pmd_page(pmd_t pmd) +static inline unsigned long pmd_page_vaddr(pmd_t pmd) { pte_t pte = __pte(pmd_val(pmd)); unsigned long pfn; @@ -855,7 +855,7 @@ static inline unsigned long pud_page_vaddr(pud_t pud) return ((unsigned long) __va(pfn << PAGE_SHIFT)); } -#define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd)) +#define pmd_page(pmd) virt_to_page((void *)pmd_page_vaddr(pmd)) #define pud_page(pud) virt_to_page((void *)pud_page_vaddr(pud)) #define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL) #define pud_present(pud) (pud_val(pud) != 0U) @@ -889,31 +889,6 @@ static inline unsigned long pud_pfn(pud_t pud) #define p4d_set(p4dp, pudp) \ (p4d_val(*(p4dp)) = (__pa((unsigned long) (pudp)))) -/* to find an entry in a page-table-directory. */ -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) -#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -/* Find an entry in the third-level page table.. */ -#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) -#define pud_offset(p4dp, address) \ - ((pud_t *) p4d_page_vaddr(*(p4dp)) + pud_index(address)) - -/* Find an entry in the second-level page table.. */ -#define pmd_offset(pudp, address) \ - ((pmd_t *) pud_page_vaddr(*(pudp)) + \ - (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))) - -/* Find an entry in the third-level page table.. */ -#define pte_index(address) \ - ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir, address) \ - ((pte_t *) __pmd_page(*(dir)) + pte_index(address)) -#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) -#define pte_unmap(pte) do { } while (0) - /* We cannot include at this point yet: */ extern struct mm_struct init_mm; @@ -1078,7 +1053,7 @@ static inline int io_remap_pfn_range(struct vm_area_struct *vma, return remap_pfn_range(vma, from, phys_base >> PAGE_SHIFT, size, prot); } -#define io_remap_pfn_range io_remap_pfn_range +#define io_remap_pfn_range io_remap_pfn_range static inline unsigned long __untagged_addr(unsigned long start) { diff --git a/arch/um/include/asm/pgtable-3level.h b/arch/um/include/asm/pgtable-3level.h index 8a3b689e0f86..36f452957cef 100644 --- a/arch/um/include/asm/pgtable-3level.h +++ b/arch/um/include/asm/pgtable-3level.h @@ -89,10 +89,6 @@ static inline void pud_clear (pud_t *pud) #define pud_page(pud) phys_to_page(pud_val(pud) & PAGE_MASK) #define pud_page_vaddr(pud) ((unsigned long) __va(pud_val(pud) & PAGE_MASK)) -/* Find an entry in the second-level page table.. */ -#define pmd_offset(pud, address) ((pmd_t *) pud_page_vaddr(*(pud)) + \ - pmd_index(address)) - static inline unsigned long pte_pfn(pte_t pte) { return phys_to_pfn(pte_val(pte)); diff --git a/arch/um/include/asm/pgtable.h b/arch/um/include/asm/pgtable.h index 9618a0b8088c..def376194dce 100644 --- a/arch/um/include/asm/pgtable.h +++ b/arch/um/include/asm/pgtable.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* +/* * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) * Copyright 2003 PathScale, Inc. * Derived from include/asm-i386/pgtable.h @@ -131,7 +131,7 @@ static inline int pte_none(pte_t pte) * Undefined behaviour if not.. */ static inline int pte_read(pte_t pte) -{ +{ return((pte_get_bits(pte, _PAGE_USER)) && !(pte_get_bits(pte, _PAGE_PROTNONE))); } @@ -163,7 +163,7 @@ static inline int pte_newpage(pte_t pte) } static inline int pte_newprot(pte_t pte) -{ +{ return(pte_present(pte) && (pte_get_bits(pte, _PAGE_NEWPROT))); } @@ -185,31 +185,31 @@ static inline pte_t pte_mkclean(pte_t pte) return(pte); } -static inline pte_t pte_mkold(pte_t pte) -{ +static inline pte_t pte_mkold(pte_t pte) +{ pte_clear_bits(pte, _PAGE_ACCESSED); return(pte); } static inline pte_t pte_wrprotect(pte_t pte) -{ +{ if (likely(pte_get_bits(pte, _PAGE_RW))) pte_clear_bits(pte, _PAGE_RW); else return pte; - return(pte_mknewprot(pte)); + return(pte_mknewprot(pte)); } static inline pte_t pte_mkread(pte_t pte) -{ +{ if (unlikely(pte_get_bits(pte, _PAGE_USER))) return pte; pte_set_bits(pte, _PAGE_USER); - return(pte_mknewprot(pte)); + return(pte_mknewprot(pte)); } static inline pte_t pte_mkdirty(pte_t pte) -{ +{ pte_set_bits(pte, _PAGE_DIRTY); return(pte); } @@ -220,20 +220,20 @@ static inline pte_t pte_mkyoung(pte_t pte) return(pte); } -static inline pte_t pte_mkwrite(pte_t pte) +static inline pte_t pte_mkwrite(pte_t pte) { if (unlikely(pte_get_bits(pte, _PAGE_RW))) return pte; pte_set_bits(pte, _PAGE_RW); - return(pte_mknewprot(pte)); + return(pte_mknewprot(pte)); } -static inline pte_t pte_mkuptodate(pte_t pte) +static inline pte_t pte_mkuptodate(pte_t pte) { pte_clear_bits(pte, _PAGE_NEWPAGE); if(pte_present(pte)) pte_clear_bits(pte, _PAGE_NEWPROT); - return(pte); + return(pte); } static inline pte_t pte_mknewpage(pte_t pte) @@ -288,29 +288,9 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b) static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { pte_set_val(pte, (pte_val(pte) & _PAGE_CHG_MASK), newprot); - return pte; + return pte; } -/* - * the pgd page can be thought of an array like this: pgd_t[PTRS_PER_PGD] - * - * this macro returns the index of the entry in the pgd page which would - * control the given virtual address - */ -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) - -/* - * pgd_offset() returns a (pgd_t *) - * pgd_index() is used get the offset into the pgd page's array of pgd_t's; - */ -#define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address)) - -/* - * a shortcut which implies the use of the kernel's pgd, instead - * of a process's - */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - /* * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD] * @@ -318,23 +298,6 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) * control the given virtual address */ #define pmd_page_vaddr(pmd) ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK)) -#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) - -#define pmd_page_vaddr(pmd) \ - ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK)) - -/* - * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE] - * - * this macro returns the index of the entry in the pte page which would - * control the given virtual address - */ -#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir, address) \ - ((pte_t *) pmd_page_vaddr(*(dir)) + pte_index(address)) -#define pte_offset_map(dir, address) \ - ((pte_t *)page_address(pmd_page(*(dir))) + pte_index(address)) -#define pte_unmap(pte) do { } while (0) struct mm_struct; extern pte_t *virt_to_pte(struct mm_struct *mm, unsigned long addr); diff --git a/arch/unicore32/include/asm/pgtable.h b/arch/unicore32/include/asm/pgtable.h index 9559e936d980..97f564c8ecba 100644 --- a/arch/unicore32/include/asm/pgtable.h +++ b/arch/unicore32/include/asm/pgtable.h @@ -153,12 +153,6 @@ extern struct page *empty_zero_page; #define pte_none(pte) (!pte_val(pte)) #define pte_clear(mm, addr, ptep) set_pte(ptep, __pte(0)) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) -#define pte_offset_kernel(dir, addr) (pmd_page_vaddr(*(dir)) \ - + __pte_index(addr)) - -#define pte_offset_map(dir, addr) (pmd_page_vaddr(*(dir)) \ - + __pte_index(addr)) -#define pte_unmap(pte) do { } while (0) #define set_pte(ptep, pte) cpu_set_pte(ptep, pte) @@ -221,17 +215,6 @@ PTE_BIT_FUNC(mkyoung, |= PTE_YOUNG); */ #define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot) -/* to find an entry in a page-table-directory */ -#define pgd_index(addr) ((addr) >> PGDIR_SHIFT) - -#define pgd_offset(mm, addr) ((mm)->pgd+pgd_index(addr)) - -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(addr) pgd_offset(&init_mm, addr) - -/* Find an entry in the third-level page table.. */ -#define __pte_index(addr) (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) - static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { const unsigned long mask = PTE_EXEC | PTE_WRITE | PTE_READ; diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index e2b4d4c08fac..76aa21e8128d 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -836,17 +836,6 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) */ #define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd)) -/* - * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD] - * - * this macro returns the index of the entry in the pmd page which would - * control the given virtual address - */ -static inline unsigned long pmd_index(unsigned long address) -{ - return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1); -} - /* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. @@ -856,25 +845,6 @@ static inline unsigned long pmd_index(unsigned long address) */ #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) -/* - * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE] - * - * this function returns the index of the entry in the pte page which would - * control the given virtual address - * - * Also define macro so we can test if pte_index is defined for arch. - */ -#define pte_index pte_index -static inline unsigned long pte_index(unsigned long address) -{ - return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); -} - -static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) -{ - return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); -} - static inline int pmd_bad(pmd_t pmd) { return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; @@ -907,12 +877,6 @@ static inline unsigned long pud_page_vaddr(pud_t pud) */ #define pud_page(pud) pfn_to_page(pud_pfn(pud)) -/* Find an entry in the second-level page table.. */ -static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) -{ - return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address); -} - #define pud_leaf pud_large static inline int pud_large(pud_t pud) { @@ -932,11 +896,6 @@ static inline int pud_large(pud_t pud) } #endif /* CONFIG_PGTABLE_LEVELS > 2 */ -static inline unsigned long pud_index(unsigned long address) -{ - return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1); -} - #if CONFIG_PGTABLE_LEVELS > 3 static inline int p4d_none(p4d_t p4d) { @@ -959,12 +918,6 @@ static inline unsigned long p4d_page_vaddr(p4d_t p4d) */ #define p4d_page(p4d) pfn_to_page(p4d_pfn(p4d)) -/* Find an entry in the third-level page table.. */ -static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) -{ - return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address); -} - static inline int p4d_bad(p4d_t p4d) { unsigned long ignore_flags = _KERNPG_TABLE | _PAGE_USER; @@ -1037,30 +990,6 @@ static inline int pgd_none(pgd_t pgd) #endif /* __ASSEMBLY__ */ -/* - * the pgd page can be thought of an array like this: pgd_t[PTRS_PER_PGD] - * - * this macro returns the index of the entry in the pgd page which would - * control the given virtual address - */ -#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) - -/* - * pgd_offset() returns a (pgd_t *) - * pgd_index() is used get the offset into the pgd page's array of pgd_t's; - */ -#define pgd_offset_pgd(pgd, address) (pgd + pgd_index((address))) -/* - * a shortcut to get a pgd_t in a given mm - */ -#define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address)) -/* - * a shortcut which implies the use of the kernel's pgd, instead - * of a process's - */ -#define pgd_offset_k(address) pgd_offset(&init_mm, (address)) - - #define KERNEL_PGD_BOUNDARY pgd_index(PAGE_OFFSET) #define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY) diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h index 82dc0d8464fa..ef76a04b4daf 100644 --- a/arch/x86/include/asm/pgtable_32.h +++ b/arch/x86/include/asm/pgtable_32.h @@ -45,17 +45,6 @@ void sync_initial_page_table(void); # include #endif -#if defined(CONFIG_HIGHPTE) -#define pte_offset_map(dir, address) \ - ((pte_t *)kmap_atomic(pmd_page(*(dir))) + \ - pte_index((address))) -#define pte_unmap(pte) kunmap_atomic((pte)) -#else -#define pte_offset_map(dir, address) \ - ((pte_t *)page_address(pmd_page(*(dir))) + pte_index((address))) -#define pte_unmap(pte) do { } while (0) -#endif - /* Clear a kernel PTE and flush it from the TLB */ #define kpte_clear_flush(ptep, vaddr) \ do { \ diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h index 8d03ffd43794..1b68d24dc6a0 100644 --- a/arch/x86/include/asm/pgtable_64.h +++ b/arch/x86/include/asm/pgtable_64.h @@ -186,10 +186,6 @@ extern void sync_global_pgds(unsigned long start, unsigned long end); /* PTE - Level 1 access. */ -/* x86-64 always has all page tables mapped. */ -#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) -#define pte_unmap(pte) ((void)(pte))/* NOP */ - /* * Encode and de-code a swap entry * diff --git a/arch/xtensa/include/asm/pgtable.h b/arch/xtensa/include/asm/pgtable.h index c57bca3fe051..fa054a1772e1 100644 --- a/arch/xtensa/include/asm/pgtable.h +++ b/arch/xtensa/include/asm/pgtable.h @@ -267,7 +267,7 @@ static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_WRITABLE; } static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; } static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } -static inline pte_t pte_wrprotect(pte_t pte) +static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) &= ~(_PAGE_WRITABLE | _PAGE_HW_WRITE); return pte; } static inline pte_t pte_mkclean(pte_t pte) { pte_val(pte) &= ~(_PAGE_DIRTY | _PAGE_HW_WRITE); return pte; } @@ -359,22 +359,6 @@ ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) update_pte(ptep, pte_wrprotect(pte)); } -/* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(address) pgd_offset(&init_mm, address) - -/* to find an entry in a page-table-directory */ -#define pgd_offset(mm,address) ((mm)->pgd + pgd_index(address)) - -#define pgd_index(address) ((address) >> PGDIR_SHIFT) - -/* Find an entry in the third-level page table.. */ -#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) -#define pte_offset_kernel(dir,addr) \ - ((pte_t*) pmd_page_vaddr(*(dir)) + pte_index(addr)) -#define pte_offset_map(dir,addr) pte_offset_kernel((dir),(addr)) -#define pte_unmap(pte) do { } while (0) - - /* * Encode and decode a swap and file entry. */ diff --git a/include/asm-generic/pgtable-nopmd.h b/include/asm-generic/pgtable-nopmd.h index 0d9b28cba16d..3e13acd019ae 100644 --- a/include/asm-generic/pgtable-nopmd.h +++ b/include/asm-generic/pgtable-nopmd.h @@ -45,6 +45,7 @@ static inline pmd_t * pmd_offset(pud_t * pud, unsigned long address) { return (pmd_t *)pud; } +#define pmd_offset pmd_offset #define pmd_val(x) (pud_val((x).pud)) #define __pmd(x) ((pmd_t) { __pud(x) } ) diff --git a/include/asm-generic/pgtable-nopud.h b/include/asm-generic/pgtable-nopud.h index ad05c1684bfc..a9d751fbda9e 100644 --- a/include/asm-generic/pgtable-nopud.h +++ b/include/asm-generic/pgtable-nopud.h @@ -43,6 +43,7 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) { return (pud_t *)p4d; } +#define pud_offset pud_offset #define pud_val(x) (p4d_val((x).p4d)) #define __pud(x) ((pud_t) { __p4d(x) }) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index ee85b222d098..90a5d0c78fca 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -28,6 +28,97 @@ #define USER_PGTABLES_CEILING 0UL #endif +/* + * A page table page can be thought of an array like this: pXd_t[PTRS_PER_PxD] + * + * The pXx_index() functions return the index of the entry in the page + * table page which would control the given virtual address + * + * As these functions may be used by the same code for different levels of + * the page table folding, they are always available, regardless of + * CONFIG_PGTABLE_LEVELS value. For the folded levels they simply return 0 + * because in such cases PTRS_PER_PxD equals 1. + */ + +static inline unsigned long pte_index(unsigned long address) +{ + return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); +} + +#ifndef pmd_index +static inline unsigned long pmd_index(unsigned long address) +{ + return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1); +} +#define pmd_index pmd_index +#endif + +#ifndef pud_index +static inline unsigned long pud_index(unsigned long address) +{ + return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1); +} +#define pud_index pud_index +#endif + +#ifndef pgd_index +/* Must be a compile-time constant, so implement it as a macro */ +#define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) +#endif + +#ifndef pte_offset_kernel +static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) +{ + return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); +} +#define pte_offset_kernel pte_offset_kernel +#endif + +#if defined(CONFIG_HIGHPTE) +#define pte_offset_map(dir, address) \ + ((pte_t *)kmap_atomic(pmd_page(*(dir))) + \ + pte_index((address))) +#define pte_unmap(pte) kunmap_atomic((pte)) +#else +#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) +#define pte_unmap(pte) ((void)(pte)) /* NOP */ +#endif + +/* Find an entry in the second-level page table.. */ +#ifndef pmd_offset +static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) +{ + return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address); +} +#define pmd_offset pmd_offset +#endif + +#ifndef pud_offset +static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) +{ + return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address); +} +#define pud_offset pud_offset +#endif + +static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address) +{ + return (pgd + pgd_index(address)); +}; + +/* + * a shortcut to get a pgd_t in a given mm + */ +#ifndef pgd_offset +#define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address)) +#endif + +/* + * a shortcut which implies the use of the kernel's pgd, instead + * of a process's + */ +#define pgd_offset_k(address) pgd_offset(&init_mm, (address)) + /* * In many cases it is known that a virtual address is mapped at PMD or PTE * level, so instead of traversing all the page table levels, we can get a -- cgit v1.2.3-70-g09d2 From c1e8d7c6a7a682e1405e3e242d32fc377fd196ff Mon Sep 17 00:00:00 2001 From: Michel Lespinasse Date: Mon, 8 Jun 2020 21:33:54 -0700 Subject: mmap locking API: convert mmap_sem comments Convert comments that reference mmap_sem to reference mmap_lock instead. [akpm@linux-foundation.org: fix up linux-next leftovers] [akpm@linux-foundation.org: s/lockaphore/lock/, per Vlastimil] [akpm@linux-foundation.org: more linux-next fixups, per Michel] Signed-off-by: Michel Lespinasse Signed-off-by: Andrew Morton Reviewed-by: Vlastimil Babka Reviewed-by: Daniel Jordan Cc: Davidlohr Bueso Cc: David Rientjes Cc: Hugh Dickins Cc: Jason Gunthorpe Cc: Jerome Glisse Cc: John Hubbard Cc: Laurent Dufour Cc: Liam Howlett Cc: Matthew Wilcox Cc: Peter Zijlstra Cc: Ying Han Link: http://lkml.kernel.org/r/20200520052908.204642-13-walken@google.com Signed-off-by: Linus Torvalds --- .../admin-guide/mm/numa_memory_policy.rst | 10 +++--- Documentation/admin-guide/mm/userfaultfd.rst | 2 +- Documentation/filesystems/locking.rst | 2 +- Documentation/vm/transhuge.rst | 4 +-- arch/arc/mm/fault.c | 2 +- arch/arm/kernel/vdso.c | 2 +- arch/arm/mm/fault.c | 2 +- arch/ia64/mm/fault.c | 2 +- arch/microblaze/mm/fault.c | 2 +- arch/nds32/mm/fault.c | 2 +- arch/powerpc/include/asm/pkeys.h | 2 +- arch/powerpc/kvm/book3s_hv_uvmem.c | 6 ++-- arch/powerpc/mm/book3s32/tlb.c | 2 +- arch/powerpc/mm/book3s64/hash_pgtable.c | 4 +-- arch/powerpc/mm/book3s64/subpage_prot.c | 2 +- arch/powerpc/mm/fault.c | 10 +++--- arch/powerpc/mm/pgtable.c | 2 +- arch/powerpc/platforms/cell/spufs/file.c | 6 ++-- arch/riscv/mm/fault.c | 2 +- arch/s390/kvm/priv.c | 2 +- arch/s390/mm/fault.c | 2 +- arch/s390/mm/gmap.c | 32 ++++++++--------- arch/s390/mm/pgalloc.c | 2 +- arch/sh/mm/cache-sh4.c | 2 +- arch/sh/mm/fault.c | 2 +- arch/sparc/mm/fault_64.c | 2 +- arch/um/kernel/skas/mmu.c | 2 +- arch/um/kernel/tlb.c | 2 +- arch/unicore32/mm/fault.c | 2 +- arch/x86/events/core.c | 2 +- arch/x86/include/asm/mmu.h | 2 +- arch/x86/include/asm/pgtable-3level.h | 8 ++--- arch/x86/kernel/cpu/resctrl/pseudo_lock.c | 6 ++-- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 6 ++-- arch/x86/kernel/ldt.c | 2 +- arch/x86/mm/fault.c | 12 +++---- drivers/char/mspec.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c | 2 +- drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 6 ++-- drivers/gpu/drm/i915/i915_perf.c | 2 +- drivers/gpu/drm/ttm/ttm_bo_vm.c | 6 ++-- drivers/infiniband/core/uverbs_main.c | 2 +- drivers/infiniband/hw/hfi1/mmu_rb.c | 2 +- drivers/media/v4l2-core/videobuf-dma-sg.c | 2 +- drivers/misc/cxl/cxllib.c | 5 ++- drivers/misc/sgi-gru/grufault.c | 8 ++--- drivers/oprofile/buffer_sync.c | 2 +- drivers/staging/android/ashmem.c | 4 +-- drivers/staging/comedi/comedi_fops.c | 2 +- drivers/tty/vt/consolemap.c | 2 +- drivers/vfio/pci/vfio_pci.c | 14 ++++---- drivers/xen/gntdev.c | 2 +- fs/coredump.c | 4 +-- fs/exec.c | 2 +- fs/ext2/file.c | 2 +- fs/ext4/super.c | 6 ++-- fs/kernfs/file.c | 4 +-- fs/proc/base.c | 6 ++-- fs/proc/task_mmu.c | 6 ++-- fs/userfaultfd.c | 18 +++++----- fs/xfs/xfs_file.c | 2 +- fs/xfs/xfs_inode.c | 14 ++++---- fs/xfs/xfs_iops.c | 4 +-- include/linux/fs.h | 4 +-- include/linux/huge_mm.h | 2 +- include/linux/mempolicy.h | 2 +- include/linux/mm.h | 10 +++--- include/linux/mm_types.h | 2 +- include/linux/mmu_notifier.h | 8 ++--- include/linux/pagemap.h | 2 +- include/linux/pgtable.h | 6 ++-- include/linux/rmap.h | 2 +- include/linux/sched/mm.h | 10 +++--- kernel/acct.c | 2 +- kernel/cgroup/cpuset.c | 4 +-- kernel/events/core.c | 6 ++-- kernel/events/uprobes.c | 4 +-- kernel/exit.c | 2 +- kernel/relay.c | 2 +- kernel/sys.c | 4 +-- lib/test_lockup.c | 8 ++--- mm/filemap.c | 38 ++++++++++---------- mm/frame_vector.c | 2 +- mm/gup.c | 38 ++++++++++---------- mm/huge_memory.c | 4 +-- mm/hugetlb.c | 2 +- mm/internal.h | 4 +-- mm/khugepaged.c | 34 +++++++++--------- mm/ksm.c | 12 +++---- mm/maccess.c | 4 +-- mm/madvise.c | 20 +++++------ mm/memcontrol.c | 2 +- mm/memory.c | 40 +++++++++++----------- mm/mempolicy.c | 12 +++---- mm/migrate.c | 4 +-- mm/mlock.c | 6 ++-- mm/mmap.c | 36 +++++++++---------- mm/mmu_gather.c | 2 +- mm/mmu_notifier.c | 10 +++--- mm/mprotect.c | 8 ++--- mm/mremap.c | 8 ++--- mm/nommu.c | 6 ++-- mm/oom_kill.c | 2 +- mm/pagewalk.c | 6 ++-- mm/rmap.c | 12 +++---- mm/shmem.c | 4 +-- mm/swap_state.c | 4 +-- mm/userfaultfd.c | 8 ++--- mm/util.c | 2 +- security/keys/keyctl.c | 2 +- sound/core/oss/pcm_oss.c | 2 +- 113 files changed, 351 insertions(+), 352 deletions(-) (limited to 'include/linux/pgtable.h') diff --git a/Documentation/admin-guide/mm/numa_memory_policy.rst b/Documentation/admin-guide/mm/numa_memory_policy.rst index 8463f5538fda..067a90a1499c 100644 --- a/Documentation/admin-guide/mm/numa_memory_policy.rst +++ b/Documentation/admin-guide/mm/numa_memory_policy.rst @@ -364,19 +364,19 @@ follows: 2) for querying the policy, we do not need to take an extra reference on the target task's task policy nor vma policies because we always acquire the - task's mm's mmap_sem for read during the query. The set_mempolicy() and - mbind() APIs [see below] always acquire the mmap_sem for write when + task's mm's mmap_lock for read during the query. The set_mempolicy() and + mbind() APIs [see below] always acquire the mmap_lock for write when installing or replacing task or vma policies. Thus, there is no possibility of a task or thread freeing a policy while another task or thread is querying it. 3) Page allocation usage of task or vma policy occurs in the fault path where - we hold them mmap_sem for read. Again, because replacing the task or vma - policy requires that the mmap_sem be held for write, the policy can't be + we hold them mmap_lock for read. Again, because replacing the task or vma + policy requires that the mmap_lock be held for write, the policy can't be freed out from under us while we're using it for page allocation. 4) Shared policies require special consideration. One task can replace a - shared memory policy while another task, with a distinct mmap_sem, is + shared memory policy while another task, with a distinct mmap_lock, is querying or allocating a page based on the policy. To resolve this potential race, the shared policy infrastructure adds an extra reference to the shared policy during lookup while holding a spin lock on the shared diff --git a/Documentation/admin-guide/mm/userfaultfd.rst b/Documentation/admin-guide/mm/userfaultfd.rst index 0bf49d7313ad..1dc2d5f823b4 100644 --- a/Documentation/admin-guide/mm/userfaultfd.rst +++ b/Documentation/admin-guide/mm/userfaultfd.rst @@ -33,7 +33,7 @@ memory ranges) provides two primary functionalities: The real advantage of userfaults if compared to regular virtual memory management of mremap/mprotect is that the userfaults in all their operations never involve heavyweight structures like vmas (in fact the -``userfaultfd`` runtime load never takes the mmap_sem for writing). +``userfaultfd`` runtime load never takes the mmap_lock for writing). Vmas are not suitable for page- (or hugepage) granular fault tracking when dealing with virtual address spaces that could span diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index 0af2e0e11461..eb71156bcb7c 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -615,7 +615,7 @@ prototypes:: locking rules: ============= ======== =========================== -ops mmap_sem PageLocked(page) +ops mmap_lock PageLocked(page) ============= ======== =========================== open: yes close: yes diff --git a/Documentation/vm/transhuge.rst b/Documentation/vm/transhuge.rst index 37c57ca32629..0ed23e59abe5 100644 --- a/Documentation/vm/transhuge.rst +++ b/Documentation/vm/transhuge.rst @@ -98,9 +98,9 @@ split_huge_page() or split_huge_pmd() has a cost. To make pagetable walks huge pmd aware, all you need to do is to call pmd_trans_huge() on the pmd returned by pmd_offset. You must hold the -mmap_sem in read (or write) mode to be sure a huge pmd cannot be +mmap_lock in read (or write) mode to be sure a huge pmd cannot be created from under you by khugepaged (khugepaged collapse_huge_page -takes the mmap_sem in write mode in addition to the anon_vma lock). If +takes the mmap_lock in write mode in addition to the anon_vma lock). If pmd_trans_huge returns false, you just fallback in the old code paths. If instead pmd_trans_huge returns true, you have to take the page table lock (pmd_lock()) and re-run pmd_trans_huge. Taking the diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c index 5b213bc0ae84..72f5405a7ec5 100644 --- a/arch/arc/mm/fault.c +++ b/arch/arc/mm/fault.c @@ -141,7 +141,7 @@ retry: } /* - * Fault retry nuances, mmap_sem already relinquished by core mm + * Fault retry nuances, mmap_lock already relinquished by core mm */ if (unlikely((fault & VM_FAULT_RETRY) && (flags & FAULT_FLAG_ALLOW_RETRY))) { diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c index e0330a25e1c6..6bfdca4769a7 100644 --- a/arch/arm/kernel/vdso.c +++ b/arch/arm/kernel/vdso.c @@ -240,7 +240,7 @@ static int install_vvar(struct mm_struct *mm, unsigned long addr) return PTR_ERR_OR_ZERO(vma); } -/* assumes mmap_sem is write-locked */ +/* assumes mmap_lock is write-locked */ void arm_install_vdso(struct mm_struct *mm, unsigned long addr) { struct vm_area_struct *vma; diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 8e56c43981b4..c6550eddfce1 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -293,7 +293,7 @@ retry: fault = __do_page_fault(mm, addr, fsr, flags, tsk); /* If we need to retry but a fatal signal is pending, handle the - * signal first. We do not need to release the mmap_sem because + * signal first. We do not need to release the mmap_lock because * it would already be released in __lock_page_or_retry in * mm/filemap.c. */ if (fault_signal_pending(fault, regs)) { diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index 6c09f43d9711..3a4dec334cc5 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -86,7 +86,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re #ifdef CONFIG_VIRTUAL_MEM_MAP /* * If fault is in region 5 and we are in the kernel, we may already - * have the mmap_sem (pfn_valid macro is called during mmap). There + * have the mmap_lock (pfn_valid macro is called during mmap). There * is no vma for region 5 addr's anyway, so skip getting the semaphore * and go directly to the exception handling code. */ diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c index 74358902a5db..a2bfe587b491 100644 --- a/arch/microblaze/mm/fault.c +++ b/arch/microblaze/mm/fault.c @@ -124,7 +124,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address, /* When running in the kernel we expect faults to occur only to * addresses in user space. All other faults represent errors in the * kernel and should generate an OOPS. Unfortunately, in the case of an - * erroneous fault occurring in a code path which already holds mmap_sem + * erroneous fault occurring in a code path which already holds mmap_lock * we will deadlock attempting to validate the fault against the * address space. Luckily the kernel only validly references user * space from well defined areas of code, which are listed in the diff --git a/arch/nds32/mm/fault.c b/arch/nds32/mm/fault.c index 89831b6e1ede..8fb73f6401a0 100644 --- a/arch/nds32/mm/fault.c +++ b/arch/nds32/mm/fault.c @@ -210,7 +210,7 @@ good_area: /* * If we need to retry but a fatal signal is pending, handle the - * signal first. We do not need to release the mmap_sem because it + * signal first. We do not need to release the mmap_lock because it * would already be released in __lock_page_or_retry in mm/filemap.c. */ if (fault_signal_pending(fault, regs)) { diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index 20ebf153c871..2fe6cae14d10 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -101,7 +101,7 @@ static inline bool mm_pkey_is_allocated(struct mm_struct *mm, int pkey) /* * Returns a positive, 5-bit key on success, or -1 on failure. - * Relies on the mmap_sem to protect against concurrency in mm_pkey_alloc() and + * Relies on the mmap_lock to protect against concurrency in mm_pkey_alloc() and * mm_pkey_free(). */ static inline int mm_pkey_alloc(struct mm_struct *mm) diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c index 305997b015b6..f91224ea034a 100644 --- a/arch/powerpc/kvm/book3s_hv_uvmem.c +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c @@ -47,7 +47,7 @@ * Locking order * * 1. kvm->srcu - Protects KVM memslots - * 2. kvm->mm->mmap_sem - find_vma, migrate_vma_pages and helpers, ksm_madvise + * 2. kvm->mm->mmap_lock - find_vma, migrate_vma_pages and helpers, ksm_madvise * 3. kvm->arch.uvmem_lock - protects read/writes to uvmem slots thus acting * as sync-points for page-in/out */ @@ -402,8 +402,8 @@ kvmppc_svm_page_in(struct vm_area_struct *vma, unsigned long start, mig.dst = &dst_pfn; /* - * We come here with mmap_sem write lock held just for - * ksm_madvise(), otherwise we only need read mmap_sem. + * We come here with mmap_lock write lock held just for + * ksm_madvise(), otherwise we only need read mmap_lock. * Hence downgrade to read lock once ksm_madvise() is done. */ ret = ksm_madvise(vma, vma->vm_start, vma->vm_end, diff --git a/arch/powerpc/mm/book3s32/tlb.c b/arch/powerpc/mm/book3s32/tlb.c index 157f98f6aea9..b6c7427daa6f 100644 --- a/arch/powerpc/mm/book3s32/tlb.c +++ b/arch/powerpc/mm/book3s32/tlb.c @@ -129,7 +129,7 @@ void flush_tlb_mm(struct mm_struct *mm) /* * It is safe to go down the mm's list of vmas when called - * from dup_mmap, holding mmap_sem. It would also be safe from + * from dup_mmap, holding mmap_lock. It would also be safe from * unmap_region or exit_mmap, but not from vmtruncate on SMP - * but it seems dup_mmap is the only SMP case which gets here. */ diff --git a/arch/powerpc/mm/book3s64/hash_pgtable.c b/arch/powerpc/mm/book3s64/hash_pgtable.c index 00af58cc8714..2a99167afbaf 100644 --- a/arch/powerpc/mm/book3s64/hash_pgtable.c +++ b/arch/powerpc/mm/book3s64/hash_pgtable.c @@ -237,7 +237,7 @@ pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long addres * to hugepage, we first clear the pmd, then invalidate all * the PTE entries. The assumption here is that any low level * page fault will see a none pmd and take the slow path that - * will wait on mmap_sem. But we could very well be in a + * will wait on mmap_lock. But we could very well be in a * hash_page with local ptep pointer value. Such a hash page * can result in adding new HPTE entries for normal subpages. * That means we could be modifying the page content as we @@ -251,7 +251,7 @@ pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long addres * Now invalidate the hpte entries in the range * covered by pmd. This make sure we take a * fault and will find the pmd as none, which will - * result in a major fault which takes mmap_sem and + * result in a major fault which takes mmap_lock and * hence wait for collapse to complete. Without this * the __collapse_huge_page_copy can result in copying * the old content. diff --git a/arch/powerpc/mm/book3s64/subpage_prot.c b/arch/powerpc/mm/book3s64/subpage_prot.c index e814d34bf7f4..60c6ea16a972 100644 --- a/arch/powerpc/mm/book3s64/subpage_prot.c +++ b/arch/powerpc/mm/book3s64/subpage_prot.c @@ -225,7 +225,7 @@ SYSCALL_DEFINE3(subpage_prot, unsigned long, addr, if (!spt) { /* * Allocate subpage prot table if not already done. - * Do this with mmap_sem held + * Do this with mmap_lock held */ spt = kzalloc(sizeof(struct subpage_prot_table), GFP_KERNEL); if (!spt) { diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index ff3653e67c7b..641fc5f3d7dd 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -138,7 +138,7 @@ static noinline int bad_access_pkey(struct pt_regs *regs, unsigned long address, * 2. T1 : set AMR to deny access to pkey=4, touches, page * 3. T1 : faults... * 4. T2: mprotect_key(foo, PAGE_SIZE, pkey=5); - * 5. T1 : enters fault handler, takes mmap_sem, etc... + * 5. T1 : enters fault handler, takes mmap_lock, etc... * 6. T1 : reaches here, sees vma_pkey(vma)=5, when we really * faulted on a pte with its pkey=4. */ @@ -525,9 +525,9 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); /* - * We want to do this outside mmap_sem, because reading code around nip + * We want to do this outside mmap_lock, because reading code around nip * can result in fault, which will cause a deadlock when called with - * mmap_sem held + * mmap_lock held */ if (is_user) flags |= FAULT_FLAG_USER; @@ -539,7 +539,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, /* When running in the kernel we expect faults to occur only to * addresses in user space. All other faults represent errors in the * kernel and should generate an OOPS. Unfortunately, in the case of an - * erroneous fault occurring in a code path which already holds mmap_sem + * erroneous fault occurring in a code path which already holds mmap_lock * we will deadlock attempting to validate the fault against the * address space. Luckily the kernel only validly references user * space from well defined areas of code, which are listed in the @@ -615,7 +615,7 @@ good_area: return user_mode(regs) ? 0 : SIGBUS; /* - * Handle the retry right now, the mmap_sem has been released in that + * Handle the retry right now, the mmap_lock has been released in that * case. */ if (unlikely(fault & VM_FAULT_RETRY)) { diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c index cea5b4e25a24..45a0556089e8 100644 --- a/arch/powerpc/mm/pgtable.c +++ b/arch/powerpc/mm/pgtable.c @@ -306,7 +306,7 @@ void assert_pte_locked(struct mm_struct *mm, unsigned long addr) pmd = pmd_offset(pud, addr); /* * khugepaged to collapse normal pages to hugepage, first set - * pmd to none to force page fault/gup to take mmap_sem. After + * pmd to none to force page fault/gup to take mmap_lock. After * pmd is set to none, we do a pte_clear which does this assertion * so if we find pmd none, return. */ diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index 324c0fb091a1..62d90a5e23d1 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -325,7 +325,7 @@ static vm_fault_t spufs_ps_fault(struct vm_fault *vmf, return VM_FAULT_SIGBUS; /* - * Because we release the mmap_sem, the context may be destroyed while + * Because we release the mmap_lock, the context may be destroyed while * we're in spu_wait. Grab an extra reference so it isn't destroyed * in the meantime. */ @@ -334,8 +334,8 @@ static vm_fault_t spufs_ps_fault(struct vm_fault *vmf, /* * We have to wait for context to be loaded before we have * pages to hand out to the user, but we don't want to wait - * with the mmap_sem held. - * It is possible to drop the mmap_sem here, but then we need + * with the mmap_lock held. + * It is possible to drop the mmap_lock here, but then we need * to return VM_FAULT_NOPAGE because the mappings may have * hanged. */ diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 996db5ebbf39..ae7b7fe24658 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -114,7 +114,7 @@ good_area: /* * If we need to retry but a fatal signal is pending, handle the - * signal first. We do not need to release the mmap_sem because it + * signal first. We do not need to release the mmap_lock because it * would already be released in __lock_page_or_retry in mm/filemap.c. */ if (fault_signal_pending(fault, regs)) diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c index 57e08b0a0a1a..96ae368aa0a2 100644 --- a/arch/s390/kvm/priv.c +++ b/arch/s390/kvm/priv.c @@ -1122,7 +1122,7 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) } /* - * Must be called with relevant read locks held (kvm->mm->mmap_sem, kvm->srcu) + * Must be called with relevant read locks held (kvm->mm->mmap_lock, kvm->srcu) */ static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc) { diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index f0fa51af8f5b..6a24751557f0 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -507,7 +507,7 @@ retry: if (IS_ENABLED(CONFIG_PGSTE) && gmap && (flags & FAULT_FLAG_RETRY_NOWAIT)) { /* FAULT_FLAG_RETRY_NOWAIT has been set, - * mmap_sem has not been released */ + * mmap_lock has not been released */ current->thread.gmap_pfault = 1; fault = VM_FAULT_PFAULT; goto out_up; diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index a8e818d10c6b..190357ff86b3 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -300,7 +300,7 @@ struct gmap *gmap_get_enabled(void) EXPORT_SYMBOL_GPL(gmap_get_enabled); /* - * gmap_alloc_table is assumed to be called with mmap_sem held + * gmap_alloc_table is assumed to be called with mmap_lock held */ static int gmap_alloc_table(struct gmap *gmap, unsigned long *table, unsigned long init, unsigned long gaddr) @@ -466,7 +466,7 @@ EXPORT_SYMBOL_GPL(gmap_map_segment); * Returns user space address which corresponds to the guest address or * -EFAULT if no such mapping exists. * This function does not establish potentially missing page table entries. - * The mmap_sem of the mm that belongs to the address space must be held + * The mmap_lock of the mm that belongs to the address space must be held * when this function gets called. * * Note: Can also be called for shadow gmaps. @@ -534,7 +534,7 @@ static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *old, pmd_t new, * * Returns 0 on success, -ENOMEM for out of memory conditions, and -EFAULT * if the vm address is already mapped to a different guest segment. - * The mmap_sem of the mm that belongs to the address space must be held + * The mmap_lock of the mm that belongs to the address space must be held * when this function gets called. */ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) @@ -655,7 +655,7 @@ retry: goto out_up; } /* - * In the case that fixup_user_fault unlocked the mmap_sem during + * In the case that fixup_user_fault unlocked the mmap_lock during * faultin redo __gmap_translate to not race with a map/unmap_segment. */ if (unlocked) @@ -669,7 +669,7 @@ out_up: EXPORT_SYMBOL_GPL(gmap_fault); /* - * this function is assumed to be called with mmap_sem held + * this function is assumed to be called with mmap_lock held */ void __gmap_zap(struct gmap *gmap, unsigned long gaddr) { @@ -882,7 +882,7 @@ static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr, if (fixup_user_fault(current, mm, vmaddr, fault_flags, &unlocked)) return -EFAULT; if (unlocked) - /* lost mmap_sem, caller has to retry __gmap_translate */ + /* lost mmap_lock, caller has to retry __gmap_translate */ return 0; /* Connect the page tables */ return __gmap_link(gmap, gaddr, vmaddr); @@ -953,7 +953,7 @@ static inline void gmap_pmd_op_end(struct gmap *gmap, pmd_t *pmdp) * -EAGAIN if a fixup is needed * -EINVAL if unsupported notifier bits have been specified * - * Expected to be called with sg->mm->mmap_sem in read and + * Expected to be called with sg->mm->mmap_lock in read and * guest_table_lock held. */ static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr, @@ -999,7 +999,7 @@ static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr, * Returns 0 if successfully protected, -ENOMEM if out of memory and * -EAGAIN if a fixup is needed. * - * Expected to be called with sg->mm->mmap_sem in read + * Expected to be called with sg->mm->mmap_lock in read */ static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr, pmd_t *pmdp, int prot, unsigned long bits) @@ -1035,7 +1035,7 @@ static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr, * Returns 0 if successfully protected, -ENOMEM if out of memory and * -EFAULT if gaddr is invalid (or mapping for shadows is missing). * - * Called with sg->mm->mmap_sem in read. + * Called with sg->mm->mmap_lock in read. */ static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr, unsigned long len, int prot, unsigned long bits) @@ -1124,7 +1124,7 @@ EXPORT_SYMBOL_GPL(gmap_mprotect_notify); * if reading using the virtual address failed. -EINVAL if called on a gmap * shadow. * - * Called with gmap->mm->mmap_sem in read. + * Called with gmap->mm->mmap_lock in read. */ int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val) { @@ -1729,7 +1729,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow); * shadow table structure is incomplete, -ENOMEM if out of memory and * -EFAULT if an address in the parent gmap could not be resolved. * - * Called with sg->mm->mmap_sem in read. + * Called with sg->mm->mmap_lock in read. */ int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, int fake) @@ -1813,7 +1813,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_r2t); * shadow table structure is incomplete, -ENOMEM if out of memory and * -EFAULT if an address in the parent gmap could not be resolved. * - * Called with sg->mm->mmap_sem in read. + * Called with sg->mm->mmap_lock in read. */ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, int fake) @@ -1897,7 +1897,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_r3t); * shadow table structure is incomplete, -ENOMEM if out of memory and * -EFAULT if an address in the parent gmap could not be resolved. * - * Called with sg->mm->mmap_sem in read. + * Called with sg->mm->mmap_lock in read. */ int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, int fake) @@ -1981,7 +1981,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_sgt); * Returns 0 if the shadow page table was found and -EAGAIN if the page * table was not found. * - * Called with sg->mm->mmap_sem in read. + * Called with sg->mm->mmap_lock in read. */ int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr, unsigned long *pgt, int *dat_protection, @@ -2021,7 +2021,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_pgt_lookup); * shadow table structure is incomplete, -ENOMEM if out of memory, * -EFAULT if an address in the parent gmap could not be resolved and * - * Called with gmap->mm->mmap_sem in read + * Called with gmap->mm->mmap_lock in read */ int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt, int fake) @@ -2100,7 +2100,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_pgt); * shadow table structure is incomplete, -ENOMEM if out of memory and * -EFAULT if an address in the parent gmap could not be resolved. * - * Called with sg->mm->mmap_sem in read. + * Called with sg->mm->mmap_lock in read. */ int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte) { diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index fff169d64711..11d2c8395e2a 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -114,7 +114,7 @@ int crst_table_upgrade(struct mm_struct *mm, unsigned long end) spin_lock_bh(&mm->page_table_lock); /* - * This routine gets called with mmap_sem lock held and there is + * This routine gets called with mmap_lock lock held and there is * no reason to optimize for the case of otherwise. However, if * that would ever change, the below check will let us know. */ diff --git a/arch/sh/mm/cache-sh4.c b/arch/sh/mm/cache-sh4.c index de9e0a60e119..ddfa9685f1ef 100644 --- a/arch/sh/mm/cache-sh4.c +++ b/arch/sh/mm/cache-sh4.c @@ -182,7 +182,7 @@ static void sh4_flush_cache_all(void *unused) * accessed with (hence cache set) is in accord with the physical * address (i.e. tag). It's no different here. * - * Caller takes mm->mmap_sem. + * Caller takes mm->mmap_lock. */ static void sh4_flush_cache_mm(void *arg) { diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index 3a125aad586d..fbe1f2fe9a8c 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -326,7 +326,7 @@ mm_fault_error(struct pt_regs *regs, unsigned long error_code, return 1; } - /* Release mmap_sem first if necessary */ + /* Release mmap_lock first if necessary */ if (!(fault & VM_FAULT_RETRY)) mmap_read_unlock(current->mm); diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index 73f95a5ba683..a3806614e4dc 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -70,7 +70,7 @@ static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr) } /* - * We now make sure that mmap_sem is held in all paths that call + * We now make sure that mmap_lock is held in all paths that call * this. Additionally, to prevent kswapd from ripping ptes from * under us, raise interrupts around the time that we look at the * pte, kswapd will have to wait to get his smp ipi response from diff --git a/arch/um/kernel/skas/mmu.c b/arch/um/kernel/skas/mmu.c index 60ea924e551d..d9961163da66 100644 --- a/arch/um/kernel/skas/mmu.c +++ b/arch/um/kernel/skas/mmu.c @@ -114,7 +114,7 @@ void uml_setup_stubs(struct mm_struct *mm) mm->context.stub_pages[0] = virt_to_page(__syscall_stub_start); mm->context.stub_pages[1] = virt_to_page(mm->context.id.stack); - /* dup_mmap already holds mmap_sem */ + /* dup_mmap already holds mmap_lock */ err = install_special_mapping(mm, STUB_START, STUB_END - STUB_START, VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC | VM_DONTCOPY | VM_PFNMAP, diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c index 6c0e76553657..61776790cd67 100644 --- a/arch/um/kernel/tlb.c +++ b/arch/um/kernel/tlb.c @@ -348,7 +348,7 @@ void fix_range_common(struct mm_struct *mm, unsigned long start_addr, if (ret) { printk(KERN_ERR "fix_range_common: failed, killing current " "process: %d\n", task_tgid_vnr(current)); - /* We are under mmap_sem, release it such that current can terminate */ + /* We are under mmap_lock, release it such that current can terminate */ mmap_write_unlock(current->mm); force_sig(SIGKILL); do_signal(¤t->thread.regs); diff --git a/arch/unicore32/mm/fault.c b/arch/unicore32/mm/fault.c index 99869b57b6cb..7654bddde133 100644 --- a/arch/unicore32/mm/fault.c +++ b/arch/unicore32/mm/fault.c @@ -246,7 +246,7 @@ retry: fault = __do_pf(mm, addr, fsr, flags, tsk); /* If we need to retry but a fatal signal is pending, handle the - * signal first. We do not need to release the mmap_sem because + * signal first. We do not need to release the mmap_lock because * it would already be released in __lock_page_or_retry in * mm/filemap.c. */ if (fault_signal_pending(fault, regs)) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 081cdfdf11f4..4103665c6e03 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2178,7 +2178,7 @@ static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) * userspace with CR4.PCE clear while another task is still * doing on_each_cpu_mask() to propagate CR4.PCE. * - * For now, this can't happen because all callers hold mmap_sem + * For now, this can't happen because all callers hold mmap_lock * for write. If this changes, we'll need a different solution. */ mmap_assert_write_locked(mm); diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h index bdeae9291e5c..0a301ad0b02f 100644 --- a/arch/x86/include/asm/mmu.h +++ b/arch/x86/include/asm/mmu.h @@ -45,7 +45,7 @@ typedef struct { #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS /* * One bit per protection key says whether userspace can - * use it or not. protected by mmap_sem. + * use it or not. protected by mmap_lock. */ u16 pkey_allocation_map; s16 execute_only_pkey; diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h index 5afb5e0fe903..e896ebef8c24 100644 --- a/arch/x86/include/asm/pgtable-3level.h +++ b/arch/x86/include/asm/pgtable-3level.h @@ -39,23 +39,23 @@ static inline void native_set_pte(pte_t *ptep, pte_t pte) * pte_offset_map_lock() on 32-bit PAE kernels was reading the pmd_t with * a "*pmdp" dereference done by GCC. Problem is, in certain places * where pte_offset_map_lock() is called, concurrent page faults are - * allowed, if the mmap_sem is hold for reading. An example is mincore + * allowed, if the mmap_lock is hold for reading. An example is mincore * vs page faults vs MADV_DONTNEED. On the page fault side * pmd_populate() rightfully does a set_64bit(), but if we're reading the * pmd_t with a "*pmdp" on the mincore side, a SMP race can happen * because GCC will not read the 64-bit value of the pmd atomically. * * To fix this all places running pte_offset_map_lock() while holding the - * mmap_sem in read mode, shall read the pmdp pointer using this + * mmap_lock in read mode, shall read the pmdp pointer using this * function to know if the pmd is null or not, and in turn to know if * they can run pte_offset_map_lock() or pmd_trans_huge() or other pmd * operations. * - * Without THP if the mmap_sem is held for reading, the pmd can only + * Without THP if the mmap_lock is held for reading, the pmd can only * transition from null to not null while pmd_read_atomic() runs. So * we can always return atomic pmd values with this function. * - * With THP if the mmap_sem is held for reading, the pmd can become + * With THP if the mmap_lock is held for reading, the pmd can become * trans_huge or none or point to a pte (and in turn become "stable") * at any time under pmd_read_atomic(). We could read it truly * atomically here with an atomic64_read() for the THP enabled case (and diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index 4bd28b388a1a..0daf2f1cf7a8 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -1326,9 +1326,9 @@ int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp) * pseudo-locked region will still be here on return. * * The mutex has to be released temporarily to avoid a potential - * deadlock with the mm->mmap_sem semaphore which is obtained in - * the device_create() and debugfs_create_dir() callpath below - * as well as before the mmap() callback is called. + * deadlock with the mm->mmap_lock which is obtained in the + * device_create() and debugfs_create_dir() callpath below as well as + * before the mmap() callback is called. */ mutex_unlock(&rdtgroup_mutex); diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index d7cb5ab0d1f0..23b4b61319d3 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -3199,10 +3199,10 @@ int __init rdtgroup_init(void) * during the debugfs directory creation also &sb->s_type->i_mutex_key * (the lockdep class of inode->i_rwsem). Other filesystem * interactions (eg. SyS_getdents) have the lock ordering: - * &sb->s_type->i_mutex_key --> &mm->mmap_sem - * During mmap(), called with &mm->mmap_sem, the rdtgroup_mutex + * &sb->s_type->i_mutex_key --> &mm->mmap_lock + * During mmap(), called with &mm->mmap_lock, the rdtgroup_mutex * is taken, thus creating dependency: - * &mm->mmap_sem --> rdtgroup_mutex for the latter that can cause + * &mm->mmap_lock --> rdtgroup_mutex for the latter that can cause * issues considering the other two lock dependencies. * By creating the debugfs directory here we avoid a dependency * that may cause deadlock (even though file operations cannot diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c index 84c3ba32f211..8748321c4486 100644 --- a/arch/x86/kernel/ldt.c +++ b/arch/x86/kernel/ldt.c @@ -8,7 +8,7 @@ * * Lock order: * contex.ldt_usr_sem - * mmap_sem + * mmap_lock * context.lock */ diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index c23bcd027ae1..0b03ae8c39cd 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -865,7 +865,7 @@ bad_area_access_error(struct pt_regs *regs, unsigned long error_code, * 2. T1 : set PKRU to deny access to pkey=4, touches page * 3. T1 : faults... * 4. T2: mprotect_key(foo, PAGE_SIZE, pkey=5); - * 5. T1 : enters fault handler, takes mmap_sem, etc... + * 5. T1 : enters fault handler, takes mmap_lock, etc... * 6. T1 : reaches here, sees vma_pkey(vma)=5, when we really * faulted on a pte with its pkey=4. */ @@ -1231,12 +1231,12 @@ void do_user_addr_fault(struct pt_regs *regs, * Kernel-mode access to the user address space should only occur * on well-defined single instructions listed in the exception * tables. But, an erroneous kernel fault occurring outside one of - * those areas which also holds mmap_sem might deadlock attempting + * those areas which also holds mmap_lock might deadlock attempting * to validate the fault against the address space. * * Only do the expensive exception table search when we might be at * risk of a deadlock. This happens if we - * 1. Failed to acquire mmap_sem, and + * 1. Failed to acquire mmap_lock, and * 2. The access did not originate in userspace. */ if (unlikely(!mmap_read_trylock(mm))) { @@ -1289,9 +1289,9 @@ good_area: * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo * the fault. Since we never set FAULT_FLAG_RETRY_NOWAIT, if - * we get VM_FAULT_RETRY back, the mmap_sem has been unlocked. + * we get VM_FAULT_RETRY back, the mmap_lock has been unlocked. * - * Note that handle_userfault() may also release and reacquire mmap_sem + * Note that handle_userfault() may also release and reacquire mmap_lock * (and not return with VM_FAULT_RETRY), when returning to userland to * repeat the page fault later with a VM_FAULT_NOPAGE retval * (potentially after handling any pending signal during the return to @@ -1310,7 +1310,7 @@ good_area: } /* - * If we need to retry the mmap_sem has already been released, + * If we need to retry the mmap_lock has already been released, * and if there is a fatal signal pending there is no guarantee * that we made any progress. Handle this case first. */ diff --git a/drivers/char/mspec.c b/drivers/char/mspec.c index d620e48bb8f7..0fae33319d2e 100644 --- a/drivers/char/mspec.c +++ b/drivers/char/mspec.c @@ -64,7 +64,7 @@ enum mspec_page_type { * This structure is shared by all vma's that are split off from the * original vma when split_vma()'s are done. * - * The refcnt is incremented atomically because mm->mmap_sem does not + * The refcnt is incremented atomically because mm->mmap_lock does not * protect in fork case where multiple tasks share the vma_data. */ struct vma_data { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 3f2b695cf19e..53b4126373a5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -186,7 +186,7 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s * disabled. The memory must be pinned and mapped to the hardware when * this is called in hqd_load functions, so it should never fault in * the first place. This resolves a circular lock dependency involving - * four locks, including the DQM lock and mmap_sem. + * four locks, including the DQM lock and mmap_lock. */ #define read_user_wptr(mmptr, wptr, dst) \ ({ \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c index 0b7e78748540..c6944739183a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c @@ -237,7 +237,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data); - /* read_user_ptr may take the mm->mmap_sem. + /* read_user_ptr may take the mm->mmap_lock. * release srbm_mutex to avoid circular dependency between * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex. */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c index ccd635b812b5..2f4bdc80a6b2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c @@ -224,7 +224,7 @@ static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1); WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, data); - /* read_user_ptr may take the mm->mmap_sem. + /* read_user_ptr may take the mm->mmap_lock. * release srbm_mutex to avoid circular dependency between * srbm_mutex->mm_sem->reservation_ww_class_mutex->srbm_mutex. */ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index 81f7656f41c3..c31a6744daee 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -203,7 +203,7 @@ i915_mmu_notifier_find(struct i915_mm_struct *mm) mmap_write_lock(mm->mm); mutex_lock(&mm->i915->mm_lock); if (mm->mn == NULL && !err) { - /* Protected by mmap_sem (write-lock) */ + /* Protected by mmap_lock (write-lock) */ err = __mmu_notifier_register(&mn->mn, mm->mm); if (!err) { /* Protected by mm_lock */ @@ -522,8 +522,8 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj) /* Spawn a worker so that we can acquire the * user pages without holding our mutex. Access - * to the user pages requires mmap_sem, and we have - * a strict lock ordering of mmap_sem, struct_mutex - + * to the user pages requires mmap_lock, and we have + * a strict lock ordering of mmap_lock, struct_mutex - * we already hold struct_mutex here and so cannot * call gup without encountering a lock inversion. * diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index 75c60c2afb7e..665bb076e84d 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -3676,7 +3676,7 @@ static int read_properties_unlocked(struct i915_perf *perf, * buffered data written by the GPU besides periodic OA metrics. * * Note we copy the properties from userspace outside of the i915 perf - * mutex to avoid an awkward lockdep with mmap_sem. + * mutex to avoid an awkward lockdep with mmap_lock. * * Most of the implementation details are handled by * i915_perf_open_ioctl_locked() after taking the &perf->lock diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 63f55685c3d8..a43aa7275f12 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -58,7 +58,7 @@ static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, goto out_clear; /* - * If possible, avoid waiting for GPU with mmap_sem + * If possible, avoid waiting for GPU with mmap_lock * held. We only do this if the fault allows retry and this * is the first attempt. */ @@ -131,14 +131,14 @@ vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, { /* * Work around locking order reversal in fault / nopfn - * between mmap_sem and bo_reserve: Perform a trylock operation + * between mmap_lock and bo_reserve: Perform a trylock operation * for reserve, and if it fails, retry the fault after waiting * for the buffer to become unreserved. */ if (unlikely(!dma_resv_trylock(bo->base.resv))) { /* * If the fault allows retry and this is the first - * fault attempt, we try to release the mmap_sem + * fault attempt, we try to release the mmap_lock * before waiting */ if (fault_flag_allow_retry_first(vmf->flags)) { diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c index 3c7367312398..69e4755cc04b 100644 --- a/drivers/infiniband/core/uverbs_main.c +++ b/drivers/infiniband/core/uverbs_main.c @@ -835,7 +835,7 @@ void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile) return; /* - * The umap_lock is nested under mmap_sem since it used within + * The umap_lock is nested under mmap_lock since it used within * the vma_ops callbacks, so we have to clean the list one mm * at a time to get the lock ordering right. Typically there * will only be one mm, so no big deal. diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.c b/drivers/infiniband/hw/hfi1/mmu_rb.c index 14d2a90964c3..24ca17b77b72 100644 --- a/drivers/infiniband/hw/hfi1/mmu_rb.c +++ b/drivers/infiniband/hw/hfi1/mmu_rb.c @@ -333,7 +333,7 @@ static void do_remove(struct mmu_rb_handler *handler, /* * Work queue function to remove all nodes that have been queued up to - * be removed. The key feature is that mm->mmap_sem is not being held + * be removed. The key feature is that mm->mmap_lock is not being held * and the remove callback can sleep while taking it, if needed. */ static void handle_remove(struct work_struct *work) diff --git a/drivers/media/v4l2-core/videobuf-dma-sg.c b/drivers/media/v4l2-core/videobuf-dma-sg.c index 672963808ab9..46ff19df9f53 100644 --- a/drivers/media/v4l2-core/videobuf-dma-sg.c +++ b/drivers/media/v4l2-core/videobuf-dma-sg.c @@ -533,7 +533,7 @@ static int __videobuf_iolock(struct videobuf_queue *q, } else { /* NOTE: HACK: videobuf_iolock on V4L2_MEMORY_MMAP buffers can only be called from videobuf_qbuf - we take current->mm->mmap_sem there, to prevent + we take current->mm->mmap_lock there, to prevent locking inversion, so don't take it here */ err = videobuf_dma_init_user_locked(&mem->dma, diff --git a/drivers/misc/cxl/cxllib.c b/drivers/misc/cxl/cxllib.c index a2c0fbb5eb28..2a1783f32254 100644 --- a/drivers/misc/cxl/cxllib.c +++ b/drivers/misc/cxl/cxllib.c @@ -245,9 +245,8 @@ int cxllib_handle_fault(struct mm_struct *mm, u64 addr, u64 size, u64 flags) dar += page_size) { if (dar < vma_start || dar >= vma_end) { /* - * We don't hold the mm->mmap_sem semaphore - * while iterating, since the semaphore is - * required by one of the lower-level page + * We don't hold mm->mmap_lock while iterating, since + * the lock is required by one of the lower-level page * fault processing functions and it could * create a deadlock. * diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 8f4e2895b565..b1521112dbbd 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -42,7 +42,7 @@ static inline int is_gru_paddr(unsigned long paddr) } /* - * Find the vma of a GRU segment. Caller must hold mmap_sem. + * Find the vma of a GRU segment. Caller must hold mmap_lock. */ struct vm_area_struct *gru_find_vma(unsigned long vaddr) { @@ -58,7 +58,7 @@ struct vm_area_struct *gru_find_vma(unsigned long vaddr) * Find and lock the gts that contains the specified user vaddr. * * Returns: - * - *gts with the mmap_sem locked for read and the GTS locked. + * - *gts with the mmap_lock locked for read and the GTS locked. * - NULL if vaddr invalid OR is not a valid GSEG vaddr. */ @@ -198,7 +198,7 @@ static int non_atomic_pte_lookup(struct vm_area_struct *vma, * Only supports Intel large pages (2MB only) on x86_64. * ZZZ - hugepage support is incomplete * - * NOTE: mmap_sem is already held on entry to this function. This + * NOTE: mmap_lock is already held on entry to this function. This * guarantees existence of the page tables. */ static int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr, @@ -569,7 +569,7 @@ static irqreturn_t gru_intr(int chiplet, int blade) } /* - * This is running in interrupt context. Trylock the mmap_sem. + * This is running in interrupt context. Trylock the mmap_lock. * If it fails, retry the fault in user context. */ gts->ustats.fmm_tlbmiss++; diff --git a/drivers/oprofile/buffer_sync.c b/drivers/oprofile/buffer_sync.c index d3b017af7758..4d7695289eda 100644 --- a/drivers/oprofile/buffer_sync.c +++ b/drivers/oprofile/buffer_sync.c @@ -486,7 +486,7 @@ typedef enum { /* Sync one of the CPU's buffers into the global event buffer. * Here we need to go through each batch of samples punctuated - * by context switch notes, taking the task's mmap_sem and doing + * by context switch notes, taking the task's mmap_lock and doing * lookup in task->mm->mmap to convert EIP into dcookie/offset * value. */ diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c index 8044510d8ec6..c05a214191da 100644 --- a/drivers/staging/android/ashmem.c +++ b/drivers/staging/android/ashmem.c @@ -555,7 +555,7 @@ static int set_name(struct ashmem_area *asma, void __user *name) /* * Holding the ashmem_mutex while doing a copy_from_user might cause - * an data abort which would try to access mmap_sem. If another + * an data abort which would try to access mmap_lock. If another * thread has invoked ashmem_mmap then it will be holding the * semaphore and will be waiting for ashmem_mutex, there by leading to * deadlock. We'll release the mutex and take the name to a local @@ -586,7 +586,7 @@ static int get_name(struct ashmem_area *asma, void __user *name) * Have a local variable to which we'll copy the content * from asma with the lock held. Later we can copy this to the user * space safely without holding any locks. So even if we proceed to - * wait for mmap_sem, it won't lead to deadlock. + * wait for mmap_lock, it won't lead to deadlock. */ char local_name[ASHMEM_NAME_LEN]; diff --git a/drivers/staging/comedi/comedi_fops.c b/drivers/staging/comedi/comedi_fops.c index a56c8f74a27b..e85a99b68f31 100644 --- a/drivers/staging/comedi/comedi_fops.c +++ b/drivers/staging/comedi/comedi_fops.c @@ -2325,7 +2325,7 @@ static int comedi_mmap(struct file *file, struct vm_area_struct *vma) int retval = 0; /* - * 'trylock' avoids circular dependency with current->mm->mmap_sem + * 'trylock' avoids circular dependency with current->mm->mmap_lock * and down-reading &dev->attach_lock should normally succeed without * contention unless the device is in the process of being attached * or detached. diff --git a/drivers/tty/vt/consolemap.c b/drivers/tty/vt/consolemap.c index b28aa0d289f8..c1be96bb3ecf 100644 --- a/drivers/tty/vt/consolemap.c +++ b/drivers/tty/vt/consolemap.c @@ -12,7 +12,7 @@ * Fix bug in inverse translation. Stanislav Voronyi , Dec 1998 * * In order to prevent the following circular lock dependency: - * &mm->mmap_sem --> cpu_hotplug.lock --> console_lock --> &mm->mmap_sem + * &mm->mmap_lock --> cpu_hotplug.lock --> console_lock --> &mm->mmap_lock * * We cannot allow page fault to happen while holding the console_lock. * Therefore, all the userspace copy operations have to be done outside diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index b5f6ef2d12f6..7c0779018b1b 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -1185,7 +1185,7 @@ reset_info_exit: /* * We need to get memory_lock for each device, but devices - * can share mmap_sem, therefore we need to zap and hold + * can share mmap_lock, therefore we need to zap and hold * the vma_lock for each device, and only then get each * memory_lock. */ @@ -1375,26 +1375,26 @@ static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try) /* * Lock ordering: - * vma_lock is nested under mmap_sem for vm_ops callback paths. + * vma_lock is nested under mmap_lock for vm_ops callback paths. * The memory_lock semaphore is used by both code paths calling * into this function to zap vmas and the vm_ops.fault callback * to protect the memory enable state of the device. * - * When zapping vmas we need to maintain the mmap_sem => vma_lock + * When zapping vmas we need to maintain the mmap_lock => vma_lock * ordering, which requires using vma_lock to walk vma_list to - * acquire an mm, then dropping vma_lock to get the mmap_sem and + * acquire an mm, then dropping vma_lock to get the mmap_lock and * reacquiring vma_lock. This logic is derived from similar * requirements in uverbs_user_mmap_disassociate(). * - * mmap_sem must always be the top-level lock when it is taken. + * mmap_lock must always be the top-level lock when it is taken. * Therefore we can only hold the memory_lock write lock when - * vma_list is empty, as we'd need to take mmap_sem to clear + * vma_list is empty, as we'd need to take mmap_lock to clear * entries. vma_list can only be guaranteed empty when holding * vma_lock, thus memory_lock is nested under vma_lock. * * This enables the vm_ops.fault callback to acquire vma_lock, * followed by memory_lock read lock, while already holding - * mmap_sem without risk of deadlock. + * mmap_lock without risk of deadlock. */ while (1) { struct mm_struct *mm = NULL; diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index 3a939e358284..64a9025a87be 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -1014,7 +1014,7 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) * to the PTE from going stale. * * Since this vma's mappings can't be touched without the - * mmap_sem, and we are holding it now, there is no need for + * mmap_lock, and we are holding it now, there is no need for * the notifier_range locking pattern. */ mmu_interval_read_begin(&map->notifier); diff --git a/fs/coredump.c b/fs/coredump.c index 9fde263af452..7237f07ff6be 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -393,7 +393,7 @@ static int zap_threads(struct task_struct *tsk, struct mm_struct *mm, * of ->siglock provides a memory barrier. * * do_exit: - * The caller holds mm->mmap_sem. This means that the task which + * The caller holds mm->mmap_lock. This means that the task which * uses this mm can't pass exit_mm(), so it can't exit or clear * its ->mm. * @@ -401,7 +401,7 @@ static int zap_threads(struct task_struct *tsk, struct mm_struct *mm, * It does list_replace_rcu(&leader->tasks, ¤t->tasks), * we must see either old or new leader, this does not matter. * However, it can change p->sighand, so lock_task_sighand(p) - * must be used. Since p->mm != NULL and we hold ->mmap_sem + * must be used. Since p->mm != NULL and we hold ->mmap_lock * it can't fail. * * Note also that "g" can be the old leader with ->mm == NULL diff --git a/fs/exec.c b/fs/exec.c index 105b91d191eb..e6e8a9a70327 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1091,7 +1091,7 @@ static int exec_mmap(struct mm_struct *mm) /* * Make sure that if there is a core dump in progress * for the old mm, we get out and die instead of going - * through with the exec. We must hold mmap_sem around + * through with the exec. We must hold mmap_lock around * checking core_state and changing tsk->mm. */ mmap_read_lock(old_mm); diff --git a/fs/ext2/file.c b/fs/ext2/file.c index b4de9a0f170d..60378ddf1424 100644 --- a/fs/ext2/file.c +++ b/fs/ext2/file.c @@ -79,7 +79,7 @@ out_unlock: /* * The lock ordering for ext2 DAX fault paths is: * - * mmap_sem (MM) + * mmap_lock (MM) * sb_start_pagefault (vfs, freeze) * ext2_inode_info->dax_sem * address_space->i_mmap_rwsem or page_lock (mutually exclusive in DAX) diff --git a/fs/ext4/super.c b/fs/ext4/super.c index a29e8ea1a7ab..c668f6b42374 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -93,11 +93,11 @@ static struct inode *ext4_get_journal_inode(struct super_block *sb, * i_mmap_rwsem (inode->i_mmap_rwsem)! * * page fault path: - * mmap_sem -> sb_start_pagefault -> i_mmap_sem (r) -> transaction start -> + * mmap_lock -> sb_start_pagefault -> i_mmap_sem (r) -> transaction start -> * page lock -> i_data_sem (rw) * * buffered write path: - * sb_start_write -> i_mutex -> mmap_sem + * sb_start_write -> i_mutex -> mmap_lock * sb_start_write -> i_mutex -> transaction start -> page lock -> * i_data_sem (rw) * @@ -107,7 +107,7 @@ static struct inode *ext4_get_journal_inode(struct super_block *sb, * i_data_sem (rw) * * direct IO: - * sb_start_write -> i_mutex -> mmap_sem + * sb_start_write -> i_mutex -> mmap_lock * sb_start_write -> i_mutex -> transaction start -> i_data_sem (rw) * * writepages: diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index fd6ddfe4cd94..06b342d8462b 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -652,9 +652,9 @@ static int kernfs_fop_open(struct inode *inode, struct file *file) * The following is done to give a different lockdep key to * @of->mutex for files which implement mmap. This is a rather * crude way to avoid false positive lockdep warning around - * mm->mmap_sem - mmap nests @of->mutex under mm->mmap_sem and + * mm->mmap_lock - mmap nests @of->mutex under mm->mmap_lock and * reading /sys/block/sda/trace/act_mask grabs sr_mutex, under - * which mm->mmap_sem nests, while holding @of->mutex. As each + * which mm->mmap_lock nests, while holding @of->mutex. As each * open file has a separate mutex, it's okay as long as those don't * happen on the same file. At this point, we can't easily give * each file a separate locking class. Let's differentiate on diff --git a/fs/proc/base.c b/fs/proc/base.c index 4f0d6f40b8f1..d86c0afc8a85 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2333,11 +2333,11 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) /* * We need two passes here: * - * 1) Collect vmas of mapped files with mmap_sem taken - * 2) Release mmap_sem and instantiate entries + * 1) Collect vmas of mapped files with mmap_lock taken + * 2) Release mmap_lock and instantiate entries * * otherwise we get lockdep complained, since filldir() - * routine might require mmap_sem taken in might_fault(). + * routine might require mmap_lock taken in might_fault(). */ for (vma = mm->mmap, pos = 2; vma; vma = vma->vm_next) { diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 81b985476c1d..dbda4499a859 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -593,7 +593,7 @@ static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, if (pmd_trans_unstable(pmd)) goto out; /* - * The mmap_sem held all the way back in m_start() is what + * The mmap_lock held all the way back in m_start() is what * keeps khugepaged out of here and from collapsing things * in here. */ @@ -752,7 +752,7 @@ static void smap_gather_stats(struct vm_area_struct *vma, } } #endif - /* mmap_sem is held in m_start */ + /* mmap_lock is held in m_start */ walk_page_vma(vma, &smaps_walk_ops, mss); } @@ -1827,7 +1827,7 @@ static int show_numa_map(struct seq_file *m, void *v) if (is_vm_hugetlb_page(vma)) seq_puts(m, " huge"); - /* mmap_sem is held by m_start */ + /* mmap_lock is held by m_start */ walk_page_vma(vma, &show_numa_ops, md); if (!md->pages) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 3a63d75ed2fd..52de29000c7e 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -369,13 +369,13 @@ static inline bool userfaultfd_signal_pending(unsigned int flags) * FAULT_FLAG_KILLABLE are not straightforward. The "Caution" * recommendation in __lock_page_or_retry is not an understatement. * - * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_sem must be released + * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is * not set. * * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not * set, VM_FAULT_RETRY can still be returned if and only if there are - * fatal_signal_pending()s, and the mmap_sem must be released before + * fatal_signal_pending()s, and the mmap_lock must be released before * returning it. */ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) @@ -396,14 +396,14 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) * FOLL_DUMP case, anon memory also checks for FOLL_DUMP with * the no_page_table() helper in follow_page_mask(), but the * shmem_vm_ops->fault method is invoked even during - * coredumping without mmap_sem and it ends up here. + * coredumping without mmap_lock and it ends up here. */ if (current->flags & (PF_EXITING|PF_DUMPCORE)) goto out; /* - * Coredumping runs without mmap_sem so we can only check that - * the mmap_sem is held, if PF_DUMPCORE was not set. + * Coredumping runs without mmap_lock so we can only check that + * the mmap_lock is held, if PF_DUMPCORE was not set. */ mmap_assert_locked(mm); @@ -422,7 +422,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) /* * If it's already released don't get it. This avoids to loop * in __get_user_pages if userfaultfd_release waits on the - * caller of handle_userfault to release the mmap_sem. + * caller of handle_userfault to release the mmap_lock. */ if (unlikely(READ_ONCE(ctx->released))) { /* @@ -481,7 +481,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) goto out; - /* take the reference before dropping the mmap_sem */ + /* take the reference before dropping the mmap_lock */ userfaultfd_ctx_get(ctx); init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function); @@ -890,9 +890,9 @@ static int userfaultfd_release(struct inode *inode, struct file *file) * Flush page faults out of all CPUs. NOTE: all page faults * must be retried without returning VM_FAULT_SIGBUS if * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx - * changes while handle_userfault released the mmap_sem. So + * changes while handle_userfault released the mmap_lock. So * it's critical that released is set to true (above), before - * taking the mmap_sem for writing. + * taking the mmap_lock for writing. */ mmap_write_lock(mm); still_valid = mmget_still_valid(mm); diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 403c90309a8f..00db81eac80d 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -1173,7 +1173,7 @@ xfs_file_llseek( * Locking for serialisation of IO during page faults. This results in a lock * ordering of: * - * mmap_sem (MM) + * mmap_lock (MM) * sb_start_pagefault(vfs, freeze) * i_mmaplock (XFS - truncate serialisation) * page_lock (MM) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 64f5f9a440ae..4c91fb25ec66 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -145,17 +145,17 @@ xfs_ilock_attr_map_shared( * * i_rwsem -> i_mmap_lock -> page_lock -> i_ilock * - * mmap_sem locking order: + * mmap_lock locking order: * - * i_rwsem -> page lock -> mmap_sem - * mmap_sem -> i_mmap_lock -> page_lock + * i_rwsem -> page lock -> mmap_lock + * mmap_lock -> i_mmap_lock -> page_lock * - * The difference in mmap_sem locking order mean that we cannot hold the + * The difference in mmap_lock locking order mean that we cannot hold the * i_mmap_lock over syscall based read(2)/write(2) based IO. These IO paths can - * fault in pages during copy in/out (for buffered IO) or require the mmap_sem + * fault in pages during copy in/out (for buffered IO) or require the mmap_lock * in get_user_pages() to map the user pages into the kernel address space for * direct IO. Similarly the i_rwsem cannot be taken inside a page fault because - * page faults already hold the mmap_sem. + * page faults already hold the mmap_lock. * * Hence to serialise fully against both syscall and mmap based IO, we need to * take both the i_rwsem and the i_mmap_lock. These locks should *only* be both @@ -1630,7 +1630,7 @@ xfs_release( return 0; /* * If we can't get the iolock just skip truncating the blocks - * past EOF because we could deadlock with the mmap_sem + * past EOF because we could deadlock with the mmap_lock * otherwise. We'll get another chance to drop them once the * last reference to the inode is dropped, so we'll never leak * blocks permanently. diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 202b2c0a9e9d..80a13c8561d8 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -28,11 +28,11 @@ #include /* - * Directories have different lock order w.r.t. mmap_sem compared to regular + * Directories have different lock order w.r.t. mmap_lock compared to regular * files. This is due to readdir potentially triggering page faults on a user * buffer inside filldir(), and this happens with the ilock on the directory * held. For regular files, the lock order is the other way around - the - * mmap_sem is taken during the page fault, and then we lock the ilock to do + * mmap_lock is taken during the page fault, and then we lock the ilock to do * block mapping. Hence we need a different class for the directory ilock so * that lockdep can tell them apart. */ diff --git a/include/linux/fs.h b/include/linux/fs.h index 01f5d296f9bb..0b026329dbed 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1679,10 +1679,10 @@ static inline int sb_start_write_trylock(struct super_block *sb) * * Since page fault freeze protection behaves as a lock, users have to preserve * ordering of freeze protection and other filesystem locks. It is advised to - * put sb_start_pagefault() close to mmap_sem in lock ordering. Page fault + * put sb_start_pagefault() close to mmap_lock in lock ordering. Page fault * handling code implies lock dependency: * - * mmap_sem + * mmap_lock * -> sb_start_pagefault */ static inline void sb_start_pagefault(struct super_block *sb) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index cfbb0a87c5f0..71f20776b06c 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -248,7 +248,7 @@ static inline int is_swap_pmd(pmd_t pmd) return !pmd_none(pmd) && !pmd_present(pmd); } -/* mmap_sem must be held on entry */ +/* mmap_lock must be held on entry */ static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) { diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 8165278c348a..ea9c15b60a96 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -31,7 +31,7 @@ struct mm_struct; * Locking policy for interlave: * In process context there is no locking because only the process accesses * its own state. All vma manipulation is somewhat protected by a down_read on - * mmap_sem. + * mmap_lock. * * Freeing policy: * Mempolicy objects are reference counted. A mempolicy will be freed when diff --git a/include/linux/mm.h b/include/linux/mm.h index 256c4b5b3b11..dc7b87310c10 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -402,7 +402,7 @@ extern pgprot_t protection_map[16]; * @FAULT_FLAG_WRITE: Fault was a write fault. * @FAULT_FLAG_MKWRITE: Fault was mkwrite of existing PTE. * @FAULT_FLAG_ALLOW_RETRY: Allow to retry the fault if blocked. - * @FAULT_FLAG_RETRY_NOWAIT: Don't drop mmap_sem and wait when retrying. + * @FAULT_FLAG_RETRY_NOWAIT: Don't drop mmap_lock and wait when retrying. * @FAULT_FLAG_KILLABLE: The fault task is in SIGKILL killable region. * @FAULT_FLAG_TRIED: The fault has been tried once. * @FAULT_FLAG_USER: The fault originated in userspace. @@ -452,10 +452,10 @@ extern pgprot_t protection_map[16]; * fault_flag_allow_retry_first - check ALLOW_RETRY the first time * * This is mostly used for places where we want to try to avoid taking - * the mmap_sem for too long a time when waiting for another condition + * the mmap_lock for too long a time when waiting for another condition * to change, in which case we can try to be polite to release the - * mmap_sem in the first round to avoid potential starvation of other - * processes that would also want the mmap_sem. + * mmap_lock in the first round to avoid potential starvation of other + * processes that would also want the mmap_lock. * * Return: true if the page fault allows retry and this is the first * attempt of the fault handling; false otherwise. @@ -582,7 +582,7 @@ struct vm_operations_struct { * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure * in mm/mempolicy.c will do this automatically. * get_policy() must NOT add a ref if the policy at (vma,addr) is not - * marked as MPOL_SHARED. vma policies are protected by the mmap_sem. + * marked as MPOL_SHARED. vma policies are protected by the mmap_lock. * If no [shared/vma] mempolicy exists at the addr, get_policy() op * must return NULL--i.e., do not "fallback" to task or system default * policy. diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index b6639b30a83b..64ede5f150dc 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -344,7 +344,7 @@ struct vm_area_struct { * can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack * or brk vma (with NULL file) can only be in an anon_vma list. */ - struct list_head anon_vma_chain; /* Serialized by mmap_sem & + struct list_head anon_vma_chain; /* Serialized by mmap_lock & * page_table_lock */ struct anon_vma *anon_vma; /* Serialized by page_table_lock */ diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index 2f462710a1a4..fc68f3570e19 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h @@ -122,7 +122,7 @@ struct mmu_notifier_ops { /* * invalidate_range_start() and invalidate_range_end() must be - * paired and are called only when the mmap_sem and/or the + * paired and are called only when the mmap_lock and/or the * locks protecting the reverse maps are held. If the subsystem * can't guarantee that no additional references are taken to * the pages in the range, it has to implement the @@ -213,13 +213,13 @@ struct mmu_notifier_ops { }; /* - * The notifier chains are protected by mmap_sem and/or the reverse map + * The notifier chains are protected by mmap_lock and/or the reverse map * semaphores. Notifier chains are only changed when all reverse maps and - * the mmap_sem locks are taken. + * the mmap_lock locks are taken. * * Therefore notifier chains can only be traversed when either * - * 1. mmap_sem is held. + * 1. mmap_lock is held. * 2. One of the reverse map locks is held (i_mmap_rwsem or anon_vma->rwsem). * 3. No other concurrent thread can access the list (release) */ diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 8e085713150c..cf2468da68e9 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -538,7 +538,7 @@ static inline int lock_page_killable(struct page *page) * lock_page_or_retry - Lock the page, unless this would block and the * caller indicated that it can handle a retry. * - * Return value and mmap_sem implications depend on flags; see + * Return value and mmap_lock implications depend on flags; see * __lock_page_or_retry(). */ static inline int lock_page_or_retry(struct page *page, struct mm_struct *mm, diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 90a5d0c78fca..32b6c52d41b9 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1134,11 +1134,11 @@ static inline pmd_t pmd_read_atomic(pmd_t *pmdp) #endif /* * This function is meant to be used by sites walking pagetables with - * the mmap_sem hold in read mode to protect against MADV_DONTNEED and + * the mmap_lock held in read mode to protect against MADV_DONTNEED and * transhuge page faults. MADV_DONTNEED can convert a transhuge pmd * into a null pmd and the transhuge page fault can convert a null pmd * into an hugepmd or into a regular pmd (if the hugepage allocation - * fails). While holding the mmap_sem in read mode the pmd becomes + * fails). While holding the mmap_lock in read mode the pmd becomes * stable and stops changing under us only if it's not null and not a * transhuge pmd. When those races occurs and this function makes a * difference vs the standard pmd_none_or_clear_bad, the result is @@ -1148,7 +1148,7 @@ static inline pmd_t pmd_read_atomic(pmd_t *pmdp) * * For 32bit kernels with a 64bit large pmd_t this automatically takes * care of reading the pmd atomically to avoid SMP race conditions - * against pmd_populate() when the mmap_sem is hold for reading by the + * against pmd_populate() when the mmap_lock is hold for reading by the * caller (a special atomic read not done by "gcc" as in the generic * version above, is also needed when THP is disabled because the page * fault can populate the pmd from under us). diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 988d176472df..3a6adfa70fb0 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -77,7 +77,7 @@ struct anon_vma { struct anon_vma_chain { struct vm_area_struct *vma; struct anon_vma *anon_vma; - struct list_head same_vma; /* locked by mmap_sem & page_table_lock */ + struct list_head same_vma; /* locked by mmap_lock & page_table_lock */ struct rb_node rb; /* locked by anon_vma->rwsem */ unsigned long rb_subtree_last; #ifdef CONFIG_DEBUG_VM_RB diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index a132d875d351..480a4d1b7dd8 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -53,7 +53,7 @@ void mmdrop(struct mm_struct *mm); /* * This has to be called after a get_task_mm()/mmget_not_zero() - * followed by taking the mmap_sem for writing before modifying the + * followed by taking the mmap_lock for writing before modifying the * vmas or anything the coredump pretends not to change from under it. * * It also has to be called when mmgrab() is used in the context of @@ -61,14 +61,14 @@ void mmdrop(struct mm_struct *mm); * the context of the process to run down_write() on that pinned mm. * * NOTE: find_extend_vma() called from GUP context is the only place - * that can modify the "mm" (notably the vm_start/end) under mmap_sem + * that can modify the "mm" (notably the vm_start/end) under mmap_lock * for reading and outside the context of the process, so it is also - * the only case that holds the mmap_sem for reading that must call - * this function. Generally if the mmap_sem is hold for reading + * the only case that holds the mmap_lock for reading that must call + * this function. Generally if the mmap_lock is hold for reading * there's no need of this check after get_task_mm()/mmget_not_zero(). * * This function can be obsoleted and the check can be removed, after - * the coredump code will hold the mmap_sem for writing before + * the coredump code will hold the mmap_lock for writing before * invoking the ->core_dump methods. */ static inline bool mmget_still_valid(struct mm_struct *mm) diff --git a/kernel/acct.c b/kernel/acct.c index c530568dd51c..b0c5b3a9f5af 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -40,7 +40,7 @@ * is one more bug... 10/11/98, AV. * * Oh, fsck... Oopsable SMP race in do_process_acct() - we must hold - * ->mmap_sem to walk the vma list of current->mm. Nasty, since it leaks + * ->mmap_lock to walk the vma list of current->mm. Nasty, since it leaks * a struct file opened for write. Fixed. 2/6/2000, AV. */ diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 729d3a5c772e..642415b8c3c9 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1655,7 +1655,7 @@ static void update_tasks_nodemask(struct cpuset *cs) guarantee_online_mems(cs, &newmems); /* - * The mpol_rebind_mm() call takes mmap_sem, which we couldn't + * The mpol_rebind_mm() call takes mmap_lock, which we couldn't * take while holding tasklist_lock. Forks can happen - the * mpol_dup() cpuset_being_rebound check will catch such forks, * and rebind their vma mempolicies too. Because we still hold @@ -1760,7 +1760,7 @@ static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) * * Call with cpuset_mutex held. May take callback_lock during call. * Will take tasklist_lock, scan tasklist for tasks in cpuset cs, - * lock each such tasks mm->mmap_sem, scan its vma's and rebind + * lock each such tasks mm->mmap_lock, scan its vma's and rebind * their mempolicies to the cpusets new mems_allowed. */ static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs, diff --git a/kernel/events/core.c b/kernel/events/core.c index 2861addad657..856d98c36f56 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -1316,7 +1316,7 @@ static void put_ctx(struct perf_event_context *ctx) * perf_event::child_mutex; * perf_event_context::lock * perf_event::mmap_mutex - * mmap_sem + * mmap_lock * perf_addr_filters_head::lock * * cpu_hotplug_lock @@ -3080,7 +3080,7 @@ static int perf_event_stop(struct perf_event *event, int restart) * pre-existing mappings, called once when new filters arrive via SET_FILTER * ioctl; * (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly - * registered mapping, called for every new mmap(), with mm::mmap_sem down + * registered mapping, called for every new mmap(), with mm::mmap_lock down * for reading; * (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process * of exec. @@ -9742,7 +9742,7 @@ static void perf_addr_filters_splice(struct perf_event *event, /* * Scan through mm's vmas and see if one of them matches the * @filter; if so, adjust filter's address range. - * Called with mm::mmap_sem down for reading. + * Called with mm::mmap_lock down for reading. */ static void perf_addr_filter_apply(struct perf_addr_filter *filter, struct mm_struct *mm, diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index cd82e1ba6b9b..4f4993dcf17a 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -457,7 +457,7 @@ static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm, * @vaddr: the virtual address to store the opcode. * @opcode: opcode to be written at @vaddr. * - * Called with mm->mmap_sem held for write. + * Called with mm->mmap_lock held for write. * Return 0 (success) or a negative errno. */ int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm, @@ -1349,7 +1349,7 @@ static int delayed_ref_ctr_inc(struct vm_area_struct *vma) } /* - * Called from mmap_region/vma_adjust with mm->mmap_sem acquired. + * Called from mmap_region/vma_adjust with mm->mmap_lock acquired. * * Currently we ignore all errors and always return 0, the callers * can't handle the failure anyway. diff --git a/kernel/exit.c b/kernel/exit.c index 36cbaa43ac80..727150f28103 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -440,7 +440,7 @@ static void exit_mm(void) sync_mm_rss(mm); /* * Serialize with any possible pending coredump. - * We must hold mmap_sem around checking core_state + * We must hold mmap_lock around checking core_state * and clearing tsk->mm. The core-inducing thread * will increment ->nr_threads for each thread in the * group with ->mm != NULL. diff --git a/kernel/relay.c b/kernel/relay.c index 204867220f8a..72fe443ea78f 100644 --- a/kernel/relay.c +++ b/kernel/relay.c @@ -91,7 +91,7 @@ static void relay_free_page_array(struct page **array) * * Returns 0 if ok, negative on error * - * Caller should already have grabbed mmap_sem. + * Caller should already have grabbed mmap_lock. */ static int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma) { diff --git a/kernel/sys.c b/kernel/sys.c index 12805750a66c..fd46865b46ba 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2007,7 +2007,7 @@ static int prctl_set_mm_map(int opt, const void __user *addr, unsigned long data } /* - * arg_lock protects concurent updates but we still need mmap_sem for + * arg_lock protects concurent updates but we still need mmap_lock for * read to exclude races with sys_brk. */ mmap_read_lock(mm); @@ -2122,7 +2122,7 @@ static int prctl_set_mm(int opt, unsigned long addr, /* * arg_lock protects concurent updates of arg boundaries, we need - * mmap_sem for a) concurrent sys_brk, b) finding VMA for addr + * mmap_lock for a) concurrent sys_brk, b) finding VMA for addr * validation. */ mmap_read_lock(mm); diff --git a/lib/test_lockup.c b/lib/test_lockup.c index fffb2a6d591b..f258743a0d83 100644 --- a/lib/test_lockup.c +++ b/lib/test_lockup.c @@ -103,7 +103,7 @@ MODULE_PARM_DESC(lock_rcu, "grab rcu_read_lock: generate rcu stalls"); static bool lock_mmap_sem; module_param(lock_mmap_sem, bool, 0400); -MODULE_PARM_DESC(lock_mmap_sem, "lock mm->mmap_sem: block procfs interfaces"); +MODULE_PARM_DESC(lock_mmap_sem, "lock mm->mmap_lock: block procfs interfaces"); static unsigned long lock_rwsem_ptr; module_param_unsafe(lock_rwsem_ptr, ulong, 0400); @@ -191,7 +191,7 @@ static void test_lock(bool master, bool verbose) if (lock_mmap_sem && master) { if (verbose) - pr_notice("lock mmap_sem pid=%d\n", main_task->pid); + pr_notice("lock mmap_lock pid=%d\n", main_task->pid); if (lock_read) mmap_read_lock(main_task->mm); else @@ -280,7 +280,7 @@ static void test_unlock(bool master, bool verbose) else mmap_write_unlock(main_task->mm); if (verbose) - pr_notice("unlock mmap_sem pid=%d\n", main_task->pid); + pr_notice("unlock mmap_lock pid=%d\n", main_task->pid); } if (lock_rwsem_ptr && master) { @@ -505,7 +505,7 @@ static int __init test_lockup_init(void) } if (lock_mmap_sem && !main_task->mm) { - pr_err("no mm to lock mmap_sem\n"); + pr_err("no mm to lock mmap_lock\n"); return -EINVAL; } diff --git a/mm/filemap.c b/mm/filemap.c index 950cf12a10fc..f0ae9a6308cb 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -76,16 +76,16 @@ * ->i_mutex * ->i_mmap_rwsem (truncate->unmap_mapping_range) * - * ->mmap_sem + * ->mmap_lock * ->i_mmap_rwsem * ->page_table_lock or pte_lock (various, mainly in memory.c) * ->i_pages lock (arch-dependent flush_dcache_mmap_lock) * - * ->mmap_sem + * ->mmap_lock * ->lock_page (access_process_vm) * * ->i_mutex (generic_perform_write) - * ->mmap_sem (fault_in_pages_readable->do_page_fault) + * ->mmap_lock (fault_in_pages_readable->do_page_fault) * * bdi->wb.list_lock * sb_lock (fs/fs-writeback.c) @@ -1371,21 +1371,21 @@ EXPORT_SYMBOL_GPL(__lock_page_killable); /* * Return values: - * 1 - page is locked; mmap_sem is still held. + * 1 - page is locked; mmap_lock is still held. * 0 - page is not locked. * mmap_lock has been released (mmap_read_unlock(), unless flags had both * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in - * which case mmap_sem is still held. + * which case mmap_lock is still held. * * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1 - * with the page locked and the mmap_sem unperturbed. + * with the page locked and the mmap_lock unperturbed. */ int __lock_page_or_retry(struct page *page, struct mm_struct *mm, unsigned int flags) { if (fault_flag_allow_retry_first(flags)) { /* - * CAUTION! In this case, mmap_sem is not released + * CAUTION! In this case, mmap_lock is not released * even though return 0. */ if (flags & FAULT_FLAG_RETRY_NOWAIT) @@ -2313,14 +2313,14 @@ EXPORT_SYMBOL(generic_file_read_iter); #ifdef CONFIG_MMU #define MMAP_LOTSAMISS (100) /* - * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_sem + * lock_page_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock * @vmf - the vm_fault for this fault. * @page - the page to lock. * @fpin - the pointer to the file we may pin (or is already pinned). * - * This works similar to lock_page_or_retry in that it can drop the mmap_sem. + * This works similar to lock_page_or_retry in that it can drop the mmap_lock. * It differs in that it actually returns the page locked if it returns 1 and 0 - * if it couldn't lock the page. If we did have to drop the mmap_sem then fpin + * if it couldn't lock the page. If we did have to drop the mmap_lock then fpin * will point to the pinned file and needs to be fput()'ed at a later point. */ static int lock_page_maybe_drop_mmap(struct vm_fault *vmf, struct page *page, @@ -2331,7 +2331,7 @@ static int lock_page_maybe_drop_mmap(struct vm_fault *vmf, struct page *page, /* * NOTE! This will make us return with VM_FAULT_RETRY, but with - * the mmap_sem still held. That's how FAULT_FLAG_RETRY_NOWAIT + * the mmap_lock still held. That's how FAULT_FLAG_RETRY_NOWAIT * is supposed to work. We have way too many special cases.. */ if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) @@ -2341,10 +2341,10 @@ static int lock_page_maybe_drop_mmap(struct vm_fault *vmf, struct page *page, if (vmf->flags & FAULT_FLAG_KILLABLE) { if (__lock_page_killable(page)) { /* - * We didn't have the right flags to drop the mmap_sem, + * We didn't have the right flags to drop the mmap_lock, * but all fault_handlers only check for fatal signals * if we return VM_FAULT_RETRY, so we need to drop the - * mmap_sem here and return 0 if we don't have a fpin. + * mmap_lock here and return 0 if we don't have a fpin. */ if (*fpin == NULL) mmap_read_unlock(vmf->vma->vm_mm); @@ -2409,7 +2409,7 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) /* * Asynchronous readahead happens when we find the page and PG_readahead, * so we want to possibly extend the readahead further. We return the file that - * was pinned if we have to drop the mmap_sem in order to do IO. + * was pinned if we have to drop the mmap_lock in order to do IO. */ static struct file *do_async_mmap_readahead(struct vm_fault *vmf, struct page *page) @@ -2444,12 +2444,12 @@ static struct file *do_async_mmap_readahead(struct vm_fault *vmf, * it in the page cache, and handles the special cases reasonably without * having a lot of duplicated code. * - * vma->vm_mm->mmap_sem must be held on entry. + * vma->vm_mm->mmap_lock must be held on entry. * - * If our return value has VM_FAULT_RETRY set, it's because the mmap_sem + * If our return value has VM_FAULT_RETRY set, it's because the mmap_lock * may be dropped before doing I/O or by lock_page_maybe_drop_mmap(). * - * If our return value does not have VM_FAULT_RETRY set, the mmap_sem + * If our return value does not have VM_FAULT_RETRY set, the mmap_lock * has not been released. * * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set. @@ -2519,7 +2519,7 @@ retry_find: goto page_not_uptodate; /* - * We've made it this far and we had to drop our mmap_sem, now is the + * We've made it this far and we had to drop our mmap_lock, now is the * time to return to the upper layer and have it re-find the vma and * redo the fault. */ @@ -2569,7 +2569,7 @@ page_not_uptodate: out_retry: /* - * We dropped the mmap_sem, we need to return to the fault handler to + * We dropped the mmap_lock, we need to return to the fault handler to * re-find the vma and come back and find our hopefully still populated * page. */ diff --git a/mm/frame_vector.c b/mm/frame_vector.c index c2578e1a2bb7..10f82d5643b6 100644 --- a/mm/frame_vector.c +++ b/mm/frame_vector.c @@ -29,7 +29,7 @@ * different type underlying the specified range of virtual addresses. * When the function isn't able to map a single page, it returns error. * - * This function takes care of grabbing mmap_sem as necessary. + * This function takes care of grabbing mmap_lock as necessary. */ int get_vaddr_frames(unsigned long start, unsigned int nr_frames, unsigned int gup_flags, struct frame_vector *vec) diff --git a/mm/gup.c b/mm/gup.c index bbb8851f4656..de9e36262ccb 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -592,7 +592,7 @@ retry: pmdval = READ_ONCE(*pmd); /* * MADV_DONTNEED may convert the pmd to null because - * mmap_sem is held in read mode + * mmap_lock is held in read mode */ if (pmd_none(pmdval)) return no_page_table(vma, flags); @@ -855,8 +855,8 @@ unmap: } /* - * mmap_sem must be held on entry. If @locked != NULL and *@flags - * does not include FOLL_NOWAIT, the mmap_sem may be released. If it + * mmap_lock must be held on entry. If @locked != NULL and *@flags + * does not include FOLL_NOWAIT, the mmap_lock may be released. If it * is, *@locked will be set to 0 and -EBUSY returned. */ static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma, @@ -979,7 +979,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) * only intends to ensure the pages are faulted in. * @vmas: array of pointers to vmas corresponding to each page. * Or NULL if the caller does not require them. - * @locked: whether we're still with the mmap_sem held + * @locked: whether we're still with the mmap_lock held * * Returns either number of pages pinned (which may be less than the * number requested), or an error. Details about the return value: @@ -992,9 +992,9 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) * * The caller is responsible for releasing returned @pages, via put_page(). * - * @vmas are valid only as long as mmap_sem is held. + * @vmas are valid only as long as mmap_lock is held. * - * Must be called with mmap_sem held. It may be released. See below. + * Must be called with mmap_lock held. It may be released. See below. * * __get_user_pages walks a process's page tables and takes a reference to * each struct page that each user address corresponds to at a given @@ -1015,12 +1015,12 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) * appropriate) must be called after the page is finished with, and * before put_page is called. * - * If @locked != NULL, *@locked will be set to 0 when mmap_sem is + * If @locked != NULL, *@locked will be set to 0 when mmap_lock is * released by an up_read(). That can happen if @gup_flags does not * have FOLL_NOWAIT. * * A caller using such a combination of @locked and @gup_flags - * must therefore hold the mmap_sem for reading only, and recognize + * must therefore hold the mmap_lock for reading only, and recognize * when it's been released. Otherwise, it must be held for either * reading or writing and will not be released. * @@ -1083,7 +1083,7 @@ static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, if (locked && *locked == 0) { /* * We've got a VM_FAULT_RETRY - * and we've lost mmap_sem. + * and we've lost mmap_lock. * We must stop here. */ BUG_ON(gup_flags & FOLL_NOWAIT); @@ -1190,7 +1190,7 @@ static bool vma_permits_fault(struct vm_area_struct *vma, * @mm: mm_struct of target mm * @address: user address * @fault_flags:flags to pass down to handle_mm_fault() - * @unlocked: did we unlock the mmap_sem while retrying, maybe NULL if caller + * @unlocked: did we unlock the mmap_lock while retrying, maybe NULL if caller * does not allow retry. If NULL, the caller must guarantee * that fault_flags does not contain FAULT_FLAG_ALLOW_RETRY. * @@ -1211,8 +1211,8 @@ static bool vma_permits_fault(struct vm_area_struct *vma, * such architectures, gup() will not be enough to make a subsequent access * succeed. * - * This function will not return with an unlocked mmap_sem. So it has not the - * same semantics wrt the @mm->mmap_sem as does filemap_fault(). + * This function will not return with an unlocked mmap_lock. So it has not the + * same semantics wrt the @mm->mmap_lock as does filemap_fault(). */ int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, unsigned long address, unsigned int fault_flags, @@ -1400,13 +1400,13 @@ retry: * @vma: target vma * @start: start address * @end: end address - * @locked: whether the mmap_sem is still held + * @locked: whether the mmap_lock is still held * * This takes care of mlocking the pages too if VM_LOCKED is set. * * return 0 on success, negative error code on error. * - * vma->vm_mm->mmap_sem must be held. + * vma->vm_mm->mmap_lock must be held. * * If @locked is NULL, it may be held for read or write and will * be unperturbed. @@ -1458,7 +1458,7 @@ long populate_vma_page_range(struct vm_area_struct *vma, * * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap * flags. VMAs must be already marked with the desired vm_flags, and - * mmap_sem must not be held. + * mmap_lock must not be held. */ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) { @@ -1525,7 +1525,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found - * allowing a hole to be left in the corefile to save diskspace. * - * Called without mmap_sem, but after all other threads have been killed. + * Called without mmap_lock, but after all other threads have been killed. */ #ifdef CONFIG_ELF_CORE struct page *get_dump_page(unsigned long addr) @@ -1886,9 +1886,9 @@ static long __get_user_pages_remote(struct task_struct *tsk, * * The caller is responsible for releasing returned @pages, via put_page(). * - * @vmas are valid only as long as mmap_sem is held. + * @vmas are valid only as long as mmap_lock is held. * - * Must be called with mmap_sem held for read or write. + * Must be called with mmap_lock held for read or write. * * get_user_pages_remote walks a process's page tables and takes a reference * to each struct page that each user address corresponds to at a given @@ -2873,7 +2873,7 @@ EXPORT_SYMBOL_GPL(get_user_pages_fast_only); * @pages: array that receives pointers to the pages pinned. * Should be at least nr_pages long. * - * Attempt to pin user pages in memory without taking mm->mmap_sem. + * Attempt to pin user pages in memory without taking mm->mmap_lock. * If not successful, it will fall back to taking the lock and * calling get_user_pages(). * diff --git a/mm/huge_memory.c b/mm/huge_memory.c index de201f0b5a4a..78c84bee7e29 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1746,7 +1746,7 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, /* * We don't have to worry about the ordering of src and dst - * ptlocks because exclusive mmap_sem prevents deadlock. + * ptlocks because exclusive mmap_lock prevents deadlock. */ old_ptl = __pmd_trans_huge_lock(old_pmd, vma); if (old_ptl) { @@ -2618,7 +2618,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) if (PageAnon(head)) { /* - * The caller does not necessarily hold an mmap_sem that would + * The caller does not necessarily hold an mmap_lock that would * prevent the anon_vma disappearing so we first we take a * reference to it and then lock the anon_vma for write. This * is similar to page_lock_anon_vma_read except the write lock diff --git a/mm/hugetlb.c b/mm/hugetlb.c index f102eb82f81d..57ece74e3aae 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4695,7 +4695,7 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, (const void __user *) src_addr, pages_per_huge_page(h), false); - /* fallback to copy_from_user outside mmap_sem */ + /* fallback to copy_from_user outside mmap_lock */ if (unlikely(ret)) { ret = -ENOENT; *pagep = page; diff --git a/mm/internal.h b/mm/internal.h index 3d3eb867030a..9886db20d94f 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -344,7 +344,7 @@ static inline void munlock_vma_pages_all(struct vm_area_struct *vma) } /* - * must be called with vma's mmap_sem held for read or write, and page locked. + * must be called with vma's mmap_lock held for read or write, and page locked. */ extern void mlock_vma_page(struct page *page); extern unsigned int munlock_vma_page(struct page *page); @@ -413,7 +413,7 @@ static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf, /* * FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or - * anything, so we only pin the file and drop the mmap_sem if only + * anything, so we only pin the file and drop the mmap_lock if only * FAULT_FLAG_ALLOW_RETRY is set, while this is the first attempt. */ if (fault_flag_allow_retry_first(flags) && diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 2c318ad1db20..b043c40a21d4 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -534,7 +534,7 @@ void __khugepaged_exit(struct mm_struct *mm) * under mmap sem read mode). Stop here (after we * return all pagetables will be destroyed) until * khugepaged has finished working on the pagetables - * under the mmap_sem. + * under the mmap_lock. */ mmap_write_lock(mm); mmap_write_unlock(mm); @@ -933,8 +933,8 @@ khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) #endif /* - * If mmap_sem temporarily dropped, revalidate vma - * before taking mmap_sem. + * If mmap_lock temporarily dropped, revalidate vma + * before taking mmap_lock. * Return 0 if succeeds, otherwise return none-zero * value (scan code). */ @@ -966,7 +966,7 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, * Only done if khugepaged_scan_pmd believes it is worthwhile. * * Called and returns without pte mapped or spinlocks held, - * but with mmap_sem held to protect against vma changes. + * but with mmap_lock held to protect against vma changes. */ static bool __collapse_huge_page_swapin(struct mm_struct *mm, @@ -993,7 +993,7 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm, swapped_in++; ret = do_swap_page(&vmf); - /* do_swap_page returns VM_FAULT_RETRY with released mmap_sem */ + /* do_swap_page returns VM_FAULT_RETRY with released mmap_lock */ if (ret & VM_FAULT_RETRY) { mmap_read_lock(mm); if (hugepage_vma_revalidate(mm, address, &vmf.vma)) { @@ -1047,9 +1047,9 @@ static void collapse_huge_page(struct mm_struct *mm, gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; /* - * Before allocating the hugepage, release the mmap_sem read lock. + * Before allocating the hugepage, release the mmap_lock read lock. * The allocation can take potentially a long time if it involves - * sync compaction, and we do not need to hold the mmap_sem during + * sync compaction, and we do not need to hold the mmap_lock during * that. We will recheck the vma after taking it again in write mode. */ mmap_read_unlock(mm); @@ -1080,8 +1080,8 @@ static void collapse_huge_page(struct mm_struct *mm, } /* - * __collapse_huge_page_swapin always returns with mmap_sem locked. - * If it fails, we release mmap_sem and jump out_nolock. + * __collapse_huge_page_swapin always returns with mmap_lock locked. + * If it fails, we release mmap_lock and jump out_nolock. * Continuing to collapse causes inconsistency. */ if (unmapped && !__collapse_huge_page_swapin(mm, vma, address, @@ -1345,7 +1345,7 @@ out_unmap: pte_unmap_unlock(pte, ptl); if (ret) { node = khugepaged_find_target_node(); - /* collapse_huge_page will return with the mmap_sem released */ + /* collapse_huge_page will return with the mmap_lock released */ collapse_huge_page(mm, address, hpage, node, referenced, unmapped); } @@ -1547,7 +1547,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) * later. * * Not that vma->anon_vma check is racy: it can be set up after - * the check but before we took mmap_sem by the fault path. + * the check but before we took mmap_lock by the fault path. * But page lock would prevent establishing any new ptes of the * page, so we are safe. * @@ -1567,10 +1567,10 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) if (!pmd) continue; /* - * We need exclusive mmap_sem to retract page table. + * We need exclusive mmap_lock to retract page table. * * We use trylock due to lock inversion: we need to acquire - * mmap_sem while holding page lock. Fault path does it in + * mmap_lock while holding page lock. Fault path does it in * reverse order. Trylock is a way to avoid deadlock. */ if (mmap_write_trylock(vma->vm_mm)) { @@ -2058,7 +2058,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, */ vma = NULL; if (unlikely(!mmap_read_trylock(mm))) - goto breakouterloop_mmap_sem; + goto breakouterloop_mmap_lock; if (likely(!khugepaged_test_exit(mm))) vma = find_vma(mm, khugepaged_scan.address); @@ -2115,15 +2115,15 @@ skip: khugepaged_scan.address += HPAGE_PMD_SIZE; progress += HPAGE_PMD_NR; if (ret) - /* we released mmap_sem so break loop */ - goto breakouterloop_mmap_sem; + /* we released mmap_lock so break loop */ + goto breakouterloop_mmap_lock; if (progress >= pages) goto breakouterloop; } } breakouterloop: mmap_read_unlock(mm); /* exit_mmap will destroy ptes after this */ -breakouterloop_mmap_sem: +breakouterloop_mmap_lock: spin_lock(&khugepaged_mm_lock); VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot); diff --git a/mm/ksm.c b/mm/ksm.c index 3efe7f28cc3f..4102034cd55a 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -442,7 +442,7 @@ static void insert_to_mm_slots_hash(struct mm_struct *mm, /* * ksmd, and unmerge_and_remove_all_rmap_items(), must not touch an mm's * page tables after it has passed through ksm_exit() - which, if necessary, - * takes mmap_sem briefly to serialize against them. ksm_exit() does not set + * takes mmap_lock briefly to serialize against them. ksm_exit() does not set * a special flag: they can just back out as soon as mm_users goes to zero. * ksm_test_exit() is used throughout to make this test for exit: in some * places for correctness, in some places just to avoid unnecessary work. @@ -831,7 +831,7 @@ static void remove_trailing_rmap_items(struct mm_slot *mm_slot, * Though it's very tempting to unmerge rmap_items from stable tree rather * than check every pte of a given vma, the locking doesn't quite work for * that - an rmap_item is assigned to the stable tree after inserting ksm - * page and upping mmap_sem. Nor does it fit with the way we skip dup'ing + * page and upping mmap_lock. Nor does it fit with the way we skip dup'ing * rmap_items from parent to child at fork time (so as not to waste time * if exit comes before the next scan reaches it). * @@ -1292,7 +1292,7 @@ static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item, /* Unstable nid is in union with stable anon_vma: remove first */ remove_rmap_item_from_tree(rmap_item); - /* Must get reference to anon_vma while still holding mmap_sem */ + /* Must get reference to anon_vma while still holding mmap_lock */ rmap_item->anon_vma = vma->anon_vma; get_anon_vma(vma->anon_vma); out: @@ -2343,13 +2343,13 @@ next_mm: struct mm_slot, mm_list); if (ksm_scan.address == 0) { /* - * We've completed a full scan of all vmas, holding mmap_sem + * We've completed a full scan of all vmas, holding mmap_lock * throughout, and found no VM_MERGEABLE: so do the same as * __ksm_exit does to remove this mm from all our lists now. * This applies either when cleaning up after __ksm_exit * (but beware: we can reach here even before __ksm_exit), * or when all VM_MERGEABLE areas have been unmapped (and - * mmap_sem then protects against race with MADV_MERGEABLE). + * mmap_lock then protects against race with MADV_MERGEABLE). */ hash_del(&slot->link); list_del(&slot->mm_list); @@ -2536,7 +2536,7 @@ void __ksm_exit(struct mm_struct *mm) * This process is exiting: if it's straightforward (as is the * case when ksmd was never running), free mm_slot immediately. * But if it's at the cursor or has rmap_items linked to it, use - * mmap_sem to synchronize with any break_cows before pagetables + * mmap_lock to synchronize with any break_cows before pagetables * are freed, and leave the mm_slot on the list for ksmd to free. * Beware: ksm may already have noticed it exiting and freed the slot. */ diff --git a/mm/maccess.c b/mm/maccess.c index 3ca8d97e5010..81df3b788284 100644 --- a/mm/maccess.c +++ b/mm/maccess.c @@ -40,9 +40,9 @@ probe_write_common(void __user *dst, const void *src, size_t size) * happens, handle that and return -EFAULT. * * We ensure that the copy_from_user is executed in atomic context so that - * do_page_fault() doesn't attempt to take mmap_sem. This makes + * do_page_fault() doesn't attempt to take mmap_lock. This makes * probe_kernel_read() suitable for use within regions where the caller - * already holds mmap_sem, or other locks which nest inside mmap_sem. + * already holds mmap_lock, or other locks which nest inside mmap_lock. * * probe_kernel_read_strict() is the same as probe_kernel_read() except for * the case where architectures have non-overlapping user and kernel address diff --git a/mm/madvise.c b/mm/madvise.c index d9b0f66b4396..dd1d43cf026d 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -40,7 +40,7 @@ struct madvise_walk_private { /* * Any behaviour which results in changes to the vma->vm_flags needs to - * take mmap_sem for writing. Others, which simply traverse vmas, need + * take mmap_lock for writing. Others, which simply traverse vmas, need * to only take it for reading. */ static int madvise_need_mmap_write(int behavior) @@ -165,7 +165,7 @@ static long madvise_behavior(struct vm_area_struct *vma, success: /* - * vm_flags is protected by the mmap_sem held in write mode. + * vm_flags is protected by the mmap_lock held in write mode. */ vma->vm_flags = new_flags; @@ -285,9 +285,9 @@ static long madvise_willneed(struct vm_area_struct *vma, * Filesystem's fadvise may need to take various locks. We need to * explicitly grab a reference because the vma (and hence the * vma's reference to the file) can go away as soon as we drop - * mmap_sem. + * mmap_lock. */ - *prev = NULL; /* tell sys_madvise we drop mmap_sem */ + *prev = NULL; /* tell sys_madvise we drop mmap_lock */ get_file(file); mmap_read_unlock(current->mm); offset = (loff_t)(start - vma->vm_start) @@ -768,7 +768,7 @@ static long madvise_dontneed_free(struct vm_area_struct *vma, return -EINVAL; if (!userfaultfd_remove(vma, start, end)) { - *prev = NULL; /* mmap_sem has been dropped, prev is stale */ + *prev = NULL; /* mmap_lock has been dropped, prev is stale */ mmap_read_lock(current->mm); vma = find_vma(current->mm, start); @@ -791,7 +791,7 @@ static long madvise_dontneed_free(struct vm_area_struct *vma, if (end > vma->vm_end) { /* * Don't fail if end > vma->vm_end. If the old - * vma was splitted while the mmap_sem was + * vma was splitted while the mmap_lock was * released the effect of the concurrent * operation may not cause madvise() to * have an undefined result. There may be an @@ -826,7 +826,7 @@ static long madvise_remove(struct vm_area_struct *vma, int error; struct file *f; - *prev = NULL; /* tell sys_madvise we drop mmap_sem */ + *prev = NULL; /* tell sys_madvise we drop mmap_lock */ if (vma->vm_flags & VM_LOCKED) return -EINVAL; @@ -847,11 +847,11 @@ static long madvise_remove(struct vm_area_struct *vma, * Filesystem's fallocate may need to take i_mutex. We need to * explicitly grab a reference because the vma (and hence the * vma's reference to the file) can go away as soon as we drop - * mmap_sem. + * mmap_lock. */ get_file(f); if (userfaultfd_remove(vma, start, end)) { - /* mmap_sem was not released by userfaultfd_remove() */ + /* mmap_lock was not released by userfaultfd_remove() */ mmap_read_unlock(current->mm); } error = vfs_fallocate(f, @@ -1153,7 +1153,7 @@ int do_madvise(unsigned long start, size_t len_in, int behavior) goto out; if (prev) vma = prev->vm_next; - else /* madvise_remove dropped mmap_sem */ + else /* madvise_remove dropped mmap_lock */ vma = find_vma(current->mm, start); } out: diff --git a/mm/memcontrol.c b/mm/memcontrol.c index ffae92d4e472..0b38b6ad547d 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5901,7 +5901,7 @@ static void mem_cgroup_move_charge(void) retry: if (unlikely(!mmap_read_trylock(mc.mm))) { /* - * Someone who are holding the mmap_sem might be waiting in + * Someone who are holding the mmap_lock might be waiting in * waitq. So we cancel all extra charges, wake up all waiters, * and retry. Because we cancel precharges, we might not be able * to move enough charges, but moving charge is a best-effort diff --git a/mm/memory.c b/mm/memory.c index 4e2e17bb1281..dc7f3543b1fd 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1185,7 +1185,7 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, * Here there can be other concurrent MADV_DONTNEED or * trans huge page faults running, and if the pmd is * none or trans huge it can change under us. This is - * because MADV_DONTNEED holds the mmap_sem in read + * because MADV_DONTNEED holds the mmap_lock in read * mode. */ if (pmd_none_or_trans_huge_or_clear_bad(pmd)) @@ -1636,7 +1636,7 @@ EXPORT_SYMBOL(vm_insert_pages); * The page does not need to be reserved. * * Usually this function is called from f_op->mmap() handler - * under mm->mmap_sem write-lock, so it can change vma->vm_flags. + * under mm->mmap_lock write-lock, so it can change vma->vm_flags. * Caller must set VM_MIXEDMAP on vma if it wants to call this * function from other places, for example from page-fault handler. * @@ -2573,7 +2573,7 @@ static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf) * mapping may be NULL here because some device drivers do not * set page.mapping but still dirty their pages * - * Drop the mmap_sem before waiting on IO, if we can. The file + * Drop the mmap_lock before waiting on IO, if we can. The file * is pinning the mapping, as per above. */ if ((dirtied || page_mkwrite) && mapping) { @@ -2623,7 +2623,7 @@ static inline void wp_page_reuse(struct vm_fault *vmf) /* * Handle the case of a page which we actually need to copy to a new page. * - * Called with mmap_sem locked and the old page referenced, but + * Called with mmap_lock locked and the old page referenced, but * without the ptl held. * * High level logic flow: @@ -2887,9 +2887,9 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf) * change only once the write actually happens. This avoids a few races, * and potentially makes it more efficient. * - * We enter with non-exclusive mmap_sem (to exclude vma changes, + * We enter with non-exclusive mmap_lock (to exclude vma changes, * but allow concurrent faults), with pte both mapped and locked. - * We return with mmap_sem still held, but pte unmapped and unlocked. + * We return with mmap_lock still held, but pte unmapped and unlocked. */ static vm_fault_t do_wp_page(struct vm_fault *vmf) __releases(vmf->ptl) @@ -3078,11 +3078,11 @@ void unmap_mapping_range(struct address_space *mapping, EXPORT_SYMBOL(unmap_mapping_range); /* - * We enter with non-exclusive mmap_sem (to exclude vma changes, + * We enter with non-exclusive mmap_lock (to exclude vma changes, * but allow concurrent faults), and pte mapped but not yet locked. * We return with pte unmapped and unlocked. * - * We return with the mmap_sem locked or unlocked in the same cases + * We return with the mmap_lock locked or unlocked in the same cases * as does filemap_fault(). */ vm_fault_t do_swap_page(struct vm_fault *vmf) @@ -3303,9 +3303,9 @@ out_release: } /* - * We enter with non-exclusive mmap_sem (to exclude vma changes, + * We enter with non-exclusive mmap_lock (to exclude vma changes, * but allow concurrent faults), and pte mapped but not yet locked. - * We return with mmap_sem still held, but pte unmapped and unlocked. + * We return with mmap_lock still held, but pte unmapped and unlocked. */ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) { @@ -3419,7 +3419,7 @@ oom: } /* - * The mmap_sem must have been held on entry, and may have been + * The mmap_lock must have been held on entry, and may have been * released depending on flags and vma->vm_ops->fault() return value. * See filemap_fault() and __lock_page_retry(). */ @@ -3928,11 +3928,11 @@ static vm_fault_t do_shared_fault(struct vm_fault *vmf) } /* - * We enter with non-exclusive mmap_sem (to exclude vma changes, + * We enter with non-exclusive mmap_lock (to exclude vma changes, * but allow concurrent faults). - * The mmap_sem may have been released depending on flags and our + * The mmap_lock may have been released depending on flags and our * return value. See filemap_fault() and __lock_page_or_retry(). - * If mmap_sem is released, vma may become invalid (for example + * If mmap_lock is released, vma may become invalid (for example * by other thread calling munmap()). */ static vm_fault_t do_fault(struct vm_fault *vmf) @@ -4161,10 +4161,10 @@ static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud) * with external mmu caches can use to update those (ie the Sparc or * PowerPC hashed page tables that act as extended TLBs). * - * We enter with non-exclusive mmap_sem (to exclude vma changes, but allow + * We enter with non-exclusive mmap_lock (to exclude vma changes, but allow * concurrent faults). * - * The mmap_sem may have been released depending on flags and our return value. + * The mmap_lock may have been released depending on flags and our return value. * See filemap_fault() and __lock_page_or_retry(). */ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) @@ -4186,7 +4186,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) /* * A regular pmd is established and it can't morph into a huge * pmd from under us anymore at this point because we hold the - * mmap_sem read mode and khugepaged takes it in write mode. + * mmap_lock read mode and khugepaged takes it in write mode. * So now it's safe to run pte_offset_map(). */ vmf->pte = pte_offset_map(vmf->pmd, vmf->address); @@ -4254,7 +4254,7 @@ unlock: /* * By the time we get here, we already hold the mm semaphore * - * The mmap_sem may have been released depending on flags and our + * The mmap_lock may have been released depending on flags and our * return value. See filemap_fault() and __lock_page_or_retry(). */ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, @@ -4349,7 +4349,7 @@ retry_pud: /* * By the time we get here, we already hold the mm semaphore * - * The mmap_sem may have been released depending on flags and our + * The mmap_lock may have been released depending on flags and our * return value. See filemap_fault() and __lock_page_or_retry(). */ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, @@ -4793,7 +4793,7 @@ void __might_fault(const char *file, int line) { /* * Some code (nfs/sunrpc) uses socket ops on kernel memory while - * holding the mmap_sem, this is safe because kernel memory doesn't + * holding the mmap_lock, this is safe because kernel memory doesn't * get paged out, therefore we'll never actually fault, and the * below annotations will generate false positives. */ diff --git a/mm/mempolicy.c b/mm/mempolicy.c index a38cd4cc3206..381320671677 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -224,7 +224,7 @@ static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes) * handle an empty nodemask with MPOL_PREFERRED here. * * Must be called holding task's alloc_lock to protect task's mems_allowed - * and mempolicy. May also be called holding the mmap_semaphore for write. + * and mempolicy. May also be called holding the mmap_lock for write. */ static int mpol_set_nodemask(struct mempolicy *pol, const nodemask_t *nodes, struct nodemask_scratch *nsc) @@ -368,7 +368,7 @@ static void mpol_rebind_preferred(struct mempolicy *pol, /* * mpol_rebind_policy - Migrate a policy to a different set of nodes * - * Per-vma policies are protected by mmap_sem. Allocations using per-task + * Per-vma policies are protected by mmap_lock. Allocations using per-task * policies are protected by task->mems_allowed_seq to prevent a premature * OOM/allocation failure due to parallel nodemask modification. */ @@ -398,7 +398,7 @@ void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new) /* * Rebind each vma in mm to new nodemask. * - * Call holding a reference to mm. Takes mm->mmap_sem during call. + * Call holding a reference to mm. Takes mm->mmap_lock during call. */ void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) @@ -764,7 +764,7 @@ queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end, /* * Apply policy to a single VMA - * This must be called with the mmap_sem held for writing. + * This must be called with the mmap_lock held for writing. */ static int vma_replace_policy(struct vm_area_struct *vma, struct mempolicy *pol) @@ -789,7 +789,7 @@ static int vma_replace_policy(struct vm_area_struct *vma, } old = vma->vm_policy; - vma->vm_policy = new; /* protected by mmap_sem */ + vma->vm_policy = new; /* protected by mmap_lock */ mpol_put(old); return 0; @@ -985,7 +985,7 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask, if (flags & MPOL_F_ADDR) { /* * Take a refcount on the mpol, lookup_node() - * wil drop the mmap_sem, so after calling + * wil drop the mmap_lock, so after calling * lookup_node() only "pol" remains valid, "vma" * is stale. */ diff --git a/mm/migrate.c b/mm/migrate.c index f69b09e0829c..f37729673558 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -2120,7 +2120,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, * pmd before doing set_pmd_at(), nor to flush the TLB after * set_pmd_at(). Clearing the pmd here would introduce a race * condition against MADV_DONTNEED, because MADV_DONTNEED only holds the - * mmap_sem for reading. If the pmd is set to NULL at any given time, + * mmap_lock for reading. If the pmd is set to NULL at any given time, * MADV_DONTNEED won't wait on the pmd lock and it'll skip clearing this * pmd. */ @@ -2675,7 +2675,7 @@ restore: * have the MIGRATE_PFN_MIGRATE flag set for their src array entry. * * It is safe to update device page table after migrate_vma_pages() because - * both destination and source page are still locked, and the mmap_sem is held + * both destination and source page are still locked, and the mmap_lock is held * in read mode (hence no one can unmap the range being migrated). * * Once the caller is done cleaning up things and updating its page table (if it diff --git a/mm/mlock.c b/mm/mlock.c index c5d806917526..f8736136fad7 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -49,7 +49,7 @@ EXPORT_SYMBOL(can_do_mlock); * When lazy mlocking via vmscan, it is important to ensure that the * vma's VM_LOCKED status is not concurrently being modified, otherwise we * may have mlocked a page that is being munlocked. So lazy mlock must take - * the mmap_sem for read, and verify that the vma really is locked + * the mmap_lock for read, and verify that the vma really is locked * (see mm/rmap.c). */ @@ -381,7 +381,7 @@ static unsigned long __munlock_pagevec_fill(struct pagevec *pvec, /* * Initialize pte walk starting at the already pinned page where we * are sure that there is a pte, as it was pinned under the same - * mmap_sem write op. + * mmap_lock write op. */ pte = get_locked_pte(vma->vm_mm, start, &ptl); /* Make sure we do not cross the page table boundary */ @@ -565,7 +565,7 @@ success: mm->locked_vm += nr_pages; /* - * vm_flags is protected by the mmap_sem held in write mode. + * vm_flags is protected by the mmap_lock held in write mode. * It's okay if try_to_unmap_one unmaps a page just after we * set VM_LOCKED, populate_vma_page_range will bring it back. */ diff --git a/mm/mmap.c b/mm/mmap.c index 79005049fbfc..59a4682ebf3f 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -132,7 +132,7 @@ void vma_set_page_prot(struct vm_area_struct *vma) vm_flags &= ~VM_SHARED; vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags); } - /* remove_protection_ptes reads vma->vm_page_prot without mmap_sem */ + /* remove_protection_ptes reads vma->vm_page_prot without mmap_lock */ WRITE_ONCE(vma->vm_page_prot, vm_page_prot); } @@ -238,14 +238,14 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) /* * Always allow shrinking brk. - * __do_munmap() may downgrade mmap_sem to read. + * __do_munmap() may downgrade mmap_lock to read. */ if (brk <= mm->brk) { int ret; /* - * mm->brk must to be protected by write mmap_sem so update it - * before downgrading mmap_sem. When __do_munmap() fails, + * mm->brk must to be protected by write mmap_lock so update it + * before downgrading mmap_lock. When __do_munmap() fails, * mm->brk will be restored from origbrk. */ mm->brk = brk; @@ -505,7 +505,7 @@ static __always_inline void vma_rb_erase(struct vm_area_struct *vma, * After the update, the vma will be reinserted using * anon_vma_interval_tree_post_update_vma(). * - * The entire update must be protected by exclusive mmap_sem and by + * The entire update must be protected by exclusive mmap_lock and by * the root anon_vma's mutex. */ static inline void @@ -2371,7 +2371,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) /* * vma->vm_start/vm_end cannot change under us because the caller - * is required to hold the mmap_sem in read mode. We need the + * is required to hold the mmap_lock in read mode. We need the * anon_vma lock to serialize against concurrent expand_stacks. */ anon_vma_lock_write(vma->anon_vma); @@ -2389,7 +2389,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) if (!error) { /* * vma_gap_update() doesn't support concurrent - * updates, but we only hold a shared mmap_sem + * updates, but we only hold a shared mmap_lock * lock here, so we need to protect against * concurrent vma expansions. * anon_vma_lock_write() doesn't help here, as @@ -2451,7 +2451,7 @@ int expand_downwards(struct vm_area_struct *vma, /* * vma->vm_start/vm_end cannot change under us because the caller - * is required to hold the mmap_sem in read mode. We need the + * is required to hold the mmap_lock in read mode. We need the * anon_vma lock to serialize against concurrent expand_stacks. */ anon_vma_lock_write(vma->anon_vma); @@ -2469,7 +2469,7 @@ int expand_downwards(struct vm_area_struct *vma, if (!error) { /* * vma_gap_update() doesn't support concurrent - * updates, but we only hold a shared mmap_sem + * updates, but we only hold a shared mmap_lock * lock here, so we need to protect against * concurrent vma expansions. * anon_vma_lock_write() doesn't help here, as @@ -2855,7 +2855,7 @@ static int __vm_munmap(unsigned long start, size_t len, bool downgrade) ret = __do_munmap(mm, start, len, &uf, downgrade); /* - * Returning 1 indicates mmap_sem is downgraded. + * Returning 1 indicates mmap_lock is downgraded. * But 1 is not legal return value of vm_munmap() and munmap(), reset * it to 0 before return. */ @@ -3107,12 +3107,12 @@ void exit_mmap(struct mm_struct *mm) /* * Manually reap the mm to free as much memory as possible. * Then, as the oom reaper does, set MMF_OOM_SKIP to disregard - * this mm from further consideration. Taking mm->mmap_sem for + * this mm from further consideration. Taking mm->mmap_lock for * write after setting MMF_OOM_SKIP will guarantee that the oom - * reaper will not run on this mm again after mmap_sem is + * reaper will not run on this mm again after mmap_lock is * dropped. * - * Nothing can be holding mm->mmap_sem here and the above call + * Nothing can be holding mm->mmap_lock here and the above call * to mmu_notifier_release(mm) ensures mmu notifier callbacks in * __oom_reap_task_mm() will not block. * @@ -3437,7 +3437,7 @@ bool vma_is_special_mapping(const struct vm_area_struct *vma, } /* - * Called with mm->mmap_sem held for writing. + * Called with mm->mmap_lock held for writing. * Insert a new vma covering the given region, with the given flags. * Its pages are supplied by the given array of struct page *. * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated. @@ -3513,11 +3513,11 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) * operations that could ever happen on a certain mm. This includes * vmtruncate, try_to_unmap, and all page faults. * - * The caller must take the mmap_sem in write mode before calling + * The caller must take the mmap_lock in write mode before calling * mm_take_all_locks(). The caller isn't allowed to release the - * mmap_sem until mm_drop_all_locks() returns. + * mmap_lock until mm_drop_all_locks() returns. * - * mmap_sem in write mode is required in order to block all operations + * mmap_lock in write mode is required in order to block all operations * that could modify pagetables and free pages without need of * altering the vma layout. It's also needed in write mode to avoid new * anon_vmas to be associated with existing vmas. @@ -3622,7 +3622,7 @@ static void vm_unlock_mapping(struct address_space *mapping) } /* - * The mmap_sem cannot be released by the caller until + * The mmap_lock cannot be released by the caller until * mm_drop_all_locks() returns. */ void mm_drop_all_locks(struct mm_struct *mm) diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index a3538cb2bcbe..03c33c93a582 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -301,7 +301,7 @@ void tlb_finish_mmu(struct mmu_gather *tlb, { /* * If there are parallel threads are doing PTE changes on same range - * under non-exclusive lock (e.g., mmap_sem read-side) but defer TLB + * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB * flush by batching, one thread may end up seeing inconsistent PTEs * and result in having stale TLB entries. So flush TLB forcefully * if we detect parallel PTE batching threads. diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 2f348b6c9c9a..352bb9f3ecc0 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -599,7 +599,7 @@ void __mmu_notifier_invalidate_range(struct mm_struct *mm, } /* - * Same as mmu_notifier_register but here the caller must hold the mmap_sem in + * Same as mmu_notifier_register but here the caller must hold the mmap_lock in * write mode. A NULL mn signals the notifier is being registered for itree * mode. */ @@ -623,7 +623,7 @@ int __mmu_notifier_register(struct mmu_notifier *subscription, /* * kmalloc cannot be called under mm_take_all_locks(), but we * know that mm->notifier_subscriptions can't change while we - * hold the write side of the mmap_sem. + * hold the write side of the mmap_lock. */ subscriptions = kzalloc( sizeof(struct mmu_notifier_subscriptions), GFP_KERNEL); @@ -655,7 +655,7 @@ int __mmu_notifier_register(struct mmu_notifier *subscription, * readers. acquire can only be used while holding the mmgrab or * mmget, and is safe because once created the * mmu_notifier_subscriptions is not freed until the mm is destroyed. - * As above, users holding the mmap_sem or one of the + * As above, users holding the mmap_lock or one of the * mm_take_all_locks() do not need to use acquire semantics. */ if (subscriptions) @@ -689,7 +689,7 @@ EXPORT_SYMBOL_GPL(__mmu_notifier_register); * @mn: The notifier to attach * @mm: The mm to attach the notifier to * - * Must not hold mmap_sem nor any other VM related lock when calling + * Must not hold mmap_lock nor any other VM related lock when calling * this registration function. Must also ensure mm_users can't go down * to zero while this runs to avoid races with mmu_notifier_release, * so mm has to be current->mm or the mm should be pinned safely such @@ -750,7 +750,7 @@ find_get_mmu_notifier(struct mm_struct *mm, const struct mmu_notifier_ops *ops) * are the same. * * Each call to mmu_notifier_get() must be paired with a call to - * mmu_notifier_put(). The caller must hold the write side of mm->mmap_sem. + * mmu_notifier_put(). The caller must hold the write side of mm->mmap_lock. * * While the caller has a mmu_notifier get the mm pointer will remain valid, * and can be converted to an active mm pointer via mmget_not_zero(). diff --git a/mm/mprotect.c b/mm/mprotect.c index ee27e44a7e07..ce8b8a5eacbb 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -49,7 +49,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; /* - * Can be called with only the mmap_sem for reading by + * Can be called with only the mmap_lock for reading by * prot_numa so we must check the pmd isn't constantly * changing from under us from pmd_none to pmd_trans_huge * and/or the other way around. @@ -59,7 +59,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, /* * The pmd points to a regular pte so the pmd can't change - * from under us even if the mmap_sem is only hold for + * from under us even if the mmap_lock is only hold for * reading. */ pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); @@ -228,7 +228,7 @@ static inline unsigned long change_pmd_range(struct vm_area_struct *vma, next = pmd_addr_end(addr, end); /* - * Automatic NUMA balancing walks the tables with mmap_sem + * Automatic NUMA balancing walks the tables with mmap_lock * held for read. It's possible a parallel update to occur * between pmd_trans_huge() and a pmd_none_or_clear_bad() * check leading to a false positive and clearing. @@ -477,7 +477,7 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, success: /* - * vm_flags and vm_page_prot are protected by the mmap_sem + * vm_flags and vm_page_prot are protected by the mmap_lock * held in write mode. */ vma->vm_flags = newflags; diff --git a/mm/mremap.c b/mm/mremap.c index 0a2f0efd2939..5dd572d57ca9 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -146,7 +146,7 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, /* * We don't have to worry about the ordering of src and dst - * pte locks because exclusive mmap_sem prevents deadlock. + * pte locks because exclusive mmap_lock prevents deadlock. */ old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl); new_pte = pte_offset_map(new_pmd, new_addr); @@ -213,7 +213,7 @@ static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr, /* * We don't have to worry about the ordering of src and dst - * ptlocks because exclusive mmap_sem prevents deadlock. + * ptlocks because exclusive mmap_lock prevents deadlock. */ old_ptl = pmd_lock(vma->vm_mm, old_pmd); new_ptl = pmd_lockptr(mm, new_pmd); @@ -710,7 +710,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, * Always allow a shrinking remap: that just unmaps * the unnecessary pages.. * __do_munmap does all the needed commit accounting, and - * downgrades mmap_sem to read if so directed. + * downgrades mmap_lock to read if so directed. */ if (old_len >= new_len) { int retval; @@ -720,7 +720,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, if (retval < 0 && old_len != new_len) { ret = retval; goto out; - /* Returning 1 indicates mmap_sem is downgraded to read. */ + /* Returning 1 indicates mmap_lock is downgraded to read. */ } else if (retval == 1) downgraded = true; ret = addr; diff --git a/mm/nommu.c b/mm/nommu.c index 8c3a04784dbe..cdcad5d61dd1 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -582,7 +582,7 @@ static void put_nommu_region(struct vm_region *region) * add a VMA into a process's mm_struct in the appropriate place in the list * and tree and add to the address space's page tree also if not an anonymous * page - * - should be called with mm->mmap_sem held writelocked + * - should be called with mm->mmap_lock held writelocked */ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) { @@ -696,7 +696,7 @@ static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma) /* * look up the first VMA in which addr resides, NULL if none - * - should be called with mm->mmap_sem at least held readlocked + * - should be called with mm->mmap_lock at least held readlocked */ struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) { @@ -742,7 +742,7 @@ int expand_stack(struct vm_area_struct *vma, unsigned long address) /* * look up the first VMA exactly that exactly matches addr - * - should be called with mm->mmap_sem at least held readlocked + * - should be called with mm->mmap_lock at least held readlocked */ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm, unsigned long addr, diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 3b5d78dfebe9..b4e9491cb320 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -898,7 +898,7 @@ static void __oom_kill_process(struct task_struct *victim, const char *message) /* * Kill all user processes sharing victim->mm in other thread groups, if * any. They don't get access to memory reserves, though, to avoid - * depletion of all memory. This prevents mm->mmap_sem livelock when an + * depletion of all memory. This prevents mm->mmap_lock livelock when an * oom killed thread cannot exit because it requires the semaphore and * its contended by another thread trying to allocate memory itself. * That thread will now get access to memory reserves since it has a diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 3b452aa05cd0..e81640d9f177 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -373,7 +373,7 @@ static int __walk_page_range(unsigned long start, unsigned long end, * caller-specific data to callbacks, @private should be helpful. * * Locking: - * Callers of walk_page_range() and walk_page_vma() should hold @mm->mmap_sem, + * Callers of walk_page_range() and walk_page_vma() should hold @mm->mmap_lock, * because these function traverse vma list and/or access to vma's data. */ int walk_page_range(struct mm_struct *mm, unsigned long start, @@ -498,11 +498,11 @@ int walk_page_vma(struct vm_area_struct *vma, const struct mm_walk_ops *ops, * Also see walk_page_range() for additional information. * * Locking: - * This function can't require that the struct mm_struct::mmap_sem is held, + * This function can't require that the struct mm_struct::mmap_lock is held, * since @mapping may be mapped by multiple processes. Instead * @mapping->i_mmap_rwsem must be held. This might have implications in the * callbacks, and it's up tho the caller to ensure that the - * struct mm_struct::mmap_sem is not needed. + * struct mm_struct::mmap_lock is not needed. * * Also this means that a caller can't rely on the struct * vm_area_struct::vm_flags to be constant across a call, diff --git a/mm/rmap.c b/mm/rmap.c index ad4a0fdcc94c..5fe2dedce1fc 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -21,7 +21,7 @@ * Lock ordering in mm: * * inode->i_mutex (while writing or truncating, not reading or faulting) - * mm->mmap_sem + * mm->mmap_lock * page->flags PG_locked (lock_page) * (see huegtlbfs below) * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share) * mapping->i_mmap_rwsem @@ -177,7 +177,7 @@ static void anon_vma_chain_link(struct vm_area_struct *vma, * to do any locking for the common case of already having * an anon_vma. * - * This must be called with the mmap_sem held for reading. + * This must be called with the mmap_lock held for reading. */ int __anon_vma_prepare(struct vm_area_struct *vma) { @@ -1444,7 +1444,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma, if (!PageTransCompound(page)) { /* * Holding pte lock, we do *not* need - * mmap_sem here + * mmap_lock here */ mlock_vma_page(page); } @@ -1817,7 +1817,7 @@ static struct anon_vma *rmap_walk_anon_lock(struct page *page, /* * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read() * because that depends on page_mapped(); but not all its usages - * are holding mmap_sem. Users without mmap_sem are required to + * are holding mmap_lock. Users without mmap_lock are required to * take a reference count to prevent the anon_vma disappearing */ anon_vma = page_anon_vma(page); @@ -1837,7 +1837,7 @@ static struct anon_vma *rmap_walk_anon_lock(struct page *page, * Find all the mappings of a page using the mapping pointer and the vma chains * contained in the anon_vma struct it points to. * - * When called from try_to_munlock(), the mmap_sem of the mm containing the vma + * When called from try_to_munlock(), the mmap_lock of the mm containing the vma * where the page was found will be held for write. So, we won't recheck * vm_flags for that VMA. That should be OK, because that vma shouldn't be * LOCKED. @@ -1889,7 +1889,7 @@ static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, * Find all the mappings of a page using the mapping pointer and the vma chains * contained in the address_space struct it points to. * - * When called from try_to_munlock(), the mmap_sem of the mm containing the vma + * When called from try_to_munlock(), the mmap_lock of the mm containing the vma * where the page was found will be held for write. So, we won't recheck * vm_flags for that VMA. That should be OK, because that vma shouldn't be * LOCKED. diff --git a/mm/shmem.c b/mm/shmem.c index 207fa263a410..a0dbe62f8042 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -2319,7 +2319,7 @@ static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, PAGE_SIZE); kunmap_atomic(page_kaddr); - /* fallback to copy_from_user outside mmap_sem */ + /* fallback to copy_from_user outside mmap_lock */ if (unlikely(ret)) { *pagep = page; shmem_inode_unacct_blocks(inode, 1); @@ -4136,7 +4136,7 @@ int shmem_zero_setup(struct vm_area_struct *vma) loff_t size = vma->vm_end - vma->vm_start; /* - * Cloning a new file under mmap_sem leads to a lock ordering conflict + * Cloning a new file under mmap_lock leads to a lock ordering conflict * between XFS directory reading and selinux: since this file is only * accessible to the user through its mapping, use S_PRIVATE flag to * bypass file security, in the same way as shmem_kernel_file_setup(). diff --git a/mm/swap_state.c b/mm/swap_state.c index 516e29d768c6..e98ff460e9e9 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -552,7 +552,7 @@ static unsigned long swapin_nr_pages(unsigned long offset) * This has been extended to use the NUMA policies from the mm triggering * the readahead. * - * Caller must hold read mmap_sem if vmf->vma is not NULL. + * Caller must hold read mmap_lock if vmf->vma is not NULL. */ struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, struct vm_fault *vmf) @@ -734,7 +734,7 @@ static void swap_ra_info(struct vm_fault *vmf, * Primitive swap readahead code. We simply read in a few pages whoes * virtual addresses are around the fault address in the same vma. * - * Caller must hold read mmap_sem if vmf->vma is not NULL. + * Caller must hold read mmap_lock if vmf->vma is not NULL. * */ static struct page *swap_vma_readahead(swp_entry_t fentry, gfp_t gfp_mask, diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 691613c5db9f..b80419320c7d 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -76,7 +76,7 @@ static int mcopy_atomic_pte(struct mm_struct *dst_mm, PAGE_SIZE); kunmap_atomic(page_kaddr); - /* fallback to copy_from_user outside mmap_sem */ + /* fallback to copy_from_user outside mmap_lock */ if (unlikely(ret)) { ret = -ENOENT; *pagep = page; @@ -200,7 +200,7 @@ static pmd_t *mm_alloc_pmd(struct mm_struct *mm, unsigned long address) #ifdef CONFIG_HUGETLB_PAGE /* * __mcopy_atomic processing for HUGETLB vmas. Note that this routine is - * called with mmap_sem held, it will release mmap_sem before returning. + * called with mmap_lock held, it will release mmap_lock before returning. */ static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma, @@ -247,7 +247,7 @@ static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm, retry: /* - * On routine entry dst_vma is set. If we had to drop mmap_sem and + * On routine entry dst_vma is set. If we had to drop mmap_lock and * retry, dst_vma will be set to NULL and we must lookup again. */ if (!dst_vma) { @@ -357,7 +357,7 @@ out: * private and shared mappings. See the routine * restore_reserve_on_error for details. Unfortunately, we * can not call restore_reserve_on_error now as it would - * require holding mmap_sem. + * require holding mmap_lock. * * If a reservation for the page existed in the reservation * map of a private mapping, the map was modified to indicate diff --git a/mm/util.c b/mm/util.c index 09f62d7d6e3e..c63c8e47be57 100644 --- a/mm/util.c +++ b/mm/util.c @@ -425,7 +425,7 @@ void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack) * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped * * Assumes @task and @mm are valid (i.e. at least one reference on each), and - * that mmap_sem is held as writer. + * that mmap_lock is held as writer. * * Return: * * 0 on success diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c index edde63a63007..e5ef20a0d05e 100644 --- a/security/keys/keyctl.c +++ b/security/keys/keyctl.c @@ -875,7 +875,7 @@ can_read_key: * * Allocating a temporary buffer to hold the keys before * transferring them to user buffer to avoid potential - * deadlock involving page fault and mmap_sem. + * deadlock involving page fault and mmap_lock. * * key_data_len = (buflen <= PAGE_SIZE) * ? buflen : actual length of key data diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c index 930def8201f4..68630244b00f 100644 --- a/sound/core/oss/pcm_oss.c +++ b/sound/core/oss/pcm_oss.c @@ -2876,7 +2876,7 @@ static int snd_pcm_oss_mmap(struct file *file, struct vm_area_struct *area) if (runtime->oss.params) { /* use mutex_trylock() for params_lock for avoiding a deadlock - * between mmap_sem and params_lock taken by + * between mmap_lock and params_lock taken by * copy_from/to_user() in snd_pcm_oss_write/read() */ err = snd_pcm_oss_change_params(substream, true); -- cgit v1.2.3-70-g09d2