diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/gup.c | 51 | ||||
| -rw-r--r-- | mm/mmap.c | 112 | ||||
| -rw-r--r-- | mm/mremap.c | 28 | ||||
| -rw-r--r-- | mm/nommu.c | 16 |
4 files changed, 114 insertions, 93 deletions
@@ -1091,6 +1091,45 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) return 0; } +/* + * This is "vma_lookup()", but with a warning if we would have + * historically expanded the stack in the GUP code. + */ +static struct vm_area_struct *gup_vma_lookup(struct mm_struct *mm, + unsigned long addr) +{ +#ifdef CONFIG_STACK_GROWSUP + return vma_lookup(mm, addr); +#else + static volatile unsigned long next_warn; + struct vm_area_struct *vma; + unsigned long now, next; + + vma = find_vma(mm, addr); + if (!vma || (addr >= vma->vm_start)) + return vma; + + /* Only warn for half-way relevant accesses */ + if (!(vma->vm_flags & VM_GROWSDOWN)) + return NULL; + if (vma->vm_start - addr > 65536) + return NULL; + + /* Let's not warn more than once an hour.. */ + now = jiffies; next = next_warn; + if (next && time_before(now, next)) + return NULL; + next_warn = now + 60*60*HZ; + + /* Let people know things may have changed. */ + pr_warn("GUP no longer grows the stack in %s (%d): %lx-%lx (%lx)\n", + current->comm, task_pid_nr(current), + vma->vm_start, vma->vm_end, addr); + dump_stack(); + return NULL; +#endif +} + /** * __get_user_pages() - pin user pages in memory * @mm: mm_struct of target mm @@ -1168,11 +1207,7 @@ static long __get_user_pages(struct mm_struct *mm, /* first iteration or cross vma bound */ if (!vma || start >= vma->vm_end) { - vma = find_vma(mm, start); - if (vma && (start < vma->vm_start)) { - WARN_ON_ONCE(vma->vm_flags & VM_GROWSDOWN); - vma = NULL; - } + vma = gup_vma_lookup(mm, start); if (!vma && in_gate_area(mm, start)) { ret = get_gate_page(mm, start & PAGE_MASK, gup_flags, &vma, @@ -1337,13 +1372,9 @@ int fixup_user_fault(struct mm_struct *mm, fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; retry: - vma = find_vma(mm, address); + vma = gup_vma_lookup(mm, address); if (!vma) return -EFAULT; - if (address < vma->vm_start ) { - WARN_ON_ONCE(vma->vm_flags & VM_GROWSDOWN); - return -EFAULT; - } if (!vma_permits_fault(vma, fault_flags)) return -EFAULT; diff --git a/mm/mmap.c b/mm/mmap.c index 3e5793ebbaae..204ddcd52625 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -193,8 +193,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) struct mm_struct *mm = current->mm; struct vm_area_struct *brkvma, *next = NULL; unsigned long min_brk; - bool populate; - bool downgraded = false; + bool populate = false; LIST_HEAD(uf); struct vma_iterator vmi; @@ -236,13 +235,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) goto success; } - /* - * Always allow shrinking brk. - * do_vma_munmap() may downgrade mmap_lock to read. - */ + /* Always allow shrinking brk. */ if (brk <= mm->brk) { - int ret; - /* Search one past newbrk */ vma_iter_init(&vmi, mm, newbrk); brkvma = vma_find(&vmi, oldbrk); @@ -250,19 +244,14 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) goto out; /* mapping intersects with an existing non-brk vma. */ /* * mm->brk must be protected by write mmap_lock. - * do_vma_munmap() may downgrade the lock, so update it + * do_vma_munmap() will drop the lock on success, so update it * before calling do_vma_munmap(). */ mm->brk = brk; - ret = do_vma_munmap(&vmi, brkvma, newbrk, oldbrk, &uf, true); - if (ret == 1) { - downgraded = true; - goto success; - } else if (!ret) - goto success; + if (do_vma_munmap(&vmi, brkvma, newbrk, oldbrk, &uf, true)) + goto out; - mm->brk = origbrk; - goto out; + goto success_unlocked; } if (check_brk_limits(oldbrk, newbrk - oldbrk)) @@ -283,19 +272,19 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) goto out; mm->brk = brk; + if (mm->def_flags & VM_LOCKED) + populate = true; success: - populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0; - if (downgraded) - mmap_read_unlock(mm); - else - mmap_write_unlock(mm); + mmap_write_unlock(mm); +success_unlocked: userfaultfd_unmap_complete(mm, &uf); if (populate) mm_populate(oldbrk, newbrk - oldbrk); return brk; out: + mm->brk = origbrk; mmap_write_unlock(mm); return origbrk; } @@ -2428,14 +2417,16 @@ int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, * @start: The aligned start address to munmap. * @end: The aligned end address to munmap. * @uf: The userfaultfd list_head - * @downgrade: Set to true to attempt a write downgrade of the mmap_lock + * @unlock: Set to true to drop the mmap_lock. unlocking only happens on + * success. * - * If @downgrade is true, check return code for potential release of the lock. + * Return: 0 on success and drops the lock if so directed, error and leaves the + * lock held otherwise. */ static int do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, struct mm_struct *mm, unsigned long start, - unsigned long end, struct list_head *uf, bool downgrade) + unsigned long end, struct list_head *uf, bool unlock) { struct vm_area_struct *prev, *next = NULL; struct maple_tree mt_detach; @@ -2551,33 +2542,23 @@ do_vmi_align_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, /* Point of no return */ mm->locked_vm -= locked_vm; mm->map_count -= count; - /* - * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or - * VM_GROWSUP VMA. Such VMAs can change their size under - * down_read(mmap_lock) and collide with the VMA we are about to unmap. - */ - if (downgrade) { - if (next && (next->vm_flags & VM_GROWSDOWN)) - downgrade = false; - else if (prev && (prev->vm_flags & VM_GROWSUP)) - downgrade = false; - else - mmap_write_downgrade(mm); - } + if (unlock) + mmap_write_downgrade(mm); /* * We can free page tables without write-locking mmap_lock because VMAs * were isolated before we downgraded mmap_lock. */ - unmap_region(mm, &mt_detach, vma, prev, next, start, end, !downgrade); + unmap_region(mm, &mt_detach, vma, prev, next, start, end, !unlock); /* Statistics and freeing VMAs */ mas_set(&mas_detach, start); remove_mt(mm, &mas_detach); __mt_destroy(&mt_detach); - - validate_mm(mm); - return downgrade ? 1 : 0; + if (unlock) + mmap_read_unlock(mm); + + return 0; clear_tree_failed: userfaultfd_error: @@ -2590,6 +2571,7 @@ end_split_failed: __mt_destroy(&mt_detach); start_split_failed: map_count_exceeded: + validate_mm(mm); return error; } @@ -2600,18 +2582,18 @@ map_count_exceeded: * @start: The start address to munmap * @len: The length of the range to munmap * @uf: The userfaultfd list_head - * @downgrade: set to true if the user wants to attempt to write_downgrade the - * mmap_lock + * @unlock: set to true if the user wants to drop the mmap_lock on success * * This function takes a @mas that is either pointing to the previous VMA or set * to MA_START and sets it up to remove the mapping(s). The @len will be * aligned and any arch_unmap work will be preformed. * - * Returns: -EINVAL on failure, 1 on success and unlock, 0 otherwise. + * Return: 0 on success and drops the lock if so directed, error and leaves the + * lock held otherwise. */ int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, - bool downgrade) + bool unlock) { unsigned long end; struct vm_area_struct *vma; @@ -2628,10 +2610,13 @@ int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, /* Find the first overlapping VMA */ vma = vma_find(vmi, end); - if (!vma) + if (!vma) { + if (unlock) + mmap_write_unlock(mm); return 0; + } - return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, downgrade); + return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock); } /* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls. @@ -2639,6 +2624,8 @@ int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, * @start: The start address to munmap * @len: The length to be munmapped. * @uf: The userfaultfd list_head + * + * Return: 0 on success, error otherwise. */ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf) @@ -2899,7 +2886,7 @@ unacct_error: return error; } -static int __vm_munmap(unsigned long start, size_t len, bool downgrade) +static int __vm_munmap(unsigned long start, size_t len, bool unlock) { int ret; struct mm_struct *mm = current->mm; @@ -2909,16 +2896,8 @@ static int __vm_munmap(unsigned long start, size_t len, bool downgrade) if (mmap_write_lock_killable(mm)) return -EINTR; - ret = do_vmi_munmap(&vmi, mm, start, len, &uf, downgrade); - /* - * Returning 1 indicates mmap_lock is downgraded. - * But 1 is not legal return value of vm_munmap() and munmap(), reset - * it to 0 before return. - */ - if (ret == 1) { - mmap_read_unlock(mm); - ret = 0; - } else + ret = do_vmi_munmap(&vmi, mm, start, len, &uf, unlock); + if (ret || !unlock) mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); @@ -3028,23 +3007,22 @@ out: * @start: the start of the address to unmap * @end: The end of the address to unmap * @uf: The userfaultfd list_head - * @downgrade: Attempt to downgrade or not + * @unlock: Drop the lock on success * - * Returns: 0 on success and not downgraded, 1 on success and downgraded. * unmaps a VMA mapping when the vma iterator is already in position. * Does not handle alignment. + * + * Return: 0 on success drops the lock of so directed, error on failure and will + * still hold the lock. */ int do_vma_munmap(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, - struct list_head *uf, bool downgrade) + unsigned long start, unsigned long end, struct list_head *uf, + bool unlock) { struct mm_struct *mm = vma->vm_mm; - int ret; arch_unmap(mm, start, end); - ret = do_vmi_align_munmap(vmi, vma, mm, start, end, uf, downgrade); - validate_mm(mm); - return ret; + return do_vmi_align_munmap(vmi, vma, mm, start, end, uf, unlock); } /* diff --git a/mm/mremap.c b/mm/mremap.c index fe6b722ae633..11e06e4ab33b 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -715,7 +715,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, } vma_iter_init(&vmi, mm, old_addr); - if (do_vmi_munmap(&vmi, mm, old_addr, old_len, uf_unmap, false) < 0) { + if (!do_vmi_munmap(&vmi, mm, old_addr, old_len, uf_unmap, false)) { /* OOM: unable to split vma, just get accounts right */ if (vm_flags & VM_ACCOUNT && !(flags & MREMAP_DONTUNMAP)) vm_acct_memory(old_len >> PAGE_SHIFT); @@ -913,7 +913,6 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, struct vm_area_struct *vma; unsigned long ret = -EINVAL; bool locked = false; - bool downgraded = false; struct vm_userfaultfd_ctx uf = NULL_VM_UFFD_CTX; LIST_HEAD(uf_unmap_early); LIST_HEAD(uf_unmap); @@ -999,24 +998,23 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, * Always allow a shrinking remap: that just unmaps * the unnecessary pages.. * do_vmi_munmap does all the needed commit accounting, and - * downgrades mmap_lock to read if so directed. + * unlocks the mmap_lock if so directed. */ if (old_len >= new_len) { - int retval; VMA_ITERATOR(vmi, mm, addr + new_len); - retval = do_vmi_munmap(&vmi, mm, addr + new_len, - old_len - new_len, &uf_unmap, true); - /* Returning 1 indicates mmap_lock is downgraded to read. */ - if (retval == 1) { - downgraded = true; - } else if (retval < 0 && old_len != new_len) { - ret = retval; + if (old_len == new_len) { + ret = addr; goto out; } + ret = do_vmi_munmap(&vmi, mm, addr + new_len, old_len - new_len, + &uf_unmap, true); + if (ret) + goto out; + ret = addr; - goto out; + goto out_unlocked; } /* @@ -1101,12 +1099,10 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, out: if (offset_in_page(ret)) locked = false; - if (downgraded) - mmap_read_unlock(current->mm); - else - mmap_write_unlock(current->mm); + mmap_write_unlock(current->mm); if (locked && new_len > old_len) mm_populate(new_addr + old_len, new_len - old_len); +out_unlocked: userfaultfd_unmap_complete(mm, &uf_unmap_early); mremap_userfaultfd_complete(&uf, addr, ret, old_len); userfaultfd_unmap_complete(mm, &uf_unmap); diff --git a/mm/nommu.c b/mm/nommu.c index 37d0b03143f1..c072a660ec2c 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -631,6 +631,22 @@ struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) EXPORT_SYMBOL(find_vma); /* + * At least xtensa ends up having protection faults even with no + * MMU.. No stack expansion, at least. + */ +struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm, + unsigned long addr, struct pt_regs *regs) +{ + struct vm_area_struct *vma; + + mmap_read_lock(mm); + vma = vma_lookup(mm, addr); + if (!vma) + mmap_read_unlock(mm); + return vma; +} + +/* * expand a stack to a given address * - not supported under NOMMU conditions */ |