mm/nommu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/mm/nommu.c
   4  *
   5  *  Replacement code for mm functions to support CPU's that don't
   6  *  have any form of memory management unit (thus no virtual memory).
   7  *
   8  *  See Documentation/nommu-mmap.txt
   9  *
  10  *  Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
  11  *  Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
  12  *  Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
  13  *  Copyright (c) 2002      Greg Ungerer <gerg@snapgear.com>
  14  *  Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org>
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/export.h>
  20 #include <linux/mm.h>
  21 #include <linux/sched/mm.h>
  22 #include <linux/vmacache.h>
  23 #include <linux/mman.h>
  24 #include <linux/swap.h>
  25 #include <linux/file.h>
  26 #include <linux/highmem.h>
  27 #include <linux/pagemap.h>
  28 #include <linux/slab.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/blkdev.h>
  31 #include <linux/backing-dev.h>
  32 #include <linux/compiler.h>
  33 #include <linux/mount.h>
  34 #include <linux/personality.h>
  35 #include <linux/security.h>
  36 #include <linux/syscalls.h>
  37 #include <linux/audit.h>
  38 #include <linux/printk.h>
  39
  40 #include <linux/uaccess.h>
  41 #include <asm/tlb.h>
  42 #include <asm/tlbflush.h>
  43 #include <asm/mmu_context.h>
  44 #include "internal.h"
  45
  46 void *high_memory;
  47 EXPORT_SYMBOL(high_memory);
  48 struct page *mem_map;
  49 unsigned long max_mapnr;
  50 EXPORT_SYMBOL(max_mapnr);
  51 unsigned long highest_memmap_pfn;
  52 int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
  53 int heap_stack_gap = 0;
  54
  55 atomic_long_t mmap_pages_allocated;
  56
  57 EXPORT_SYMBOL(mem_map);
  58
  59 /* list of mapped, potentially shareable regions */
  60 static struct kmem_cache *vm_region_jar;
  61 struct rb_root nommu_region_tree = RB_ROOT;
  62 DECLARE_RWSEM(nommu_region_sem);
  63
  64 const struct vm_operations_struct generic_file_vm_ops = {
  65 };
  66
  67 /*
  68  * Return the total memory allocated for this pointer, not
  69  * just what the caller asked for.
  70  *
  71  * Doesn't have to be accurate, i.e. may have races.
  72  */
  73 unsigned int kobjsize(const void *objp)
  74 {
  75         struct page *page;
  76
  77         /*
  78          * If the object we have should not have ksize performed on it,
  79          * return size of 0
  80          */
  81         if (!objp || !virt_addr_valid(objp))
  82                 return 0;
  83
  84         page = virt_to_head_page(objp);
  85
  86         /*
  87          * If the allocator sets PageSlab, we know the pointer came from
  88          * kmalloc().
  89          */
  90         if (PageSlab(page))
  91                 return ksize(objp);
  92
  93         /*
  94          * If it's not a compound page, see if we have a matching VMA
  95          * region. This test is intentionally done in reverse order,
  96          * so if there's no VMA, we still fall through and hand back
  97          * PAGE_SIZE for 0-order pages.
  98          */
  99         if (!PageCompound(page)) {
 100                 struct vm_area_struct *vma;
 101
 102                 vma = find_vma(current->mm, (unsigned long)objp);
 103                 if (vma)
 104                         return vma->vm_end - vma->vm_start;
 105         }
 106
 107         /*
 108          * The ksize() function is only guaranteed to work for pointers
 109          * returned by kmalloc(). So handle arbitrary pointers here.
 110          */
 111         return page_size(page);
 112 }
 113
 114 /**
 115  * follow_pfn - look up PFN at a user virtual address
 116  * @vma: memory mapping
 117  * @address: user virtual address
 118  * @pfn: location to store found PFN
 119  *
 120  * Only IO mappings and raw PFN mappings are allowed.
 121  *
 122  * Returns zero and the pfn at @pfn on success, -ve otherwise.
 123  */
 124 int follow_pfn(struct vm_area_struct *vma, unsigned long address,
 125         unsigned long *pfn)
 126 {
 127         if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
 128                 return -EINVAL;
 129
 130         *pfn = address >> PAGE_SHIFT;
 131         return 0;
 132 }
 133 EXPORT_SYMBOL(follow_pfn);
 134
 135 LIST_HEAD(vmap_area_list);
 136
 137 void vfree(const void *addr)
 138 {
 139         kfree(addr);
 140 }
 141 EXPORT_SYMBOL(vfree);
 142
 143 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
 144 {
 145         /*
 146          *  You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc()
 147          * returns only a logical address.
 148          */
 149         return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM);
 150 }
 151 EXPORT_SYMBOL(__vmalloc);
 152
 153 void *__vmalloc_node_flags(unsigned long size, int node, gfp_t flags)
 154 {
 155         return __vmalloc(size, flags, PAGE_KERNEL);
 156 }
 157
 158 static void *__vmalloc_user_flags(unsigned long size, gfp_t flags)
 159 {
 160         void *ret;
 161
 162         ret = __vmalloc(size, flags, PAGE_KERNEL);
 163         if (ret) {
 164                 struct vm_area_struct *vma;
 165
 166                 down_write(&current->mm->mmap_sem);
 167                 vma = find_vma(current->mm, (unsigned long)ret);
 168                 if (vma)
 169                         vma->vm_flags |= VM_USERMAP;
 170                 up_write(&current->mm->mmap_sem);
 171         }
 172
 173         return ret;
 174 }
 175
 176 void *vmalloc_user(unsigned long size)
 177 {
 178         return __vmalloc_user_flags(size, GFP_KERNEL | __GFP_ZERO);
 179 }
 180 EXPORT_SYMBOL(vmalloc_user);
 181
 182 void *vmalloc_user_node_flags(unsigned long size, int node, gfp_t flags)
 183 {
 184         return __vmalloc_user_flags(size, flags | __GFP_ZERO);
 185 }
 186 EXPORT_SYMBOL(vmalloc_user_node_flags);
 187
 188 struct page *vmalloc_to_page(const void *addr)
 189 {
 190         return virt_to_page(addr);
 191 }
 192 EXPORT_SYMBOL(vmalloc_to_page);
 193
 194 unsigned long vmalloc_to_pfn(const void *addr)
 195 {
 196         return page_to_pfn(virt_to_page(addr));
 197 }
 198 EXPORT_SYMBOL(vmalloc_to_pfn);
 199
 200 long vread(char *buf, char *addr, unsigned long count)
 201 {
 202         /* Don't allow overflow */
 203         if ((unsigned long) buf + count < count)
 204                 count = -(unsigned long) buf;
 205
 206         memcpy(buf, addr, count);
 207         return count;
 208 }
 209
 210 long vwrite(char *buf, char *addr, unsigned long count)
 211 {
 212         /* Don't allow overflow */
 213         if ((unsigned long) addr + count < count)
 214                 count = -(unsigned long) addr;
 215
 216         memcpy(addr, buf, count);
 217         return count;
 218 }
 219
 220 /*
 221  *      vmalloc  -  allocate virtually contiguous memory
 222  *
 223  *      @size:          allocation size
 224  *
 225  *      Allocate enough pages to cover @size from the page level
 226  *      allocator and map them into contiguous kernel virtual space.
 227  *
 228  *      For tight control over page level allocator and protection flags
 229  *      use __vmalloc() instead.
 230  */
 231 void *vmalloc(unsigned long size)
 232 {
 233        return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
 234 }
 235 EXPORT_SYMBOL(vmalloc);
 236
 237 /*
 238  *      vzalloc - allocate virtually contiguous memory with zero fill
 239  *
 240  *      @size:          allocation size
 241  *
 242  *      Allocate enough pages to cover @size from the page level
 243  *      allocator and map them into contiguous kernel virtual space.
 244  *      The memory allocated is set to zero.
 245  *
 246  *      For tight control over page level allocator and protection flags
 247  *      use __vmalloc() instead.
 248  */
 249 void *vzalloc(unsigned long size)
 250 {
 251         return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
 252                         PAGE_KERNEL);
 253 }
 254 EXPORT_SYMBOL(vzalloc);
 255
 256 /**
 257  * vmalloc_node - allocate memory on a specific node
 258  * @size:       allocation size
 259  * @node:       numa node
 260  *
 261  * Allocate enough pages to cover @size from the page level
 262  * allocator and map them into contiguous kernel virtual space.
 263  *
 264  * For tight control over page level allocator and protection flags
 265  * use __vmalloc() instead.
 266  */
 267 void *vmalloc_node(unsigned long size, int node)
 268 {
 269         return vmalloc(size);
 270 }
 271 EXPORT_SYMBOL(vmalloc_node);
 272
 273 /**
 274  * vzalloc_node - allocate memory on a specific node with zero fill
 275  * @size:       allocation size
 276  * @node:       numa node
 277  *
 278  * Allocate enough pages to cover @size from the page level
 279  * allocator and map them into contiguous kernel virtual space.
 280  * The memory allocated is set to zero.
 281  *
 282  * For tight control over page level allocator and protection flags
 283  * use __vmalloc() instead.
 284  */
 285 void *vzalloc_node(unsigned long size, int node)
 286 {
 287         return vzalloc(size);
 288 }
 289 EXPORT_SYMBOL(vzalloc_node);
 290
 291 /**
 292  *      vmalloc_exec  -  allocate virtually contiguous, executable memory
 293  *      @size:          allocation size
 294  *
 295  *      Kernel-internal function to allocate enough pages to cover @size
 296  *      the page level allocator and map them into contiguous and
 297  *      executable kernel virtual space.
 298  *
 299  *      For tight control over page level allocator and protection flags
 300  *      use __vmalloc() instead.
 301  */
 302
 303 void *vmalloc_exec(unsigned long size)
 304 {
 305         return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
 306 }
 307
 308 /**
 309  * vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
 310  *      @size:          allocation size
 311  *
 312  *      Allocate enough 32bit PA addressable pages to cover @size from the
 313  *      page level allocator and map them into contiguous kernel virtual space.
 314  */
 315 void *vmalloc_32(unsigned long size)
 316 {
 317         return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
 318 }
 319 EXPORT_SYMBOL(vmalloc_32);
 320
 321 /**
 322  * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
 323  *      @size:          allocation size
 324  *
 325  * The resulting memory area is 32bit addressable and zeroed so it can be
 326  * mapped to userspace without leaking data.
 327  *
 328  * VM_USERMAP is set on the corresponding VMA so that subsequent calls to
 329  * remap_vmalloc_range() are permissible.
 330  */
 331 void *vmalloc_32_user(unsigned long size)
 332 {
 333         /*
 334          * We'll have to sort out the ZONE_DMA bits for 64-bit,
 335          * but for now this can simply use vmalloc_user() directly.
 336          */
 337         return vmalloc_user(size);
 338 }
 339 EXPORT_SYMBOL(vmalloc_32_user);
 340
 341 void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot)
 342 {
 343         BUG();
 344         return NULL;
 345 }
 346 EXPORT_SYMBOL(vmap);
 347
 348 void vunmap(const void *addr)
 349 {
 350         BUG();
 351 }
 352 EXPORT_SYMBOL(vunmap);
 353
 354 void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
 355 {
 356         BUG();
 357         return NULL;
 358 }
 359 EXPORT_SYMBOL(vm_map_ram);
 360
 361 void vm_unmap_ram(const void *mem, unsigned int count)
 362 {
 363         BUG();
 364 }
 365 EXPORT_SYMBOL(vm_unmap_ram);
 366
 367 void vm_unmap_aliases(void)
 368 {
 369 }
 370 EXPORT_SYMBOL_GPL(vm_unmap_aliases);
 371
 372 /*
 373  * Implement a stub for vmalloc_sync_all() if the architecture chose not to
 374  * have one.
 375  */
 376 void __weak vmalloc_sync_all(void)
 377 {
 378 }
 379
 380 struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
 381 {
 382         BUG();
 383         return NULL;
 384 }
 385 EXPORT_SYMBOL_GPL(alloc_vm_area);
 386
 387 void free_vm_area(struct vm_struct *area)
 388 {
 389         BUG();
 390 }
 391 EXPORT_SYMBOL_GPL(free_vm_area);
 392
 393 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
 394                    struct page *page)
 395 {
 396         return -EINVAL;
 397 }
 398 EXPORT_SYMBOL(vm_insert_page);
 399
 400 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
 401                         unsigned long num)
 402 {
 403         return -EINVAL;
 404 }
 405 EXPORT_SYMBOL(vm_map_pages);
 406
 407 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
 408                                 unsigned long num)
 409 {
 410         return -EINVAL;
 411 }
 412 EXPORT_SYMBOL(vm_map_pages_zero);
 413
 414 /*
 415  *  sys_brk() for the most part doesn't need the global kernel
 416  *  lock, except when an application is doing something nasty
 417  *  like trying to un-brk an area that has already been mapped
 418  *  to a regular file.  in this case, the unmapping will need
 419  *  to invoke file system routines that need the global lock.
 420  */
 421 SYSCALL_DEFINE1(brk, unsigned long, brk)
 422 {
 423         struct mm_struct *mm = current->mm;
 424
 425         if (brk < mm->start_brk || brk > mm->context.end_brk)
 426                 return mm->brk;
 427
 428         if (mm->brk == brk)
 429                 return mm->brk;
 430
 431         /*
 432          * Always allow shrinking brk
 433          */
 434         if (brk <= mm->brk) {
 435                 mm->brk = brk;
 436                 return brk;
 437         }
 438
 439         /*
 440          * Ok, looks good - let it rip.
 441          */
 442         flush_icache_range(mm->brk, brk);
 443         return mm->brk = brk;
 444 }
 445
 446 /*
 447  * initialise the percpu counter for VM and region record slabs
 448  */
 449 void __init mmap_init(void)
 450 {
 451         int ret;
 452
 453         ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
 454         VM_BUG_ON(ret);
 455         vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC|SLAB_ACCOUNT);
 456 }
 457
 458 /*
 459  * validate the region tree
 460  * - the caller must hold the region lock
 461  */
 462 #ifdef CONFIG_DEBUG_NOMMU_REGIONS
 463 static noinline void validate_nommu_regions(void)
 464 {
 465         struct vm_region *region, *last;
 466         struct rb_node *p, *lastp;
 467
 468         lastp = rb_first(&nommu_region_tree);
 469         if (!lastp)
 470                 return;
 471
 472         last = rb_entry(lastp, struct vm_region, vm_rb);
 473         BUG_ON(last->vm_end <= last->vm_start);
 474         BUG_ON(last->vm_top < last->vm_end);
 475
 476         while ((p = rb_next(lastp))) {
 477                 region = rb_entry(p, struct vm_region, vm_rb);
 478                 last = rb_entry(lastp, struct vm_region, vm_rb);
 479
 480                 BUG_ON(region->vm_end <= region->vm_start);
 481                 BUG_ON(region->vm_top < region->vm_end);
 482                 BUG_ON(region->vm_start < last->vm_top);
 483
 484                 lastp = p;
 485         }
 486 }
 487 #else
 488 static void validate_nommu_regions(void)
 489 {
 490 }
 491 #endif
 492
 493 /*
 494  * add a region into the global tree
 495  */
 496 static void add_nommu_region(struct vm_region *region)
 497 {
 498         struct vm_region *pregion;
 499         struct rb_node **p, *parent;
 500
 501         validate_nommu_regions();
 502
 503         parent = NULL;
 504         p = &nommu_region_tree.rb_node;
 505         while (*p) {
 506                 parent = *p;
 507                 pregion = rb_entry(parent, struct vm_region, vm_rb);
 508                 if (region->vm_start < pregion->vm_start)
 509                         p = &(*p)->rb_left;
 510                 else if (region->vm_start > pregion->vm_start)
 511                         p = &(*p)->rb_right;
 512                 else if (pregion == region)
 513                         return;
 514                 else
 515                         BUG();
 516         }
 517
 518         rb_link_node(&region->vm_rb, parent, p);
 519         rb_insert_color(&region->vm_rb, &nommu_region_tree);
 520
 521         validate_nommu_regions();
 522 }
 523
 524 /*
 525  * delete a region from the global tree
 526  */
 527 static void delete_nommu_region(struct vm_region *region)
 528 {
 529         BUG_ON(!nommu_region_tree.rb_node);
 530
 531         validate_nommu_regions();
 532         rb_erase(&region->vm_rb, &nommu_region_tree);
 533         validate_nommu_regions();
 534 }
 535
 536 /*
 537  * free a contiguous series of pages
 538  */
 539 static void free_page_series(unsigned long from, unsigned long to)
 540 {
 541         for (; from < to; from += PAGE_SIZE) {
 542                 struct page *page = virt_to_page(from);
 543
 544                 atomic_long_dec(&mmap_pages_allocated);
 545                 put_page(page);
 546         }
 547 }
 548
 549 /*
 550  * release a reference to a region
 551  * - the caller must hold the region semaphore for writing, which this releases
 552  * - the region may not have been added to the tree yet, in which case vm_top
 553  *   will equal vm_start
 554  */
 555 static void __put_nommu_region(struct vm_region *region)
 556         __releases(nommu_region_sem)
 557 {
 558         BUG_ON(!nommu_region_tree.rb_node);
 559
 560         if (--region->vm_usage == 0) {
 561                 if (region->vm_top > region->vm_start)
 562                         delete_nommu_region(region);
 563                 up_write(&nommu_region_sem);
 564
 565                 if (region->vm_file)
 566                         fput(region->vm_file);
 567
 568                 /* IO memory and memory shared directly out of the pagecache
 569                  * from ramfs/tmpfs mustn't be released here */
 570                 if (region->vm_flags & VM_MAPPED_COPY)
 571                         free_page_series(region->vm_start, region->vm_top);
 572                 kmem_cache_free(vm_region_jar, region);
 573         } else {
 574                 up_write(&nommu_region_sem);
 575         }
 576 }
 577
 578 /*
 579  * release a reference to a region
 580  */
 581 static void put_nommu_region(struct vm_region *region)
 582 {
 583         down_write(&nommu_region_sem);
 584         __put_nommu_region(region);
 585 }
 586
 587 /*
 588  * add a VMA into a process's mm_struct in the appropriate place in the list
 589  * and tree and add to the address space's page tree also if not an anonymous
 590  * page
 591  * - should be called with mm->mmap_sem held writelocked
 592  */
 593 static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
 594 {
 595         struct vm_area_struct *pvma, *prev;
 596         struct address_space *mapping;
 597         struct rb_node **p, *parent, *rb_prev;
 598
 599         BUG_ON(!vma->vm_region);
 600
 601         mm->map_count++;
 602         vma->vm_mm = mm;
 603
 604         /* add the VMA to the mapping */
 605         if (vma->vm_file) {
 606                 mapping = vma->vm_file->f_mapping;
 607
 608                 i_mmap_lock_write(mapping);
 609                 flush_dcache_mmap_lock(mapping);
 610                 vma_interval_tree_insert(vma, &mapping->i_mmap);
 611                 flush_dcache_mmap_unlock(mapping);
 612                 i_mmap_unlock_write(mapping);
 613         }
 614
 615         /* add the VMA to the tree */
 616         parent = rb_prev = NULL;
 617         p = &mm->mm_rb.rb_node;
 618         while (*p) {
 619                 parent = *p;
 620                 pvma = rb_entry(parent, struct vm_area_struct, vm_rb);
 621
 622                 /* sort by: start addr, end addr, VMA struct addr in that order
 623                  * (the latter is necessary as we may get identical VMAs) */
 624                 if (vma->vm_start < pvma->vm_start)
 625                         p = &(*p)->rb_left;
 626                 else if (vma->vm_start > pvma->vm_start) {
 627                         rb_prev = parent;
 628                         p = &(*p)->rb_right;
 629                 } else if (vma->vm_end < pvma->vm_end)
 630                         p = &(*p)->rb_left;
 631                 else if (vma->vm_end > pvma->vm_end) {
 632                         rb_prev = parent;
 633                         p = &(*p)->rb_right;
 634                 } else if (vma < pvma)
 635                         p = &(*p)->rb_left;
 636                 else if (vma > pvma) {
 637                         rb_prev = parent;
 638                         p = &(*p)->rb_right;
 639                 } else
 640                         BUG();
 641         }
 642
 643         rb_link_node(&vma->vm_rb, parent, p);
 644         rb_insert_color(&vma->vm_rb, &mm->mm_rb);
 645
 646         /* add VMA to the VMA list also */
 647         prev = NULL;
 648         if (rb_prev)
 649                 prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
 650
 651         __vma_link_list(mm, vma, prev, parent);
 652 }
 653
 654 /*
 655  * delete a VMA from its owning mm_struct and address space
 656  */
 657 static void delete_vma_from_mm(struct vm_area_struct *vma)
 658 {
 659         int i;
 660         struct address_space *mapping;
 661         struct mm_struct *mm = vma->vm_mm;
 662         struct task_struct *curr = current;
 663
 664         mm->map_count--;
 665         for (i = 0; i < VMACACHE_SIZE; i++) {
 666                 /* if the vma is cached, invalidate the entire cache */
 667                 if (curr->vmacache.vmas[i] == vma) {
 668                         vmacache_invalidate(mm);
 669                         break;
 670                 }
 671         }
 672
 673         /* remove the VMA from the mapping */
 674         if (vma->vm_file) {
 675                 mapping = vma->vm_file->f_mapping;
 676
 677                 i_mmap_lock_write(mapping);
 678                 flush_dcache_mmap_lock(mapping);
 679                 vma_interval_tree_remove(vma, &mapping->i_mmap);
 680                 flush_dcache_mmap_unlock(mapping);
 681                 i_mmap_unlock_write(mapping);
 682         }
 683
 684         /* remove from the MM's tree and list */
 685         rb_erase(&vma->vm_rb, &mm->mm_rb);
 686
 687         if (vma->vm_prev)
 688                 vma->vm_prev->vm_next = vma->vm_next;
 689         else
 690                 mm->mmap = vma->vm_next;
 691
 692         if (vma->vm_next)
 693                 vma->vm_next->vm_prev = vma->vm_prev;
 694 }
 695
 696 /*
 697  * destroy a VMA record
 698  */
 699 static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma)
 700 {
 701         if (vma->vm_ops && vma->vm_ops->close)
 702                 vma->vm_ops->close(vma);
 703         if (vma->vm_file)
 704                 fput(vma->vm_file);
 705         put_nommu_region(vma->vm_region);
 706         vm_area_free(vma);
 707 }
 708
 709 /*
 710  * look up the first VMA in which addr resides, NULL if none
 711  * - should be called with mm->mmap_sem at least held readlocked
 712  */
 713 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
 714 {
 715         struct vm_area_struct *vma;
 716
 717         /* check the cache first */
 718         vma = vmacache_find(mm, addr);
 719         if (likely(vma))
 720                 return vma;
 721
 722         /* trawl the list (there may be multiple mappings in which addr
 723          * resides) */
 724         for (vma = mm->mmap; vma; vma = vma->vm_next) {
 725                 if (vma->vm_start > addr)
 726                         return NULL;
 727                 if (vma->vm_end > addr) {
 728                         vmacache_update(addr, vma);
 729                         return vma;
 730                 }
 731         }
 732
 733         return NULL;
 734 }
 735 EXPORT_SYMBOL(find_vma);
 736
 737 /*
 738  * find a VMA
 739  * - we don't extend stack VMAs under NOMMU conditions
 740  */
 741 struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
 742 {
 743         return find_vma(mm, addr);
 744 }
 745
 746 /*
 747  * expand a stack to a given address
 748  * - not supported under NOMMU conditions
 749  */
 750 int expand_stack(struct vm_area_struct *vma, unsigned long address)
 751 {
 752         return -ENOMEM;
 753 }
 754
 755 /*
 756  * look up the first VMA exactly that exactly matches addr
 757  * - should be called with mm->mmap_sem at least held readlocked
 758  */
 759 static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
 760                                              unsigned long addr,
 761                                              unsigned long len)
 762 {
 763         struct vm_area_struct *vma;
 764         unsigned long end = addr + len;
 765
 766         /* check the cache first */
 767         vma = vmacache_find_exact(mm, addr, end);
 768         if (vma)
 769                 return vma;
 770
 771         /* trawl the list (there may be multiple mappings in which addr
 772          * resides) */
 773         for (vma = mm->mmap; vma; vma = vma->vm_next) {
 774                 if (vma->vm_start < addr)
 775                         continue;
 776                 if (vma->vm_start > addr)
 777                         return NULL;
 778                 if (vma->vm_end == end) {
 779                         vmacache_update(addr, vma);
 780                         return vma;
 781                 }
 782         }
 783
 784         return NULL;
 785 }
 786
 787 /*
 788  * determine whether a mapping should be permitted and, if so, what sort of
 789  * mapping we're capable of supporting
 790  */
 791 static int validate_mmap_request(struct file *file,
 792                                  unsigned long addr,
 793                                  unsigned long len,
 794                                  unsigned long prot,
 795                                  unsigned long flags,
 796                                  unsigned long pgoff,
 797                                  unsigned long *_capabilities)
 798 {
 799         unsigned long capabilities, rlen;
 800         int ret;
 801
 802         /* do the simple checks first */
 803         if (flags & MAP_FIXED)
 804                 return -EINVAL;
 805
 806         if ((flags & MAP_TYPE) != MAP_PRIVATE &&
 807             (flags & MAP_TYPE) != MAP_SHARED)
 808                 return -EINVAL;
 809
 810         if (!len)
 811                 return -EINVAL;
 812
 813         /* Careful about overflows.. */
 814         rlen = PAGE_ALIGN(len);
 815         if (!rlen || rlen > TASK_SIZE)
 816                 return -ENOMEM;
 817
 818         /* offset overflow? */
 819         if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
 820                 return -EOVERFLOW;
 821
 822         if (file) {
 823                 /* files must support mmap */
 824                 if (!file->f_op->mmap)
 825                         return -ENODEV;
 826
 827                 /* work out if what we've got could possibly be shared
 828                  * - we support chardevs that provide their own "memory"
 829                  * - we support files/blockdevs that are memory backed
 830                  */
 831                 if (file->f_op->mmap_capabilities) {
 832                         capabilities = file->f_op->mmap_capabilities(file);
 833                 } else {
 834                         /* no explicit capabilities set, so assume some
 835                          * defaults */
 836                         switch (file_inode(file)->i_mode & S_IFMT) {
 837                         case S_IFREG:
 838                         case S_IFBLK:
 839                                 capabilities = NOMMU_MAP_COPY;
 840                                 break;
 841
 842                         case S_IFCHR:
 843                                 capabilities =
 844                                         NOMMU_MAP_DIRECT |
 845                                         NOMMU_MAP_READ |
 846                                         NOMMU_MAP_WRITE;
 847                                 break;
 848
 849                         default:
 850                                 return -EINVAL;
 851                         }
 852                 }
 853
 854                 /* eliminate any capabilities that we can't support on this
 855                  * device */
 856                 if (!file->f_op->get_unmapped_area)
 857                         capabilities &= ~NOMMU_MAP_DIRECT;
 858                 if (!(file->f_mode & FMODE_CAN_READ))
 859                         capabilities &= ~NOMMU_MAP_COPY;
 860
 861                 /* The file shall have been opened with read permission. */
 862                 if (!(file->f_mode & FMODE_READ))
 863                         return -EACCES;
 864
 865                 if (flags & MAP_SHARED) {
 866                         /* do checks for writing, appending and locking */
 867                         if ((prot & PROT_WRITE) &&
 868                             !(file->f_mode & FMODE_WRITE))
 869                                 return -EACCES;
 870
 871                         if (IS_APPEND(file_inode(file)) &&
 872                             (file->f_mode & FMODE_WRITE))
 873                                 return -EACCES;
 874
 875                         if (locks_verify_locked(file))
 876                                 return -EAGAIN;
 877
 878                         if (!(capabilities & NOMMU_MAP_DIRECT))
 879                                 return -ENODEV;
 880
 881                         /* we mustn't privatise shared mappings */
 882                         capabilities &= ~NOMMU_MAP_COPY;
 883                 } else {
 884                         /* we're going to read the file into private memory we
 885                          * allocate */
 886                         if (!(capabilities & NOMMU_MAP_COPY))
 887                                 return -ENODEV;
 888
 889                         /* we don't permit a private writable mapping to be
 890                          * shared with the backing device */
 891                         if (prot & PROT_WRITE)
 892                                 capabilities &= ~NOMMU_MAP_DIRECT;
 893                 }
 894
 895                 if (capabilities & NOMMU_MAP_DIRECT) {
 896                         if (((prot & PROT_READ)  && !(capabilities & NOMMU_MAP_READ))  ||
 897                             ((prot & PROT_WRITE) && !(capabilities & NOMMU_MAP_WRITE)) ||
 898                             ((prot & PROT_EXEC)  && !(capabilities & NOMMU_MAP_EXEC))
 899                             ) {
 900                                 capabilities &= ~NOMMU_MAP_DIRECT;
 901                                 if (flags & MAP_SHARED) {
 902                                         pr_warn("MAP_SHARED not completely supported on !MMU\n");
 903                                         return -EINVAL;
 904                                 }
 905                         }
 906                 }
 907
 908                 /* handle executable mappings and implied executable
 909                  * mappings */
 910                 if (path_noexec(&file->f_path)) {
 911                         if (prot & PROT_EXEC)
 912                                 return -EPERM;
 913                 } else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) {
 914                         /* handle implication of PROT_EXEC by PROT_READ */
 915                         if (current->personality & READ_IMPLIES_EXEC) {
 916                                 if (capabilities & NOMMU_MAP_EXEC)
 917                                         prot |= PROT_EXEC;
 918                         }
 919                 } else if ((prot & PROT_READ) &&
 920                          (prot & PROT_EXEC) &&
 921                          !(capabilities & NOMMU_MAP_EXEC)
 922                          ) {
 923                         /* backing file is not executable, try to copy */
 924                         capabilities &= ~NOMMU_MAP_DIRECT;
 925                 }
 926         } else {
 927                 /* anonymous mappings are always memory backed and can be
 928                  * privately mapped
 929                  */
 930                 capabilities = NOMMU_MAP_COPY;
 931
 932                 /* handle PROT_EXEC implication by PROT_READ */
 933                 if ((prot & PROT_READ) &&
 934                     (current->personality & READ_IMPLIES_EXEC))
 935                         prot |= PROT_EXEC;
 936         }
 937
 938         /* allow the security API to have its say */
 939         ret = security_mmap_addr(addr);
 940         if (ret < 0)
 941                 return ret;
 942
 943         /* looks okay */
 944         *_capabilities = capabilities;
 945         return 0;
 946 }
 947
 948 /*
 949  * we've determined that we can make the mapping, now translate what we
 950  * now know into VMA flags
 951  */
 952 static unsigned long determine_vm_flags(struct file *file,
 953                                         unsigned long prot,
 954                                         unsigned long flags,
 955                                         unsigned long capabilities)
 956 {
 957         unsigned long vm_flags;
 958
 959         vm_flags = calc_vm_prot_bits(prot, 0) | calc_vm_flag_bits(flags);
 960         /* vm_flags |= mm->def_flags; */
 961
 962         if (!(capabilities & NOMMU_MAP_DIRECT)) {
 963                 /* attempt to share read-only copies of mapped file chunks */
 964                 vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
 965                 if (file && !(prot & PROT_WRITE))
 966                         vm_flags |= VM_MAYSHARE;
 967         } else {
 968                 /* overlay a shareable mapping on the backing device or inode
 969                  * if possible - used for chardevs, ramfs/tmpfs/shmfs and
 970                  * romfs/cramfs */
 971                 vm_flags |= VM_MAYSHARE | (capabilities & NOMMU_VMFLAGS);
 972                 if (flags & MAP_SHARED)
 973                         vm_flags |= VM_SHARED;
 974         }
 975
 976         /* refuse to let anyone share private mappings with this process if
 977          * it's being traced - otherwise breakpoints set in it may interfere
 978          * with another untraced process
 979          */
 980         if ((flags & MAP_PRIVATE) && current->ptrace)
 981                 vm_flags &= ~VM_MAYSHARE;
 982
 983         return vm_flags;
 984 }
 985
 986 /*
 987  * set up a shared mapping on a file (the driver or filesystem provides and
 988  * pins the storage)
 989  */
 990 static int do_mmap_shared_file(struct vm_area_struct *vma)
 991 {
 992         int ret;
 993
 994         ret = call_mmap(vma->vm_file, vma);
 995         if (ret == 0) {
 996                 vma->vm_region->vm_top = vma->vm_region->vm_end;
 997                 return 0;
 998         }
 999         if (ret != -ENOSYS)
1000                 return ret;
1001
1002         /* getting -ENOSYS indicates that direct mmap isn't possible (as
1003          * opposed to tried but failed) so we can only give a suitable error as
1004          * it's not possible to make a private copy if MAP_SHARED was given */
1005         return -ENODEV;
1006 }
1007
1008 /*
1009  * set up a private mapping or an anonymous shared mapping
1010  */
1011 static int do_mmap_private(struct vm_area_struct *vma,
1012                            struct vm_region *region,
1013                            unsigned long len,
1014                            unsigned long capabilities)
1015 {
1016         unsigned long total, point;
1017         void *base;
1018         int ret, order;
1019
1020         /* invoke the file's mapping function so that it can keep track of
1021          * shared mappings on devices or memory
1022          * - VM_MAYSHARE will be set if it may attempt to share
1023          */
1024         if (capabilities & NOMMU_MAP_DIRECT) {
1025                 ret = call_mmap(vma->vm_file, vma);
1026                 if (ret == 0) {
1027                         /* shouldn't return success if we're not sharing */
1028                         BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
1029                         vma->vm_region->vm_top = vma->vm_region->vm_end;
1030                         return 0;
1031                 }
1032                 if (ret != -ENOSYS)
1033                         return ret;
1034
1035                 /* getting an ENOSYS error indicates that direct mmap isn't
1036                  * possible (as opposed to tried but failed) so we'll try to
1037                  * make a private copy of the data and map that instead */
1038         }
1039
1040
1041         /* allocate some memory to hold the mapping
1042          * - note that this may not return a page-aligned address if the object
1043          *   we're allocating is smaller than a page
1044          */
1045         order = get_order(len);
1046         total = 1 << order;
1047         point = len >> PAGE_SHIFT;
1048
1049         /* we don't want to allocate a power-of-2 sized page set */
1050         if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages)
1051                 total = point;
1052
1053         base = alloc_pages_exact(total << PAGE_SHIFT, GFP_KERNEL);
1054         if (!base)
1055                 goto enomem;
1056
1057         atomic_long_add(total, &mmap_pages_allocated);
1058
1059         region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
1060         region->vm_start = (unsigned long) base;
1061         region->vm_end   = region->vm_start + len;
1062         region->vm_top   = region->vm_start + (total << PAGE_SHIFT);
1063
1064         vma->vm_start = region->vm_start;
1065         vma->vm_end   = region->vm_start + len;
1066
1067         if (vma->vm_file) {
1068                 /* read the contents of a file into the copy */
1069                 loff_t fpos;
1070
1071                 fpos = vma->vm_pgoff;
1072                 fpos <<= PAGE_SHIFT;
1073
1074                 ret = kernel_read(vma->vm_file, base, len, &fpos);
1075                 if (ret < 0)
1076                         goto error_free;
1077
1078                 /* clear the last little bit */
1079                 if (ret < len)
1080                         memset(base + ret, 0, len - ret);
1081
1082         } else {
1083                 vma_set_anonymous(vma);
1084         }
1085
1086         return 0;
1087
1088 error_free:
1089         free_page_series(region->vm_start, region->vm_top);
1090         region->vm_start = vma->vm_start = 0;
1091         region->vm_end   = vma->vm_end = 0;
1092         region->vm_top   = 0;
1093         return ret;
1094
1095 enomem:
1096         pr_err("Allocation of length %lu from process %d (%s) failed\n",
1097                len, current->pid, current->comm);
1098         show_free_areas(0, NULL);
1099         return -ENOMEM;
1100 }
1101
1102 /*
1103  * handle mapping creation for uClinux
1104  */
1105 unsigned long do_mmap(struct file *file,
1106                         unsigned long addr,
1107                         unsigned long len,
1108                         unsigned long prot,
1109                         unsigned long flags,
1110                         vm_flags_t vm_flags,
1111                         unsigned long pgoff,
1112                         unsigned long *populate,
1113                         struct list_head *uf)
1114 {
1115         struct vm_area_struct *vma;
1116         struct vm_region *region;
1117         struct rb_node *rb;
1118         unsigned long capabilities, result;
1119         int ret;
1120
1121         *populate = 0;
1122
1123         /* decide whether we should attempt the mapping, and if so what sort of
1124          * mapping */
1125         ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
1126                                     &capabilities);
1127         if (ret < 0)
1128                 return ret;
1129
1130         /* we ignore the address hint */
1131         addr = 0;
1132         len = PAGE_ALIGN(len);
1133
1134         /* we've determined that we can make the mapping, now translate what we
1135          * now know into VMA flags */
1136         vm_flags |= determine_vm_flags(file, prot, flags, capabilities);
1137
1138         /* we're going to need to record the mapping */
1139         region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
1140         if (!region)
1141                 goto error_getting_region;
1142
1143         vma = vm_area_alloc(current->mm);
1144         if (!vma)
1145                 goto error_getting_vma;
1146
1147         region->vm_usage = 1;
1148         region->vm_flags = vm_flags;
1149         region->vm_pgoff = pgoff;
1150
1151         vma->vm_flags = vm_flags;
1152         vma->vm_pgoff = pgoff;
1153
1154         if (file) {
1155                 region->vm_file = get_file(file);
1156                 vma->vm_file = get_file(file);
1157         }
1158
1159         down_write(&nommu_region_sem);
1160
1161         /* if we want to share, we need to check for regions created by other
1162          * mmap() calls that overlap with our proposed mapping
1163          * - we can only share with a superset match on most regular files
1164          * - shared mappings on character devices and memory backed files are
1165          *   permitted to overlap inexactly as far as we are concerned for in
1166          *   these cases, sharing is handled in the driver or filesystem rather
1167          *   than here
1168          */
1169         if (vm_flags & VM_MAYSHARE) {
1170                 struct vm_region *pregion;
1171                 unsigned long pglen, rpglen, pgend, rpgend, start;
1172
1173                 pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1174                 pgend = pgoff + pglen;
1175
1176                 for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
1177                         pregion = rb_entry(rb, struct vm_region, vm_rb);
1178
1179                         if (!(pregion->vm_flags & VM_MAYSHARE))
1180                                 continue;
1181
1182                         /* search for overlapping mappings on the same file */
1183                         if (file_inode(pregion->vm_file) !=
1184                             file_inode(file))
1185                                 continue;
1186
1187                         if (pregion->vm_pgoff >= pgend)
1188                                 continue;
1189
1190                         rpglen = pregion->vm_end - pregion->vm_start;
1191                         rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
1192                         rpgend = pregion->vm_pgoff + rpglen;
1193                         if (pgoff >= rpgend)
1194                                 continue;
1195
1196                         /* handle inexactly overlapping matches between
1197                          * mappings */
1198                         if ((pregion->vm_pgoff != pgoff || rpglen != pglen) &&
1199                             !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
1200                                 /* new mapping is not a subset of the region */
1201                                 if (!(capabilities & NOMMU_MAP_DIRECT))
1202                                         goto sharing_violation;
1203                                 continue;
1204                         }
1205
1206                         /* we've found a region we can share */
1207                         pregion->vm_usage++;
1208                         vma->vm_region = pregion;
1209                         start = pregion->vm_start;
1210                         start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
1211                         vma->vm_start = start;
1212                         vma->vm_end = start + len;
1213
1214                         if (pregion->vm_flags & VM_MAPPED_COPY)
1215                                 vma->vm_flags |= VM_MAPPED_COPY;
1216                         else {
1217                                 ret = do_mmap_shared_file(vma);
1218                                 if (ret < 0) {
1219                                         vma->vm_region = NULL;
1220                                         vma->vm_start = 0;
1221                                         vma->vm_end = 0;
1222                                         pregion->vm_usage--;
1223                                         pregion = NULL;
1224                                         goto error_just_free;
1225                                 }
1226                         }
1227                         fput(region->vm_file);
1228                         kmem_cache_free(vm_region_jar, region);
1229                         region = pregion;
1230                         result = start;
1231                         goto share;
1232                 }
1233
1234                 /* obtain the address at which to make a shared mapping
1235                  * - this is the hook for quasi-memory character devices to
1236                  *   tell us the location of a shared mapping
1237                  */
1238                 if (capabilities & NOMMU_MAP_DIRECT) {
1239                         addr = file->f_op->get_unmapped_area(file, addr, len,
1240                                                              pgoff, flags);
1241                         if (IS_ERR_VALUE(addr)) {
1242                                 ret = addr;
1243                                 if (ret != -ENOSYS)
1244                                         goto error_just_free;
1245
1246                                 /* the driver refused to tell us where to site
1247                                  * the mapping so we'll have to attempt to copy
1248                                  * it */
1249                                 ret = -ENODEV;
1250                                 if (!(capabilities & NOMMU_MAP_COPY))
1251                                         goto error_just_free;
1252
1253                                 capabilities &= ~NOMMU_MAP_DIRECT;
1254                         } else {
1255                                 vma->vm_start = region->vm_start = addr;
1256                                 vma->vm_end = region->vm_end = addr + len;
1257                         }
1258                 }
1259         }
1260
1261         vma->vm_region = region;
1262
1263         /* set up the mapping
1264          * - the region is filled in if NOMMU_MAP_DIRECT is still set
1265          */
1266         if (file && vma->vm_flags & VM_SHARED)
1267                 ret = do_mmap_shared_file(vma);
1268         else
1269                 ret = do_mmap_private(vma, region, len, capabilities);
1270         if (ret < 0)
1271                 goto error_just_free;
1272         add_nommu_region(region);
1273
1274         /* clear anonymous mappings that don't ask for uninitialized data */
1275         if (!vma->vm_file &&
1276             (!IS_ENABLED(CONFIG_MMAP_ALLOW_UNINITIALIZED) ||
1277              !(flags & MAP_UNINITIALIZED)))
1278                 memset((void *)region->vm_start, 0,
1279                        region->vm_end - region->vm_start);
1280
1281         /* okay... we have a mapping; now we have to register it */
1282         result = vma->vm_start;
1283
1284         current->mm->total_vm += len >> PAGE_SHIFT;
1285
1286 share:
1287         add_vma_to_mm(current->mm, vma);
1288
1289         /* we flush the region from the icache only when the first executable
1290          * mapping of it is made  */
1291         if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) {
1292                 flush_icache_range(region->vm_start, region->vm_end);
1293                 region->vm_icache_flushed = true;
1294         }
1295
1296         up_write(&nommu_region_sem);
1297
1298         return result;
1299
1300 error_just_free:
1301         up_write(&nommu_region_sem);
1302 error:
1303         if (region->vm_file)
1304                 fput(region->vm_file);
1305         kmem_cache_free(vm_region_jar, region);
1306         if (vma->vm_file)
1307                 fput(vma->vm_file);
1308         vm_area_free(vma);
1309         return ret;
1310
1311 sharing_violation:
1312         up_write(&nommu_region_sem);
1313         pr_warn("Attempt to share mismatched mappings\n");
1314         ret = -EINVAL;
1315         goto error;
1316
1317 error_getting_vma:
1318         kmem_cache_free(vm_region_jar, region);
1319         pr_warn("Allocation of vma for %lu byte allocation from process %d failed\n",
1320                         len, current->pid);
1321         show_free_areas(0, NULL);
1322         return -ENOMEM;
1323
1324 error_getting_region:
1325         pr_warn("Allocation of vm region for %lu byte allocation from process %d failed\n",
1326                         len, current->pid);
1327         show_free_areas(0, NULL);
1328         return -ENOMEM;
1329 }
1330
1331 unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
1332                               unsigned long prot, unsigned long flags,
1333                               unsigned long fd, unsigned long pgoff)
1334 {
1335         struct file *file = NULL;
1336         unsigned long retval = -EBADF;
1337
1338         audit_mmap_fd(fd, flags);
1339         if (!(flags & MAP_ANONYMOUS)) {
1340                 file = fget(fd);
1341                 if (!file)
1342                         goto out;
1343         }
1344
1345         flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
1346
1347         retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1348
1349         if (file)
1350                 fput(file);
1351 out:
1352         return retval;
1353 }
1354
1355 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1356                 unsigned long, prot, unsigned long, flags,
1357                 unsigned long, fd, unsigned long, pgoff)
1358 {
1359         return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
1360 }
1361
1362 #ifdef __ARCH_WANT_SYS_OLD_MMAP
1363 struct mmap_arg_struct {
1364         unsigned long addr;
1365         unsigned long len;
1366         unsigned long prot;
1367         unsigned long flags;
1368         unsigned long fd;
1369         unsigned long offset;
1370 };
1371
1372 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1373 {
1374         struct mmap_arg_struct a;
1375
1376         if (copy_from_user(&a, arg, sizeof(a)))
1377                 return -EFAULT;
1378         if (offset_in_page(a.offset))
1379                 return -EINVAL;
1380
1381         return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1382                                a.offset >> PAGE_SHIFT);
1383 }
1384 #endif /* __ARCH_WANT_SYS_OLD_MMAP */
1385
1386 /*
1387  * split a vma into two pieces at address 'addr', a new vma is allocated either
1388  * for the first part or the tail.
1389  */
1390 int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
1391               unsigned long addr, int new_below)
1392 {
1393         struct vm_area_struct *new;
1394         struct vm_region *region;
1395         unsigned long npages;
1396
1397         /* we're only permitted to split anonymous regions (these should have
1398          * only a single usage on the region) */
1399         if (vma->vm_file)
1400                 return -ENOMEM;
1401
1402         if (mm->map_count >= sysctl_max_map_count)
1403                 return -ENOMEM;
1404
1405         region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
1406         if (!region)
1407                 return -ENOMEM;
1408
1409         new = vm_area_dup(vma);
1410         if (!new) {
1411                 kmem_cache_free(vm_region_jar, region);
1412                 return -ENOMEM;
1413         }
1414
1415         /* most fields are the same, copy all, and then fixup */
1416         *region = *vma->vm_region;
1417         new->vm_region = region;
1418
1419         npages = (addr - vma->vm_start) >> PAGE_SHIFT;
1420
1421         if (new_below) {
1422                 region->vm_top = region->vm_end = new->vm_end = addr;
1423         } else {
1424                 region->vm_start = new->vm_start = addr;
1425                 region->vm_pgoff = new->vm_pgoff += npages;
1426         }
1427
1428         if (new->vm_ops && new->vm_ops->open)
1429                 new->vm_ops->open(new);
1430
1431         delete_vma_from_mm(vma);
1432         down_write(&nommu_region_sem);
1433         delete_nommu_region(vma->vm_region);
1434         if (new_below) {
1435                 vma->vm_region->vm_start = vma->vm_start = addr;
1436                 vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
1437         } else {
1438                 vma->vm_region->vm_end = vma->vm_end = addr;
1439                 vma->vm_region->vm_top = addr;
1440         }
1441         add_nommu_region(vma->vm_region);
1442         add_nommu_region(new->vm_region);
1443         up_write(&nommu_region_sem);
1444         add_vma_to_mm(mm, vma);
1445         add_vma_to_mm(mm, new);
1446         return 0;
1447 }
1448
1449 /*
1450  * shrink a VMA by removing the specified chunk from either the beginning or
1451  * the end
1452  */
1453 static int shrink_vma(struct mm_struct *mm,
1454                       struct vm_area_struct *vma,
1455                       unsigned long from, unsigned long to)
1456 {
1457         struct vm_region *region;
1458
1459         /* adjust the VMA's pointers, which may reposition it in the MM's tree
1460          * and list */
1461         delete_vma_from_mm(vma);
1462         if (from > vma->vm_start)
1463                 vma->vm_end = from;
1464         else
1465                 vma->vm_start = to;
1466         add_vma_to_mm(mm, vma);
1467
1468         /* cut the backing region down to size */
1469         region = vma->vm_region;
1470         BUG_ON(region->vm_usage != 1);
1471
1472         down_write(&nommu_region_sem);
1473         delete_nommu_region(region);
1474         if (from > region->vm_start) {
1475                 to = region->vm_top;
1476                 region->vm_top = region->vm_end = from;
1477         } else {
1478                 region->vm_start = to;
1479         }
1480         add_nommu_region(region);
1481         up_write(&nommu_region_sem);
1482
1483         free_page_series(from, to);
1484         return 0;
1485 }
1486
1487 /*
1488  * release a mapping
1489  * - under NOMMU conditions the chunk to be unmapped must be backed by a single
1490  *   VMA, though it need not cover the whole VMA
1491  */
1492 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf)
1493 {
1494         struct vm_area_struct *vma;
1495         unsigned long end;
1496         int ret;
1497
1498         len = PAGE_ALIGN(len);
1499         if (len == 0)
1500                 return -EINVAL;
1501
1502         end = start + len;
1503
1504         /* find the first potentially overlapping VMA */
1505         vma = find_vma(mm, start);
1506         if (!vma) {
1507                 static int limit;
1508                 if (limit < 5) {
1509                         pr_warn("munmap of memory not mmapped by process %d (%s): 0x%lx-0x%lx\n",
1510                                         current->pid, current->comm,
1511                                         start, start + len - 1);
1512                         limit++;
1513                 }
1514                 return -EINVAL;
1515         }
1516
1517         /* we're allowed to split an anonymous VMA but not a file-backed one */
1518         if (vma->vm_file) {
1519                 do {
1520                         if (start > vma->vm_start)
1521                                 return -EINVAL;
1522                         if (end == vma->vm_end)
1523                                 goto erase_whole_vma;
1524                         vma = vma->vm_next;
1525                 } while (vma);
1526                 return -EINVAL;
1527         } else {
1528                 /* the chunk must be a subset of the VMA found */
1529                 if (start == vma->vm_start && end == vma->vm_end)
1530                         goto erase_whole_vma;
1531                 if (start < vma->vm_start || end > vma->vm_end)
1532                         return -EINVAL;
1533                 if (offset_in_page(start))
1534                         return -EINVAL;
1535                 if (end != vma->vm_end && offset_in_page(end))
1536                         return -EINVAL;
1537                 if (start != vma->vm_start && end != vma->vm_end) {
1538                         ret = split_vma(mm, vma, start, 1);
1539                         if (ret < 0)
1540                                 return ret;
1541                 }
1542                 return shrink_vma(mm, vma, start, end);
1543         }
1544
1545 erase_whole_vma:
1546         delete_vma_from_mm(vma);
1547         delete_vma(mm, vma);
1548         return 0;
1549 }
1550 EXPORT_SYMBOL(do_munmap);
1551
1552 int vm_munmap(unsigned long addr, size_t len)
1553 {
1554         struct mm_struct *mm = current->mm;
1555         int ret;
1556
1557         down_write(&mm->mmap_sem);
1558         ret = do_munmap(mm, addr, len, NULL);
1559         up_write(&mm->mmap_sem);
1560         return ret;
1561 }
1562 EXPORT_SYMBOL(vm_munmap);
1563
1564 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
1565 {
1566         return vm_munmap(addr, len);
1567 }
1568
1569 /*
1570  * release all the mappings made in a process's VM space
1571  */
1572 void exit_mmap(struct mm_struct *mm)
1573 {
1574         struct vm_area_struct *vma;
1575
1576         if (!mm)
1577                 return;
1578
1579         mm->total_vm = 0;
1580
1581         while ((vma = mm->mmap)) {
1582                 mm->mmap = vma->vm_next;
1583                 delete_vma_from_mm(vma);
1584                 delete_vma(mm, vma);
1585                 cond_resched();
1586         }
1587 }
1588
1589 int vm_brk(unsigned long addr, unsigned long len)
1590 {
1591         return -ENOMEM;
1592 }
1593
1594 /*
1595  * expand (or shrink) an existing mapping, potentially moving it at the same
1596  * time (controlled by the MREMAP_MAYMOVE flag and available VM space)
1597  *
1598  * under NOMMU conditions, we only permit changing a mapping's size, and only
1599  * as long as it stays within the region allocated by do_mmap_private() and the
1600  * block is not shareable
1601  *
1602  * MREMAP_FIXED is not supported under NOMMU conditions
1603  */
1604 static unsigned long do_mremap(unsigned long addr,
1605                         unsigned long old_len, unsigned long new_len,
1606                         unsigned long flags, unsigned long new_addr)
1607 {
1608         struct vm_area_struct *vma;
1609
1610         /* insanity checks first */
1611         old_len = PAGE_ALIGN(old_len);
1612         new_len = PAGE_ALIGN(new_len);
1613         if (old_len == 0 || new_len == 0)
1614                 return (unsigned long) -EINVAL;
1615
1616         if (offset_in_page(addr))
1617                 return -EINVAL;
1618
1619         if (flags & MREMAP_FIXED && new_addr != addr)
1620                 return (unsigned long) -EINVAL;
1621
1622         vma = find_vma_exact(current->mm, addr, old_len);
1623         if (!vma)
1624                 return (unsigned long) -EINVAL;
1625
1626         if (vma->vm_end != vma->vm_start + old_len)
1627                 return (unsigned long) -EFAULT;
1628
1629         if (vma->vm_flags & VM_MAYSHARE)
1630                 return (unsigned long) -EPERM;
1631
1632         if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
1633                 return (unsigned long) -ENOMEM;
1634
1635         /* all checks complete - do it */
1636         vma->vm_end = vma->vm_start + new_len;
1637         return vma->vm_start;
1638 }
1639
1640 SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
1641                 unsigned long, new_len, unsigned long, flags,
1642                 unsigned long, new_addr)
1643 {
1644         unsigned long ret;
1645
1646         down_write(&current->mm->mmap_sem);
1647         ret = do_mremap(addr, old_len, new_len, flags, new_addr);
1648         up_write(&current->mm->mmap_sem);
1649         return ret;
1650 }
1651
1652 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
1653                          unsigned int foll_flags)
1654 {
1655         return NULL;
1656 }
1657
1658 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
1659                 unsigned long pfn, unsigned long size, pgprot_t prot)
1660 {
1661         if (addr != (pfn << PAGE_SHIFT))
1662                 return -EINVAL;
1663
1664         vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
1665         return 0;
1666 }
1667 EXPORT_SYMBOL(remap_pfn_range);
1668
1669 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
1670 {
1671         unsigned long pfn = start >> PAGE_SHIFT;
1672         unsigned long vm_len = vma->vm_end - vma->vm_start;
1673
1674         pfn += vma->vm_pgoff;
1675         return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
1676 }
1677 EXPORT_SYMBOL(vm_iomap_memory);
1678
1679 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
1680                         unsigned long pgoff)
1681 {
1682         unsigned int size = vma->vm_end - vma->vm_start;
1683
1684         if (!(vma->vm_flags & VM_USERMAP))
1685                 return -EINVAL;
1686
1687         vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT));
1688         vma->vm_end = vma->vm_start + size;
1689
1690         return 0;
1691 }
1692 EXPORT_SYMBOL(remap_vmalloc_range);
1693
1694 unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr,
1695         unsigned long len, unsigned long pgoff, unsigned long flags)
1696 {
1697         return -ENOMEM;
1698 }
1699
1700 vm_fault_t filemap_fault(struct vm_fault *vmf)
1701 {
1702         BUG();
1703         return 0;
1704 }
1705 EXPORT_SYMBOL(filemap_fault);
1706
1707 void filemap_map_pages(struct vm_fault *vmf,
1708                 pgoff_t start_pgoff, pgoff_t end_pgoff)
1709 {
1710         BUG();
1711 }
1712 EXPORT_SYMBOL(filemap_map_pages);
1713
1714 int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
1715                 unsigned long addr, void *buf, int len, unsigned int gup_flags)
1716 {
1717         struct vm_area_struct *vma;
1718         int write = gup_flags & FOLL_WRITE;
1719
1720         if (down_read_killable(&mm->mmap_sem))
1721                 return 0;
1722
1723         /* the access must start within one of the target process's mappings */
1724         vma = find_vma(mm, addr);
1725         if (vma) {
1726                 /* don't overrun this mapping */
1727                 if (addr + len >= vma->vm_end)
1728                         len = vma->vm_end - addr;
1729
1730                 /* only read or write mappings where it is permitted */
1731                 if (write && vma->vm_flags & VM_MAYWRITE)
1732                         copy_to_user_page(vma, NULL, addr,
1733                                          (void *) addr, buf, len);
1734                 else if (!write && vma->vm_flags & VM_MAYREAD)
1735                         copy_from_user_page(vma, NULL, addr,
1736                                             buf, (void *) addr, len);
1737                 else
1738                         len = 0;
1739         } else {
1740                 len = 0;
1741         }
1742
1743         up_read(&mm->mmap_sem);
1744
1745         return len;
1746 }
1747
1748 /**
1749  * access_remote_vm - access another process' address space
1750  * @mm:         the mm_struct of the target address space
1751  * @addr:       start address to access
1752  * @buf:        source or destination buffer
1753  * @len:        number of bytes to transfer
1754  * @gup_flags:  flags modifying lookup behaviour
1755  *
1756  * The caller must hold a reference on @mm.
1757  */
1758 int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1759                 void *buf, int len, unsigned int gup_flags)
1760 {
1761         return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags);
1762 }
1763
1764 /*
1765  * Access another process' address space.
1766  * - source/target buffer must be kernel space
1767  */
1768 int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len,
1769                 unsigned int gup_flags)
1770 {
1771         struct mm_struct *mm;
1772
1773         if (addr + len < addr)
1774                 return 0;
1775
1776         mm = get_task_mm(tsk);
1777         if (!mm)
1778                 return 0;
1779
1780         len = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags);
1781
1782         mmput(mm);
1783         return len;
1784 }
1785 EXPORT_SYMBOL_GPL(access_process_vm);
1786
1787 /**
1788  * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
1789  * @inode: The inode to check
1790  * @size: The current filesize of the inode
1791  * @newsize: The proposed filesize of the inode
1792  *
1793  * Check the shared mappings on an inode on behalf of a shrinking truncate to
1794  * make sure that that any outstanding VMAs aren't broken and then shrink the
1795  * vm_regions that extend that beyond so that do_mmap_pgoff() doesn't
1796  * automatically grant mappings that are too large.
1797  */
1798 int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
1799                                 size_t newsize)
1800 {
1801         struct vm_area_struct *vma;
1802         struct vm_region *region;
1803         pgoff_t low, high;
1804         size_t r_size, r_top;
1805
1806         low = newsize >> PAGE_SHIFT;
1807         high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1808
1809         down_write(&nommu_region_sem);
1810         i_mmap_lock_read(inode->i_mapping);
1811
1812         /* search for VMAs that fall within the dead zone */
1813         vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) {
1814                 /* found one - only interested if it's shared out of the page
1815                  * cache */
1816                 if (vma->vm_flags & VM_SHARED) {
1817                         i_mmap_unlock_read(inode->i_mapping);
1818                         up_write(&nommu_region_sem);
1819                         return -ETXTBSY; /* not quite true, but near enough */
1820                 }
1821         }
1822
1823         /* reduce any regions that overlap the dead zone - if in existence,
1824          * these will be pointed to by VMAs that don't overlap the dead zone
1825          *
1826          * we don't check for any regions that start beyond the EOF as there
1827          * shouldn't be any
1828          */
1829         vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) {
1830                 if (!(vma->vm_flags & VM_SHARED))
1831                         continue;
1832
1833                 region = vma->vm_region;
1834                 r_size = region->vm_top - region->vm_start;
1835                 r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size;
1836
1837                 if (r_top > newsize) {
1838                         region->vm_top -= r_top - newsize;
1839                         if (region->vm_end > region->vm_top)
1840                                 region->vm_end = region->vm_top;
1841                 }
1842         }
1843
1844         i_mmap_unlock_read(inode->i_mapping);
1845         up_write(&nommu_region_sem);
1846         return 0;
1847 }
1848
1849 /*
1850  * Initialise sysctl_user_reserve_kbytes.
1851  *
1852  * This is intended to prevent a user from starting a single memory hogging
1853  * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
1854  * mode.
1855  *
1856  * The default value is min(3% of free memory, 128MB)
1857  * 128MB is enough to recover with sshd/login, bash, and top/kill.
1858  */
1859 static int __meminit init_user_reserve(void)
1860 {
1861         unsigned long free_kbytes;
1862
1863         free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
1864
1865         sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
1866         return 0;
1867 }
1868 subsys_initcall(init_user_reserve);
1869
1870 /*
1871  * Initialise sysctl_admin_reserve_kbytes.
1872  *
1873  * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
1874  * to log in and kill a memory hogging process.
1875  *
1876  * Systems with more than 256MB will reserve 8MB, enough to recover
1877  * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
1878  * only reserve 3% of free pages by default.
1879  */
1880 static int __meminit init_admin_reserve(void)
1881 {
1882         unsigned long free_kbytes;
1883
1884         free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
1885
1886         sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
1887         return 0;
1888 }
1889 subsys_initcall(init_admin_reserve);