arch/sparc/mm/init_64.c

   1 /*
   2  *  arch/sparc64/mm/init.c
   3  *
   4  *  Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
   5  *  Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   6  */
   7
   8 #include <linux/extable.h>
   9 #include <linux/kernel.h>
  10 #include <linux/sched.h>
  11 #include <linux/string.h>
  12 #include <linux/init.h>
  13 #include <linux/bootmem.h>
  14 #include <linux/mm.h>
  15 #include <linux/hugetlb.h>
  16 #include <linux/initrd.h>
  17 #include <linux/swap.h>
  18 #include <linux/pagemap.h>
  19 #include <linux/poison.h>
  20 #include <linux/fs.h>
  21 #include <linux/seq_file.h>
  22 #include <linux/kprobes.h>
  23 #include <linux/cache.h>
  24 #include <linux/sort.h>
  25 #include <linux/ioport.h>
  26 #include <linux/percpu.h>
  27 #include <linux/memblock.h>
  28 #include <linux/mmzone.h>
  29 #include <linux/gfp.h>
  30
  31 #include <asm/head.h>
  32 #include <asm/page.h>
  33 #include <asm/pgalloc.h>
  34 #include <asm/pgtable.h>
  35 #include <asm/oplib.h>
  36 #include <asm/iommu.h>
  37 #include <asm/io.h>
  38 #include <linux/uaccess.h>
  39 #include <asm/mmu_context.h>
  40 #include <asm/tlbflush.h>
  41 #include <asm/dma.h>
  42 #include <asm/starfire.h>
  43 #include <asm/tlb.h>
  44 #include <asm/spitfire.h>
  45 #include <asm/sections.h>
  46 #include <asm/tsb.h>
  47 #include <asm/hypervisor.h>
  48 #include <asm/prom.h>
  49 #include <asm/mdesc.h>
  50 #include <asm/cpudata.h>
  51 #include <asm/setup.h>
  52 #include <asm/irq.h>
  53
  54 #include "init_64.h"
  55
  56 unsigned long kern_linear_pte_xor[4] __read_mostly;
  57 static unsigned long page_cache4v_flag;
  58
  59 /* A bitmap, two bits for every 256MB of physical memory.  These two
  60  * bits determine what page size we use for kernel linear
  61  * translations.  They form an index into kern_linear_pte_xor[].  The
  62  * value in the indexed slot is XOR'd with the TLB miss virtual
  63  * address to form the resulting TTE.  The mapping is:
  64  *
  65  *      0       ==>     4MB
  66  *      1       ==>     256MB
  67  *      2       ==>     2GB
  68  *      3       ==>     16GB
  69  *
  70  * All sun4v chips support 256MB pages.  Only SPARC-T4 and later
  71  * support 2GB pages, and hopefully future cpus will support the 16GB
  72  * pages as well.  For slots 2 and 3, we encode a 256MB TTE xor there
  73  * if these larger page sizes are not supported by the cpu.
  74  *
  75  * It would be nice to determine this from the machine description
  76  * 'cpu' properties, but we need to have this table setup before the
  77  * MDESC is initialized.
  78  */
  79
  80 #ifndef CONFIG_DEBUG_PAGEALLOC
  81 /* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings.
  82  * Space is allocated for this right after the trap table in
  83  * arch/sparc64/kernel/head.S
  84  */
  85 extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];
  86 #endif
  87 extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
  88
  89 static unsigned long cpu_pgsz_mask;
  90
  91 #define MAX_BANKS       1024
  92
  93 static struct linux_prom64_registers pavail[MAX_BANKS];
  94 static int pavail_ents;
  95
  96 u64 numa_latency[MAX_NUMNODES][MAX_NUMNODES];
  97
  98 static int cmp_p64(const void *a, const void *b)
  99 {
 100         const struct linux_prom64_registers *x = a, *y = b;
 101
 102         if (x->phys_addr > y->phys_addr)
 103                 return 1;
 104         if (x->phys_addr < y->phys_addr)
 105                 return -1;
 106         return 0;
 107 }
 108
 109 static void __init read_obp_memory(const char *property,
 110                                    struct linux_prom64_registers *regs,
 111                                    int *num_ents)
 112 {
 113         phandle node = prom_finddevice("/memory");
 114         int prop_size = prom_getproplen(node, property);
 115         int ents, ret, i;
 116
 117         ents = prop_size / sizeof(struct linux_prom64_registers);
 118         if (ents > MAX_BANKS) {
 119                 prom_printf("The machine has more %s property entries than "
 120                             "this kernel can support (%d).\n",
 121                             property, MAX_BANKS);
 122                 prom_halt();
 123         }
 124
 125         ret = prom_getproperty(node, property, (char *) regs, prop_size);
 126         if (ret == -1) {
 127                 prom_printf("Couldn't get %s property from /memory.\n",
 128                                 property);
 129                 prom_halt();
 130         }
 131
 132         /* Sanitize what we got from the firmware, by page aligning
 133          * everything.
 134          */
 135         for (i = 0; i < ents; i++) {
 136                 unsigned long base, size;
 137
 138                 base = regs[i].phys_addr;
 139                 size = regs[i].reg_size;
 140
 141                 size &= PAGE_MASK;
 142                 if (base & ~PAGE_MASK) {
 143                         unsigned long new_base = PAGE_ALIGN(base);
 144
 145                         size -= new_base - base;
 146                         if ((long) size < 0L)
 147                                 size = 0UL;
 148                         base = new_base;
 149                 }
 150                 if (size == 0UL) {
 151                         /* If it is empty, simply get rid of it.
 152                          * This simplifies the logic of the other
 153                          * functions that process these arrays.
 154                          */
 155                         memmove(&regs[i], &regs[i + 1],
 156                                 (ents - i - 1) * sizeof(regs[0]));
 157                         i--;
 158                         ents--;
 159                         continue;
 160                 }
 161                 regs[i].phys_addr = base;
 162                 regs[i].reg_size = size;
 163         }
 164
 165         *num_ents = ents;
 166
 167         sort(regs, ents, sizeof(struct linux_prom64_registers),
 168              cmp_p64, NULL);
 169 }
 170
 171 /* Kernel physical address base and size in bytes.  */
 172 unsigned long kern_base __read_mostly;
 173 unsigned long kern_size __read_mostly;
 174
 175 /* Initial ramdisk setup */
 176 extern unsigned long sparc_ramdisk_image64;
 177 extern unsigned int sparc_ramdisk_image;
 178 extern unsigned int sparc_ramdisk_size;
 179
 180 struct page *mem_map_zero __read_mostly;
 181 EXPORT_SYMBOL(mem_map_zero);
 182
 183 unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly;
 184
 185 unsigned long sparc64_kern_pri_context __read_mostly;
 186 unsigned long sparc64_kern_pri_nuc_bits __read_mostly;
 187 unsigned long sparc64_kern_sec_context __read_mostly;
 188
 189 int num_kernel_image_mappings;
 190
 191 #ifdef CONFIG_DEBUG_DCFLUSH
 192 atomic_t dcpage_flushes = ATOMIC_INIT(0);
 193 #ifdef CONFIG_SMP
 194 atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
 195 #endif
 196 #endif
 197
 198 inline void flush_dcache_page_impl(struct page *page)
 199 {
 200         BUG_ON(tlb_type == hypervisor);
 201 #ifdef CONFIG_DEBUG_DCFLUSH
 202         atomic_inc(&dcpage_flushes);
 203 #endif
 204
 205 #ifdef DCACHE_ALIASING_POSSIBLE
 206         __flush_dcache_page(page_address(page),
 207                             ((tlb_type == spitfire) &&
 208                              page_mapping(page) != NULL));
 209 #else
 210         if (page_mapping(page) != NULL &&
 211             tlb_type == spitfire)
 212                 __flush_icache_page(__pa(page_address(page)));
 213 #endif
 214 }
 215
 216 #define PG_dcache_dirty         PG_arch_1
 217 #define PG_dcache_cpu_shift     32UL
 218 #define PG_dcache_cpu_mask      \
 219         ((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL)
 220
 221 #define dcache_dirty_cpu(page) \
 222         (((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
 223
 224 static inline void set_dcache_dirty(struct page *page, int this_cpu)
 225 {
 226         unsigned long mask = this_cpu;
 227         unsigned long non_cpu_bits;
 228
 229         non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift);
 230         mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty);
 231
 232         __asm__ __volatile__("1:\n\t"
 233                              "ldx       [%2], %%g7\n\t"
 234                              "and       %%g7, %1, %%g1\n\t"
 235                              "or        %%g1, %0, %%g1\n\t"
 236                              "casx      [%2], %%g7, %%g1\n\t"
 237                              "cmp       %%g7, %%g1\n\t"
 238                              "bne,pn    %%xcc, 1b\n\t"
 239                              " nop"
 240                              : /* no outputs */
 241                              : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)
 242                              : "g1", "g7");
 243 }
 244
 245 static inline void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu)
 246 {
 247         unsigned long mask = (1UL << PG_dcache_dirty);
 248
 249         __asm__ __volatile__("! test_and_clear_dcache_dirty\n"
 250                              "1:\n\t"
 251                              "ldx       [%2], %%g7\n\t"
 252                              "srlx      %%g7, %4, %%g1\n\t"
 253                              "and       %%g1, %3, %%g1\n\t"
 254                              "cmp       %%g1, %0\n\t"
 255                              "bne,pn    %%icc, 2f\n\t"
 256                              " andn     %%g7, %1, %%g1\n\t"
 257                              "casx      [%2], %%g7, %%g1\n\t"
 258                              "cmp       %%g7, %%g1\n\t"
 259                              "bne,pn    %%xcc, 1b\n\t"
 260                              " nop\n"
 261                              "2:"
 262                              : /* no outputs */
 263                              : "r" (cpu), "r" (mask), "r" (&page->flags),
 264                                "i" (PG_dcache_cpu_mask),
 265                                "i" (PG_dcache_cpu_shift)
 266                              : "g1", "g7");
 267 }
 268
 269 static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long pte)
 270 {
 271         unsigned long tsb_addr = (unsigned long) ent;
 272
 273         if (tlb_type == cheetah_plus || tlb_type == hypervisor)
 274                 tsb_addr = __pa(tsb_addr);
 275
 276         __tsb_insert(tsb_addr, tag, pte);
 277 }
 278
 279 unsigned long _PAGE_ALL_SZ_BITS __read_mostly;
 280
 281 static void flush_dcache(unsigned long pfn)
 282 {
 283         struct page *page;
 284
 285         page = pfn_to_page(pfn);
 286         if (page) {
 287                 unsigned long pg_flags;
 288
 289                 pg_flags = page->flags;
 290                 if (pg_flags & (1UL << PG_dcache_dirty)) {
 291                         int cpu = ((pg_flags >> PG_dcache_cpu_shift) &
 292                                    PG_dcache_cpu_mask);
 293                         int this_cpu = get_cpu();
 294
 295                         /* This is just to optimize away some function calls
 296                          * in the SMP case.
 297                          */
 298                         if (cpu == this_cpu)
 299                                 flush_dcache_page_impl(page);
 300                         else
 301                                 smp_flush_dcache_page_impl(page, cpu);
 302
 303                         clear_dcache_dirty_cpu(page, cpu);
 304
 305                         put_cpu();
 306                 }
 307         }
 308 }
 309
 310 /* mm->context.lock must be held */
 311 static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_index,
 312                                     unsigned long tsb_hash_shift, unsigned long address,
 313                                     unsigned long tte)
 314 {
 315         struct tsb *tsb = mm->context.tsb_block[tsb_index].tsb;
 316         unsigned long tag;
 317
 318         if (unlikely(!tsb))
 319                 return;
 320
 321         tsb += ((address >> tsb_hash_shift) &
 322                 (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
 323         tag = (address >> 22UL);
 324         tsb_insert(tsb, tag, tte);
 325 }
 326
 327 #ifdef CONFIG_HUGETLB_PAGE
 328 static int __init setup_hugepagesz(char *string)
 329 {
 330         unsigned long long hugepage_size;
 331         unsigned int hugepage_shift;
 332         unsigned short hv_pgsz_idx;
 333         unsigned int hv_pgsz_mask;
 334         int rc = 0;
 335
 336         hugepage_size = memparse(string, &string);
 337         hugepage_shift = ilog2(hugepage_size);
 338
 339         switch (hugepage_shift) {
 340         case HPAGE_2GB_SHIFT:
 341                 hv_pgsz_mask = HV_PGSZ_MASK_2GB;
 342                 hv_pgsz_idx = HV_PGSZ_IDX_2GB;
 343                 break;
 344         case HPAGE_256MB_SHIFT:
 345                 hv_pgsz_mask = HV_PGSZ_MASK_256MB;
 346                 hv_pgsz_idx = HV_PGSZ_IDX_256MB;
 347                 break;
 348         case HPAGE_SHIFT:
 349                 hv_pgsz_mask = HV_PGSZ_MASK_4MB;
 350                 hv_pgsz_idx = HV_PGSZ_IDX_4MB;
 351                 break;
 352         case HPAGE_64K_SHIFT:
 353                 hv_pgsz_mask = HV_PGSZ_MASK_64K;
 354                 hv_pgsz_idx = HV_PGSZ_IDX_64K;
 355                 break;
 356         default:
 357                 hv_pgsz_mask = 0;
 358         }
 359
 360         if ((hv_pgsz_mask & cpu_pgsz_mask) == 0U) {
 361                 pr_warn("hugepagesz=%llu not supported by MMU.\n",
 362                         hugepage_size);
 363                 goto out;
 364         }
 365
 366         hugetlb_add_hstate(hugepage_shift - PAGE_SHIFT);
 367         rc = 1;
 368
 369 out:
 370         return rc;
 371 }
 372 __setup("hugepagesz=", setup_hugepagesz);
 373 #endif  /* CONFIG_HUGETLB_PAGE */
 374
 375 void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
 376 {
 377         struct mm_struct *mm;
 378         unsigned long flags;
 379         pte_t pte = *ptep;
 380
 381         if (tlb_type != hypervisor) {
 382                 unsigned long pfn = pte_pfn(pte);
 383
 384                 if (pfn_valid(pfn))
 385                         flush_dcache(pfn);
 386         }
 387
 388         mm = vma->vm_mm;
 389
 390         /* Don't insert a non-valid PTE into the TSB, we'll deadlock.  */
 391         if (!pte_accessible(mm, pte))
 392                 return;
 393
 394         spin_lock_irqsave(&mm->context.lock, flags);
 395
 396 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
 397         if ((mm->context.hugetlb_pte_count || mm->context.thp_pte_count) &&
 398             is_hugetlb_pmd(__pmd(pte_val(pte)))) {
 399                 /* We are fabricating 8MB pages using 4MB real hw pages.  */
 400                 pte_val(pte) |= (address & (1UL << REAL_HPAGE_SHIFT));
 401                 __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
 402                                         address, pte_val(pte));
 403         } else
 404 #endif
 405                 __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT,
 406                                         address, pte_val(pte));
 407
 408         spin_unlock_irqrestore(&mm->context.lock, flags);
 409 }
 410
 411 void flush_dcache_page(struct page *page)
 412 {
 413         struct address_space *mapping;
 414         int this_cpu;
 415
 416         if (tlb_type == hypervisor)
 417                 return;
 418
 419         /* Do not bother with the expensive D-cache flush if it
 420          * is merely the zero page.  The 'bigcore' testcase in GDB
 421          * causes this case to run millions of times.
 422          */
 423         if (page == ZERO_PAGE(0))
 424                 return;
 425
 426         this_cpu = get_cpu();
 427
 428         mapping = page_mapping(page);
 429         if (mapping && !mapping_mapped(mapping)) {
 430                 int dirty = test_bit(PG_dcache_dirty, &page->flags);
 431                 if (dirty) {
 432                         int dirty_cpu = dcache_dirty_cpu(page);
 433
 434                         if (dirty_cpu == this_cpu)
 435                                 goto out;
 436                         smp_flush_dcache_page_impl(page, dirty_cpu);
 437                 }
 438                 set_dcache_dirty(page, this_cpu);
 439         } else {
 440                 /* We could delay the flush for the !page_mapping
 441                  * case too.  But that case is for exec env/arg
 442                  * pages and those are %99 certainly going to get
 443                  * faulted into the tlb (and thus flushed) anyways.
 444                  */
 445                 flush_dcache_page_impl(page);
 446         }
 447
 448 out:
 449         put_cpu();
 450 }
 451 EXPORT_SYMBOL(flush_dcache_page);
 452
 453 void __kprobes flush_icache_range(unsigned long start, unsigned long end)
 454 {
 455         /* Cheetah and Hypervisor platform cpus have coherent I-cache. */
 456         if (tlb_type == spitfire) {
 457                 unsigned long kaddr;
 458
 459                 /* This code only runs on Spitfire cpus so this is
 460                  * why we can assume _PAGE_PADDR_4U.
 461                  */
 462                 for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE) {
 463                         unsigned long paddr, mask = _PAGE_PADDR_4U;
 464
 465                         if (kaddr >= PAGE_OFFSET)
 466                                 paddr = kaddr & mask;
 467                         else {
 468                                 pgd_t *pgdp = pgd_offset_k(kaddr);
 469                                 pud_t *pudp = pud_offset(pgdp, kaddr);
 470                                 pmd_t *pmdp = pmd_offset(pudp, kaddr);
 471                                 pte_t *ptep = pte_offset_kernel(pmdp, kaddr);
 472
 473                                 paddr = pte_val(*ptep) & mask;
 474                         }
 475                         __flush_icache_page(paddr);
 476                 }
 477         }
 478 }
 479 EXPORT_SYMBOL(flush_icache_range);
 480
 481 void mmu_info(struct seq_file *m)
 482 {
 483         static const char *pgsz_strings[] = {
 484                 "8K", "64K", "512K", "4MB", "32MB",
 485                 "256MB", "2GB", "16GB",
 486         };
 487         int i, printed;
 488
 489         if (tlb_type == cheetah)
 490                 seq_printf(m, "MMU Type\t: Cheetah\n");
 491         else if (tlb_type == cheetah_plus)
 492                 seq_printf(m, "MMU Type\t: Cheetah+\n");
 493         else if (tlb_type == spitfire)
 494                 seq_printf(m, "MMU Type\t: Spitfire\n");
 495         else if (tlb_type == hypervisor)
 496                 seq_printf(m, "MMU Type\t: Hypervisor (sun4v)\n");
 497         else
 498                 seq_printf(m, "MMU Type\t: ???\n");
 499
 500         seq_printf(m, "MMU PGSZs\t: ");
 501         printed = 0;
 502         for (i = 0; i < ARRAY_SIZE(pgsz_strings); i++) {
 503                 if (cpu_pgsz_mask & (1UL << i)) {
 504                         seq_printf(m, "%s%s",
 505                                    printed ? "," : "", pgsz_strings[i]);
 506                         printed++;
 507                 }
 508         }
 509         seq_putc(m, '\n');
 510
 511 #ifdef CONFIG_DEBUG_DCFLUSH
 512         seq_printf(m, "DCPageFlushes\t: %d\n",
 513                    atomic_read(&dcpage_flushes));
 514 #ifdef CONFIG_SMP
 515         seq_printf(m, "DCPageFlushesXC\t: %d\n",
 516                    atomic_read(&dcpage_flushes_xcall));
 517 #endif /* CONFIG_SMP */
 518 #endif /* CONFIG_DEBUG_DCFLUSH */
 519 }
 520
 521 struct linux_prom_translation prom_trans[512] __read_mostly;
 522 unsigned int prom_trans_ents __read_mostly;
 523
 524 unsigned long kern_locked_tte_data;
 525
 526 /* The obp translations are saved based on 8k pagesize, since obp can
 527  * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
 528  * HI_OBP_ADDRESS range are handled in ktlb.S.
 529  */
 530 static inline int in_obp_range(unsigned long vaddr)
 531 {
 532         return (vaddr >= LOW_OBP_ADDRESS &&
 533                 vaddr < HI_OBP_ADDRESS);
 534 }
 535
 536 static int cmp_ptrans(const void *a, const void *b)
 537 {
 538         const struct linux_prom_translation *x = a, *y = b;
 539
 540         if (x->virt > y->virt)
 541                 return 1;
 542         if (x->virt < y->virt)
 543                 return -1;
 544         return 0;
 545 }
 546
 547 /* Read OBP translations property into 'prom_trans[]'.  */
 548 static void __init read_obp_translations(void)
 549 {
 550         int n, node, ents, first, last, i;
 551
 552         node = prom_finddevice("/virtual-memory");
 553         n = prom_getproplen(node, "translations");
 554         if (unlikely(n == 0 || n == -1)) {
 555                 prom_printf("prom_mappings: Couldn't get size.\n");
 556                 prom_halt();
 557         }
 558         if (unlikely(n > sizeof(prom_trans))) {
 559                 prom_printf("prom_mappings: Size %d is too big.\n", n);
 560                 prom_halt();
 561         }
 562
 563         if ((n = prom_getproperty(node, "translations",
 564                                   (char *)&prom_trans[0],
 565                                   sizeof(prom_trans))) == -1) {
 566                 prom_printf("prom_mappings: Couldn't get property.\n");
 567                 prom_halt();
 568         }
 569
 570         n = n / sizeof(struct linux_prom_translation);
 571
 572         ents = n;
 573
 574         sort(prom_trans, ents, sizeof(struct linux_prom_translation),
 575              cmp_ptrans, NULL);
 576
 577         /* Now kick out all the non-OBP entries.  */
 578         for (i = 0; i < ents; i++) {
 579                 if (in_obp_range(prom_trans[i].virt))
 580                         break;
 581         }
 582         first = i;
 583         for (; i < ents; i++) {
 584                 if (!in_obp_range(prom_trans[i].virt))
 585                         break;
 586         }
 587         last = i;
 588
 589         for (i = 0; i < (last - first); i++) {
 590                 struct linux_prom_translation *src = &prom_trans[i + first];
 591                 struct linux_prom_translation *dest = &prom_trans[i];
 592
 593                 *dest = *src;
 594         }
 595         for (; i < ents; i++) {
 596                 struct linux_prom_translation *dest = &prom_trans[i];
 597                 dest->virt = dest->size = dest->data = 0x0UL;
 598         }
 599
 600         prom_trans_ents = last - first;
 601
 602         if (tlb_type == spitfire) {
 603                 /* Clear diag TTE bits. */
 604                 for (i = 0; i < prom_trans_ents; i++)
 605                         prom_trans[i].data &= ~0x0003fe0000000000UL;
 606         }
 607
 608         /* Force execute bit on.  */
 609         for (i = 0; i < prom_trans_ents; i++)
 610                 prom_trans[i].data |= (tlb_type == hypervisor ?
 611                                        _PAGE_EXEC_4V : _PAGE_EXEC_4U);
 612 }
 613
 614 static void __init hypervisor_tlb_lock(unsigned long vaddr,
 615                                        unsigned long pte,
 616                                        unsigned long mmu)
 617 {
 618         unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu);
 619
 620         if (ret != 0) {
 621                 prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: "
 622                             "errors with %lx\n", vaddr, 0, pte, mmu, ret);
 623                 prom_halt();
 624         }
 625 }
 626
 627 static unsigned long kern_large_tte(unsigned long paddr);
 628
 629 static void __init remap_kernel(void)
 630 {
 631         unsigned long phys_page, tte_vaddr, tte_data;
 632         int i, tlb_ent = sparc64_highest_locked_tlbent();
 633
 634         tte_vaddr = (unsigned long) KERNBASE;
 635         phys_page = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;
 636         tte_data = kern_large_tte(phys_page);
 637
 638         kern_locked_tte_data = tte_data;
 639
 640         /* Now lock us into the TLBs via Hypervisor or OBP. */
 641         if (tlb_type == hypervisor) {
 642                 for (i = 0; i < num_kernel_image_mappings; i++) {
 643                         hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU);
 644                         hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU);
 645                         tte_vaddr += 0x400000;
 646                         tte_data += 0x400000;
 647                 }
 648         } else {
 649                 for (i = 0; i < num_kernel_image_mappings; i++) {
 650                         prom_dtlb_load(tlb_ent - i, tte_data, tte_vaddr);
 651                         prom_itlb_load(tlb_ent - i, tte_data, tte_vaddr);
 652                         tte_vaddr += 0x400000;
 653                         tte_data += 0x400000;
 654                 }
 655                 sparc64_highest_unlocked_tlb_ent = tlb_ent - i;
 656         }
 657         if (tlb_type == cheetah_plus) {
 658                 sparc64_kern_pri_context = (CTX_CHEETAH_PLUS_CTX0 |
 659                                             CTX_CHEETAH_PLUS_NUC);
 660                 sparc64_kern_pri_nuc_bits = CTX_CHEETAH_PLUS_NUC;
 661                 sparc64_kern_sec_context = CTX_CHEETAH_PLUS_CTX0;
 662         }
 663 }
 664
 665
 666 static void __init inherit_prom_mappings(void)
 667 {
 668         /* Now fixup OBP's idea about where we really are mapped. */
 669         printk("Remapping the kernel... ");
 670         remap_kernel();
 671         printk("done.\n");
 672 }
 673
 674 void prom_world(int enter)
 675 {
 676         if (!enter)
 677                 set_fs(get_fs());
 678
 679         __asm__ __volatile__("flushw");
 680 }
 681
 682 void __flush_dcache_range(unsigned long start, unsigned long end)
 683 {
 684         unsigned long va;
 685
 686         if (tlb_type == spitfire) {
 687                 int n = 0;
 688
 689                 for (va = start; va < end; va += 32) {
 690                         spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
 691                         if (++n >= 512)
 692                                 break;
 693                 }
 694         } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 695                 start = __pa(start);
 696                 end = __pa(end);
 697                 for (va = start; va < end; va += 32)
 698                         __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
 699                                              "membar #Sync"
 700                                              : /* no outputs */
 701                                              : "r" (va),
 702                                                "i" (ASI_DCACHE_INVALIDATE));
 703         }
 704 }
 705 EXPORT_SYMBOL(__flush_dcache_range);
 706
 707 /* get_new_mmu_context() uses "cache + 1".  */
 708 DEFINE_SPINLOCK(ctx_alloc_lock);
 709 unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;
 710 #define MAX_CTX_NR      (1UL << CTX_NR_BITS)
 711 #define CTX_BMAP_SLOTS  BITS_TO_LONGS(MAX_CTX_NR)
 712 DECLARE_BITMAP(mmu_context_bmap, MAX_CTX_NR);
 713
 714 /* Caller does TLB context flushing on local CPU if necessary.
 715  * The caller also ensures that CTX_VALID(mm->context) is false.
 716  *
 717  * We must be careful about boundary cases so that we never
 718  * let the user have CTX 0 (nucleus) or we ever use a CTX
 719  * version of zero (and thus NO_CONTEXT would not be caught
 720  * by version mis-match tests in mmu_context.h).
 721  *
 722  * Always invoked with interrupts disabled.
 723  */
 724 void get_new_mmu_context(struct mm_struct *mm)
 725 {
 726         unsigned long ctx, new_ctx;
 727         unsigned long orig_pgsz_bits;
 728         int new_version;
 729
 730         spin_lock(&ctx_alloc_lock);
 731         orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);
 732         ctx = (tlb_context_cache + 1) & CTX_NR_MASK;
 733         new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);
 734         new_version = 0;
 735         if (new_ctx >= (1 << CTX_NR_BITS)) {
 736                 new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
 737                 if (new_ctx >= ctx) {
 738                         int i;
 739                         new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +
 740                                 CTX_FIRST_VERSION;
 741                         if (new_ctx == 1)
 742                                 new_ctx = CTX_FIRST_VERSION;
 743
 744                         /* Don't call memset, for 16 entries that's just
 745                          * plain silly...
 746                          */
 747                         mmu_context_bmap[0] = 3;
 748                         mmu_context_bmap[1] = 0;
 749                         mmu_context_bmap[2] = 0;
 750                         mmu_context_bmap[3] = 0;
 751                         for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {
 752                                 mmu_context_bmap[i + 0] = 0;
 753                                 mmu_context_bmap[i + 1] = 0;
 754                                 mmu_context_bmap[i + 2] = 0;
 755                                 mmu_context_bmap[i + 3] = 0;
 756                         }
 757                         new_version = 1;
 758                         goto out;
 759                 }
 760         }
 761         mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
 762         new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
 763 out:
 764         tlb_context_cache = new_ctx;
 765         mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;
 766         spin_unlock(&ctx_alloc_lock);
 767
 768         if (unlikely(new_version))
 769                 smp_new_mmu_context_version();
 770 }
 771
 772 static int numa_enabled = 1;
 773 static int numa_debug;
 774
 775 static int __init early_numa(char *p)
 776 {
 777         if (!p)
 778                 return 0;
 779
 780         if (strstr(p, "off"))
 781                 numa_enabled = 0;
 782
 783         if (strstr(p, "debug"))
 784                 numa_debug = 1;
 785
 786         return 0;
 787 }
 788 early_param("numa", early_numa);
 789
 790 #define numadbg(f, a...) \
 791 do {    if (numa_debug) \
 792                 printk(KERN_INFO f, ## a); \
 793 } while (0)
 794
 795 static void __init find_ramdisk(unsigned long phys_base)
 796 {
 797 #ifdef CONFIG_BLK_DEV_INITRD
 798         if (sparc_ramdisk_image || sparc_ramdisk_image64) {
 799                 unsigned long ramdisk_image;
 800
 801                 /* Older versions of the bootloader only supported a
 802                  * 32-bit physical address for the ramdisk image
 803                  * location, stored at sparc_ramdisk_image.  Newer
 804                  * SILO versions set sparc_ramdisk_image to zero and
 805                  * provide a full 64-bit physical address at
 806                  * sparc_ramdisk_image64.
 807                  */
 808                 ramdisk_image = sparc_ramdisk_image;
 809                 if (!ramdisk_image)
 810                         ramdisk_image = sparc_ramdisk_image64;
 811
 812                 /* Another bootloader quirk.  The bootloader normalizes
 813                  * the physical address to KERNBASE, so we have to
 814                  * factor that back out and add in the lowest valid
 815                  * physical page address to get the true physical address.
 816                  */
 817                 ramdisk_image -= KERNBASE;
 818                 ramdisk_image += phys_base;
 819
 820                 numadbg("Found ramdisk at physical address 0x%lx, size %u\n",
 821                         ramdisk_image, sparc_ramdisk_size);
 822
 823                 initrd_start = ramdisk_image;
 824                 initrd_end = ramdisk_image + sparc_ramdisk_size;
 825
 826                 memblock_reserve(initrd_start, sparc_ramdisk_size);
 827
 828                 initrd_start += PAGE_OFFSET;
 829                 initrd_end += PAGE_OFFSET;
 830         }
 831 #endif
 832 }
 833
 834 struct node_mem_mask {
 835         unsigned long mask;
 836         unsigned long match;
 837 };
 838 static struct node_mem_mask node_masks[MAX_NUMNODES];
 839 static int num_node_masks;
 840
 841 #ifdef CONFIG_NEED_MULTIPLE_NODES
 842
 843 struct mdesc_mlgroup {
 844         u64     node;
 845         u64     latency;
 846         u64     match;
 847         u64     mask;
 848 };
 849
 850 static struct mdesc_mlgroup *mlgroups;
 851 static int num_mlgroups;
 852
 853 int numa_cpu_lookup_table[NR_CPUS];
 854 cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES];
 855
 856 struct mdesc_mblock {
 857         u64     base;
 858         u64     size;
 859         u64     offset; /* RA-to-PA */
 860 };
 861 static struct mdesc_mblock *mblocks;
 862 static int num_mblocks;
 863
 864 static struct mdesc_mblock * __init addr_to_mblock(unsigned long addr)
 865 {
 866         struct mdesc_mblock *m = NULL;
 867         int i;
 868
 869         for (i = 0; i < num_mblocks; i++) {
 870                 m = &mblocks[i];
 871
 872                 if (addr >= m->base &&
 873                     addr < (m->base + m->size)) {
 874                         break;
 875                 }
 876         }
 877
 878         return m;
 879 }
 880
 881 static u64 __init memblock_nid_range_sun4u(u64 start, u64 end, int *nid)
 882 {
 883         int prev_nid, new_nid;
 884
 885         prev_nid = -1;
 886         for ( ; start < end; start += PAGE_SIZE) {
 887                 for (new_nid = 0; new_nid < num_node_masks; new_nid++) {
 888                         struct node_mem_mask *p = &node_masks[new_nid];
 889
 890                         if ((start & p->mask) == p->match) {
 891                                 if (prev_nid == -1)
 892                                         prev_nid = new_nid;
 893                                 break;
 894                         }
 895                 }
 896
 897                 if (new_nid == num_node_masks) {
 898                         prev_nid = 0;
 899                         WARN_ONCE(1, "addr[%Lx] doesn't match a NUMA node rule. Some memory will be owned by node 0.",
 900                                   start);
 901                         break;
 902                 }
 903
 904                 if (prev_nid != new_nid)
 905                         break;
 906         }
 907         *nid = prev_nid;
 908
 909         return start > end ? end : start;
 910 }
 911
 912 static u64 __init memblock_nid_range(u64 start, u64 end, int *nid)
 913 {
 914         u64 ret_end, pa_start, m_mask, m_match, m_end;
 915         struct mdesc_mblock *mblock;
 916         int _nid, i;
 917
 918         if (tlb_type != hypervisor)
 919                 return memblock_nid_range_sun4u(start, end, nid);
 920
 921         mblock = addr_to_mblock(start);
 922         if (!mblock) {
 923                 WARN_ONCE(1, "memblock_nid_range: Can't find mblock addr[%Lx]",
 924                           start);
 925
 926                 _nid = 0;
 927                 ret_end = end;
 928                 goto done;
 929         }
 930
 931         pa_start = start + mblock->offset;
 932         m_match = 0;
 933         m_mask = 0;
 934
 935         for (_nid = 0; _nid < num_node_masks; _nid++) {
 936                 struct node_mem_mask *const m = &node_masks[_nid];
 937
 938                 if ((pa_start & m->mask) == m->match) {
 939                         m_match = m->match;
 940                         m_mask = m->mask;
 941                         break;
 942                 }
 943         }
 944
 945         if (num_node_masks == _nid) {
 946                 /* We could not find NUMA group, so default to 0, but lets
 947                  * search for latency group, so we could calculate the correct
 948                  * end address that we return
 949                  */
 950                 _nid = 0;
 951
 952                 for (i = 0; i < num_mlgroups; i++) {
 953                         struct mdesc_mlgroup *const m = &mlgroups[i];
 954
 955                         if ((pa_start & m->mask) == m->match) {
 956                                 m_match = m->match;
 957                                 m_mask = m->mask;
 958                                 break;
 959                         }
 960                 }
 961
 962                 if (i == num_mlgroups) {
 963                         WARN_ONCE(1, "memblock_nid_range: Can't find latency group addr[%Lx]",
 964                                   start);
 965
 966                         ret_end = end;
 967                         goto done;
 968                 }
 969         }
 970
 971         /*
 972          * Each latency group has match and mask, and each memory block has an
 973          * offset.  An address belongs to a latency group if its address matches
 974          * the following formula: ((addr + offset) & mask) == match
 975          * It is, however, slow to check every single page if it matches a
 976          * particular latency group. As optimization we calculate end value by
 977          * using bit arithmetics.
 978          */
 979         m_end = m_match + (1ul << __ffs(m_mask)) - mblock->offset;
 980         m_end += pa_start & ~((1ul << fls64(m_mask)) - 1);
 981         ret_end = m_end > end ? end : m_end;
 982
 983 done:
 984         *nid = _nid;
 985         return ret_end;
 986 }
 987 #endif
 988
 989 /* This must be invoked after performing all of the necessary
 990  * memblock_set_node() calls for 'nid'.  We need to be able to get
 991  * correct data from get_pfn_range_for_nid().
 992  */
 993 static void __init allocate_node_data(int nid)
 994 {
 995         struct pglist_data *p;
 996         unsigned long start_pfn, end_pfn;
 997 #ifdef CONFIG_NEED_MULTIPLE_NODES
 998         unsigned long paddr;
 999
1000         paddr = memblock_alloc_try_nid(sizeof(struct pglist_data), SMP_CACHE_BYTES, nid);
1001         if (!paddr) {
1002                 prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid);
1003                 prom_halt();
1004         }
1005         NODE_DATA(nid) = __va(paddr);
1006         memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));
1007
1008         NODE_DATA(nid)->node_id = nid;
1009 #endif
1010
1011         p = NODE_DATA(nid);
1012
1013         get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
1014         p->node_start_pfn = start_pfn;
1015         p->node_spanned_pages = end_pfn - start_pfn;
1016 }
1017
1018 static void init_node_masks_nonnuma(void)
1019 {
1020 #ifdef CONFIG_NEED_MULTIPLE_NODES
1021         int i;
1022 #endif
1023
1024         numadbg("Initializing tables for non-numa.\n");
1025
1026         node_masks[0].mask = 0;
1027         node_masks[0].match = 0;
1028         num_node_masks = 1;
1029
1030 #ifdef CONFIG_NEED_MULTIPLE_NODES
1031         for (i = 0; i < NR_CPUS; i++)
1032                 numa_cpu_lookup_table[i] = 0;
1033
1034         cpumask_setall(&numa_cpumask_lookup_table[0]);
1035 #endif
1036 }
1037
1038 #ifdef CONFIG_NEED_MULTIPLE_NODES
1039 struct pglist_data *node_data[MAX_NUMNODES];
1040
1041 EXPORT_SYMBOL(numa_cpu_lookup_table);
1042 EXPORT_SYMBOL(numa_cpumask_lookup_table);
1043 EXPORT_SYMBOL(node_data);
1044
1045 static int scan_pio_for_cfg_handle(struct mdesc_handle *md, u64 pio,
1046                                    u32 cfg_handle)
1047 {
1048         u64 arc;
1049
1050         mdesc_for_each_arc(arc, md, pio, MDESC_ARC_TYPE_FWD) {
1051                 u64 target = mdesc_arc_target(md, arc);
1052                 const u64 *val;
1053
1054                 val = mdesc_get_property(md, target,
1055                                          "cfg-handle", NULL);
1056                 if (val && *val == cfg_handle)
1057                         return 0;
1058         }
1059         return -ENODEV;
1060 }
1061
1062 static int scan_arcs_for_cfg_handle(struct mdesc_handle *md, u64 grp,
1063                                     u32 cfg_handle)
1064 {
1065         u64 arc, candidate, best_latency = ~(u64)0;
1066
1067         candidate = MDESC_NODE_NULL;
1068         mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
1069                 u64 target = mdesc_arc_target(md, arc);
1070                 const char *name = mdesc_node_name(md, target);
1071                 const u64 *val;
1072
1073                 if (strcmp(name, "pio-latency-group"))
1074                         continue;
1075
1076                 val = mdesc_get_property(md, target, "latency", NULL);
1077                 if (!val)
1078                         continue;
1079
1080                 if (*val < best_latency) {
1081                         candidate = target;
1082                         best_latency = *val;
1083                 }
1084         }
1085
1086         if (candidate == MDESC_NODE_NULL)
1087                 return -ENODEV;
1088
1089         return scan_pio_for_cfg_handle(md, candidate, cfg_handle);
1090 }
1091
1092 int of_node_to_nid(struct device_node *dp)
1093 {
1094         const struct linux_prom64_registers *regs;
1095         struct mdesc_handle *md;
1096         u32 cfg_handle;
1097         int count, nid;
1098         u64 grp;
1099
1100         /* This is the right thing to do on currently supported
1101          * SUN4U NUMA platforms as well, as the PCI controller does
1102          * not sit behind any particular memory controller.
1103          */
1104         if (!mlgroups)
1105                 return -1;
1106
1107         regs = of_get_property(dp, "reg", NULL);
1108         if (!regs)
1109                 return -1;
1110
1111         cfg_handle = (regs->phys_addr >> 32UL) & 0x0fffffff;
1112
1113         md = mdesc_grab();
1114
1115         count = 0;
1116         nid = -1;
1117         mdesc_for_each_node_by_name(md, grp, "group") {
1118                 if (!scan_arcs_for_cfg_handle(md, grp, cfg_handle)) {
1119                         nid = count;
1120                         break;
1121                 }
1122                 count++;
1123         }
1124
1125         mdesc_release(md);
1126
1127         return nid;
1128 }
1129
1130 static void __init add_node_ranges(void)
1131 {
1132         struct memblock_region *reg;
1133         unsigned long prev_max;
1134
1135 memblock_resized:
1136         prev_max = memblock.memory.max;
1137
1138         for_each_memblock(memory, reg) {
1139                 unsigned long size = reg->size;
1140                 unsigned long start, end;
1141
1142                 start = reg->base;
1143                 end = start + size;
1144                 while (start < end) {
1145                         unsigned long this_end;
1146                         int nid;
1147
1148                         this_end = memblock_nid_range(start, end, &nid);
1149
1150                         numadbg("Setting memblock NUMA node nid[%d] "
1151                                 "start[%lx] end[%lx]\n",
1152                                 nid, start, this_end);
1153
1154                         memblock_set_node(start, this_end - start,
1155                                           &memblock.memory, nid);
1156                         if (memblock.memory.max != prev_max)
1157                                 goto memblock_resized;
1158                         start = this_end;
1159                 }
1160         }
1161 }
1162
1163 static int __init grab_mlgroups(struct mdesc_handle *md)
1164 {
1165         unsigned long paddr;
1166         int count = 0;
1167         u64 node;
1168
1169         mdesc_for_each_node_by_name(md, node, "memory-latency-group")
1170                 count++;
1171         if (!count)
1172                 return -ENOENT;
1173
1174         paddr = memblock_alloc(count * sizeof(struct mdesc_mlgroup),
1175                           SMP_CACHE_BYTES);
1176         if (!paddr)
1177                 return -ENOMEM;
1178
1179         mlgroups = __va(paddr);
1180         num_mlgroups = count;
1181
1182         count = 0;
1183         mdesc_for_each_node_by_name(md, node, "memory-latency-group") {
1184                 struct mdesc_mlgroup *m = &mlgroups[count++];
1185                 const u64 *val;
1186
1187                 m->node = node;
1188
1189                 val = mdesc_get_property(md, node, "latency", NULL);
1190                 m->latency = *val;
1191                 val = mdesc_get_property(md, node, "address-match", NULL);
1192                 m->match = *val;
1193                 val = mdesc_get_property(md, node, "address-mask", NULL);
1194                 m->mask = *val;
1195
1196                 numadbg("MLGROUP[%d]: node[%llx] latency[%llx] "
1197                         "match[%llx] mask[%llx]\n",
1198                         count - 1, m->node, m->latency, m->match, m->mask);
1199         }
1200
1201         return 0;
1202 }
1203
1204 static int __init grab_mblocks(struct mdesc_handle *md)
1205 {
1206         unsigned long paddr;
1207         int count = 0;
1208         u64 node;
1209
1210         mdesc_for_each_node_by_name(md, node, "mblock")
1211                 count++;
1212         if (!count)
1213                 return -ENOENT;
1214
1215         paddr = memblock_alloc(count * sizeof(struct mdesc_mblock),
1216                           SMP_CACHE_BYTES);
1217         if (!paddr)
1218                 return -ENOMEM;
1219
1220         mblocks = __va(paddr);
1221         num_mblocks = count;
1222
1223         count = 0;
1224         mdesc_for_each_node_by_name(md, node, "mblock") {
1225                 struct mdesc_mblock *m = &mblocks[count++];
1226                 const u64 *val;
1227
1228                 val = mdesc_get_property(md, node, "base", NULL);
1229                 m->base = *val;
1230                 val = mdesc_get_property(md, node, "size", NULL);
1231                 m->size = *val;
1232                 val = mdesc_get_property(md, node,
1233                                          "address-congruence-offset", NULL);
1234
1235                 /* The address-congruence-offset property is optional.
1236                  * Explicity zero it be identifty this.
1237                  */
1238                 if (val)
1239                         m->offset = *val;
1240                 else
1241                         m->offset = 0UL;
1242
1243                 numadbg("MBLOCK[%d]: base[%llx] size[%llx] offset[%llx]\n",
1244                         count - 1, m->base, m->size, m->offset);
1245         }
1246
1247         return 0;
1248 }
1249
1250 static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md,
1251                                                u64 grp, cpumask_t *mask)
1252 {
1253         u64 arc;
1254
1255         cpumask_clear(mask);
1256
1257         mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_BACK) {
1258                 u64 target = mdesc_arc_target(md, arc);
1259                 const char *name = mdesc_node_name(md, target);
1260                 const u64 *id;
1261
1262                 if (strcmp(name, "cpu"))
1263                         continue;
1264                 id = mdesc_get_property(md, target, "id", NULL);
1265                 if (*id < nr_cpu_ids)
1266                         cpumask_set_cpu(*id, mask);
1267         }
1268 }
1269
1270 static struct mdesc_mlgroup * __init find_mlgroup(u64 node)
1271 {
1272         int i;
1273
1274         for (i = 0; i < num_mlgroups; i++) {
1275                 struct mdesc_mlgroup *m = &mlgroups[i];
1276                 if (m->node == node)
1277                         return m;
1278         }
1279         return NULL;
1280 }
1281
1282 int __node_distance(int from, int to)
1283 {
1284         if ((from >= MAX_NUMNODES) || (to >= MAX_NUMNODES)) {
1285                 pr_warn("Returning default NUMA distance value for %d->%d\n",
1286                         from, to);
1287                 return (from == to) ? LOCAL_DISTANCE : REMOTE_DISTANCE;
1288         }
1289         return numa_latency[from][to];
1290 }
1291
1292 static int __init find_best_numa_node_for_mlgroup(struct mdesc_mlgroup *grp)
1293 {
1294         int i;
1295
1296         for (i = 0; i < MAX_NUMNODES; i++) {
1297                 struct node_mem_mask *n = &node_masks[i];
1298
1299                 if ((grp->mask == n->mask) && (grp->match == n->match))
1300                         break;
1301         }
1302         return i;
1303 }
1304
1305 static void __init find_numa_latencies_for_group(struct mdesc_handle *md,
1306                                                  u64 grp, int index)
1307 {
1308         u64 arc;
1309
1310         mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
1311                 int tnode;
1312                 u64 target = mdesc_arc_target(md, arc);
1313                 struct mdesc_mlgroup *m = find_mlgroup(target);
1314
1315                 if (!m)
1316                         continue;
1317                 tnode = find_best_numa_node_for_mlgroup(m);
1318                 if (tnode == MAX_NUMNODES)
1319                         continue;
1320                 numa_latency[index][tnode] = m->latency;
1321         }
1322 }
1323
1324 static int __init numa_attach_mlgroup(struct mdesc_handle *md, u64 grp,
1325                                       int index)
1326 {
1327         struct mdesc_mlgroup *candidate = NULL;
1328         u64 arc, best_latency = ~(u64)0;
1329         struct node_mem_mask *n;
1330
1331         mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
1332                 u64 target = mdesc_arc_target(md, arc);
1333                 struct mdesc_mlgroup *m = find_mlgroup(target);
1334                 if (!m)
1335                         continue;
1336                 if (m->latency < best_latency) {
1337                         candidate = m;
1338                         best_latency = m->latency;
1339                 }
1340         }
1341         if (!candidate)
1342                 return -ENOENT;
1343
1344         if (num_node_masks != index) {
1345                 printk(KERN_ERR "Inconsistent NUMA state, "
1346                        "index[%d] != num_node_masks[%d]\n",
1347                        index, num_node_masks);
1348                 return -EINVAL;
1349         }
1350
1351         n = &node_masks[num_node_masks++];
1352
1353         n->mask = candidate->mask;
1354         n->match = candidate->match;
1355
1356         numadbg("NUMA NODE[%d]: mask[%lx] match[%lx] (latency[%llx])\n",
1357                 index, n->mask, n->match, candidate->latency);
1358
1359         return 0;
1360 }
1361
1362 static int __init numa_parse_mdesc_group(struct mdesc_handle *md, u64 grp,
1363                                          int index)
1364 {
1365         cpumask_t mask;
1366         int cpu;
1367
1368         numa_parse_mdesc_group_cpus(md, grp, &mask);
1369
1370         for_each_cpu(cpu, &mask)
1371                 numa_cpu_lookup_table[cpu] = index;
1372         cpumask_copy(&numa_cpumask_lookup_table[index], &mask);
1373
1374         if (numa_debug) {
1375                 printk(KERN_INFO "NUMA GROUP[%d]: cpus [ ", index);
1376                 for_each_cpu(cpu, &mask)
1377                         printk("%d ", cpu);
1378                 printk("]\n");
1379         }
1380
1381         return numa_attach_mlgroup(md, grp, index);
1382 }
1383
1384 static int __init numa_parse_mdesc(void)
1385 {
1386         struct mdesc_handle *md = mdesc_grab();
1387         int i, j, err, count;
1388         u64 node;
1389
1390         node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups");
1391         if (node == MDESC_NODE_NULL) {
1392                 mdesc_release(md);
1393                 return -ENOENT;
1394         }
1395
1396         err = grab_mblocks(md);
1397         if (err < 0)
1398                 goto out;
1399
1400         err = grab_mlgroups(md);
1401         if (err < 0)
1402                 goto out;
1403
1404         count = 0;
1405         mdesc_for_each_node_by_name(md, node, "group") {
1406                 err = numa_parse_mdesc_group(md, node, count);
1407                 if (err < 0)
1408                         break;
1409                 count++;
1410         }
1411
1412         count = 0;
1413         mdesc_for_each_node_by_name(md, node, "group") {
1414                 find_numa_latencies_for_group(md, node, count);
1415                 count++;
1416         }
1417
1418         /* Normalize numa latency matrix according to ACPI SLIT spec. */
1419         for (i = 0; i < MAX_NUMNODES; i++) {
1420                 u64 self_latency = numa_latency[i][i];
1421
1422                 for (j = 0; j < MAX_NUMNODES; j++) {
1423                         numa_latency[i][j] =
1424                                 (numa_latency[i][j] * LOCAL_DISTANCE) /
1425                                 self_latency;
1426                 }
1427         }
1428
1429         add_node_ranges();
1430
1431         for (i = 0; i < num_node_masks; i++) {
1432                 allocate_node_data(i);
1433                 node_set_online(i);
1434         }
1435
1436         err = 0;
1437 out:
1438         mdesc_release(md);
1439         return err;
1440 }
1441
1442 static int __init numa_parse_jbus(void)
1443 {
1444         unsigned long cpu, index;
1445
1446         /* NUMA node id is encoded in bits 36 and higher, and there is
1447          * a 1-to-1 mapping from CPU ID to NUMA node ID.
1448          */
1449         index = 0;
1450         for_each_present_cpu(cpu) {
1451                 numa_cpu_lookup_table[cpu] = index;
1452                 cpumask_copy(&numa_cpumask_lookup_table[index], cpumask_of(cpu));
1453                 node_masks[index].mask = ~((1UL << 36UL) - 1UL);
1454                 node_masks[index].match = cpu << 36UL;
1455
1456                 index++;
1457         }
1458         num_node_masks = index;
1459
1460         add_node_ranges();
1461
1462         for (index = 0; index < num_node_masks; index++) {
1463                 allocate_node_data(index);
1464                 node_set_online(index);
1465         }
1466
1467         return 0;
1468 }
1469
1470 static int __init numa_parse_sun4u(void)
1471 {
1472         if (tlb_type == cheetah || tlb_type == cheetah_plus) {
1473                 unsigned long ver;
1474
1475                 __asm__ ("rdpr %%ver, %0" : "=r" (ver));
1476                 if ((ver >> 32UL) == __JALAPENO_ID ||
1477                     (ver >> 32UL) == __SERRANO_ID)
1478                         return numa_parse_jbus();
1479         }
1480         return -1;
1481 }
1482
1483 static int __init bootmem_init_numa(void)
1484 {
1485         int i, j;
1486         int err = -1;
1487
1488         numadbg("bootmem_init_numa()\n");
1489
1490         /* Some sane defaults for numa latency values */
1491         for (i = 0; i < MAX_NUMNODES; i++) {
1492                 for (j = 0; j < MAX_NUMNODES; j++)
1493                         numa_latency[i][j] = (i == j) ?
1494                                 LOCAL_DISTANCE : REMOTE_DISTANCE;
1495         }
1496
1497         if (numa_enabled) {
1498                 if (tlb_type == hypervisor)
1499                         err = numa_parse_mdesc();
1500                 else
1501                         err = numa_parse_sun4u();
1502         }
1503         return err;
1504 }
1505
1506 #else
1507
1508 static int bootmem_init_numa(void)
1509 {
1510         return -1;
1511 }
1512
1513 #endif
1514
1515 static void __init bootmem_init_nonnuma(void)
1516 {
1517         unsigned long top_of_ram = memblock_end_of_DRAM();
1518         unsigned long total_ram = memblock_phys_mem_size();
1519
1520         numadbg("bootmem_init_nonnuma()\n");
1521
1522         printk(KERN_INFO "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
1523                top_of_ram, total_ram);
1524         printk(KERN_INFO "Memory hole size: %ldMB\n",
1525                (top_of_ram - total_ram) >> 20);
1526
1527         init_node_masks_nonnuma();
1528         memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);
1529         allocate_node_data(0);
1530         node_set_online(0);
1531 }
1532
1533 static unsigned long __init bootmem_init(unsigned long phys_base)
1534 {
1535         unsigned long end_pfn;
1536
1537         end_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
1538         max_pfn = max_low_pfn = end_pfn;
1539         min_low_pfn = (phys_base >> PAGE_SHIFT);
1540
1541         if (bootmem_init_numa() < 0)
1542                 bootmem_init_nonnuma();
1543
1544         /* Dump memblock with node info. */
1545         memblock_dump_all();
1546
1547         /* XXX cpu notifier XXX */
1548
1549         sparse_memory_present_with_active_regions(MAX_NUMNODES);
1550         sparse_init();
1551
1552         return end_pfn;
1553 }
1554
1555 static struct linux_prom64_registers pall[MAX_BANKS] __initdata;
1556 static int pall_ents __initdata;
1557
1558 static unsigned long max_phys_bits = 40;
1559
1560 bool kern_addr_valid(unsigned long addr)
1561 {
1562         pgd_t *pgd;
1563         pud_t *pud;
1564         pmd_t *pmd;
1565         pte_t *pte;
1566
1567         if ((long)addr < 0L) {
1568                 unsigned long pa = __pa(addr);
1569
1570                 if ((pa >> max_phys_bits) != 0UL)
1571                         return false;
1572
1573                 return pfn_valid(pa >> PAGE_SHIFT);
1574         }
1575
1576         if (addr >= (unsigned long) KERNBASE &&
1577             addr < (unsigned long)&_end)
1578                 return true;
1579
1580         pgd = pgd_offset_k(addr);
1581         if (pgd_none(*pgd))
1582                 return 0;
1583
1584         pud = pud_offset(pgd, addr);
1585         if (pud_none(*pud))
1586                 return 0;
1587
1588         if (pud_large(*pud))
1589                 return pfn_valid(pud_pfn(*pud));
1590
1591         pmd = pmd_offset(pud, addr);
1592         if (pmd_none(*pmd))
1593                 return 0;
1594
1595         if (pmd_large(*pmd))
1596                 return pfn_valid(pmd_pfn(*pmd));
1597
1598         pte = pte_offset_kernel(pmd, addr);
1599         if (pte_none(*pte))
1600                 return 0;
1601
1602         return pfn_valid(pte_pfn(*pte));
1603 }
1604 EXPORT_SYMBOL(kern_addr_valid);
1605
1606 static unsigned long __ref kernel_map_hugepud(unsigned long vstart,
1607                                               unsigned long vend,
1608                                               pud_t *pud)
1609 {
1610         const unsigned long mask16gb = (1UL << 34) - 1UL;
1611         u64 pte_val = vstart;
1612
1613         /* Each PUD is 8GB */
1614         if ((vstart & mask16gb) ||
1615             (vend - vstart <= mask16gb)) {
1616                 pte_val ^= kern_linear_pte_xor[2];
1617                 pud_val(*pud) = pte_val | _PAGE_PUD_HUGE;
1618
1619                 return vstart + PUD_SIZE;
1620         }
1621
1622         pte_val ^= kern_linear_pte_xor[3];
1623         pte_val |= _PAGE_PUD_HUGE;
1624
1625         vend = vstart + mask16gb + 1UL;
1626         while (vstart < vend) {
1627                 pud_val(*pud) = pte_val;
1628
1629                 pte_val += PUD_SIZE;
1630                 vstart += PUD_SIZE;
1631                 pud++;
1632         }
1633         return vstart;
1634 }
1635
1636 static bool kernel_can_map_hugepud(unsigned long vstart, unsigned long vend,
1637                                    bool guard)
1638 {
1639         if (guard && !(vstart & ~PUD_MASK) && (vend - vstart) >= PUD_SIZE)
1640                 return true;
1641
1642         return false;
1643 }
1644
1645 static unsigned long __ref kernel_map_hugepmd(unsigned long vstart,
1646                                               unsigned long vend,
1647                                               pmd_t *pmd)
1648 {
1649         const unsigned long mask256mb = (1UL << 28) - 1UL;
1650         const unsigned long mask2gb = (1UL << 31) - 1UL;
1651         u64 pte_val = vstart;
1652
1653         /* Each PMD is 8MB */
1654         if ((vstart & mask256mb) ||
1655             (vend - vstart <= mask256mb)) {
1656                 pte_val ^= kern_linear_pte_xor[0];
1657                 pmd_val(*pmd) = pte_val | _PAGE_PMD_HUGE;
1658
1659                 return vstart + PMD_SIZE;
1660         }
1661
1662         if ((vstart & mask2gb) ||
1663             (vend - vstart <= mask2gb)) {
1664                 pte_val ^= kern_linear_pte_xor[1];
1665                 pte_val |= _PAGE_PMD_HUGE;
1666                 vend = vstart + mask256mb + 1UL;
1667         } else {
1668                 pte_val ^= kern_linear_pte_xor[2];
1669                 pte_val |= _PAGE_PMD_HUGE;
1670                 vend = vstart + mask2gb + 1UL;
1671         }
1672
1673         while (vstart < vend) {
1674                 pmd_val(*pmd) = pte_val;
1675
1676                 pte_val += PMD_SIZE;
1677                 vstart += PMD_SIZE;
1678                 pmd++;
1679         }
1680
1681         return vstart;
1682 }
1683
1684 static bool kernel_can_map_hugepmd(unsigned long vstart, unsigned long vend,
1685                                    bool guard)
1686 {
1687         if (guard && !(vstart & ~PMD_MASK) && (vend - vstart) >= PMD_SIZE)
1688                 return true;
1689
1690         return false;
1691 }
1692
1693 static unsigned long __ref kernel_map_range(unsigned long pstart,
1694                                             unsigned long pend, pgprot_t prot,
1695                                             bool use_huge)
1696 {
1697         unsigned long vstart = PAGE_OFFSET + pstart;
1698         unsigned long vend = PAGE_OFFSET + pend;
1699         unsigned long alloc_bytes = 0UL;
1700
1701         if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) {
1702                 prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n",
1703                             vstart, vend);
1704                 prom_halt();
1705         }
1706
1707         while (vstart < vend) {
1708                 unsigned long this_end, paddr = __pa(vstart);
1709                 pgd_t *pgd = pgd_offset_k(vstart);
1710                 pud_t *pud;
1711                 pmd_t *pmd;
1712                 pte_t *pte;
1713
1714                 if (pgd_none(*pgd)) {
1715                         pud_t *new;
1716
1717                         new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1718                         alloc_bytes += PAGE_SIZE;
1719                         pgd_populate(&init_mm, pgd, new);
1720                 }
1721                 pud = pud_offset(pgd, vstart);
1722                 if (pud_none(*pud)) {
1723                         pmd_t *new;
1724
1725                         if (kernel_can_map_hugepud(vstart, vend, use_huge)) {
1726                                 vstart = kernel_map_hugepud(vstart, vend, pud);
1727                                 continue;
1728                         }
1729                         new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1730                         alloc_bytes += PAGE_SIZE;
1731                         pud_populate(&init_mm, pud, new);
1732                 }
1733
1734                 pmd = pmd_offset(pud, vstart);
1735                 if (pmd_none(*pmd)) {
1736                         pte_t *new;
1737
1738                         if (kernel_can_map_hugepmd(vstart, vend, use_huge)) {
1739                                 vstart = kernel_map_hugepmd(vstart, vend, pmd);
1740                                 continue;
1741                         }
1742                         new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
1743                         alloc_bytes += PAGE_SIZE;
1744                         pmd_populate_kernel(&init_mm, pmd, new);
1745                 }
1746
1747                 pte = pte_offset_kernel(pmd, vstart);
1748                 this_end = (vstart + PMD_SIZE) & PMD_MASK;
1749                 if (this_end > vend)
1750                         this_end = vend;
1751
1752                 while (vstart < this_end) {
1753                         pte_val(*pte) = (paddr | pgprot_val(prot));
1754
1755                         vstart += PAGE_SIZE;
1756                         paddr += PAGE_SIZE;
1757                         pte++;
1758                 }
1759         }
1760
1761         return alloc_bytes;
1762 }
1763
1764 static void __init flush_all_kernel_tsbs(void)
1765 {
1766         int i;
1767
1768         for (i = 0; i < KERNEL_TSB_NENTRIES; i++) {
1769                 struct tsb *ent = &swapper_tsb[i];
1770
1771                 ent->tag = (1UL << TSB_TAG_INVALID_BIT);
1772         }
1773 #ifndef CONFIG_DEBUG_PAGEALLOC
1774         for (i = 0; i < KERNEL_TSB4M_NENTRIES; i++) {
1775                 struct tsb *ent = &swapper_4m_tsb[i];
1776
1777                 ent->tag = (1UL << TSB_TAG_INVALID_BIT);
1778         }
1779 #endif
1780 }
1781
1782 extern unsigned int kvmap_linear_patch[1];
1783
1784 static void __init kernel_physical_mapping_init(void)
1785 {
1786         unsigned long i, mem_alloced = 0UL;
1787         bool use_huge = true;
1788
1789 #ifdef CONFIG_DEBUG_PAGEALLOC
1790         use_huge = false;
1791 #endif
1792         for (i = 0; i < pall_ents; i++) {
1793                 unsigned long phys_start, phys_end;
1794
1795                 phys_start = pall[i].phys_addr;
1796                 phys_end = phys_start + pall[i].reg_size;
1797
1798                 mem_alloced += kernel_map_range(phys_start, phys_end,
1799                                                 PAGE_KERNEL, use_huge);
1800         }
1801
1802         printk("Allocated %ld bytes for kernel page tables.\n",
1803                mem_alloced);
1804
1805         kvmap_linear_patch[0] = 0x01000000; /* nop */
1806         flushi(&kvmap_linear_patch[0]);
1807
1808         flush_all_kernel_tsbs();
1809
1810         __flush_tlb_all();
1811 }
1812
1813 #ifdef CONFIG_DEBUG_PAGEALLOC
1814 void __kernel_map_pages(struct page *page, int numpages, int enable)
1815 {
1816         unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT;
1817         unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);
1818
1819         kernel_map_range(phys_start, phys_end,
1820                          (enable ? PAGE_KERNEL : __pgprot(0)), false);
1821
1822         flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
1823                                PAGE_OFFSET + phys_end);
1824
1825         /* we should perform an IPI and flush all tlbs,
1826          * but that can deadlock->flush only current cpu.
1827          */
1828         __flush_tlb_kernel_range(PAGE_OFFSET + phys_start,
1829                                  PAGE_OFFSET + phys_end);
1830 }
1831 #endif
1832
1833 unsigned long __init find_ecache_flush_span(unsigned long size)
1834 {
1835         int i;
1836
1837         for (i = 0; i < pavail_ents; i++) {
1838                 if (pavail[i].reg_size >= size)
1839                         return pavail[i].phys_addr;
1840         }
1841
1842         return ~0UL;
1843 }
1844
1845 unsigned long PAGE_OFFSET;
1846 EXPORT_SYMBOL(PAGE_OFFSET);
1847
1848 unsigned long VMALLOC_END   = 0x0000010000000000UL;
1849 EXPORT_SYMBOL(VMALLOC_END);
1850
1851 unsigned long sparc64_va_hole_top =    0xfffff80000000000UL;
1852 unsigned long sparc64_va_hole_bottom = 0x0000080000000000UL;
1853
1854 static void __init setup_page_offset(void)
1855 {
1856         if (tlb_type == cheetah || tlb_type == cheetah_plus) {
1857                 /* Cheetah/Panther support a full 64-bit virtual
1858                  * address, so we can use all that our page tables
1859                  * support.
1860                  */
1861                 sparc64_va_hole_top =    0xfff0000000000000UL;
1862                 sparc64_va_hole_bottom = 0x0010000000000000UL;
1863
1864                 max_phys_bits = 42;
1865         } else if (tlb_type == hypervisor) {
1866                 switch (sun4v_chip_type) {
1867                 case SUN4V_CHIP_NIAGARA1:
1868                 case SUN4V_CHIP_NIAGARA2:
1869                         /* T1 and T2 support 48-bit virtual addresses.  */
1870                         sparc64_va_hole_top =    0xffff800000000000UL;
1871                         sparc64_va_hole_bottom = 0x0000800000000000UL;
1872
1873                         max_phys_bits = 39;
1874                         break;
1875                 case SUN4V_CHIP_NIAGARA3:
1876                         /* T3 supports 48-bit virtual addresses.  */
1877                         sparc64_va_hole_top =    0xffff800000000000UL;
1878                         sparc64_va_hole_bottom = 0x0000800000000000UL;
1879
1880                         max_phys_bits = 43;
1881                         break;
1882                 case SUN4V_CHIP_NIAGARA4:
1883                 case SUN4V_CHIP_NIAGARA5:
1884                 case SUN4V_CHIP_SPARC64X:
1885                 case SUN4V_CHIP_SPARC_M6:
1886                         /* T4 and later support 52-bit virtual addresses.  */
1887                         sparc64_va_hole_top =    0xfff8000000000000UL;
1888                         sparc64_va_hole_bottom = 0x0008000000000000UL;
1889                         max_phys_bits = 47;
1890                         break;
1891                 case SUN4V_CHIP_SPARC_M7:
1892                 case SUN4V_CHIP_SPARC_SN:
1893                 default:
1894                         /* M7 and later support 52-bit virtual addresses.  */
1895                         sparc64_va_hole_top =    0xfff8000000000000UL;
1896                         sparc64_va_hole_bottom = 0x0008000000000000UL;
1897                         max_phys_bits = 49;
1898                         break;
1899                 }
1900         }
1901
1902         if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) {
1903                 prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n",
1904                             max_phys_bits);
1905                 prom_halt();
1906         }
1907
1908         PAGE_OFFSET = sparc64_va_hole_top;
1909         VMALLOC_END = ((sparc64_va_hole_bottom >> 1) +
1910                        (sparc64_va_hole_bottom >> 2));
1911
1912         pr_info("MM: PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n",
1913                 PAGE_OFFSET, max_phys_bits);
1914         pr_info("MM: VMALLOC [0x%016lx --> 0x%016lx]\n",
1915                 VMALLOC_START, VMALLOC_END);
1916         pr_info("MM: VMEMMAP [0x%016lx --> 0x%016lx]\n",
1917                 VMEMMAP_BASE, VMEMMAP_BASE << 1);
1918 }
1919
1920 static void __init tsb_phys_patch(void)
1921 {
1922         struct tsb_ldquad_phys_patch_entry *pquad;
1923         struct tsb_phys_patch_entry *p;
1924
1925         pquad = &__tsb_ldquad_phys_patch;
1926         while (pquad < &__tsb_ldquad_phys_patch_end) {
1927                 unsigned long addr = pquad->addr;
1928
1929                 if (tlb_type == hypervisor)
1930                         *(unsigned int *) addr = pquad->sun4v_insn;
1931                 else
1932                         *(unsigned int *) addr = pquad->sun4u_insn;
1933                 wmb();
1934                 __asm__ __volatile__("flush     %0"
1935                                      : /* no outputs */
1936                                      : "r" (addr));
1937
1938                 pquad++;
1939         }
1940
1941         p = &__tsb_phys_patch;
1942         while (p < &__tsb_phys_patch_end) {
1943                 unsigned long addr = p->addr;
1944
1945                 *(unsigned int *) addr = p->insn;
1946                 wmb();
1947                 __asm__ __volatile__("flush     %0"
1948                                      : /* no outputs */
1949                                      : "r" (addr));
1950
1951                 p++;
1952         }
1953 }
1954
1955 /* Don't mark as init, we give this to the Hypervisor.  */
1956 #ifndef CONFIG_DEBUG_PAGEALLOC
1957 #define NUM_KTSB_DESCR  2
1958 #else
1959 #define NUM_KTSB_DESCR  1
1960 #endif
1961 static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR];
1962
1963 /* The swapper TSBs are loaded with a base sequence of:
1964  *
1965  *      sethi   %uhi(SYMBOL), REG1
1966  *      sethi   %hi(SYMBOL), REG2
1967  *      or      REG1, %ulo(SYMBOL), REG1
1968  *      or      REG2, %lo(SYMBOL), REG2
1969  *      sllx    REG1, 32, REG1
1970  *      or      REG1, REG2, REG1
1971  *
1972  * When we use physical addressing for the TSB accesses, we patch the
1973  * first four instructions in the above sequence.
1974  */
1975
1976 static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa)
1977 {
1978         unsigned long high_bits, low_bits;
1979
1980         high_bits = (pa >> 32) & 0xffffffff;
1981         low_bits = (pa >> 0) & 0xffffffff;
1982
1983         while (start < end) {
1984                 unsigned int *ia = (unsigned int *)(unsigned long)*start;
1985
1986                 ia[0] = (ia[0] & ~0x3fffff) | (high_bits >> 10);
1987                 __asm__ __volatile__("flush     %0" : : "r" (ia));
1988
1989                 ia[1] = (ia[1] & ~0x3fffff) | (low_bits >> 10);
1990                 __asm__ __volatile__("flush     %0" : : "r" (ia + 1));
1991
1992                 ia[2] = (ia[2] & ~0x1fff) | (high_bits & 0x3ff);
1993                 __asm__ __volatile__("flush     %0" : : "r" (ia + 2));
1994
1995                 ia[3] = (ia[3] & ~0x1fff) | (low_bits & 0x3ff);
1996                 __asm__ __volatile__("flush     %0" : : "r" (ia + 3));
1997
1998                 start++;
1999         }
2000 }
2001
2002 static void ktsb_phys_patch(void)
2003 {
2004         extern unsigned int __swapper_tsb_phys_patch;
2005         extern unsigned int __swapper_tsb_phys_patch_end;
2006         unsigned long ktsb_pa;
2007
2008         ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
2009         patch_one_ktsb_phys(&__swapper_tsb_phys_patch,
2010                             &__swapper_tsb_phys_patch_end, ktsb_pa);
2011 #ifndef CONFIG_DEBUG_PAGEALLOC
2012         {
2013         extern unsigned int __swapper_4m_tsb_phys_patch;
2014         extern unsigned int __swapper_4m_tsb_phys_patch_end;
2015         ktsb_pa = (kern_base +
2016                    ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
2017         patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch,
2018                             &__swapper_4m_tsb_phys_patch_end, ktsb_pa);
2019         }
2020 #endif
2021 }
2022
2023 static void __init sun4v_ktsb_init(void)
2024 {
2025         unsigned long ktsb_pa;
2026
2027         /* First KTSB for PAGE_SIZE mappings.  */
2028         ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
2029
2030         switch (PAGE_SIZE) {
2031         case 8 * 1024:
2032         default:
2033                 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_8K;
2034                 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_8K;
2035                 break;
2036
2037         case 64 * 1024:
2038                 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_64K;
2039                 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_64K;
2040                 break;
2041
2042         case 512 * 1024:
2043                 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_512K;
2044                 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_512K;
2045                 break;
2046
2047         case 4 * 1024 * 1024:
2048                 ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB;
2049                 ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB;
2050                 break;
2051         }
2052
2053         ktsb_descr[0].assoc = 1;
2054         ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES;
2055         ktsb_descr[0].ctx_idx = 0;
2056         ktsb_descr[0].tsb_base = ktsb_pa;
2057         ktsb_descr[0].resv = 0;
2058
2059 #ifndef CONFIG_DEBUG_PAGEALLOC
2060         /* Second KTSB for 4MB/256MB/2GB/16GB mappings.  */
2061         ktsb_pa = (kern_base +
2062                    ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
2063
2064         ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB;
2065         ktsb_descr[1].pgsz_mask = ((HV_PGSZ_MASK_4MB |
2066                                     HV_PGSZ_MASK_256MB |
2067                                     HV_PGSZ_MASK_2GB |
2068                                     HV_PGSZ_MASK_16GB) &
2069                                    cpu_pgsz_mask);
2070         ktsb_descr[1].assoc = 1;
2071         ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES;
2072         ktsb_descr[1].ctx_idx = 0;
2073         ktsb_descr[1].tsb_base = ktsb_pa;
2074         ktsb_descr[1].resv = 0;
2075 #endif
2076 }
2077
2078 void sun4v_ktsb_register(void)
2079 {
2080         unsigned long pa, ret;
2081
2082         pa = kern_base + ((unsigned long)&ktsb_descr[0] - KERNBASE);
2083
2084         ret = sun4v_mmu_tsb_ctx0(NUM_KTSB_DESCR, pa);
2085         if (ret != 0) {
2086                 prom_printf("hypervisor_mmu_tsb_ctx0[%lx]: "
2087                             "errors with %lx\n", pa, ret);
2088                 prom_halt();
2089         }
2090 }
2091
2092 static void __init sun4u_linear_pte_xor_finalize(void)
2093 {
2094 #ifndef CONFIG_DEBUG_PAGEALLOC
2095         /* This is where we would add Panther support for
2096          * 32MB and 256MB pages.
2097          */
2098 #endif
2099 }
2100
2101 static void __init sun4v_linear_pte_xor_finalize(void)
2102 {
2103         unsigned long pagecv_flag;
2104
2105         /* Bit 9 of TTE is no longer CV bit on M7 processor and it instead
2106          * enables MCD error. Do not set bit 9 on M7 processor.
2107          */
2108         switch (sun4v_chip_type) {
2109         case SUN4V_CHIP_SPARC_M7:
2110         case SUN4V_CHIP_SPARC_SN:
2111                 pagecv_flag = 0x00;
2112                 break;
2113         default:
2114                 pagecv_flag = _PAGE_CV_4V;
2115                 break;
2116         }
2117 #ifndef CONFIG_DEBUG_PAGEALLOC
2118         if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) {
2119                 kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
2120                         PAGE_OFFSET;
2121                 kern_linear_pte_xor[1] |= (_PAGE_CP_4V | pagecv_flag |
2122                                            _PAGE_P_4V | _PAGE_W_4V);
2123         } else {
2124                 kern_linear_pte_xor[1] = kern_linear_pte_xor[0];
2125         }
2126
2127         if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) {
2128                 kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^
2129                         PAGE_OFFSET;
2130                 kern_linear_pte_xor[2] |= (_PAGE_CP_4V | pagecv_flag |
2131                                            _PAGE_P_4V | _PAGE_W_4V);
2132         } else {
2133                 kern_linear_pte_xor[2] = kern_linear_pte_xor[1];
2134         }
2135
2136         if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) {
2137                 kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^
2138                         PAGE_OFFSET;
2139                 kern_linear_pte_xor[3] |= (_PAGE_CP_4V | pagecv_flag |
2140                                            _PAGE_P_4V | _PAGE_W_4V);
2141         } else {
2142                 kern_linear_pte_xor[3] = kern_linear_pte_xor[2];
2143         }
2144 #endif
2145 }
2146
2147 /* paging_init() sets up the page tables */
2148
2149 static unsigned long last_valid_pfn;
2150
2151 static void sun4u_pgprot_init(void);
2152 static void sun4v_pgprot_init(void);
2153
2154 static phys_addr_t __init available_memory(void)
2155 {
2156         phys_addr_t available = 0ULL;
2157         phys_addr_t pa_start, pa_end;
2158         u64 i;
2159
2160         for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &pa_start,
2161                                 &pa_end, NULL)
2162                 available = available + (pa_end  - pa_start);
2163
2164         return available;
2165 }
2166
2167 #define _PAGE_CACHE_4U  (_PAGE_CP_4U | _PAGE_CV_4U)
2168 #define _PAGE_CACHE_4V  (_PAGE_CP_4V | _PAGE_CV_4V)
2169 #define __DIRTY_BITS_4U  (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)
2170 #define __DIRTY_BITS_4V  (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V)
2171 #define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R)
2172 #define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R)
2173
2174 /* We need to exclude reserved regions. This exclusion will include
2175  * vmlinux and initrd. To be more precise the initrd size could be used to
2176  * compute a new lower limit because it is freed later during initialization.
2177  */
2178 static void __init reduce_memory(phys_addr_t limit_ram)
2179 {
2180         phys_addr_t avail_ram = available_memory();
2181         phys_addr_t pa_start, pa_end;
2182         u64 i;
2183
2184         if (limit_ram >= avail_ram)
2185                 return;
2186
2187         for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &pa_start,
2188                                 &pa_end, NULL) {
2189                 phys_addr_t region_size = pa_end - pa_start;
2190                 phys_addr_t clip_start = pa_start;
2191
2192                 avail_ram = avail_ram - region_size;
2193                 /* Are we consuming too much? */
2194                 if (avail_ram < limit_ram) {
2195                         phys_addr_t give_back = limit_ram - avail_ram;
2196
2197                         region_size = region_size - give_back;
2198                         clip_start = clip_start + give_back;
2199                 }
2200
2201                 memblock_remove(clip_start, region_size);
2202
2203                 if (avail_ram <= limit_ram)
2204                         break;
2205                 i = 0UL;
2206         }
2207 }
2208
2209 void __init paging_init(void)
2210 {
2211         unsigned long end_pfn, shift, phys_base;
2212         unsigned long real_end, i;
2213
2214         setup_page_offset();
2215
2216         /* These build time checkes make sure that the dcache_dirty_cpu()
2217          * page->flags usage will work.
2218          *
2219          * When a page gets marked as dcache-dirty, we store the
2220          * cpu number starting at bit 32 in the page->flags.  Also,
2221          * functions like clear_dcache_dirty_cpu use the cpu mask
2222          * in 13-bit signed-immediate instruction fields.
2223          */
2224
2225         /*
2226          * Page flags must not reach into upper 32 bits that are used
2227          * for the cpu number
2228          */
2229         BUILD_BUG_ON(NR_PAGEFLAGS > 32);
2230
2231         /*
2232          * The bit fields placed in the high range must not reach below
2233          * the 32 bit boundary. Otherwise we cannot place the cpu field
2234          * at the 32 bit boundary.
2235          */
2236         BUILD_BUG_ON(SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH +
2237                 ilog2(roundup_pow_of_two(NR_CPUS)) > 32);
2238
2239         BUILD_BUG_ON(NR_CPUS > 4096);
2240
2241         kern_base = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;
2242         kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;
2243
2244         /* Invalidate both kernel TSBs.  */
2245         memset(swapper_tsb, 0x40, sizeof(swapper_tsb));
2246 #ifndef CONFIG_DEBUG_PAGEALLOC
2247         memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
2248 #endif
2249
2250         /* TTE.cv bit on sparc v9 occupies the same position as TTE.mcde
2251          * bit on M7 processor. This is a conflicting usage of the same
2252          * bit. Enabling TTE.cv on M7 would turn on Memory Corruption
2253          * Detection error on all pages and this will lead to problems
2254          * later. Kernel does not run with MCD enabled and hence rest
2255          * of the required steps to fully configure memory corruption
2256          * detection are not taken. We need to ensure TTE.mcde is not
2257          * set on M7 processor. Compute the value of cacheability
2258          * flag for use later taking this into consideration.
2259          */
2260         switch (sun4v_chip_type) {
2261         case SUN4V_CHIP_SPARC_M7:
2262         case SUN4V_CHIP_SPARC_SN:
2263                 page_cache4v_flag = _PAGE_CP_4V;
2264                 break;
2265         default:
2266                 page_cache4v_flag = _PAGE_CACHE_4V;
2267                 break;
2268         }
2269
2270         if (tlb_type == hypervisor)
2271                 sun4v_pgprot_init();
2272         else
2273                 sun4u_pgprot_init();
2274
2275         if (tlb_type == cheetah_plus ||
2276             tlb_type == hypervisor) {
2277                 tsb_phys_patch();
2278                 ktsb_phys_patch();
2279         }
2280
2281         if (tlb_type == hypervisor)
2282                 sun4v_patch_tlb_handlers();
2283
2284         /* Find available physical memory...
2285          *
2286          * Read it twice in order to work around a bug in openfirmware.
2287          * The call to grab this table itself can cause openfirmware to
2288          * allocate memory, which in turn can take away some space from
2289          * the list of available memory.  Reading it twice makes sure
2290          * we really do get the final value.
2291          */
2292         read_obp_translations();
2293         read_obp_memory("reg", &pall[0], &pall_ents);
2294         read_obp_memory("available", &pavail[0], &pavail_ents);
2295         read_obp_memory("available", &pavail[0], &pavail_ents);
2296
2297         phys_base = 0xffffffffffffffffUL;
2298         for (i = 0; i < pavail_ents; i++) {
2299                 phys_base = min(phys_base, pavail[i].phys_addr);
2300                 memblock_add(pavail[i].phys_addr, pavail[i].reg_size);
2301         }
2302
2303         memblock_reserve(kern_base, kern_size);
2304
2305         find_ramdisk(phys_base);
2306
2307         if (cmdline_memory_size)
2308                 reduce_memory(cmdline_memory_size);
2309
2310         memblock_allow_resize();
2311         memblock_dump_all();
2312
2313         set_bit(0, mmu_context_bmap);
2314
2315         shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE);
2316
2317         real_end = (unsigned long)_end;
2318         num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << ILOG2_4MB);
2319         printk("Kernel: Using %d locked TLB entries for main kernel image.\n",
2320                num_kernel_image_mappings);
2321
2322         /* Set kernel pgd to upper alias so physical page computations
2323          * work.
2324          */
2325         init_mm.pgd += ((shift) / (sizeof(pgd_t)));
2326
2327         memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir));
2328
2329         inherit_prom_mappings();
2330
2331         /* Ok, we can use our TLB miss and window trap handlers safely.  */
2332         setup_tba();
2333
2334         __flush_tlb_all();
2335
2336         prom_build_devicetree();
2337         of_populate_present_mask();
2338 #ifndef CONFIG_SMP
2339         of_fill_in_cpu_data();
2340 #endif
2341
2342         if (tlb_type == hypervisor) {
2343                 sun4v_mdesc_init();
2344                 mdesc_populate_present_mask(cpu_all_mask);
2345 #ifndef CONFIG_SMP
2346                 mdesc_fill_in_cpu_data(cpu_all_mask);
2347 #endif
2348                 mdesc_get_page_sizes(cpu_all_mask, &cpu_pgsz_mask);
2349
2350                 sun4v_linear_pte_xor_finalize();
2351
2352                 sun4v_ktsb_init();
2353                 sun4v_ktsb_register();
2354         } else {
2355                 unsigned long impl, ver;
2356
2357                 cpu_pgsz_mask = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K |
2358                                  HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB);
2359
2360                 __asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver));
2361                 impl = ((ver >> 32) & 0xffff);
2362                 if (impl == PANTHER_IMPL)
2363                         cpu_pgsz_mask |= (HV_PGSZ_MASK_32MB |
2364                                           HV_PGSZ_MASK_256MB);
2365
2366                 sun4u_linear_pte_xor_finalize();
2367         }
2368
2369         /* Flush the TLBs and the 4M TSB so that the updated linear
2370          * pte XOR settings are realized for all mappings.
2371          */
2372         __flush_tlb_all();
2373 #ifndef CONFIG_DEBUG_PAGEALLOC
2374         memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
2375 #endif
2376         __flush_tlb_all();
2377
2378         /* Setup bootmem... */
2379         last_valid_pfn = end_pfn = bootmem_init(phys_base);
2380
2381         kernel_physical_mapping_init();
2382
2383         {
2384                 unsigned long max_zone_pfns[MAX_NR_ZONES];
2385
2386                 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
2387
2388                 max_zone_pfns[ZONE_NORMAL] = end_pfn;
2389
2390                 free_area_init_nodes(max_zone_pfns);
2391         }
2392
2393         printk("Booting Linux...\n");
2394 }
2395
2396 int page_in_phys_avail(unsigned long paddr)
2397 {
2398         int i;
2399
2400         paddr &= PAGE_MASK;
2401
2402         for (i = 0; i < pavail_ents; i++) {
2403                 unsigned long start, end;
2404
2405                 start = pavail[i].phys_addr;
2406                 end = start + pavail[i].reg_size;
2407
2408                 if (paddr >= start && paddr < end)
2409                         return 1;
2410         }
2411         if (paddr >= kern_base && paddr < (kern_base + kern_size))
2412                 return 1;
2413 #ifdef CONFIG_BLK_DEV_INITRD
2414         if (paddr >= __pa(initrd_start) &&
2415             paddr < __pa(PAGE_ALIGN(initrd_end)))
2416                 return 1;
2417 #endif
2418
2419         return 0;
2420 }
2421
2422 static void __init register_page_bootmem_info(void)
2423 {
2424 #ifdef CONFIG_NEED_MULTIPLE_NODES
2425         int i;
2426
2427         for_each_online_node(i)
2428                 if (NODE_DATA(i)->node_spanned_pages)
2429                         register_page_bootmem_info_node(NODE_DATA(i));
2430 #endif
2431 }
2432 void __init mem_init(void)
2433 {
2434         high_memory = __va(last_valid_pfn << PAGE_SHIFT);
2435
2436         register_page_bootmem_info();
2437         free_all_bootmem();
2438
2439         /*
2440          * Set up the zero page, mark it reserved, so that page count
2441          * is not manipulated when freeing the page from user ptes.
2442          */
2443         mem_map_zero = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);
2444         if (mem_map_zero == NULL) {
2445                 prom_printf("paging_init: Cannot alloc zero page.\n");
2446                 prom_halt();
2447         }
2448         mark_page_reserved(mem_map_zero);
2449
2450         mem_init_print_info(NULL);
2451
2452         if (tlb_type == cheetah || tlb_type == cheetah_plus)
2453                 cheetah_ecache_flush_init();
2454 }
2455
2456 void free_initmem(void)
2457 {
2458         unsigned long addr, initend;
2459         int do_free = 1;
2460
2461         /* If the physical memory maps were trimmed by kernel command
2462          * line options, don't even try freeing this initmem stuff up.
2463          * The kernel image could have been in the trimmed out region
2464          * and if so the freeing below will free invalid page structs.
2465          */
2466         if (cmdline_memory_size)
2467                 do_free = 0;
2468
2469         /*
2470          * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
2471          */
2472         addr = PAGE_ALIGN((unsigned long)(__init_begin));
2473         initend = (unsigned long)(__init_end) & PAGE_MASK;
2474         for (; addr < initend; addr += PAGE_SIZE) {
2475                 unsigned long page;
2476
2477                 page = (addr +
2478                         ((unsigned long) __va(kern_base)) -
2479                         ((unsigned long) KERNBASE));
2480                 memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
2481
2482                 if (do_free)
2483                         free_reserved_page(virt_to_page(page));
2484         }
2485 }
2486
2487 #ifdef CONFIG_BLK_DEV_INITRD
2488 void free_initrd_mem(unsigned long start, unsigned long end)
2489 {
2490         free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM,
2491                            "initrd");
2492 }
2493 #endif
2494
2495 pgprot_t PAGE_KERNEL __read_mostly;
2496 EXPORT_SYMBOL(PAGE_KERNEL);
2497
2498 pgprot_t PAGE_KERNEL_LOCKED __read_mostly;
2499 pgprot_t PAGE_COPY __read_mostly;
2500
2501 pgprot_t PAGE_SHARED __read_mostly;
2502 EXPORT_SYMBOL(PAGE_SHARED);
2503
2504 unsigned long pg_iobits __read_mostly;
2505
2506 unsigned long _PAGE_IE __read_mostly;
2507 EXPORT_SYMBOL(_PAGE_IE);
2508
2509 unsigned long _PAGE_E __read_mostly;
2510 EXPORT_SYMBOL(_PAGE_E);
2511
2512 unsigned long _PAGE_CACHE __read_mostly;
2513 EXPORT_SYMBOL(_PAGE_CACHE);
2514
2515 #ifdef CONFIG_SPARSEMEM_VMEMMAP
2516 int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
2517                                int node)
2518 {
2519         unsigned long pte_base;
2520
2521         pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U |
2522                     _PAGE_CP_4U | _PAGE_CV_4U |
2523                     _PAGE_P_4U | _PAGE_W_4U);
2524         if (tlb_type == hypervisor)
2525                 pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4V |
2526                             page_cache4v_flag | _PAGE_P_4V | _PAGE_W_4V);
2527
2528         pte_base |= _PAGE_PMD_HUGE;
2529
2530         vstart = vstart & PMD_MASK;
2531         vend = ALIGN(vend, PMD_SIZE);
2532         for (; vstart < vend; vstart += PMD_SIZE) {
2533                 pgd_t *pgd = pgd_offset_k(vstart);
2534                 unsigned long pte;
2535                 pud_t *pud;
2536                 pmd_t *pmd;
2537
2538                 if (pgd_none(*pgd)) {
2539                         pud_t *new = vmemmap_alloc_block(PAGE_SIZE, node);
2540
2541                         if (!new)
2542                                 return -ENOMEM;
2543                         pgd_populate(&init_mm, pgd, new);
2544                 }
2545
2546                 pud = pud_offset(pgd, vstart);
2547                 if (pud_none(*pud)) {
2548                         pmd_t *new = vmemmap_alloc_block(PAGE_SIZE, node);
2549
2550                         if (!new)
2551                                 return -ENOMEM;
2552                         pud_populate(&init_mm, pud, new);
2553                 }
2554
2555                 pmd = pmd_offset(pud, vstart);
2556
2557                 pte = pmd_val(*pmd);
2558                 if (!(pte & _PAGE_VALID)) {
2559                         void *block = vmemmap_alloc_block(PMD_SIZE, node);
2560
2561                         if (!block)
2562                                 return -ENOMEM;
2563
2564                         pmd_val(*pmd) = pte_base | __pa(block);
2565                 }
2566         }
2567
2568         return 0;
2569 }
2570
2571 void vmemmap_free(unsigned long start, unsigned long end)
2572 {
2573 }
2574 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
2575
2576 static void prot_init_common(unsigned long page_none,
2577                              unsigned long page_shared,
2578                              unsigned long page_copy,
2579                              unsigned long page_readonly,
2580                              unsigned long page_exec_bit)
2581 {
2582         PAGE_COPY = __pgprot(page_copy);
2583         PAGE_SHARED = __pgprot(page_shared);
2584
2585         protection_map[0x0] = __pgprot(page_none);
2586         protection_map[0x1] = __pgprot(page_readonly & ~page_exec_bit);
2587         protection_map[0x2] = __pgprot(page_copy & ~page_exec_bit);
2588         protection_map[0x3] = __pgprot(page_copy & ~page_exec_bit);
2589         protection_map[0x4] = __pgprot(page_readonly);
2590         protection_map[0x5] = __pgprot(page_readonly);
2591         protection_map[0x6] = __pgprot(page_copy);
2592         protection_map[0x7] = __pgprot(page_copy);
2593         protection_map[0x8] = __pgprot(page_none);
2594         protection_map[0x9] = __pgprot(page_readonly & ~page_exec_bit);
2595         protection_map[0xa] = __pgprot(page_shared & ~page_exec_bit);
2596         protection_map[0xb] = __pgprot(page_shared & ~page_exec_bit);
2597         protection_map[0xc] = __pgprot(page_readonly);
2598         protection_map[0xd] = __pgprot(page_readonly);
2599         protection_map[0xe] = __pgprot(page_shared);
2600         protection_map[0xf] = __pgprot(page_shared);
2601 }
2602
2603 static void __init sun4u_pgprot_init(void)
2604 {
2605         unsigned long page_none, page_shared, page_copy, page_readonly;
2606         unsigned long page_exec_bit;
2607         int i;
2608
2609         PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
2610                                 _PAGE_CACHE_4U | _PAGE_P_4U |
2611                                 __ACCESS_BITS_4U | __DIRTY_BITS_4U |
2612                                 _PAGE_EXEC_4U);
2613         PAGE_KERNEL_LOCKED = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
2614                                        _PAGE_CACHE_4U | _PAGE_P_4U |
2615                                        __ACCESS_BITS_4U | __DIRTY_BITS_4U |
2616                                        _PAGE_EXEC_4U | _PAGE_L_4U);
2617
2618         _PAGE_IE = _PAGE_IE_4U;
2619         _PAGE_E = _PAGE_E_4U;
2620         _PAGE_CACHE = _PAGE_CACHE_4U;
2621
2622         pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4U | __DIRTY_BITS_4U |
2623                      __ACCESS_BITS_4U | _PAGE_E_4U);
2624
2625 #ifdef CONFIG_DEBUG_PAGEALLOC
2626         kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
2627 #else
2628         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^
2629                 PAGE_OFFSET;
2630 #endif
2631         kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |
2632                                    _PAGE_P_4U | _PAGE_W_4U);
2633
2634         for (i = 1; i < 4; i++)
2635                 kern_linear_pte_xor[i] = kern_linear_pte_xor[0];
2636
2637         _PAGE_ALL_SZ_BITS =  (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U |
2638                               _PAGE_SZ64K_4U | _PAGE_SZ8K_4U |
2639                               _PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U);
2640
2641
2642         page_none = _PAGE_PRESENT_4U | _PAGE_ACCESSED_4U | _PAGE_CACHE_4U;
2643         page_shared = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
2644                        __ACCESS_BITS_4U | _PAGE_WRITE_4U | _PAGE_EXEC_4U);
2645         page_copy   = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
2646                        __ACCESS_BITS_4U | _PAGE_EXEC_4U);
2647         page_readonly   = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
2648                            __ACCESS_BITS_4U | _PAGE_EXEC_4U);
2649
2650         page_exec_bit = _PAGE_EXEC_4U;
2651
2652         prot_init_common(page_none, page_shared, page_copy, page_readonly,
2653                          page_exec_bit);
2654 }
2655
2656 static void __init sun4v_pgprot_init(void)
2657 {
2658         unsigned long page_none, page_shared, page_copy, page_readonly;
2659         unsigned long page_exec_bit;
2660         int i;
2661
2662         PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID |
2663                                 page_cache4v_flag | _PAGE_P_4V |
2664                                 __ACCESS_BITS_4V | __DIRTY_BITS_4V |
2665                                 _PAGE_EXEC_4V);
2666         PAGE_KERNEL_LOCKED = PAGE_KERNEL;
2667
2668         _PAGE_IE = _PAGE_IE_4V;
2669         _PAGE_E = _PAGE_E_4V;
2670         _PAGE_CACHE = page_cache4v_flag;
2671
2672 #ifdef CONFIG_DEBUG_PAGEALLOC
2673         kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
2674 #else
2675         kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^
2676                 PAGE_OFFSET;
2677 #endif
2678         kern_linear_pte_xor[0] |= (page_cache4v_flag | _PAGE_P_4V |
2679                                    _PAGE_W_4V);
2680
2681         for (i = 1; i < 4; i++)
2682                 kern_linear_pte_xor[i] = kern_linear_pte_xor[0];
2683
2684         pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V |
2685                      __ACCESS_BITS_4V | _PAGE_E_4V);
2686
2687         _PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V |
2688                              _PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V |
2689                              _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V |
2690                              _PAGE_SZ64K_4V | _PAGE_SZ8K_4V);
2691
2692         page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | page_cache4v_flag;
2693         page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag |
2694                        __ACCESS_BITS_4V | _PAGE_WRITE_4V | _PAGE_EXEC_4V);
2695         page_copy   = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag |
2696                        __ACCESS_BITS_4V | _PAGE_EXEC_4V);
2697         page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag |
2698                          __ACCESS_BITS_4V | _PAGE_EXEC_4V);
2699
2700         page_exec_bit = _PAGE_EXEC_4V;
2701
2702         prot_init_common(page_none, page_shared, page_copy, page_readonly,
2703                          page_exec_bit);
2704 }
2705
2706 unsigned long pte_sz_bits(unsigned long sz)
2707 {
2708         if (tlb_type == hypervisor) {
2709                 switch (sz) {
2710                 case 8 * 1024:
2711                 default:
2712                         return _PAGE_SZ8K_4V;
2713                 case 64 * 1024:
2714                         return _PAGE_SZ64K_4V;
2715                 case 512 * 1024:
2716                         return _PAGE_SZ512K_4V;
2717                 case 4 * 1024 * 1024:
2718                         return _PAGE_SZ4MB_4V;
2719                 }
2720         } else {
2721                 switch (sz) {
2722                 case 8 * 1024:
2723                 default:
2724                         return _PAGE_SZ8K_4U;
2725                 case 64 * 1024:
2726                         return _PAGE_SZ64K_4U;
2727                 case 512 * 1024:
2728                         return _PAGE_SZ512K_4U;
2729                 case 4 * 1024 * 1024:
2730                         return _PAGE_SZ4MB_4U;
2731                 }
2732         }
2733 }
2734
2735 pte_t mk_pte_io(unsigned long page, pgprot_t prot, int space, unsigned long page_size)
2736 {
2737         pte_t pte;
2738
2739         pte_val(pte)  = page | pgprot_val(pgprot_noncached(prot));
2740         pte_val(pte) |= (((unsigned long)space) << 32);
2741         pte_val(pte) |= pte_sz_bits(page_size);
2742
2743         return pte;
2744 }
2745
2746 static unsigned long kern_large_tte(unsigned long paddr)
2747 {
2748         unsigned long val;
2749
2750         val = (_PAGE_VALID | _PAGE_SZ4MB_4U |
2751                _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U |
2752                _PAGE_EXEC_4U | _PAGE_L_4U | _PAGE_W_4U);
2753         if (tlb_type == hypervisor)
2754                 val = (_PAGE_VALID | _PAGE_SZ4MB_4V |
2755                        page_cache4v_flag | _PAGE_P_4V |
2756                        _PAGE_EXEC_4V | _PAGE_W_4V);
2757
2758         return val | paddr;
2759 }
2760
2761 /* If not locked, zap it. */
2762 void __flush_tlb_all(void)
2763 {
2764         unsigned long pstate;
2765         int i;
2766
2767         __asm__ __volatile__("flushw\n\t"
2768                              "rdpr      %%pstate, %0\n\t"
2769                              "wrpr      %0, %1, %%pstate"
2770                              : "=r" (pstate)
2771                              : "i" (PSTATE_IE));
2772         if (tlb_type == hypervisor) {
2773                 sun4v_mmu_demap_all();
2774         } else if (tlb_type == spitfire) {
2775                 for (i = 0; i < 64; i++) {
2776                         /* Spitfire Errata #32 workaround */
2777                         /* NOTE: Always runs on spitfire, so no
2778                          *       cheetah+ page size encodings.
2779                          */
2780                         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
2781                                              "flush     %%g6"
2782                                              : /* No outputs */
2783                                              : "r" (0),
2784                                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
2785
2786                         if (!(spitfire_get_dtlb_data(i) & _PAGE_L_4U)) {
2787                                 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
2788                                                      "membar #Sync"
2789                                                      : /* no outputs */
2790                                                      : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
2791                                 spitfire_put_dtlb_data(i, 0x0UL);
2792                         }
2793
2794                         /* Spitfire Errata #32 workaround */
2795                         /* NOTE: Always runs on spitfire, so no
2796                          *       cheetah+ page size encodings.
2797                          */
2798                         __asm__ __volatile__("stxa      %0, [%1] %2\n\t"
2799                                              "flush     %%g6"
2800                                              : /* No outputs */
2801                                              : "r" (0),
2802                                              "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
2803
2804                         if (!(spitfire_get_itlb_data(i) & _PAGE_L_4U)) {
2805                                 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
2806                                                      "membar #Sync"
2807                                                      : /* no outputs */
2808                                                      : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
2809                                 spitfire_put_itlb_data(i, 0x0UL);
2810                         }
2811                 }
2812         } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
2813                 cheetah_flush_dtlb_all();
2814                 cheetah_flush_itlb_all();
2815         }
2816         __asm__ __volatile__("wrpr      %0, 0, %%pstate"
2817                              : : "r" (pstate));
2818 }
2819
2820 pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
2821                             unsigned long address)
2822 {
2823         struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO);
2824         pte_t *pte = NULL;
2825
2826         if (page)
2827                 pte = (pte_t *) page_address(page);
2828
2829         return pte;
2830 }
2831
2832 pgtable_t pte_alloc_one(struct mm_struct *mm,
2833                         unsigned long address)
2834 {
2835         struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO);
2836         if (!page)
2837                 return NULL;
2838         if (!pgtable_page_ctor(page)) {
2839                 free_hot_cold_page(page, 0);
2840                 return NULL;
2841         }
2842         return (pte_t *) page_address(page);
2843 }
2844
2845 void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
2846 {
2847         free_page((unsigned long)pte);
2848 }
2849
2850 static void __pte_free(pgtable_t pte)
2851 {
2852         struct page *page = virt_to_page(pte);
2853
2854         pgtable_page_dtor(page);
2855         __free_page(page);
2856 }
2857
2858 void pte_free(struct mm_struct *mm, pgtable_t pte)
2859 {
2860         __pte_free(pte);
2861 }
2862
2863 void pgtable_free(void *table, bool is_page)
2864 {
2865         if (is_page)
2866                 __pte_free(table);
2867         else
2868                 kmem_cache_free(pgtable_cache, table);
2869 }
2870
2871 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
2872 void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
2873                           pmd_t *pmd)
2874 {
2875         unsigned long pte, flags;
2876         struct mm_struct *mm;
2877         pmd_t entry = *pmd;
2878
2879         if (!pmd_large(entry) || !pmd_young(entry))
2880                 return;
2881
2882         pte = pmd_val(entry);
2883
2884         /* Don't insert a non-valid PMD into the TSB, we'll deadlock.  */
2885         if (!(pte & _PAGE_VALID))
2886                 return;
2887
2888         /* We are fabricating 8MB pages using 4MB real hw pages.  */
2889         pte |= (addr & (1UL << REAL_HPAGE_SHIFT));
2890
2891         mm = vma->vm_mm;
2892
2893         spin_lock_irqsave(&mm->context.lock, flags);
2894
2895         if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL)
2896                 __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
2897                                         addr, pte);
2898
2899         spin_unlock_irqrestore(&mm->context.lock, flags);
2900 }
2901 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
2902
2903 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
2904 static void context_reload(void *__data)
2905 {
2906         struct mm_struct *mm = __data;
2907
2908         if (mm == current->mm)
2909                 load_secondary_context(mm);
2910 }
2911
2912 void hugetlb_setup(struct pt_regs *regs)
2913 {
2914         struct mm_struct *mm = current->mm;
2915         struct tsb_config *tp;
2916
2917         if (faulthandler_disabled() || !mm) {
2918                 const struct exception_table_entry *entry;
2919
2920                 entry = search_exception_tables(regs->tpc);
2921                 if (entry) {
2922                         regs->tpc = entry->fixup;
2923                         regs->tnpc = regs->tpc + 4;
2924                         return;
2925                 }
2926                 pr_alert("Unexpected HugeTLB setup in atomic context.\n");
2927                 die_if_kernel("HugeTSB in atomic", regs);
2928         }
2929
2930         tp = &mm->context.tsb_block[MM_TSB_HUGE];
2931         if (likely(tp->tsb == NULL))
2932                 tsb_grow(mm, MM_TSB_HUGE, 0);
2933
2934         tsb_context_switch(mm);
2935         smp_tsb_sync(mm);
2936
2937         /* On UltraSPARC-III+ and later, configure the second half of
2938          * the Data-TLB for huge pages.
2939          */
2940         if (tlb_type == cheetah_plus) {
2941                 bool need_context_reload = false;
2942                 unsigned long ctx;
2943
2944                 spin_lock_irq(&ctx_alloc_lock);
2945                 ctx = mm->context.sparc64_ctx_val;
2946                 ctx &= ~CTX_PGSZ_MASK;
2947                 ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT;
2948                 ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT;
2949
2950                 if (ctx != mm->context.sparc64_ctx_val) {
2951                         /* When changing the page size fields, we
2952                          * must perform a context flush so that no
2953                          * stale entries match.  This flush must
2954                          * occur with the original context register
2955                          * settings.
2956                          */
2957                         do_flush_tlb_mm(mm);
2958
2959                         /* Reload the context register of all processors
2960                          * also executing in this address space.
2961                          */
2962                         mm->context.sparc64_ctx_val = ctx;
2963                         need_context_reload = true;
2964                 }
2965                 spin_unlock_irq(&ctx_alloc_lock);
2966
2967                 if (need_context_reload)
2968                         on_each_cpu(context_reload, mm, 0);
2969         }
2970 }
2971 #endif
2972
2973 static struct resource code_resource = {
2974         .name   = "Kernel code",
2975         .flags  = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
2976 };
2977
2978 static struct resource data_resource = {
2979         .name   = "Kernel data",
2980         .flags  = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
2981 };
2982
2983 static struct resource bss_resource = {
2984         .name   = "Kernel bss",
2985         .flags  = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM
2986 };
2987
2988 static inline resource_size_t compute_kern_paddr(void *addr)
2989 {
2990         return (resource_size_t) (addr - KERNBASE + kern_base);
2991 }
2992
2993 static void __init kernel_lds_init(void)
2994 {
2995         code_resource.start = compute_kern_paddr(_text);
2996         code_resource.end   = compute_kern_paddr(_etext - 1);
2997         data_resource.start = compute_kern_paddr(_etext);
2998         data_resource.end   = compute_kern_paddr(_edata - 1);
2999         bss_resource.start  = compute_kern_paddr(__bss_start);
3000         bss_resource.end    = compute_kern_paddr(_end - 1);
3001 }
3002
3003 static int __init report_memory(void)
3004 {
3005         int i;
3006         struct resource *res;
3007
3008         kernel_lds_init();
3009
3010         for (i = 0; i < pavail_ents; i++) {
3011                 res = kzalloc(sizeof(struct resource), GFP_KERNEL);
3012
3013                 if (!res) {
3014                         pr_warn("Failed to allocate source.\n");
3015                         break;
3016                 }
3017
3018                 res->name = "System RAM";
3019                 res->start = pavail[i].phys_addr;
3020                 res->end = pavail[i].phys_addr + pavail[i].reg_size - 1;
3021                 res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM;
3022
3023                 if (insert_resource(&iomem_resource, res) < 0) {
3024                         pr_warn("Resource insertion failed.\n");
3025                         break;
3026                 }
3027
3028                 insert_resource(res, &code_resource);
3029                 insert_resource(res, &data_resource);
3030                 insert_resource(res, &bss_resource);
3031         }
3032
3033         return 0;
3034 }
3035 arch_initcall(report_memory);
3036
3037 #ifdef CONFIG_SMP
3038 #define do_flush_tlb_kernel_range       smp_flush_tlb_kernel_range
3039 #else
3040 #define do_flush_tlb_kernel_range       __flush_tlb_kernel_range
3041 #endif
3042
3043 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
3044 {
3045         if (start < HI_OBP_ADDRESS && end > LOW_OBP_ADDRESS) {
3046                 if (start < LOW_OBP_ADDRESS) {
3047                         flush_tsb_kernel_range(start, LOW_OBP_ADDRESS);
3048                         do_flush_tlb_kernel_range(start, LOW_OBP_ADDRESS);
3049                 }
3050                 if (end > HI_OBP_ADDRESS) {
3051                         flush_tsb_kernel_range(HI_OBP_ADDRESS, end);
3052                         do_flush_tlb_kernel_range(HI_OBP_ADDRESS, end);
3053                 }
3054         } else {
3055                 flush_tsb_kernel_range(start, end);
3056                 do_flush_tlb_kernel_range(start, end);
3057         }
3058 }