arch/mips/mm/tlb-r4k.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
   7  * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
   8  * Carsten Langgaard, carstenl@mips.com
   9  * Copyright (C) 2002 MIPS Technologies, Inc.  All rights reserved.
  10  */
  11 #include <linux/config.h>
  12 #include <linux/init.h>
  13 #include <linux/sched.h>
  14 #include <linux/mm.h>
  15
  16 #include <asm/cpu.h>
  17 #include <asm/bootinfo.h>
  18 #include <asm/mmu_context.h>
  19 #include <asm/pgtable.h>
  20 #include <asm/system.h>
  21
  22 extern void build_tlb_refill_handler(void);
  23
  24 /*
  25  * Make sure all entries differ.  If they're not different
  26  * MIPS32 will take revenge ...
  27  */
  28 #define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
  29
  30 /* CP0 hazard avoidance. */
  31 #define BARRIER __asm__ __volatile__(".set noreorder\n\t" \
  32                                      "nop; nop; nop; nop; nop; nop;\n\t" \
  33                                      ".set reorder\n\t")
  34
  35 void local_flush_tlb_all(void)
  36 {
  37         unsigned long flags;
  38         unsigned long old_ctx;
  39         int entry;
  40
  41         local_irq_save(flags);
  42         /* Save old context and create impossible VPN2 value */
  43         old_ctx = read_c0_entryhi();
  44         write_c0_entrylo0(0);
  45         write_c0_entrylo1(0);
  46
  47         entry = read_c0_wired();
  48
  49         /* Blast 'em all away. */
  50         while (entry < current_cpu_data.tlbsize) {
  51                 /* Make sure all entries differ. */
  52                 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
  53                 write_c0_index(entry);
  54                 mtc0_tlbw_hazard();
  55                 tlb_write_indexed();
  56                 entry++;
  57         }
  58         tlbw_use_hazard();
  59         write_c0_entryhi(old_ctx);
  60         local_irq_restore(flags);
  61 }
  62
  63 /* All entries common to a mm share an asid.  To effectively flush
  64    these entries, we just bump the asid. */
  65 void local_flush_tlb_mm(struct mm_struct *mm)
  66 {
  67         int cpu;
  68
  69         preempt_disable();
  70
  71         cpu = smp_processor_id();
  72
  73         if (cpu_context(cpu, mm) != 0) {
  74                 drop_mmu_context(mm, cpu);
  75         }
  76
  77         preempt_enable();
  78 }
  79
  80 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
  81         unsigned long end)
  82 {
  83         struct mm_struct *mm = vma->vm_mm;
  84         int cpu = smp_processor_id();
  85
  86         if (cpu_context(cpu, mm) != 0) {
  87                 unsigned long flags;
  88                 int size;
  89
  90                 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
  91                 size = (size + 1) >> 1;
  92                 local_irq_save(flags);
  93                 if (size <= current_cpu_data.tlbsize/2) {
  94                         int oldpid = read_c0_entryhi();
  95                         int newpid = cpu_asid(cpu, mm);
  96
  97                         start &= (PAGE_MASK << 1);
  98                         end += ((PAGE_SIZE << 1) - 1);
  99                         end &= (PAGE_MASK << 1);
 100                         while (start < end) {
 101                                 int idx;
 102
 103                                 write_c0_entryhi(start | newpid);
 104                                 start += (PAGE_SIZE << 1);
 105                                 mtc0_tlbw_hazard();
 106                                 tlb_probe();
 107                                 BARRIER;
 108                                 idx = read_c0_index();
 109                                 write_c0_entrylo0(0);
 110                                 write_c0_entrylo1(0);
 111                                 if (idx < 0)
 112                                         continue;
 113                                 /* Make sure all entries differ. */
 114                                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 115                                 mtc0_tlbw_hazard();
 116                                 tlb_write_indexed();
 117                         }
 118                         tlbw_use_hazard();
 119                         write_c0_entryhi(oldpid);
 120                 } else {
 121                         drop_mmu_context(mm, cpu);
 122                 }
 123                 local_irq_restore(flags);
 124         }
 125 }
 126
 127 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
 128 {
 129         unsigned long flags;
 130         int size;
 131
 132         size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
 133         size = (size + 1) >> 1;
 134         local_irq_save(flags);
 135         if (size <= current_cpu_data.tlbsize / 2) {
 136                 int pid = read_c0_entryhi();
 137
 138                 start &= (PAGE_MASK << 1);
 139                 end += ((PAGE_SIZE << 1) - 1);
 140                 end &= (PAGE_MASK << 1);
 141
 142                 while (start < end) {
 143                         int idx;
 144
 145                         write_c0_entryhi(start);
 146                         start += (PAGE_SIZE << 1);
 147                         mtc0_tlbw_hazard();
 148                         tlb_probe();
 149                         BARRIER;
 150                         idx = read_c0_index();
 151                         write_c0_entrylo0(0);
 152                         write_c0_entrylo1(0);
 153                         if (idx < 0)
 154                                 continue;
 155                         /* Make sure all entries differ. */
 156                         write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 157                         mtc0_tlbw_hazard();
 158                         tlb_write_indexed();
 159                 }
 160                 tlbw_use_hazard();
 161                 write_c0_entryhi(pid);
 162         } else {
 163                 local_flush_tlb_all();
 164         }
 165         local_irq_restore(flags);
 166 }
 167
 168 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 169 {
 170         int cpu = smp_processor_id();
 171
 172         if (cpu_context(cpu, vma->vm_mm) != 0) {
 173                 unsigned long flags;
 174                 int oldpid, newpid, idx;
 175
 176                 newpid = cpu_asid(cpu, vma->vm_mm);
 177                 page &= (PAGE_MASK << 1);
 178                 local_irq_save(flags);
 179                 oldpid = read_c0_entryhi();
 180                 write_c0_entryhi(page | newpid);
 181                 mtc0_tlbw_hazard();
 182                 tlb_probe();
 183                 BARRIER;
 184                 idx = read_c0_index();
 185                 write_c0_entrylo0(0);
 186                 write_c0_entrylo1(0);
 187                 if (idx < 0)
 188                         goto finish;
 189                 /* Make sure all entries differ. */
 190                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 191                 mtc0_tlbw_hazard();
 192                 tlb_write_indexed();
 193                 tlbw_use_hazard();
 194
 195         finish:
 196                 write_c0_entryhi(oldpid);
 197                 local_irq_restore(flags);
 198         }
 199 }
 200
 201 /*
 202  * This one is only used for pages with the global bit set so we don't care
 203  * much about the ASID.
 204  */
 205 void local_flush_tlb_one(unsigned long page)
 206 {
 207         unsigned long flags;
 208         int oldpid, idx;
 209
 210         local_irq_save(flags);
 211         oldpid = read_c0_entryhi();
 212         page &= (PAGE_MASK << 1);
 213         write_c0_entryhi(page);
 214         mtc0_tlbw_hazard();
 215         tlb_probe();
 216         BARRIER;
 217         idx = read_c0_index();
 218         write_c0_entrylo0(0);
 219         write_c0_entrylo1(0);
 220         if (idx >= 0) {
 221                 /* Make sure all entries differ. */
 222                 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
 223                 mtc0_tlbw_hazard();
 224                 tlb_write_indexed();
 225                 tlbw_use_hazard();
 226         }
 227         write_c0_entryhi(oldpid);
 228
 229         local_irq_restore(flags);
 230 }
 231
 232 /*
 233  * We will need multiple versions of update_mmu_cache(), one that just
 234  * updates the TLB with the new pte(s), and another which also checks
 235  * for the R4k "end of page" hardware bug and does the needy.
 236  */
 237 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
 238 {
 239         unsigned long flags;
 240         pgd_t *pgdp;
 241         pud_t *pudp;
 242         pmd_t *pmdp;
 243         pte_t *ptep;
 244         int idx, pid;
 245
 246         /*
 247          * Handle debugger faulting in for debugee.
 248          */
 249         if (current->active_mm != vma->vm_mm)
 250                 return;
 251
 252         local_irq_save(flags);
 253
 254         pid = read_c0_entryhi() & ASID_MASK;
 255         address &= (PAGE_MASK << 1);
 256         write_c0_entryhi(address | pid);
 257         pgdp = pgd_offset(vma->vm_mm, address);
 258         mtc0_tlbw_hazard();
 259         tlb_probe();
 260         BARRIER;
 261         pudp = pud_offset(pgdp, address);
 262         pmdp = pmd_offset(pudp, address);
 263         idx = read_c0_index();
 264         ptep = pte_offset_map(pmdp, address);
 265
 266 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32_R1)
 267         write_c0_entrylo0(ptep->pte_high);
 268         ptep++;
 269         write_c0_entrylo1(ptep->pte_high);
 270 #else
 271         write_c0_entrylo0(pte_val(*ptep++) >> 6);
 272         write_c0_entrylo1(pte_val(*ptep) >> 6);
 273 #endif
 274         mtc0_tlbw_hazard();
 275         if (idx < 0)
 276                 tlb_write_random();
 277         else
 278                 tlb_write_indexed();
 279         tlbw_use_hazard();
 280         local_irq_restore(flags);
 281 }
 282
 283 #if 0
 284 static void r4k_update_mmu_cache_hwbug(struct vm_area_struct * vma,
 285                                        unsigned long address, pte_t pte)
 286 {
 287         unsigned long flags;
 288         unsigned int asid;
 289         pgd_t *pgdp;
 290         pmd_t *pmdp;
 291         pte_t *ptep;
 292         int idx;
 293
 294         local_irq_save(flags);
 295         address &= (PAGE_MASK << 1);
 296         asid = read_c0_entryhi() & ASID_MASK;
 297         write_c0_entryhi(address | asid);
 298         pgdp = pgd_offset(vma->vm_mm, address);
 299         mtc0_tlbw_hazard();
 300         tlb_probe();
 301         BARRIER;
 302         pmdp = pmd_offset(pgdp, address);
 303         idx = read_c0_index();
 304         ptep = pte_offset_map(pmdp, address);
 305         write_c0_entrylo0(pte_val(*ptep++) >> 6);
 306         write_c0_entrylo1(pte_val(*ptep) >> 6);
 307         mtc0_tlbw_hazard();
 308         if (idx < 0)
 309                 tlb_write_random();
 310         else
 311                 tlb_write_indexed();
 312         tlbw_use_hazard();
 313         local_irq_restore(flags);
 314 }
 315 #endif
 316
 317 void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
 318         unsigned long entryhi, unsigned long pagemask)
 319 {
 320         unsigned long flags;
 321         unsigned long wired;
 322         unsigned long old_pagemask;
 323         unsigned long old_ctx;
 324
 325         local_irq_save(flags);
 326         /* Save old context and create impossible VPN2 value */
 327         old_ctx = read_c0_entryhi();
 328         old_pagemask = read_c0_pagemask();
 329         wired = read_c0_wired();
 330         write_c0_wired(wired + 1);
 331         write_c0_index(wired);
 332         BARRIER;
 333         write_c0_pagemask(pagemask);
 334         write_c0_entryhi(entryhi);
 335         write_c0_entrylo0(entrylo0);
 336         write_c0_entrylo1(entrylo1);
 337         mtc0_tlbw_hazard();
 338         tlb_write_indexed();
 339         tlbw_use_hazard();
 340
 341         write_c0_entryhi(old_ctx);
 342         BARRIER;
 343         write_c0_pagemask(old_pagemask);
 344         local_flush_tlb_all();
 345         local_irq_restore(flags);
 346 }
 347
 348 /*
 349  * Used for loading TLB entries before trap_init() has started, when we
 350  * don't actually want to add a wired entry which remains throughout the
 351  * lifetime of the system
 352  */
 353
 354 static int temp_tlb_entry __initdata;
 355
 356 __init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1,
 357                                unsigned long entryhi, unsigned long pagemask)
 358 {
 359         int ret = 0;
 360         unsigned long flags;
 361         unsigned long wired;
 362         unsigned long old_pagemask;
 363         unsigned long old_ctx;
 364
 365         local_irq_save(flags);
 366         /* Save old context and create impossible VPN2 value */
 367         old_ctx = read_c0_entryhi();
 368         old_pagemask = read_c0_pagemask();
 369         wired = read_c0_wired();
 370         if (--temp_tlb_entry < wired) {
 371                 printk(KERN_WARNING
 372                        "No TLB space left for add_temporary_entry\n");
 373                 ret = -ENOSPC;
 374                 goto out;
 375         }
 376
 377         write_c0_index(temp_tlb_entry);
 378         write_c0_pagemask(pagemask);
 379         write_c0_entryhi(entryhi);
 380         write_c0_entrylo0(entrylo0);
 381         write_c0_entrylo1(entrylo1);
 382         mtc0_tlbw_hazard();
 383         tlb_write_indexed();
 384         tlbw_use_hazard();
 385
 386         write_c0_entryhi(old_ctx);
 387         write_c0_pagemask(old_pagemask);
 388 out:
 389         local_irq_restore(flags);
 390         return ret;
 391 }
 392
 393 static void __init probe_tlb(unsigned long config)
 394 {
 395         struct cpuinfo_mips *c = &current_cpu_data;
 396         unsigned int reg;
 397
 398         /*
 399          * If this isn't a MIPS32 / MIPS64 compliant CPU.  Config 1 register
 400          * is not supported, we assume R4k style.  Cpu probing already figured
 401          * out the number of tlb entries.
 402          */
 403         if ((c->processor_id & 0xff0000) == PRID_COMP_LEGACY)
 404                 return;
 405
 406         reg = read_c0_config1();
 407         if (!((config >> 7) & 3))
 408                 panic("No TLB present");
 409
 410         c->tlbsize = ((reg >> 25) & 0x3f) + 1;
 411 }
 412
 413 void __init tlb_init(void)
 414 {
 415         unsigned int config = read_c0_config();
 416
 417         /*
 418          * You should never change this register:
 419          *   - On R4600 1.7 the tlbp never hits for pages smaller than
 420          *     the value in the c0_pagemask register.
 421          *   - The entire mm handling assumes the c0_pagemask register to
 422          *     be set for 4kb pages.
 423          */
 424         probe_tlb(config);
 425         write_c0_pagemask(PM_DEFAULT_MASK);
 426         write_c0_wired(0);
 427         write_c0_framemask(0);
 428         temp_tlb_entry = current_cpu_data.tlbsize - 1;
 429
 430         /* From this point on the ARC firmware is dead.  */
 431         local_flush_tlb_all();
 432
 433         /* Did I tell you that ARC SUCKS?  */
 434
 435         build_tlb_refill_handler();
 436 }