arch/x86/mm/ioremap.c

   1 /*
   2  * Re-map IO memory to kernel address space so that we can access it.
   3  * This is needed for high PCI addresses that aren't mapped in the
   4  * 640k-1MB IO memory area on PC's
   5  *
   6  * (C) Copyright 1995 1996 Linus Torvalds
   7  */
   8
   9 #include <linux/bootmem.h>
  10 #include <linux/init.h>
  11 #include <linux/io.h>
  12 #include <linux/module.h>
  13 #include <linux/slab.h>
  14 #include <linux/vmalloc.h>
  15
  16 #include <asm/cacheflush.h>
  17 #include <asm/e820.h>
  18 #include <asm/fixmap.h>
  19 #include <asm/pgtable.h>
  20 #include <asm/tlbflush.h>
  21 #include <asm/pgalloc.h>
  22 #include <asm/pat.h>
  23
  24 #ifdef CONFIG_X86_64
  25
  26 unsigned long __phys_addr(unsigned long x)
  27 {
  28         if (x >= __START_KERNEL_map)
  29                 return x - __START_KERNEL_map + phys_base;
  30         return x - PAGE_OFFSET;
  31 }
  32 EXPORT_SYMBOL(__phys_addr);
  33
  34 static inline int phys_addr_valid(unsigned long addr)
  35 {
  36         return addr < (1UL << boot_cpu_data.x86_phys_bits);
  37 }
  38
  39 #else
  40
  41 static inline int phys_addr_valid(unsigned long addr)
  42 {
  43         return 1;
  44 }
  45
  46 #endif
  47
  48 int page_is_ram(unsigned long pagenr)
  49 {
  50         unsigned long addr, end;
  51         int i;
  52
  53         /*
  54          * A special case is the first 4Kb of memory;
  55          * This is a BIOS owned area, not kernel ram, but generally
  56          * not listed as such in the E820 table.
  57          */
  58         if (pagenr == 0)
  59                 return 0;
  60
  61         /*
  62          * Second special case: Some BIOSen report the PC BIOS
  63          * area (640->1Mb) as ram even though it is not.
  64          */
  65         if (pagenr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
  66                     pagenr < (BIOS_END >> PAGE_SHIFT))
  67                 return 0;
  68
  69         for (i = 0; i < e820.nr_map; i++) {
  70                 /*
  71                  * Not usable memory:
  72                  */
  73                 if (e820.map[i].type != E820_RAM)
  74                         continue;
  75                 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
  76                 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
  77
  78
  79                 if ((pagenr >= addr) && (pagenr < end))
  80                         return 1;
  81         }
  82         return 0;
  83 }
  84
  85 /*
  86  * Fix up the linear direct mapping of the kernel to avoid cache attribute
  87  * conflicts.
  88  */
  89 int ioremap_change_attr(unsigned long vaddr, unsigned long size,
  90                                unsigned long prot_val)
  91 {
  92         unsigned long nrpages = size >> PAGE_SHIFT;
  93         int err;
  94
  95         switch (prot_val) {
  96         case _PAGE_CACHE_UC:
  97         default:
  98                 err = _set_memory_uc(vaddr, nrpages);
  99                 break;
 100         case _PAGE_CACHE_WC:
 101                 err = _set_memory_wc(vaddr, nrpages);
 102                 break;
 103         case _PAGE_CACHE_WB:
 104                 err = _set_memory_wb(vaddr, nrpages);
 105                 break;
 106         }
 107
 108         return err;
 109 }
 110
 111 /*
 112  * Remap an arbitrary physical address space into the kernel virtual
 113  * address space. Needed when the kernel wants to access high addresses
 114  * directly.
 115  *
 116  * NOTE! We need to allow non-page-aligned mappings too: we will obviously
 117  * have to convert them into an offset in a page-aligned mapping, but the
 118  * caller shouldn't need to know that small detail.
 119  */
 120 static void __iomem *__ioremap(resource_size_t phys_addr, unsigned long size,
 121                                unsigned long prot_val)
 122 {
 123         unsigned long pfn, offset, last_addr, vaddr;
 124         struct vm_struct *area;
 125         unsigned long new_prot_val;
 126         pgprot_t prot;
 127
 128         /* Don't allow wraparound or zero size */
 129         last_addr = phys_addr + size - 1;
 130         if (!size || last_addr < phys_addr)
 131                 return NULL;
 132
 133         if (!phys_addr_valid(phys_addr)) {
 134                 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
 135                        phys_addr);
 136                 WARN_ON_ONCE(1);
 137                 return NULL;
 138         }
 139
 140         /*
 141          * Don't remap the low PCI/ISA area, it's always mapped..
 142          */
 143         if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS)
 144                 return (__force void __iomem *)phys_to_virt(phys_addr);
 145
 146         /*
 147          * Don't allow anybody to remap normal RAM that we're using..
 148          */
 149         for (pfn = phys_addr >> PAGE_SHIFT;
 150                                 (pfn << PAGE_SHIFT) < last_addr; pfn++) {
 151
 152                 int is_ram = page_is_ram(pfn);
 153
 154                 if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
 155                         return NULL;
 156                 WARN_ON_ONCE(is_ram);
 157         }
 158
 159         /*
 160          * Mappings have to be page-aligned
 161          */
 162         offset = phys_addr & ~PAGE_MASK;
 163         phys_addr &= PAGE_MASK;
 164         size = PAGE_ALIGN(last_addr+1) - phys_addr;
 165
 166         if (reserve_memtype(phys_addr, phys_addr + size,
 167                             prot_val, &new_prot_val)) {
 168                 /*
 169                  * Do not fallback to certain memory types with certain
 170                  * requested type:
 171                  * - request is uncached, return cannot be write-back
 172                  * - request is uncached, return cannot be write-combine
 173                  * - request is write-combine, return cannot be write-back
 174                  */
 175                 if ((prot_val == _PAGE_CACHE_UC &&
 176                      (new_prot_val == _PAGE_CACHE_WB ||
 177                       new_prot_val == _PAGE_CACHE_WC)) ||
 178                     (prot_val == _PAGE_CACHE_WC &&
 179                      new_prot_val == _PAGE_CACHE_WB)) {
 180                         printk(
 181                 "ioremap error for 0x%llx-0x%llx, requested 0x%lx, got 0x%lx\n",
 182                                 phys_addr, phys_addr + size,
 183                                 prot_val, new_prot_val);
 184                         free_memtype(phys_addr, phys_addr + size);
 185                         return NULL;
 186                 }
 187                 prot_val = new_prot_val;
 188         }
 189
 190         switch (prot_val) {
 191         case _PAGE_CACHE_UC:
 192         default:
 193                 prot = PAGE_KERNEL_NOCACHE;
 194                 break;
 195         case _PAGE_CACHE_WC:
 196                 prot = PAGE_KERNEL_WC;
 197                 break;
 198         case _PAGE_CACHE_WB:
 199                 prot = PAGE_KERNEL;
 200                 break;
 201         }
 202
 203         /*
 204          * Ok, go for it..
 205          */
 206         area = get_vm_area(size, VM_IOREMAP);
 207         if (!area)
 208                 return NULL;
 209         area->phys_addr = phys_addr;
 210         vaddr = (unsigned long) area->addr;
 211         if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) {
 212                 free_memtype(phys_addr, phys_addr + size);
 213                 free_vm_area(area);
 214                 return NULL;
 215         }
 216
 217         if (ioremap_change_attr(vaddr, size, prot_val) < 0) {
 218                 free_memtype(phys_addr, phys_addr + size);
 219                 vunmap(area->addr);
 220                 return NULL;
 221         }
 222
 223         return (void __iomem *) (vaddr + offset);
 224 }
 225
 226 /**
 227  * ioremap_nocache     -   map bus memory into CPU space
 228  * @offset:    bus address of the memory
 229  * @size:      size of the resource to map
 230  *
 231  * ioremap_nocache performs a platform specific sequence of operations to
 232  * make bus memory CPU accessible via the readb/readw/readl/writeb/
 233  * writew/writel functions and the other mmio helpers. The returned
 234  * address is not guaranteed to be usable directly as a virtual
 235  * address.
 236  *
 237  * This version of ioremap ensures that the memory is marked uncachable
 238  * on the CPU as well as honouring existing caching rules from things like
 239  * the PCI bus. Note that there are other caches and buffers on many
 240  * busses. In particular driver authors should read up on PCI writes
 241  *
 242  * It's useful if some control registers are in such an area and
 243  * write combining or read caching is not desirable:
 244  *
 245  * Must be freed with iounmap.
 246  */
 247 void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
 248 {
 249         return __ioremap(phys_addr, size, _PAGE_CACHE_UC);
 250 }
 251 EXPORT_SYMBOL(ioremap_nocache);
 252
 253 /**
 254  * ioremap_wc   -       map memory into CPU space write combined
 255  * @offset:     bus address of the memory
 256  * @size:       size of the resource to map
 257  *
 258  * This version of ioremap ensures that the memory is marked write combining.
 259  * Write combining allows faster writes to some hardware devices.
 260  *
 261  * Must be freed with iounmap.
 262  */
 263 void __iomem *ioremap_wc(unsigned long phys_addr, unsigned long size)
 264 {
 265         if (pat_wc_enabled)
 266                 return __ioremap(phys_addr, size, _PAGE_CACHE_WC);
 267         else
 268                 return ioremap_nocache(phys_addr, size);
 269 }
 270 EXPORT_SYMBOL(ioremap_wc);
 271
 272 void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
 273 {
 274         return __ioremap(phys_addr, size, _PAGE_CACHE_WB);
 275 }
 276 EXPORT_SYMBOL(ioremap_cache);
 277
 278 /**
 279  * iounmap - Free a IO remapping
 280  * @addr: virtual address from ioremap_*
 281  *
 282  * Caller must ensure there is only one unmapping for the same pointer.
 283  */
 284 void iounmap(volatile void __iomem *addr)
 285 {
 286         struct vm_struct *p, *o;
 287
 288         if ((void __force *)addr <= high_memory)
 289                 return;
 290
 291         /*
 292          * __ioremap special-cases the PCI/ISA range by not instantiating a
 293          * vm_area and by simply returning an address into the kernel mapping
 294          * of ISA space.   So handle that here.
 295          */
 296         if (addr >= phys_to_virt(ISA_START_ADDRESS) &&
 297             addr < phys_to_virt(ISA_END_ADDRESS))
 298                 return;
 299
 300         addr = (volatile void __iomem *)
 301                 (PAGE_MASK & (unsigned long __force)addr);
 302
 303         /* Use the vm area unlocked, assuming the caller
 304            ensures there isn't another iounmap for the same address
 305            in parallel. Reuse of the virtual address is prevented by
 306            leaving it in the global lists until we're done with it.
 307            cpa takes care of the direct mappings. */
 308         read_lock(&vmlist_lock);
 309         for (p = vmlist; p; p = p->next) {
 310                 if (p->addr == addr)
 311                         break;
 312         }
 313         read_unlock(&vmlist_lock);
 314
 315         if (!p) {
 316                 printk(KERN_ERR "iounmap: bad address %p\n", addr);
 317                 dump_stack();
 318                 return;
 319         }
 320
 321         free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
 322
 323         /* Finally remove it */
 324         o = remove_vm_area((void *)addr);
 325         BUG_ON(p != o || o == NULL);
 326         kfree(p);
 327 }
 328 EXPORT_SYMBOL(iounmap);
 329
 330 #ifdef CONFIG_X86_32
 331
 332 int __initdata early_ioremap_debug;
 333
 334 static int __init early_ioremap_debug_setup(char *str)
 335 {
 336         early_ioremap_debug = 1;
 337
 338         return 0;
 339 }
 340 early_param("early_ioremap_debug", early_ioremap_debug_setup);
 341
 342 static __initdata int after_paging_init;
 343 static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)]
 344                 __section(.bss.page_aligned);
 345
 346 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
 347 {
 348         /* Don't assume we're using swapper_pg_dir at this point */
 349         pgd_t *base = __va(read_cr3());
 350         pgd_t *pgd = &base[pgd_index(addr)];
 351         pud_t *pud = pud_offset(pgd, addr);
 352         pmd_t *pmd = pmd_offset(pud, addr);
 353
 354         return pmd;
 355 }
 356
 357 static inline pte_t * __init early_ioremap_pte(unsigned long addr)
 358 {
 359         return &bm_pte[pte_index(addr)];
 360 }
 361
 362 void __init early_ioremap_init(void)
 363 {
 364         pmd_t *pmd;
 365
 366         if (early_ioremap_debug)
 367                 printk(KERN_INFO "early_ioremap_init()\n");
 368
 369         pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
 370         memset(bm_pte, 0, sizeof(bm_pte));
 371         pmd_populate_kernel(&init_mm, pmd, bm_pte);
 372
 373         /*
 374          * The boot-ioremap range spans multiple pmds, for which
 375          * we are not prepared:
 376          */
 377         if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
 378                 WARN_ON(1);
 379                 printk(KERN_WARNING "pmd %p != %p\n",
 380                        pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
 381                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
 382                         fix_to_virt(FIX_BTMAP_BEGIN));
 383                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END):   %08lx\n",
 384                         fix_to_virt(FIX_BTMAP_END));
 385
 386                 printk(KERN_WARNING "FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
 387                 printk(KERN_WARNING "FIX_BTMAP_BEGIN:     %d\n",
 388                        FIX_BTMAP_BEGIN);
 389         }
 390 }
 391
 392 void __init early_ioremap_clear(void)
 393 {
 394         pmd_t *pmd;
 395
 396         if (early_ioremap_debug)
 397                 printk(KERN_INFO "early_ioremap_clear()\n");
 398
 399         pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
 400         pmd_clear(pmd);
 401         paravirt_release_pt(__pa(bm_pte) >> PAGE_SHIFT);
 402         __flush_tlb_all();
 403 }
 404
 405 void __init early_ioremap_reset(void)
 406 {
 407         enum fixed_addresses idx;
 408         unsigned long addr, phys;
 409         pte_t *pte;
 410
 411         after_paging_init = 1;
 412         for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
 413                 addr = fix_to_virt(idx);
 414                 pte = early_ioremap_pte(addr);
 415                 if (pte_present(*pte)) {
 416                         phys = pte_val(*pte) & PAGE_MASK;
 417                         set_fixmap(idx, phys);
 418                 }
 419         }
 420 }
 421
 422 static void __init __early_set_fixmap(enum fixed_addresses idx,
 423                                    unsigned long phys, pgprot_t flags)
 424 {
 425         unsigned long addr = __fix_to_virt(idx);
 426         pte_t *pte;
 427
 428         if (idx >= __end_of_fixed_addresses) {
 429                 BUG();
 430                 return;
 431         }
 432         pte = early_ioremap_pte(addr);
 433         if (pgprot_val(flags))
 434                 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
 435         else
 436                 pte_clear(NULL, addr, pte);
 437         __flush_tlb_one(addr);
 438 }
 439
 440 static inline void __init early_set_fixmap(enum fixed_addresses idx,
 441                                         unsigned long phys)
 442 {
 443         if (after_paging_init)
 444                 set_fixmap(idx, phys);
 445         else
 446                 __early_set_fixmap(idx, phys, PAGE_KERNEL);
 447 }
 448
 449 static inline void __init early_clear_fixmap(enum fixed_addresses idx)
 450 {
 451         if (after_paging_init)
 452                 clear_fixmap(idx);
 453         else
 454                 __early_set_fixmap(idx, 0, __pgprot(0));
 455 }
 456
 457
 458 int __initdata early_ioremap_nested;
 459
 460 static int __init check_early_ioremap_leak(void)
 461 {
 462         if (!early_ioremap_nested)
 463                 return 0;
 464
 465         printk(KERN_WARNING
 466                "Debug warning: early ioremap leak of %d areas detected.\n",
 467                early_ioremap_nested);
 468         printk(KERN_WARNING
 469                "please boot with early_ioremap_debug and report the dmesg.\n");
 470         WARN_ON(1);
 471
 472         return 1;
 473 }
 474 late_initcall(check_early_ioremap_leak);
 475
 476 void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
 477 {
 478         unsigned long offset, last_addr;
 479         unsigned int nrpages, nesting;
 480         enum fixed_addresses idx0, idx;
 481
 482         WARN_ON(system_state != SYSTEM_BOOTING);
 483
 484         nesting = early_ioremap_nested;
 485         if (early_ioremap_debug) {
 486                 printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
 487                        phys_addr, size, nesting);
 488                 dump_stack();
 489         }
 490
 491         /* Don't allow wraparound or zero size */
 492         last_addr = phys_addr + size - 1;
 493         if (!size || last_addr < phys_addr) {
 494                 WARN_ON(1);
 495                 return NULL;
 496         }
 497
 498         if (nesting >= FIX_BTMAPS_NESTING) {
 499                 WARN_ON(1);
 500                 return NULL;
 501         }
 502         early_ioremap_nested++;
 503         /*
 504          * Mappings have to be page-aligned
 505          */
 506         offset = phys_addr & ~PAGE_MASK;
 507         phys_addr &= PAGE_MASK;
 508         size = PAGE_ALIGN(last_addr) - phys_addr;
 509
 510         /*
 511          * Mappings have to fit in the FIX_BTMAP area.
 512          */
 513         nrpages = size >> PAGE_SHIFT;
 514         if (nrpages > NR_FIX_BTMAPS) {
 515                 WARN_ON(1);
 516                 return NULL;
 517         }
 518
 519         /*
 520          * Ok, go for it..
 521          */
 522         idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
 523         idx = idx0;
 524         while (nrpages > 0) {
 525                 early_set_fixmap(idx, phys_addr);
 526                 phys_addr += PAGE_SIZE;
 527                 --idx;
 528                 --nrpages;
 529         }
 530         if (early_ioremap_debug)
 531                 printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
 532
 533         return (void *) (offset + fix_to_virt(idx0));
 534 }
 535
 536 void __init early_iounmap(void *addr, unsigned long size)
 537 {
 538         unsigned long virt_addr;
 539         unsigned long offset;
 540         unsigned int nrpages;
 541         enum fixed_addresses idx;
 542         unsigned int nesting;
 543
 544         nesting = --early_ioremap_nested;
 545         WARN_ON(nesting < 0);
 546
 547         if (early_ioremap_debug) {
 548                 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
 549                        size, nesting);
 550                 dump_stack();
 551         }
 552
 553         virt_addr = (unsigned long)addr;
 554         if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
 555                 WARN_ON(1);
 556                 return;
 557         }
 558         offset = virt_addr & ~PAGE_MASK;
 559         nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
 560
 561         idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
 562         while (nrpages > 0) {
 563                 early_clear_fixmap(idx);
 564                 --idx;
 565                 --nrpages;
 566         }
 567 }
 568
 569 void __this_fixmap_does_not_exist(void)
 570 {
 571         WARN_ON(1);
 572 }
 573
 574 #endif /* CONFIG_X86_32 */