arch/mips/mm/dma-noncoherent.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2000  Ani Joshi <ajoshi@unixbox.com>
   4  * Copyright (C) 2000, 2001, 06  Ralf Baechle <ralf@linux-mips.org>
   5  * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
   6  */
   7 #include <linux/dma-direct.h>
   8 #include <linux/dma-noncoherent.h>
   9 #include <linux/dma-contiguous.h>
  10 #include <linux/highmem.h>
  11
  12 #include <asm/cache.h>
  13 #include <asm/cpu-type.h>
  14 #include <asm/dma-coherence.h>
  15 #include <asm/io.h>
  16
  17 #ifdef CONFIG_DMA_PERDEV_COHERENT
  18 static inline int dev_is_coherent(struct device *dev)
  19 {
  20         return dev->archdata.dma_coherent;
  21 }
  22 #else
  23 static inline int dev_is_coherent(struct device *dev)
  24 {
  25         switch (coherentio) {
  26         default:
  27         case IO_COHERENCE_DEFAULT:
  28                 return hw_coherentio;
  29         case IO_COHERENCE_ENABLED:
  30                 return 1;
  31         case IO_COHERENCE_DISABLED:
  32                 return 0;
  33         }
  34 }
  35 #endif /* CONFIG_DMA_PERDEV_COHERENT */
  36
  37 /*
  38  * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively
  39  * fill random cachelines with stale data at any time, requiring an extra
  40  * flush post-DMA.
  41  *
  42  * Warning on the terminology - Linux calls an uncached area coherent;  MIPS
  43  * terminology calls memory areas with hardware maintained coherency coherent.
  44  *
  45  * Note that the R14000 and R16000 should also be checked for in this condition.
  46  * However this function is only called on non-I/O-coherent systems and only the
  47  * R10000 and R12000 are used in such systems, the SGI IP28 Indigo² rsp.
  48  * SGI IP32 aka O2.
  49  */
  50 static inline bool cpu_needs_post_dma_flush(struct device *dev)
  51 {
  52         if (dev_is_coherent(dev))
  53                 return false;
  54
  55         switch (boot_cpu_type()) {
  56         case CPU_R10000:
  57         case CPU_R12000:
  58         case CPU_BMIPS5000:
  59                 return true;
  60         default:
  61                 /*
  62                  * Presence of MAARs suggests that the CPU supports
  63                  * speculatively prefetching data, and therefore requires
  64                  * the post-DMA flush/invalidate.
  65                  */
  66                 return cpu_has_maar;
  67         }
  68 }
  69
  70 void *arch_dma_alloc(struct device *dev, size_t size,
  71                 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
  72 {
  73         void *ret;
  74
  75         ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs);
  76         if (!ret)
  77                 return NULL;
  78
  79         if (!dev_is_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) {
  80                 dma_cache_wback_inv((unsigned long) ret, size);
  81                 ret = (void *)UNCAC_ADDR(ret);
  82         }
  83
  84         return ret;
  85 }
  86
  87 void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
  88                 dma_addr_t dma_addr, unsigned long attrs)
  89 {
  90         if (!(attrs & DMA_ATTR_NON_CONSISTENT) && !dev_is_coherent(dev))
  91                 cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr);
  92         dma_direct_free(dev, size, cpu_addr, dma_addr, attrs);
  93 }
  94
  95 int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma,
  96                 void *cpu_addr, dma_addr_t dma_addr, size_t size,
  97                 unsigned long attrs)
  98 {
  99         unsigned long user_count = vma_pages(vma);
 100         unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 101         unsigned long addr = (unsigned long)cpu_addr;
 102         unsigned long off = vma->vm_pgoff;
 103         unsigned long pfn;
 104         int ret = -ENXIO;
 105
 106         if (!dev_is_coherent(dev))
 107                 addr = CAC_ADDR(addr);
 108
 109         pfn = page_to_pfn(virt_to_page((void *)addr));
 110
 111         if (attrs & DMA_ATTR_WRITE_COMBINE)
 112                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 113         else
 114                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 115
 116         if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
 117                 return ret;
 118
 119         if (off < count && user_count <= (count - off)) {
 120                 ret = remap_pfn_range(vma, vma->vm_start,
 121                                       pfn + off,
 122                                       user_count << PAGE_SHIFT,
 123                                       vma->vm_page_prot);
 124         }
 125
 126         return ret;
 127 }
 128
 129 static inline void dma_sync_virt(void *addr, size_t size,
 130                 enum dma_data_direction dir)
 131 {
 132         switch (dir) {
 133         case DMA_TO_DEVICE:
 134                 dma_cache_wback((unsigned long)addr, size);
 135                 break;
 136
 137         case DMA_FROM_DEVICE:
 138                 dma_cache_inv((unsigned long)addr, size);
 139                 break;
 140
 141         case DMA_BIDIRECTIONAL:
 142                 dma_cache_wback_inv((unsigned long)addr, size);
 143                 break;
 144
 145         default:
 146                 BUG();
 147         }
 148 }
 149
 150 /*
 151  * A single sg entry may refer to multiple physically contiguous pages.  But
 152  * we still need to process highmem pages individually.  If highmem is not
 153  * configured then the bulk of this loop gets optimized out.
 154  */
 155 static inline void dma_sync_phys(phys_addr_t paddr, size_t size,
 156                 enum dma_data_direction dir)
 157 {
 158         struct page *page = pfn_to_page(paddr >> PAGE_SHIFT);
 159         unsigned long offset = paddr & ~PAGE_MASK;
 160         size_t left = size;
 161
 162         do {
 163                 size_t len = left;
 164
 165                 if (PageHighMem(page)) {
 166                         void *addr;
 167
 168                         if (offset + len > PAGE_SIZE) {
 169                                 if (offset >= PAGE_SIZE) {
 170                                         page += offset >> PAGE_SHIFT;
 171                                         offset &= ~PAGE_MASK;
 172                                 }
 173                                 len = PAGE_SIZE - offset;
 174                         }
 175
 176                         addr = kmap_atomic(page);
 177                         dma_sync_virt(addr + offset, len, dir);
 178                         kunmap_atomic(addr);
 179                 } else
 180                         dma_sync_virt(page_address(page) + offset, size, dir);
 181                 offset = 0;
 182                 page++;
 183                 left -= len;
 184         } while (left);
 185 }
 186
 187 void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
 188                 size_t size, enum dma_data_direction dir)
 189 {
 190         if (!dev_is_coherent(dev))
 191                 dma_sync_phys(paddr, size, dir);
 192 }
 193
 194 void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
 195                 size_t size, enum dma_data_direction dir)
 196 {
 197         if (cpu_needs_post_dma_flush(dev))
 198                 dma_sync_phys(paddr, size, dir);
 199 }
 200
 201 void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size,
 202                 enum dma_data_direction direction)
 203 {
 204         BUG_ON(direction == DMA_NONE);
 205
 206         if (!dev_is_coherent(dev))
 207                 dma_sync_virt(vaddr, size, direction);
 208 }