arch/csky/mm/dma-mapping.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.
   3
   4 #include <linux/cache.h>
   5 #include <linux/dma-mapping.h>
   6 #include <linux/dma-contiguous.h>
   7 #include <linux/dma-noncoherent.h>
   8 #include <linux/genalloc.h>
   9 #include <linux/highmem.h>
  10 #include <linux/io.h>
  11 #include <linux/mm.h>
  12 #include <linux/scatterlist.h>
  13 #include <linux/types.h>
  14 #include <linux/version.h>
  15 #include <asm/cache.h>
  16
  17 static struct gen_pool *atomic_pool;
  18 static size_t atomic_pool_size __initdata = SZ_256K;
  19
  20 static int __init early_coherent_pool(char *p)
  21 {
  22         atomic_pool_size = memparse(p, &p);
  23         return 0;
  24 }
  25 early_param("coherent_pool", early_coherent_pool);
  26
  27 static int __init atomic_pool_init(void)
  28 {
  29         struct page *page;
  30         size_t size = atomic_pool_size;
  31         void *ptr;
  32         int ret;
  33
  34         atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
  35         if (!atomic_pool)
  36                 BUG();
  37
  38         page = alloc_pages(GFP_KERNEL | GFP_DMA, get_order(size));
  39         if (!page)
  40                 BUG();
  41
  42         ptr = dma_common_contiguous_remap(page, size, VM_ALLOC,
  43                                           pgprot_noncached(PAGE_KERNEL),
  44                                           __builtin_return_address(0));
  45         if (!ptr)
  46                 BUG();
  47
  48         ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,
  49                                 page_to_phys(page), atomic_pool_size, -1);
  50         if (ret)
  51                 BUG();
  52
  53         gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);
  54
  55         pr_info("DMA: preallocated %zu KiB pool for atomic coherent pool\n",
  56                 atomic_pool_size / 1024);
  57
  58         pr_info("DMA: vaddr: 0x%x phy: 0x%lx,\n", (unsigned int)ptr,
  59                 page_to_phys(page));
  60
  61         return 0;
  62 }
  63 postcore_initcall(atomic_pool_init);
  64
  65 static void *csky_dma_alloc_atomic(struct device *dev, size_t size,
  66                                    dma_addr_t *dma_handle)
  67 {
  68         unsigned long addr;
  69
  70         addr = gen_pool_alloc(atomic_pool, size);
  71         if (addr)
  72                 *dma_handle = gen_pool_virt_to_phys(atomic_pool, addr);
  73
  74         return (void *)addr;
  75 }
  76
  77 static void csky_dma_free_atomic(struct device *dev, size_t size, void *vaddr,
  78                                  dma_addr_t dma_handle, unsigned long attrs)
  79 {
  80         gen_pool_free(atomic_pool, (unsigned long)vaddr, size);
  81 }
  82
  83 static void __dma_clear_buffer(struct page *page, size_t size)
  84 {
  85         if (PageHighMem(page)) {
  86                 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
  87
  88                 do {
  89                         void *ptr = kmap_atomic(page);
  90                         size_t _size = (size < PAGE_SIZE) ? size : PAGE_SIZE;
  91
  92                         memset(ptr, 0, _size);
  93                         dma_wbinv_range((unsigned long)ptr,
  94                                         (unsigned long)ptr + _size);
  95
  96                         kunmap_atomic(ptr);
  97
  98                         page++;
  99                         size -= PAGE_SIZE;
 100                         count--;
 101                 } while (count);
 102         } else {
 103                 void *ptr = page_address(page);
 104
 105                 memset(ptr, 0, size);
 106                 dma_wbinv_range((unsigned long)ptr, (unsigned long)ptr + size);
 107         }
 108 }
 109
 110 static void *csky_dma_alloc_nonatomic(struct device *dev, size_t size,
 111                                       dma_addr_t *dma_handle, gfp_t gfp,
 112                                       unsigned long attrs)
 113 {
 114         void  *vaddr;
 115         struct page *page;
 116         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 117
 118         if (DMA_ATTR_NON_CONSISTENT & attrs) {
 119                 pr_err("csky %s can't support DMA_ATTR_NON_CONSISTENT.\n", __func__);
 120                 return NULL;
 121         }
 122
 123         if (IS_ENABLED(CONFIG_DMA_CMA))
 124                 page = dma_alloc_from_contiguous(dev, count, get_order(size),
 125                                                  gfp);
 126         else
 127                 page = alloc_pages(gfp, get_order(size));
 128
 129         if (!page) {
 130                 pr_err("csky %s no more free pages.\n", __func__);
 131                 return NULL;
 132         }
 133
 134         *dma_handle = page_to_phys(page);
 135
 136         __dma_clear_buffer(page, size);
 137
 138         if (attrs & DMA_ATTR_NO_KERNEL_MAPPING)
 139                 return page;
 140
 141         vaddr = dma_common_contiguous_remap(page, PAGE_ALIGN(size), VM_USERMAP,
 142                 pgprot_noncached(PAGE_KERNEL), __builtin_return_address(0));
 143         if (!vaddr)
 144                 BUG();
 145
 146         return vaddr;
 147 }
 148
 149 static void csky_dma_free_nonatomic(
 150         struct device *dev,
 151         size_t size,
 152         void *vaddr,
 153         dma_addr_t dma_handle,
 154         unsigned long attrs
 155         )
 156 {
 157         struct page *page = phys_to_page(dma_handle);
 158         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 159
 160         if ((unsigned int)vaddr >= VMALLOC_START)
 161                 dma_common_free_remap(vaddr, size, VM_USERMAP);
 162
 163         if (IS_ENABLED(CONFIG_DMA_CMA))
 164                 dma_release_from_contiguous(dev, page, count);
 165         else
 166                 __free_pages(page, get_order(size));
 167 }
 168
 169 void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
 170                      gfp_t gfp, unsigned long attrs)
 171 {
 172         if (gfpflags_allow_blocking(gfp))
 173                 return csky_dma_alloc_nonatomic(dev, size, dma_handle, gfp,
 174                                                 attrs);
 175         else
 176                 return csky_dma_alloc_atomic(dev, size, dma_handle);
 177 }
 178
 179 void arch_dma_free(struct device *dev, size_t size, void *vaddr,
 180                    dma_addr_t dma_handle, unsigned long attrs)
 181 {
 182         if (!addr_in_gen_pool(atomic_pool, (unsigned int) vaddr, size))
 183                 csky_dma_free_nonatomic(dev, size, vaddr, dma_handle, attrs);
 184         else
 185                 csky_dma_free_atomic(dev, size, vaddr, dma_handle, attrs);
 186 }
 187
 188 static inline void cache_op(phys_addr_t paddr, size_t size,
 189                             void (*fn)(unsigned long start, unsigned long end))
 190 {
 191         struct page *page = pfn_to_page(paddr >> PAGE_SHIFT);
 192         unsigned int offset = paddr & ~PAGE_MASK;
 193         size_t left = size;
 194         unsigned long start;
 195
 196         do {
 197                 size_t len = left;
 198
 199                 if (PageHighMem(page)) {
 200                         void *addr;
 201
 202                         if (offset + len > PAGE_SIZE) {
 203                                 if (offset >= PAGE_SIZE) {
 204                                         page += offset >> PAGE_SHIFT;
 205                                         offset &= ~PAGE_MASK;
 206                                 }
 207                                 len = PAGE_SIZE - offset;
 208                         }
 209
 210                         addr = kmap_atomic(page);
 211                         start = (unsigned long)(addr + offset);
 212                         fn(start, start + len);
 213                         kunmap_atomic(addr);
 214                 } else {
 215                         start = (unsigned long)phys_to_virt(paddr);
 216                         fn(start, start + size);
 217                 }
 218                 offset = 0;
 219                 page++;
 220                 left -= len;
 221         } while (left);
 222 }
 223
 224 void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
 225                               size_t size, enum dma_data_direction dir)
 226 {
 227         switch (dir) {
 228         case DMA_TO_DEVICE:
 229                 cache_op(paddr, size, dma_wb_range);
 230                 break;
 231         case DMA_FROM_DEVICE:
 232         case DMA_BIDIRECTIONAL:
 233                 cache_op(paddr, size, dma_wbinv_range);
 234                 break;
 235         default:
 236                 BUG();
 237         }
 238 }
 239
 240 void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
 241                            size_t size, enum dma_data_direction dir)
 242 {
 243         switch (dir) {
 244         case DMA_TO_DEVICE:
 245                 cache_op(paddr, size, dma_wb_range);
 246                 break;
 247         case DMA_FROM_DEVICE:
 248         case DMA_BIDIRECTIONAL:
 249                 cache_op(paddr, size, dma_wbinv_range);
 250                 break;
 251         default:
 252                 BUG();
 253         }
 254 }