Merge branch 'work.get_user_pages_fast' of git://git.kernel.org/pub/scm/linux/kernel...
[sfrench/cifs-2.6.git] / drivers / staging / media / atomisp / pci / atomisp2 / hmm / hmm.c
1 /*
2  * Support for Medifield PNW Camera Imaging ISP subsystem.
3  *
4  * Copyright (c) 2010-2017 Intel Corporation. All Rights Reserved.
5  *
6  * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
7  *
8  * This program is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU General Public License version
10  * 2 as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  *
17  *
18  */
19 /*
20  * This file contains entry functions for memory management of ISP driver
21  */
22 #include <linux/kernel.h>
23 #include <linux/types.h>
24 #include <linux/mm.h>
25 #include <linux/highmem.h>      /* for kmap */
26 #include <linux/io.h>           /* for page_to_phys */
27 #include <linux/sysfs.h>
28
29 #include "hmm/hmm.h"
30 #include "hmm/hmm_pool.h"
31 #include "hmm/hmm_bo.h"
32
33 #include "atomisp_internal.h"
34 #include "asm/cacheflush.h"
35 #include "mmu/isp_mmu.h"
36 #include "mmu/sh_mmu_mrfld.h"
37
38 struct hmm_bo_device bo_device;
39 struct hmm_pool dynamic_pool;
40 struct hmm_pool reserved_pool;
41 static ia_css_ptr dummy_ptr;
42 static bool hmm_initialized;
43 struct _hmm_mem_stat hmm_mem_stat;
44
45 /*
46  * p: private
47  * s: shared
48  * u: user
49  * i: ion
50  */
51 static const char hmm_bo_type_string[] = "psui";
52
53 static ssize_t bo_show(struct device *dev, struct device_attribute *attr,
54                        char *buf, struct list_head *bo_list, bool active)
55 {
56         ssize_t ret = 0;
57         struct hmm_buffer_object *bo;
58         unsigned long flags;
59         int i;
60         long total[HMM_BO_LAST] = { 0 };
61         long count[HMM_BO_LAST] = { 0 };
62         int index1 = 0;
63         int index2 = 0;
64
65         ret = scnprintf(buf, PAGE_SIZE, "type pgnr\n");
66         if (ret <= 0)
67                 return 0;
68
69         index1 += ret;
70
71         spin_lock_irqsave(&bo_device.list_lock, flags);
72         list_for_each_entry(bo, bo_list, list) {
73                 if ((active && (bo->status & HMM_BO_ALLOCED)) ||
74                     (!active && !(bo->status & HMM_BO_ALLOCED))) {
75                         ret = scnprintf(buf + index1, PAGE_SIZE - index1,
76                                         "%c %d\n",
77                                         hmm_bo_type_string[bo->type], bo->pgnr);
78
79                         total[bo->type] += bo->pgnr;
80                         count[bo->type]++;
81                         if (ret > 0)
82                                 index1 += ret;
83                 }
84         }
85         spin_unlock_irqrestore(&bo_device.list_lock, flags);
86
87         for (i = 0; i < HMM_BO_LAST; i++) {
88                 if (count[i]) {
89                         ret = scnprintf(buf + index1 + index2,
90                                         PAGE_SIZE - index1 - index2,
91                                         "%ld %c buffer objects: %ld KB\n",
92                                         count[i], hmm_bo_type_string[i],
93                                         total[i] * 4);
94                         if (ret > 0)
95                                 index2 += ret;
96                 }
97         }
98
99         /* Add trailing zero, not included by scnprintf */
100         return index1 + index2 + 1;
101 }
102
103 static ssize_t active_bo_show(struct device *dev, struct device_attribute *attr,
104                               char *buf)
105 {
106         return bo_show(dev, attr, buf, &bo_device.entire_bo_list, true);
107 }
108
109 static ssize_t free_bo_show(struct device *dev, struct device_attribute *attr,
110                             char *buf)
111 {
112         return bo_show(dev, attr, buf, &bo_device.entire_bo_list, false);
113 }
114
115 static ssize_t reserved_pool_show(struct device *dev,
116                                   struct device_attribute *attr,
117                                   char *buf)
118 {
119         ssize_t ret = 0;
120
121         struct hmm_reserved_pool_info *pinfo = reserved_pool.pool_info;
122         unsigned long flags;
123
124         if (!pinfo || !pinfo->initialized)
125                 return 0;
126
127         spin_lock_irqsave(&pinfo->list_lock, flags);
128         ret = scnprintf(buf, PAGE_SIZE, "%d out of %d pages available\n",
129                         pinfo->index, pinfo->pgnr);
130         spin_unlock_irqrestore(&pinfo->list_lock, flags);
131
132         if (ret > 0)
133                 ret++; /* Add trailing zero, not included by scnprintf */
134
135         return ret;
136 };
137
138 static ssize_t dynamic_pool_show(struct device *dev,
139                                  struct device_attribute *attr,
140                                  char *buf)
141 {
142         ssize_t ret = 0;
143
144         struct hmm_dynamic_pool_info *pinfo = dynamic_pool.pool_info;
145         unsigned long flags;
146
147         if (!pinfo || !pinfo->initialized)
148                 return 0;
149
150         spin_lock_irqsave(&pinfo->list_lock, flags);
151         ret = scnprintf(buf, PAGE_SIZE, "%d (max %d) pages available\n",
152                         pinfo->pgnr, pinfo->pool_size);
153         spin_unlock_irqrestore(&pinfo->list_lock, flags);
154
155         if (ret > 0)
156                 ret++; /* Add trailing zero, not included by scnprintf */
157
158         return ret;
159 };
160
161 static DEVICE_ATTR(active_bo, 0444, active_bo_show, NULL);
162 static DEVICE_ATTR(free_bo, 0444, free_bo_show, NULL);
163 static DEVICE_ATTR(reserved_pool, 0444, reserved_pool_show, NULL);
164 static DEVICE_ATTR(dynamic_pool, 0444, dynamic_pool_show, NULL);
165
166 static struct attribute *sysfs_attrs_ctrl[] = {
167         &dev_attr_active_bo.attr,
168         &dev_attr_free_bo.attr,
169         &dev_attr_reserved_pool.attr,
170         &dev_attr_dynamic_pool.attr,
171         NULL
172 };
173
174 static struct attribute_group atomisp_attribute_group[] = {
175         {.attrs = sysfs_attrs_ctrl },
176 };
177
178 int hmm_init(void)
179 {
180         int ret;
181
182         ret = hmm_bo_device_init(&bo_device, &sh_mmu_mrfld,
183                                  ISP_VM_START, ISP_VM_SIZE);
184         if (ret)
185                 dev_err(atomisp_dev, "hmm_bo_device_init failed.\n");
186
187         hmm_initialized = true;
188
189         /*
190          * As hmm use NULL to indicate invalid ISP virtual address,
191          * and ISP_VM_START is defined to 0 too, so we allocate
192          * one piece of dummy memory, which should return value 0,
193          * at the beginning, to avoid hmm_alloc return 0 in the
194          * further allocation.
195          */
196         dummy_ptr = hmm_alloc(1, HMM_BO_PRIVATE, 0, NULL, HMM_UNCACHED);
197
198         if (!ret) {
199                 ret = sysfs_create_group(&atomisp_dev->kobj,
200                                          atomisp_attribute_group);
201                 if (ret)
202                         dev_err(atomisp_dev,
203                                 "%s Failed to create sysfs\n", __func__);
204         }
205
206         return ret;
207 }
208
209 void hmm_cleanup(void)
210 {
211         sysfs_remove_group(&atomisp_dev->kobj, atomisp_attribute_group);
212
213         /* free dummy memory first */
214         hmm_free(dummy_ptr);
215         dummy_ptr = 0;
216
217         hmm_bo_device_exit(&bo_device);
218         hmm_initialized = false;
219 }
220
221 ia_css_ptr hmm_alloc(size_t bytes, enum hmm_bo_type type,
222                      int from_highmem, void *userptr, bool cached)
223 {
224         unsigned int pgnr;
225         struct hmm_buffer_object *bo;
226         int ret;
227
228         /*
229          * Check if we are initialized. In the ideal world we wouldn't need
230          * this but we can tackle it once the driver is a lot cleaner
231          */
232
233         if (!hmm_initialized)
234                 hmm_init();
235         /* Get page number from size */
236         pgnr = size_to_pgnr_ceil(bytes);
237
238         /* Buffer object structure init */
239         bo = hmm_bo_alloc(&bo_device, pgnr);
240         if (!bo) {
241                 dev_err(atomisp_dev, "hmm_bo_create failed.\n");
242                 goto create_bo_err;
243         }
244
245         /* Allocate pages for memory */
246         ret = hmm_bo_alloc_pages(bo, type, from_highmem, userptr, cached);
247         if (ret) {
248                 dev_err(atomisp_dev, "hmm_bo_alloc_pages failed.\n");
249                 goto alloc_page_err;
250         }
251
252         /* Combind the virtual address and pages togather */
253         ret = hmm_bo_bind(bo);
254         if (ret) {
255                 dev_err(atomisp_dev, "hmm_bo_bind failed.\n");
256                 goto bind_err;
257         }
258
259         hmm_mem_stat.tol_cnt += pgnr;
260
261         return bo->start;
262
263 bind_err:
264         hmm_bo_free_pages(bo);
265 alloc_page_err:
266         hmm_bo_unref(bo);
267 create_bo_err:
268         return 0;
269 }
270
271 void hmm_free(ia_css_ptr virt)
272 {
273         struct hmm_buffer_object *bo;
274
275         WARN_ON(!virt);
276
277         bo = hmm_bo_device_search_start(&bo_device, (unsigned int)virt);
278
279         if (!bo) {
280                 dev_err(atomisp_dev,
281                         "can not find buffer object start with address 0x%x\n",
282                         (unsigned int)virt);
283                 return;
284         }
285
286         hmm_mem_stat.tol_cnt -= bo->pgnr;
287
288         hmm_bo_unbind(bo);
289         hmm_bo_free_pages(bo);
290         hmm_bo_unref(bo);
291 }
292
293 static inline int hmm_check_bo(struct hmm_buffer_object *bo, unsigned int ptr)
294 {
295         if (!bo) {
296                 dev_err(atomisp_dev,
297                         "can not find buffer object contains address 0x%x\n",
298                         ptr);
299                 return -EINVAL;
300         }
301
302         if (!hmm_bo_page_allocated(bo)) {
303                 dev_err(atomisp_dev,
304                         "buffer object has no page allocated.\n");
305                 return -EINVAL;
306         }
307
308         if (!hmm_bo_allocated(bo)) {
309                 dev_err(atomisp_dev,
310                         "buffer object has no virtual address space allocated.\n");
311                 return -EINVAL;
312         }
313
314         return 0;
315 }
316
317 /* Read function in ISP memory management */
318 static int load_and_flush_by_kmap(ia_css_ptr virt, void *data,
319                                   unsigned int bytes)
320 {
321         struct hmm_buffer_object *bo;
322         unsigned int idx, offset, len;
323         char *src, *des;
324         int ret;
325
326         bo = hmm_bo_device_search_in_range(&bo_device, virt);
327         ret = hmm_check_bo(bo, virt);
328         if (ret)
329                 return ret;
330
331         des = (char *)data;
332         while (bytes) {
333                 idx = (virt - bo->start) >> PAGE_SHIFT;
334                 offset = (virt - bo->start) - (idx << PAGE_SHIFT);
335
336                 src = (char *)kmap(bo->page_obj[idx].page) + offset;
337
338                 if ((bytes + offset) >= PAGE_SIZE) {
339                         len = PAGE_SIZE - offset;
340                         bytes -= len;
341                 } else {
342                         len = bytes;
343                         bytes = 0;
344                 }
345
346                 virt += len;    /* update virt for next loop */
347
348                 if (des) {
349                         memcpy(des, src, len);
350                         des += len;
351                 }
352
353                 clflush_cache_range(src, len);
354
355                 kunmap(bo->page_obj[idx].page);
356         }
357
358         return 0;
359 }
360
361 /* Read function in ISP memory management */
362 static int load_and_flush(ia_css_ptr virt, void *data, unsigned int bytes)
363 {
364         struct hmm_buffer_object *bo;
365         int ret;
366
367         bo = hmm_bo_device_search_in_range(&bo_device, virt);
368         ret = hmm_check_bo(bo, virt);
369         if (ret)
370                 return ret;
371
372         if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
373                 void *src = bo->vmap_addr;
374
375                 src += (virt - bo->start);
376                 memcpy(data, src, bytes);
377                 if (bo->status & HMM_BO_VMAPED_CACHED)
378                         clflush_cache_range(src, bytes);
379         } else {
380                 void *vptr;
381
382                 vptr = hmm_bo_vmap(bo, true);
383                 if (!vptr)
384                         return load_and_flush_by_kmap(virt, data, bytes);
385                 else
386                         vptr = vptr + (virt - bo->start);
387
388                 memcpy(data, vptr, bytes);
389                 clflush_cache_range(vptr, bytes);
390                 hmm_bo_vunmap(bo);
391         }
392
393         return 0;
394 }
395
396 /* Read function in ISP memory management */
397 int hmm_load(ia_css_ptr virt, void *data, unsigned int bytes)
398 {
399         if (!data) {
400                 dev_err(atomisp_dev,
401                         "hmm_load NULL argument\n");
402                 return -EINVAL;
403         }
404         return load_and_flush(virt, data, bytes);
405 }
406
407 /* Flush hmm data from the data cache */
408 int hmm_flush(ia_css_ptr virt, unsigned int bytes)
409 {
410         return load_and_flush(virt, NULL, bytes);
411 }
412
413 /* Write function in ISP memory management */
414 int hmm_store(ia_css_ptr virt, const void *data, unsigned int bytes)
415 {
416         struct hmm_buffer_object *bo;
417         unsigned int idx, offset, len;
418         char *src, *des;
419         int ret;
420
421         bo = hmm_bo_device_search_in_range(&bo_device, virt);
422         ret = hmm_check_bo(bo, virt);
423         if (ret)
424                 return ret;
425
426         if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
427                 void *dst = bo->vmap_addr;
428
429                 dst += (virt - bo->start);
430                 memcpy(dst, data, bytes);
431                 if (bo->status & HMM_BO_VMAPED_CACHED)
432                         clflush_cache_range(dst, bytes);
433         } else {
434                 void *vptr;
435
436                 vptr = hmm_bo_vmap(bo, true);
437                 if (vptr) {
438                         vptr = vptr + (virt - bo->start);
439
440                         memcpy(vptr, data, bytes);
441                         clflush_cache_range(vptr, bytes);
442                         hmm_bo_vunmap(bo);
443                         return 0;
444                 }
445         }
446
447         src = (char *)data;
448         while (bytes) {
449                 idx = (virt - bo->start) >> PAGE_SHIFT;
450                 offset = (virt - bo->start) - (idx << PAGE_SHIFT);
451
452                 if (in_atomic())
453                         des = (char *)kmap_atomic(bo->page_obj[idx].page);
454                 else
455                         des = (char *)kmap(bo->page_obj[idx].page);
456
457                 if (!des) {
458                         dev_err(atomisp_dev,
459                                 "kmap buffer object page failed: pg_idx = %d\n",
460                                 idx);
461                         return -EINVAL;
462                 }
463
464                 des += offset;
465
466                 if ((bytes + offset) >= PAGE_SIZE) {
467                         len = PAGE_SIZE - offset;
468                         bytes -= len;
469                 } else {
470                         len = bytes;
471                         bytes = 0;
472                 }
473
474                 virt += len;
475
476                 memcpy(des, src, len);
477
478                 src += len;
479
480                 clflush_cache_range(des, len);
481
482                 if (in_atomic())
483                         /*
484                          * Note: kunmap_atomic requires return addr from
485                          * kmap_atomic, not the page. See linux/highmem.h
486                          */
487                         kunmap_atomic(des - offset);
488                 else
489                         kunmap(bo->page_obj[idx].page);
490         }
491
492         return 0;
493 }
494
495 /* memset function in ISP memory management */
496 int hmm_set(ia_css_ptr virt, int c, unsigned int bytes)
497 {
498         struct hmm_buffer_object *bo;
499         unsigned int idx, offset, len;
500         char *des;
501         int ret;
502
503         bo = hmm_bo_device_search_in_range(&bo_device, virt);
504         ret = hmm_check_bo(bo, virt);
505         if (ret)
506                 return ret;
507
508         if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
509                 void *dst = bo->vmap_addr;
510
511                 dst += (virt - bo->start);
512                 memset(dst, c, bytes);
513
514                 if (bo->status & HMM_BO_VMAPED_CACHED)
515                         clflush_cache_range(dst, bytes);
516         } else {
517                 void *vptr;
518
519                 vptr = hmm_bo_vmap(bo, true);
520                 if (vptr) {
521                         vptr = vptr + (virt - bo->start);
522                         memset(vptr, c, bytes);
523                         clflush_cache_range(vptr, bytes);
524                         hmm_bo_vunmap(bo);
525                         return 0;
526                 }
527         }
528
529         while (bytes) {
530                 idx = (virt - bo->start) >> PAGE_SHIFT;
531                 offset = (virt - bo->start) - (idx << PAGE_SHIFT);
532
533                 des = (char *)kmap(bo->page_obj[idx].page) + offset;
534
535                 if ((bytes + offset) >= PAGE_SIZE) {
536                         len = PAGE_SIZE - offset;
537                         bytes -= len;
538                 } else {
539                         len = bytes;
540                         bytes = 0;
541                 }
542
543                 virt += len;
544
545                 memset(des, c, len);
546
547                 clflush_cache_range(des, len);
548
549                 kunmap(bo->page_obj[idx].page);
550         }
551
552         return 0;
553 }
554
555 /* Virtual address to physical address convert */
556 phys_addr_t hmm_virt_to_phys(ia_css_ptr virt)
557 {
558         unsigned int idx, offset;
559         struct hmm_buffer_object *bo;
560
561         bo = hmm_bo_device_search_in_range(&bo_device, virt);
562         if (!bo) {
563                 dev_err(atomisp_dev,
564                         "can not find buffer object contains address 0x%x\n",
565                         virt);
566                 return -1;
567         }
568
569         idx = (virt - bo->start) >> PAGE_SHIFT;
570         offset = (virt - bo->start) - (idx << PAGE_SHIFT);
571
572         return page_to_phys(bo->page_obj[idx].page) + offset;
573 }
574
575 int hmm_mmap(struct vm_area_struct *vma, ia_css_ptr virt)
576 {
577         struct hmm_buffer_object *bo;
578
579         bo = hmm_bo_device_search_start(&bo_device, virt);
580         if (!bo) {
581                 dev_err(atomisp_dev,
582                         "can not find buffer object start with address 0x%x\n",
583                         virt);
584                 return -EINVAL;
585         }
586
587         return hmm_bo_mmap(vma, bo);
588 }
589
590 /* Map ISP virtual address into IA virtual address */
591 void *hmm_vmap(ia_css_ptr virt, bool cached)
592 {
593         struct hmm_buffer_object *bo;
594         void *ptr;
595
596         bo = hmm_bo_device_search_in_range(&bo_device, virt);
597         if (!bo) {
598                 dev_err(atomisp_dev,
599                         "can not find buffer object contains address 0x%x\n",
600                         virt);
601                 return NULL;
602         }
603
604         ptr = hmm_bo_vmap(bo, cached);
605         if (ptr)
606                 return ptr + (virt - bo->start);
607         else
608                 return NULL;
609 }
610
611 /* Flush the memory which is mapped as cached memory through hmm_vmap */
612 void hmm_flush_vmap(ia_css_ptr virt)
613 {
614         struct hmm_buffer_object *bo;
615
616         bo = hmm_bo_device_search_in_range(&bo_device, virt);
617         if (!bo) {
618                 dev_warn(atomisp_dev,
619                          "can not find buffer object contains address 0x%x\n",
620                          virt);
621                 return;
622         }
623
624         hmm_bo_flush_vmap(bo);
625 }
626
627 void hmm_vunmap(ia_css_ptr virt)
628 {
629         struct hmm_buffer_object *bo;
630
631         bo = hmm_bo_device_search_in_range(&bo_device, virt);
632         if (!bo) {
633                 dev_warn(atomisp_dev,
634                          "can not find buffer object contains address 0x%x\n",
635                          virt);
636                 return;
637         }
638
639         hmm_bo_vunmap(bo);
640 }
641
642 int hmm_pool_register(unsigned int pool_size, enum hmm_pool_type pool_type)
643 {
644         switch (pool_type) {
645         case HMM_POOL_TYPE_RESERVED:
646                 reserved_pool.pops = &reserved_pops;
647                 return reserved_pool.pops->pool_init(&reserved_pool.pool_info,
648                                                      pool_size);
649         case HMM_POOL_TYPE_DYNAMIC:
650                 dynamic_pool.pops = &dynamic_pops;
651                 return dynamic_pool.pops->pool_init(&dynamic_pool.pool_info,
652                                                     pool_size);
653         default:
654                 dev_err(atomisp_dev, "invalid pool type.\n");
655                 return -EINVAL;
656         }
657 }
658
659 void hmm_pool_unregister(enum hmm_pool_type pool_type)
660 {
661         switch (pool_type) {
662         case HMM_POOL_TYPE_RESERVED:
663                 if (reserved_pool.pops && reserved_pool.pops->pool_exit)
664                         reserved_pool.pops->pool_exit(&reserved_pool.pool_info);
665                 break;
666         case HMM_POOL_TYPE_DYNAMIC:
667                 if (dynamic_pool.pops && dynamic_pool.pops->pool_exit)
668                         dynamic_pool.pops->pool_exit(&dynamic_pool.pool_info);
669                 break;
670         default:
671                 dev_err(atomisp_dev, "invalid pool type.\n");
672                 break;
673         }
674
675         return;
676 }
677
678 void *hmm_isp_vaddr_to_host_vaddr(ia_css_ptr ptr, bool cached)
679 {
680         return hmm_vmap(ptr, cached);
681         /* vmunmap will be done in hmm_bo_release() */
682 }
683
684 ia_css_ptr hmm_host_vaddr_to_hrt_vaddr(const void *ptr)
685 {
686         struct hmm_buffer_object *bo;
687
688         bo = hmm_bo_device_search_vmap_start(&bo_device, ptr);
689         if (bo)
690                 return bo->start;
691
692         dev_err(atomisp_dev,
693                 "can not find buffer object whose kernel virtual address is %p\n",
694                 ptr);
695         return 0;
696 }
697
698 void hmm_show_mem_stat(const char *func, const int line)
699 {
700         trace_printk("tol_cnt=%d usr_size=%d res_size=%d res_cnt=%d sys_size=%d  dyc_thr=%d dyc_size=%d.\n",
701                      hmm_mem_stat.tol_cnt,
702                      hmm_mem_stat.usr_size, hmm_mem_stat.res_size,
703                      hmm_mem_stat.res_cnt, hmm_mem_stat.sys_size,
704                      hmm_mem_stat.dyc_thr, hmm_mem_stat.dyc_size);
705 }
706
707 void hmm_init_mem_stat(int res_pgnr, int dyc_en, int dyc_pgnr)
708 {
709         hmm_mem_stat.res_size = res_pgnr;
710         /* If reserved mem pool is not enabled, set its "mem stat" values as -1. */
711         if (0 == hmm_mem_stat.res_size) {
712                 hmm_mem_stat.res_size = -1;
713                 hmm_mem_stat.res_cnt = -1;
714         }
715
716         /* If dynamic memory pool is not enabled, set its "mem stat" values as -1. */
717         if (!dyc_en) {
718                 hmm_mem_stat.dyc_size = -1;
719                 hmm_mem_stat.dyc_thr = -1;
720         } else {
721                 hmm_mem_stat.dyc_size = 0;
722                 hmm_mem_stat.dyc_thr = dyc_pgnr;
723         }
724         hmm_mem_stat.usr_size = 0;
725         hmm_mem_stat.sys_size = 0;
726         hmm_mem_stat.tol_cnt = 0;
727 }