drivers/staging/media/atomisp/pci/atomisp2/hmm/hmm_bo.c

   1 /*
   2  * Support for Medifield PNW Camera Imaging ISP subsystem.
   3  *
   4  * Copyright (c) 2010 Intel Corporation. All Rights Reserved.
   5  *
   6  * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License version
  10  * 2 as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  *
  18  */
  19 /*
  20  * This file contains functions for buffer object structure management
  21  */
  22 #include <linux/kernel.h>
  23 #include <linux/types.h>
  24 #include <linux/gfp.h>          /* for GFP_ATOMIC */
  25 #include <linux/mm.h>
  26 #include <linux/mm_types.h>
  27 #include <linux/hugetlb.h>
  28 #include <linux/highmem.h>
  29 #include <linux/slab.h>         /* for kmalloc */
  30 #include <linux/module.h>
  31 #include <linux/moduleparam.h>
  32 #include <linux/string.h>
  33 #include <linux/list.h>
  34 #include <linux/errno.h>
  35 #include <linux/io.h>
  36 #include <asm/current.h>
  37 #include <linux/sched/signal.h>
  38 #include <linux/file.h>
  39
  40 #include <asm/set_memory.h>
  41
  42 #include "atomisp_internal.h"
  43 #include "hmm/hmm_common.h"
  44 #include "hmm/hmm_pool.h"
  45 #include "hmm/hmm_bo.h"
  46
  47 static unsigned int order_to_nr(unsigned int order)
  48 {
  49         return 1U << order;
  50 }
  51
  52 static unsigned int nr_to_order_bottom(unsigned int nr)
  53 {
  54         return fls(nr) - 1;
  55 }
  56
  57 static struct hmm_buffer_object *__bo_alloc(struct kmem_cache *bo_cache)
  58 {
  59         struct hmm_buffer_object *bo;
  60
  61         bo = kmem_cache_alloc(bo_cache, GFP_KERNEL);
  62         if (!bo)
  63                 dev_err(atomisp_dev, "%s: failed!\n", __func__);
  64
  65         return bo;
  66 }
  67
  68 static int __bo_init(struct hmm_bo_device *bdev, struct hmm_buffer_object *bo,
  69                                         unsigned int pgnr)
  70 {
  71         check_bodev_null_return(bdev, -EINVAL);
  72         var_equal_return(hmm_bo_device_inited(bdev), 0, -EINVAL,
  73                         "hmm_bo_device not inited yet.\n");
  74         /* prevent zero size buffer object */
  75         if (pgnr == 0) {
  76                 dev_err(atomisp_dev, "0 size buffer is not allowed.\n");
  77                 return -EINVAL;
  78         }
  79
  80         memset(bo, 0, sizeof(*bo));
  81         mutex_init(&bo->mutex);
  82
  83         /* init the bo->list HEAD as an element of entire_bo_list */
  84         INIT_LIST_HEAD(&bo->list);
  85
  86         bo->bdev = bdev;
  87         bo->vmap_addr = NULL;
  88         bo->status = HMM_BO_FREE;
  89         bo->start = bdev->start;
  90         bo->pgnr = pgnr;
  91         bo->end = bo->start + pgnr_to_size(pgnr);
  92         bo->prev = NULL;
  93         bo->next = NULL;
  94
  95         return 0;
  96 }
  97
  98 static struct hmm_buffer_object *__bo_search_and_remove_from_free_rbtree(
  99                                 struct rb_node *node, unsigned int pgnr)
 100 {
 101         struct hmm_buffer_object *this, *ret_bo, *temp_bo;
 102
 103         this = rb_entry(node, struct hmm_buffer_object, node);
 104         if (this->pgnr == pgnr ||
 105                 (this->pgnr > pgnr && this->node.rb_left == NULL)) {
 106                 goto remove_bo_and_return;
 107         } else {
 108                 if (this->pgnr < pgnr) {
 109                         if (!this->node.rb_right)
 110                                 return NULL;
 111                         ret_bo = __bo_search_and_remove_from_free_rbtree(
 112                                 this->node.rb_right, pgnr);
 113                 } else {
 114                         ret_bo = __bo_search_and_remove_from_free_rbtree(
 115                                 this->node.rb_left, pgnr);
 116                 }
 117                 if (!ret_bo) {
 118                         if (this->pgnr > pgnr)
 119                                 goto remove_bo_and_return;
 120                         else
 121                                 return NULL;
 122                 }
 123                 return ret_bo;
 124         }
 125
 126 remove_bo_and_return:
 127         /* NOTE: All nodes on free rbtree have a 'prev' that points to NULL.
 128          * 1. check if 'this->next' is NULL:
 129          *      yes: erase 'this' node and rebalance rbtree, return 'this'.
 130          */
 131         if (this->next == NULL) {
 132                 rb_erase(&this->node, &this->bdev->free_rbtree);
 133                 return this;
 134         }
 135         /* NOTE: if 'this->next' is not NULL, always return 'this->next' bo.
 136          * 2. check if 'this->next->next' is NULL:
 137          *      yes: change the related 'next/prev' pointer,
 138          *              return 'this->next' but the rbtree stays unchanged.
 139          */
 140         temp_bo = this->next;
 141         this->next = temp_bo->next;
 142         if (temp_bo->next)
 143                 temp_bo->next->prev = this;
 144         temp_bo->next = NULL;
 145         temp_bo->prev = NULL;
 146         return temp_bo;
 147 }
 148
 149 static struct hmm_buffer_object *__bo_search_by_addr(struct rb_root *root,
 150                                                         ia_css_ptr start)
 151 {
 152         struct rb_node *n = root->rb_node;
 153         struct hmm_buffer_object *bo;
 154
 155         do {
 156                 bo = rb_entry(n, struct hmm_buffer_object, node);
 157
 158                 if (bo->start > start) {
 159                         if (n->rb_left == NULL)
 160                                 return NULL;
 161                         n = n->rb_left;
 162                 } else if (bo->start < start) {
 163                         if (n->rb_right == NULL)
 164                                 return NULL;
 165                         n = n->rb_right;
 166                 } else {
 167                         return bo;
 168                 }
 169         } while (n);
 170
 171         return NULL;
 172 }
 173
 174 static struct hmm_buffer_object *__bo_search_by_addr_in_range(
 175                 struct rb_root *root, unsigned int start)
 176 {
 177         struct rb_node *n = root->rb_node;
 178         struct hmm_buffer_object *bo;
 179
 180         do {
 181                 bo = rb_entry(n, struct hmm_buffer_object, node);
 182
 183                 if (bo->start > start) {
 184                         if (n->rb_left == NULL)
 185                                 return NULL;
 186                         n = n->rb_left;
 187                 } else {
 188                         if (bo->end > start)
 189                                 return bo;
 190                         if (n->rb_right == NULL)
 191                                 return NULL;
 192                         n = n->rb_right;
 193                 }
 194         } while (n);
 195
 196         return NULL;
 197 }
 198
 199 static void __bo_insert_to_free_rbtree(struct rb_root *root,
 200                                         struct hmm_buffer_object *bo)
 201 {
 202         struct rb_node **new = &(root->rb_node);
 203         struct rb_node *parent = NULL;
 204         struct hmm_buffer_object *this;
 205         unsigned int pgnr = bo->pgnr;
 206
 207         while (*new) {
 208                 parent = *new;
 209                 this = container_of(*new, struct hmm_buffer_object, node);
 210
 211                 if (pgnr < this->pgnr) {
 212                         new = &((*new)->rb_left);
 213                 } else if (pgnr > this->pgnr) {
 214                         new = &((*new)->rb_right);
 215                 } else {
 216                         bo->prev = this;
 217                         bo->next = this->next;
 218                         if (this->next)
 219                                 this->next->prev = bo;
 220                         this->next = bo;
 221                         bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_FREE;
 222                         return;
 223                 }
 224         }
 225
 226         bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_FREE;
 227
 228         rb_link_node(&bo->node, parent, new);
 229         rb_insert_color(&bo->node, root);
 230 }
 231
 232 static void __bo_insert_to_alloc_rbtree(struct rb_root *root,
 233                                         struct hmm_buffer_object *bo)
 234 {
 235         struct rb_node **new = &(root->rb_node);
 236         struct rb_node *parent = NULL;
 237         struct hmm_buffer_object *this;
 238         unsigned int start = bo->start;
 239
 240         while (*new) {
 241                 parent = *new;
 242                 this = container_of(*new, struct hmm_buffer_object, node);
 243
 244                 if (start < this->start)
 245                         new = &((*new)->rb_left);
 246                 else
 247                         new = &((*new)->rb_right);
 248         }
 249
 250         kref_init(&bo->kref);
 251         bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_ALLOCED;
 252
 253         rb_link_node(&bo->node, parent, new);
 254         rb_insert_color(&bo->node, root);
 255 }
 256
 257 static struct hmm_buffer_object *__bo_break_up(struct hmm_bo_device *bdev,
 258                                         struct hmm_buffer_object *bo,
 259                                         unsigned int pgnr)
 260 {
 261         struct hmm_buffer_object *new_bo;
 262         unsigned long flags;
 263         int ret;
 264
 265         new_bo = __bo_alloc(bdev->bo_cache);
 266         if (!new_bo) {
 267                 dev_err(atomisp_dev, "%s: __bo_alloc failed!\n", __func__);
 268                 return NULL;
 269         }
 270         ret = __bo_init(bdev, new_bo, pgnr);
 271         if (ret) {
 272                 dev_err(atomisp_dev, "%s: __bo_init failed!\n", __func__);
 273                 kmem_cache_free(bdev->bo_cache, new_bo);
 274                 return NULL;
 275         }
 276
 277         new_bo->start = bo->start;
 278         new_bo->end = new_bo->start + pgnr_to_size(pgnr);
 279         bo->start = new_bo->end;
 280         bo->pgnr = bo->pgnr - pgnr;
 281
 282         spin_lock_irqsave(&bdev->list_lock, flags);
 283         list_add_tail(&new_bo->list, &bo->list);
 284         spin_unlock_irqrestore(&bdev->list_lock, flags);
 285
 286         return new_bo;
 287 }
 288
 289 static void __bo_take_off_handling(struct hmm_buffer_object *bo)
 290 {
 291         struct hmm_bo_device *bdev = bo->bdev;
 292         /* There are 4 situations when we take off a known bo from free rbtree:
 293          * 1. if bo->next && bo->prev == NULL, bo is a rbtree node
 294          *      and does not have a linked list after bo, to take off this bo,
 295          *      we just need erase bo directly and rebalance the free rbtree
 296          */
 297         if (bo->prev == NULL && bo->next == NULL) {
 298                 rb_erase(&bo->node, &bdev->free_rbtree);
 299         /* 2. when bo->next != NULL && bo->prev == NULL, bo is a rbtree node,
 300          *      and has a linked list,to take off this bo we need erase bo
 301          *      first, then, insert bo->next into free rbtree and rebalance
 302          *      the free rbtree
 303          */
 304         } else if (bo->prev == NULL && bo->next != NULL) {
 305                 bo->next->prev = NULL;
 306                 rb_erase(&bo->node, &bdev->free_rbtree);
 307                 __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo->next);
 308                 bo->next = NULL;
 309         /* 3. when bo->prev != NULL && bo->next == NULL, bo is not a rbtree
 310          *      node, bo is the last element of the linked list after rbtree
 311          *      node, to take off this bo, we just need set the "prev/next"
 312          *      pointers to NULL, the free rbtree stays unchaged
 313          */
 314         } else if (bo->prev != NULL && bo->next == NULL) {
 315                 bo->prev->next = NULL;
 316                 bo->prev = NULL;
 317         /* 4. when bo->prev != NULL && bo->next != NULL ,bo is not a rbtree
 318          *      node, bo is in the middle of the linked list after rbtree node,
 319          *      to take off this bo, we just set take the "prev/next" pointers
 320          *      to NULL, the free rbtree stays unchaged
 321          */
 322         } else {
 323                 bo->next->prev = bo->prev;
 324                 bo->prev->next = bo->next;
 325                 bo->next = NULL;
 326                 bo->prev = NULL;
 327         }
 328 }
 329
 330 static struct hmm_buffer_object *__bo_merge(struct hmm_buffer_object *bo,
 331                                         struct hmm_buffer_object *next_bo)
 332 {
 333         struct hmm_bo_device *bdev;
 334         unsigned long flags;
 335
 336         bdev = bo->bdev;
 337         next_bo->start = bo->start;
 338         next_bo->pgnr = next_bo->pgnr + bo->pgnr;
 339
 340         spin_lock_irqsave(&bdev->list_lock, flags);
 341         list_del(&bo->list);
 342         spin_unlock_irqrestore(&bdev->list_lock, flags);
 343
 344         kmem_cache_free(bo->bdev->bo_cache, bo);
 345
 346         return next_bo;
 347 }
 348
 349 /*
 350  * hmm_bo_device functions.
 351  */
 352 int hmm_bo_device_init(struct hmm_bo_device *bdev,
 353                                 struct isp_mmu_client *mmu_driver,
 354                                 unsigned int vaddr_start,
 355                                 unsigned int size)
 356 {
 357         struct hmm_buffer_object *bo;
 358         unsigned long flags;
 359         int ret;
 360
 361         check_bodev_null_return(bdev, -EINVAL);
 362
 363         ret = isp_mmu_init(&bdev->mmu, mmu_driver);
 364         if (ret) {
 365                 dev_err(atomisp_dev, "isp_mmu_init failed.\n");
 366                 return ret;
 367         }
 368
 369         bdev->start = vaddr_start;
 370         bdev->pgnr = size_to_pgnr_ceil(size);
 371         bdev->size = pgnr_to_size(bdev->pgnr);
 372
 373         spin_lock_init(&bdev->list_lock);
 374         mutex_init(&bdev->rbtree_mutex);
 375
 376         bdev->flag = HMM_BO_DEVICE_INITED;
 377
 378         INIT_LIST_HEAD(&bdev->entire_bo_list);
 379         bdev->allocated_rbtree = RB_ROOT;
 380         bdev->free_rbtree = RB_ROOT;
 381
 382         bdev->bo_cache = kmem_cache_create("bo_cache",
 383                                 sizeof(struct hmm_buffer_object), 0, 0, NULL);
 384         if (!bdev->bo_cache) {
 385                 dev_err(atomisp_dev, "%s: create cache failed!\n", __func__);
 386                 isp_mmu_exit(&bdev->mmu);
 387                 return -ENOMEM;
 388         }
 389
 390         bo = __bo_alloc(bdev->bo_cache);
 391         if (!bo) {
 392                 dev_err(atomisp_dev, "%s: __bo_alloc failed!\n", __func__);
 393                 isp_mmu_exit(&bdev->mmu);
 394                 return -ENOMEM;
 395         }
 396
 397         ret = __bo_init(bdev, bo, bdev->pgnr);
 398         if (ret) {
 399                 dev_err(atomisp_dev, "%s: __bo_init failed!\n", __func__);
 400                 kmem_cache_free(bdev->bo_cache, bo);
 401                 isp_mmu_exit(&bdev->mmu);
 402                 return -EINVAL;
 403         }
 404
 405         spin_lock_irqsave(&bdev->list_lock, flags);
 406         list_add_tail(&bo->list, &bdev->entire_bo_list);
 407         spin_unlock_irqrestore(&bdev->list_lock, flags);
 408
 409         __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo);
 410
 411         return 0;
 412 }
 413
 414 struct hmm_buffer_object *hmm_bo_alloc(struct hmm_bo_device *bdev,
 415                                         unsigned int pgnr)
 416 {
 417         struct hmm_buffer_object *bo, *new_bo;
 418         struct rb_root *root = &bdev->free_rbtree;
 419
 420         check_bodev_null_return(bdev, NULL);
 421         var_equal_return(hmm_bo_device_inited(bdev), 0, NULL,
 422                         "hmm_bo_device not inited yet.\n");
 423
 424         if (pgnr == 0) {
 425                 dev_err(atomisp_dev, "0 size buffer is not allowed.\n");
 426                 return NULL;
 427         }
 428
 429         mutex_lock(&bdev->rbtree_mutex);
 430         bo = __bo_search_and_remove_from_free_rbtree(root->rb_node, pgnr);
 431         if (!bo) {
 432                 mutex_unlock(&bdev->rbtree_mutex);
 433                 dev_err(atomisp_dev, "%s: Out of Memory! hmm_bo_alloc failed",
 434                         __func__);
 435                 return NULL;
 436         }
 437
 438         if (bo->pgnr > pgnr) {
 439                 new_bo = __bo_break_up(bdev, bo, pgnr);
 440                 if (!new_bo) {
 441                         mutex_unlock(&bdev->rbtree_mutex);
 442                         dev_err(atomisp_dev, "%s: __bo_break_up failed!\n",
 443                                 __func__);
 444                         return NULL;
 445                 }
 446
 447                 __bo_insert_to_alloc_rbtree(&bdev->allocated_rbtree, new_bo);
 448                 __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo);
 449
 450                 mutex_unlock(&bdev->rbtree_mutex);
 451                 return new_bo;
 452         }
 453
 454         __bo_insert_to_alloc_rbtree(&bdev->allocated_rbtree, bo);
 455
 456         mutex_unlock(&bdev->rbtree_mutex);
 457         return bo;
 458 }
 459
 460 void hmm_bo_release(struct hmm_buffer_object *bo)
 461 {
 462         struct hmm_bo_device *bdev = bo->bdev;
 463         struct hmm_buffer_object *next_bo, *prev_bo;
 464
 465         mutex_lock(&bdev->rbtree_mutex);
 466
 467         /*
 468          * FIX ME:
 469          *
 470          * how to destroy the bo when it is stilled MMAPED?
 471          *
 472          * ideally, this will not happened as hmm_bo_release
 473          * will only be called when kref reaches 0, and in mmap
 474          * operation the hmm_bo_ref will eventually be called.
 475          * so, if this happened, something goes wrong.
 476          */
 477         if (bo->status & HMM_BO_MMAPED) {
 478                 mutex_unlock(&bdev->rbtree_mutex);
 479                 dev_dbg(atomisp_dev, "destroy bo which is MMAPED, do nothing\n");
 480                 return;
 481         }
 482
 483         if (bo->status & HMM_BO_BINDED) {
 484                 dev_warn(atomisp_dev, "the bo is still binded, unbind it first...\n");
 485                 hmm_bo_unbind(bo);
 486         }
 487
 488         if (bo->status & HMM_BO_PAGE_ALLOCED) {
 489                 dev_warn(atomisp_dev, "the pages is not freed, free pages first\n");
 490                 hmm_bo_free_pages(bo);
 491         }
 492         if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
 493                 dev_warn(atomisp_dev, "the vunmap is not done, do it...\n");
 494                 hmm_bo_vunmap(bo);
 495         }
 496
 497         rb_erase(&bo->node, &bdev->allocated_rbtree);
 498
 499         prev_bo = list_entry(bo->list.prev, struct hmm_buffer_object, list);
 500         next_bo = list_entry(bo->list.next, struct hmm_buffer_object, list);
 501
 502         if (bo->list.prev != &bdev->entire_bo_list &&
 503                 prev_bo->end == bo->start &&
 504                 (prev_bo->status & HMM_BO_MASK) == HMM_BO_FREE) {
 505                 __bo_take_off_handling(prev_bo);
 506                 bo = __bo_merge(prev_bo, bo);
 507         }
 508
 509         if (bo->list.next != &bdev->entire_bo_list &&
 510                 next_bo->start == bo->end &&
 511                 (next_bo->status & HMM_BO_MASK) == HMM_BO_FREE) {
 512                 __bo_take_off_handling(next_bo);
 513                 bo = __bo_merge(bo, next_bo);
 514         }
 515
 516         __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo);
 517
 518         mutex_unlock(&bdev->rbtree_mutex);
 519         return;
 520 }
 521
 522 void hmm_bo_device_exit(struct hmm_bo_device *bdev)
 523 {
 524         struct hmm_buffer_object *bo;
 525         unsigned long flags;
 526
 527         dev_dbg(atomisp_dev, "%s: entering!\n", __func__);
 528
 529         check_bodev_null_return_void(bdev);
 530
 531         /*
 532          * release all allocated bos even they a in use
 533          * and all bos will be merged into a big bo
 534          */
 535         while (!RB_EMPTY_ROOT(&bdev->allocated_rbtree))
 536                 hmm_bo_release(
 537                         rbtree_node_to_hmm_bo(bdev->allocated_rbtree.rb_node));
 538
 539         dev_dbg(atomisp_dev, "%s: finished releasing all allocated bos!\n",
 540                 __func__);
 541
 542         /* free all bos to release all ISP virtual memory */
 543         while (!list_empty(&bdev->entire_bo_list)) {
 544                 bo = list_to_hmm_bo(bdev->entire_bo_list.next);
 545
 546                 spin_lock_irqsave(&bdev->list_lock, flags);
 547                 list_del(&bo->list);
 548                 spin_unlock_irqrestore(&bdev->list_lock, flags);
 549
 550                 kmem_cache_free(bdev->bo_cache, bo);
 551         }
 552
 553         dev_dbg(atomisp_dev, "%s: finished to free all bos!\n", __func__);
 554
 555         kmem_cache_destroy(bdev->bo_cache);
 556
 557         isp_mmu_exit(&bdev->mmu);
 558 }
 559
 560 int hmm_bo_device_inited(struct hmm_bo_device *bdev)
 561 {
 562         check_bodev_null_return(bdev, -EINVAL);
 563
 564         return bdev->flag == HMM_BO_DEVICE_INITED;
 565 }
 566
 567 int hmm_bo_allocated(struct hmm_buffer_object *bo)
 568 {
 569         check_bo_null_return(bo, 0);
 570
 571         return bo->status & HMM_BO_ALLOCED;
 572 }
 573
 574 struct hmm_buffer_object *hmm_bo_device_search_start(
 575         struct hmm_bo_device *bdev, ia_css_ptr vaddr)
 576 {
 577         struct hmm_buffer_object *bo;
 578
 579         check_bodev_null_return(bdev, NULL);
 580
 581         mutex_lock(&bdev->rbtree_mutex);
 582         bo = __bo_search_by_addr(&bdev->allocated_rbtree, vaddr);
 583         if (!bo) {
 584                 mutex_unlock(&bdev->rbtree_mutex);
 585                 dev_err(atomisp_dev, "%s can not find bo with addr: 0x%x\n",
 586                         __func__, vaddr);
 587                 return NULL;
 588         }
 589         mutex_unlock(&bdev->rbtree_mutex);
 590
 591         return bo;
 592 }
 593
 594 struct hmm_buffer_object *hmm_bo_device_search_in_range(
 595         struct hmm_bo_device *bdev, unsigned int vaddr)
 596 {
 597         struct hmm_buffer_object *bo;
 598
 599         check_bodev_null_return(bdev, NULL);
 600
 601         mutex_lock(&bdev->rbtree_mutex);
 602         bo = __bo_search_by_addr_in_range(&bdev->allocated_rbtree, vaddr);
 603         if (!bo) {
 604                 mutex_unlock(&bdev->rbtree_mutex);
 605                 dev_err(atomisp_dev, "%s can not find bo contain addr: 0x%x\n",
 606                         __func__, vaddr);
 607                 return NULL;
 608         }
 609         mutex_unlock(&bdev->rbtree_mutex);
 610
 611         return bo;
 612 }
 613
 614 struct hmm_buffer_object *hmm_bo_device_search_vmap_start(
 615         struct hmm_bo_device *bdev, const void *vaddr)
 616 {
 617         struct list_head *pos;
 618         struct hmm_buffer_object *bo;
 619         unsigned long flags;
 620
 621         check_bodev_null_return(bdev, NULL);
 622
 623         spin_lock_irqsave(&bdev->list_lock, flags);
 624         list_for_each(pos, &bdev->entire_bo_list) {
 625                 bo = list_to_hmm_bo(pos);
 626                 /* pass bo which has no vm_node allocated */
 627                 if ((bo->status & HMM_BO_MASK) == HMM_BO_FREE)
 628                         continue;
 629                 if (bo->vmap_addr == vaddr)
 630                         goto found;
 631         }
 632         spin_unlock_irqrestore(&bdev->list_lock, flags);
 633         return NULL;
 634 found:
 635         spin_unlock_irqrestore(&bdev->list_lock, flags);
 636         return bo;
 637
 638 }
 639
 640
 641 static void free_private_bo_pages(struct hmm_buffer_object *bo,
 642                                 struct hmm_pool *dypool,
 643                                 struct hmm_pool *repool,
 644                                 int free_pgnr)
 645 {
 646         int i, ret;
 647
 648         for (i = 0; i < free_pgnr; i++) {
 649                 switch (bo->page_obj[i].type) {
 650                 case HMM_PAGE_TYPE_RESERVED:
 651                         if (repool->pops
 652                             && repool->pops->pool_free_pages) {
 653                                 repool->pops->pool_free_pages(repool->pool_info,
 654                                                         &bo->page_obj[i]);
 655                                 hmm_mem_stat.res_cnt--;
 656                         }
 657                         break;
 658                 /*
 659                  * HMM_PAGE_TYPE_GENERAL indicates that pages are from system
 660                  * memory, so when free them, they should be put into dynamic
 661                  * pool.
 662                  */
 663                 case HMM_PAGE_TYPE_DYNAMIC:
 664                 case HMM_PAGE_TYPE_GENERAL:
 665                         if (dypool->pops
 666                             && dypool->pops->pool_inited
 667                             && dypool->pops->pool_inited(dypool->pool_info)) {
 668                                 if (dypool->pops->pool_free_pages)
 669                                         dypool->pops->pool_free_pages(
 670                                                               dypool->pool_info,
 671                                                               &bo->page_obj[i]);
 672                                 break;
 673                         }
 674
 675                         /*
 676                          * if dynamic memory pool doesn't exist, need to free
 677                          * pages to system directly.
 678                          */
 679                 default:
 680                         ret = set_pages_wb(bo->page_obj[i].page, 1);
 681                         if (ret)
 682                                 dev_err(atomisp_dev,
 683                                                 "set page to WB err ...ret = %d\n",
 684                                                         ret);
 685                         /*
 686                         W/A: set_pages_wb seldom return value = -EFAULT
 687                         indicate that address of page is not in valid
 688                         range(0xffff880000000000~0xffffc7ffffffffff)
 689                         then, _free_pages would panic; Do not know why page
 690                         address be valid,it maybe memory corruption by lowmemory
 691                         */
 692                         if (!ret) {
 693                                 __free_pages(bo->page_obj[i].page, 0);
 694                                 hmm_mem_stat.sys_size--;
 695                         }
 696                         break;
 697                 }
 698         }
 699
 700         return;
 701 }
 702
 703 /*Allocate pages which will be used only by ISP*/
 704 static int alloc_private_pages(struct hmm_buffer_object *bo,
 705                                 int from_highmem,
 706                                 bool cached,
 707                                 struct hmm_pool *dypool,
 708                                 struct hmm_pool *repool)
 709 {
 710         int ret;
 711         unsigned int pgnr, order, blk_pgnr, alloc_pgnr;
 712         struct page *pages;
 713         gfp_t gfp = GFP_NOWAIT | __GFP_NOWARN; /* REVISIT: need __GFP_FS too? */
 714         int i, j;
 715         int failure_number = 0;
 716         bool reduce_order = false;
 717         bool lack_mem = true;
 718
 719         if (from_highmem)
 720                 gfp |= __GFP_HIGHMEM;
 721
 722         pgnr = bo->pgnr;
 723
 724         bo->page_obj = kmalloc_array(pgnr, sizeof(struct hmm_page_object),
 725                                 GFP_KERNEL);
 726         if (unlikely(!bo->page_obj))
 727                 return -ENOMEM;
 728
 729         i = 0;
 730         alloc_pgnr = 0;
 731
 732         /*
 733          * get physical pages from dynamic pages pool.
 734          */
 735         if (dypool->pops && dypool->pops->pool_alloc_pages) {
 736                 alloc_pgnr = dypool->pops->pool_alloc_pages(dypool->pool_info,
 737                                                         bo->page_obj, pgnr,
 738                                                         cached);
 739                 hmm_mem_stat.dyc_size -= alloc_pgnr;
 740
 741                 if (alloc_pgnr == pgnr)
 742                         return 0;
 743         }
 744
 745         pgnr -= alloc_pgnr;
 746         i += alloc_pgnr;
 747
 748         /*
 749          * get physical pages from reserved pages pool for atomisp.
 750          */
 751         if (repool->pops && repool->pops->pool_alloc_pages) {
 752                 alloc_pgnr = repool->pops->pool_alloc_pages(repool->pool_info,
 753                                                         &bo->page_obj[i], pgnr,
 754                                                         cached);
 755                 hmm_mem_stat.res_cnt += alloc_pgnr;
 756                 if (alloc_pgnr == pgnr)
 757                         return 0;
 758         }
 759
 760         pgnr -= alloc_pgnr;
 761         i += alloc_pgnr;
 762
 763         while (pgnr) {
 764                 order = nr_to_order_bottom(pgnr);
 765                 /*
 766                  * if be short of memory, we will set order to 0
 767                  * everytime.
 768                  */
 769                 if (lack_mem)
 770                         order = HMM_MIN_ORDER;
 771                 else if (order > HMM_MAX_ORDER)
 772                         order = HMM_MAX_ORDER;
 773 retry:
 774                 /*
 775                  * When order > HMM_MIN_ORDER, for performance reasons we don't
 776                  * want alloc_pages() to sleep. In case it fails and fallbacks
 777                  * to HMM_MIN_ORDER or in case the requested order is originally
 778                  * the minimum value, we can allow alloc_pages() to sleep for
 779                  * robustness purpose.
 780                  *
 781                  * REVISIT: why __GFP_FS is necessary?
 782                  */
 783                 if (order == HMM_MIN_ORDER) {
 784                         gfp &= ~GFP_NOWAIT;
 785                         gfp |= __GFP_RECLAIM | __GFP_FS;
 786                 }
 787
 788                 pages = alloc_pages(gfp, order);
 789                 if (unlikely(!pages)) {
 790                         /*
 791                          * in low memory case, if allocation page fails,
 792                          * we turn to try if order=0 allocation could
 793                          * succeed. if order=0 fails too, that means there is
 794                          * no memory left.
 795                          */
 796                         if (order == HMM_MIN_ORDER) {
 797                                 dev_err(atomisp_dev,
 798                                         "%s: cannot allocate pages\n",
 799                                          __func__);
 800                                 goto cleanup;
 801                         }
 802                         order = HMM_MIN_ORDER;
 803                         failure_number++;
 804                         reduce_order = true;
 805                         /*
 806                          * if fail two times continuously, we think be short
 807                          * of memory now.
 808                          */
 809                         if (failure_number == 2) {
 810                                 lack_mem = true;
 811                                 failure_number = 0;
 812                         }
 813                         goto retry;
 814                 } else {
 815                         blk_pgnr = order_to_nr(order);
 816
 817                         if (!cached) {
 818                                 /*
 819                                  * set memory to uncacheable -- UC_MINUS
 820                                  */
 821                                 ret = set_pages_uc(pages, blk_pgnr);
 822                                 if (ret) {
 823                                         dev_err(atomisp_dev,
 824                                                      "set page uncacheable"
 825                                                         "failed.\n");
 826
 827                                         __free_pages(pages, order);
 828
 829                                         goto cleanup;
 830                                 }
 831                         }
 832
 833                         for (j = 0; j < blk_pgnr; j++) {
 834                                 bo->page_obj[i].page = pages + j;
 835                                 bo->page_obj[i++].type = HMM_PAGE_TYPE_GENERAL;
 836                         }
 837
 838                         pgnr -= blk_pgnr;
 839                         hmm_mem_stat.sys_size += blk_pgnr;
 840
 841                         /*
 842                          * if order is not reduced this time, clear
 843                          * failure_number.
 844                          */
 845                         if (reduce_order)
 846                                 reduce_order = false;
 847                         else
 848                                 failure_number = 0;
 849                 }
 850         }
 851
 852         return 0;
 853 cleanup:
 854         alloc_pgnr = i;
 855         free_private_bo_pages(bo, dypool, repool, alloc_pgnr);
 856
 857         kfree(bo->page_obj);
 858
 859         return -ENOMEM;
 860 }
 861
 862 static void free_private_pages(struct hmm_buffer_object *bo,
 863                                 struct hmm_pool *dypool,
 864                                 struct hmm_pool *repool)
 865 {
 866         free_private_bo_pages(bo, dypool, repool, bo->pgnr);
 867
 868         kfree(bo->page_obj);
 869 }
 870
 871 /*
 872  * Hacked from kernel function __get_user_pages in mm/memory.c
 873  *
 874  * Handle buffers allocated by other kernel space driver and mmaped into user
 875  * space, function Ignore the VM_PFNMAP and VM_IO flag in VMA structure
 876  *
 877  * Get physical pages from user space virtual address and update into page list
 878  */
 879 static int __get_pfnmap_pages(struct task_struct *tsk, struct mm_struct *mm,
 880                               unsigned long start, int nr_pages,
 881                               unsigned int gup_flags, struct page **pages,
 882                               struct vm_area_struct **vmas)
 883 {
 884         int i, ret;
 885         unsigned long vm_flags;
 886
 887         if (nr_pages <= 0)
 888                 return 0;
 889
 890         VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
 891
 892         /*
 893          * Require read or write permissions.
 894          * If FOLL_FORCE is set, we only require the "MAY" flags.
 895          */
 896         vm_flags  = (gup_flags & FOLL_WRITE) ?
 897                         (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
 898         vm_flags &= (gup_flags & FOLL_FORCE) ?
 899                         (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
 900         i = 0;
 901
 902         do {
 903                 struct vm_area_struct *vma;
 904
 905                 vma = find_vma(mm, start);
 906                 if (!vma) {
 907                         dev_err(atomisp_dev, "find_vma failed\n");
 908                         return i ? : -EFAULT;
 909                 }
 910
 911                 if (is_vm_hugetlb_page(vma)) {
 912                         /*
 913                         i = follow_hugetlb_page(mm, vma, pages, vmas,
 914                                         &start, &nr_pages, i, gup_flags);
 915                         */
 916                         continue;
 917                 }
 918
 919                 do {
 920                         struct page *page;
 921                         unsigned long pfn;
 922
 923                         /*
 924                          * If we have a pending SIGKILL, don't keep faulting
 925                          * pages and potentially allocating memory.
 926                          */
 927                         if (unlikely(fatal_signal_pending(current))) {
 928                                 dev_err(atomisp_dev,
 929                                         "fatal_signal_pending in %s\n",
 930                                         __func__);
 931                                 return i ? i : -ERESTARTSYS;
 932                         }
 933
 934                         ret = follow_pfn(vma, start, &pfn);
 935                         if (ret) {
 936                                 dev_err(atomisp_dev, "follow_pfn() failed\n");
 937                                 return i ? : -EFAULT;
 938                         }
 939
 940                         page = pfn_to_page(pfn);
 941                         if (IS_ERR(page))
 942                                 return i ? i : PTR_ERR(page);
 943                         if (pages) {
 944                                 pages[i] = page;
 945                                 get_page(page);
 946                                 flush_anon_page(vma, page, start);
 947                                 flush_dcache_page(page);
 948                         }
 949                         if (vmas)
 950                                 vmas[i] = vma;
 951                         i++;
 952                         start += PAGE_SIZE;
 953                         nr_pages--;
 954                 } while (nr_pages && start < vma->vm_end);
 955         } while (nr_pages);
 956
 957         return i;
 958 }
 959
 960 static int get_pfnmap_pages(struct task_struct *tsk, struct mm_struct *mm,
 961                      unsigned long start, int nr_pages, int write, int force,
 962                      struct page **pages, struct vm_area_struct **vmas)
 963 {
 964         int flags = FOLL_TOUCH;
 965
 966         if (pages)
 967                 flags |= FOLL_GET;
 968         if (write)
 969                 flags |= FOLL_WRITE;
 970         if (force)
 971                 flags |= FOLL_FORCE;
 972
 973         return __get_pfnmap_pages(tsk, mm, start, nr_pages, flags, pages, vmas);
 974 }
 975
 976 /*
 977  * Convert user space virtual address into pages list
 978  */
 979 static int alloc_user_pages(struct hmm_buffer_object *bo,
 980                               void *userptr, bool cached)
 981 {
 982         int page_nr;
 983         int i;
 984         struct vm_area_struct *vma;
 985         struct page **pages;
 986
 987         pages = kmalloc_array(bo->pgnr, sizeof(struct page *), GFP_KERNEL);
 988         if (unlikely(!pages))
 989                 return -ENOMEM;
 990
 991         bo->page_obj = kmalloc_array(bo->pgnr, sizeof(struct hmm_page_object),
 992                 GFP_KERNEL);
 993         if (unlikely(!bo->page_obj)) {
 994                 kfree(pages);
 995                 return -ENOMEM;
 996         }
 997
 998         mutex_unlock(&bo->mutex);
 999         down_read(&current->mm->mmap_sem);
1000         vma = find_vma(current->mm, (unsigned long)userptr);
1001         up_read(&current->mm->mmap_sem);
1002         if (vma == NULL) {
1003                 dev_err(atomisp_dev, "find_vma failed\n");
1004                 kfree(bo->page_obj);
1005                 kfree(pages);
1006                 mutex_lock(&bo->mutex);
1007                 return -EFAULT;
1008         }
1009         mutex_lock(&bo->mutex);
1010         /*
1011          * Handle frame buffer allocated in other kerenl space driver
1012          * and map to user space
1013          */
1014         if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
1015                 page_nr = get_pfnmap_pages(current, current->mm,
1016                                            (unsigned long)userptr,
1017                                            (int)(bo->pgnr), 1, 0,
1018                                            pages, NULL);
1019                 bo->mem_type = HMM_BO_MEM_TYPE_PFN;
1020         } else {
1021                 /*Handle frame buffer allocated in user space*/
1022                 mutex_unlock(&bo->mutex);
1023                 down_read(&current->mm->mmap_sem);
1024                 page_nr = get_user_pages((unsigned long)userptr,
1025                                          (int)(bo->pgnr), 1, pages, NULL);
1026                 up_read(&current->mm->mmap_sem);
1027                 mutex_lock(&bo->mutex);
1028                 bo->mem_type = HMM_BO_MEM_TYPE_USER;
1029         }
1030
1031         /* can be written by caller, not forced */
1032         if (page_nr != bo->pgnr) {
1033                 dev_err(atomisp_dev,
1034                                 "get_user_pages err: bo->pgnr = %d, "
1035                                 "pgnr actually pinned = %d.\n",
1036                                 bo->pgnr, page_nr);
1037                 goto out_of_mem;
1038         }
1039
1040         for (i = 0; i < bo->pgnr; i++) {
1041                 bo->page_obj[i].page = pages[i];
1042                 bo->page_obj[i].type = HMM_PAGE_TYPE_GENERAL;
1043         }
1044         hmm_mem_stat.usr_size += bo->pgnr;
1045         kfree(pages);
1046
1047         return 0;
1048
1049 out_of_mem:
1050         for (i = 0; i < page_nr; i++)
1051                 put_page(pages[i]);
1052         kfree(pages);
1053         kfree(bo->page_obj);
1054
1055         return -ENOMEM;
1056 }
1057
1058 static void free_user_pages(struct hmm_buffer_object *bo)
1059 {
1060         int i;
1061
1062         for (i = 0; i < bo->pgnr; i++)
1063                 put_page(bo->page_obj[i].page);
1064         hmm_mem_stat.usr_size -= bo->pgnr;
1065
1066         kfree(bo->page_obj);
1067 }
1068
1069 /*
1070  * allocate/free physical pages for the bo.
1071  *
1072  * type indicate where are the pages from. currently we have 3 types
1073  * of memory: HMM_BO_PRIVATE, HMM_BO_USER, HMM_BO_SHARE.
1074  *
1075  * from_highmem is only valid when type is HMM_BO_PRIVATE, it will
1076  * try to alloc memory from highmem if from_highmem is set.
1077  *
1078  * userptr is only valid when type is HMM_BO_USER, it indicates
1079  * the start address from user space task.
1080  *
1081  * from_highmem and userptr will both be ignored when type is
1082  * HMM_BO_SHARE.
1083  */
1084 int hmm_bo_alloc_pages(struct hmm_buffer_object *bo,
1085                        enum hmm_bo_type type, int from_highmem,
1086                        void *userptr, bool cached)
1087 {
1088         int ret = -EINVAL;
1089
1090         check_bo_null_return(bo, -EINVAL);
1091
1092         mutex_lock(&bo->mutex);
1093         check_bo_status_no_goto(bo, HMM_BO_PAGE_ALLOCED, status_err);
1094
1095         /*
1096          * TO DO:
1097          * add HMM_BO_USER type
1098          */
1099         if (type == HMM_BO_PRIVATE) {
1100                 ret = alloc_private_pages(bo, from_highmem,
1101                                 cached, &dynamic_pool, &reserved_pool);
1102         } else if (type == HMM_BO_USER) {
1103                 ret = alloc_user_pages(bo, userptr, cached);
1104         } else {
1105                 dev_err(atomisp_dev, "invalid buffer type.\n");
1106                 ret = -EINVAL;
1107         }
1108         if (ret)
1109                 goto alloc_err;
1110
1111         bo->type = type;
1112
1113         bo->status |= HMM_BO_PAGE_ALLOCED;
1114
1115         mutex_unlock(&bo->mutex);
1116
1117         return 0;
1118
1119 alloc_err:
1120         mutex_unlock(&bo->mutex);
1121         dev_err(atomisp_dev, "alloc pages err...\n");
1122         return ret;
1123 status_err:
1124         mutex_unlock(&bo->mutex);
1125         dev_err(atomisp_dev,
1126                         "buffer object has already page allocated.\n");
1127         return -EINVAL;
1128 }
1129
1130 /*
1131  * free physical pages of the bo.
1132  */
1133 void hmm_bo_free_pages(struct hmm_buffer_object *bo)
1134 {
1135         check_bo_null_return_void(bo);
1136
1137         mutex_lock(&bo->mutex);
1138
1139         check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err2);
1140
1141         /* clear the flag anyway. */
1142         bo->status &= (~HMM_BO_PAGE_ALLOCED);
1143
1144         if (bo->type == HMM_BO_PRIVATE)
1145                 free_private_pages(bo, &dynamic_pool, &reserved_pool);
1146         else if (bo->type == HMM_BO_USER)
1147                 free_user_pages(bo);
1148         else
1149                 dev_err(atomisp_dev, "invalid buffer type.\n");
1150         mutex_unlock(&bo->mutex);
1151
1152         return;
1153
1154 status_err2:
1155         mutex_unlock(&bo->mutex);
1156         dev_err(atomisp_dev,
1157                         "buffer object not page allocated yet.\n");
1158 }
1159
1160 int hmm_bo_page_allocated(struct hmm_buffer_object *bo)
1161 {
1162         check_bo_null_return(bo, 0);
1163
1164         return bo->status & HMM_BO_PAGE_ALLOCED;
1165 }
1166
1167 /*
1168  * get physical page info of the bo.
1169  */
1170 int hmm_bo_get_page_info(struct hmm_buffer_object *bo,
1171                          struct hmm_page_object **page_obj, int *pgnr)
1172 {
1173         check_bo_null_return(bo, -EINVAL);
1174
1175         mutex_lock(&bo->mutex);
1176
1177         check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err);
1178
1179         *page_obj = bo->page_obj;
1180         *pgnr = bo->pgnr;
1181
1182         mutex_unlock(&bo->mutex);
1183
1184         return 0;
1185
1186 status_err:
1187         dev_err(atomisp_dev,
1188                         "buffer object not page allocated yet.\n");
1189         mutex_unlock(&bo->mutex);
1190         return -EINVAL;
1191 }
1192
1193 /*
1194  * bind the physical pages to a virtual address space.
1195  */
1196 int hmm_bo_bind(struct hmm_buffer_object *bo)
1197 {
1198         int ret;
1199         unsigned int virt;
1200         struct hmm_bo_device *bdev;
1201         unsigned int i;
1202
1203         check_bo_null_return(bo, -EINVAL);
1204
1205         mutex_lock(&bo->mutex);
1206
1207         check_bo_status_yes_goto(bo,
1208                                    HMM_BO_PAGE_ALLOCED | HMM_BO_ALLOCED,
1209                                    status_err1);
1210
1211         check_bo_status_no_goto(bo, HMM_BO_BINDED, status_err2);
1212
1213         bdev = bo->bdev;
1214
1215         virt = bo->start;
1216
1217         for (i = 0; i < bo->pgnr; i++) {
1218                 ret =
1219                     isp_mmu_map(&bdev->mmu, virt,
1220                                 page_to_phys(bo->page_obj[i].page), 1);
1221                 if (ret)
1222                         goto map_err;
1223                 virt += (1 << PAGE_SHIFT);
1224         }
1225
1226         /*
1227          * flush TBL here.
1228          *
1229          * theoretically, we donot need to flush TLB as we didnot change
1230          * any existed address mappings, but for Silicon Hive's MMU, its
1231          * really a bug here. I guess when fetching PTEs (page table entity)
1232          * to TLB, its MMU will fetch additional INVALID PTEs automatically
1233          * for performance issue. EX, we only set up 1 page address mapping,
1234          * meaning updating 1 PTE, but the MMU fetches 4 PTE at one time,
1235          * so the additional 3 PTEs are invalid.
1236          */
1237         if (bo->start != 0x0)
1238                 isp_mmu_flush_tlb_range(&bdev->mmu, bo->start,
1239                                                 (bo->pgnr << PAGE_SHIFT));
1240
1241         bo->status |= HMM_BO_BINDED;
1242
1243         mutex_unlock(&bo->mutex);
1244
1245         return 0;
1246
1247 map_err:
1248         /* unbind the physical pages with related virtual address space */
1249         virt = bo->start;
1250         for ( ; i > 0; i--) {
1251                 isp_mmu_unmap(&bdev->mmu, virt, 1);
1252                 virt += pgnr_to_size(1);
1253         }
1254
1255         mutex_unlock(&bo->mutex);
1256         dev_err(atomisp_dev,
1257                         "setup MMU address mapping failed.\n");
1258         return ret;
1259
1260 status_err2:
1261         mutex_unlock(&bo->mutex);
1262         dev_err(atomisp_dev, "buffer object already binded.\n");
1263         return -EINVAL;
1264 status_err1:
1265         mutex_unlock(&bo->mutex);
1266         dev_err(atomisp_dev,
1267                      "buffer object vm_node or page not allocated.\n");
1268         return -EINVAL;
1269 }
1270
1271 /*
1272  * unbind the physical pages with related virtual address space.
1273  */
1274 void hmm_bo_unbind(struct hmm_buffer_object *bo)
1275 {
1276         unsigned int virt;
1277         struct hmm_bo_device *bdev;
1278         unsigned int i;
1279
1280         check_bo_null_return_void(bo);
1281
1282         mutex_lock(&bo->mutex);
1283
1284         check_bo_status_yes_goto(bo,
1285                                    HMM_BO_PAGE_ALLOCED |
1286                                    HMM_BO_ALLOCED |
1287                                    HMM_BO_BINDED, status_err);
1288
1289         bdev = bo->bdev;
1290
1291         virt = bo->start;
1292
1293         for (i = 0; i < bo->pgnr; i++) {
1294                 isp_mmu_unmap(&bdev->mmu, virt, 1);
1295                 virt += pgnr_to_size(1);
1296         }
1297
1298         /*
1299          * flush TLB as the address mapping has been removed and
1300          * related TLBs should be invalidated.
1301          */
1302         isp_mmu_flush_tlb_range(&bdev->mmu, bo->start,
1303                                 (bo->pgnr << PAGE_SHIFT));
1304
1305         bo->status &= (~HMM_BO_BINDED);
1306
1307         mutex_unlock(&bo->mutex);
1308
1309         return;
1310
1311 status_err:
1312         mutex_unlock(&bo->mutex);
1313         dev_err(atomisp_dev,
1314                      "buffer vm or page not allocated or not binded yet.\n");
1315 }
1316
1317 int hmm_bo_binded(struct hmm_buffer_object *bo)
1318 {
1319         int ret;
1320
1321         check_bo_null_return(bo, 0);
1322
1323         mutex_lock(&bo->mutex);
1324
1325         ret = bo->status & HMM_BO_BINDED;
1326
1327         mutex_unlock(&bo->mutex);
1328
1329         return ret;
1330 }
1331
1332 void *hmm_bo_vmap(struct hmm_buffer_object *bo, bool cached)
1333 {
1334         struct page **pages;
1335         int i;
1336
1337         check_bo_null_return(bo, NULL);
1338
1339         mutex_lock(&bo->mutex);
1340         if (((bo->status & HMM_BO_VMAPED) && !cached) ||
1341             ((bo->status & HMM_BO_VMAPED_CACHED) && cached)) {
1342                 mutex_unlock(&bo->mutex);
1343                 return bo->vmap_addr;
1344         }
1345
1346         /* cached status need to be changed, so vunmap first */
1347         if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
1348                 vunmap(bo->vmap_addr);
1349                 bo->vmap_addr = NULL;
1350                 bo->status &= ~(HMM_BO_VMAPED | HMM_BO_VMAPED_CACHED);
1351         }
1352
1353         pages = kmalloc_array(bo->pgnr, sizeof(*pages), GFP_KERNEL);
1354         if (unlikely(!pages)) {
1355                 mutex_unlock(&bo->mutex);
1356                 return NULL;
1357         }
1358
1359         for (i = 0; i < bo->pgnr; i++)
1360                 pages[i] = bo->page_obj[i].page;
1361
1362         bo->vmap_addr = vmap(pages, bo->pgnr, VM_MAP,
1363                 cached ? PAGE_KERNEL : PAGE_KERNEL_NOCACHE);
1364         if (unlikely(!bo->vmap_addr)) {
1365                 kfree(pages);
1366                 mutex_unlock(&bo->mutex);
1367                 dev_err(atomisp_dev, "vmap failed...\n");
1368                 return NULL;
1369         }
1370         bo->status |= (cached ? HMM_BO_VMAPED_CACHED : HMM_BO_VMAPED);
1371
1372         kfree(pages);
1373
1374         mutex_unlock(&bo->mutex);
1375         return bo->vmap_addr;
1376 }
1377
1378 void hmm_bo_flush_vmap(struct hmm_buffer_object *bo)
1379 {
1380         check_bo_null_return_void(bo);
1381
1382         mutex_lock(&bo->mutex);
1383         if (!(bo->status & HMM_BO_VMAPED_CACHED) || !bo->vmap_addr) {
1384                 mutex_unlock(&bo->mutex);
1385                 return;
1386         }
1387
1388         clflush_cache_range(bo->vmap_addr, bo->pgnr * PAGE_SIZE);
1389         mutex_unlock(&bo->mutex);
1390 }
1391
1392 void hmm_bo_vunmap(struct hmm_buffer_object *bo)
1393 {
1394         check_bo_null_return_void(bo);
1395
1396         mutex_lock(&bo->mutex);
1397         if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
1398                 vunmap(bo->vmap_addr);
1399                 bo->vmap_addr = NULL;
1400                 bo->status &= ~(HMM_BO_VMAPED | HMM_BO_VMAPED_CACHED);
1401         }
1402
1403         mutex_unlock(&bo->mutex);
1404         return;
1405 }
1406
1407 void hmm_bo_ref(struct hmm_buffer_object *bo)
1408 {
1409         check_bo_null_return_void(bo);
1410
1411         kref_get(&bo->kref);
1412 }
1413
1414 static void kref_hmm_bo_release(struct kref *kref)
1415 {
1416         if (!kref)
1417                 return;
1418
1419         hmm_bo_release(kref_to_hmm_bo(kref));
1420 }
1421
1422 void hmm_bo_unref(struct hmm_buffer_object *bo)
1423 {
1424         check_bo_null_return_void(bo);
1425
1426         kref_put(&bo->kref, kref_hmm_bo_release);
1427 }
1428
1429 static void hmm_bo_vm_open(struct vm_area_struct *vma)
1430 {
1431         struct hmm_buffer_object *bo =
1432             (struct hmm_buffer_object *)vma->vm_private_data;
1433
1434         check_bo_null_return_void(bo);
1435
1436         hmm_bo_ref(bo);
1437
1438         mutex_lock(&bo->mutex);
1439
1440         bo->status |= HMM_BO_MMAPED;
1441
1442         bo->mmap_count++;
1443
1444         mutex_unlock(&bo->mutex);
1445 }
1446
1447 static void hmm_bo_vm_close(struct vm_area_struct *vma)
1448 {
1449         struct hmm_buffer_object *bo =
1450             (struct hmm_buffer_object *)vma->vm_private_data;
1451
1452         check_bo_null_return_void(bo);
1453
1454         hmm_bo_unref(bo);
1455
1456         mutex_lock(&bo->mutex);
1457
1458         bo->mmap_count--;
1459
1460         if (!bo->mmap_count) {
1461                 bo->status &= (~HMM_BO_MMAPED);
1462                 vma->vm_private_data = NULL;
1463         }
1464
1465         mutex_unlock(&bo->mutex);
1466 }
1467
1468 static const struct vm_operations_struct hmm_bo_vm_ops = {
1469         .open = hmm_bo_vm_open,
1470         .close = hmm_bo_vm_close,
1471 };
1472
1473 /*
1474  * mmap the bo to user space.
1475  */
1476 int hmm_bo_mmap(struct vm_area_struct *vma, struct hmm_buffer_object *bo)
1477 {
1478         unsigned int start, end;
1479         unsigned int virt;
1480         unsigned int pgnr, i;
1481         unsigned int pfn;
1482
1483         check_bo_null_return(bo, -EINVAL);
1484
1485         check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err);
1486
1487         pgnr = bo->pgnr;
1488         start = vma->vm_start;
1489         end = vma->vm_end;
1490
1491         /*
1492          * check vma's virtual address space size and buffer object's size.
1493          * must be the same.
1494          */
1495         if ((start + pgnr_to_size(pgnr)) != end) {
1496                 dev_warn(atomisp_dev,
1497                              "vma's address space size not equal"
1498                              " to buffer object's size");
1499                 return -EINVAL;
1500         }
1501
1502         virt = vma->vm_start;
1503         for (i = 0; i < pgnr; i++) {
1504                 pfn = page_to_pfn(bo->page_obj[i].page);
1505                 if (remap_pfn_range(vma, virt, pfn, PAGE_SIZE, PAGE_SHARED)) {
1506                         dev_warn(atomisp_dev,
1507                                         "remap_pfn_range failed:"
1508                                         " virt = 0x%x, pfn = 0x%x,"
1509                                         " mapped_pgnr = %d\n", virt, pfn, 1);
1510                         return -EINVAL;
1511                 }
1512                 virt += PAGE_SIZE;
1513         }
1514
1515         vma->vm_private_data = bo;
1516
1517         vma->vm_ops = &hmm_bo_vm_ops;
1518         vma->vm_flags |= VM_IO|VM_DONTEXPAND|VM_DONTDUMP;
1519
1520         /*
1521          * call hmm_bo_vm_open explictly.
1522          */
1523         hmm_bo_vm_open(vma);
1524
1525         return 0;
1526
1527 status_err:
1528         dev_err(atomisp_dev, "buffer page not allocated yet.\n");
1529         return -EINVAL;
1530 }