drivers/infiniband/hw/hns/hns_roce_db.c

   1 /* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
   2 /*
   3  * Copyright (c) 2017 Hisilicon Limited.
   4  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   5  */
   6
   7 #include <rdma/ib_umem.h>
   8 #include "hns_roce_device.h"
   9
  10 int hns_roce_db_map_user(struct hns_roce_ucontext *context, unsigned long virt,
  11                          struct hns_roce_db *db)
  12 {
  13         unsigned long page_addr = virt & PAGE_MASK;
  14         struct hns_roce_user_db_page *page;
  15         unsigned int offset;
  16         int ret = 0;
  17
  18         mutex_lock(&context->page_mutex);
  19
  20         list_for_each_entry(page, &context->page_list, list)
  21                 if (page->user_virt == page_addr)
  22                         goto found;
  23
  24         page = kmalloc(sizeof(*page), GFP_KERNEL);
  25         if (!page) {
  26                 ret = -ENOMEM;
  27                 goto out;
  28         }
  29
  30         refcount_set(&page->refcount, 1);
  31         page->user_virt = page_addr;
  32         page->umem = ib_umem_get(context->ibucontext.device, page_addr,
  33                                  PAGE_SIZE, 0);
  34         if (IS_ERR(page->umem)) {
  35                 ret = PTR_ERR(page->umem);
  36                 kfree(page);
  37                 goto out;
  38         }
  39
  40         list_add(&page->list, &context->page_list);
  41
  42 found:
  43         offset = virt - page_addr;
  44         db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + offset;
  45         db->virt_addr = sg_virt(page->umem->sgt_append.sgt.sgl) + offset;
  46         db->u.user_page = page;
  47         refcount_inc(&page->refcount);
  48
  49 out:
  50         mutex_unlock(&context->page_mutex);
  51
  52         return ret;
  53 }
  54
  55 void hns_roce_db_unmap_user(struct hns_roce_ucontext *context,
  56                             struct hns_roce_db *db)
  57 {
  58         mutex_lock(&context->page_mutex);
  59
  60         refcount_dec(&db->u.user_page->refcount);
  61         if (refcount_dec_if_one(&db->u.user_page->refcount)) {
  62                 list_del(&db->u.user_page->list);
  63                 ib_umem_release(db->u.user_page->umem);
  64                 kfree(db->u.user_page);
  65         }
  66
  67         mutex_unlock(&context->page_mutex);
  68 }
  69
  70 static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir(
  71                                         struct device *dma_device)
  72 {
  73         struct hns_roce_db_pgdir *pgdir;
  74
  75         pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL);
  76         if (!pgdir)
  77                 return NULL;
  78
  79         bitmap_fill(pgdir->order1,
  80                     HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT);
  81         pgdir->bits[0] = pgdir->order0;
  82         pgdir->bits[1] = pgdir->order1;
  83         pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE,
  84                                          &pgdir->db_dma, GFP_KERNEL);
  85         if (!pgdir->page) {
  86                 kfree(pgdir);
  87                 return NULL;
  88         }
  89
  90         return pgdir;
  91 }
  92
  93 static int hns_roce_alloc_db_from_pgdir(struct hns_roce_db_pgdir *pgdir,
  94                                         struct hns_roce_db *db, int order)
  95 {
  96         unsigned long o;
  97         unsigned long i;
  98
  99         for (o = order; o <= 1; ++o) {
 100                 i = find_first_bit(pgdir->bits[o], HNS_ROCE_DB_PER_PAGE >> o);
 101                 if (i < HNS_ROCE_DB_PER_PAGE >> o)
 102                         goto found;
 103         }
 104
 105         return -ENOMEM;
 106
 107 found:
 108         clear_bit(i, pgdir->bits[o]);
 109
 110         i <<= o;
 111
 112         if (o > order)
 113                 set_bit(i ^ 1, pgdir->bits[order]);
 114
 115         db->u.pgdir     = pgdir;
 116         db->index       = i;
 117         db->db_record   = pgdir->page + db->index;
 118         db->dma         = pgdir->db_dma  + db->index * HNS_ROCE_DB_UNIT_SIZE;
 119         db->order       = order;
 120
 121         return 0;
 122 }
 123
 124 int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db,
 125                       int order)
 126 {
 127         struct hns_roce_db_pgdir *pgdir;
 128         int ret = 0;
 129
 130         mutex_lock(&hr_dev->pgdir_mutex);
 131
 132         list_for_each_entry(pgdir, &hr_dev->pgdir_list, list)
 133                 if (!hns_roce_alloc_db_from_pgdir(pgdir, db, order))
 134                         goto out;
 135
 136         pgdir = hns_roce_alloc_db_pgdir(hr_dev->dev);
 137         if (!pgdir) {
 138                 ret = -ENOMEM;
 139                 goto out;
 140         }
 141
 142         list_add(&pgdir->list, &hr_dev->pgdir_list);
 143
 144         /* This should never fail -- we just allocated an empty page: */
 145         WARN_ON(hns_roce_alloc_db_from_pgdir(pgdir, db, order));
 146
 147 out:
 148         mutex_unlock(&hr_dev->pgdir_mutex);
 149
 150         return ret;
 151 }
 152
 153 void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db)
 154 {
 155         unsigned long o;
 156         unsigned long i;
 157
 158         mutex_lock(&hr_dev->pgdir_mutex);
 159
 160         o = db->order;
 161         i = db->index;
 162
 163         if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
 164                 clear_bit(i ^ 1, db->u.pgdir->order0);
 165                 ++o;
 166         }
 167
 168         i >>= o;
 169         set_bit(i, db->u.pgdir->bits[o]);
 170
 171         if (bitmap_full(db->u.pgdir->order1,
 172                         HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) {
 173                 dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page,
 174                                   db->u.pgdir->db_dma);
 175                 list_del(&db->u.pgdir->list);
 176                 kfree(db->u.pgdir);
 177         }
 178
 179         mutex_unlock(&hr_dev->pgdir_mutex);
 180 }