drivers/gpu/drm/i915/i915_scatterlist.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2016 Intel Corporation
   5  */
   6
   7 #include "i915_scatterlist.h"
   8 #include "i915_ttm_buddy_manager.h"
   9
  10 #include <drm/drm_buddy.h>
  11 #include <drm/drm_mm.h>
  12
  13 #include <linux/slab.h>
  14
  15 bool i915_sg_trim(struct sg_table *orig_st)
  16 {
  17         struct sg_table new_st;
  18         struct scatterlist *sg, *new_sg;
  19         unsigned int i;
  20
  21         if (orig_st->nents == orig_st->orig_nents)
  22                 return false;
  23
  24         if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN))
  25                 return false;
  26
  27         new_sg = new_st.sgl;
  28         for_each_sg(orig_st->sgl, sg, orig_st->nents, i) {
  29                 sg_set_page(new_sg, sg_page(sg), sg->length, 0);
  30                 sg_dma_address(new_sg) = sg_dma_address(sg);
  31                 sg_dma_len(new_sg) = sg_dma_len(sg);
  32
  33                 new_sg = sg_next(new_sg);
  34         }
  35         GEM_BUG_ON(new_sg); /* Should walk exactly nents and hit the end */
  36
  37         sg_free_table(orig_st);
  38
  39         *orig_st = new_st;
  40         return true;
  41 }
  42
  43 static void i915_refct_sgt_release(struct kref *ref)
  44 {
  45         struct i915_refct_sgt *rsgt =
  46                 container_of(ref, typeof(*rsgt), kref);
  47
  48         sg_free_table(&rsgt->table);
  49         kfree(rsgt);
  50 }
  51
  52 static const struct i915_refct_sgt_ops rsgt_ops = {
  53         .release = i915_refct_sgt_release
  54 };
  55
  56 /**
  57  * i915_refct_sgt_init - Initialize a struct i915_refct_sgt with default ops
  58  * @rsgt: The struct i915_refct_sgt to initialize.
  59  * @size: The size of the underlying memory buffer.
  60  */
  61 void i915_refct_sgt_init(struct i915_refct_sgt *rsgt, size_t size)
  62 {
  63         __i915_refct_sgt_init(rsgt, size, &rsgt_ops);
  64 }
  65
  66 /**
  67  * i915_rsgt_from_mm_node - Create a refcounted sg_table from a struct
  68  * drm_mm_node
  69  * @node: The drm_mm_node.
  70  * @region_start: An offset to add to the dma addresses of the sg list.
  71  * @page_alignment: Required page alignment for each sg entry. Power of two.
  72  *
  73  * Create a struct sg_table, initializing it from a struct drm_mm_node,
  74  * taking a maximum segment length into account, splitting into segments
  75  * if necessary.
  76  *
  77  * Return: A pointer to a kmalloced struct i915_refct_sgt on success, negative
  78  * error code cast to an error pointer on failure.
  79  */
  80 struct i915_refct_sgt *i915_rsgt_from_mm_node(const struct drm_mm_node *node,
  81                                               u64 region_start,
  82                                               u32 page_alignment)
  83 {
  84         const u32 max_segment = round_down(UINT_MAX, page_alignment);
  85         const u32 segment_pages = max_segment >> PAGE_SHIFT;
  86         u64 block_size, offset, prev_end;
  87         struct i915_refct_sgt *rsgt;
  88         struct sg_table *st;
  89         struct scatterlist *sg;
  90
  91         GEM_BUG_ON(!max_segment);
  92
  93         rsgt = kmalloc(sizeof(*rsgt), GFP_KERNEL);
  94         if (!rsgt)
  95                 return ERR_PTR(-ENOMEM);
  96
  97         i915_refct_sgt_init(rsgt, node->size << PAGE_SHIFT);
  98         st = &rsgt->table;
  99         /* restricted by sg_alloc_table */
 100         if (WARN_ON(overflows_type(DIV_ROUND_UP_ULL(node->size, segment_pages),
 101                                    unsigned int))) {
 102                 i915_refct_sgt_put(rsgt);
 103                 return ERR_PTR(-E2BIG);
 104         }
 105
 106         if (sg_alloc_table(st, DIV_ROUND_UP_ULL(node->size, segment_pages),
 107                            GFP_KERNEL)) {
 108                 i915_refct_sgt_put(rsgt);
 109                 return ERR_PTR(-ENOMEM);
 110         }
 111
 112         sg = st->sgl;
 113         st->nents = 0;
 114         prev_end = (resource_size_t)-1;
 115         block_size = node->size << PAGE_SHIFT;
 116         offset = node->start << PAGE_SHIFT;
 117
 118         while (block_size) {
 119                 u64 len;
 120
 121                 if (offset != prev_end || sg->length >= max_segment) {
 122                         if (st->nents)
 123                                 sg = __sg_next(sg);
 124
 125                         sg_dma_address(sg) = region_start + offset;
 126                         GEM_BUG_ON(!IS_ALIGNED(sg_dma_address(sg),
 127                                                page_alignment));
 128                         sg_dma_len(sg) = 0;
 129                         sg->length = 0;
 130                         st->nents++;
 131                 }
 132
 133                 len = min_t(u64, block_size, max_segment - sg->length);
 134                 sg->length += len;
 135                 sg_dma_len(sg) += len;
 136
 137                 offset += len;
 138                 block_size -= len;
 139
 140                 prev_end = offset;
 141         }
 142
 143         sg_mark_end(sg);
 144         i915_sg_trim(st);
 145
 146         return rsgt;
 147 }
 148
 149 /**
 150  * i915_rsgt_from_buddy_resource - Create a refcounted sg_table from a struct
 151  * i915_buddy_block list
 152  * @res: The struct i915_ttm_buddy_resource.
 153  * @region_start: An offset to add to the dma addresses of the sg list.
 154  * @page_alignment: Required page alignment for each sg entry. Power of two.
 155  *
 156  * Create a struct sg_table, initializing it from struct i915_buddy_block list,
 157  * taking a maximum segment length into account, splitting into segments
 158  * if necessary.
 159  *
 160  * Return: A pointer to a kmalloced struct i915_refct_sgts on success, negative
 161  * error code cast to an error pointer on failure.
 162  */
 163 struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res,
 164                                                      u64 region_start,
 165                                                      u32 page_alignment)
 166 {
 167         struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
 168         const u64 size = res->size;
 169         const u32 max_segment = round_down(UINT_MAX, page_alignment);
 170         struct drm_buddy *mm = bman_res->mm;
 171         struct list_head *blocks = &bman_res->blocks;
 172         struct drm_buddy_block *block;
 173         struct i915_refct_sgt *rsgt;
 174         struct scatterlist *sg;
 175         struct sg_table *st;
 176         resource_size_t prev_end;
 177
 178         GEM_BUG_ON(list_empty(blocks));
 179         GEM_BUG_ON(!max_segment);
 180
 181         rsgt = kmalloc(sizeof(*rsgt), GFP_KERNEL);
 182         if (!rsgt)
 183                 return ERR_PTR(-ENOMEM);
 184
 185         i915_refct_sgt_init(rsgt, size);
 186         st = &rsgt->table;
 187         /* restricted by sg_alloc_table */
 188         if (WARN_ON(overflows_type(PFN_UP(res->size), unsigned int))) {
 189                 i915_refct_sgt_put(rsgt);
 190                 return ERR_PTR(-E2BIG);
 191         }
 192
 193         if (sg_alloc_table(st, PFN_UP(res->size), GFP_KERNEL)) {
 194                 i915_refct_sgt_put(rsgt);
 195                 return ERR_PTR(-ENOMEM);
 196         }
 197
 198         sg = st->sgl;
 199         st->nents = 0;
 200         prev_end = (resource_size_t)-1;
 201
 202         list_for_each_entry(block, blocks, link) {
 203                 u64 block_size, offset;
 204
 205                 block_size = min_t(u64, size, drm_buddy_block_size(mm, block));
 206                 offset = drm_buddy_block_offset(block);
 207
 208                 while (block_size) {
 209                         u64 len;
 210
 211                         if (offset != prev_end || sg->length >= max_segment) {
 212                                 if (st->nents)
 213                                         sg = __sg_next(sg);
 214
 215                                 sg_dma_address(sg) = region_start + offset;
 216                                 GEM_BUG_ON(!IS_ALIGNED(sg_dma_address(sg),
 217                                                        page_alignment));
 218                                 sg_dma_len(sg) = 0;
 219                                 sg->length = 0;
 220                                 st->nents++;
 221                         }
 222
 223                         len = min_t(u64, block_size, max_segment - sg->length);
 224                         sg->length += len;
 225                         sg_dma_len(sg) += len;
 226
 227                         offset += len;
 228                         block_size -= len;
 229
 230                         prev_end = offset;
 231                 }
 232         }
 233
 234         sg_mark_end(sg);
 235         i915_sg_trim(st);
 236
 237         return rsgt;
 238 }
 239
 240 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 241 #include "selftests/scatterlist.c"
 242 #endif