drivers/gpu/drm/vmwgfx/vmwgfx_ttm_buffer.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /**************************************************************************
   3  *
   4  * Copyright 2009-2023 VMware, Inc., Palo Alto, CA., USA
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include "vmwgfx_bo.h"
  29 #include "vmwgfx_drv.h"
  30 #include <drm/ttm/ttm_placement.h>
  31
  32 static const struct ttm_place vram_placement_flags = {
  33         .fpfn = 0,
  34         .lpfn = 0,
  35         .mem_type = TTM_PL_VRAM,
  36         .flags = 0
  37 };
  38
  39 static const struct ttm_place sys_placement_flags = {
  40         .fpfn = 0,
  41         .lpfn = 0,
  42         .mem_type = TTM_PL_SYSTEM,
  43         .flags = 0
  44 };
  45
  46 static const struct ttm_place gmr_placement_flags = {
  47         .fpfn = 0,
  48         .lpfn = 0,
  49         .mem_type = VMW_PL_GMR,
  50         .flags = 0
  51 };
  52
  53 static const struct ttm_place mob_placement_flags = {
  54         .fpfn = 0,
  55         .lpfn = 0,
  56         .mem_type = VMW_PL_MOB,
  57         .flags = 0
  58 };
  59
  60 struct ttm_placement vmw_vram_placement = {
  61         .num_placement = 1,
  62         .placement = &vram_placement_flags,
  63         .num_busy_placement = 1,
  64         .busy_placement = &vram_placement_flags
  65 };
  66
  67 static const struct ttm_place vram_gmr_placement_flags[] = {
  68         {
  69                 .fpfn = 0,
  70                 .lpfn = 0,
  71                 .mem_type = TTM_PL_VRAM,
  72                 .flags = 0
  73         }, {
  74                 .fpfn = 0,
  75                 .lpfn = 0,
  76                 .mem_type = VMW_PL_GMR,
  77                 .flags = 0
  78         }
  79 };
  80
  81 static const struct ttm_place vmw_sys_placement_flags = {
  82         .fpfn = 0,
  83         .lpfn = 0,
  84         .mem_type = VMW_PL_SYSTEM,
  85         .flags = 0
  86 };
  87
  88 struct ttm_placement vmw_vram_gmr_placement = {
  89         .num_placement = 2,
  90         .placement = vram_gmr_placement_flags,
  91         .num_busy_placement = 1,
  92         .busy_placement = &gmr_placement_flags
  93 };
  94
  95 struct ttm_placement vmw_vram_sys_placement = {
  96         .num_placement = 1,
  97         .placement = &vram_placement_flags,
  98         .num_busy_placement = 1,
  99         .busy_placement = &sys_placement_flags
 100 };
 101
 102 struct ttm_placement vmw_sys_placement = {
 103         .num_placement = 1,
 104         .placement = &sys_placement_flags,
 105         .num_busy_placement = 1,
 106         .busy_placement = &sys_placement_flags
 107 };
 108
 109 struct ttm_placement vmw_pt_sys_placement = {
 110         .num_placement = 1,
 111         .placement = &vmw_sys_placement_flags,
 112         .num_busy_placement = 1,
 113         .busy_placement = &vmw_sys_placement_flags
 114 };
 115
 116 struct ttm_placement vmw_mob_placement = {
 117         .num_placement = 1,
 118         .num_busy_placement = 1,
 119         .placement = &mob_placement_flags,
 120         .busy_placement = &mob_placement_flags
 121 };
 122
 123 const size_t vmw_tt_size = sizeof(struct vmw_ttm_tt);
 124
 125 /**
 126  * __vmw_piter_non_sg_next: Helper functions to advance
 127  * a struct vmw_piter iterator.
 128  *
 129  * @viter: Pointer to the iterator.
 130  *
 131  * These functions return false if past the end of the list,
 132  * true otherwise. Functions are selected depending on the current
 133  * DMA mapping mode.
 134  */
 135 static bool __vmw_piter_non_sg_next(struct vmw_piter *viter)
 136 {
 137         return ++(viter->i) < viter->num_pages;
 138 }
 139
 140 static bool __vmw_piter_sg_next(struct vmw_piter *viter)
 141 {
 142         bool ret = __vmw_piter_non_sg_next(viter);
 143
 144         return __sg_page_iter_dma_next(&viter->iter) && ret;
 145 }
 146
 147
 148 static dma_addr_t __vmw_piter_dma_addr(struct vmw_piter *viter)
 149 {
 150         return viter->addrs[viter->i];
 151 }
 152
 153 static dma_addr_t __vmw_piter_sg_addr(struct vmw_piter *viter)
 154 {
 155         return sg_page_iter_dma_address(&viter->iter);
 156 }
 157
 158
 159 /**
 160  * vmw_piter_start - Initialize a struct vmw_piter.
 161  *
 162  * @viter: Pointer to the iterator to initialize
 163  * @vsgt: Pointer to a struct vmw_sg_table to initialize from
 164  * @p_offset: Pointer offset used to update current array position
 165  *
 166  * Note that we're following the convention of __sg_page_iter_start, so that
 167  * the iterator doesn't point to a valid page after initialization; it has
 168  * to be advanced one step first.
 169  */
 170 void vmw_piter_start(struct vmw_piter *viter, const struct vmw_sg_table *vsgt,
 171                      unsigned long p_offset)
 172 {
 173         viter->i = p_offset - 1;
 174         viter->num_pages = vsgt->num_pages;
 175         viter->pages = vsgt->pages;
 176         switch (vsgt->mode) {
 177         case vmw_dma_alloc_coherent:
 178                 viter->next = &__vmw_piter_non_sg_next;
 179                 viter->dma_address = &__vmw_piter_dma_addr;
 180                 viter->addrs = vsgt->addrs;
 181                 break;
 182         case vmw_dma_map_populate:
 183         case vmw_dma_map_bind:
 184                 viter->next = &__vmw_piter_sg_next;
 185                 viter->dma_address = &__vmw_piter_sg_addr;
 186                 __sg_page_iter_start(&viter->iter.base, vsgt->sgt->sgl,
 187                                      vsgt->sgt->orig_nents, p_offset);
 188                 break;
 189         default:
 190                 BUG();
 191         }
 192 }
 193
 194 /**
 195  * vmw_ttm_unmap_from_dma - unmap  device addresses previsouly mapped for
 196  * TTM pages
 197  *
 198  * @vmw_tt: Pointer to a struct vmw_ttm_backend
 199  *
 200  * Used to free dma mappings previously mapped by vmw_ttm_map_for_dma.
 201  */
 202 static void vmw_ttm_unmap_from_dma(struct vmw_ttm_tt *vmw_tt)
 203 {
 204         struct device *dev = vmw_tt->dev_priv->drm.dev;
 205
 206         dma_unmap_sgtable(dev, &vmw_tt->sgt, DMA_BIDIRECTIONAL, 0);
 207         vmw_tt->sgt.nents = vmw_tt->sgt.orig_nents;
 208 }
 209
 210 /**
 211  * vmw_ttm_map_for_dma - map TTM pages to get device addresses
 212  *
 213  * @vmw_tt: Pointer to a struct vmw_ttm_backend
 214  *
 215  * This function is used to get device addresses from the kernel DMA layer.
 216  * However, it's violating the DMA API in that when this operation has been
 217  * performed, it's illegal for the CPU to write to the pages without first
 218  * unmapping the DMA mappings, or calling dma_sync_sg_for_cpu(). It is
 219  * therefore only legal to call this function if we know that the function
 220  * dma_sync_sg_for_cpu() is a NOP, and dma_sync_sg_for_device() is at most
 221  * a CPU write buffer flush.
 222  */
 223 static int vmw_ttm_map_for_dma(struct vmw_ttm_tt *vmw_tt)
 224 {
 225         struct device *dev = vmw_tt->dev_priv->drm.dev;
 226
 227         return dma_map_sgtable(dev, &vmw_tt->sgt, DMA_BIDIRECTIONAL, 0);
 228 }
 229
 230 /**
 231  * vmw_ttm_map_dma - Make sure TTM pages are visible to the device
 232  *
 233  * @vmw_tt: Pointer to a struct vmw_ttm_tt
 234  *
 235  * Select the correct function for and make sure the TTM pages are
 236  * visible to the device. Allocate storage for the device mappings.
 237  * If a mapping has already been performed, indicated by the storage
 238  * pointer being non NULL, the function returns success.
 239  */
 240 static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)
 241 {
 242         struct vmw_private *dev_priv = vmw_tt->dev_priv;
 243         struct vmw_sg_table *vsgt = &vmw_tt->vsgt;
 244         int ret = 0;
 245
 246         if (vmw_tt->mapped)
 247                 return 0;
 248
 249         vsgt->mode = dev_priv->map_mode;
 250         vsgt->pages = vmw_tt->dma_ttm.pages;
 251         vsgt->num_pages = vmw_tt->dma_ttm.num_pages;
 252         vsgt->addrs = vmw_tt->dma_ttm.dma_address;
 253         vsgt->sgt = NULL;
 254
 255         switch (dev_priv->map_mode) {
 256         case vmw_dma_map_bind:
 257         case vmw_dma_map_populate:
 258                 vsgt->sgt = &vmw_tt->sgt;
 259                 ret = sg_alloc_table_from_pages_segment(
 260                         &vmw_tt->sgt, vsgt->pages, vsgt->num_pages, 0,
 261                         (unsigned long)vsgt->num_pages << PAGE_SHIFT,
 262                         dma_get_max_seg_size(dev_priv->drm.dev), GFP_KERNEL);
 263                 if (ret)
 264                         goto out_sg_alloc_fail;
 265
 266                 ret = vmw_ttm_map_for_dma(vmw_tt);
 267                 if (unlikely(ret != 0))
 268                         goto out_map_fail;
 269
 270                 break;
 271         default:
 272                 break;
 273         }
 274
 275         vmw_tt->mapped = true;
 276         return 0;
 277
 278 out_map_fail:
 279         sg_free_table(vmw_tt->vsgt.sgt);
 280         vmw_tt->vsgt.sgt = NULL;
 281 out_sg_alloc_fail:
 282         return ret;
 283 }
 284
 285 /**
 286  * vmw_ttm_unmap_dma - Tear down any TTM page device mappings
 287  *
 288  * @vmw_tt: Pointer to a struct vmw_ttm_tt
 289  *
 290  * Tear down any previously set up device DMA mappings and free
 291  * any storage space allocated for them. If there are no mappings set up,
 292  * this function is a NOP.
 293  */
 294 static void vmw_ttm_unmap_dma(struct vmw_ttm_tt *vmw_tt)
 295 {
 296         struct vmw_private *dev_priv = vmw_tt->dev_priv;
 297
 298         if (!vmw_tt->vsgt.sgt)
 299                 return;
 300
 301         switch (dev_priv->map_mode) {
 302         case vmw_dma_map_bind:
 303         case vmw_dma_map_populate:
 304                 vmw_ttm_unmap_from_dma(vmw_tt);
 305                 sg_free_table(vmw_tt->vsgt.sgt);
 306                 vmw_tt->vsgt.sgt = NULL;
 307                 break;
 308         default:
 309                 break;
 310         }
 311         vmw_tt->mapped = false;
 312 }
 313
 314 /**
 315  * vmw_bo_sg_table - Return a struct vmw_sg_table object for a
 316  * TTM buffer object
 317  *
 318  * @bo: Pointer to a struct ttm_buffer_object
 319  *
 320  * Returns a pointer to a struct vmw_sg_table object. The object should
 321  * not be freed after use.
 322  * Note that for the device addresses to be valid, the buffer object must
 323  * either be reserved or pinned.
 324  */
 325 const struct vmw_sg_table *vmw_bo_sg_table(struct ttm_buffer_object *bo)
 326 {
 327         struct vmw_ttm_tt *vmw_tt =
 328                 container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm);
 329
 330         return &vmw_tt->vsgt;
 331 }
 332
 333
 334 static int vmw_ttm_bind(struct ttm_device *bdev,
 335                         struct ttm_tt *ttm, struct ttm_resource *bo_mem)
 336 {
 337         struct vmw_ttm_tt *vmw_be =
 338                 container_of(ttm, struct vmw_ttm_tt, dma_ttm);
 339         int ret = 0;
 340
 341         if (!bo_mem)
 342                 return -EINVAL;
 343
 344         if (vmw_be->bound)
 345                 return 0;
 346
 347         ret = vmw_ttm_map_dma(vmw_be);
 348         if (unlikely(ret != 0))
 349                 return ret;
 350
 351         vmw_be->gmr_id = bo_mem->start;
 352         vmw_be->mem_type = bo_mem->mem_type;
 353
 354         switch (bo_mem->mem_type) {
 355         case VMW_PL_GMR:
 356                 ret = vmw_gmr_bind(vmw_be->dev_priv, &vmw_be->vsgt,
 357                                     ttm->num_pages, vmw_be->gmr_id);
 358                 break;
 359         case VMW_PL_MOB:
 360                 if (unlikely(vmw_be->mob == NULL)) {
 361                         vmw_be->mob =
 362                                 vmw_mob_create(ttm->num_pages);
 363                         if (unlikely(vmw_be->mob == NULL))
 364                                 return -ENOMEM;
 365                 }
 366
 367                 ret = vmw_mob_bind(vmw_be->dev_priv, vmw_be->mob,
 368                                     &vmw_be->vsgt, ttm->num_pages,
 369                                     vmw_be->gmr_id);
 370                 break;
 371         case VMW_PL_SYSTEM:
 372                 /* Nothing to be done for a system bind */
 373                 break;
 374         default:
 375                 BUG();
 376         }
 377         vmw_be->bound = true;
 378         return ret;
 379 }
 380
 381 static void vmw_ttm_unbind(struct ttm_device *bdev,
 382                            struct ttm_tt *ttm)
 383 {
 384         struct vmw_ttm_tt *vmw_be =
 385                 container_of(ttm, struct vmw_ttm_tt, dma_ttm);
 386
 387         if (!vmw_be->bound)
 388                 return;
 389
 390         switch (vmw_be->mem_type) {
 391         case VMW_PL_GMR:
 392                 vmw_gmr_unbind(vmw_be->dev_priv, vmw_be->gmr_id);
 393                 break;
 394         case VMW_PL_MOB:
 395                 vmw_mob_unbind(vmw_be->dev_priv, vmw_be->mob);
 396                 break;
 397         case VMW_PL_SYSTEM:
 398                 break;
 399         default:
 400                 BUG();
 401         }
 402
 403         if (vmw_be->dev_priv->map_mode == vmw_dma_map_bind)
 404                 vmw_ttm_unmap_dma(vmw_be);
 405         vmw_be->bound = false;
 406 }
 407
 408
 409 static void vmw_ttm_destroy(struct ttm_device *bdev, struct ttm_tt *ttm)
 410 {
 411         struct vmw_ttm_tt *vmw_be =
 412                 container_of(ttm, struct vmw_ttm_tt, dma_ttm);
 413
 414         vmw_ttm_unmap_dma(vmw_be);
 415         ttm_tt_fini(ttm);
 416         if (vmw_be->mob)
 417                 vmw_mob_destroy(vmw_be->mob);
 418
 419         kfree(vmw_be);
 420 }
 421
 422
 423 static int vmw_ttm_populate(struct ttm_device *bdev,
 424                             struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
 425 {
 426         int ret;
 427
 428         /* TODO: maybe completely drop this ? */
 429         if (ttm_tt_is_populated(ttm))
 430                 return 0;
 431
 432         ret = ttm_pool_alloc(&bdev->pool, ttm, ctx);
 433
 434         return ret;
 435 }
 436
 437 static void vmw_ttm_unpopulate(struct ttm_device *bdev,
 438                                struct ttm_tt *ttm)
 439 {
 440         struct vmw_ttm_tt *vmw_tt = container_of(ttm, struct vmw_ttm_tt,
 441                                                  dma_ttm);
 442
 443         vmw_ttm_unbind(bdev, ttm);
 444
 445         if (vmw_tt->mob) {
 446                 vmw_mob_destroy(vmw_tt->mob);
 447                 vmw_tt->mob = NULL;
 448         }
 449
 450         vmw_ttm_unmap_dma(vmw_tt);
 451
 452         ttm_pool_free(&bdev->pool, ttm);
 453 }
 454
 455 static struct ttm_tt *vmw_ttm_tt_create(struct ttm_buffer_object *bo,
 456                                         uint32_t page_flags)
 457 {
 458         struct vmw_ttm_tt *vmw_be;
 459         int ret;
 460
 461         vmw_be = kzalloc(sizeof(*vmw_be), GFP_KERNEL);
 462         if (!vmw_be)
 463                 return NULL;
 464
 465         vmw_be->dev_priv = container_of(bo->bdev, struct vmw_private, bdev);
 466         vmw_be->mob = NULL;
 467
 468         if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent)
 469                 ret = ttm_sg_tt_init(&vmw_be->dma_ttm, bo, page_flags,
 470                                      ttm_cached);
 471         else
 472                 ret = ttm_tt_init(&vmw_be->dma_ttm, bo, page_flags,
 473                                   ttm_cached, 0);
 474         if (unlikely(ret != 0))
 475                 goto out_no_init;
 476
 477         return &vmw_be->dma_ttm;
 478 out_no_init:
 479         kfree(vmw_be);
 480         return NULL;
 481 }
 482
 483 static void vmw_evict_flags(struct ttm_buffer_object *bo,
 484                      struct ttm_placement *placement)
 485 {
 486         *placement = vmw_sys_placement;
 487 }
 488
 489 static int vmw_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *mem)
 490 {
 491         struct vmw_private *dev_priv = container_of(bdev, struct vmw_private, bdev);
 492
 493         switch (mem->mem_type) {
 494         case TTM_PL_SYSTEM:
 495         case VMW_PL_SYSTEM:
 496         case VMW_PL_GMR:
 497         case VMW_PL_MOB:
 498                 return 0;
 499         case TTM_PL_VRAM:
 500                 mem->bus.offset = (mem->start << PAGE_SHIFT) +
 501                         dev_priv->vram_start;
 502                 mem->bus.is_iomem = true;
 503                 mem->bus.caching = ttm_cached;
 504                 break;
 505         default:
 506                 return -EINVAL;
 507         }
 508         return 0;
 509 }
 510
 511 /**
 512  * vmw_move_notify - TTM move_notify_callback
 513  *
 514  * @bo: The TTM buffer object about to move.
 515  * @old_mem: The old memory where we move from
 516  * @new_mem: The struct ttm_resource indicating to what memory
 517  *       region the move is taking place.
 518  *
 519  * Calls move_notify for all subsystems needing it.
 520  * (currently only resources).
 521  */
 522 static void vmw_move_notify(struct ttm_buffer_object *bo,
 523                             struct ttm_resource *old_mem,
 524                             struct ttm_resource *new_mem)
 525 {
 526         vmw_bo_move_notify(bo, new_mem);
 527         vmw_query_move_notify(bo, old_mem, new_mem);
 528 }
 529
 530
 531 /**
 532  * vmw_swap_notify - TTM move_notify_callback
 533  *
 534  * @bo: The TTM buffer object about to be swapped out.
 535  */
 536 static void vmw_swap_notify(struct ttm_buffer_object *bo)
 537 {
 538         vmw_bo_swap_notify(bo);
 539         (void) ttm_bo_wait(bo, false, false);
 540 }
 541
 542 static bool vmw_memtype_is_system(uint32_t mem_type)
 543 {
 544         return mem_type == TTM_PL_SYSTEM || mem_type == VMW_PL_SYSTEM;
 545 }
 546
 547 static int vmw_move(struct ttm_buffer_object *bo,
 548                     bool evict,
 549                     struct ttm_operation_ctx *ctx,
 550                     struct ttm_resource *new_mem,
 551                     struct ttm_place *hop)
 552 {
 553         struct ttm_resource_manager *old_man = ttm_manager_type(bo->bdev, bo->resource->mem_type);
 554         struct ttm_resource_manager *new_man = ttm_manager_type(bo->bdev, new_mem->mem_type);
 555         int ret;
 556
 557         if (new_man->use_tt && !vmw_memtype_is_system(new_mem->mem_type)) {
 558                 ret = vmw_ttm_bind(bo->bdev, bo->ttm, new_mem);
 559                 if (ret)
 560                         return ret;
 561         }
 562
 563         vmw_move_notify(bo, bo->resource, new_mem);
 564
 565         if (old_man->use_tt && new_man->use_tt) {
 566                 if (vmw_memtype_is_system(bo->resource->mem_type)) {
 567                         ttm_bo_move_null(bo, new_mem);
 568                         return 0;
 569                 }
 570                 ret = ttm_bo_wait_ctx(bo, ctx);
 571                 if (ret)
 572                         goto fail;
 573
 574                 vmw_ttm_unbind(bo->bdev, bo->ttm);
 575                 ttm_resource_free(bo, &bo->resource);
 576                 ttm_bo_assign_mem(bo, new_mem);
 577                 return 0;
 578         } else {
 579                 ret = ttm_bo_move_memcpy(bo, ctx, new_mem);
 580                 if (ret)
 581                         goto fail;
 582         }
 583         return 0;
 584 fail:
 585         vmw_move_notify(bo, new_mem, bo->resource);
 586         return ret;
 587 }
 588
 589 struct ttm_device_funcs vmw_bo_driver = {
 590         .ttm_tt_create = &vmw_ttm_tt_create,
 591         .ttm_tt_populate = &vmw_ttm_populate,
 592         .ttm_tt_unpopulate = &vmw_ttm_unpopulate,
 593         .ttm_tt_destroy = &vmw_ttm_destroy,
 594         .eviction_valuable = ttm_bo_eviction_valuable,
 595         .evict_flags = vmw_evict_flags,
 596         .move = vmw_move,
 597         .swap_notify = vmw_swap_notify,
 598         .io_mem_reserve = &vmw_ttm_io_mem_reserve,
 599 };
 600
 601 int vmw_bo_create_and_populate(struct vmw_private *dev_priv,
 602                                unsigned long bo_size,
 603                                struct ttm_buffer_object **bo_p)
 604 {
 605         struct ttm_operation_ctx ctx = {
 606                 .interruptible = false,
 607                 .no_wait_gpu = false
 608         };
 609         struct ttm_buffer_object *bo;
 610         int ret;
 611
 612         ret = vmw_bo_create_kernel(dev_priv, bo_size,
 613                                    &vmw_pt_sys_placement,
 614                                    &bo);
 615         if (unlikely(ret != 0))
 616                 return ret;
 617
 618         ret = ttm_bo_reserve(bo, false, true, NULL);
 619         BUG_ON(ret != 0);
 620         ret = vmw_ttm_populate(bo->bdev, bo->ttm, &ctx);
 621         if (likely(ret == 0)) {
 622                 struct vmw_ttm_tt *vmw_tt =
 623                         container_of(bo->ttm, struct vmw_ttm_tt, dma_ttm);
 624                 ret = vmw_ttm_map_dma(vmw_tt);
 625         }
 626
 627         ttm_bo_unreserve(bo);
 628
 629         if (likely(ret == 0))
 630                 *bo_p = bo;
 631         return ret;
 632 }