drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c

   1 /*
   2  * Huawei HiNIC PCI Express Linux driver
   3  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * for more details.
  13  *
  14  */
  15
  16 #include <linux/kernel.h>
  17 #include <linux/types.h>
  18 #include <linux/pci.h>
  19 #include <linux/device.h>
  20 #include <linux/dma-mapping.h>
  21 #include <linux/slab.h>
  22 #include <linux/atomic.h>
  23 #include <linux/semaphore.h>
  24 #include <linux/errno.h>
  25 #include <linux/vmalloc.h>
  26 #include <linux/err.h>
  27 #include <asm/byteorder.h>
  28
  29 #include "hinic_hw_if.h"
  30 #include "hinic_hw_wqe.h"
  31 #include "hinic_hw_wq.h"
  32 #include "hinic_hw_cmdq.h"
  33
  34 #define WQS_BLOCKS_PER_PAGE             4
  35
  36 #define WQ_BLOCK_SIZE                   4096
  37 #define WQS_PAGE_SIZE                   (WQS_BLOCKS_PER_PAGE * WQ_BLOCK_SIZE)
  38
  39 #define WQS_MAX_NUM_BLOCKS              128
  40 #define WQS_FREE_BLOCKS_SIZE(wqs)       (WQS_MAX_NUM_BLOCKS * \
  41                                          sizeof((wqs)->free_blocks[0]))
  42
  43 #define WQ_SIZE(wq)                     ((wq)->q_depth * (wq)->wqebb_size)
  44
  45 #define WQ_PAGE_ADDR_SIZE               sizeof(u64)
  46 #define WQ_MAX_PAGES                    (WQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE)
  47
  48 #define CMDQ_BLOCK_SIZE                 512
  49 #define CMDQ_PAGE_SIZE                  4096
  50
  51 #define CMDQ_WQ_MAX_PAGES               (CMDQ_BLOCK_SIZE / WQ_PAGE_ADDR_SIZE)
  52
  53 #define WQ_BASE_VADDR(wqs, wq)          \
  54                         ((void *)((wqs)->page_vaddr[(wq)->page_idx]) \
  55                                 + (wq)->block_idx * WQ_BLOCK_SIZE)
  56
  57 #define WQ_BASE_PADDR(wqs, wq)          \
  58                         ((wqs)->page_paddr[(wq)->page_idx] \
  59                                 + (wq)->block_idx * WQ_BLOCK_SIZE)
  60
  61 #define WQ_BASE_ADDR(wqs, wq)           \
  62                         ((void *)((wqs)->shadow_page_vaddr[(wq)->page_idx]) \
  63                                 + (wq)->block_idx * WQ_BLOCK_SIZE)
  64
  65 #define CMDQ_BASE_VADDR(cmdq_pages, wq) \
  66                         ((void *)((cmdq_pages)->page_vaddr) \
  67                                 + (wq)->block_idx * CMDQ_BLOCK_SIZE)
  68
  69 #define CMDQ_BASE_PADDR(cmdq_pages, wq) \
  70                         ((cmdq_pages)->page_paddr \
  71                                 + (wq)->block_idx * CMDQ_BLOCK_SIZE)
  72
  73 #define CMDQ_BASE_ADDR(cmdq_pages, wq)  \
  74                         ((void *)((cmdq_pages)->shadow_page_vaddr) \
  75                                 + (wq)->block_idx * CMDQ_BLOCK_SIZE)
  76
  77 #define WQ_PAGE_ADDR(wq, idx)           \
  78                         ((wq)->shadow_block_vaddr[WQE_PAGE_NUM(wq, idx)])
  79
  80 #define MASKED_WQE_IDX(wq, idx)         ((idx) & (wq)->mask)
  81
  82 #define WQE_IN_RANGE(wqe, start, end)   \
  83                 (((unsigned long)(wqe) >= (unsigned long)(start)) && \
  84                  ((unsigned long)(wqe) < (unsigned long)(end)))
  85
  86 #define WQE_SHADOW_PAGE(wq, wqe)        \
  87                 (((unsigned long)(wqe) - (unsigned long)(wq)->shadow_wqe) \
  88                         / (wq)->max_wqe_size)
  89
  90 static inline int WQE_PAGE_OFF(struct hinic_wq *wq, u16 idx)
  91 {
  92         return (((idx) & ((wq)->num_wqebbs_per_page - 1))
  93                 << (wq)->wqebb_size_shift);
  94 }
  95
  96 static inline int WQE_PAGE_NUM(struct hinic_wq *wq, u16 idx)
  97 {
  98         return (((idx) >> ((wq)->wqebbs_per_page_shift))
  99                 & ((wq)->num_q_pages - 1));
 100 }
 101 /**
 102  * queue_alloc_page - allocate page for Queue
 103  * @hwif: HW interface for allocating DMA
 104  * @vaddr: virtual address will be returned in this address
 105  * @paddr: physical address will be returned in this address
 106  * @shadow_vaddr: VM area will be return here for holding WQ page addresses
 107  * @page_sz: page size of each WQ page
 108  *
 109  * Return 0 - Success, negative - Failure
 110  **/
 111 static int queue_alloc_page(struct hinic_hwif *hwif, u64 **vaddr, u64 *paddr,
 112                             void ***shadow_vaddr, size_t page_sz)
 113 {
 114         struct pci_dev *pdev = hwif->pdev;
 115         dma_addr_t dma_addr;
 116
 117         *vaddr = dma_alloc_coherent(&pdev->dev, page_sz, &dma_addr,
 118                                     GFP_KERNEL);
 119         if (!*vaddr) {
 120                 dev_err(&pdev->dev, "Failed to allocate dma for wqs page\n");
 121                 return -ENOMEM;
 122         }
 123
 124         *paddr = (u64)dma_addr;
 125
 126         /* use vzalloc for big mem */
 127         *shadow_vaddr = vzalloc(page_sz);
 128         if (!*shadow_vaddr)
 129                 goto err_shadow_vaddr;
 130
 131         return 0;
 132
 133 err_shadow_vaddr:
 134         dma_free_coherent(&pdev->dev, page_sz, *vaddr, dma_addr);
 135         return -ENOMEM;
 136 }
 137
 138 /**
 139  * wqs_allocate_page - allocate page for WQ set
 140  * @wqs: Work Queue Set
 141  * @page_idx: the page index of the page will be allocated
 142  *
 143  * Return 0 - Success, negative - Failure
 144  **/
 145 static int wqs_allocate_page(struct hinic_wqs *wqs, int page_idx)
 146 {
 147         return queue_alloc_page(wqs->hwif, &wqs->page_vaddr[page_idx],
 148                                 &wqs->page_paddr[page_idx],
 149                                 &wqs->shadow_page_vaddr[page_idx],
 150                                 WQS_PAGE_SIZE);
 151 }
 152
 153 /**
 154  * wqs_free_page - free page of WQ set
 155  * @wqs: Work Queue Set
 156  * @page_idx: the page index of the page will be freed
 157  **/
 158 static void wqs_free_page(struct hinic_wqs *wqs, int page_idx)
 159 {
 160         struct hinic_hwif *hwif = wqs->hwif;
 161         struct pci_dev *pdev = hwif->pdev;
 162
 163         dma_free_coherent(&pdev->dev, WQS_PAGE_SIZE,
 164                           wqs->page_vaddr[page_idx],
 165                           (dma_addr_t)wqs->page_paddr[page_idx]);
 166         vfree(wqs->shadow_page_vaddr[page_idx]);
 167 }
 168
 169 /**
 170  * cmdq_allocate_page - allocate page for cmdq
 171  * @cmdq_pages: the pages of the cmdq queue struct to hold the page
 172  *
 173  * Return 0 - Success, negative - Failure
 174  **/
 175 static int cmdq_allocate_page(struct hinic_cmdq_pages *cmdq_pages)
 176 {
 177         return queue_alloc_page(cmdq_pages->hwif, &cmdq_pages->page_vaddr,
 178                                 &cmdq_pages->page_paddr,
 179                                 &cmdq_pages->shadow_page_vaddr,
 180                                 CMDQ_PAGE_SIZE);
 181 }
 182
 183 /**
 184  * cmdq_free_page - free page from cmdq
 185  * @cmdq_pages: the pages of the cmdq queue struct that hold the page
 186  *
 187  * Return 0 - Success, negative - Failure
 188  **/
 189 static void cmdq_free_page(struct hinic_cmdq_pages *cmdq_pages)
 190 {
 191         struct hinic_hwif *hwif = cmdq_pages->hwif;
 192         struct pci_dev *pdev = hwif->pdev;
 193
 194         dma_free_coherent(&pdev->dev, CMDQ_PAGE_SIZE,
 195                           cmdq_pages->page_vaddr,
 196                           (dma_addr_t)cmdq_pages->page_paddr);
 197         vfree(cmdq_pages->shadow_page_vaddr);
 198 }
 199
 200 static int alloc_page_arrays(struct hinic_wqs *wqs)
 201 {
 202         struct hinic_hwif *hwif = wqs->hwif;
 203         struct pci_dev *pdev = hwif->pdev;
 204         size_t size;
 205
 206         size = wqs->num_pages * sizeof(*wqs->page_paddr);
 207         wqs->page_paddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 208         if (!wqs->page_paddr)
 209                 return -ENOMEM;
 210
 211         size = wqs->num_pages * sizeof(*wqs->page_vaddr);
 212         wqs->page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 213         if (!wqs->page_vaddr)
 214                 goto err_page_vaddr;
 215
 216         size = wqs->num_pages * sizeof(*wqs->shadow_page_vaddr);
 217         wqs->shadow_page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 218         if (!wqs->shadow_page_vaddr)
 219                 goto err_page_shadow_vaddr;
 220
 221         return 0;
 222
 223 err_page_shadow_vaddr:
 224         devm_kfree(&pdev->dev, wqs->page_vaddr);
 225
 226 err_page_vaddr:
 227         devm_kfree(&pdev->dev, wqs->page_paddr);
 228         return -ENOMEM;
 229 }
 230
 231 static void free_page_arrays(struct hinic_wqs *wqs)
 232 {
 233         struct hinic_hwif *hwif = wqs->hwif;
 234         struct pci_dev *pdev = hwif->pdev;
 235
 236         devm_kfree(&pdev->dev, wqs->shadow_page_vaddr);
 237         devm_kfree(&pdev->dev, wqs->page_vaddr);
 238         devm_kfree(&pdev->dev, wqs->page_paddr);
 239 }
 240
 241 static int wqs_next_block(struct hinic_wqs *wqs, int *page_idx,
 242                           int *block_idx)
 243 {
 244         int pos;
 245
 246         down(&wqs->alloc_blocks_lock);
 247
 248         wqs->num_free_blks--;
 249
 250         if (wqs->num_free_blks < 0) {
 251                 wqs->num_free_blks++;
 252                 up(&wqs->alloc_blocks_lock);
 253                 return -ENOMEM;
 254         }
 255
 256         pos = wqs->alloc_blk_pos++;
 257         pos &= WQS_MAX_NUM_BLOCKS - 1;
 258
 259         *page_idx = wqs->free_blocks[pos].page_idx;
 260         *block_idx = wqs->free_blocks[pos].block_idx;
 261
 262         wqs->free_blocks[pos].page_idx = -1;
 263         wqs->free_blocks[pos].block_idx = -1;
 264
 265         up(&wqs->alloc_blocks_lock);
 266         return 0;
 267 }
 268
 269 static void wqs_return_block(struct hinic_wqs *wqs, int page_idx,
 270                              int block_idx)
 271 {
 272         int pos;
 273
 274         down(&wqs->alloc_blocks_lock);
 275
 276         pos = wqs->return_blk_pos++;
 277         pos &= WQS_MAX_NUM_BLOCKS - 1;
 278
 279         wqs->free_blocks[pos].page_idx = page_idx;
 280         wqs->free_blocks[pos].block_idx = block_idx;
 281
 282         wqs->num_free_blks++;
 283
 284         up(&wqs->alloc_blocks_lock);
 285 }
 286
 287 static void init_wqs_blocks_arr(struct hinic_wqs *wqs)
 288 {
 289         int page_idx, blk_idx, pos = 0;
 290
 291         for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) {
 292                 for (blk_idx = 0; blk_idx < WQS_BLOCKS_PER_PAGE; blk_idx++) {
 293                         wqs->free_blocks[pos].page_idx = page_idx;
 294                         wqs->free_blocks[pos].block_idx = blk_idx;
 295                         pos++;
 296                 }
 297         }
 298
 299         wqs->alloc_blk_pos = 0;
 300         wqs->return_blk_pos = pos;
 301         wqs->num_free_blks = pos;
 302
 303         sema_init(&wqs->alloc_blocks_lock, 1);
 304 }
 305
 306 /**
 307  * hinic_wqs_alloc - allocate Work Queues set
 308  * @wqs: Work Queue Set
 309  * @max_wqs: maximum wqs to allocate
 310  * @hwif: HW interface for use for the allocation
 311  *
 312  * Return 0 - Success, negative - Failure
 313  **/
 314 int hinic_wqs_alloc(struct hinic_wqs *wqs, int max_wqs,
 315                     struct hinic_hwif *hwif)
 316 {
 317         struct pci_dev *pdev = hwif->pdev;
 318         int err, i, page_idx;
 319
 320         max_wqs = ALIGN(max_wqs, WQS_BLOCKS_PER_PAGE);
 321         if (max_wqs > WQS_MAX_NUM_BLOCKS)  {
 322                 dev_err(&pdev->dev, "Invalid max_wqs = %d\n", max_wqs);
 323                 return -EINVAL;
 324         }
 325
 326         wqs->hwif = hwif;
 327         wqs->num_pages = max_wqs / WQS_BLOCKS_PER_PAGE;
 328
 329         if (alloc_page_arrays(wqs)) {
 330                 dev_err(&pdev->dev,
 331                         "Failed to allocate mem for page addresses\n");
 332                 return -ENOMEM;
 333         }
 334
 335         for (page_idx = 0; page_idx < wqs->num_pages; page_idx++) {
 336                 err = wqs_allocate_page(wqs, page_idx);
 337                 if (err) {
 338                         dev_err(&pdev->dev, "Failed wq page allocation\n");
 339                         goto err_wq_allocate_page;
 340                 }
 341         }
 342
 343         wqs->free_blocks = devm_kzalloc(&pdev->dev, WQS_FREE_BLOCKS_SIZE(wqs),
 344                                         GFP_KERNEL);
 345         if (!wqs->free_blocks) {
 346                 err = -ENOMEM;
 347                 goto err_alloc_blocks;
 348         }
 349
 350         init_wqs_blocks_arr(wqs);
 351         return 0;
 352
 353 err_alloc_blocks:
 354 err_wq_allocate_page:
 355         for (i = 0; i < page_idx; i++)
 356                 wqs_free_page(wqs, i);
 357
 358         free_page_arrays(wqs);
 359         return err;
 360 }
 361
 362 /**
 363  * hinic_wqs_free - free Work Queues set
 364  * @wqs: Work Queue Set
 365  **/
 366 void hinic_wqs_free(struct hinic_wqs *wqs)
 367 {
 368         struct hinic_hwif *hwif = wqs->hwif;
 369         struct pci_dev *pdev = hwif->pdev;
 370         int page_idx;
 371
 372         devm_kfree(&pdev->dev, wqs->free_blocks);
 373
 374         for (page_idx = 0; page_idx < wqs->num_pages; page_idx++)
 375                 wqs_free_page(wqs, page_idx);
 376
 377         free_page_arrays(wqs);
 378 }
 379
 380 /**
 381  * alloc_wqes_shadow - allocate WQE shadows for WQ
 382  * @wq: WQ to allocate shadows for
 383  *
 384  * Return 0 - Success, negative - Failure
 385  **/
 386 static int alloc_wqes_shadow(struct hinic_wq *wq)
 387 {
 388         struct hinic_hwif *hwif = wq->hwif;
 389         struct pci_dev *pdev = hwif->pdev;
 390         size_t size;
 391
 392         size = wq->num_q_pages * wq->max_wqe_size;
 393         wq->shadow_wqe = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 394         if (!wq->shadow_wqe)
 395                 return -ENOMEM;
 396
 397         size = wq->num_q_pages * sizeof(wq->prod_idx);
 398         wq->shadow_idx = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
 399         if (!wq->shadow_idx)
 400                 goto err_shadow_idx;
 401
 402         return 0;
 403
 404 err_shadow_idx:
 405         devm_kfree(&pdev->dev, wq->shadow_wqe);
 406         return -ENOMEM;
 407 }
 408
 409 /**
 410  * free_wqes_shadow - free WQE shadows of WQ
 411  * @wq: WQ to free shadows from
 412  **/
 413 static void free_wqes_shadow(struct hinic_wq *wq)
 414 {
 415         struct hinic_hwif *hwif = wq->hwif;
 416         struct pci_dev *pdev = hwif->pdev;
 417
 418         devm_kfree(&pdev->dev, wq->shadow_idx);
 419         devm_kfree(&pdev->dev, wq->shadow_wqe);
 420 }
 421
 422 /**
 423  * free_wq_pages - free pages of WQ
 424  * @hwif: HW interface for releasing dma addresses
 425  * @wq: WQ to free pages from
 426  * @num_q_pages: number pages to free
 427  **/
 428 static void free_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif,
 429                           int num_q_pages)
 430 {
 431         struct pci_dev *pdev = hwif->pdev;
 432         int i;
 433
 434         for (i = 0; i < num_q_pages; i++) {
 435                 void **vaddr = &wq->shadow_block_vaddr[i];
 436                 u64 *paddr = &wq->block_vaddr[i];
 437                 dma_addr_t dma_addr;
 438
 439                 dma_addr = (dma_addr_t)be64_to_cpu(*paddr);
 440                 dma_free_coherent(&pdev->dev, wq->wq_page_size, *vaddr,
 441                                   dma_addr);
 442         }
 443
 444         free_wqes_shadow(wq);
 445 }
 446
 447 /**
 448  * alloc_wq_pages - alloc pages for WQ
 449  * @hwif: HW interface for allocating dma addresses
 450  * @wq: WQ to allocate pages for
 451  * @max_pages: maximum pages allowed
 452  *
 453  * Return 0 - Success, negative - Failure
 454  **/
 455 static int alloc_wq_pages(struct hinic_wq *wq, struct hinic_hwif *hwif,
 456                           int max_pages)
 457 {
 458         struct pci_dev *pdev = hwif->pdev;
 459         int i, err, num_q_pages;
 460
 461         num_q_pages = ALIGN(WQ_SIZE(wq), wq->wq_page_size) / wq->wq_page_size;
 462         if (num_q_pages > max_pages) {
 463                 dev_err(&pdev->dev, "Number wq pages exceeds the limit\n");
 464                 return -EINVAL;
 465         }
 466
 467         if (num_q_pages & (num_q_pages - 1)) {
 468                 dev_err(&pdev->dev, "Number wq pages must be power of 2\n");
 469                 return -EINVAL;
 470         }
 471
 472         wq->num_q_pages = num_q_pages;
 473
 474         err = alloc_wqes_shadow(wq);
 475         if (err) {
 476                 dev_err(&pdev->dev, "Failed to allocate wqe shadow\n");
 477                 return err;
 478         }
 479
 480         for (i = 0; i < num_q_pages; i++) {
 481                 void **vaddr = &wq->shadow_block_vaddr[i];
 482                 u64 *paddr = &wq->block_vaddr[i];
 483                 dma_addr_t dma_addr;
 484
 485                 *vaddr = dma_alloc_coherent(&pdev->dev, wq->wq_page_size,
 486                                             &dma_addr, GFP_KERNEL);
 487                 if (!*vaddr) {
 488                         dev_err(&pdev->dev, "Failed to allocate wq page\n");
 489                         goto err_alloc_wq_pages;
 490                 }
 491
 492                 /* HW uses Big Endian Format */
 493                 *paddr = cpu_to_be64(dma_addr);
 494         }
 495
 496         return 0;
 497
 498 err_alloc_wq_pages:
 499         free_wq_pages(wq, hwif, i);
 500         return -ENOMEM;
 501 }
 502
 503 /**
 504  * hinic_wq_allocate - Allocate the WQ resources from the WQS
 505  * @wqs: WQ set from which to allocate the WQ resources
 506  * @wq: WQ to allocate resources for it from the WQ set
 507  * @wqebb_size: Work Queue Block Byte Size
 508  * @wq_page_size: the page size in the Work Queue
 509  * @q_depth: number of wqebbs in WQ
 510  * @max_wqe_size: maximum WQE size that will be used in the WQ
 511  *
 512  * Return 0 - Success, negative - Failure
 513  **/
 514 int hinic_wq_allocate(struct hinic_wqs *wqs, struct hinic_wq *wq,
 515                       u16 wqebb_size, u16 wq_page_size, u16 q_depth,
 516                       u16 max_wqe_size)
 517 {
 518         struct hinic_hwif *hwif = wqs->hwif;
 519         struct pci_dev *pdev = hwif->pdev;
 520         u16 num_wqebbs_per_page;
 521         u16 wqebb_size_shift;
 522         int err;
 523
 524         if (!is_power_of_2(wqebb_size)) {
 525                 dev_err(&pdev->dev, "wqebb_size must be power of 2\n");
 526                 return -EINVAL;
 527         }
 528
 529         if (wq_page_size == 0) {
 530                 dev_err(&pdev->dev, "wq_page_size must be > 0\n");
 531                 return -EINVAL;
 532         }
 533
 534         if (q_depth & (q_depth - 1)) {
 535                 dev_err(&pdev->dev, "WQ q_depth must be power of 2\n");
 536                 return -EINVAL;
 537         }
 538
 539         wqebb_size_shift = ilog2(wqebb_size);
 540         num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size)
 541                                 >> wqebb_size_shift;
 542
 543         if (!is_power_of_2(num_wqebbs_per_page)) {
 544                 dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n");
 545                 return -EINVAL;
 546         }
 547
 548         wq->hwif = hwif;
 549
 550         err = wqs_next_block(wqs, &wq->page_idx, &wq->block_idx);
 551         if (err) {
 552                 dev_err(&pdev->dev, "Failed to get free wqs next block\n");
 553                 return err;
 554         }
 555
 556         wq->wqebb_size = wqebb_size;
 557         wq->wq_page_size = wq_page_size;
 558         wq->q_depth = q_depth;
 559         wq->max_wqe_size = max_wqe_size;
 560         wq->num_wqebbs_per_page = num_wqebbs_per_page;
 561         wq->wqebbs_per_page_shift = ilog2(num_wqebbs_per_page);
 562         wq->wqebb_size_shift = wqebb_size_shift;
 563         wq->block_vaddr = WQ_BASE_VADDR(wqs, wq);
 564         wq->shadow_block_vaddr = WQ_BASE_ADDR(wqs, wq);
 565         wq->block_paddr = WQ_BASE_PADDR(wqs, wq);
 566
 567         err = alloc_wq_pages(wq, wqs->hwif, WQ_MAX_PAGES);
 568         if (err) {
 569                 dev_err(&pdev->dev, "Failed to allocate wq pages\n");
 570                 goto err_alloc_wq_pages;
 571         }
 572
 573         atomic_set(&wq->cons_idx, 0);
 574         atomic_set(&wq->prod_idx, 0);
 575         atomic_set(&wq->delta, q_depth);
 576         wq->mask = q_depth - 1;
 577
 578         return 0;
 579
 580 err_alloc_wq_pages:
 581         wqs_return_block(wqs, wq->page_idx, wq->block_idx);
 582         return err;
 583 }
 584
 585 /**
 586  * hinic_wq_free - Free the WQ resources to the WQS
 587  * @wqs: WQ set to free the WQ resources to it
 588  * @wq: WQ to free its resources to the WQ set resources
 589  **/
 590 void hinic_wq_free(struct hinic_wqs *wqs, struct hinic_wq *wq)
 591 {
 592         free_wq_pages(wq, wqs->hwif, wq->num_q_pages);
 593
 594         wqs_return_block(wqs, wq->page_idx, wq->block_idx);
 595 }
 596
 597 /**
 598  * hinic_wqs_cmdq_alloc - Allocate wqs for cmdqs
 599  * @cmdq_pages: will hold the pages of the cmdq
 600  * @wq: returned wqs
 601  * @hwif: HW interface
 602  * @cmdq_blocks: number of cmdq blocks/wq to allocate
 603  * @wqebb_size: Work Queue Block Byte Size
 604  * @wq_page_size: the page size in the Work Queue
 605  * @q_depth: number of wqebbs in WQ
 606  * @max_wqe_size: maximum WQE size that will be used in the WQ
 607  *
 608  * Return 0 - Success, negative - Failure
 609  **/
 610 int hinic_wqs_cmdq_alloc(struct hinic_cmdq_pages *cmdq_pages,
 611                          struct hinic_wq *wq, struct hinic_hwif *hwif,
 612                          int cmdq_blocks, u16 wqebb_size, u16 wq_page_size,
 613                          u16 q_depth, u16 max_wqe_size)
 614 {
 615         struct pci_dev *pdev = hwif->pdev;
 616         u16 num_wqebbs_per_page_shift;
 617         u16 num_wqebbs_per_page;
 618         u16 wqebb_size_shift;
 619         int i, j, err = -ENOMEM;
 620
 621         if (!is_power_of_2(wqebb_size)) {
 622                 dev_err(&pdev->dev, "wqebb_size must be power of 2\n");
 623                 return -EINVAL;
 624         }
 625
 626         if (wq_page_size == 0) {
 627                 dev_err(&pdev->dev, "wq_page_size must be > 0\n");
 628                 return -EINVAL;
 629         }
 630
 631         if (q_depth & (q_depth - 1)) {
 632                 dev_err(&pdev->dev, "WQ q_depth must be power of 2\n");
 633                 return -EINVAL;
 634         }
 635
 636         wqebb_size_shift = ilog2(wqebb_size);
 637         num_wqebbs_per_page = ALIGN(wq_page_size, wqebb_size)
 638                                 >> wqebb_size_shift;
 639
 640         if (!is_power_of_2(num_wqebbs_per_page)) {
 641                 dev_err(&pdev->dev, "num wqebbs per page must be power of 2\n");
 642                 return -EINVAL;
 643         }
 644
 645         cmdq_pages->hwif = hwif;
 646
 647         err = cmdq_allocate_page(cmdq_pages);
 648         if (err) {
 649                 dev_err(&pdev->dev, "Failed to allocate CMDQ page\n");
 650                 return err;
 651         }
 652         num_wqebbs_per_page_shift = ilog2(num_wqebbs_per_page);
 653
 654         for (i = 0; i < cmdq_blocks; i++) {
 655                 wq[i].hwif = hwif;
 656                 wq[i].page_idx = 0;
 657                 wq[i].block_idx = i;
 658
 659                 wq[i].wqebb_size = wqebb_size;
 660                 wq[i].wq_page_size = wq_page_size;
 661                 wq[i].q_depth = q_depth;
 662                 wq[i].max_wqe_size = max_wqe_size;
 663                 wq[i].num_wqebbs_per_page = num_wqebbs_per_page;
 664                 wq[i].wqebbs_per_page_shift = num_wqebbs_per_page_shift;
 665                 wq[i].wqebb_size_shift = wqebb_size_shift;
 666                 wq[i].block_vaddr = CMDQ_BASE_VADDR(cmdq_pages, &wq[i]);
 667                 wq[i].shadow_block_vaddr = CMDQ_BASE_ADDR(cmdq_pages, &wq[i]);
 668                 wq[i].block_paddr = CMDQ_BASE_PADDR(cmdq_pages, &wq[i]);
 669
 670                 err = alloc_wq_pages(&wq[i], cmdq_pages->hwif,
 671                                      CMDQ_WQ_MAX_PAGES);
 672                 if (err) {
 673                         dev_err(&pdev->dev, "Failed to alloc CMDQ blocks\n");
 674                         goto err_cmdq_block;
 675                 }
 676
 677                 atomic_set(&wq[i].cons_idx, 0);
 678                 atomic_set(&wq[i].prod_idx, 0);
 679                 atomic_set(&wq[i].delta, q_depth);
 680                 wq[i].mask = q_depth - 1;
 681         }
 682
 683         return 0;
 684
 685 err_cmdq_block:
 686         for (j = 0; j < i; j++)
 687                 free_wq_pages(&wq[j], cmdq_pages->hwif, wq[j].num_q_pages);
 688
 689         cmdq_free_page(cmdq_pages);
 690         return err;
 691 }
 692
 693 /**
 694  * hinic_wqs_cmdq_free - Free wqs from cmdqs
 695  * @cmdq_pages: hold the pages of the cmdq
 696  * @wq: wqs to free
 697  * @cmdq_blocks: number of wqs to free
 698  **/
 699 void hinic_wqs_cmdq_free(struct hinic_cmdq_pages *cmdq_pages,
 700                          struct hinic_wq *wq, int cmdq_blocks)
 701 {
 702         int i;
 703
 704         for (i = 0; i < cmdq_blocks; i++)
 705                 free_wq_pages(&wq[i], cmdq_pages->hwif, wq[i].num_q_pages);
 706
 707         cmdq_free_page(cmdq_pages);
 708 }
 709
 710 static void copy_wqe_to_shadow(struct hinic_wq *wq, void *shadow_addr,
 711                                int num_wqebbs, u16 idx)
 712 {
 713         void *wqebb_addr;
 714         int i;
 715
 716         for (i = 0; i < num_wqebbs; i++, idx++) {
 717                 idx = MASKED_WQE_IDX(wq, idx);
 718                 wqebb_addr = WQ_PAGE_ADDR(wq, idx) +
 719                              WQE_PAGE_OFF(wq, idx);
 720
 721                 memcpy(shadow_addr, wqebb_addr, wq->wqebb_size);
 722
 723                 shadow_addr += wq->wqebb_size;
 724         }
 725 }
 726
 727 static void copy_wqe_from_shadow(struct hinic_wq *wq, void *shadow_addr,
 728                                  int num_wqebbs, u16 idx)
 729 {
 730         void *wqebb_addr;
 731         int i;
 732
 733         for (i = 0; i < num_wqebbs; i++, idx++) {
 734                 idx = MASKED_WQE_IDX(wq, idx);
 735                 wqebb_addr = WQ_PAGE_ADDR(wq, idx) +
 736                              WQE_PAGE_OFF(wq, idx);
 737
 738                 memcpy(wqebb_addr, shadow_addr, wq->wqebb_size);
 739                 shadow_addr += wq->wqebb_size;
 740         }
 741 }
 742
 743 /**
 744  * hinic_get_wqe - get wqe ptr in the current pi and update the pi
 745  * @wq: wq to get wqe from
 746  * @wqe_size: wqe size
 747  * @prod_idx: returned pi
 748  *
 749  * Return wqe pointer
 750  **/
 751 struct hinic_hw_wqe *hinic_get_wqe(struct hinic_wq *wq, unsigned int wqe_size,
 752                                    u16 *prod_idx)
 753 {
 754         int curr_pg, end_pg, num_wqebbs;
 755         u16 curr_prod_idx, end_prod_idx;
 756
 757         *prod_idx = MASKED_WQE_IDX(wq, atomic_read(&wq->prod_idx));
 758
 759         num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) >> wq->wqebb_size_shift;
 760
 761         if (atomic_sub_return(num_wqebbs, &wq->delta) <= 0) {
 762                 atomic_add(num_wqebbs, &wq->delta);
 763                 return ERR_PTR(-EBUSY);
 764         }
 765
 766         end_prod_idx = atomic_add_return(num_wqebbs, &wq->prod_idx);
 767
 768         end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx);
 769         curr_prod_idx = end_prod_idx - num_wqebbs;
 770         curr_prod_idx = MASKED_WQE_IDX(wq, curr_prod_idx);
 771
 772         /* end prod index points to the next wqebb, therefore minus 1 */
 773         end_prod_idx = MASKED_WQE_IDX(wq, end_prod_idx - 1);
 774
 775         curr_pg = WQE_PAGE_NUM(wq, curr_prod_idx);
 776         end_pg = WQE_PAGE_NUM(wq, end_prod_idx);
 777
 778         *prod_idx = curr_prod_idx;
 779
 780         if (curr_pg != end_pg) {
 781                 void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
 782
 783                 copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *prod_idx);
 784
 785                 wq->shadow_idx[curr_pg] = *prod_idx;
 786                 return shadow_addr;
 787         }
 788
 789         return WQ_PAGE_ADDR(wq, *prod_idx) + WQE_PAGE_OFF(wq, *prod_idx);
 790 }
 791
 792 /**
 793  * hinic_return_wqe - return the wqe when transmit failed
 794  * @wq: wq to return wqe
 795  * @wqe_size: wqe size
 796  **/
 797 void hinic_return_wqe(struct hinic_wq *wq, unsigned int wqe_size)
 798 {
 799         int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
 800
 801         atomic_sub(num_wqebbs, &wq->prod_idx);
 802
 803         atomic_add(num_wqebbs, &wq->delta);
 804 }
 805
 806 /**
 807  * hinic_put_wqe - return the wqe place to use for a new wqe
 808  * @wq: wq to return wqe
 809  * @wqe_size: wqe size
 810  **/
 811 void hinic_put_wqe(struct hinic_wq *wq, unsigned int wqe_size)
 812 {
 813         int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size)
 814                         >> wq->wqebb_size_shift;
 815
 816         atomic_add(num_wqebbs, &wq->cons_idx);
 817
 818         atomic_add(num_wqebbs, &wq->delta);
 819 }
 820
 821 /**
 822  * hinic_read_wqe - read wqe ptr in the current ci
 823  * @wq: wq to get read from
 824  * @wqe_size: wqe size
 825  * @cons_idx: returned ci
 826  *
 827  * Return wqe pointer
 828  **/
 829 struct hinic_hw_wqe *hinic_read_wqe(struct hinic_wq *wq, unsigned int wqe_size,
 830                                     u16 *cons_idx)
 831 {
 832         int num_wqebbs = ALIGN(wqe_size, wq->wqebb_size)
 833                         >> wq->wqebb_size_shift;
 834         u16 curr_cons_idx, end_cons_idx;
 835         int curr_pg, end_pg;
 836
 837         if ((atomic_read(&wq->delta) + num_wqebbs) > wq->q_depth)
 838                 return ERR_PTR(-EBUSY);
 839
 840         curr_cons_idx = atomic_read(&wq->cons_idx);
 841
 842         curr_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx);
 843         end_cons_idx = MASKED_WQE_IDX(wq, curr_cons_idx + num_wqebbs - 1);
 844
 845         curr_pg = WQE_PAGE_NUM(wq, curr_cons_idx);
 846         end_pg = WQE_PAGE_NUM(wq, end_cons_idx);
 847
 848         *cons_idx = curr_cons_idx;
 849
 850         if (curr_pg != end_pg) {
 851                 void *shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
 852
 853                 copy_wqe_to_shadow(wq, shadow_addr, num_wqebbs, *cons_idx);
 854                 return shadow_addr;
 855         }
 856
 857         return WQ_PAGE_ADDR(wq, *cons_idx) + WQE_PAGE_OFF(wq, *cons_idx);
 858 }
 859
 860 /**
 861  * hinic_read_wqe_direct - read wqe directly from ci position
 862  * @wq: wq
 863  * @cons_idx: ci position
 864  *
 865  * Return wqe
 866  **/
 867 struct hinic_hw_wqe *hinic_read_wqe_direct(struct hinic_wq *wq, u16 cons_idx)
 868 {
 869         return WQ_PAGE_ADDR(wq, cons_idx) + WQE_PAGE_OFF(wq, cons_idx);
 870 }
 871
 872 /**
 873  * wqe_shadow - check if a wqe is shadow
 874  * @wq: wq of the wqe
 875  * @wqe: the wqe for shadow checking
 876  *
 877  * Return true - shadow, false - Not shadow
 878  **/
 879 static inline bool wqe_shadow(struct hinic_wq *wq, struct hinic_hw_wqe *wqe)
 880 {
 881         size_t wqe_shadow_size = wq->num_q_pages * wq->max_wqe_size;
 882
 883         return WQE_IN_RANGE(wqe, wq->shadow_wqe,
 884                             &wq->shadow_wqe[wqe_shadow_size]);
 885 }
 886
 887 /**
 888  * hinic_write_wqe - write the wqe to the wq
 889  * @wq: wq to write wqe to
 890  * @wqe: wqe to write
 891  * @wqe_size: wqe size
 892  **/
 893 void hinic_write_wqe(struct hinic_wq *wq, struct hinic_hw_wqe *wqe,
 894                      unsigned int wqe_size)
 895 {
 896         int curr_pg, num_wqebbs;
 897         void *shadow_addr;
 898         u16 prod_idx;
 899
 900         if (wqe_shadow(wq, wqe)) {
 901                 curr_pg = WQE_SHADOW_PAGE(wq, wqe);
 902
 903                 prod_idx = wq->shadow_idx[curr_pg];
 904                 num_wqebbs = ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
 905                 shadow_addr = &wq->shadow_wqe[curr_pg * wq->max_wqe_size];
 906
 907                 copy_wqe_from_shadow(wq, shadow_addr, num_wqebbs, prod_idx);
 908         }
 909 }