drivers/i3c/master/mipi-i3c-hci/dma.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /*
   3  * Copyright (c) 2020, MIPI Alliance, Inc.
   4  *
   5  * Author: Nicolas Pitre <npitre@baylibre.com>
   6  *
   7  * Note: The I3C HCI v2.0 spec is still in flux. The IBI support is based on
   8  * v1.x of the spec and v2.0 will likely be split out.
   9  */
  10
  11 #include <linux/bitfield.h>
  12 #include <linux/device.h>
  13 #include <linux/dma-mapping.h>
  14 #include <linux/errno.h>
  15 #include <linux/i3c/master.h>
  16 #include <linux/io.h>
  17
  18 #include "hci.h"
  19 #include "cmd.h"
  20 #include "ibi.h"
  21
  22
  23 /*
  24  * Software Parameter Values (somewhat arb itrary for now).
  25  * Some of them could be determined at run time eventually.
  26  */
  27
  28 #define XFER_RINGS                      1       /* max: 8 */
  29 #define XFER_RING_ENTRIES               16      /* max: 255 */
  30
  31 #define IBI_RINGS                       1       /* max: 8 */
  32 #define IBI_STATUS_RING_ENTRIES         32      /* max: 255 */
  33 #define IBI_CHUNK_CACHELINES            1       /* max: 256 bytes equivalent */
  34 #define IBI_CHUNK_POOL_SIZE             128     /* max: 1023 */
  35
  36 /*
  37  * Ring Header Preamble
  38  */
  39
  40 #define rhs_reg_read(r)         readl(hci->RHS_regs + (RHS_##r))
  41 #define rhs_reg_write(r, v)     writel(v, hci->RHS_regs + (RHS_##r))
  42
  43 #define RHS_CONTROL                     0x00
  44 #define PREAMBLE_SIZE                   GENMASK(31, 24) /* Preamble Section Size */
  45 #define HEADER_SIZE                     GENMASK(23, 16) /* Ring Header Size */
  46 #define MAX_HEADER_COUNT_CAP            GENMASK(7, 4) /* HC Max Header Count */
  47 #define MAX_HEADER_COUNT                GENMASK(3, 0) /* Driver Max Header Count */
  48
  49 #define RHS_RHn_OFFSET(n)               (0x04 + (n)*4)
  50
  51 /*
  52  * Ring Header (Per-Ring Bundle)
  53  */
  54
  55 #define rh_reg_read(r)          readl(rh->regs + (RH_##r))
  56 #define rh_reg_write(r, v)      writel(v, rh->regs + (RH_##r))
  57
  58 #define RH_CR_SETUP                     0x00    /* Command/Response Ring */
  59 #define CR_XFER_STRUCT_SIZE             GENMASK(31, 24)
  60 #define CR_RESP_STRUCT_SIZE             GENMASK(23, 16)
  61 #define CR_RING_SIZE                    GENMASK(8, 0)
  62
  63 #define RH_IBI_SETUP                    0x04
  64 #define IBI_STATUS_STRUCT_SIZE          GENMASK(31, 24)
  65 #define IBI_STATUS_RING_SIZE            GENMASK(23, 16)
  66 #define IBI_DATA_CHUNK_SIZE             GENMASK(12, 10)
  67 #define IBI_DATA_CHUNK_COUNT            GENMASK(9, 0)
  68
  69 #define RH_CHUNK_CONTROL                        0x08
  70
  71 #define RH_INTR_STATUS                  0x10
  72 #define RH_INTR_STATUS_ENABLE           0x14
  73 #define RH_INTR_SIGNAL_ENABLE           0x18
  74 #define RH_INTR_FORCE                   0x1c
  75 #define INTR_IBI_READY                  BIT(12)
  76 #define INTR_TRANSFER_COMPLETION        BIT(11)
  77 #define INTR_RING_OP                    BIT(10)
  78 #define INTR_TRANSFER_ERR               BIT(9)
  79 #define INTR_WARN_INS_STOP_MODE         BIT(7)
  80 #define INTR_IBI_RING_FULL              BIT(6)
  81 #define INTR_TRANSFER_ABORT             BIT(5)
  82
  83 #define RH_RING_STATUS                  0x20
  84 #define RING_STATUS_LOCKED              BIT(3)
  85 #define RING_STATUS_ABORTED             BIT(2)
  86 #define RING_STATUS_RUNNING             BIT(1)
  87 #define RING_STATUS_ENABLED             BIT(0)
  88
  89 #define RH_RING_CONTROL                 0x24
  90 #define RING_CTRL_ABORT                 BIT(2)
  91 #define RING_CTRL_RUN_STOP              BIT(1)
  92 #define RING_CTRL_ENABLE                BIT(0)
  93
  94 #define RH_RING_OPERATION1              0x28
  95 #define RING_OP1_IBI_DEQ_PTR            GENMASK(23, 16)
  96 #define RING_OP1_CR_SW_DEQ_PTR          GENMASK(15, 8)
  97 #define RING_OP1_CR_ENQ_PTR             GENMASK(7, 0)
  98
  99 #define RH_RING_OPERATION2              0x2c
 100 #define RING_OP2_IBI_ENQ_PTR            GENMASK(23, 16)
 101 #define RING_OP2_CR_DEQ_PTR             GENMASK(7, 0)
 102
 103 #define RH_CMD_RING_BASE_LO             0x30
 104 #define RH_CMD_RING_BASE_HI             0x34
 105 #define RH_RESP_RING_BASE_LO            0x38
 106 #define RH_RESP_RING_BASE_HI            0x3c
 107 #define RH_IBI_STATUS_RING_BASE_LO      0x40
 108 #define RH_IBI_STATUS_RING_BASE_HI      0x44
 109 #define RH_IBI_DATA_RING_BASE_LO        0x48
 110 #define RH_IBI_DATA_RING_BASE_HI        0x4c
 111
 112 #define RH_CMD_RING_SG                  0x50    /* Ring Scatter Gather Support */
 113 #define RH_RESP_RING_SG                 0x54
 114 #define RH_IBI_STATUS_RING_SG           0x58
 115 #define RH_IBI_DATA_RING_SG             0x5c
 116 #define RING_SG_BLP                     BIT(31) /* Buffer Vs. List Pointer */
 117 #define RING_SG_LIST_SIZE               GENMASK(15, 0)
 118
 119 /*
 120  * Data Buffer Descriptor (in memory)
 121  */
 122
 123 #define DATA_BUF_BLP                    BIT(31) /* Buffer Vs. List Pointer */
 124 #define DATA_BUF_IOC                    BIT(30) /* Interrupt on Completion */
 125 #define DATA_BUF_BLOCK_SIZE             GENMASK(15, 0)
 126
 127
 128 struct hci_rh_data {
 129         void __iomem *regs;
 130         void *xfer, *resp, *ibi_status, *ibi_data;
 131         dma_addr_t xfer_dma, resp_dma, ibi_status_dma, ibi_data_dma;
 132         unsigned int xfer_entries, ibi_status_entries, ibi_chunks_total;
 133         unsigned int xfer_struct_sz, resp_struct_sz, ibi_status_sz, ibi_chunk_sz;
 134         unsigned int done_ptr, ibi_chunk_ptr;
 135         struct hci_xfer **src_xfers;
 136         spinlock_t lock;
 137         struct completion op_done;
 138 };
 139
 140 struct hci_rings_data {
 141         unsigned int total;
 142         struct hci_rh_data headers[] __counted_by(total);
 143 };
 144
 145 struct hci_dma_dev_ibi_data {
 146         struct i3c_generic_ibi_pool *pool;
 147         unsigned int max_len;
 148 };
 149
 150 static inline u32 lo32(dma_addr_t physaddr)
 151 {
 152         return physaddr;
 153 }
 154
 155 static inline u32 hi32(dma_addr_t physaddr)
 156 {
 157         /* trickery to avoid compiler warnings on 32-bit build targets */
 158         if (sizeof(dma_addr_t) > 4) {
 159                 u64 hi = physaddr;
 160                 return hi >> 32;
 161         }
 162         return 0;
 163 }
 164
 165 static void hci_dma_cleanup(struct i3c_hci *hci)
 166 {
 167         struct hci_rings_data *rings = hci->io_data;
 168         struct hci_rh_data *rh;
 169         unsigned int i;
 170
 171         if (!rings)
 172                 return;
 173
 174         for (i = 0; i < rings->total; i++) {
 175                 rh = &rings->headers[i];
 176
 177                 rh_reg_write(RING_CONTROL, 0);
 178                 rh_reg_write(CR_SETUP, 0);
 179                 rh_reg_write(IBI_SETUP, 0);
 180                 rh_reg_write(INTR_SIGNAL_ENABLE, 0);
 181
 182                 if (rh->xfer)
 183                         dma_free_coherent(&hci->master.dev,
 184                                           rh->xfer_struct_sz * rh->xfer_entries,
 185                                           rh->xfer, rh->xfer_dma);
 186                 if (rh->resp)
 187                         dma_free_coherent(&hci->master.dev,
 188                                           rh->resp_struct_sz * rh->xfer_entries,
 189                                           rh->resp, rh->resp_dma);
 190                 kfree(rh->src_xfers);
 191                 if (rh->ibi_status)
 192                         dma_free_coherent(&hci->master.dev,
 193                                           rh->ibi_status_sz * rh->ibi_status_entries,
 194                                           rh->ibi_status, rh->ibi_status_dma);
 195                 if (rh->ibi_data_dma)
 196                         dma_unmap_single(&hci->master.dev, rh->ibi_data_dma,
 197                                          rh->ibi_chunk_sz * rh->ibi_chunks_total,
 198                                          DMA_FROM_DEVICE);
 199                 kfree(rh->ibi_data);
 200         }
 201
 202         rhs_reg_write(CONTROL, 0);
 203
 204         kfree(rings);
 205         hci->io_data = NULL;
 206 }
 207
 208 static int hci_dma_init(struct i3c_hci *hci)
 209 {
 210         struct hci_rings_data *rings;
 211         struct hci_rh_data *rh;
 212         u32 regval;
 213         unsigned int i, nr_rings, xfers_sz, resps_sz;
 214         unsigned int ibi_status_ring_sz, ibi_data_ring_sz;
 215         int ret;
 216
 217         regval = rhs_reg_read(CONTROL);
 218         nr_rings = FIELD_GET(MAX_HEADER_COUNT_CAP, regval);
 219         dev_info(&hci->master.dev, "%d DMA rings available\n", nr_rings);
 220         if (unlikely(nr_rings > 8)) {
 221                 dev_err(&hci->master.dev, "number of rings should be <= 8\n");
 222                 nr_rings = 8;
 223         }
 224         if (nr_rings > XFER_RINGS)
 225                 nr_rings = XFER_RINGS;
 226         rings = kzalloc(struct_size(rings, headers, nr_rings), GFP_KERNEL);
 227         if (!rings)
 228                 return -ENOMEM;
 229         hci->io_data = rings;
 230         rings->total = nr_rings;
 231
 232         regval = FIELD_PREP(MAX_HEADER_COUNT, rings->total);
 233         rhs_reg_write(CONTROL, regval);
 234
 235         for (i = 0; i < rings->total; i++) {
 236                 u32 offset = rhs_reg_read(RHn_OFFSET(i));
 237
 238                 dev_info(&hci->master.dev, "Ring %d at offset %#x\n", i, offset);
 239                 ret = -EINVAL;
 240                 if (!offset)
 241                         goto err_out;
 242                 rh = &rings->headers[i];
 243                 rh->regs = hci->base_regs + offset;
 244                 spin_lock_init(&rh->lock);
 245                 init_completion(&rh->op_done);
 246
 247                 rh->xfer_entries = XFER_RING_ENTRIES;
 248
 249                 regval = rh_reg_read(CR_SETUP);
 250                 rh->xfer_struct_sz = FIELD_GET(CR_XFER_STRUCT_SIZE, regval);
 251                 rh->resp_struct_sz = FIELD_GET(CR_RESP_STRUCT_SIZE, regval);
 252                 DBG("xfer_struct_sz = %d, resp_struct_sz = %d",
 253                     rh->xfer_struct_sz, rh->resp_struct_sz);
 254                 xfers_sz = rh->xfer_struct_sz * rh->xfer_entries;
 255                 resps_sz = rh->resp_struct_sz * rh->xfer_entries;
 256
 257                 rh->xfer = dma_alloc_coherent(&hci->master.dev, xfers_sz,
 258                                               &rh->xfer_dma, GFP_KERNEL);
 259                 rh->resp = dma_alloc_coherent(&hci->master.dev, resps_sz,
 260                                               &rh->resp_dma, GFP_KERNEL);
 261                 rh->src_xfers =
 262                         kmalloc_array(rh->xfer_entries, sizeof(*rh->src_xfers),
 263                                       GFP_KERNEL);
 264                 ret = -ENOMEM;
 265                 if (!rh->xfer || !rh->resp || !rh->src_xfers)
 266                         goto err_out;
 267
 268                 rh_reg_write(CMD_RING_BASE_LO, lo32(rh->xfer_dma));
 269                 rh_reg_write(CMD_RING_BASE_HI, hi32(rh->xfer_dma));
 270                 rh_reg_write(RESP_RING_BASE_LO, lo32(rh->resp_dma));
 271                 rh_reg_write(RESP_RING_BASE_HI, hi32(rh->resp_dma));
 272
 273                 regval = FIELD_PREP(CR_RING_SIZE, rh->xfer_entries);
 274                 rh_reg_write(CR_SETUP, regval);
 275
 276                 rh_reg_write(INTR_STATUS_ENABLE, 0xffffffff);
 277                 rh_reg_write(INTR_SIGNAL_ENABLE, INTR_IBI_READY |
 278                                                  INTR_TRANSFER_COMPLETION |
 279                                                  INTR_RING_OP |
 280                                                  INTR_TRANSFER_ERR |
 281                                                  INTR_WARN_INS_STOP_MODE |
 282                                                  INTR_IBI_RING_FULL |
 283                                                  INTR_TRANSFER_ABORT);
 284
 285                 /* IBIs */
 286
 287                 if (i >= IBI_RINGS)
 288                         goto ring_ready;
 289
 290                 regval = rh_reg_read(IBI_SETUP);
 291                 rh->ibi_status_sz = FIELD_GET(IBI_STATUS_STRUCT_SIZE, regval);
 292                 rh->ibi_status_entries = IBI_STATUS_RING_ENTRIES;
 293                 rh->ibi_chunks_total = IBI_CHUNK_POOL_SIZE;
 294
 295                 rh->ibi_chunk_sz = dma_get_cache_alignment();
 296                 rh->ibi_chunk_sz *= IBI_CHUNK_CACHELINES;
 297                 BUG_ON(rh->ibi_chunk_sz > 256);
 298
 299                 ibi_status_ring_sz = rh->ibi_status_sz * rh->ibi_status_entries;
 300                 ibi_data_ring_sz = rh->ibi_chunk_sz * rh->ibi_chunks_total;
 301
 302                 rh->ibi_status =
 303                         dma_alloc_coherent(&hci->master.dev, ibi_status_ring_sz,
 304                                            &rh->ibi_status_dma, GFP_KERNEL);
 305                 rh->ibi_data = kmalloc(ibi_data_ring_sz, GFP_KERNEL);
 306                 ret = -ENOMEM;
 307                 if (!rh->ibi_status || !rh->ibi_data)
 308                         goto err_out;
 309                 rh->ibi_data_dma =
 310                         dma_map_single(&hci->master.dev, rh->ibi_data,
 311                                        ibi_data_ring_sz, DMA_FROM_DEVICE);
 312                 if (dma_mapping_error(&hci->master.dev, rh->ibi_data_dma)) {
 313                         rh->ibi_data_dma = 0;
 314                         ret = -ENOMEM;
 315                         goto err_out;
 316                 }
 317
 318                 regval = FIELD_PREP(IBI_STATUS_RING_SIZE,
 319                                     rh->ibi_status_entries) |
 320                          FIELD_PREP(IBI_DATA_CHUNK_SIZE,
 321                                     ilog2(rh->ibi_chunk_sz) - 2) |
 322                          FIELD_PREP(IBI_DATA_CHUNK_COUNT,
 323                                     rh->ibi_chunks_total);
 324                 rh_reg_write(IBI_SETUP, regval);
 325
 326                 regval = rh_reg_read(INTR_SIGNAL_ENABLE);
 327                 regval |= INTR_IBI_READY;
 328                 rh_reg_write(INTR_SIGNAL_ENABLE, regval);
 329
 330 ring_ready:
 331                 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE |
 332                                            RING_CTRL_RUN_STOP);
 333         }
 334
 335         return 0;
 336
 337 err_out:
 338         hci_dma_cleanup(hci);
 339         return ret;
 340 }
 341
 342 static void hci_dma_unmap_xfer(struct i3c_hci *hci,
 343                                struct hci_xfer *xfer_list, unsigned int n)
 344 {
 345         struct hci_xfer *xfer;
 346         unsigned int i;
 347
 348         for (i = 0; i < n; i++) {
 349                 xfer = xfer_list + i;
 350                 if (!xfer->data)
 351                         continue;
 352                 dma_unmap_single(&hci->master.dev,
 353                                  xfer->data_dma, xfer->data_len,
 354                                  xfer->rnw ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
 355         }
 356 }
 357
 358 static int hci_dma_queue_xfer(struct i3c_hci *hci,
 359                               struct hci_xfer *xfer_list, int n)
 360 {
 361         struct hci_rings_data *rings = hci->io_data;
 362         struct hci_rh_data *rh;
 363         unsigned int i, ring, enqueue_ptr;
 364         u32 op1_val, op2_val;
 365
 366         /* For now we only use ring 0 */
 367         ring = 0;
 368         rh = &rings->headers[ring];
 369
 370         op1_val = rh_reg_read(RING_OPERATION1);
 371         enqueue_ptr = FIELD_GET(RING_OP1_CR_ENQ_PTR, op1_val);
 372         for (i = 0; i < n; i++) {
 373                 struct hci_xfer *xfer = xfer_list + i;
 374                 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * enqueue_ptr;
 375
 376                 /* store cmd descriptor */
 377                 *ring_data++ = xfer->cmd_desc[0];
 378                 *ring_data++ = xfer->cmd_desc[1];
 379                 if (hci->cmd == &mipi_i3c_hci_cmd_v2) {
 380                         *ring_data++ = xfer->cmd_desc[2];
 381                         *ring_data++ = xfer->cmd_desc[3];
 382                 }
 383
 384                 /* first word of Data Buffer Descriptor Structure */
 385                 if (!xfer->data)
 386                         xfer->data_len = 0;
 387                 *ring_data++ =
 388                         FIELD_PREP(DATA_BUF_BLOCK_SIZE, xfer->data_len) |
 389                         ((i == n - 1) ? DATA_BUF_IOC : 0);
 390
 391                 /* 2nd and 3rd words of Data Buffer Descriptor Structure */
 392                 if (xfer->data) {
 393                         xfer->data_dma =
 394                                 dma_map_single(&hci->master.dev,
 395                                                xfer->data,
 396                                                xfer->data_len,
 397                                                xfer->rnw ?
 398                                                   DMA_FROM_DEVICE :
 399                                                   DMA_TO_DEVICE);
 400                         if (dma_mapping_error(&hci->master.dev,
 401                                               xfer->data_dma)) {
 402                                 hci_dma_unmap_xfer(hci, xfer_list, i);
 403                                 return -ENOMEM;
 404                         }
 405                         *ring_data++ = lo32(xfer->data_dma);
 406                         *ring_data++ = hi32(xfer->data_dma);
 407                 } else {
 408                         *ring_data++ = 0;
 409                         *ring_data++ = 0;
 410                 }
 411
 412                 /* remember corresponding xfer struct */
 413                 rh->src_xfers[enqueue_ptr] = xfer;
 414                 /* remember corresponding ring/entry for this xfer structure */
 415                 xfer->ring_number = ring;
 416                 xfer->ring_entry = enqueue_ptr;
 417
 418                 enqueue_ptr = (enqueue_ptr + 1) % rh->xfer_entries;
 419
 420                 /*
 421                  * We may update the hardware view of the enqueue pointer
 422                  * only if we didn't reach its dequeue pointer.
 423                  */
 424                 op2_val = rh_reg_read(RING_OPERATION2);
 425                 if (enqueue_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val)) {
 426                         /* the ring is full */
 427                         hci_dma_unmap_xfer(hci, xfer_list, i + 1);
 428                         return -EBUSY;
 429                 }
 430         }
 431
 432         /* take care to update the hardware enqueue pointer atomically */
 433         spin_lock_irq(&rh->lock);
 434         op1_val = rh_reg_read(RING_OPERATION1);
 435         op1_val &= ~RING_OP1_CR_ENQ_PTR;
 436         op1_val |= FIELD_PREP(RING_OP1_CR_ENQ_PTR, enqueue_ptr);
 437         rh_reg_write(RING_OPERATION1, op1_val);
 438         spin_unlock_irq(&rh->lock);
 439
 440         return 0;
 441 }
 442
 443 static bool hci_dma_dequeue_xfer(struct i3c_hci *hci,
 444                                  struct hci_xfer *xfer_list, int n)
 445 {
 446         struct hci_rings_data *rings = hci->io_data;
 447         struct hci_rh_data *rh = &rings->headers[xfer_list[0].ring_number];
 448         unsigned int i;
 449         bool did_unqueue = false;
 450
 451         /* stop the ring */
 452         rh_reg_write(RING_CONTROL, RING_CTRL_ABORT);
 453         if (wait_for_completion_timeout(&rh->op_done, HZ) == 0) {
 454                 /*
 455                  * We're deep in it if ever this condition is ever met.
 456                  * Hardware might still be writing to memory, etc.
 457                  */
 458                 dev_crit(&hci->master.dev, "unable to abort the ring\n");
 459                 WARN_ON(1);
 460         }
 461
 462         for (i = 0; i < n; i++) {
 463                 struct hci_xfer *xfer = xfer_list + i;
 464                 int idx = xfer->ring_entry;
 465
 466                 /*
 467                  * At the time the abort happened, the xfer might have
 468                  * completed already. If not then replace corresponding
 469                  * descriptor entries with a no-op.
 470                  */
 471                 if (idx >= 0) {
 472                         u32 *ring_data = rh->xfer + rh->xfer_struct_sz * idx;
 473
 474                         /* store no-op cmd descriptor */
 475                         *ring_data++ = FIELD_PREP(CMD_0_ATTR, 0x7);
 476                         *ring_data++ = 0;
 477                         if (hci->cmd == &mipi_i3c_hci_cmd_v2) {
 478                                 *ring_data++ = 0;
 479                                 *ring_data++ = 0;
 480                         }
 481
 482                         /* disassociate this xfer struct */
 483                         rh->src_xfers[idx] = NULL;
 484
 485                         /* and unmap it */
 486                         hci_dma_unmap_xfer(hci, xfer, 1);
 487
 488                         did_unqueue = true;
 489                 }
 490         }
 491
 492         /* restart the ring */
 493         rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE);
 494
 495         return did_unqueue;
 496 }
 497
 498 static void hci_dma_xfer_done(struct i3c_hci *hci, struct hci_rh_data *rh)
 499 {
 500         u32 op1_val, op2_val, resp, *ring_resp;
 501         unsigned int tid, done_ptr = rh->done_ptr;
 502         struct hci_xfer *xfer;
 503
 504         for (;;) {
 505                 op2_val = rh_reg_read(RING_OPERATION2);
 506                 if (done_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val))
 507                         break;
 508
 509                 ring_resp = rh->resp + rh->resp_struct_sz * done_ptr;
 510                 resp = *ring_resp;
 511                 tid = RESP_TID(resp);
 512                 DBG("resp = 0x%08x", resp);
 513
 514                 xfer = rh->src_xfers[done_ptr];
 515                 if (!xfer) {
 516                         DBG("orphaned ring entry");
 517                 } else {
 518                         hci_dma_unmap_xfer(hci, xfer, 1);
 519                         xfer->ring_entry = -1;
 520                         xfer->response = resp;
 521                         if (tid != xfer->cmd_tid) {
 522                                 dev_err(&hci->master.dev,
 523                                         "response tid=%d when expecting %d\n",
 524                                         tid, xfer->cmd_tid);
 525                                 /* TODO: do something about it? */
 526                         }
 527                         if (xfer->completion)
 528                                 complete(xfer->completion);
 529                 }
 530
 531                 done_ptr = (done_ptr + 1) % rh->xfer_entries;
 532                 rh->done_ptr = done_ptr;
 533         }
 534
 535         /* take care to update the software dequeue pointer atomically */
 536         spin_lock(&rh->lock);
 537         op1_val = rh_reg_read(RING_OPERATION1);
 538         op1_val &= ~RING_OP1_CR_SW_DEQ_PTR;
 539         op1_val |= FIELD_PREP(RING_OP1_CR_SW_DEQ_PTR, done_ptr);
 540         rh_reg_write(RING_OPERATION1, op1_val);
 541         spin_unlock(&rh->lock);
 542 }
 543
 544 static int hci_dma_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev,
 545                                const struct i3c_ibi_setup *req)
 546 {
 547         struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
 548         struct i3c_generic_ibi_pool *pool;
 549         struct hci_dma_dev_ibi_data *dev_ibi;
 550
 551         dev_ibi = kmalloc(sizeof(*dev_ibi), GFP_KERNEL);
 552         if (!dev_ibi)
 553                 return -ENOMEM;
 554         pool = i3c_generic_ibi_alloc_pool(dev, req);
 555         if (IS_ERR(pool)) {
 556                 kfree(dev_ibi);
 557                 return PTR_ERR(pool);
 558         }
 559         dev_ibi->pool = pool;
 560         dev_ibi->max_len = req->max_payload_len;
 561         dev_data->ibi_data = dev_ibi;
 562         return 0;
 563 }
 564
 565 static void hci_dma_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev)
 566 {
 567         struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
 568         struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data;
 569
 570         dev_data->ibi_data = NULL;
 571         i3c_generic_ibi_free_pool(dev_ibi->pool);
 572         kfree(dev_ibi);
 573 }
 574
 575 static void hci_dma_recycle_ibi_slot(struct i3c_hci *hci,
 576                                      struct i3c_dev_desc *dev,
 577                                      struct i3c_ibi_slot *slot)
 578 {
 579         struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
 580         struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data;
 581
 582         i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot);
 583 }
 584
 585 static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh)
 586 {
 587         struct i3c_dev_desc *dev;
 588         struct i3c_hci_dev_data *dev_data;
 589         struct hci_dma_dev_ibi_data *dev_ibi;
 590         struct i3c_ibi_slot *slot;
 591         u32 op1_val, op2_val, ibi_status_error;
 592         unsigned int ptr, enq_ptr, deq_ptr;
 593         unsigned int ibi_size, ibi_chunks, ibi_data_offset, first_part;
 594         int ibi_addr, last_ptr;
 595         void *ring_ibi_data;
 596         dma_addr_t ring_ibi_data_dma;
 597
 598         op1_val = rh_reg_read(RING_OPERATION1);
 599         deq_ptr = FIELD_GET(RING_OP1_IBI_DEQ_PTR, op1_val);
 600
 601         op2_val = rh_reg_read(RING_OPERATION2);
 602         enq_ptr = FIELD_GET(RING_OP2_IBI_ENQ_PTR, op2_val);
 603
 604         ibi_status_error = 0;
 605         ibi_addr = -1;
 606         ibi_chunks = 0;
 607         ibi_size = 0;
 608         last_ptr = -1;
 609
 610         /* let's find all we can about this IBI */
 611         for (ptr = deq_ptr; ptr != enq_ptr;
 612              ptr = (ptr + 1) % rh->ibi_status_entries) {
 613                 u32 ibi_status, *ring_ibi_status;
 614                 unsigned int chunks;
 615
 616                 ring_ibi_status = rh->ibi_status + rh->ibi_status_sz * ptr;
 617                 ibi_status = *ring_ibi_status;
 618                 DBG("status = %#x", ibi_status);
 619
 620                 if (ibi_status_error) {
 621                         /* we no longer care */
 622                 } else if (ibi_status & IBI_ERROR) {
 623                         ibi_status_error = ibi_status;
 624                 } else if (ibi_addr ==  -1) {
 625                         ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status);
 626                 } else if (ibi_addr != FIELD_GET(IBI_TARGET_ADDR, ibi_status)) {
 627                         /* the address changed unexpectedly */
 628                         ibi_status_error = ibi_status;
 629                 }
 630
 631                 chunks = FIELD_GET(IBI_CHUNKS, ibi_status);
 632                 ibi_chunks += chunks;
 633                 if (!(ibi_status & IBI_LAST_STATUS)) {
 634                         ibi_size += chunks * rh->ibi_chunk_sz;
 635                 } else {
 636                         ibi_size += FIELD_GET(IBI_DATA_LENGTH, ibi_status);
 637                         last_ptr = ptr;
 638                         break;
 639                 }
 640         }
 641
 642         /* validate what we've got */
 643
 644         if (last_ptr == -1) {
 645                 /* this IBI sequence is not yet complete */
 646                 DBG("no LAST_STATUS available (e=%d d=%d)", enq_ptr, deq_ptr);
 647                 return;
 648         }
 649         deq_ptr = last_ptr + 1;
 650         deq_ptr %= rh->ibi_status_entries;
 651
 652         if (ibi_status_error) {
 653                 dev_err(&hci->master.dev, "IBI error from %#x\n", ibi_addr);
 654                 goto done;
 655         }
 656
 657         /* determine who this is for */
 658         dev = i3c_hci_addr_to_dev(hci, ibi_addr);
 659         if (!dev) {
 660                 dev_err(&hci->master.dev,
 661                         "IBI for unknown device %#x\n", ibi_addr);
 662                 goto done;
 663         }
 664
 665         dev_data = i3c_dev_get_master_data(dev);
 666         dev_ibi = dev_data->ibi_data;
 667         if (ibi_size > dev_ibi->max_len) {
 668                 dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n",
 669                         ibi_size, dev_ibi->max_len);
 670                 goto done;
 671         }
 672
 673         /*
 674          * This ring model is not suitable for zero-copy processing of IBIs.
 675          * We have the data chunk ring wrap-around to deal with, meaning
 676          * that the payload might span multiple chunks beginning at the
 677          * end of the ring and wrap to the start of the ring. Furthermore
 678          * there is no guarantee that those chunks will be released in order
 679          * and in a timely manner by the upper driver. So let's just copy
 680          * them to a discrete buffer. In practice they're supposed to be
 681          * small anyway.
 682          */
 683         slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool);
 684         if (!slot) {
 685                 dev_err(&hci->master.dev, "no free slot for IBI\n");
 686                 goto done;
 687         }
 688
 689         /* copy first part of the payload */
 690         ibi_data_offset = rh->ibi_chunk_sz * rh->ibi_chunk_ptr;
 691         ring_ibi_data = rh->ibi_data + ibi_data_offset;
 692         ring_ibi_data_dma = rh->ibi_data_dma + ibi_data_offset;
 693         first_part = (rh->ibi_chunks_total - rh->ibi_chunk_ptr)
 694                         * rh->ibi_chunk_sz;
 695         if (first_part > ibi_size)
 696                 first_part = ibi_size;
 697         dma_sync_single_for_cpu(&hci->master.dev, ring_ibi_data_dma,
 698                                 first_part, DMA_FROM_DEVICE);
 699         memcpy(slot->data, ring_ibi_data, first_part);
 700
 701         /* copy second part if any */
 702         if (ibi_size > first_part) {
 703                 /* we wrap back to the start and copy remaining data */
 704                 ring_ibi_data = rh->ibi_data;
 705                 ring_ibi_data_dma = rh->ibi_data_dma;
 706                 dma_sync_single_for_cpu(&hci->master.dev, ring_ibi_data_dma,
 707                                         ibi_size - first_part, DMA_FROM_DEVICE);
 708                 memcpy(slot->data + first_part, ring_ibi_data,
 709                        ibi_size - first_part);
 710         }
 711
 712         /* submit it */
 713         slot->dev = dev;
 714         slot->len = ibi_size;
 715         i3c_master_queue_ibi(dev, slot);
 716
 717 done:
 718         /* take care to update the ibi dequeue pointer atomically */
 719         spin_lock(&rh->lock);
 720         op1_val = rh_reg_read(RING_OPERATION1);
 721         op1_val &= ~RING_OP1_IBI_DEQ_PTR;
 722         op1_val |= FIELD_PREP(RING_OP1_IBI_DEQ_PTR, deq_ptr);
 723         rh_reg_write(RING_OPERATION1, op1_val);
 724         spin_unlock(&rh->lock);
 725
 726         /* update the chunk pointer */
 727         rh->ibi_chunk_ptr += ibi_chunks;
 728         rh->ibi_chunk_ptr %= rh->ibi_chunks_total;
 729
 730         /* and tell the hardware about freed chunks */
 731         rh_reg_write(CHUNK_CONTROL, rh_reg_read(CHUNK_CONTROL) + ibi_chunks);
 732 }
 733
 734 static bool hci_dma_irq_handler(struct i3c_hci *hci, unsigned int mask)
 735 {
 736         struct hci_rings_data *rings = hci->io_data;
 737         unsigned int i;
 738         bool handled = false;
 739
 740         for (i = 0; mask && i < rings->total; i++) {
 741                 struct hci_rh_data *rh;
 742                 u32 status;
 743
 744                 if (!(mask & BIT(i)))
 745                         continue;
 746                 mask &= ~BIT(i);
 747
 748                 rh = &rings->headers[i];
 749                 status = rh_reg_read(INTR_STATUS);
 750                 DBG("rh%d status: %#x", i, status);
 751                 if (!status)
 752                         continue;
 753                 rh_reg_write(INTR_STATUS, status);
 754
 755                 if (status & INTR_IBI_READY)
 756                         hci_dma_process_ibi(hci, rh);
 757                 if (status & (INTR_TRANSFER_COMPLETION | INTR_TRANSFER_ERR))
 758                         hci_dma_xfer_done(hci, rh);
 759                 if (status & INTR_RING_OP)
 760                         complete(&rh->op_done);
 761
 762                 if (status & INTR_TRANSFER_ABORT) {
 763                         dev_notice_ratelimited(&hci->master.dev,
 764                                 "ring %d: Transfer Aborted\n", i);
 765                         mipi_i3c_hci_resume(hci);
 766                 }
 767                 if (status & INTR_WARN_INS_STOP_MODE)
 768                         dev_warn_ratelimited(&hci->master.dev,
 769                                 "ring %d: Inserted Stop on Mode Change\n", i);
 770                 if (status & INTR_IBI_RING_FULL)
 771                         dev_err_ratelimited(&hci->master.dev,
 772                                 "ring %d: IBI Ring Full Condition\n", i);
 773
 774                 handled = true;
 775         }
 776
 777         return handled;
 778 }
 779
 780 const struct hci_io_ops mipi_i3c_hci_dma = {
 781         .init                   = hci_dma_init,
 782         .cleanup                = hci_dma_cleanup,
 783         .queue_xfer             = hci_dma_queue_xfer,
 784         .dequeue_xfer           = hci_dma_dequeue_xfer,
 785         .irq_handler            = hci_dma_irq_handler,
 786         .request_ibi            = hci_dma_request_ibi,
 787         .free_ibi               = hci_dma_free_ibi,
 788         .recycle_ibi_slot       = hci_dma_recycle_ibi_slot,
 789 };