drivers/dma/ti/k3-udma.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
   4  *  Author: Peter Ujfalusi <peter.ujfalusi@ti.com>
   5  */
   6
   7 #include <linux/kernel.h>
   8 #include <linux/module.h>
   9 #include <linux/delay.h>
  10 #include <linux/dmaengine.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/dmapool.h>
  13 #include <linux/err.h>
  14 #include <linux/init.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/list.h>
  17 #include <linux/platform_device.h>
  18 #include <linux/slab.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/sys_soc.h>
  21 #include <linux/of.h>
  22 #include <linux/of_dma.h>
  23 #include <linux/of_device.h>
  24 #include <linux/of_irq.h>
  25 #include <linux/workqueue.h>
  26 #include <linux/completion.h>
  27 #include <linux/soc/ti/k3-ringacc.h>
  28 #include <linux/soc/ti/ti_sci_protocol.h>
  29 #include <linux/soc/ti/ti_sci_inta_msi.h>
  30 #include <linux/dma/k3-event-router.h>
  31 #include <linux/dma/ti-cppi5.h>
  32
  33 #include "../virt-dma.h"
  34 #include "k3-udma.h"
  35 #include "k3-psil-priv.h"
  36
  37 struct udma_static_tr {
  38         u8 elsize; /* RPSTR0 */
  39         u16 elcnt; /* RPSTR0 */
  40         u16 bstcnt; /* RPSTR1 */
  41 };
  42
  43 #define K3_UDMA_MAX_RFLOWS              1024
  44 #define K3_UDMA_DEFAULT_RING_SIZE       16
  45
  46 /* How SRC/DST tag should be updated by UDMA in the descriptor's Word 3 */
  47 #define UDMA_RFLOW_SRCTAG_NONE          0
  48 #define UDMA_RFLOW_SRCTAG_CFG_TAG       1
  49 #define UDMA_RFLOW_SRCTAG_FLOW_ID       2
  50 #define UDMA_RFLOW_SRCTAG_SRC_TAG       4
  51
  52 #define UDMA_RFLOW_DSTTAG_NONE          0
  53 #define UDMA_RFLOW_DSTTAG_CFG_TAG       1
  54 #define UDMA_RFLOW_DSTTAG_FLOW_ID       2
  55 #define UDMA_RFLOW_DSTTAG_DST_TAG_LO    4
  56 #define UDMA_RFLOW_DSTTAG_DST_TAG_HI    5
  57
  58 struct udma_chan;
  59
  60 enum k3_dma_type {
  61         DMA_TYPE_UDMA = 0,
  62         DMA_TYPE_BCDMA,
  63         DMA_TYPE_PKTDMA,
  64 };
  65
  66 enum udma_mmr {
  67         MMR_GCFG = 0,
  68         MMR_BCHANRT,
  69         MMR_RCHANRT,
  70         MMR_TCHANRT,
  71         MMR_LAST,
  72 };
  73
  74 static const char * const mmr_names[] = {
  75         [MMR_GCFG] = "gcfg",
  76         [MMR_BCHANRT] = "bchanrt",
  77         [MMR_RCHANRT] = "rchanrt",
  78         [MMR_TCHANRT] = "tchanrt",
  79 };
  80
  81 struct udma_tchan {
  82         void __iomem *reg_rt;
  83
  84         int id;
  85         struct k3_ring *t_ring; /* Transmit ring */
  86         struct k3_ring *tc_ring; /* Transmit Completion ring */
  87         int tflow_id; /* applicable only for PKTDMA */
  88
  89 };
  90
  91 #define udma_bchan udma_tchan
  92
  93 struct udma_rflow {
  94         int id;
  95         struct k3_ring *fd_ring; /* Free Descriptor ring */
  96         struct k3_ring *r_ring; /* Receive ring */
  97 };
  98
  99 struct udma_rchan {
 100         void __iomem *reg_rt;
 101
 102         int id;
 103 };
 104
 105 struct udma_oes_offsets {
 106         /* K3 UDMA Output Event Offset */
 107         u32 udma_rchan;
 108
 109         /* BCDMA Output Event Offsets */
 110         u32 bcdma_bchan_data;
 111         u32 bcdma_bchan_ring;
 112         u32 bcdma_tchan_data;
 113         u32 bcdma_tchan_ring;
 114         u32 bcdma_rchan_data;
 115         u32 bcdma_rchan_ring;
 116
 117         /* PKTDMA Output Event Offsets */
 118         u32 pktdma_tchan_flow;
 119         u32 pktdma_rchan_flow;
 120 };
 121
 122 #define UDMA_FLAG_PDMA_ACC32            BIT(0)
 123 #define UDMA_FLAG_PDMA_BURST            BIT(1)
 124 #define UDMA_FLAG_TDTYPE                BIT(2)
 125 #define UDMA_FLAG_BURST_SIZE            BIT(3)
 126 #define UDMA_FLAGS_J7_CLASS             (UDMA_FLAG_PDMA_ACC32 | \
 127                                          UDMA_FLAG_PDMA_BURST | \
 128                                          UDMA_FLAG_TDTYPE | \
 129                                          UDMA_FLAG_BURST_SIZE)
 130
 131 struct udma_match_data {
 132         enum k3_dma_type type;
 133         u32 psil_base;
 134         bool enable_memcpy_support;
 135         u32 flags;
 136         u32 statictr_z_mask;
 137         u8 burst_size[3];
 138         struct udma_soc_data *soc_data;
 139 };
 140
 141 struct udma_soc_data {
 142         struct udma_oes_offsets oes;
 143         u32 bcdma_trigger_event_offset;
 144 };
 145
 146 struct udma_hwdesc {
 147         size_t cppi5_desc_size;
 148         void *cppi5_desc_vaddr;
 149         dma_addr_t cppi5_desc_paddr;
 150
 151         /* TR descriptor internal pointers */
 152         void *tr_req_base;
 153         struct cppi5_tr_resp_t *tr_resp_base;
 154 };
 155
 156 struct udma_rx_flush {
 157         struct udma_hwdesc hwdescs[2];
 158
 159         size_t buffer_size;
 160         void *buffer_vaddr;
 161         dma_addr_t buffer_paddr;
 162 };
 163
 164 struct udma_tpl {
 165         u8 levels;
 166         u32 start_idx[3];
 167 };
 168
 169 struct udma_dev {
 170         struct dma_device ddev;
 171         struct device *dev;
 172         void __iomem *mmrs[MMR_LAST];
 173         const struct udma_match_data *match_data;
 174         const struct udma_soc_data *soc_data;
 175
 176         struct udma_tpl bchan_tpl;
 177         struct udma_tpl tchan_tpl;
 178         struct udma_tpl rchan_tpl;
 179
 180         size_t desc_align; /* alignment to use for descriptors */
 181
 182         struct udma_tisci_rm tisci_rm;
 183
 184         struct k3_ringacc *ringacc;
 185
 186         struct work_struct purge_work;
 187         struct list_head desc_to_purge;
 188         spinlock_t lock;
 189
 190         struct udma_rx_flush rx_flush;
 191
 192         int bchan_cnt;
 193         int tchan_cnt;
 194         int echan_cnt;
 195         int rchan_cnt;
 196         int rflow_cnt;
 197         int tflow_cnt;
 198         unsigned long *bchan_map;
 199         unsigned long *tchan_map;
 200         unsigned long *rchan_map;
 201         unsigned long *rflow_gp_map;
 202         unsigned long *rflow_gp_map_allocated;
 203         unsigned long *rflow_in_use;
 204         unsigned long *tflow_map;
 205
 206         struct udma_bchan *bchans;
 207         struct udma_tchan *tchans;
 208         struct udma_rchan *rchans;
 209         struct udma_rflow *rflows;
 210
 211         struct udma_chan *channels;
 212         u32 psil_base;
 213         u32 atype;
 214         u32 asel;
 215 };
 216
 217 struct udma_desc {
 218         struct virt_dma_desc vd;
 219
 220         bool terminated;
 221
 222         enum dma_transfer_direction dir;
 223
 224         struct udma_static_tr static_tr;
 225         u32 residue;
 226
 227         unsigned int sglen;
 228         unsigned int desc_idx; /* Only used for cyclic in packet mode */
 229         unsigned int tr_idx;
 230
 231         u32 metadata_size;
 232         void *metadata; /* pointer to provided metadata buffer (EPIP, PSdata) */
 233
 234         unsigned int hwdesc_count;
 235         struct udma_hwdesc hwdesc[];
 236 };
 237
 238 enum udma_chan_state {
 239         UDMA_CHAN_IS_IDLE = 0, /* not active, no teardown is in progress */
 240         UDMA_CHAN_IS_ACTIVE, /* Normal operation */
 241         UDMA_CHAN_IS_TERMINATING, /* channel is being terminated */
 242 };
 243
 244 struct udma_tx_drain {
 245         struct delayed_work work;
 246         ktime_t tstamp;
 247         u32 residue;
 248 };
 249
 250 struct udma_chan_config {
 251         bool pkt_mode; /* TR or packet */
 252         bool needs_epib; /* EPIB is needed for the communication or not */
 253         u32 psd_size; /* size of Protocol Specific Data */
 254         u32 metadata_size; /* (needs_epib ? 16:0) + psd_size */
 255         u32 hdesc_size; /* Size of a packet descriptor in packet mode */
 256         bool notdpkt; /* Suppress sending TDC packet */
 257         int remote_thread_id;
 258         u32 atype;
 259         u32 asel;
 260         u32 src_thread;
 261         u32 dst_thread;
 262         enum psil_endpoint_type ep_type;
 263         bool enable_acc32;
 264         bool enable_burst;
 265         enum udma_tp_level channel_tpl; /* Channel Throughput Level */
 266
 267         u32 tr_trigger_type;
 268         unsigned long tx_flags;
 269
 270         /* PKDMA mapped channel */
 271         int mapped_channel_id;
 272         /* PKTDMA default tflow or rflow for mapped channel */
 273         int default_flow_id;
 274
 275         enum dma_transfer_direction dir;
 276 };
 277
 278 struct udma_chan {
 279         struct virt_dma_chan vc;
 280         struct dma_slave_config cfg;
 281         struct udma_dev *ud;
 282         struct device *dma_dev;
 283         struct udma_desc *desc;
 284         struct udma_desc *terminated_desc;
 285         struct udma_static_tr static_tr;
 286         char *name;
 287
 288         struct udma_bchan *bchan;
 289         struct udma_tchan *tchan;
 290         struct udma_rchan *rchan;
 291         struct udma_rflow *rflow;
 292
 293         bool psil_paired;
 294
 295         int irq_num_ring;
 296         int irq_num_udma;
 297
 298         bool cyclic;
 299         bool paused;
 300
 301         enum udma_chan_state state;
 302         struct completion teardown_completed;
 303
 304         struct udma_tx_drain tx_drain;
 305
 306         /* Channel configuration parameters */
 307         struct udma_chan_config config;
 308
 309         /* dmapool for packet mode descriptors */
 310         bool use_dma_pool;
 311         struct dma_pool *hdesc_pool;
 312
 313         u32 id;
 314 };
 315
 316 static inline struct udma_dev *to_udma_dev(struct dma_device *d)
 317 {
 318         return container_of(d, struct udma_dev, ddev);
 319 }
 320
 321 static inline struct udma_chan *to_udma_chan(struct dma_chan *c)
 322 {
 323         return container_of(c, struct udma_chan, vc.chan);
 324 }
 325
 326 static inline struct udma_desc *to_udma_desc(struct dma_async_tx_descriptor *t)
 327 {
 328         return container_of(t, struct udma_desc, vd.tx);
 329 }
 330
 331 /* Generic register access functions */
 332 static inline u32 udma_read(void __iomem *base, int reg)
 333 {
 334         return readl(base + reg);
 335 }
 336
 337 static inline void udma_write(void __iomem *base, int reg, u32 val)
 338 {
 339         writel(val, base + reg);
 340 }
 341
 342 static inline void udma_update_bits(void __iomem *base, int reg,
 343                                     u32 mask, u32 val)
 344 {
 345         u32 tmp, orig;
 346
 347         orig = readl(base + reg);
 348         tmp = orig & ~mask;
 349         tmp |= (val & mask);
 350
 351         if (tmp != orig)
 352                 writel(tmp, base + reg);
 353 }
 354
 355 /* TCHANRT */
 356 static inline u32 udma_tchanrt_read(struct udma_chan *uc, int reg)
 357 {
 358         if (!uc->tchan)
 359                 return 0;
 360         return udma_read(uc->tchan->reg_rt, reg);
 361 }
 362
 363 static inline void udma_tchanrt_write(struct udma_chan *uc, int reg, u32 val)
 364 {
 365         if (!uc->tchan)
 366                 return;
 367         udma_write(uc->tchan->reg_rt, reg, val);
 368 }
 369
 370 static inline void udma_tchanrt_update_bits(struct udma_chan *uc, int reg,
 371                                             u32 mask, u32 val)
 372 {
 373         if (!uc->tchan)
 374                 return;
 375         udma_update_bits(uc->tchan->reg_rt, reg, mask, val);
 376 }
 377
 378 /* RCHANRT */
 379 static inline u32 udma_rchanrt_read(struct udma_chan *uc, int reg)
 380 {
 381         if (!uc->rchan)
 382                 return 0;
 383         return udma_read(uc->rchan->reg_rt, reg);
 384 }
 385
 386 static inline void udma_rchanrt_write(struct udma_chan *uc, int reg, u32 val)
 387 {
 388         if (!uc->rchan)
 389                 return;
 390         udma_write(uc->rchan->reg_rt, reg, val);
 391 }
 392
 393 static inline void udma_rchanrt_update_bits(struct udma_chan *uc, int reg,
 394                                             u32 mask, u32 val)
 395 {
 396         if (!uc->rchan)
 397                 return;
 398         udma_update_bits(uc->rchan->reg_rt, reg, mask, val);
 399 }
 400
 401 static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread)
 402 {
 403         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
 404
 405         dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
 406         return tisci_rm->tisci_psil_ops->pair(tisci_rm->tisci,
 407                                               tisci_rm->tisci_navss_dev_id,
 408                                               src_thread, dst_thread);
 409 }
 410
 411 static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread,
 412                              u32 dst_thread)
 413 {
 414         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
 415
 416         dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
 417         return tisci_rm->tisci_psil_ops->unpair(tisci_rm->tisci,
 418                                                 tisci_rm->tisci_navss_dev_id,
 419                                                 src_thread, dst_thread);
 420 }
 421
 422 static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel)
 423 {
 424         struct device *chan_dev = &chan->dev->device;
 425
 426         if (asel == 0) {
 427                 /* No special handling for the channel */
 428                 chan->dev->chan_dma_dev = false;
 429
 430                 chan_dev->dma_coherent = false;
 431                 chan_dev->dma_parms = NULL;
 432         } else if (asel == 14 || asel == 15) {
 433                 chan->dev->chan_dma_dev = true;
 434
 435                 chan_dev->dma_coherent = true;
 436                 dma_coerce_mask_and_coherent(chan_dev, DMA_BIT_MASK(48));
 437                 chan_dev->dma_parms = chan_dev->parent->dma_parms;
 438         } else {
 439                 dev_warn(chan->device->dev, "Invalid ASEL value: %u\n", asel);
 440
 441                 chan_dev->dma_coherent = false;
 442                 chan_dev->dma_parms = NULL;
 443         }
 444 }
 445
 446 static u8 udma_get_chan_tpl_index(struct udma_tpl *tpl_map, int chan_id)
 447 {
 448         int i;
 449
 450         for (i = 0; i < tpl_map->levels; i++) {
 451                 if (chan_id >= tpl_map->start_idx[i])
 452                         return i;
 453         }
 454
 455         return 0;
 456 }
 457
 458 static void udma_reset_uchan(struct udma_chan *uc)
 459 {
 460         memset(&uc->config, 0, sizeof(uc->config));
 461         uc->config.remote_thread_id = -1;
 462         uc->config.mapped_channel_id = -1;
 463         uc->config.default_flow_id = -1;
 464         uc->state = UDMA_CHAN_IS_IDLE;
 465 }
 466
 467 static void udma_dump_chan_stdata(struct udma_chan *uc)
 468 {
 469         struct device *dev = uc->ud->dev;
 470         u32 offset;
 471         int i;
 472
 473         if (uc->config.dir == DMA_MEM_TO_DEV || uc->config.dir == DMA_MEM_TO_MEM) {
 474                 dev_dbg(dev, "TCHAN State data:\n");
 475                 for (i = 0; i < 32; i++) {
 476                         offset = UDMA_CHAN_RT_STDATA_REG + i * 4;
 477                         dev_dbg(dev, "TRT_STDATA[%02d]: 0x%08x\n", i,
 478                                 udma_tchanrt_read(uc, offset));
 479                 }
 480         }
 481
 482         if (uc->config.dir == DMA_DEV_TO_MEM || uc->config.dir == DMA_MEM_TO_MEM) {
 483                 dev_dbg(dev, "RCHAN State data:\n");
 484                 for (i = 0; i < 32; i++) {
 485                         offset = UDMA_CHAN_RT_STDATA_REG + i * 4;
 486                         dev_dbg(dev, "RRT_STDATA[%02d]: 0x%08x\n", i,
 487                                 udma_rchanrt_read(uc, offset));
 488                 }
 489         }
 490 }
 491
 492 static inline dma_addr_t udma_curr_cppi5_desc_paddr(struct udma_desc *d,
 493                                                     int idx)
 494 {
 495         return d->hwdesc[idx].cppi5_desc_paddr;
 496 }
 497
 498 static inline void *udma_curr_cppi5_desc_vaddr(struct udma_desc *d, int idx)
 499 {
 500         return d->hwdesc[idx].cppi5_desc_vaddr;
 501 }
 502
 503 static struct udma_desc *udma_udma_desc_from_paddr(struct udma_chan *uc,
 504                                                    dma_addr_t paddr)
 505 {
 506         struct udma_desc *d = uc->terminated_desc;
 507
 508         if (d) {
 509                 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
 510                                                                    d->desc_idx);
 511
 512                 if (desc_paddr != paddr)
 513                         d = NULL;
 514         }
 515
 516         if (!d) {
 517                 d = uc->desc;
 518                 if (d) {
 519                         dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
 520                                                                 d->desc_idx);
 521
 522                         if (desc_paddr != paddr)
 523                                 d = NULL;
 524                 }
 525         }
 526
 527         return d;
 528 }
 529
 530 static void udma_free_hwdesc(struct udma_chan *uc, struct udma_desc *d)
 531 {
 532         if (uc->use_dma_pool) {
 533                 int i;
 534
 535                 for (i = 0; i < d->hwdesc_count; i++) {
 536                         if (!d->hwdesc[i].cppi5_desc_vaddr)
 537                                 continue;
 538
 539                         dma_pool_free(uc->hdesc_pool,
 540                                       d->hwdesc[i].cppi5_desc_vaddr,
 541                                       d->hwdesc[i].cppi5_desc_paddr);
 542
 543                         d->hwdesc[i].cppi5_desc_vaddr = NULL;
 544                 }
 545         } else if (d->hwdesc[0].cppi5_desc_vaddr) {
 546                 dma_free_coherent(uc->dma_dev, d->hwdesc[0].cppi5_desc_size,
 547                                   d->hwdesc[0].cppi5_desc_vaddr,
 548                                   d->hwdesc[0].cppi5_desc_paddr);
 549
 550                 d->hwdesc[0].cppi5_desc_vaddr = NULL;
 551         }
 552 }
 553
 554 static void udma_purge_desc_work(struct work_struct *work)
 555 {
 556         struct udma_dev *ud = container_of(work, typeof(*ud), purge_work);
 557         struct virt_dma_desc *vd, *_vd;
 558         unsigned long flags;
 559         LIST_HEAD(head);
 560
 561         spin_lock_irqsave(&ud->lock, flags);
 562         list_splice_tail_init(&ud->desc_to_purge, &head);
 563         spin_unlock_irqrestore(&ud->lock, flags);
 564
 565         list_for_each_entry_safe(vd, _vd, &head, node) {
 566                 struct udma_chan *uc = to_udma_chan(vd->tx.chan);
 567                 struct udma_desc *d = to_udma_desc(&vd->tx);
 568
 569                 udma_free_hwdesc(uc, d);
 570                 list_del(&vd->node);
 571                 kfree(d);
 572         }
 573
 574         /* If more to purge, schedule the work again */
 575         if (!list_empty(&ud->desc_to_purge))
 576                 schedule_work(&ud->purge_work);
 577 }
 578
 579 static void udma_desc_free(struct virt_dma_desc *vd)
 580 {
 581         struct udma_dev *ud = to_udma_dev(vd->tx.chan->device);
 582         struct udma_chan *uc = to_udma_chan(vd->tx.chan);
 583         struct udma_desc *d = to_udma_desc(&vd->tx);
 584         unsigned long flags;
 585
 586         if (uc->terminated_desc == d)
 587                 uc->terminated_desc = NULL;
 588
 589         if (uc->use_dma_pool) {
 590                 udma_free_hwdesc(uc, d);
 591                 kfree(d);
 592                 return;
 593         }
 594
 595         spin_lock_irqsave(&ud->lock, flags);
 596         list_add_tail(&vd->node, &ud->desc_to_purge);
 597         spin_unlock_irqrestore(&ud->lock, flags);
 598
 599         schedule_work(&ud->purge_work);
 600 }
 601
 602 static bool udma_is_chan_running(struct udma_chan *uc)
 603 {
 604         u32 trt_ctl = 0;
 605         u32 rrt_ctl = 0;
 606
 607         if (uc->tchan)
 608                 trt_ctl = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
 609         if (uc->rchan)
 610                 rrt_ctl = udma_rchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
 611
 612         if (trt_ctl & UDMA_CHAN_RT_CTL_EN || rrt_ctl & UDMA_CHAN_RT_CTL_EN)
 613                 return true;
 614
 615         return false;
 616 }
 617
 618 static bool udma_is_chan_paused(struct udma_chan *uc)
 619 {
 620         u32 val, pause_mask;
 621
 622         switch (uc->config.dir) {
 623         case DMA_DEV_TO_MEM:
 624                 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG);
 625                 pause_mask = UDMA_PEER_RT_EN_PAUSE;
 626                 break;
 627         case DMA_MEM_TO_DEV:
 628                 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_RT_EN_REG);
 629                 pause_mask = UDMA_PEER_RT_EN_PAUSE;
 630                 break;
 631         case DMA_MEM_TO_MEM:
 632                 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_CTL_REG);
 633                 pause_mask = UDMA_CHAN_RT_CTL_PAUSE;
 634                 break;
 635         default:
 636                 return false;
 637         }
 638
 639         if (val & pause_mask)
 640                 return true;
 641
 642         return false;
 643 }
 644
 645 static inline dma_addr_t udma_get_rx_flush_hwdesc_paddr(struct udma_chan *uc)
 646 {
 647         return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr;
 648 }
 649
 650 static int udma_push_to_ring(struct udma_chan *uc, int idx)
 651 {
 652         struct udma_desc *d = uc->desc;
 653         struct k3_ring *ring = NULL;
 654         dma_addr_t paddr;
 655
 656         switch (uc->config.dir) {
 657         case DMA_DEV_TO_MEM:
 658                 ring = uc->rflow->fd_ring;
 659                 break;
 660         case DMA_MEM_TO_DEV:
 661         case DMA_MEM_TO_MEM:
 662                 ring = uc->tchan->t_ring;
 663                 break;
 664         default:
 665                 return -EINVAL;
 666         }
 667
 668         /* RX flush packet: idx == -1 is only passed in case of DEV_TO_MEM */
 669         if (idx == -1) {
 670                 paddr = udma_get_rx_flush_hwdesc_paddr(uc);
 671         } else {
 672                 paddr = udma_curr_cppi5_desc_paddr(d, idx);
 673
 674                 wmb(); /* Ensure that writes are not moved over this point */
 675         }
 676
 677         return k3_ringacc_ring_push(ring, &paddr);
 678 }
 679
 680 static bool udma_desc_is_rx_flush(struct udma_chan *uc, dma_addr_t addr)
 681 {
 682         if (uc->config.dir != DMA_DEV_TO_MEM)
 683                 return false;
 684
 685         if (addr == udma_get_rx_flush_hwdesc_paddr(uc))
 686                 return true;
 687
 688         return false;
 689 }
 690
 691 static int udma_pop_from_ring(struct udma_chan *uc, dma_addr_t *addr)
 692 {
 693         struct k3_ring *ring = NULL;
 694         int ret;
 695
 696         switch (uc->config.dir) {
 697         case DMA_DEV_TO_MEM:
 698                 ring = uc->rflow->r_ring;
 699                 break;
 700         case DMA_MEM_TO_DEV:
 701         case DMA_MEM_TO_MEM:
 702                 ring = uc->tchan->tc_ring;
 703                 break;
 704         default:
 705                 return -ENOENT;
 706         }
 707
 708         ret = k3_ringacc_ring_pop(ring, addr);
 709         if (ret)
 710                 return ret;
 711
 712         rmb(); /* Ensure that reads are not moved before this point */
 713
 714         /* Teardown completion */
 715         if (cppi5_desc_is_tdcm(*addr))
 716                 return 0;
 717
 718         /* Check for flush descriptor */
 719         if (udma_desc_is_rx_flush(uc, *addr))
 720                 return -ENOENT;
 721
 722         return 0;
 723 }
 724
 725 static void udma_reset_rings(struct udma_chan *uc)
 726 {
 727         struct k3_ring *ring1 = NULL;
 728         struct k3_ring *ring2 = NULL;
 729
 730         switch (uc->config.dir) {
 731         case DMA_DEV_TO_MEM:
 732                 if (uc->rchan) {
 733                         ring1 = uc->rflow->fd_ring;
 734                         ring2 = uc->rflow->r_ring;
 735                 }
 736                 break;
 737         case DMA_MEM_TO_DEV:
 738         case DMA_MEM_TO_MEM:
 739                 if (uc->tchan) {
 740                         ring1 = uc->tchan->t_ring;
 741                         ring2 = uc->tchan->tc_ring;
 742                 }
 743                 break;
 744         default:
 745                 break;
 746         }
 747
 748         if (ring1)
 749                 k3_ringacc_ring_reset_dma(ring1,
 750                                           k3_ringacc_ring_get_occ(ring1));
 751         if (ring2)
 752                 k3_ringacc_ring_reset(ring2);
 753
 754         /* make sure we are not leaking memory by stalled descriptor */
 755         if (uc->terminated_desc) {
 756                 udma_desc_free(&uc->terminated_desc->vd);
 757                 uc->terminated_desc = NULL;
 758         }
 759 }
 760
 761 static void udma_decrement_byte_counters(struct udma_chan *uc, u32 val)
 762 {
 763         if (uc->desc->dir == DMA_DEV_TO_MEM) {
 764                 udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
 765                 udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
 766                 if (uc->config.ep_type != PSIL_EP_NATIVE)
 767                         udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
 768         } else {
 769                 udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
 770                 udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
 771                 if (!uc->bchan && uc->config.ep_type != PSIL_EP_NATIVE)
 772                         udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
 773         }
 774 }
 775
 776 static void udma_reset_counters(struct udma_chan *uc)
 777 {
 778         u32 val;
 779
 780         if (uc->tchan) {
 781                 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
 782                 udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
 783
 784                 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
 785                 udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
 786
 787                 val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG);
 788                 udma_tchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val);
 789
 790                 if (!uc->bchan) {
 791                         val = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
 792                         udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
 793                 }
 794         }
 795
 796         if (uc->rchan) {
 797                 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
 798                 udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val);
 799
 800                 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
 801                 udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val);
 802
 803                 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PCNT_REG);
 804                 udma_rchanrt_write(uc, UDMA_CHAN_RT_PCNT_REG, val);
 805
 806                 val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
 807                 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val);
 808         }
 809 }
 810
 811 static int udma_reset_chan(struct udma_chan *uc, bool hard)
 812 {
 813         switch (uc->config.dir) {
 814         case DMA_DEV_TO_MEM:
 815                 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
 816                 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
 817                 break;
 818         case DMA_MEM_TO_DEV:
 819                 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
 820                 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
 821                 break;
 822         case DMA_MEM_TO_MEM:
 823                 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
 824                 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG, 0);
 825                 break;
 826         default:
 827                 return -EINVAL;
 828         }
 829
 830         /* Reset all counters */
 831         udma_reset_counters(uc);
 832
 833         /* Hard reset: re-initialize the channel to reset */
 834         if (hard) {
 835                 struct udma_chan_config ucc_backup;
 836                 int ret;
 837
 838                 memcpy(&ucc_backup, &uc->config, sizeof(uc->config));
 839                 uc->ud->ddev.device_free_chan_resources(&uc->vc.chan);
 840
 841                 /* restore the channel configuration */
 842                 memcpy(&uc->config, &ucc_backup, sizeof(uc->config));
 843                 ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan);
 844                 if (ret)
 845                         return ret;
 846
 847                 /*
 848                  * Setting forced teardown after forced reset helps recovering
 849                  * the rchan.
 850                  */
 851                 if (uc->config.dir == DMA_DEV_TO_MEM)
 852                         udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
 853                                            UDMA_CHAN_RT_CTL_EN |
 854                                            UDMA_CHAN_RT_CTL_TDOWN |
 855                                            UDMA_CHAN_RT_CTL_FTDOWN);
 856         }
 857         uc->state = UDMA_CHAN_IS_IDLE;
 858
 859         return 0;
 860 }
 861
 862 static void udma_start_desc(struct udma_chan *uc)
 863 {
 864         struct udma_chan_config *ucc = &uc->config;
 865
 866         if (uc->ud->match_data->type == DMA_TYPE_UDMA && ucc->pkt_mode &&
 867             (uc->cyclic || ucc->dir == DMA_DEV_TO_MEM)) {
 868                 int i;
 869
 870                 /*
 871                  * UDMA only: Push all descriptors to ring for packet mode
 872                  * cyclic or RX
 873                  * PKTDMA supports pre-linked descriptor and cyclic is not
 874                  * supported
 875                  */
 876                 for (i = 0; i < uc->desc->sglen; i++)
 877                         udma_push_to_ring(uc, i);
 878         } else {
 879                 udma_push_to_ring(uc, 0);
 880         }
 881 }
 882
 883 static bool udma_chan_needs_reconfiguration(struct udma_chan *uc)
 884 {
 885         /* Only PDMAs have staticTR */
 886         if (uc->config.ep_type == PSIL_EP_NATIVE)
 887                 return false;
 888
 889         /* Check if the staticTR configuration has changed for TX */
 890         if (memcmp(&uc->static_tr, &uc->desc->static_tr, sizeof(uc->static_tr)))
 891                 return true;
 892
 893         return false;
 894 }
 895
 896 static int udma_start(struct udma_chan *uc)
 897 {
 898         struct virt_dma_desc *vd = vchan_next_desc(&uc->vc);
 899
 900         if (!vd) {
 901                 uc->desc = NULL;
 902                 return -ENOENT;
 903         }
 904
 905         list_del(&vd->node);
 906
 907         uc->desc = to_udma_desc(&vd->tx);
 908
 909         /* Channel is already running and does not need reconfiguration */
 910         if (udma_is_chan_running(uc) && !udma_chan_needs_reconfiguration(uc)) {
 911                 udma_start_desc(uc);
 912                 goto out;
 913         }
 914
 915         /* Make sure that we clear the teardown bit, if it is set */
 916         udma_reset_chan(uc, false);
 917
 918         /* Push descriptors before we start the channel */
 919         udma_start_desc(uc);
 920
 921         switch (uc->desc->dir) {
 922         case DMA_DEV_TO_MEM:
 923                 /* Config remote TR */
 924                 if (uc->config.ep_type == PSIL_EP_PDMA_XY) {
 925                         u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
 926                                   PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
 927                         const struct udma_match_data *match_data =
 928                                                         uc->ud->match_data;
 929
 930                         if (uc->config.enable_acc32)
 931                                 val |= PDMA_STATIC_TR_XY_ACC32;
 932                         if (uc->config.enable_burst)
 933                                 val |= PDMA_STATIC_TR_XY_BURST;
 934
 935                         udma_rchanrt_write(uc,
 936                                            UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG,
 937                                            val);
 938
 939                         udma_rchanrt_write(uc,
 940                                 UDMA_CHAN_RT_PEER_STATIC_TR_Z_REG,
 941                                 PDMA_STATIC_TR_Z(uc->desc->static_tr.bstcnt,
 942                                                  match_data->statictr_z_mask));
 943
 944                         /* save the current staticTR configuration */
 945                         memcpy(&uc->static_tr, &uc->desc->static_tr,
 946                                sizeof(uc->static_tr));
 947                 }
 948
 949                 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
 950                                    UDMA_CHAN_RT_CTL_EN);
 951
 952                 /* Enable remote */
 953                 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
 954                                    UDMA_PEER_RT_EN_ENABLE);
 955
 956                 break;
 957         case DMA_MEM_TO_DEV:
 958                 /* Config remote TR */
 959                 if (uc->config.ep_type == PSIL_EP_PDMA_XY) {
 960                         u32 val = PDMA_STATIC_TR_Y(uc->desc->static_tr.elcnt) |
 961                                   PDMA_STATIC_TR_X(uc->desc->static_tr.elsize);
 962
 963                         if (uc->config.enable_acc32)
 964                                 val |= PDMA_STATIC_TR_XY_ACC32;
 965                         if (uc->config.enable_burst)
 966                                 val |= PDMA_STATIC_TR_XY_BURST;
 967
 968                         udma_tchanrt_write(uc,
 969                                            UDMA_CHAN_RT_PEER_STATIC_TR_XY_REG,
 970                                            val);
 971
 972                         /* save the current staticTR configuration */
 973                         memcpy(&uc->static_tr, &uc->desc->static_tr,
 974                                sizeof(uc->static_tr));
 975                 }
 976
 977                 /* Enable remote */
 978                 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
 979                                    UDMA_PEER_RT_EN_ENABLE);
 980
 981                 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
 982                                    UDMA_CHAN_RT_CTL_EN);
 983
 984                 break;
 985         case DMA_MEM_TO_MEM:
 986                 udma_rchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
 987                                    UDMA_CHAN_RT_CTL_EN);
 988                 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
 989                                    UDMA_CHAN_RT_CTL_EN);
 990
 991                 break;
 992         default:
 993                 return -EINVAL;
 994         }
 995
 996         uc->state = UDMA_CHAN_IS_ACTIVE;
 997 out:
 998
 999         return 0;
1000 }
1001
1002 static int udma_stop(struct udma_chan *uc)
1003 {
1004         enum udma_chan_state old_state = uc->state;
1005
1006         uc->state = UDMA_CHAN_IS_TERMINATING;
1007         reinit_completion(&uc->teardown_completed);
1008
1009         switch (uc->config.dir) {
1010         case DMA_DEV_TO_MEM:
1011                 if (!uc->cyclic && !uc->desc)
1012                         udma_push_to_ring(uc, -1);
1013
1014                 udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
1015                                    UDMA_PEER_RT_EN_ENABLE |
1016                                    UDMA_PEER_RT_EN_TEARDOWN);
1017                 break;
1018         case DMA_MEM_TO_DEV:
1019                 udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
1020                                    UDMA_PEER_RT_EN_ENABLE |
1021                                    UDMA_PEER_RT_EN_FLUSH);
1022                 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
1023                                    UDMA_CHAN_RT_CTL_EN |
1024                                    UDMA_CHAN_RT_CTL_TDOWN);
1025                 break;
1026         case DMA_MEM_TO_MEM:
1027                 udma_tchanrt_write(uc, UDMA_CHAN_RT_CTL_REG,
1028                                    UDMA_CHAN_RT_CTL_EN |
1029                                    UDMA_CHAN_RT_CTL_TDOWN);
1030                 break;
1031         default:
1032                 uc->state = old_state;
1033                 complete_all(&uc->teardown_completed);
1034                 return -EINVAL;
1035         }
1036
1037         return 0;
1038 }
1039
1040 static void udma_cyclic_packet_elapsed(struct udma_chan *uc)
1041 {
1042         struct udma_desc *d = uc->desc;
1043         struct cppi5_host_desc_t *h_desc;
1044
1045         h_desc = d->hwdesc[d->desc_idx].cppi5_desc_vaddr;
1046         cppi5_hdesc_reset_to_original(h_desc);
1047         udma_push_to_ring(uc, d->desc_idx);
1048         d->desc_idx = (d->desc_idx + 1) % d->sglen;
1049 }
1050
1051 static inline void udma_fetch_epib(struct udma_chan *uc, struct udma_desc *d)
1052 {
1053         struct cppi5_host_desc_t *h_desc = d->hwdesc[0].cppi5_desc_vaddr;
1054
1055         memcpy(d->metadata, h_desc->epib, d->metadata_size);
1056 }
1057
1058 static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d)
1059 {
1060         u32 peer_bcnt, bcnt;
1061
1062         /*
1063          * Only TX towards PDMA is affected.
1064          * If DMA_PREP_INTERRUPT is not set by consumer then skip the transfer
1065          * completion calculation, consumer must ensure that there is no stale
1066          * data in DMA fabric in this case.
1067          */
1068         if (uc->config.ep_type == PSIL_EP_NATIVE ||
1069             uc->config.dir != DMA_MEM_TO_DEV || !(uc->config.tx_flags & DMA_PREP_INTERRUPT))
1070                 return true;
1071
1072         peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG);
1073         bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
1074
1075         /* Transfer is incomplete, store current residue and time stamp */
1076         if (peer_bcnt < bcnt) {
1077                 uc->tx_drain.residue = bcnt - peer_bcnt;
1078                 uc->tx_drain.tstamp = ktime_get();
1079                 return false;
1080         }
1081
1082         return true;
1083 }
1084
1085 static void udma_check_tx_completion(struct work_struct *work)
1086 {
1087         struct udma_chan *uc = container_of(work, typeof(*uc),
1088                                             tx_drain.work.work);
1089         bool desc_done = true;
1090         u32 residue_diff;
1091         ktime_t time_diff;
1092         unsigned long delay;
1093
1094         while (1) {
1095                 if (uc->desc) {
1096                         /* Get previous residue and time stamp */
1097                         residue_diff = uc->tx_drain.residue;
1098                         time_diff = uc->tx_drain.tstamp;
1099                         /*
1100                          * Get current residue and time stamp or see if
1101                          * transfer is complete
1102                          */
1103                         desc_done = udma_is_desc_really_done(uc, uc->desc);
1104                 }
1105
1106                 if (!desc_done) {
1107                         /*
1108                          * Find the time delta and residue delta w.r.t
1109                          * previous poll
1110                          */
1111                         time_diff = ktime_sub(uc->tx_drain.tstamp,
1112                                               time_diff) + 1;
1113                         residue_diff -= uc->tx_drain.residue;
1114                         if (residue_diff) {
1115                                 /*
1116                                  * Try to guess when we should check
1117                                  * next time by calculating rate at
1118                                  * which data is being drained at the
1119                                  * peer device
1120                                  */
1121                                 delay = (time_diff / residue_diff) *
1122                                         uc->tx_drain.residue;
1123                         } else {
1124                                 /* No progress, check again in 1 second  */
1125                                 schedule_delayed_work(&uc->tx_drain.work, HZ);
1126                                 break;
1127                         }
1128
1129                         usleep_range(ktime_to_us(delay),
1130                                      ktime_to_us(delay) + 10);
1131                         continue;
1132                 }
1133
1134                 if (uc->desc) {
1135                         struct udma_desc *d = uc->desc;
1136
1137                         udma_decrement_byte_counters(uc, d->residue);
1138                         udma_start(uc);
1139                         vchan_cookie_complete(&d->vd);
1140                         break;
1141                 }
1142
1143                 break;
1144         }
1145 }
1146
1147 static irqreturn_t udma_ring_irq_handler(int irq, void *data)
1148 {
1149         struct udma_chan *uc = data;
1150         struct udma_desc *d;
1151         dma_addr_t paddr = 0;
1152
1153         if (udma_pop_from_ring(uc, &paddr) || !paddr)
1154                 return IRQ_HANDLED;
1155
1156         spin_lock(&uc->vc.lock);
1157
1158         /* Teardown completion message */
1159         if (cppi5_desc_is_tdcm(paddr)) {
1160                 complete_all(&uc->teardown_completed);
1161
1162                 if (uc->terminated_desc) {
1163                         udma_desc_free(&uc->terminated_desc->vd);
1164                         uc->terminated_desc = NULL;
1165                 }
1166
1167                 if (!uc->desc)
1168                         udma_start(uc);
1169
1170                 goto out;
1171         }
1172
1173         d = udma_udma_desc_from_paddr(uc, paddr);
1174
1175         if (d) {
1176                 dma_addr_t desc_paddr = udma_curr_cppi5_desc_paddr(d,
1177                                                                    d->desc_idx);
1178                 if (desc_paddr != paddr) {
1179                         dev_err(uc->ud->dev, "not matching descriptors!\n");
1180                         goto out;
1181                 }
1182
1183                 if (d == uc->desc) {
1184                         /* active descriptor */
1185                         if (uc->cyclic) {
1186                                 udma_cyclic_packet_elapsed(uc);
1187                                 vchan_cyclic_callback(&d->vd);
1188                         } else {
1189                                 if (udma_is_desc_really_done(uc, d)) {
1190                                         udma_decrement_byte_counters(uc, d->residue);
1191                                         udma_start(uc);
1192                                         vchan_cookie_complete(&d->vd);
1193                                 } else {
1194                                         schedule_delayed_work(&uc->tx_drain.work,
1195                                                               0);
1196                                 }
1197                         }
1198                 } else {
1199                         /*
1200                          * terminated descriptor, mark the descriptor as
1201                          * completed to update the channel's cookie marker
1202                          */
1203                         dma_cookie_complete(&d->vd.tx);
1204                 }
1205         }
1206 out:
1207         spin_unlock(&uc->vc.lock);
1208
1209         return IRQ_HANDLED;
1210 }
1211
1212 static irqreturn_t udma_udma_irq_handler(int irq, void *data)
1213 {
1214         struct udma_chan *uc = data;
1215         struct udma_desc *d;
1216
1217         spin_lock(&uc->vc.lock);
1218         d = uc->desc;
1219         if (d) {
1220                 d->tr_idx = (d->tr_idx + 1) % d->sglen;
1221
1222                 if (uc->cyclic) {
1223                         vchan_cyclic_callback(&d->vd);
1224                 } else {
1225                         /* TODO: figure out the real amount of data */
1226                         udma_decrement_byte_counters(uc, d->residue);
1227                         udma_start(uc);
1228                         vchan_cookie_complete(&d->vd);
1229                 }
1230         }
1231
1232         spin_unlock(&uc->vc.lock);
1233
1234         return IRQ_HANDLED;
1235 }
1236
1237 /**
1238  * __udma_alloc_gp_rflow_range - alloc range of GP RX flows
1239  * @ud: UDMA device
1240  * @from: Start the search from this flow id number
1241  * @cnt: Number of consecutive flow ids to allocate
1242  *
1243  * Allocate range of RX flow ids for future use, those flows can be requested
1244  * only using explicit flow id number. if @from is set to -1 it will try to find
1245  * first free range. if @from is positive value it will force allocation only
1246  * of the specified range of flows.
1247  *
1248  * Returns -ENOMEM if can't find free range.
1249  * -EEXIST if requested range is busy.
1250  * -EINVAL if wrong input values passed.
1251  * Returns flow id on success.
1252  */
1253 static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
1254 {
1255         int start, tmp_from;
1256         DECLARE_BITMAP(tmp, K3_UDMA_MAX_RFLOWS);
1257
1258         tmp_from = from;
1259         if (tmp_from < 0)
1260                 tmp_from = ud->rchan_cnt;
1261         /* default flows can't be allocated and accessible only by id */
1262         if (tmp_from < ud->rchan_cnt)
1263                 return -EINVAL;
1264
1265         if (tmp_from + cnt > ud->rflow_cnt)
1266                 return -EINVAL;
1267
1268         bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated,
1269                   ud->rflow_cnt);
1270
1271         start = bitmap_find_next_zero_area(tmp,
1272                                            ud->rflow_cnt,
1273                                            tmp_from, cnt, 0);
1274         if (start >= ud->rflow_cnt)
1275                 return -ENOMEM;
1276
1277         if (from >= 0 && start != from)
1278                 return -EEXIST;
1279
1280         bitmap_set(ud->rflow_gp_map_allocated, start, cnt);
1281         return start;
1282 }
1283
1284 static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt)
1285 {
1286         if (from < ud->rchan_cnt)
1287                 return -EINVAL;
1288         if (from + cnt > ud->rflow_cnt)
1289                 return -EINVAL;
1290
1291         bitmap_clear(ud->rflow_gp_map_allocated, from, cnt);
1292         return 0;
1293 }
1294
1295 static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id)
1296 {
1297         /*
1298          * Attempt to request rflow by ID can be made for any rflow
1299          * if not in use with assumption that caller knows what's doing.
1300          * TI-SCI FW will perform additional permission check ant way, it's
1301          * safe
1302          */
1303
1304         if (id < 0 || id >= ud->rflow_cnt)
1305                 return ERR_PTR(-ENOENT);
1306
1307         if (test_bit(id, ud->rflow_in_use))
1308                 return ERR_PTR(-ENOENT);
1309
1310         if (ud->rflow_gp_map) {
1311                 /* GP rflow has to be allocated first */
1312                 if (!test_bit(id, ud->rflow_gp_map) &&
1313                     !test_bit(id, ud->rflow_gp_map_allocated))
1314                         return ERR_PTR(-EINVAL);
1315         }
1316
1317         dev_dbg(ud->dev, "get rflow%d\n", id);
1318         set_bit(id, ud->rflow_in_use);
1319         return &ud->rflows[id];
1320 }
1321
1322 static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow)
1323 {
1324         if (!test_bit(rflow->id, ud->rflow_in_use)) {
1325                 dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id);
1326                 return;
1327         }
1328
1329         dev_dbg(ud->dev, "put rflow%d\n", rflow->id);
1330         clear_bit(rflow->id, ud->rflow_in_use);
1331 }
1332
1333 #define UDMA_RESERVE_RESOURCE(res)                                      \
1334 static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud,     \
1335                                                enum udma_tp_level tpl,  \
1336                                                int id)                  \
1337 {                                                                       \
1338         if (id >= 0) {                                                  \
1339                 if (test_bit(id, ud->res##_map)) {                      \
1340                         dev_err(ud->dev, "res##%d is in use\n", id);    \
1341                         return ERR_PTR(-ENOENT);                        \
1342                 }                                                       \
1343         } else {                                                        \
1344                 int start;                                              \
1345                                                                         \
1346                 if (tpl >= ud->res##_tpl.levels)                        \
1347                         tpl = ud->res##_tpl.levels - 1;                 \
1348                                                                         \
1349                 start = ud->res##_tpl.start_idx[tpl];                   \
1350                                                                         \
1351                 id = find_next_zero_bit(ud->res##_map, ud->res##_cnt,   \
1352                                         start);                         \
1353                 if (id == ud->res##_cnt) {                              \
1354                         return ERR_PTR(-ENOENT);                        \
1355                 }                                                       \
1356         }                                                               \
1357                                                                         \
1358         set_bit(id, ud->res##_map);                                     \
1359         return &ud->res##s[id];                                         \
1360 }
1361
1362 UDMA_RESERVE_RESOURCE(bchan);
1363 UDMA_RESERVE_RESOURCE(tchan);
1364 UDMA_RESERVE_RESOURCE(rchan);
1365
1366 static int bcdma_get_bchan(struct udma_chan *uc)
1367 {
1368         struct udma_dev *ud = uc->ud;
1369         enum udma_tp_level tpl;
1370         int ret;
1371
1372         if (uc->bchan) {
1373                 dev_dbg(ud->dev, "chan%d: already have bchan%d allocated\n",
1374                         uc->id, uc->bchan->id);
1375                 return 0;
1376         }
1377
1378         /*
1379          * Use normal channels for peripherals, and highest TPL channel for
1380          * mem2mem
1381          */
1382         if (uc->config.tr_trigger_type)
1383                 tpl = 0;
1384         else
1385                 tpl = ud->bchan_tpl.levels - 1;
1386
1387         uc->bchan = __udma_reserve_bchan(ud, tpl, -1);
1388         if (IS_ERR(uc->bchan)) {
1389                 ret = PTR_ERR(uc->bchan);
1390                 uc->bchan = NULL;
1391                 return ret;
1392         }
1393
1394         uc->tchan = uc->bchan;
1395
1396         return 0;
1397 }
1398
1399 static int udma_get_tchan(struct udma_chan *uc)
1400 {
1401         struct udma_dev *ud = uc->ud;
1402         int ret;
1403
1404         if (uc->tchan) {
1405                 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n",
1406                         uc->id, uc->tchan->id);
1407                 return 0;
1408         }
1409
1410         /*
1411          * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels.
1412          * For PKTDMA mapped channels it is configured to a channel which must
1413          * be used to service the peripheral.
1414          */
1415         uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl,
1416                                          uc->config.mapped_channel_id);
1417         if (IS_ERR(uc->tchan)) {
1418                 ret = PTR_ERR(uc->tchan);
1419                 uc->tchan = NULL;
1420                 return ret;
1421         }
1422
1423         if (ud->tflow_cnt) {
1424                 int tflow_id;
1425
1426                 /* Only PKTDMA have support for tx flows */
1427                 if (uc->config.default_flow_id >= 0)
1428                         tflow_id = uc->config.default_flow_id;
1429                 else
1430                         tflow_id = uc->tchan->id;
1431
1432                 if (test_bit(tflow_id, ud->tflow_map)) {
1433                         dev_err(ud->dev, "tflow%d is in use\n", tflow_id);
1434                         clear_bit(uc->tchan->id, ud->tchan_map);
1435                         uc->tchan = NULL;
1436                         return -ENOENT;
1437                 }
1438
1439                 uc->tchan->tflow_id = tflow_id;
1440                 set_bit(tflow_id, ud->tflow_map);
1441         } else {
1442                 uc->tchan->tflow_id = -1;
1443         }
1444
1445         return 0;
1446 }
1447
1448 static int udma_get_rchan(struct udma_chan *uc)
1449 {
1450         struct udma_dev *ud = uc->ud;
1451         int ret;
1452
1453         if (uc->rchan) {
1454                 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n",
1455                         uc->id, uc->rchan->id);
1456                 return 0;
1457         }
1458
1459         /*
1460          * mapped_channel_id is -1 for UDMA, BCDMA and PKTDMA unmapped channels.
1461          * For PKTDMA mapped channels it is configured to a channel which must
1462          * be used to service the peripheral.
1463          */
1464         uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl,
1465                                          uc->config.mapped_channel_id);
1466         if (IS_ERR(uc->rchan)) {
1467                 ret = PTR_ERR(uc->rchan);
1468                 uc->rchan = NULL;
1469                 return ret;
1470         }
1471
1472         return 0;
1473 }
1474
1475 static int udma_get_chan_pair(struct udma_chan *uc)
1476 {
1477         struct udma_dev *ud = uc->ud;
1478         int chan_id, end;
1479
1480         if ((uc->tchan && uc->rchan) && uc->tchan->id == uc->rchan->id) {
1481                 dev_info(ud->dev, "chan%d: already have %d pair allocated\n",
1482                          uc->id, uc->tchan->id);
1483                 return 0;
1484         }
1485
1486         if (uc->tchan) {
1487                 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n",
1488                         uc->id, uc->tchan->id);
1489                 return -EBUSY;
1490         } else if (uc->rchan) {
1491                 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n",
1492                         uc->id, uc->rchan->id);
1493                 return -EBUSY;
1494         }
1495
1496         /* Can be optimized, but let's have it like this for now */
1497         end = min(ud->tchan_cnt, ud->rchan_cnt);
1498         /*
1499          * Try to use the highest TPL channel pair for MEM_TO_MEM channels
1500          * Note: in UDMAP the channel TPL is symmetric between tchan and rchan
1501          */
1502         chan_id = ud->tchan_tpl.start_idx[ud->tchan_tpl.levels - 1];
1503         for (; chan_id < end; chan_id++) {
1504                 if (!test_bit(chan_id, ud->tchan_map) &&
1505                     !test_bit(chan_id, ud->rchan_map))
1506                         break;
1507         }
1508
1509         if (chan_id == end)
1510                 return -ENOENT;
1511
1512         set_bit(chan_id, ud->tchan_map);
1513         set_bit(chan_id, ud->rchan_map);
1514         uc->tchan = &ud->tchans[chan_id];
1515         uc->rchan = &ud->rchans[chan_id];
1516
1517         /* UDMA does not use tx flows */
1518         uc->tchan->tflow_id = -1;
1519
1520         return 0;
1521 }
1522
1523 static int udma_get_rflow(struct udma_chan *uc, int flow_id)
1524 {
1525         struct udma_dev *ud = uc->ud;
1526         int ret;
1527
1528         if (!uc->rchan) {
1529                 dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id);
1530                 return -EINVAL;
1531         }
1532
1533         if (uc->rflow) {
1534                 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n",
1535                         uc->id, uc->rflow->id);
1536                 return 0;
1537         }
1538
1539         uc->rflow = __udma_get_rflow(ud, flow_id);
1540         if (IS_ERR(uc->rflow)) {
1541                 ret = PTR_ERR(uc->rflow);
1542                 uc->rflow = NULL;
1543                 return ret;
1544         }
1545
1546         return 0;
1547 }
1548
1549 static void bcdma_put_bchan(struct udma_chan *uc)
1550 {
1551         struct udma_dev *ud = uc->ud;
1552
1553         if (uc->bchan) {
1554                 dev_dbg(ud->dev, "chan%d: put bchan%d\n", uc->id,
1555                         uc->bchan->id);
1556                 clear_bit(uc->bchan->id, ud->bchan_map);
1557                 uc->bchan = NULL;
1558                 uc->tchan = NULL;
1559         }
1560 }
1561
1562 static void udma_put_rchan(struct udma_chan *uc)
1563 {
1564         struct udma_dev *ud = uc->ud;
1565
1566         if (uc->rchan) {
1567                 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id,
1568                         uc->rchan->id);
1569                 clear_bit(uc->rchan->id, ud->rchan_map);
1570                 uc->rchan = NULL;
1571         }
1572 }
1573
1574 static void udma_put_tchan(struct udma_chan *uc)
1575 {
1576         struct udma_dev *ud = uc->ud;
1577
1578         if (uc->tchan) {
1579                 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id,
1580                         uc->tchan->id);
1581                 clear_bit(uc->tchan->id, ud->tchan_map);
1582
1583                 if (uc->tchan->tflow_id >= 0)
1584                         clear_bit(uc->tchan->tflow_id, ud->tflow_map);
1585
1586                 uc->tchan = NULL;
1587         }
1588 }
1589
1590 static void udma_put_rflow(struct udma_chan *uc)
1591 {
1592         struct udma_dev *ud = uc->ud;
1593
1594         if (uc->rflow) {
1595                 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id,
1596                         uc->rflow->id);
1597                 __udma_put_rflow(ud, uc->rflow);
1598                 uc->rflow = NULL;
1599         }
1600 }
1601
1602 static void bcdma_free_bchan_resources(struct udma_chan *uc)
1603 {
1604         if (!uc->bchan)
1605                 return;
1606
1607         k3_ringacc_ring_free(uc->bchan->tc_ring);
1608         k3_ringacc_ring_free(uc->bchan->t_ring);
1609         uc->bchan->tc_ring = NULL;
1610         uc->bchan->t_ring = NULL;
1611         k3_configure_chan_coherency(&uc->vc.chan, 0);
1612
1613         bcdma_put_bchan(uc);
1614 }
1615
1616 static int bcdma_alloc_bchan_resources(struct udma_chan *uc)
1617 {
1618         struct k3_ring_cfg ring_cfg;
1619         struct udma_dev *ud = uc->ud;
1620         int ret;
1621
1622         ret = bcdma_get_bchan(uc);
1623         if (ret)
1624                 return ret;
1625
1626         ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->bchan->id, -1,
1627                                             &uc->bchan->t_ring,
1628                                             &uc->bchan->tc_ring);
1629         if (ret) {
1630                 ret = -EBUSY;
1631                 goto err_ring;
1632         }
1633
1634         memset(&ring_cfg, 0, sizeof(ring_cfg));
1635         ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1636         ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
1637         ring_cfg.mode = K3_RINGACC_RING_MODE_RING;
1638
1639         k3_configure_chan_coherency(&uc->vc.chan, ud->asel);
1640         ring_cfg.asel = ud->asel;
1641         ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);
1642
1643         ret = k3_ringacc_ring_cfg(uc->bchan->t_ring, &ring_cfg);
1644         if (ret)
1645                 goto err_ringcfg;
1646
1647         return 0;
1648
1649 err_ringcfg:
1650         k3_ringacc_ring_free(uc->bchan->tc_ring);
1651         uc->bchan->tc_ring = NULL;
1652         k3_ringacc_ring_free(uc->bchan->t_ring);
1653         uc->bchan->t_ring = NULL;
1654         k3_configure_chan_coherency(&uc->vc.chan, 0);
1655 err_ring:
1656         bcdma_put_bchan(uc);
1657
1658         return ret;
1659 }
1660
1661 static void udma_free_tx_resources(struct udma_chan *uc)
1662 {
1663         if (!uc->tchan)
1664                 return;
1665
1666         k3_ringacc_ring_free(uc->tchan->t_ring);
1667         k3_ringacc_ring_free(uc->tchan->tc_ring);
1668         uc->tchan->t_ring = NULL;
1669         uc->tchan->tc_ring = NULL;
1670
1671         udma_put_tchan(uc);
1672 }
1673
1674 static int udma_alloc_tx_resources(struct udma_chan *uc)
1675 {
1676         struct k3_ring_cfg ring_cfg;
1677         struct udma_dev *ud = uc->ud;
1678         struct udma_tchan *tchan;
1679         int ring_idx, ret;
1680
1681         ret = udma_get_tchan(uc);
1682         if (ret)
1683                 return ret;
1684
1685         tchan = uc->tchan;
1686         if (tchan->tflow_id >= 0)
1687                 ring_idx = tchan->tflow_id;
1688         else
1689                 ring_idx = ud->bchan_cnt + tchan->id;
1690
1691         ret = k3_ringacc_request_rings_pair(ud->ringacc, ring_idx, -1,
1692                                             &tchan->t_ring,
1693                                             &tchan->tc_ring);
1694         if (ret) {
1695                 ret = -EBUSY;
1696                 goto err_ring;
1697         }
1698
1699         memset(&ring_cfg, 0, sizeof(ring_cfg));
1700         ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1701         ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
1702         if (ud->match_data->type == DMA_TYPE_UDMA) {
1703                 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;
1704         } else {
1705                 ring_cfg.mode = K3_RINGACC_RING_MODE_RING;
1706
1707                 k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel);
1708                 ring_cfg.asel = uc->config.asel;
1709                 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);
1710         }
1711
1712         ret = k3_ringacc_ring_cfg(tchan->t_ring, &ring_cfg);
1713         ret |= k3_ringacc_ring_cfg(tchan->tc_ring, &ring_cfg);
1714
1715         if (ret)
1716                 goto err_ringcfg;
1717
1718         return 0;
1719
1720 err_ringcfg:
1721         k3_ringacc_ring_free(uc->tchan->tc_ring);
1722         uc->tchan->tc_ring = NULL;
1723         k3_ringacc_ring_free(uc->tchan->t_ring);
1724         uc->tchan->t_ring = NULL;
1725 err_ring:
1726         udma_put_tchan(uc);
1727
1728         return ret;
1729 }
1730
1731 static void udma_free_rx_resources(struct udma_chan *uc)
1732 {
1733         if (!uc->rchan)
1734                 return;
1735
1736         if (uc->rflow) {
1737                 struct udma_rflow *rflow = uc->rflow;
1738
1739                 k3_ringacc_ring_free(rflow->fd_ring);
1740                 k3_ringacc_ring_free(rflow->r_ring);
1741                 rflow->fd_ring = NULL;
1742                 rflow->r_ring = NULL;
1743
1744                 udma_put_rflow(uc);
1745         }
1746
1747         udma_put_rchan(uc);
1748 }
1749
1750 static int udma_alloc_rx_resources(struct udma_chan *uc)
1751 {
1752         struct udma_dev *ud = uc->ud;
1753         struct k3_ring_cfg ring_cfg;
1754         struct udma_rflow *rflow;
1755         int fd_ring_id;
1756         int ret;
1757
1758         ret = udma_get_rchan(uc);
1759         if (ret)
1760                 return ret;
1761
1762         /* For MEM_TO_MEM we don't need rflow or rings */
1763         if (uc->config.dir == DMA_MEM_TO_MEM)
1764                 return 0;
1765
1766         if (uc->config.default_flow_id >= 0)
1767                 ret = udma_get_rflow(uc, uc->config.default_flow_id);
1768         else
1769                 ret = udma_get_rflow(uc, uc->rchan->id);
1770
1771         if (ret) {
1772                 ret = -EBUSY;
1773                 goto err_rflow;
1774         }
1775
1776         rflow = uc->rflow;
1777         if (ud->tflow_cnt)
1778                 fd_ring_id = ud->tflow_cnt + rflow->id;
1779         else
1780                 fd_ring_id = ud->bchan_cnt + ud->tchan_cnt + ud->echan_cnt +
1781                              uc->rchan->id;
1782
1783         ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1,
1784                                             &rflow->fd_ring, &rflow->r_ring);
1785         if (ret) {
1786                 ret = -EBUSY;
1787                 goto err_ring;
1788         }
1789
1790         memset(&ring_cfg, 0, sizeof(ring_cfg));
1791
1792         ring_cfg.elm_size = K3_RINGACC_RING_ELSIZE_8;
1793         if (ud->match_data->type == DMA_TYPE_UDMA) {
1794                 if (uc->config.pkt_mode)
1795                         ring_cfg.size = SG_MAX_SEGMENTS;
1796                 else
1797                         ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1798
1799                 ring_cfg.mode = K3_RINGACC_RING_MODE_MESSAGE;
1800         } else {
1801                 ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1802                 ring_cfg.mode = K3_RINGACC_RING_MODE_RING;
1803
1804                 k3_configure_chan_coherency(&uc->vc.chan, uc->config.asel);
1805                 ring_cfg.asel = uc->config.asel;
1806                 ring_cfg.dma_dev = dmaengine_get_dma_device(&uc->vc.chan);
1807         }
1808
1809         ret = k3_ringacc_ring_cfg(rflow->fd_ring, &ring_cfg);
1810
1811         ring_cfg.size = K3_UDMA_DEFAULT_RING_SIZE;
1812         ret |= k3_ringacc_ring_cfg(rflow->r_ring, &ring_cfg);
1813
1814         if (ret)
1815                 goto err_ringcfg;
1816
1817         return 0;
1818
1819 err_ringcfg:
1820         k3_ringacc_ring_free(rflow->r_ring);
1821         rflow->r_ring = NULL;
1822         k3_ringacc_ring_free(rflow->fd_ring);
1823         rflow->fd_ring = NULL;
1824 err_ring:
1825         udma_put_rflow(uc);
1826 err_rflow:
1827         udma_put_rchan(uc);
1828
1829         return ret;
1830 }
1831
1832 #define TISCI_BCDMA_BCHAN_VALID_PARAMS (                        \
1833         TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |       \
1834         TI_SCI_MSG_VALUE_RM_UDMAP_CH_EXTENDED_CH_TYPE_VALID)
1835
1836 #define TISCI_BCDMA_TCHAN_VALID_PARAMS (                        \
1837         TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |       \
1838         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID)
1839
1840 #define TISCI_BCDMA_RCHAN_VALID_PARAMS (                        \
1841         TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID)
1842
1843 #define TISCI_UDMA_TCHAN_VALID_PARAMS (                         \
1844         TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |       \
1845         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |      \
1846         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |    \
1847         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |          \
1848         TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |      \
1849         TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |         \
1850         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |            \
1851         TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1852
1853 #define TISCI_UDMA_RCHAN_VALID_PARAMS (                         \
1854         TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |       \
1855         TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |         \
1856         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |            \
1857         TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |          \
1858         TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_SHORT_VALID |    \
1859         TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_IGNORE_LONG_VALID |     \
1860         TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |    \
1861         TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID |      \
1862         TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID)
1863
1864 static int udma_tisci_m2m_channel_config(struct udma_chan *uc)
1865 {
1866         struct udma_dev *ud = uc->ud;
1867         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1868         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1869         struct udma_tchan *tchan = uc->tchan;
1870         struct udma_rchan *rchan = uc->rchan;
1871         u8 burst_size = 0;
1872         int ret;
1873         u8 tpl;
1874
1875         /* Non synchronized - mem to mem type of transfer */
1876         int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
1877         struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1878         struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
1879
1880         if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) {
1881                 tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, tchan->id);
1882
1883                 burst_size = ud->match_data->burst_size[tpl];
1884         }
1885
1886         req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS;
1887         req_tx.nav_id = tisci_rm->tisci_dev_id;
1888         req_tx.index = tchan->id;
1889         req_tx.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1890         req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1891         req_tx.txcq_qnum = tc_ring;
1892         req_tx.tx_atype = ud->atype;
1893         if (burst_size) {
1894                 req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID;
1895                 req_tx.tx_burst_size = burst_size;
1896         }
1897
1898         ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1899         if (ret) {
1900                 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
1901                 return ret;
1902         }
1903
1904         req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS;
1905         req_rx.nav_id = tisci_rm->tisci_dev_id;
1906         req_rx.index = rchan->id;
1907         req_rx.rx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2;
1908         req_rx.rxcq_qnum = tc_ring;
1909         req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR;
1910         req_rx.rx_atype = ud->atype;
1911         if (burst_size) {
1912                 req_rx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID;
1913                 req_rx.rx_burst_size = burst_size;
1914         }
1915
1916         ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
1917         if (ret)
1918                 dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret);
1919
1920         return ret;
1921 }
1922
1923 static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc)
1924 {
1925         struct udma_dev *ud = uc->ud;
1926         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1927         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1928         struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1929         struct udma_bchan *bchan = uc->bchan;
1930         u8 burst_size = 0;
1931         int ret;
1932         u8 tpl;
1933
1934         if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) {
1935                 tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, bchan->id);
1936
1937                 burst_size = ud->match_data->burst_size[tpl];
1938         }
1939
1940         req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS;
1941         req_tx.nav_id = tisci_rm->tisci_dev_id;
1942         req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN;
1943         req_tx.index = bchan->id;
1944         if (burst_size) {
1945                 req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID;
1946                 req_tx.tx_burst_size = burst_size;
1947         }
1948
1949         ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1950         if (ret)
1951                 dev_err(ud->dev, "bchan%d cfg failed %d\n", bchan->id, ret);
1952
1953         return ret;
1954 }
1955
1956 static int udma_tisci_tx_channel_config(struct udma_chan *uc)
1957 {
1958         struct udma_dev *ud = uc->ud;
1959         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
1960         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
1961         struct udma_tchan *tchan = uc->tchan;
1962         int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring);
1963         struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
1964         u32 mode, fetch_size;
1965         int ret;
1966
1967         if (uc->config.pkt_mode) {
1968                 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
1969                 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
1970                                                    uc->config.psd_size, 0);
1971         } else {
1972                 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;
1973                 fetch_size = sizeof(struct cppi5_desc_hdr_t);
1974         }
1975
1976         req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS;
1977         req_tx.nav_id = tisci_rm->tisci_dev_id;
1978         req_tx.index = tchan->id;
1979         req_tx.tx_chan_type = mode;
1980         req_tx.tx_supr_tdpkt = uc->config.notdpkt;
1981         req_tx.tx_fetch_size = fetch_size >> 2;
1982         req_tx.txcq_qnum = tc_ring;
1983         req_tx.tx_atype = uc->config.atype;
1984         if (uc->config.ep_type == PSIL_EP_PDMA_XY &&
1985             ud->match_data->flags & UDMA_FLAG_TDTYPE) {
1986                 /* wait for peer to complete the teardown for PDMAs */
1987                 req_tx.valid_params |=
1988                                 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID;
1989                 req_tx.tx_tdtype = 1;
1990         }
1991
1992         ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
1993         if (ret)
1994                 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
1995
1996         return ret;
1997 }
1998
1999 static int bcdma_tisci_tx_channel_config(struct udma_chan *uc)
2000 {
2001         struct udma_dev *ud = uc->ud;
2002         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2003         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2004         struct udma_tchan *tchan = uc->tchan;
2005         struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 };
2006         int ret;
2007
2008         req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS;
2009         req_tx.nav_id = tisci_rm->tisci_dev_id;
2010         req_tx.index = tchan->id;
2011         req_tx.tx_supr_tdpkt = uc->config.notdpkt;
2012         if (ud->match_data->flags & UDMA_FLAG_TDTYPE) {
2013                 /* wait for peer to complete the teardown for PDMAs */
2014                 req_tx.valid_params |=
2015                                 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_TDTYPE_VALID;
2016                 req_tx.tx_tdtype = 1;
2017         }
2018
2019         ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx);
2020         if (ret)
2021                 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret);
2022
2023         return ret;
2024 }
2025
2026 #define pktdma_tisci_tx_channel_config bcdma_tisci_tx_channel_config
2027
2028 static int udma_tisci_rx_channel_config(struct udma_chan *uc)
2029 {
2030         struct udma_dev *ud = uc->ud;
2031         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2032         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2033         struct udma_rchan *rchan = uc->rchan;
2034         int fd_ring = k3_ringacc_get_ring_id(uc->rflow->fd_ring);
2035         int rx_ring = k3_ringacc_get_ring_id(uc->rflow->r_ring);
2036         struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
2037         struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
2038         u32 mode, fetch_size;
2039         int ret;
2040
2041         if (uc->config.pkt_mode) {
2042                 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
2043                 fetch_size = cppi5_hdesc_calc_size(uc->config.needs_epib,
2044                                                    uc->config.psd_size, 0);
2045         } else {
2046                 mode = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_PBRR;
2047                 fetch_size = sizeof(struct cppi5_desc_hdr_t);
2048         }
2049
2050         req_rx.valid_params = TISCI_UDMA_RCHAN_VALID_PARAMS;
2051         req_rx.nav_id = tisci_rm->tisci_dev_id;
2052         req_rx.index = rchan->id;
2053         req_rx.rx_fetch_size =  fetch_size >> 2;
2054         req_rx.rxcq_qnum = rx_ring;
2055         req_rx.rx_chan_type = mode;
2056         req_rx.rx_atype = uc->config.atype;
2057
2058         ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2059         if (ret) {
2060                 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret);
2061                 return ret;
2062         }
2063
2064         flow_req.valid_params =
2065                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
2066                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
2067                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
2068                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
2069                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
2070                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
2071                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
2072                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
2073                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
2074                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
2075                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
2076                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
2077                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
2078
2079         flow_req.nav_id = tisci_rm->tisci_dev_id;
2080         flow_req.flow_index = rchan->id;
2081
2082         if (uc->config.needs_epib)
2083                 flow_req.rx_einfo_present = 1;
2084         else
2085                 flow_req.rx_einfo_present = 0;
2086         if (uc->config.psd_size)
2087                 flow_req.rx_psinfo_present = 1;
2088         else
2089                 flow_req.rx_psinfo_present = 0;
2090         flow_req.rx_error_handling = 1;
2091         flow_req.rx_dest_qnum = rx_ring;
2092         flow_req.rx_src_tag_hi_sel = UDMA_RFLOW_SRCTAG_NONE;
2093         flow_req.rx_src_tag_lo_sel = UDMA_RFLOW_SRCTAG_SRC_TAG;
2094         flow_req.rx_dest_tag_hi_sel = UDMA_RFLOW_DSTTAG_DST_TAG_HI;
2095         flow_req.rx_dest_tag_lo_sel = UDMA_RFLOW_DSTTAG_DST_TAG_LO;
2096         flow_req.rx_fdq0_sz0_qnum = fd_ring;
2097         flow_req.rx_fdq1_qnum = fd_ring;
2098         flow_req.rx_fdq2_qnum = fd_ring;
2099         flow_req.rx_fdq3_qnum = fd_ring;
2100
2101         ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
2102
2103         if (ret)
2104                 dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret);
2105
2106         return 0;
2107 }
2108
2109 static int bcdma_tisci_rx_channel_config(struct udma_chan *uc)
2110 {
2111         struct udma_dev *ud = uc->ud;
2112         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2113         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2114         struct udma_rchan *rchan = uc->rchan;
2115         struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
2116         int ret;
2117
2118         req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS;
2119         req_rx.nav_id = tisci_rm->tisci_dev_id;
2120         req_rx.index = rchan->id;
2121
2122         ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2123         if (ret)
2124                 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret);
2125
2126         return ret;
2127 }
2128
2129 static int pktdma_tisci_rx_channel_config(struct udma_chan *uc)
2130 {
2131         struct udma_dev *ud = uc->ud;
2132         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
2133         const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops;
2134         struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 };
2135         struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 };
2136         int ret;
2137
2138         req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS;
2139         req_rx.nav_id = tisci_rm->tisci_dev_id;
2140         req_rx.index = uc->rchan->id;
2141
2142         ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx);
2143         if (ret) {
2144                 dev_err(ud->dev, "rchan%d cfg failed %d\n", uc->rchan->id, ret);
2145                 return ret;
2146         }
2147
2148         flow_req.valid_params =
2149                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
2150                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
2151                 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID;
2152
2153         flow_req.nav_id = tisci_rm->tisci_dev_id;
2154         flow_req.flow_index = uc->rflow->id;
2155
2156         if (uc->config.needs_epib)
2157                 flow_req.rx_einfo_present = 1;
2158         else
2159                 flow_req.rx_einfo_present = 0;
2160         if (uc->config.psd_size)
2161                 flow_req.rx_psinfo_present = 1;
2162         else
2163                 flow_req.rx_psinfo_present = 0;
2164         flow_req.rx_error_handling = 1;
2165
2166         ret = tisci_ops->rx_flow_cfg(tisci_rm->tisci, &flow_req);
2167
2168         if (ret)
2169                 dev_err(ud->dev, "flow%d config failed: %d\n", uc->rflow->id,
2170                         ret);
2171
2172         return ret;
2173 }
2174
2175 static int udma_alloc_chan_resources(struct dma_chan *chan)
2176 {
2177         struct udma_chan *uc = to_udma_chan(chan);
2178         struct udma_dev *ud = to_udma_dev(chan->device);
2179         const struct udma_soc_data *soc_data = ud->soc_data;
2180         struct k3_ring *irq_ring;
2181         u32 irq_udma_idx;
2182         int ret;
2183
2184         uc->dma_dev = ud->dev;
2185
2186         if (uc->config.pkt_mode || uc->config.dir == DMA_MEM_TO_MEM) {
2187                 uc->use_dma_pool = true;
2188                 /* in case of MEM_TO_MEM we have maximum of two TRs */
2189                 if (uc->config.dir == DMA_MEM_TO_MEM) {
2190                         uc->config.hdesc_size = cppi5_trdesc_calc_size(
2191                                         sizeof(struct cppi5_tr_type15_t), 2);
2192                         uc->config.pkt_mode = false;
2193                 }
2194         }
2195
2196         if (uc->use_dma_pool) {
2197                 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,
2198                                                  uc->config.hdesc_size,
2199                                                  ud->desc_align,
2200                                                  0);
2201                 if (!uc->hdesc_pool) {
2202                         dev_err(ud->ddev.dev,
2203                                 "Descriptor pool allocation failed\n");
2204                         uc->use_dma_pool = false;
2205                         ret = -ENOMEM;
2206                         goto err_cleanup;
2207                 }
2208         }
2209
2210         /*
2211          * Make sure that the completion is in a known state:
2212          * No teardown, the channel is idle
2213          */
2214         reinit_completion(&uc->teardown_completed);
2215         complete_all(&uc->teardown_completed);
2216         uc->state = UDMA_CHAN_IS_IDLE;
2217
2218         switch (uc->config.dir) {
2219         case DMA_MEM_TO_MEM:
2220                 /* Non synchronized - mem to mem type of transfer */
2221                 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
2222                         uc->id);
2223
2224                 ret = udma_get_chan_pair(uc);
2225                 if (ret)
2226                         goto err_cleanup;
2227
2228                 ret = udma_alloc_tx_resources(uc);
2229                 if (ret) {
2230                         udma_put_rchan(uc);
2231                         goto err_cleanup;
2232                 }
2233
2234                 ret = udma_alloc_rx_resources(uc);
2235                 if (ret) {
2236                         udma_free_tx_resources(uc);
2237                         goto err_cleanup;
2238                 }
2239
2240                 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2241                 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2242                                         K3_PSIL_DST_THREAD_ID_OFFSET;
2243
2244                 irq_ring = uc->tchan->tc_ring;
2245                 irq_udma_idx = uc->tchan->id;
2246
2247                 ret = udma_tisci_m2m_channel_config(uc);
2248                 break;
2249         case DMA_MEM_TO_DEV:
2250                 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
2251                 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2252                         uc->id);
2253
2254                 ret = udma_alloc_tx_resources(uc);
2255                 if (ret)
2256                         goto err_cleanup;
2257
2258                 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2259                 uc->config.dst_thread = uc->config.remote_thread_id;
2260                 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2261
2262                 irq_ring = uc->tchan->tc_ring;
2263                 irq_udma_idx = uc->tchan->id;
2264
2265                 ret = udma_tisci_tx_channel_config(uc);
2266                 break;
2267         case DMA_DEV_TO_MEM:
2268                 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
2269                 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2270                         uc->id);
2271
2272                 ret = udma_alloc_rx_resources(uc);
2273                 if (ret)
2274                         goto err_cleanup;
2275
2276                 uc->config.src_thread = uc->config.remote_thread_id;
2277                 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2278                                         K3_PSIL_DST_THREAD_ID_OFFSET;
2279
2280                 irq_ring = uc->rflow->r_ring;
2281                 irq_udma_idx = soc_data->oes.udma_rchan + uc->rchan->id;
2282
2283                 ret = udma_tisci_rx_channel_config(uc);
2284                 break;
2285         default:
2286                 /* Can not happen */
2287                 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2288                         __func__, uc->id, uc->config.dir);
2289                 ret = -EINVAL;
2290                 goto err_cleanup;
2291
2292         }
2293
2294         /* check if the channel configuration was successful */
2295         if (ret)
2296                 goto err_res_free;
2297
2298         if (udma_is_chan_running(uc)) {
2299                 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2300                 udma_reset_chan(uc, false);
2301                 if (udma_is_chan_running(uc)) {
2302                         dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2303                         ret = -EBUSY;
2304                         goto err_res_free;
2305                 }
2306         }
2307
2308         /* PSI-L pairing */
2309         ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
2310         if (ret) {
2311                 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2312                         uc->config.src_thread, uc->config.dst_thread);
2313                 goto err_res_free;
2314         }
2315
2316         uc->psil_paired = true;
2317
2318         uc->irq_num_ring = k3_ringacc_get_ring_irq_num(irq_ring);
2319         if (uc->irq_num_ring <= 0) {
2320                 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
2321                         k3_ringacc_get_ring_id(irq_ring));
2322                 ret = -EINVAL;
2323                 goto err_psi_free;
2324         }
2325
2326         ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
2327                           IRQF_TRIGGER_HIGH, uc->name, uc);
2328         if (ret) {
2329                 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
2330                 goto err_irq_free;
2331         }
2332
2333         /* Event from UDMA (TR events) only needed for slave TR mode channels */
2334         if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) {
2335                 uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx);
2336                 if (uc->irq_num_udma <= 0) {
2337                         dev_err(ud->dev, "Failed to get udma irq (index: %u)\n",
2338                                 irq_udma_idx);
2339                         free_irq(uc->irq_num_ring, uc);
2340                         ret = -EINVAL;
2341                         goto err_irq_free;
2342                 }
2343
2344                 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,
2345                                   uc->name, uc);
2346                 if (ret) {
2347                         dev_err(ud->dev, "chan%d: UDMA irq request failed\n",
2348                                 uc->id);
2349                         free_irq(uc->irq_num_ring, uc);
2350                         goto err_irq_free;
2351                 }
2352         } else {
2353                 uc->irq_num_udma = 0;
2354         }
2355
2356         udma_reset_rings(uc);
2357
2358         return 0;
2359
2360 err_irq_free:
2361         uc->irq_num_ring = 0;
2362         uc->irq_num_udma = 0;
2363 err_psi_free:
2364         navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread);
2365         uc->psil_paired = false;
2366 err_res_free:
2367         udma_free_tx_resources(uc);
2368         udma_free_rx_resources(uc);
2369 err_cleanup:
2370         udma_reset_uchan(uc);
2371
2372         if (uc->use_dma_pool) {
2373                 dma_pool_destroy(uc->hdesc_pool);
2374                 uc->use_dma_pool = false;
2375         }
2376
2377         return ret;
2378 }
2379
2380 static int bcdma_alloc_chan_resources(struct dma_chan *chan)
2381 {
2382         struct udma_chan *uc = to_udma_chan(chan);
2383         struct udma_dev *ud = to_udma_dev(chan->device);
2384         const struct udma_oes_offsets *oes = &ud->soc_data->oes;
2385         u32 irq_udma_idx, irq_ring_idx;
2386         int ret;
2387
2388         /* Only TR mode is supported */
2389         uc->config.pkt_mode = false;
2390
2391         /*
2392          * Make sure that the completion is in a known state:
2393          * No teardown, the channel is idle
2394          */
2395         reinit_completion(&uc->teardown_completed);
2396         complete_all(&uc->teardown_completed);
2397         uc->state = UDMA_CHAN_IS_IDLE;
2398
2399         switch (uc->config.dir) {
2400         case DMA_MEM_TO_MEM:
2401                 /* Non synchronized - mem to mem type of transfer */
2402                 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__,
2403                         uc->id);
2404
2405                 ret = bcdma_alloc_bchan_resources(uc);
2406                 if (ret)
2407                         return ret;
2408
2409                 irq_ring_idx = uc->bchan->id + oes->bcdma_bchan_ring;
2410                 irq_udma_idx = uc->bchan->id + oes->bcdma_bchan_data;
2411
2412                 ret = bcdma_tisci_m2m_channel_config(uc);
2413                 break;
2414         case DMA_MEM_TO_DEV:
2415                 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
2416                 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2417                         uc->id);
2418
2419                 ret = udma_alloc_tx_resources(uc);
2420                 if (ret) {
2421                         uc->config.remote_thread_id = -1;
2422                         return ret;
2423                 }
2424
2425                 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2426                 uc->config.dst_thread = uc->config.remote_thread_id;
2427                 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2428
2429                 irq_ring_idx = uc->tchan->id + oes->bcdma_tchan_ring;
2430                 irq_udma_idx = uc->tchan->id + oes->bcdma_tchan_data;
2431
2432                 ret = bcdma_tisci_tx_channel_config(uc);
2433                 break;
2434         case DMA_DEV_TO_MEM:
2435                 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
2436                 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2437                         uc->id);
2438
2439                 ret = udma_alloc_rx_resources(uc);
2440                 if (ret) {
2441                         uc->config.remote_thread_id = -1;
2442                         return ret;
2443                 }
2444
2445                 uc->config.src_thread = uc->config.remote_thread_id;
2446                 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2447                                         K3_PSIL_DST_THREAD_ID_OFFSET;
2448
2449                 irq_ring_idx = uc->rchan->id + oes->bcdma_rchan_ring;
2450                 irq_udma_idx = uc->rchan->id + oes->bcdma_rchan_data;
2451
2452                 ret = bcdma_tisci_rx_channel_config(uc);
2453                 break;
2454         default:
2455                 /* Can not happen */
2456                 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2457                         __func__, uc->id, uc->config.dir);
2458                 return -EINVAL;
2459         }
2460
2461         /* check if the channel configuration was successful */
2462         if (ret)
2463                 goto err_res_free;
2464
2465         if (udma_is_chan_running(uc)) {
2466                 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2467                 udma_reset_chan(uc, false);
2468                 if (udma_is_chan_running(uc)) {
2469                         dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2470                         ret = -EBUSY;
2471                         goto err_res_free;
2472                 }
2473         }
2474
2475         uc->dma_dev = dmaengine_get_dma_device(chan);
2476         if (uc->config.dir == DMA_MEM_TO_MEM  && !uc->config.tr_trigger_type) {
2477                 uc->config.hdesc_size = cppi5_trdesc_calc_size(
2478                                         sizeof(struct cppi5_tr_type15_t), 2);
2479
2480                 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev,
2481                                                  uc->config.hdesc_size,
2482                                                  ud->desc_align,
2483                                                  0);
2484                 if (!uc->hdesc_pool) {
2485                         dev_err(ud->ddev.dev,
2486                                 "Descriptor pool allocation failed\n");
2487                         uc->use_dma_pool = false;
2488                         ret = -ENOMEM;
2489                         goto err_res_free;
2490                 }
2491
2492                 uc->use_dma_pool = true;
2493         } else if (uc->config.dir != DMA_MEM_TO_MEM) {
2494                 /* PSI-L pairing */
2495                 ret = navss_psil_pair(ud, uc->config.src_thread,
2496                                       uc->config.dst_thread);
2497                 if (ret) {
2498                         dev_err(ud->dev,
2499                                 "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2500                                 uc->config.src_thread, uc->config.dst_thread);
2501                         goto err_res_free;
2502                 }
2503
2504                 uc->psil_paired = true;
2505         }
2506
2507         uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx);
2508         if (uc->irq_num_ring <= 0) {
2509                 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
2510                         irq_ring_idx);
2511                 ret = -EINVAL;
2512                 goto err_psi_free;
2513         }
2514
2515         ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
2516                           IRQF_TRIGGER_HIGH, uc->name, uc);
2517         if (ret) {
2518                 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
2519                 goto err_irq_free;
2520         }
2521
2522         /* Event from BCDMA (TR events) only needed for slave channels */
2523         if (is_slave_direction(uc->config.dir)) {
2524                 uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx);
2525                 if (uc->irq_num_udma <= 0) {
2526                         dev_err(ud->dev, "Failed to get bcdma irq (index: %u)\n",
2527                                 irq_udma_idx);
2528                         free_irq(uc->irq_num_ring, uc);
2529                         ret = -EINVAL;
2530                         goto err_irq_free;
2531                 }
2532
2533                 ret = request_irq(uc->irq_num_udma, udma_udma_irq_handler, 0,
2534                                   uc->name, uc);
2535                 if (ret) {
2536                         dev_err(ud->dev, "chan%d: BCDMA irq request failed\n",
2537                                 uc->id);
2538                         free_irq(uc->irq_num_ring, uc);
2539                         goto err_irq_free;
2540                 }
2541         } else {
2542                 uc->irq_num_udma = 0;
2543         }
2544
2545         udma_reset_rings(uc);
2546
2547         INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work,
2548                                   udma_check_tx_completion);
2549         return 0;
2550
2551 err_irq_free:
2552         uc->irq_num_ring = 0;
2553         uc->irq_num_udma = 0;
2554 err_psi_free:
2555         if (uc->psil_paired)
2556                 navss_psil_unpair(ud, uc->config.src_thread,
2557                                   uc->config.dst_thread);
2558         uc->psil_paired = false;
2559 err_res_free:
2560         bcdma_free_bchan_resources(uc);
2561         udma_free_tx_resources(uc);
2562         udma_free_rx_resources(uc);
2563
2564         udma_reset_uchan(uc);
2565
2566         if (uc->use_dma_pool) {
2567                 dma_pool_destroy(uc->hdesc_pool);
2568                 uc->use_dma_pool = false;
2569         }
2570
2571         return ret;
2572 }
2573
2574 static int bcdma_router_config(struct dma_chan *chan)
2575 {
2576         struct k3_event_route_data *router_data = chan->route_data;
2577         struct udma_chan *uc = to_udma_chan(chan);
2578         u32 trigger_event;
2579
2580         if (!uc->bchan)
2581                 return -EINVAL;
2582
2583         if (uc->config.tr_trigger_type != 1 && uc->config.tr_trigger_type != 2)
2584                 return -EINVAL;
2585
2586         trigger_event = uc->ud->soc_data->bcdma_trigger_event_offset;
2587         trigger_event += (uc->bchan->id * 2) + uc->config.tr_trigger_type - 1;
2588
2589         return router_data->set_event(router_data->priv, trigger_event);
2590 }
2591
2592 static int pktdma_alloc_chan_resources(struct dma_chan *chan)
2593 {
2594         struct udma_chan *uc = to_udma_chan(chan);
2595         struct udma_dev *ud = to_udma_dev(chan->device);
2596         const struct udma_oes_offsets *oes = &ud->soc_data->oes;
2597         u32 irq_ring_idx;
2598         int ret;
2599
2600         /*
2601          * Make sure that the completion is in a known state:
2602          * No teardown, the channel is idle
2603          */
2604         reinit_completion(&uc->teardown_completed);
2605         complete_all(&uc->teardown_completed);
2606         uc->state = UDMA_CHAN_IS_IDLE;
2607
2608         switch (uc->config.dir) {
2609         case DMA_MEM_TO_DEV:
2610                 /* Slave transfer synchronized - mem to dev (TX) trasnfer */
2611                 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__,
2612                         uc->id);
2613
2614                 ret = udma_alloc_tx_resources(uc);
2615                 if (ret) {
2616                         uc->config.remote_thread_id = -1;
2617                         return ret;
2618                 }
2619
2620                 uc->config.src_thread = ud->psil_base + uc->tchan->id;
2621                 uc->config.dst_thread = uc->config.remote_thread_id;
2622                 uc->config.dst_thread |= K3_PSIL_DST_THREAD_ID_OFFSET;
2623
2624                 irq_ring_idx = uc->tchan->tflow_id + oes->pktdma_tchan_flow;
2625
2626                 ret = pktdma_tisci_tx_channel_config(uc);
2627                 break;
2628         case DMA_DEV_TO_MEM:
2629                 /* Slave transfer synchronized - dev to mem (RX) trasnfer */
2630                 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__,
2631                         uc->id);
2632
2633                 ret = udma_alloc_rx_resources(uc);
2634                 if (ret) {
2635                         uc->config.remote_thread_id = -1;
2636                         return ret;
2637                 }
2638
2639                 uc->config.src_thread = uc->config.remote_thread_id;
2640                 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) |
2641                                         K3_PSIL_DST_THREAD_ID_OFFSET;
2642
2643                 irq_ring_idx = uc->rflow->id + oes->pktdma_rchan_flow;
2644
2645                 ret = pktdma_tisci_rx_channel_config(uc);
2646                 break;
2647         default:
2648                 /* Can not happen */
2649                 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n",
2650                         __func__, uc->id, uc->config.dir);
2651                 return -EINVAL;
2652         }
2653
2654         /* check if the channel configuration was successful */
2655         if (ret)
2656                 goto err_res_free;
2657
2658         if (udma_is_chan_running(uc)) {
2659                 dev_warn(ud->dev, "chan%d: is running!\n", uc->id);
2660                 udma_reset_chan(uc, false);
2661                 if (udma_is_chan_running(uc)) {
2662                         dev_err(ud->dev, "chan%d: won't stop!\n", uc->id);
2663                         ret = -EBUSY;
2664                         goto err_res_free;
2665                 }
2666         }
2667
2668         uc->dma_dev = dmaengine_get_dma_device(chan);
2669         uc->hdesc_pool = dma_pool_create(uc->name, uc->dma_dev,
2670                                          uc->config.hdesc_size, ud->desc_align,
2671                                          0);
2672         if (!uc->hdesc_pool) {
2673                 dev_err(ud->ddev.dev,
2674                         "Descriptor pool allocation failed\n");
2675                 uc->use_dma_pool = false;
2676                 ret = -ENOMEM;
2677                 goto err_res_free;
2678         }
2679
2680         uc->use_dma_pool = true;
2681
2682         /* PSI-L pairing */
2683         ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread);
2684         if (ret) {
2685                 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n",
2686                         uc->config.src_thread, uc->config.dst_thread);
2687                 goto err_res_free;
2688         }
2689
2690         uc->psil_paired = true;
2691
2692         uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx);
2693         if (uc->irq_num_ring <= 0) {
2694                 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n",
2695                         irq_ring_idx);
2696                 ret = -EINVAL;
2697                 goto err_psi_free;
2698         }
2699
2700         ret = request_irq(uc->irq_num_ring, udma_ring_irq_handler,
2701                           IRQF_TRIGGER_HIGH, uc->name, uc);
2702         if (ret) {
2703                 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id);
2704                 goto err_irq_free;
2705         }
2706
2707         uc->irq_num_udma = 0;
2708
2709         udma_reset_rings(uc);
2710
2711         INIT_DELAYED_WORK_ONSTACK(&uc->tx_drain.work,
2712                                   udma_check_tx_completion);
2713
2714         if (uc->tchan)
2715                 dev_dbg(ud->dev,
2716                         "chan%d: tchan%d, tflow%d, Remote thread: 0x%04x\n",
2717                         uc->id, uc->tchan->id, uc->tchan->tflow_id,
2718                         uc->config.remote_thread_id);
2719         else if (uc->rchan)
2720                 dev_dbg(ud->dev,
2721                         "chan%d: rchan%d, rflow%d, Remote thread: 0x%04x\n",
2722                         uc->id, uc->rchan->id, uc->rflow->id,
2723                         uc->config.remote_thread_id);
2724         return 0;
2725
2726 err_irq_free:
2727         uc->irq_num_ring = 0;
2728 err_psi_free:
2729         navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread);
2730         uc->psil_paired = false;
2731 err_res_free:
2732         udma_free_tx_resources(uc);
2733         udma_free_rx_resources(uc);
2734
2735         udma_reset_uchan(uc);
2736
2737         dma_pool_destroy(uc->hdesc_pool);
2738         uc->use_dma_pool = false;
2739
2740         return ret;
2741 }
2742
2743 static int udma_slave_config(struct dma_chan *chan,
2744                              struct dma_slave_config *cfg)
2745 {
2746         struct udma_chan *uc = to_udma_chan(chan);
2747
2748         memcpy(&uc->cfg, cfg, sizeof(uc->cfg));
2749
2750         return 0;
2751 }
2752
2753 static struct udma_desc *udma_alloc_tr_desc(struct udma_chan *uc,
2754                                             size_t tr_size, int tr_count,
2755                                             enum dma_transfer_direction dir)
2756 {
2757         struct udma_hwdesc *hwdesc;
2758         struct cppi5_desc_hdr_t *tr_desc;
2759         struct udma_desc *d;
2760         u32 reload_count = 0;
2761         u32 ring_id;
2762
2763         switch (tr_size) {
2764         case 16:
2765         case 32:
2766         case 64:
2767         case 128:
2768                 break;
2769         default:
2770                 dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size);
2771                 return NULL;
2772         }
2773
2774         /* We have only one descriptor containing multiple TRs */
2775         d = kzalloc(sizeof(*d) + sizeof(d->hwdesc[0]), GFP_NOWAIT);
2776         if (!d)
2777                 return NULL;
2778
2779         d->sglen = tr_count;
2780
2781         d->hwdesc_count = 1;
2782         hwdesc = &d->hwdesc[0];
2783
2784         /* Allocate memory for DMA ring descriptor */
2785         if (uc->use_dma_pool) {
2786                 hwdesc->cppi5_desc_size = uc->config.hdesc_size;
2787                 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
2788                                                 GFP_NOWAIT,
2789                                                 &hwdesc->cppi5_desc_paddr);
2790         } else {
2791                 hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size,
2792                                                                  tr_count);
2793                 hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
2794                                                 uc->ud->desc_align);
2795                 hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev,
2796                                                 hwdesc->cppi5_desc_size,
2797                                                 &hwdesc->cppi5_desc_paddr,
2798                                                 GFP_NOWAIT);
2799         }
2800
2801         if (!hwdesc->cppi5_desc_vaddr) {
2802                 kfree(d);
2803                 return NULL;
2804         }
2805
2806         /* Start of the TR req records */
2807         hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
2808         /* Start address of the TR response array */
2809         hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size * tr_count;
2810
2811         tr_desc = hwdesc->cppi5_desc_vaddr;
2812
2813         if (uc->cyclic)
2814                 reload_count = CPPI5_INFO0_TRDESC_RLDCNT_INFINITE;
2815
2816         if (dir == DMA_DEV_TO_MEM)
2817                 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
2818         else
2819                 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
2820
2821         cppi5_trdesc_init(tr_desc, tr_count, tr_size, 0, reload_count);
2822         cppi5_desc_set_pktids(tr_desc, uc->id,
2823                               CPPI5_INFO1_DESC_FLOWID_DEFAULT);
2824         cppi5_desc_set_retpolicy(tr_desc, 0, ring_id);
2825
2826         return d;
2827 }
2828
2829 /**
2830  * udma_get_tr_counters - calculate TR counters for a given length
2831  * @len: Length of the trasnfer
2832  * @align_to: Preferred alignment
2833  * @tr0_cnt0: First TR icnt0
2834  * @tr0_cnt1: First TR icnt1
2835  * @tr1_cnt0: Second (if used) TR icnt0
2836  *
2837  * For len < SZ_64K only one TR is enough, tr1_cnt0 is not updated
2838  * For len >= SZ_64K two TRs are used in a simple way:
2839  * First TR: SZ_64K-alignment blocks (tr0_cnt0, tr0_cnt1)
2840  * Second TR: the remaining length (tr1_cnt0)
2841  *
2842  * Returns the number of TRs the length needs (1 or 2)
2843  * -EINVAL if the length can not be supported
2844  */
2845 static int udma_get_tr_counters(size_t len, unsigned long align_to,
2846                                 u16 *tr0_cnt0, u16 *tr0_cnt1, u16 *tr1_cnt0)
2847 {
2848         if (len < SZ_64K) {
2849                 *tr0_cnt0 = len;
2850                 *tr0_cnt1 = 1;
2851
2852                 return 1;
2853         }
2854
2855         if (align_to > 3)
2856                 align_to = 3;
2857
2858 realign:
2859         *tr0_cnt0 = SZ_64K - BIT(align_to);
2860         if (len / *tr0_cnt0 >= SZ_64K) {
2861                 if (align_to) {
2862                         align_to--;
2863                         goto realign;
2864                 }
2865                 return -EINVAL;
2866         }
2867
2868         *tr0_cnt1 = len / *tr0_cnt0;
2869         *tr1_cnt0 = len % *tr0_cnt0;
2870
2871         return 2;
2872 }
2873
2874 static struct udma_desc *
2875 udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl,
2876                       unsigned int sglen, enum dma_transfer_direction dir,
2877                       unsigned long tx_flags, void *context)
2878 {
2879         struct scatterlist *sgent;
2880         struct udma_desc *d;
2881         struct cppi5_tr_type1_t *tr_req = NULL;
2882         u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
2883         unsigned int i;
2884         size_t tr_size;
2885         int num_tr = 0;
2886         int tr_idx = 0;
2887         u64 asel;
2888
2889         /* estimate the number of TRs we will need */
2890         for_each_sg(sgl, sgent, sglen, i) {
2891                 if (sg_dma_len(sgent) < SZ_64K)
2892                         num_tr++;
2893                 else
2894                         num_tr += 2;
2895         }
2896
2897         /* Now allocate and setup the descriptor. */
2898         tr_size = sizeof(struct cppi5_tr_type1_t);
2899         d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);
2900         if (!d)
2901                 return NULL;
2902
2903         d->sglen = sglen;
2904
2905         if (uc->ud->match_data->type == DMA_TYPE_UDMA)
2906                 asel = 0;
2907         else
2908                 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
2909
2910         tr_req = d->hwdesc[0].tr_req_base;
2911         for_each_sg(sgl, sgent, sglen, i) {
2912                 dma_addr_t sg_addr = sg_dma_address(sgent);
2913
2914                 num_tr = udma_get_tr_counters(sg_dma_len(sgent), __ffs(sg_addr),
2915                                               &tr0_cnt0, &tr0_cnt1, &tr1_cnt0);
2916                 if (num_tr < 0) {
2917                         dev_err(uc->ud->dev, "size %u is not supported\n",
2918                                 sg_dma_len(sgent));
2919                         udma_free_hwdesc(uc, d);
2920                         kfree(d);
2921                         return NULL;
2922                 }
2923
2924                 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
2925                               false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
2926                 cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT);
2927
2928                 sg_addr |= asel;
2929                 tr_req[tr_idx].addr = sg_addr;
2930                 tr_req[tr_idx].icnt0 = tr0_cnt0;
2931                 tr_req[tr_idx].icnt1 = tr0_cnt1;
2932                 tr_req[tr_idx].dim1 = tr0_cnt0;
2933                 tr_idx++;
2934
2935                 if (num_tr == 2) {
2936                         cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
2937                                       false, false,
2938                                       CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
2939                         cppi5_tr_csf_set(&tr_req[tr_idx].flags,
2940                                          CPPI5_TR_CSF_SUPR_EVT);
2941
2942                         tr_req[tr_idx].addr = sg_addr + tr0_cnt1 * tr0_cnt0;
2943                         tr_req[tr_idx].icnt0 = tr1_cnt0;
2944                         tr_req[tr_idx].icnt1 = 1;
2945                         tr_req[tr_idx].dim1 = tr1_cnt0;
2946                         tr_idx++;
2947                 }
2948
2949                 d->residue += sg_dma_len(sgent);
2950         }
2951
2952         cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags,
2953                          CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
2954
2955         return d;
2956 }
2957
2958 static struct udma_desc *
2959 udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl,
2960                                 unsigned int sglen,
2961                                 enum dma_transfer_direction dir,
2962                                 unsigned long tx_flags, void *context)
2963 {
2964         struct scatterlist *sgent;
2965         struct cppi5_tr_type15_t *tr_req = NULL;
2966         enum dma_slave_buswidth dev_width;
2967         u16 tr_cnt0, tr_cnt1;
2968         dma_addr_t dev_addr;
2969         struct udma_desc *d;
2970         unsigned int i;
2971         size_t tr_size, sg_len;
2972         int num_tr = 0;
2973         int tr_idx = 0;
2974         u32 burst, trigger_size, port_window;
2975         u64 asel;
2976
2977         if (dir == DMA_DEV_TO_MEM) {
2978                 dev_addr = uc->cfg.src_addr;
2979                 dev_width = uc->cfg.src_addr_width;
2980                 burst = uc->cfg.src_maxburst;
2981                 port_window = uc->cfg.src_port_window_size;
2982         } else if (dir == DMA_MEM_TO_DEV) {
2983                 dev_addr = uc->cfg.dst_addr;
2984                 dev_width = uc->cfg.dst_addr_width;
2985                 burst = uc->cfg.dst_maxburst;
2986                 port_window = uc->cfg.dst_port_window_size;
2987         } else {
2988                 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
2989                 return NULL;
2990         }
2991
2992         if (!burst)
2993                 burst = 1;
2994
2995         if (port_window) {
2996                 if (port_window != burst) {
2997                         dev_err(uc->ud->dev,
2998                                 "The burst must be equal to port_window\n");
2999                         return NULL;
3000                 }
3001
3002                 tr_cnt0 = dev_width * port_window;
3003                 tr_cnt1 = 1;
3004         } else {
3005                 tr_cnt0 = dev_width;
3006                 tr_cnt1 = burst;
3007         }
3008         trigger_size = tr_cnt0 * tr_cnt1;
3009
3010         /* estimate the number of TRs we will need */
3011         for_each_sg(sgl, sgent, sglen, i) {
3012                 sg_len = sg_dma_len(sgent);
3013
3014                 if (sg_len % trigger_size) {
3015                         dev_err(uc->ud->dev,
3016                                 "Not aligned SG entry (%zu for %u)\n", sg_len,
3017                                 trigger_size);
3018                         return NULL;
3019                 }
3020
3021                 if (sg_len / trigger_size < SZ_64K)
3022                         num_tr++;
3023                 else
3024                         num_tr += 2;
3025         }
3026
3027         /* Now allocate and setup the descriptor. */
3028         tr_size = sizeof(struct cppi5_tr_type15_t);
3029         d = udma_alloc_tr_desc(uc, tr_size, num_tr, dir);
3030         if (!d)
3031                 return NULL;
3032
3033         d->sglen = sglen;
3034
3035         if (uc->ud->match_data->type == DMA_TYPE_UDMA) {
3036                 asel = 0;
3037         } else {
3038                 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
3039                 dev_addr |= asel;
3040         }
3041
3042         tr_req = d->hwdesc[0].tr_req_base;
3043         for_each_sg(sgl, sgent, sglen, i) {
3044                 u16 tr0_cnt2, tr0_cnt3, tr1_cnt2;
3045                 dma_addr_t sg_addr = sg_dma_address(sgent);
3046
3047                 sg_len = sg_dma_len(sgent);
3048                 num_tr = udma_get_tr_counters(sg_len / trigger_size, 0,
3049                                               &tr0_cnt2, &tr0_cnt3, &tr1_cnt2);
3050                 if (num_tr < 0) {
3051                         dev_err(uc->ud->dev, "size %zu is not supported\n",
3052                                 sg_len);
3053                         udma_free_hwdesc(uc, d);
3054                         kfree(d);
3055                         return NULL;
3056                 }
3057
3058                 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false,
3059                               true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3060                 cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT);
3061                 cppi5_tr_set_trigger(&tr_req[tr_idx].flags,
3062                                      uc->config.tr_trigger_type,
3063                                      CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 0, 0);
3064
3065                 sg_addr |= asel;
3066                 if (dir == DMA_DEV_TO_MEM) {
3067                         tr_req[tr_idx].addr = dev_addr;
3068                         tr_req[tr_idx].icnt0 = tr_cnt0;
3069                         tr_req[tr_idx].icnt1 = tr_cnt1;
3070                         tr_req[tr_idx].icnt2 = tr0_cnt2;
3071                         tr_req[tr_idx].icnt3 = tr0_cnt3;
3072                         tr_req[tr_idx].dim1 = (-1) * tr_cnt0;
3073
3074                         tr_req[tr_idx].daddr = sg_addr;
3075                         tr_req[tr_idx].dicnt0 = tr_cnt0;
3076                         tr_req[tr_idx].dicnt1 = tr_cnt1;
3077                         tr_req[tr_idx].dicnt2 = tr0_cnt2;
3078                         tr_req[tr_idx].dicnt3 = tr0_cnt3;
3079                         tr_req[tr_idx].ddim1 = tr_cnt0;
3080                         tr_req[tr_idx].ddim2 = trigger_size;
3081                         tr_req[tr_idx].ddim3 = trigger_size * tr0_cnt2;
3082                 } else {
3083                         tr_req[tr_idx].addr = sg_addr;
3084                         tr_req[tr_idx].icnt0 = tr_cnt0;
3085                         tr_req[tr_idx].icnt1 = tr_cnt1;
3086                         tr_req[tr_idx].icnt2 = tr0_cnt2;
3087                         tr_req[tr_idx].icnt3 = tr0_cnt3;
3088                         tr_req[tr_idx].dim1 = tr_cnt0;
3089                         tr_req[tr_idx].dim2 = trigger_size;
3090                         tr_req[tr_idx].dim3 = trigger_size * tr0_cnt2;
3091
3092                         tr_req[tr_idx].daddr = dev_addr;
3093                         tr_req[tr_idx].dicnt0 = tr_cnt0;
3094                         tr_req[tr_idx].dicnt1 = tr_cnt1;
3095                         tr_req[tr_idx].dicnt2 = tr0_cnt2;
3096                         tr_req[tr_idx].dicnt3 = tr0_cnt3;
3097                         tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;
3098                 }
3099
3100                 tr_idx++;
3101
3102                 if (num_tr == 2) {
3103                         cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15,
3104                                       false, true,
3105                                       CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3106                         cppi5_tr_csf_set(&tr_req[tr_idx].flags,
3107                                          CPPI5_TR_CSF_SUPR_EVT);
3108                         cppi5_tr_set_trigger(&tr_req[tr_idx].flags,
3109                                              uc->config.tr_trigger_type,
3110                                              CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC,
3111                                              0, 0);
3112
3113                         sg_addr += trigger_size * tr0_cnt2 * tr0_cnt3;
3114                         if (dir == DMA_DEV_TO_MEM) {
3115                                 tr_req[tr_idx].addr = dev_addr;
3116                                 tr_req[tr_idx].icnt0 = tr_cnt0;
3117                                 tr_req[tr_idx].icnt1 = tr_cnt1;
3118                                 tr_req[tr_idx].icnt2 = tr1_cnt2;
3119                                 tr_req[tr_idx].icnt3 = 1;
3120                                 tr_req[tr_idx].dim1 = (-1) * tr_cnt0;
3121
3122                                 tr_req[tr_idx].daddr = sg_addr;
3123                                 tr_req[tr_idx].dicnt0 = tr_cnt0;
3124                                 tr_req[tr_idx].dicnt1 = tr_cnt1;
3125                                 tr_req[tr_idx].dicnt2 = tr1_cnt2;
3126                                 tr_req[tr_idx].dicnt3 = 1;
3127                                 tr_req[tr_idx].ddim1 = tr_cnt0;
3128                                 tr_req[tr_idx].ddim2 = trigger_size;
3129                         } else {
3130                                 tr_req[tr_idx].addr = sg_addr;
3131                                 tr_req[tr_idx].icnt0 = tr_cnt0;
3132                                 tr_req[tr_idx].icnt1 = tr_cnt1;
3133                                 tr_req[tr_idx].icnt2 = tr1_cnt2;
3134                                 tr_req[tr_idx].icnt3 = 1;
3135                                 tr_req[tr_idx].dim1 = tr_cnt0;
3136                                 tr_req[tr_idx].dim2 = trigger_size;
3137
3138                                 tr_req[tr_idx].daddr = dev_addr;
3139                                 tr_req[tr_idx].dicnt0 = tr_cnt0;
3140                                 tr_req[tr_idx].dicnt1 = tr_cnt1;
3141                                 tr_req[tr_idx].dicnt2 = tr1_cnt2;
3142                                 tr_req[tr_idx].dicnt3 = 1;
3143                                 tr_req[tr_idx].ddim1 = (-1) * tr_cnt0;
3144                         }
3145                         tr_idx++;
3146                 }
3147
3148                 d->residue += sg_len;
3149         }
3150
3151         cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags,
3152                          CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
3153
3154         return d;
3155 }
3156
3157 static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d,
3158                                    enum dma_slave_buswidth dev_width,
3159                                    u16 elcnt)
3160 {
3161         if (uc->config.ep_type != PSIL_EP_PDMA_XY)
3162                 return 0;
3163
3164         /* Bus width translates to the element size (ES) */
3165         switch (dev_width) {
3166         case DMA_SLAVE_BUSWIDTH_1_BYTE:
3167                 d->static_tr.elsize = 0;
3168                 break;
3169         case DMA_SLAVE_BUSWIDTH_2_BYTES:
3170                 d->static_tr.elsize = 1;
3171                 break;
3172         case DMA_SLAVE_BUSWIDTH_3_BYTES:
3173                 d->static_tr.elsize = 2;
3174                 break;
3175         case DMA_SLAVE_BUSWIDTH_4_BYTES:
3176                 d->static_tr.elsize = 3;
3177                 break;
3178         case DMA_SLAVE_BUSWIDTH_8_BYTES:
3179                 d->static_tr.elsize = 4;
3180                 break;
3181         default: /* not reached */
3182                 return -EINVAL;
3183         }
3184
3185         d->static_tr.elcnt = elcnt;
3186
3187         /*
3188          * PDMA must to close the packet when the channel is in packet mode.
3189          * For TR mode when the channel is not cyclic we also need PDMA to close
3190          * the packet otherwise the transfer will stall because PDMA holds on
3191          * the data it has received from the peripheral.
3192          */
3193         if (uc->config.pkt_mode || !uc->cyclic) {
3194                 unsigned int div = dev_width * elcnt;
3195
3196                 if (uc->cyclic)
3197                         d->static_tr.bstcnt = d->residue / d->sglen / div;
3198                 else
3199                         d->static_tr.bstcnt = d->residue / div;
3200
3201                 if (uc->config.dir == DMA_DEV_TO_MEM &&
3202                     d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask)
3203                         return -EINVAL;
3204         } else {
3205                 d->static_tr.bstcnt = 0;
3206         }
3207
3208         return 0;
3209 }
3210
3211 static struct udma_desc *
3212 udma_prep_slave_sg_pkt(struct udma_chan *uc, struct scatterlist *sgl,
3213                        unsigned int sglen, enum dma_transfer_direction dir,
3214                        unsigned long tx_flags, void *context)
3215 {
3216         struct scatterlist *sgent;
3217         struct cppi5_host_desc_t *h_desc = NULL;
3218         struct udma_desc *d;
3219         u32 ring_id;
3220         unsigned int i;
3221         u64 asel;
3222
3223         d = kzalloc(struct_size(d, hwdesc, sglen), GFP_NOWAIT);
3224         if (!d)
3225                 return NULL;
3226
3227         d->sglen = sglen;
3228         d->hwdesc_count = sglen;
3229
3230         if (dir == DMA_DEV_TO_MEM)
3231                 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
3232         else
3233                 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
3234
3235         if (uc->ud->match_data->type == DMA_TYPE_UDMA)
3236                 asel = 0;
3237         else
3238                 asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
3239
3240         for_each_sg(sgl, sgent, sglen, i) {
3241                 struct udma_hwdesc *hwdesc = &d->hwdesc[i];
3242                 dma_addr_t sg_addr = sg_dma_address(sgent);
3243                 struct cppi5_host_desc_t *desc;
3244                 size_t sg_len = sg_dma_len(sgent);
3245
3246                 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
3247                                                 GFP_NOWAIT,
3248                                                 &hwdesc->cppi5_desc_paddr);
3249                 if (!hwdesc->cppi5_desc_vaddr) {
3250                         dev_err(uc->ud->dev,
3251                                 "descriptor%d allocation failed\n", i);
3252
3253                         udma_free_hwdesc(uc, d);
3254                         kfree(d);
3255                         return NULL;
3256                 }
3257
3258                 d->residue += sg_len;
3259                 hwdesc->cppi5_desc_size = uc->config.hdesc_size;
3260                 desc = hwdesc->cppi5_desc_vaddr;
3261
3262                 if (i == 0) {
3263                         cppi5_hdesc_init(desc, 0, 0);
3264                         /* Flow and Packed ID */
3265                         cppi5_desc_set_pktids(&desc->hdr, uc->id,
3266                                               CPPI5_INFO1_DESC_FLOWID_DEFAULT);
3267                         cppi5_desc_set_retpolicy(&desc->hdr, 0, ring_id);
3268                 } else {
3269                         cppi5_hdesc_reset_hbdesc(desc);
3270                         cppi5_desc_set_retpolicy(&desc->hdr, 0, 0xffff);
3271                 }
3272
3273                 /* attach the sg buffer to the descriptor */
3274                 sg_addr |= asel;
3275                 cppi5_hdesc_attach_buf(desc, sg_addr, sg_len, sg_addr, sg_len);
3276
3277                 /* Attach link as host buffer descriptor */
3278                 if (h_desc)
3279                         cppi5_hdesc_link_hbdesc(h_desc,
3280                                                 hwdesc->cppi5_desc_paddr | asel);
3281
3282                 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA ||
3283                     dir == DMA_MEM_TO_DEV)
3284                         h_desc = desc;
3285         }
3286
3287         if (d->residue >= SZ_4M) {
3288                 dev_err(uc->ud->dev,
3289                         "%s: Transfer size %u is over the supported 4M range\n",
3290                         __func__, d->residue);
3291                 udma_free_hwdesc(uc, d);
3292                 kfree(d);
3293                 return NULL;
3294         }
3295
3296         h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3297         cppi5_hdesc_set_pktlen(h_desc, d->residue);
3298
3299         return d;
3300 }
3301
3302 static int udma_attach_metadata(struct dma_async_tx_descriptor *desc,
3303                                 void *data, size_t len)
3304 {
3305         struct udma_desc *d = to_udma_desc(desc);
3306         struct udma_chan *uc = to_udma_chan(desc->chan);
3307         struct cppi5_host_desc_t *h_desc;
3308         u32 psd_size = len;
3309         u32 flags = 0;
3310
3311         if (!uc->config.pkt_mode || !uc->config.metadata_size)
3312                 return -ENOTSUPP;
3313
3314         if (!data || len > uc->config.metadata_size)
3315                 return -EINVAL;
3316
3317         if (uc->config.needs_epib && len < CPPI5_INFO0_HDESC_EPIB_SIZE)
3318                 return -EINVAL;
3319
3320         h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3321         if (d->dir == DMA_MEM_TO_DEV)
3322                 memcpy(h_desc->epib, data, len);
3323
3324         if (uc->config.needs_epib)
3325                 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
3326
3327         d->metadata = data;
3328         d->metadata_size = len;
3329         if (uc->config.needs_epib)
3330                 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
3331
3332         cppi5_hdesc_update_flags(h_desc, flags);
3333         cppi5_hdesc_update_psdata_size(h_desc, psd_size);
3334
3335         return 0;
3336 }
3337
3338 static void *udma_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
3339                                    size_t *payload_len, size_t *max_len)
3340 {
3341         struct udma_desc *d = to_udma_desc(desc);
3342         struct udma_chan *uc = to_udma_chan(desc->chan);
3343         struct cppi5_host_desc_t *h_desc;
3344
3345         if (!uc->config.pkt_mode || !uc->config.metadata_size)
3346                 return ERR_PTR(-ENOTSUPP);
3347
3348         h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3349
3350         *max_len = uc->config.metadata_size;
3351
3352         *payload_len = cppi5_hdesc_epib_present(&h_desc->hdr) ?
3353                        CPPI5_INFO0_HDESC_EPIB_SIZE : 0;
3354         *payload_len += cppi5_hdesc_get_psdata_size(h_desc);
3355
3356         return h_desc->epib;
3357 }
3358
3359 static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc,
3360                                  size_t payload_len)
3361 {
3362         struct udma_desc *d = to_udma_desc(desc);
3363         struct udma_chan *uc = to_udma_chan(desc->chan);
3364         struct cppi5_host_desc_t *h_desc;
3365         u32 psd_size = payload_len;
3366         u32 flags = 0;
3367
3368         if (!uc->config.pkt_mode || !uc->config.metadata_size)
3369                 return -ENOTSUPP;
3370
3371         if (payload_len > uc->config.metadata_size)
3372                 return -EINVAL;
3373
3374         if (uc->config.needs_epib && payload_len < CPPI5_INFO0_HDESC_EPIB_SIZE)
3375                 return -EINVAL;
3376
3377         h_desc = d->hwdesc[0].cppi5_desc_vaddr;
3378
3379         if (uc->config.needs_epib) {
3380                 psd_size -= CPPI5_INFO0_HDESC_EPIB_SIZE;
3381                 flags |= CPPI5_INFO0_HDESC_EPIB_PRESENT;
3382         }
3383
3384         cppi5_hdesc_update_flags(h_desc, flags);
3385         cppi5_hdesc_update_psdata_size(h_desc, psd_size);
3386
3387         return 0;
3388 }
3389
3390 static struct dma_descriptor_metadata_ops metadata_ops = {
3391         .attach = udma_attach_metadata,
3392         .get_ptr = udma_get_metadata_ptr,
3393         .set_len = udma_set_metadata_len,
3394 };
3395
3396 static struct dma_async_tx_descriptor *
3397 udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
3398                    unsigned int sglen, enum dma_transfer_direction dir,
3399                    unsigned long tx_flags, void *context)
3400 {
3401         struct udma_chan *uc = to_udma_chan(chan);
3402         enum dma_slave_buswidth dev_width;
3403         struct udma_desc *d;
3404         u32 burst;
3405
3406         if (dir != uc->config.dir &&
3407             (uc->config.dir == DMA_MEM_TO_MEM && !uc->config.tr_trigger_type)) {
3408                 dev_err(chan->device->dev,
3409                         "%s: chan%d is for %s, not supporting %s\n",
3410                         __func__, uc->id,
3411                         dmaengine_get_direction_text(uc->config.dir),
3412                         dmaengine_get_direction_text(dir));
3413                 return NULL;
3414         }
3415
3416         if (dir == DMA_DEV_TO_MEM) {
3417                 dev_width = uc->cfg.src_addr_width;
3418                 burst = uc->cfg.src_maxburst;
3419         } else if (dir == DMA_MEM_TO_DEV) {
3420                 dev_width = uc->cfg.dst_addr_width;
3421                 burst = uc->cfg.dst_maxburst;
3422         } else {
3423                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
3424                 return NULL;
3425         }
3426
3427         if (!burst)
3428                 burst = 1;
3429
3430         uc->config.tx_flags = tx_flags;
3431
3432         if (uc->config.pkt_mode)
3433                 d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags,
3434                                            context);
3435         else if (is_slave_direction(uc->config.dir))
3436                 d = udma_prep_slave_sg_tr(uc, sgl, sglen, dir, tx_flags,
3437                                           context);
3438         else
3439                 d = udma_prep_slave_sg_triggered_tr(uc, sgl, sglen, dir,
3440                                                     tx_flags, context);
3441
3442         if (!d)
3443                 return NULL;
3444
3445         d->dir = dir;
3446         d->desc_idx = 0;
3447         d->tr_idx = 0;
3448
3449         /* static TR for remote PDMA */
3450         if (udma_configure_statictr(uc, d, dev_width, burst)) {
3451                 dev_err(uc->ud->dev,
3452                         "%s: StaticTR Z is limited to maximum 4095 (%u)\n",
3453                         __func__, d->static_tr.bstcnt);
3454
3455                 udma_free_hwdesc(uc, d);
3456                 kfree(d);
3457                 return NULL;
3458         }
3459
3460         if (uc->config.metadata_size)
3461                 d->vd.tx.metadata_ops = &metadata_ops;
3462
3463         return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
3464 }
3465
3466 static struct udma_desc *
3467 udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr,
3468                         size_t buf_len, size_t period_len,
3469                         enum dma_transfer_direction dir, unsigned long flags)
3470 {
3471         struct udma_desc *d;
3472         size_t tr_size, period_addr;
3473         struct cppi5_tr_type1_t *tr_req;
3474         unsigned int periods = buf_len / period_len;
3475         u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
3476         unsigned int i;
3477         int num_tr;
3478
3479         num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0,
3480                                       &tr0_cnt1, &tr1_cnt0);
3481         if (num_tr < 0) {
3482                 dev_err(uc->ud->dev, "size %zu is not supported\n",
3483                         period_len);
3484                 return NULL;
3485         }
3486
3487         /* Now allocate and setup the descriptor. */
3488         tr_size = sizeof(struct cppi5_tr_type1_t);
3489         d = udma_alloc_tr_desc(uc, tr_size, periods * num_tr, dir);
3490         if (!d)
3491                 return NULL;
3492
3493         tr_req = d->hwdesc[0].tr_req_base;
3494         if (uc->ud->match_data->type == DMA_TYPE_UDMA)
3495                 period_addr = buf_addr;
3496         else
3497                 period_addr = buf_addr |
3498                         ((u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT);
3499
3500         for (i = 0; i < periods; i++) {
3501                 int tr_idx = i * num_tr;
3502
3503                 cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1, false,
3504                               false, CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3505
3506                 tr_req[tr_idx].addr = period_addr;
3507                 tr_req[tr_idx].icnt0 = tr0_cnt0;
3508                 tr_req[tr_idx].icnt1 = tr0_cnt1;
3509                 tr_req[tr_idx].dim1 = tr0_cnt0;
3510
3511                 if (num_tr == 2) {
3512                         cppi5_tr_csf_set(&tr_req[tr_idx].flags,
3513                                          CPPI5_TR_CSF_SUPR_EVT);
3514                         tr_idx++;
3515
3516                         cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE1,
3517                                       false, false,
3518                                       CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3519
3520                         tr_req[tr_idx].addr = period_addr + tr0_cnt1 * tr0_cnt0;
3521                         tr_req[tr_idx].icnt0 = tr1_cnt0;
3522                         tr_req[tr_idx].icnt1 = 1;
3523                         tr_req[tr_idx].dim1 = tr1_cnt0;
3524                 }
3525
3526                 if (!(flags & DMA_PREP_INTERRUPT))
3527                         cppi5_tr_csf_set(&tr_req[tr_idx].flags,
3528                                          CPPI5_TR_CSF_SUPR_EVT);
3529
3530                 period_addr += period_len;
3531         }
3532
3533         return d;
3534 }
3535
3536 static struct udma_desc *
3537 udma_prep_dma_cyclic_pkt(struct udma_chan *uc, dma_addr_t buf_addr,
3538                          size_t buf_len, size_t period_len,
3539                          enum dma_transfer_direction dir, unsigned long flags)
3540 {
3541         struct udma_desc *d;
3542         u32 ring_id;
3543         int i;
3544         int periods = buf_len / period_len;
3545
3546         if (periods > (K3_UDMA_DEFAULT_RING_SIZE - 1))
3547                 return NULL;
3548
3549         if (period_len >= SZ_4M)
3550                 return NULL;
3551
3552         d = kzalloc(struct_size(d, hwdesc, periods), GFP_NOWAIT);
3553         if (!d)
3554                 return NULL;
3555
3556         d->hwdesc_count = periods;
3557
3558         /* TODO: re-check this... */
3559         if (dir == DMA_DEV_TO_MEM)
3560                 ring_id = k3_ringacc_get_ring_id(uc->rflow->r_ring);
3561         else
3562                 ring_id = k3_ringacc_get_ring_id(uc->tchan->tc_ring);
3563
3564         if (uc->ud->match_data->type != DMA_TYPE_UDMA)
3565                 buf_addr |= (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT;
3566
3567         for (i = 0; i < periods; i++) {
3568                 struct udma_hwdesc *hwdesc = &d->hwdesc[i];
3569                 dma_addr_t period_addr = buf_addr + (period_len * i);
3570                 struct cppi5_host_desc_t *h_desc;
3571
3572                 hwdesc->cppi5_desc_vaddr = dma_pool_zalloc(uc->hdesc_pool,
3573                                                 GFP_NOWAIT,
3574                                                 &hwdesc->cppi5_desc_paddr);
3575                 if (!hwdesc->cppi5_desc_vaddr) {
3576                         dev_err(uc->ud->dev,
3577                                 "descriptor%d allocation failed\n", i);
3578
3579                         udma_free_hwdesc(uc, d);
3580                         kfree(d);
3581                         return NULL;
3582                 }
3583
3584                 hwdesc->cppi5_desc_size = uc->config.hdesc_size;
3585                 h_desc = hwdesc->cppi5_desc_vaddr;
3586
3587                 cppi5_hdesc_init(h_desc, 0, 0);
3588                 cppi5_hdesc_set_pktlen(h_desc, period_len);
3589
3590                 /* Flow and Packed ID */
3591                 cppi5_desc_set_pktids(&h_desc->hdr, uc->id,
3592                                       CPPI5_INFO1_DESC_FLOWID_DEFAULT);
3593                 cppi5_desc_set_retpolicy(&h_desc->hdr, 0, ring_id);
3594
3595                 /* attach each period to a new descriptor */
3596                 cppi5_hdesc_attach_buf(h_desc,
3597                                        period_addr, period_len,
3598                                        period_addr, period_len);
3599         }
3600
3601         return d;
3602 }
3603
3604 static struct dma_async_tx_descriptor *
3605 udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
3606                      size_t period_len, enum dma_transfer_direction dir,
3607                      unsigned long flags)
3608 {
3609         struct udma_chan *uc = to_udma_chan(chan);
3610         enum dma_slave_buswidth dev_width;
3611         struct udma_desc *d;
3612         u32 burst;
3613
3614         if (dir != uc->config.dir) {
3615                 dev_err(chan->device->dev,
3616                         "%s: chan%d is for %s, not supporting %s\n",
3617                         __func__, uc->id,
3618                         dmaengine_get_direction_text(uc->config.dir),
3619                         dmaengine_get_direction_text(dir));
3620                 return NULL;
3621         }
3622
3623         uc->cyclic = true;
3624
3625         if (dir == DMA_DEV_TO_MEM) {
3626                 dev_width = uc->cfg.src_addr_width;
3627                 burst = uc->cfg.src_maxburst;
3628         } else if (dir == DMA_MEM_TO_DEV) {
3629                 dev_width = uc->cfg.dst_addr_width;
3630                 burst = uc->cfg.dst_maxburst;
3631         } else {
3632                 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__);
3633                 return NULL;
3634         }
3635
3636         if (!burst)
3637                 burst = 1;
3638
3639         if (uc->config.pkt_mode)
3640                 d = udma_prep_dma_cyclic_pkt(uc, buf_addr, buf_len, period_len,
3641                                              dir, flags);
3642         else
3643                 d = udma_prep_dma_cyclic_tr(uc, buf_addr, buf_len, period_len,
3644                                             dir, flags);
3645
3646         if (!d)
3647                 return NULL;
3648
3649         d->sglen = buf_len / period_len;
3650
3651         d->dir = dir;
3652         d->residue = buf_len;
3653
3654         /* static TR for remote PDMA */
3655         if (udma_configure_statictr(uc, d, dev_width, burst)) {
3656                 dev_err(uc->ud->dev,
3657                         "%s: StaticTR Z is limited to maximum 4095 (%u)\n",
3658                         __func__, d->static_tr.bstcnt);
3659
3660                 udma_free_hwdesc(uc, d);
3661                 kfree(d);
3662                 return NULL;
3663         }
3664
3665         if (uc->config.metadata_size)
3666                 d->vd.tx.metadata_ops = &metadata_ops;
3667
3668         return vchan_tx_prep(&uc->vc, &d->vd, flags);
3669 }
3670
3671 static struct dma_async_tx_descriptor *
3672 udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
3673                      size_t len, unsigned long tx_flags)
3674 {
3675         struct udma_chan *uc = to_udma_chan(chan);
3676         struct udma_desc *d;
3677         struct cppi5_tr_type15_t *tr_req;
3678         int num_tr;
3679         size_t tr_size = sizeof(struct cppi5_tr_type15_t);
3680         u16 tr0_cnt0, tr0_cnt1, tr1_cnt0;
3681
3682         if (uc->config.dir != DMA_MEM_TO_MEM) {
3683                 dev_err(chan->device->dev,
3684                         "%s: chan%d is for %s, not supporting %s\n",
3685                         __func__, uc->id,
3686                         dmaengine_get_direction_text(uc->config.dir),
3687                         dmaengine_get_direction_text(DMA_MEM_TO_MEM));
3688                 return NULL;
3689         }
3690
3691         num_tr = udma_get_tr_counters(len, __ffs(src | dest), &tr0_cnt0,
3692                                       &tr0_cnt1, &tr1_cnt0);
3693         if (num_tr < 0) {
3694                 dev_err(uc->ud->dev, "size %zu is not supported\n",
3695                         len);
3696                 return NULL;
3697         }
3698
3699         d = udma_alloc_tr_desc(uc, tr_size, num_tr, DMA_MEM_TO_MEM);
3700         if (!d)
3701                 return NULL;
3702
3703         d->dir = DMA_MEM_TO_MEM;
3704         d->desc_idx = 0;
3705         d->tr_idx = 0;
3706         d->residue = len;
3707
3708         if (uc->ud->match_data->type != DMA_TYPE_UDMA) {
3709                 src |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;
3710                 dest |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT;
3711         }
3712
3713         tr_req = d->hwdesc[0].tr_req_base;
3714
3715         cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true,
3716                       CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3717         cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT);
3718
3719         tr_req[0].addr = src;
3720         tr_req[0].icnt0 = tr0_cnt0;
3721         tr_req[0].icnt1 = tr0_cnt1;
3722         tr_req[0].icnt2 = 1;
3723         tr_req[0].icnt3 = 1;
3724         tr_req[0].dim1 = tr0_cnt0;
3725
3726         tr_req[0].daddr = dest;
3727         tr_req[0].dicnt0 = tr0_cnt0;
3728         tr_req[0].dicnt1 = tr0_cnt1;
3729         tr_req[0].dicnt2 = 1;
3730         tr_req[0].dicnt3 = 1;
3731         tr_req[0].ddim1 = tr0_cnt0;
3732
3733         if (num_tr == 2) {
3734                 cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true,
3735                               CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
3736                 cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT);
3737
3738                 tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0;
3739                 tr_req[1].icnt0 = tr1_cnt0;
3740                 tr_req[1].icnt1 = 1;
3741                 tr_req[1].icnt2 = 1;
3742                 tr_req[1].icnt3 = 1;
3743
3744                 tr_req[1].daddr = dest + tr0_cnt1 * tr0_cnt0;
3745                 tr_req[1].dicnt0 = tr1_cnt0;
3746                 tr_req[1].dicnt1 = 1;
3747                 tr_req[1].dicnt2 = 1;
3748                 tr_req[1].dicnt3 = 1;
3749         }
3750
3751         cppi5_tr_csf_set(&tr_req[num_tr - 1].flags,
3752                          CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
3753
3754         if (uc->config.metadata_size)
3755                 d->vd.tx.metadata_ops = &metadata_ops;
3756
3757         return vchan_tx_prep(&uc->vc, &d->vd, tx_flags);
3758 }
3759
3760 static void udma_issue_pending(struct dma_chan *chan)
3761 {
3762         struct udma_chan *uc = to_udma_chan(chan);
3763         unsigned long flags;
3764
3765         spin_lock_irqsave(&uc->vc.lock, flags);
3766
3767         /* If we have something pending and no active descriptor, then */
3768         if (vchan_issue_pending(&uc->vc) && !uc->desc) {
3769                 /*
3770                  * start a descriptor if the channel is NOT [marked as
3771                  * terminating _and_ it is still running (teardown has not
3772                  * completed yet)].
3773                  */
3774                 if (!(uc->state == UDMA_CHAN_IS_TERMINATING &&
3775                       udma_is_chan_running(uc)))
3776                         udma_start(uc);
3777         }
3778
3779         spin_unlock_irqrestore(&uc->vc.lock, flags);
3780 }
3781
3782 static enum dma_status udma_tx_status(struct dma_chan *chan,
3783                                       dma_cookie_t cookie,
3784                                       struct dma_tx_state *txstate)
3785 {
3786         struct udma_chan *uc = to_udma_chan(chan);
3787         enum dma_status ret;
3788         unsigned long flags;
3789
3790         spin_lock_irqsave(&uc->vc.lock, flags);
3791
3792         ret = dma_cookie_status(chan, cookie, txstate);
3793
3794         if (!udma_is_chan_running(uc))
3795                 ret = DMA_COMPLETE;
3796
3797         if (ret == DMA_IN_PROGRESS && udma_is_chan_paused(uc))
3798                 ret = DMA_PAUSED;
3799
3800         if (ret == DMA_COMPLETE || !txstate)
3801                 goto out;
3802
3803         if (uc->desc && uc->desc->vd.tx.cookie == cookie) {
3804                 u32 peer_bcnt = 0;
3805                 u32 bcnt = 0;
3806                 u32 residue = uc->desc->residue;
3807                 u32 delay = 0;
3808
3809                 if (uc->desc->dir == DMA_MEM_TO_DEV) {
3810                         bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_SBCNT_REG);
3811
3812                         if (uc->config.ep_type != PSIL_EP_NATIVE) {
3813                                 peer_bcnt = udma_tchanrt_read(uc,
3814                                                 UDMA_CHAN_RT_PEER_BCNT_REG);
3815
3816                                 if (bcnt > peer_bcnt)
3817                                         delay = bcnt - peer_bcnt;
3818                         }
3819                 } else if (uc->desc->dir == DMA_DEV_TO_MEM) {
3820                         bcnt = udma_rchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
3821
3822                         if (uc->config.ep_type != PSIL_EP_NATIVE) {
3823                                 peer_bcnt = udma_rchanrt_read(uc,
3824                                                 UDMA_CHAN_RT_PEER_BCNT_REG);
3825
3826                                 if (peer_bcnt > bcnt)
3827                                         delay = peer_bcnt - bcnt;
3828                         }
3829                 } else {
3830                         bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG);
3831                 }
3832
3833                 if (bcnt && !(bcnt % uc->desc->residue))
3834                         residue = 0;
3835                 else
3836                         residue -= bcnt % uc->desc->residue;
3837
3838                 if (!residue && (uc->config.dir == DMA_DEV_TO_MEM || !delay)) {
3839                         ret = DMA_COMPLETE;
3840                         delay = 0;
3841                 }
3842
3843                 dma_set_residue(txstate, residue);
3844                 dma_set_in_flight_bytes(txstate, delay);
3845
3846         } else {
3847                 ret = DMA_COMPLETE;
3848         }
3849
3850 out:
3851         spin_unlock_irqrestore(&uc->vc.lock, flags);
3852         return ret;
3853 }
3854
3855 static int udma_pause(struct dma_chan *chan)
3856 {
3857         struct udma_chan *uc = to_udma_chan(chan);
3858
3859         /* pause the channel */
3860         switch (uc->config.dir) {
3861         case DMA_DEV_TO_MEM:
3862                 udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
3863                                          UDMA_PEER_RT_EN_PAUSE,
3864                                          UDMA_PEER_RT_EN_PAUSE);
3865                 break;
3866         case DMA_MEM_TO_DEV:
3867                 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
3868                                          UDMA_PEER_RT_EN_PAUSE,
3869                                          UDMA_PEER_RT_EN_PAUSE);
3870                 break;
3871         case DMA_MEM_TO_MEM:
3872                 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG,
3873                                          UDMA_CHAN_RT_CTL_PAUSE,
3874                                          UDMA_CHAN_RT_CTL_PAUSE);
3875                 break;
3876         default:
3877                 return -EINVAL;
3878         }
3879
3880         return 0;
3881 }
3882
3883 static int udma_resume(struct dma_chan *chan)
3884 {
3885         struct udma_chan *uc = to_udma_chan(chan);
3886
3887         /* resume the channel */
3888         switch (uc->config.dir) {
3889         case DMA_DEV_TO_MEM:
3890                 udma_rchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
3891                                          UDMA_PEER_RT_EN_PAUSE, 0);
3892
3893                 break;
3894         case DMA_MEM_TO_DEV:
3895                 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_PEER_RT_EN_REG,
3896                                          UDMA_PEER_RT_EN_PAUSE, 0);
3897                 break;
3898         case DMA_MEM_TO_MEM:
3899                 udma_tchanrt_update_bits(uc, UDMA_CHAN_RT_CTL_REG,
3900                                          UDMA_CHAN_RT_CTL_PAUSE, 0);
3901                 break;
3902         default:
3903                 return -EINVAL;
3904         }
3905
3906         return 0;
3907 }
3908
3909 static int udma_terminate_all(struct dma_chan *chan)
3910 {
3911         struct udma_chan *uc = to_udma_chan(chan);
3912         unsigned long flags;
3913         LIST_HEAD(head);
3914
3915         spin_lock_irqsave(&uc->vc.lock, flags);
3916
3917         if (udma_is_chan_running(uc))
3918                 udma_stop(uc);
3919
3920         if (uc->desc) {
3921                 uc->terminated_desc = uc->desc;
3922                 uc->desc = NULL;
3923                 uc->terminated_desc->terminated = true;
3924                 cancel_delayed_work(&uc->tx_drain.work);
3925         }
3926
3927         uc->paused = false;
3928
3929         vchan_get_all_descriptors(&uc->vc, &head);
3930         spin_unlock_irqrestore(&uc->vc.lock, flags);
3931         vchan_dma_desc_free_list(&uc->vc, &head);
3932
3933         return 0;
3934 }
3935
3936 static void udma_synchronize(struct dma_chan *chan)
3937 {
3938         struct udma_chan *uc = to_udma_chan(chan);
3939         unsigned long timeout = msecs_to_jiffies(1000);
3940
3941         vchan_synchronize(&uc->vc);
3942
3943         if (uc->state == UDMA_CHAN_IS_TERMINATING) {
3944                 timeout = wait_for_completion_timeout(&uc->teardown_completed,
3945                                                       timeout);
3946                 if (!timeout) {
3947                         dev_warn(uc->ud->dev, "chan%d teardown timeout!\n",
3948                                  uc->id);
3949                         udma_dump_chan_stdata(uc);
3950                         udma_reset_chan(uc, true);
3951                 }
3952         }
3953
3954         udma_reset_chan(uc, false);
3955         if (udma_is_chan_running(uc))
3956                 dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id);
3957
3958         cancel_delayed_work_sync(&uc->tx_drain.work);
3959         udma_reset_rings(uc);
3960 }
3961
3962 static void udma_desc_pre_callback(struct virt_dma_chan *vc,
3963                                    struct virt_dma_desc *vd,
3964                                    struct dmaengine_result *result)
3965 {
3966         struct udma_chan *uc = to_udma_chan(&vc->chan);
3967         struct udma_desc *d;
3968
3969         if (!vd)
3970                 return;
3971
3972         d = to_udma_desc(&vd->tx);
3973
3974         if (d->metadata_size)
3975                 udma_fetch_epib(uc, d);
3976
3977         /* Provide residue information for the client */
3978         if (result) {
3979                 void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx);
3980
3981                 if (cppi5_desc_get_type(desc_vaddr) ==
3982                     CPPI5_INFO0_DESC_TYPE_VAL_HOST) {
3983                         result->residue = d->residue -
3984                                           cppi5_hdesc_get_pktlen(desc_vaddr);
3985                         if (result->residue)
3986                                 result->result = DMA_TRANS_ABORTED;
3987                         else
3988                                 result->result = DMA_TRANS_NOERROR;
3989                 } else {
3990                         result->residue = 0;
3991                         result->result = DMA_TRANS_NOERROR;
3992                 }
3993         }
3994 }
3995
3996 /*
3997  * This tasklet handles the completion of a DMA descriptor by
3998  * calling its callback and freeing it.
3999  */
4000 static void udma_vchan_complete(struct tasklet_struct *t)
4001 {
4002         struct virt_dma_chan *vc = from_tasklet(vc, t, task);
4003         struct virt_dma_desc *vd, *_vd;
4004         struct dmaengine_desc_callback cb;
4005         LIST_HEAD(head);
4006
4007         spin_lock_irq(&vc->lock);
4008         list_splice_tail_init(&vc->desc_completed, &head);
4009         vd = vc->cyclic;
4010         if (vd) {
4011                 vc->cyclic = NULL;
4012                 dmaengine_desc_get_callback(&vd->tx, &cb);
4013         } else {
4014                 memset(&cb, 0, sizeof(cb));
4015         }
4016         spin_unlock_irq(&vc->lock);
4017
4018         udma_desc_pre_callback(vc, vd, NULL);
4019         dmaengine_desc_callback_invoke(&cb, NULL);
4020
4021         list_for_each_entry_safe(vd, _vd, &head, node) {
4022                 struct dmaengine_result result;
4023
4024                 dmaengine_desc_get_callback(&vd->tx, &cb);
4025
4026                 list_del(&vd->node);
4027
4028                 udma_desc_pre_callback(vc, vd, &result);
4029                 dmaengine_desc_callback_invoke(&cb, &result);
4030
4031                 vchan_vdesc_fini(vd);
4032         }
4033 }
4034
4035 static void udma_free_chan_resources(struct dma_chan *chan)
4036 {
4037         struct udma_chan *uc = to_udma_chan(chan);
4038         struct udma_dev *ud = to_udma_dev(chan->device);
4039
4040         udma_terminate_all(chan);
4041         if (uc->terminated_desc) {
4042                 udma_reset_chan(uc, false);
4043                 udma_reset_rings(uc);
4044         }
4045
4046         cancel_delayed_work_sync(&uc->tx_drain.work);
4047
4048         if (uc->irq_num_ring > 0) {
4049                 free_irq(uc->irq_num_ring, uc);
4050
4051                 uc->irq_num_ring = 0;
4052         }
4053         if (uc->irq_num_udma > 0) {
4054                 free_irq(uc->irq_num_udma, uc);
4055
4056                 uc->irq_num_udma = 0;
4057         }
4058
4059         /* Release PSI-L pairing */
4060         if (uc->psil_paired) {
4061                 navss_psil_unpair(ud, uc->config.src_thread,
4062                                   uc->config.dst_thread);
4063                 uc->psil_paired = false;
4064         }
4065
4066         vchan_free_chan_resources(&uc->vc);
4067         tasklet_kill(&uc->vc.task);
4068
4069         bcdma_free_bchan_resources(uc);
4070         udma_free_tx_resources(uc);
4071         udma_free_rx_resources(uc);
4072         udma_reset_uchan(uc);
4073
4074         if (uc->use_dma_pool) {
4075                 dma_pool_destroy(uc->hdesc_pool);
4076                 uc->use_dma_pool = false;
4077         }
4078 }
4079
4080 static struct platform_driver udma_driver;
4081 static struct platform_driver bcdma_driver;
4082 static struct platform_driver pktdma_driver;
4083
4084 struct udma_filter_param {
4085         int remote_thread_id;
4086         u32 atype;
4087         u32 asel;
4088         u32 tr_trigger_type;
4089 };
4090
4091 static bool udma_dma_filter_fn(struct dma_chan *chan, void *param)
4092 {
4093         struct udma_chan_config *ucc;
4094         struct psil_endpoint_config *ep_config;
4095         struct udma_filter_param *filter_param;
4096         struct udma_chan *uc;
4097         struct udma_dev *ud;
4098
4099         if (chan->device->dev->driver != &udma_driver.driver &&
4100             chan->device->dev->driver != &bcdma_driver.driver &&
4101             chan->device->dev->driver != &pktdma_driver.driver)
4102                 return false;
4103
4104         uc = to_udma_chan(chan);
4105         ucc = &uc->config;
4106         ud = uc->ud;
4107         filter_param = param;
4108
4109         if (filter_param->atype > 2) {
4110                 dev_err(ud->dev, "Invalid channel atype: %u\n",
4111                         filter_param->atype);
4112                 return false;
4113         }
4114
4115         if (filter_param->asel > 15) {
4116                 dev_err(ud->dev, "Invalid channel asel: %u\n",
4117                         filter_param->asel);
4118                 return false;
4119         }
4120
4121         ucc->remote_thread_id = filter_param->remote_thread_id;
4122         ucc->atype = filter_param->atype;
4123         ucc->asel = filter_param->asel;
4124         ucc->tr_trigger_type = filter_param->tr_trigger_type;
4125
4126         if (ucc->tr_trigger_type) {
4127                 ucc->dir = DMA_MEM_TO_MEM;
4128                 goto triggered_bchan;
4129         } else if (ucc->remote_thread_id & K3_PSIL_DST_THREAD_ID_OFFSET) {
4130                 ucc->dir = DMA_MEM_TO_DEV;
4131         } else {
4132                 ucc->dir = DMA_DEV_TO_MEM;
4133         }
4134
4135         ep_config = psil_get_ep_config(ucc->remote_thread_id);
4136         if (IS_ERR(ep_config)) {
4137                 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n",
4138                         ucc->remote_thread_id);
4139                 ucc->dir = DMA_MEM_TO_MEM;
4140                 ucc->remote_thread_id = -1;
4141                 ucc->atype = 0;
4142                 ucc->asel = 0;
4143                 return false;
4144         }
4145
4146         if (ud->match_data->type == DMA_TYPE_BCDMA &&
4147             ep_config->pkt_mode) {
4148                 dev_err(ud->dev,
4149                         "Only TR mode is supported (psi-l thread 0x%04x)\n",
4150                         ucc->remote_thread_id);
4151                 ucc->dir = DMA_MEM_TO_MEM;
4152                 ucc->remote_thread_id = -1;
4153                 ucc->atype = 0;
4154                 ucc->asel = 0;
4155                 return false;
4156         }
4157
4158         ucc->pkt_mode = ep_config->pkt_mode;
4159         ucc->channel_tpl = ep_config->channel_tpl;
4160         ucc->notdpkt = ep_config->notdpkt;
4161         ucc->ep_type = ep_config->ep_type;
4162
4163         if (ud->match_data->type == DMA_TYPE_PKTDMA &&
4164             ep_config->mapped_channel_id >= 0) {
4165                 ucc->mapped_channel_id = ep_config->mapped_channel_id;
4166                 ucc->default_flow_id = ep_config->default_flow_id;
4167         } else {
4168                 ucc->mapped_channel_id = -1;
4169                 ucc->default_flow_id = -1;
4170         }
4171
4172         if (ucc->ep_type != PSIL_EP_NATIVE) {
4173                 const struct udma_match_data *match_data = ud->match_data;
4174
4175                 if (match_data->flags & UDMA_FLAG_PDMA_ACC32)
4176                         ucc->enable_acc32 = ep_config->pdma_acc32;
4177                 if (match_data->flags & UDMA_FLAG_PDMA_BURST)
4178                         ucc->enable_burst = ep_config->pdma_burst;
4179         }
4180
4181         ucc->needs_epib = ep_config->needs_epib;
4182         ucc->psd_size = ep_config->psd_size;
4183         ucc->metadata_size =
4184                         (ucc->needs_epib ? CPPI5_INFO0_HDESC_EPIB_SIZE : 0) +
4185                         ucc->psd_size;
4186
4187         if (ucc->pkt_mode)
4188                 ucc->hdesc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
4189                                  ucc->metadata_size, ud->desc_align);
4190
4191         dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id,
4192                 ucc->remote_thread_id, dmaengine_get_direction_text(ucc->dir));
4193
4194         return true;
4195
4196 triggered_bchan:
4197         dev_dbg(ud->dev, "chan%d: triggered channel (type: %u)\n", uc->id,
4198                 ucc->tr_trigger_type);
4199
4200         return true;
4201
4202 }
4203
4204 static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec,
4205                                       struct of_dma *ofdma)
4206 {
4207         struct udma_dev *ud = ofdma->of_dma_data;
4208         dma_cap_mask_t mask = ud->ddev.cap_mask;
4209         struct udma_filter_param filter_param;
4210         struct dma_chan *chan;
4211
4212         if (ud->match_data->type == DMA_TYPE_BCDMA) {
4213                 if (dma_spec->args_count != 3)
4214                         return NULL;
4215
4216                 filter_param.tr_trigger_type = dma_spec->args[0];
4217                 filter_param.remote_thread_id = dma_spec->args[1];
4218                 filter_param.asel = dma_spec->args[2];
4219                 filter_param.atype = 0;
4220         } else {
4221                 if (dma_spec->args_count != 1 && dma_spec->args_count != 2)
4222                         return NULL;
4223
4224                 filter_param.remote_thread_id = dma_spec->args[0];
4225                 filter_param.tr_trigger_type = 0;
4226                 if (dma_spec->args_count == 2) {
4227                         if (ud->match_data->type == DMA_TYPE_UDMA) {
4228                                 filter_param.atype = dma_spec->args[1];
4229                                 filter_param.asel = 0;
4230                         } else {
4231                                 filter_param.atype = 0;
4232                                 filter_param.asel = dma_spec->args[1];
4233                         }
4234                 } else {
4235                         filter_param.atype = 0;
4236                         filter_param.asel = 0;
4237                 }
4238         }
4239
4240         chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param,
4241                                      ofdma->of_node);
4242         if (!chan) {
4243                 dev_err(ud->dev, "get channel fail in %s.\n", __func__);
4244                 return ERR_PTR(-EINVAL);
4245         }
4246
4247         return chan;
4248 }
4249
4250 static struct udma_match_data am654_main_data = {
4251         .type = DMA_TYPE_UDMA,
4252         .psil_base = 0x1000,
4253         .enable_memcpy_support = true,
4254         .statictr_z_mask = GENMASK(11, 0),
4255         .burst_size = {
4256                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4257                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */
4258                 0, /* No UH Channels */
4259         },
4260 };
4261
4262 static struct udma_match_data am654_mcu_data = {
4263         .type = DMA_TYPE_UDMA,
4264         .psil_base = 0x6000,
4265         .enable_memcpy_support = false,
4266         .statictr_z_mask = GENMASK(11, 0),
4267         .burst_size = {
4268                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4269                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */
4270                 0, /* No UH Channels */
4271         },
4272 };
4273
4274 static struct udma_match_data j721e_main_data = {
4275         .type = DMA_TYPE_UDMA,
4276         .psil_base = 0x1000,
4277         .enable_memcpy_support = true,
4278         .flags = UDMA_FLAGS_J7_CLASS,
4279         .statictr_z_mask = GENMASK(23, 0),
4280         .burst_size = {
4281                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4282                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* H Channels */
4283                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* UH Channels */
4284         },
4285 };
4286
4287 static struct udma_match_data j721e_mcu_data = {
4288         .type = DMA_TYPE_UDMA,
4289         .psil_base = 0x6000,
4290         .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */
4291         .flags = UDMA_FLAGS_J7_CLASS,
4292         .statictr_z_mask = GENMASK(23, 0),
4293         .burst_size = {
4294                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4295                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES, /* H Channels */
4296                 0, /* No UH Channels */
4297         },
4298 };
4299
4300 static struct udma_soc_data am62a_dmss_csi_soc_data = {
4301         .oes = {
4302                 .bcdma_rchan_data = 0xe00,
4303                 .bcdma_rchan_ring = 0x1000,
4304         },
4305 };
4306
4307 static struct udma_match_data am62a_bcdma_csirx_data = {
4308         .type = DMA_TYPE_BCDMA,
4309         .psil_base = 0x3100,
4310         .enable_memcpy_support = false,
4311         .burst_size = {
4312                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4313                 0, /* No H Channels */
4314                 0, /* No UH Channels */
4315         },
4316         .soc_data = &am62a_dmss_csi_soc_data,
4317 };
4318
4319 static struct udma_match_data am64_bcdma_data = {
4320         .type = DMA_TYPE_BCDMA,
4321         .psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */
4322         .enable_memcpy_support = true, /* Supported via bchan */
4323         .flags = UDMA_FLAGS_J7_CLASS,
4324         .statictr_z_mask = GENMASK(23, 0),
4325         .burst_size = {
4326                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4327                 0, /* No H Channels */
4328                 0, /* No UH Channels */
4329         },
4330 };
4331
4332 static struct udma_match_data am64_pktdma_data = {
4333         .type = DMA_TYPE_PKTDMA,
4334         .psil_base = 0x1000,
4335         .enable_memcpy_support = false, /* PKTDMA does not support MEM_TO_MEM */
4336         .flags = UDMA_FLAGS_J7_CLASS,
4337         .statictr_z_mask = GENMASK(23, 0),
4338         .burst_size = {
4339                 TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */
4340                 0, /* No H Channels */
4341                 0, /* No UH Channels */
4342         },
4343 };
4344
4345 static const struct of_device_id udma_of_match[] = {
4346         {
4347                 .compatible = "ti,am654-navss-main-udmap",
4348                 .data = &am654_main_data,
4349         },
4350         {
4351                 .compatible = "ti,am654-navss-mcu-udmap",
4352                 .data = &am654_mcu_data,
4353         }, {
4354                 .compatible = "ti,j721e-navss-main-udmap",
4355                 .data = &j721e_main_data,
4356         }, {
4357                 .compatible = "ti,j721e-navss-mcu-udmap",
4358                 .data = &j721e_mcu_data,
4359         },
4360         {
4361                 .compatible = "ti,am64-dmss-bcdma",
4362                 .data = &am64_bcdma_data,
4363         },
4364         {
4365                 .compatible = "ti,am64-dmss-pktdma",
4366                 .data = &am64_pktdma_data,
4367         },
4368         {
4369                 .compatible = "ti,am62a-dmss-bcdma-csirx",
4370                 .data = &am62a_bcdma_csirx_data,
4371         },
4372         { /* Sentinel */ },
4373 };
4374
4375 static struct udma_soc_data am654_soc_data = {
4376         .oes = {
4377                 .udma_rchan = 0x200,
4378         },
4379 };
4380
4381 static struct udma_soc_data j721e_soc_data = {
4382         .oes = {
4383                 .udma_rchan = 0x400,
4384         },
4385 };
4386
4387 static struct udma_soc_data j7200_soc_data = {
4388         .oes = {
4389                 .udma_rchan = 0x80,
4390         },
4391 };
4392
4393 static struct udma_soc_data am64_soc_data = {
4394         .oes = {
4395                 .bcdma_bchan_data = 0x2200,
4396                 .bcdma_bchan_ring = 0x2400,
4397                 .bcdma_tchan_data = 0x2800,
4398                 .bcdma_tchan_ring = 0x2a00,
4399                 .bcdma_rchan_data = 0x2e00,
4400                 .bcdma_rchan_ring = 0x3000,
4401                 .pktdma_tchan_flow = 0x1200,
4402                 .pktdma_rchan_flow = 0x1600,
4403         },
4404         .bcdma_trigger_event_offset = 0xc400,
4405 };
4406
4407 static const struct soc_device_attribute k3_soc_devices[] = {
4408         { .family = "AM65X", .data = &am654_soc_data },
4409         { .family = "J721E", .data = &j721e_soc_data },
4410         { .family = "J7200", .data = &j7200_soc_data },
4411         { .family = "AM64X", .data = &am64_soc_data },
4412         { .family = "J721S2", .data = &j721e_soc_data},
4413         { .family = "AM62X", .data = &am64_soc_data },
4414         { .family = "AM62AX", .data = &am64_soc_data },
4415         { /* sentinel */ }
4416 };
4417
4418 static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud)
4419 {
4420         u32 cap2, cap3, cap4;
4421         int i;
4422
4423         ud->mmrs[MMR_GCFG] = devm_platform_ioremap_resource_byname(pdev, mmr_names[MMR_GCFG]);
4424         if (IS_ERR(ud->mmrs[MMR_GCFG]))
4425                 return PTR_ERR(ud->mmrs[MMR_GCFG]);
4426
4427         cap2 = udma_read(ud->mmrs[MMR_GCFG], 0x28);
4428         cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4429
4430         switch (ud->match_data->type) {
4431         case DMA_TYPE_UDMA:
4432                 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3);
4433                 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2);
4434                 ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2);
4435                 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2);
4436                 break;
4437         case DMA_TYPE_BCDMA:
4438                 ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2);
4439                 ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2);
4440                 ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2);
4441                 ud->rflow_cnt = ud->rchan_cnt;
4442                 break;
4443         case DMA_TYPE_PKTDMA:
4444                 cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30);
4445                 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2);
4446                 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2);
4447                 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3);
4448                 ud->tflow_cnt = PKTDMA_CAP4_TFLOW_CNT(cap4);
4449                 break;
4450         default:
4451                 return -EINVAL;
4452         }
4453
4454         for (i = 1; i < MMR_LAST; i++) {
4455                 if (i == MMR_BCHANRT && ud->bchan_cnt == 0)
4456                         continue;
4457                 if (i == MMR_TCHANRT && ud->tchan_cnt == 0)
4458                         continue;
4459                 if (i == MMR_RCHANRT && ud->rchan_cnt == 0)
4460                         continue;
4461
4462                 ud->mmrs[i] = devm_platform_ioremap_resource_byname(pdev, mmr_names[i]);
4463                 if (IS_ERR(ud->mmrs[i]))
4464                         return PTR_ERR(ud->mmrs[i]);
4465         }
4466
4467         return 0;
4468 }
4469
4470 static void udma_mark_resource_ranges(struct udma_dev *ud, unsigned long *map,
4471                                       struct ti_sci_resource_desc *rm_desc,
4472                                       char *name)
4473 {
4474         bitmap_clear(map, rm_desc->start, rm_desc->num);
4475         bitmap_clear(map, rm_desc->start_sec, rm_desc->num_sec);
4476         dev_dbg(ud->dev, "ti_sci resource range for %s: %d:%d | %d:%d\n", name,
4477                 rm_desc->start, rm_desc->num, rm_desc->start_sec,
4478                 rm_desc->num_sec);
4479 }
4480
4481 static const char * const range_names[] = {
4482         [RM_RANGE_BCHAN] = "ti,sci-rm-range-bchan",
4483         [RM_RANGE_TCHAN] = "ti,sci-rm-range-tchan",
4484         [RM_RANGE_RCHAN] = "ti,sci-rm-range-rchan",
4485         [RM_RANGE_RFLOW] = "ti,sci-rm-range-rflow",
4486         [RM_RANGE_TFLOW] = "ti,sci-rm-range-tflow",
4487 };
4488
4489 static int udma_setup_resources(struct udma_dev *ud)
4490 {
4491         int ret, i, j;
4492         struct device *dev = ud->dev;
4493         struct ti_sci_resource *rm_res, irq_res;
4494         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
4495         u32 cap3;
4496
4497         /* Set up the throughput level start indexes */
4498         cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4499         if (of_device_is_compatible(dev->of_node,
4500                                     "ti,am654-navss-main-udmap")) {
4501                 ud->tchan_tpl.levels = 2;
4502                 ud->tchan_tpl.start_idx[0] = 8;
4503         } else if (of_device_is_compatible(dev->of_node,
4504                                            "ti,am654-navss-mcu-udmap")) {
4505                 ud->tchan_tpl.levels = 2;
4506                 ud->tchan_tpl.start_idx[0] = 2;
4507         } else if (UDMA_CAP3_UCHAN_CNT(cap3)) {
4508                 ud->tchan_tpl.levels = 3;
4509                 ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3);
4510                 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4511         } else if (UDMA_CAP3_HCHAN_CNT(cap3)) {
4512                 ud->tchan_tpl.levels = 2;
4513                 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4514         } else {
4515                 ud->tchan_tpl.levels = 1;
4516         }
4517
4518         ud->rchan_tpl.levels = ud->tchan_tpl.levels;
4519         ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0];
4520         ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1];
4521
4522         ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
4523                                            sizeof(unsigned long), GFP_KERNEL);
4524         ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
4525                                   GFP_KERNEL);
4526         ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
4527                                            sizeof(unsigned long), GFP_KERNEL);
4528         ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
4529                                   GFP_KERNEL);
4530         ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt),
4531                                               sizeof(unsigned long),
4532                                               GFP_KERNEL);
4533         ud->rflow_gp_map_allocated = devm_kcalloc(dev,
4534                                                   BITS_TO_LONGS(ud->rflow_cnt),
4535                                                   sizeof(unsigned long),
4536                                                   GFP_KERNEL);
4537         ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
4538                                         sizeof(unsigned long),
4539                                         GFP_KERNEL);
4540         ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
4541                                   GFP_KERNEL);
4542
4543         if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map ||
4544             !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans ||
4545             !ud->rflows || !ud->rflow_in_use)
4546                 return -ENOMEM;
4547
4548         /*
4549          * RX flows with the same Ids as RX channels are reserved to be used
4550          * as default flows if remote HW can't generate flow_ids. Those
4551          * RX flows can be requested only explicitly by id.
4552          */
4553         bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt);
4554
4555         /* by default no GP rflows are assigned to Linux */
4556         bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt);
4557
4558         /* Get resource ranges from tisci */
4559         for (i = 0; i < RM_RANGE_LAST; i++) {
4560                 if (i == RM_RANGE_BCHAN || i == RM_RANGE_TFLOW)
4561                         continue;
4562
4563                 tisci_rm->rm_ranges[i] =
4564                         devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
4565                                                     tisci_rm->tisci_dev_id,
4566                                                     (char *)range_names[i]);
4567         }
4568
4569         /* tchan ranges */
4570         rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4571         if (IS_ERR(rm_res)) {
4572                 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
4573                 irq_res.sets = 1;
4574         } else {
4575                 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
4576                 for (i = 0; i < rm_res->sets; i++)
4577                         udma_mark_resource_ranges(ud, ud->tchan_map,
4578                                                   &rm_res->desc[i], "tchan");
4579                 irq_res.sets = rm_res->sets;
4580         }
4581
4582         /* rchan and matching default flow ranges */
4583         rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4584         if (IS_ERR(rm_res)) {
4585                 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
4586                 irq_res.sets++;
4587         } else {
4588                 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
4589                 for (i = 0; i < rm_res->sets; i++)
4590                         udma_mark_resource_ranges(ud, ud->rchan_map,
4591                                                   &rm_res->desc[i], "rchan");
4592                 irq_res.sets += rm_res->sets;
4593         }
4594
4595         irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);
4596         if (!irq_res.desc)
4597                 return -ENOMEM;
4598         rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4599         if (IS_ERR(rm_res)) {
4600                 irq_res.desc[0].start = 0;
4601                 irq_res.desc[0].num = ud->tchan_cnt;
4602                 i = 1;
4603         } else {
4604                 for (i = 0; i < rm_res->sets; i++) {
4605                         irq_res.desc[i].start = rm_res->desc[i].start;
4606                         irq_res.desc[i].num = rm_res->desc[i].num;
4607                         irq_res.desc[i].start_sec = rm_res->desc[i].start_sec;
4608                         irq_res.desc[i].num_sec = rm_res->desc[i].num_sec;
4609                 }
4610         }
4611         rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4612         if (IS_ERR(rm_res)) {
4613                 irq_res.desc[i].start = 0;
4614                 irq_res.desc[i].num = ud->rchan_cnt;
4615         } else {
4616                 for (j = 0; j < rm_res->sets; j++, i++) {
4617                         if (rm_res->desc[j].num) {
4618                                 irq_res.desc[i].start = rm_res->desc[j].start +
4619                                                 ud->soc_data->oes.udma_rchan;
4620                                 irq_res.desc[i].num = rm_res->desc[j].num;
4621                         }
4622                         if (rm_res->desc[j].num_sec) {
4623                                 irq_res.desc[i].start_sec = rm_res->desc[j].start_sec +
4624                                                 ud->soc_data->oes.udma_rchan;
4625                                 irq_res.desc[i].num_sec = rm_res->desc[j].num_sec;
4626                         }
4627                 }
4628         }
4629         ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
4630         kfree(irq_res.desc);
4631         if (ret) {
4632                 dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
4633                 return ret;
4634         }
4635
4636         /* GP rflow ranges */
4637         rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
4638         if (IS_ERR(rm_res)) {
4639                 /* all gp flows are assigned exclusively to Linux */
4640                 bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt,
4641                              ud->rflow_cnt - ud->rchan_cnt);
4642         } else {
4643                 for (i = 0; i < rm_res->sets; i++)
4644                         udma_mark_resource_ranges(ud, ud->rflow_gp_map,
4645                                                   &rm_res->desc[i], "gp-rflow");
4646         }
4647
4648         return 0;
4649 }
4650
4651 static int bcdma_setup_resources(struct udma_dev *ud)
4652 {
4653         int ret, i, j;
4654         struct device *dev = ud->dev;
4655         struct ti_sci_resource *rm_res, irq_res;
4656         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
4657         const struct udma_oes_offsets *oes = &ud->soc_data->oes;
4658         u32 cap;
4659
4660         /* Set up the throughput level start indexes */
4661         cap = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4662         if (BCDMA_CAP3_UBCHAN_CNT(cap)) {
4663                 ud->bchan_tpl.levels = 3;
4664                 ud->bchan_tpl.start_idx[1] = BCDMA_CAP3_UBCHAN_CNT(cap);
4665                 ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap);
4666         } else if (BCDMA_CAP3_HBCHAN_CNT(cap)) {
4667                 ud->bchan_tpl.levels = 2;
4668                 ud->bchan_tpl.start_idx[0] = BCDMA_CAP3_HBCHAN_CNT(cap);
4669         } else {
4670                 ud->bchan_tpl.levels = 1;
4671         }
4672
4673         cap = udma_read(ud->mmrs[MMR_GCFG], 0x30);
4674         if (BCDMA_CAP4_URCHAN_CNT(cap)) {
4675                 ud->rchan_tpl.levels = 3;
4676                 ud->rchan_tpl.start_idx[1] = BCDMA_CAP4_URCHAN_CNT(cap);
4677                 ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap);
4678         } else if (BCDMA_CAP4_HRCHAN_CNT(cap)) {
4679                 ud->rchan_tpl.levels = 2;
4680                 ud->rchan_tpl.start_idx[0] = BCDMA_CAP4_HRCHAN_CNT(cap);
4681         } else {
4682                 ud->rchan_tpl.levels = 1;
4683         }
4684
4685         if (BCDMA_CAP4_UTCHAN_CNT(cap)) {
4686                 ud->tchan_tpl.levels = 3;
4687                 ud->tchan_tpl.start_idx[1] = BCDMA_CAP4_UTCHAN_CNT(cap);
4688                 ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap);
4689         } else if (BCDMA_CAP4_HTCHAN_CNT(cap)) {
4690                 ud->tchan_tpl.levels = 2;
4691                 ud->tchan_tpl.start_idx[0] = BCDMA_CAP4_HTCHAN_CNT(cap);
4692         } else {
4693                 ud->tchan_tpl.levels = 1;
4694         }
4695
4696         ud->bchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->bchan_cnt),
4697                                            sizeof(unsigned long), GFP_KERNEL);
4698         ud->bchans = devm_kcalloc(dev, ud->bchan_cnt, sizeof(*ud->bchans),
4699                                   GFP_KERNEL);
4700         ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
4701                                            sizeof(unsigned long), GFP_KERNEL);
4702         ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
4703                                   GFP_KERNEL);
4704         ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
4705                                            sizeof(unsigned long), GFP_KERNEL);
4706         ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
4707                                   GFP_KERNEL);
4708         /* BCDMA do not really have flows, but the driver expect it */
4709         ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rchan_cnt),
4710                                         sizeof(unsigned long),
4711                                         GFP_KERNEL);
4712         ud->rflows = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rflows),
4713                                   GFP_KERNEL);
4714
4715         if (!ud->bchan_map || !ud->tchan_map || !ud->rchan_map ||
4716             !ud->rflow_in_use || !ud->bchans || !ud->tchans || !ud->rchans ||
4717             !ud->rflows)
4718                 return -ENOMEM;
4719
4720         /* Get resource ranges from tisci */
4721         for (i = 0; i < RM_RANGE_LAST; i++) {
4722                 if (i == RM_RANGE_RFLOW || i == RM_RANGE_TFLOW)
4723                         continue;
4724                 if (i == RM_RANGE_BCHAN && ud->bchan_cnt == 0)
4725                         continue;
4726                 if (i == RM_RANGE_TCHAN && ud->tchan_cnt == 0)
4727                         continue;
4728                 if (i == RM_RANGE_RCHAN && ud->rchan_cnt == 0)
4729                         continue;
4730
4731                 tisci_rm->rm_ranges[i] =
4732                         devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
4733                                                     tisci_rm->tisci_dev_id,
4734                                                     (char *)range_names[i]);
4735         }
4736
4737         irq_res.sets = 0;
4738
4739         /* bchan ranges */
4740         if (ud->bchan_cnt) {
4741                 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];
4742                 if (IS_ERR(rm_res)) {
4743                         bitmap_zero(ud->bchan_map, ud->bchan_cnt);
4744                         irq_res.sets++;
4745                 } else {
4746                         bitmap_fill(ud->bchan_map, ud->bchan_cnt);
4747                         for (i = 0; i < rm_res->sets; i++)
4748                                 udma_mark_resource_ranges(ud, ud->bchan_map,
4749                                                           &rm_res->desc[i],
4750                                                           "bchan");
4751                         irq_res.sets += rm_res->sets;
4752                 }
4753         }
4754
4755         /* tchan ranges */
4756         if (ud->tchan_cnt) {
4757                 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4758                 if (IS_ERR(rm_res)) {
4759                         bitmap_zero(ud->tchan_map, ud->tchan_cnt);
4760                         irq_res.sets += 2;
4761                 } else {
4762                         bitmap_fill(ud->tchan_map, ud->tchan_cnt);
4763                         for (i = 0; i < rm_res->sets; i++)
4764                                 udma_mark_resource_ranges(ud, ud->tchan_map,
4765                                                           &rm_res->desc[i],
4766                                                           "tchan");
4767                         irq_res.sets += rm_res->sets * 2;
4768                 }
4769         }
4770
4771         /* rchan ranges */
4772         if (ud->rchan_cnt) {
4773                 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4774                 if (IS_ERR(rm_res)) {
4775                         bitmap_zero(ud->rchan_map, ud->rchan_cnt);
4776                         irq_res.sets += 2;
4777                 } else {
4778                         bitmap_fill(ud->rchan_map, ud->rchan_cnt);
4779                         for (i = 0; i < rm_res->sets; i++)
4780                                 udma_mark_resource_ranges(ud, ud->rchan_map,
4781                                                           &rm_res->desc[i],
4782                                                           "rchan");
4783                         irq_res.sets += rm_res->sets * 2;
4784                 }
4785         }
4786
4787         irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);
4788         if (!irq_res.desc)
4789                 return -ENOMEM;
4790         if (ud->bchan_cnt) {
4791                 rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN];
4792                 if (IS_ERR(rm_res)) {
4793                         irq_res.desc[0].start = oes->bcdma_bchan_ring;
4794                         irq_res.desc[0].num = ud->bchan_cnt;
4795                         i = 1;
4796                 } else {
4797                         for (i = 0; i < rm_res->sets; i++) {
4798                                 irq_res.desc[i].start = rm_res->desc[i].start +
4799                                                         oes->bcdma_bchan_ring;
4800                                 irq_res.desc[i].num = rm_res->desc[i].num;
4801                         }
4802                 }
4803         } else {
4804                 i = 0;
4805         }
4806
4807         if (ud->tchan_cnt) {
4808                 rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4809                 if (IS_ERR(rm_res)) {
4810                         irq_res.desc[i].start = oes->bcdma_tchan_data;
4811                         irq_res.desc[i].num = ud->tchan_cnt;
4812                         irq_res.desc[i + 1].start = oes->bcdma_tchan_ring;
4813                         irq_res.desc[i + 1].num = ud->tchan_cnt;
4814                         i += 2;
4815                 } else {
4816                         for (j = 0; j < rm_res->sets; j++, i += 2) {
4817                                 irq_res.desc[i].start = rm_res->desc[j].start +
4818                                                         oes->bcdma_tchan_data;
4819                                 irq_res.desc[i].num = rm_res->desc[j].num;
4820
4821                                 irq_res.desc[i + 1].start = rm_res->desc[j].start +
4822                                                         oes->bcdma_tchan_ring;
4823                                 irq_res.desc[i + 1].num = rm_res->desc[j].num;
4824                         }
4825                 }
4826         }
4827         if (ud->rchan_cnt) {
4828                 rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4829                 if (IS_ERR(rm_res)) {
4830                         irq_res.desc[i].start = oes->bcdma_rchan_data;
4831                         irq_res.desc[i].num = ud->rchan_cnt;
4832                         irq_res.desc[i + 1].start = oes->bcdma_rchan_ring;
4833                         irq_res.desc[i + 1].num = ud->rchan_cnt;
4834                         i += 2;
4835                 } else {
4836                         for (j = 0; j < rm_res->sets; j++, i += 2) {
4837                                 irq_res.desc[i].start = rm_res->desc[j].start +
4838                                                         oes->bcdma_rchan_data;
4839                                 irq_res.desc[i].num = rm_res->desc[j].num;
4840
4841                                 irq_res.desc[i + 1].start = rm_res->desc[j].start +
4842                                                         oes->bcdma_rchan_ring;
4843                                 irq_res.desc[i + 1].num = rm_res->desc[j].num;
4844                         }
4845                 }
4846         }
4847
4848         ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
4849         kfree(irq_res.desc);
4850         if (ret) {
4851                 dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
4852                 return ret;
4853         }
4854
4855         return 0;
4856 }
4857
4858 static int pktdma_setup_resources(struct udma_dev *ud)
4859 {
4860         int ret, i, j;
4861         struct device *dev = ud->dev;
4862         struct ti_sci_resource *rm_res, irq_res;
4863         struct udma_tisci_rm *tisci_rm = &ud->tisci_rm;
4864         const struct udma_oes_offsets *oes = &ud->soc_data->oes;
4865         u32 cap3;
4866
4867         /* Set up the throughput level start indexes */
4868         cap3 = udma_read(ud->mmrs[MMR_GCFG], 0x2c);
4869         if (UDMA_CAP3_UCHAN_CNT(cap3)) {
4870                 ud->tchan_tpl.levels = 3;
4871                 ud->tchan_tpl.start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3);
4872                 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4873         } else if (UDMA_CAP3_HCHAN_CNT(cap3)) {
4874                 ud->tchan_tpl.levels = 2;
4875                 ud->tchan_tpl.start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3);
4876         } else {
4877                 ud->tchan_tpl.levels = 1;
4878         }
4879
4880         ud->rchan_tpl.levels = ud->tchan_tpl.levels;
4881         ud->rchan_tpl.start_idx[0] = ud->tchan_tpl.start_idx[0];
4882         ud->rchan_tpl.start_idx[1] = ud->tchan_tpl.start_idx[1];
4883
4884         ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt),
4885                                            sizeof(unsigned long), GFP_KERNEL);
4886         ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans),
4887                                   GFP_KERNEL);
4888         ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt),
4889                                            sizeof(unsigned long), GFP_KERNEL);
4890         ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans),
4891                                   GFP_KERNEL);
4892         ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt),
4893                                         sizeof(unsigned long),
4894                                         GFP_KERNEL);
4895         ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows),
4896                                   GFP_KERNEL);
4897         ud->tflow_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tflow_cnt),
4898                                            sizeof(unsigned long), GFP_KERNEL);
4899
4900         if (!ud->tchan_map || !ud->rchan_map || !ud->tflow_map || !ud->tchans ||
4901             !ud->rchans || !ud->rflows || !ud->rflow_in_use)
4902                 return -ENOMEM;
4903
4904         /* Get resource ranges from tisci */
4905         for (i = 0; i < RM_RANGE_LAST; i++) {
4906                 if (i == RM_RANGE_BCHAN)
4907                         continue;
4908
4909                 tisci_rm->rm_ranges[i] =
4910                         devm_ti_sci_get_of_resource(tisci_rm->tisci, dev,
4911                                                     tisci_rm->tisci_dev_id,
4912                                                     (char *)range_names[i]);
4913         }
4914
4915         /* tchan ranges */
4916         rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN];
4917         if (IS_ERR(rm_res)) {
4918                 bitmap_zero(ud->tchan_map, ud->tchan_cnt);
4919         } else {
4920                 bitmap_fill(ud->tchan_map, ud->tchan_cnt);
4921                 for (i = 0; i < rm_res->sets; i++)
4922                         udma_mark_resource_ranges(ud, ud->tchan_map,
4923                                                   &rm_res->desc[i], "tchan");
4924         }
4925
4926         /* rchan ranges */
4927         rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN];
4928         if (IS_ERR(rm_res)) {
4929                 bitmap_zero(ud->rchan_map, ud->rchan_cnt);
4930         } else {
4931                 bitmap_fill(ud->rchan_map, ud->rchan_cnt);
4932                 for (i = 0; i < rm_res->sets; i++)
4933                         udma_mark_resource_ranges(ud, ud->rchan_map,
4934                                                   &rm_res->desc[i], "rchan");
4935         }
4936
4937         /* rflow ranges */
4938         rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
4939         if (IS_ERR(rm_res)) {
4940                 /* all rflows are assigned exclusively to Linux */
4941                 bitmap_zero(ud->rflow_in_use, ud->rflow_cnt);
4942                 irq_res.sets = 1;
4943         } else {
4944                 bitmap_fill(ud->rflow_in_use, ud->rflow_cnt);
4945                 for (i = 0; i < rm_res->sets; i++)
4946                         udma_mark_resource_ranges(ud, ud->rflow_in_use,
4947                                                   &rm_res->desc[i], "rflow");
4948                 irq_res.sets = rm_res->sets;
4949         }
4950
4951         /* tflow ranges */
4952         rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];
4953         if (IS_ERR(rm_res)) {
4954                 /* all tflows are assigned exclusively to Linux */
4955                 bitmap_zero(ud->tflow_map, ud->tflow_cnt);
4956                 irq_res.sets++;
4957         } else {
4958                 bitmap_fill(ud->tflow_map, ud->tflow_cnt);
4959                 for (i = 0; i < rm_res->sets; i++)
4960                         udma_mark_resource_ranges(ud, ud->tflow_map,
4961                                                   &rm_res->desc[i], "tflow");
4962                 irq_res.sets += rm_res->sets;
4963         }
4964
4965         irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL);
4966         if (!irq_res.desc)
4967                 return -ENOMEM;
4968         rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW];
4969         if (IS_ERR(rm_res)) {
4970                 irq_res.desc[0].start = oes->pktdma_tchan_flow;
4971                 irq_res.desc[0].num = ud->tflow_cnt;
4972                 i = 1;
4973         } else {
4974                 for (i = 0; i < rm_res->sets; i++) {
4975                         irq_res.desc[i].start = rm_res->desc[i].start +
4976                                                 oes->pktdma_tchan_flow;
4977                         irq_res.desc[i].num = rm_res->desc[i].num;
4978                 }
4979         }
4980         rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW];
4981         if (IS_ERR(rm_res)) {
4982                 irq_res.desc[i].start = oes->pktdma_rchan_flow;
4983                 irq_res.desc[i].num = ud->rflow_cnt;
4984         } else {
4985                 for (j = 0; j < rm_res->sets; j++, i++) {
4986                         irq_res.desc[i].start = rm_res->desc[j].start +
4987                                                 oes->pktdma_rchan_flow;
4988                         irq_res.desc[i].num = rm_res->desc[j].num;
4989                 }
4990         }
4991         ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res);
4992         kfree(irq_res.desc);
4993         if (ret) {
4994                 dev_err(ud->dev, "Failed to allocate MSI interrupts\n");
4995                 return ret;
4996         }
4997
4998         return 0;
4999 }
5000
5001 static int setup_resources(struct udma_dev *ud)
5002 {
5003         struct device *dev = ud->dev;
5004         int ch_count, ret;
5005
5006         switch (ud->match_data->type) {
5007         case DMA_TYPE_UDMA:
5008                 ret = udma_setup_resources(ud);
5009                 break;
5010         case DMA_TYPE_BCDMA:
5011                 ret = bcdma_setup_resources(ud);
5012                 break;
5013         case DMA_TYPE_PKTDMA:
5014                 ret = pktdma_setup_resources(ud);
5015                 break;
5016         default:
5017                 return -EINVAL;
5018         }
5019
5020         if (ret)
5021                 return ret;
5022
5023         ch_count  = ud->bchan_cnt + ud->tchan_cnt + ud->rchan_cnt;
5024         if (ud->bchan_cnt)
5025                 ch_count -= bitmap_weight(ud->bchan_map, ud->bchan_cnt);
5026         ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt);
5027         ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt);
5028         if (!ch_count)
5029                 return -ENODEV;
5030
5031         ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels),
5032                                     GFP_KERNEL);
5033         if (!ud->channels)
5034                 return -ENOMEM;
5035
5036         switch (ud->match_data->type) {
5037         case DMA_TYPE_UDMA:
5038                 dev_info(dev,
5039                          "Channels: %d (tchan: %u, rchan: %u, gp-rflow: %u)\n",
5040                          ch_count,
5041                          ud->tchan_cnt - bitmap_weight(ud->tchan_map,
5042                                                        ud->tchan_cnt),
5043                          ud->rchan_cnt - bitmap_weight(ud->rchan_map,
5044                                                        ud->rchan_cnt),
5045                          ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map,
5046                                                        ud->rflow_cnt));
5047                 break;
5048         case DMA_TYPE_BCDMA:
5049                 dev_info(dev,
5050                          "Channels: %d (bchan: %u, tchan: %u, rchan: %u)\n",
5051                          ch_count,
5052                          ud->bchan_cnt - bitmap_weight(ud->bchan_map,
5053                                                        ud->bchan_cnt),
5054                          ud->tchan_cnt - bitmap_weight(ud->tchan_map,
5055                                                        ud->tchan_cnt),
5056                          ud->rchan_cnt - bitmap_weight(ud->rchan_map,
5057                                                        ud->rchan_cnt));
5058                 break;
5059         case DMA_TYPE_PKTDMA:
5060                 dev_info(dev,
5061                          "Channels: %d (tchan: %u, rchan: %u)\n",
5062                          ch_count,
5063                          ud->tchan_cnt - bitmap_weight(ud->tchan_map,
5064                                                        ud->tchan_cnt),
5065                          ud->rchan_cnt - bitmap_weight(ud->rchan_map,
5066                                                        ud->rchan_cnt));
5067                 break;
5068         default:
5069                 break;
5070         }
5071
5072         return ch_count;
5073 }
5074
5075 static int udma_setup_rx_flush(struct udma_dev *ud)
5076 {
5077         struct udma_rx_flush *rx_flush = &ud->rx_flush;
5078         struct cppi5_desc_hdr_t *tr_desc;
5079         struct cppi5_tr_type1_t *tr_req;
5080         struct cppi5_host_desc_t *desc;
5081         struct device *dev = ud->dev;
5082         struct udma_hwdesc *hwdesc;
5083         size_t tr_size;
5084
5085         /* Allocate 1K buffer for discarded data on RX channel teardown */
5086         rx_flush->buffer_size = SZ_1K;
5087         rx_flush->buffer_vaddr = devm_kzalloc(dev, rx_flush->buffer_size,
5088                                               GFP_KERNEL);
5089         if (!rx_flush->buffer_vaddr)
5090                 return -ENOMEM;
5091
5092         rx_flush->buffer_paddr = dma_map_single(dev, rx_flush->buffer_vaddr,
5093                                                 rx_flush->buffer_size,
5094                                                 DMA_TO_DEVICE);
5095         if (dma_mapping_error(dev, rx_flush->buffer_paddr))
5096                 return -ENOMEM;
5097
5098         /* Set up descriptor to be used for TR mode */
5099         hwdesc = &rx_flush->hwdescs[0];
5100         tr_size = sizeof(struct cppi5_tr_type1_t);
5101         hwdesc->cppi5_desc_size = cppi5_trdesc_calc_size(tr_size, 1);
5102         hwdesc->cppi5_desc_size = ALIGN(hwdesc->cppi5_desc_size,
5103                                         ud->desc_align);
5104
5105         hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
5106                                                 GFP_KERNEL);
5107         if (!hwdesc->cppi5_desc_vaddr)
5108                 return -ENOMEM;
5109
5110         hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
5111                                                   hwdesc->cppi5_desc_size,
5112                                                   DMA_TO_DEVICE);
5113         if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
5114                 return -ENOMEM;
5115
5116         /* Start of the TR req records */
5117         hwdesc->tr_req_base = hwdesc->cppi5_desc_vaddr + tr_size;
5118         /* Start address of the TR response array */
5119         hwdesc->tr_resp_base = hwdesc->tr_req_base + tr_size;
5120
5121         tr_desc = hwdesc->cppi5_desc_vaddr;
5122         cppi5_trdesc_init(tr_desc, 1, tr_size, 0, 0);
5123         cppi5_desc_set_pktids(tr_desc, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);
5124         cppi5_desc_set_retpolicy(tr_desc, 0, 0);
5125
5126         tr_req = hwdesc->tr_req_base;
5127         cppi5_tr_init(&tr_req->flags, CPPI5_TR_TYPE1, false, false,
5128                       CPPI5_TR_EVENT_SIZE_COMPLETION, 0);
5129         cppi5_tr_csf_set(&tr_req->flags, CPPI5_TR_CSF_SUPR_EVT);
5130
5131         tr_req->addr = rx_flush->buffer_paddr;
5132         tr_req->icnt0 = rx_flush->buffer_size;
5133         tr_req->icnt1 = 1;
5134
5135         dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
5136                                    hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
5137
5138         /* Set up descriptor to be used for packet mode */
5139         hwdesc = &rx_flush->hwdescs[1];
5140         hwdesc->cppi5_desc_size = ALIGN(sizeof(struct cppi5_host_desc_t) +
5141                                         CPPI5_INFO0_HDESC_EPIB_SIZE +
5142                                         CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE,
5143                                         ud->desc_align);
5144
5145         hwdesc->cppi5_desc_vaddr = devm_kzalloc(dev, hwdesc->cppi5_desc_size,
5146                                                 GFP_KERNEL);
5147         if (!hwdesc->cppi5_desc_vaddr)
5148                 return -ENOMEM;
5149
5150         hwdesc->cppi5_desc_paddr = dma_map_single(dev, hwdesc->cppi5_desc_vaddr,
5151                                                   hwdesc->cppi5_desc_size,
5152                                                   DMA_TO_DEVICE);
5153         if (dma_mapping_error(dev, hwdesc->cppi5_desc_paddr))
5154                 return -ENOMEM;
5155
5156         desc = hwdesc->cppi5_desc_vaddr;
5157         cppi5_hdesc_init(desc, 0, 0);
5158         cppi5_desc_set_pktids(&desc->hdr, 0, CPPI5_INFO1_DESC_FLOWID_DEFAULT);
5159         cppi5_desc_set_retpolicy(&desc->hdr, 0, 0);
5160
5161         cppi5_hdesc_attach_buf(desc,
5162                                rx_flush->buffer_paddr, rx_flush->buffer_size,
5163                                rx_flush->buffer_paddr, rx_flush->buffer_size);
5164
5165         dma_sync_single_for_device(dev, hwdesc->cppi5_desc_paddr,
5166                                    hwdesc->cppi5_desc_size, DMA_TO_DEVICE);
5167         return 0;
5168 }
5169
5170 #ifdef CONFIG_DEBUG_FS
5171 static void udma_dbg_summary_show_chan(struct seq_file *s,
5172                                        struct dma_chan *chan)
5173 {
5174         struct udma_chan *uc = to_udma_chan(chan);
5175         struct udma_chan_config *ucc = &uc->config;
5176
5177         seq_printf(s, " %-13s| %s", dma_chan_name(chan),
5178                    chan->dbg_client_name ?: "in-use");
5179         if (ucc->tr_trigger_type)
5180                 seq_puts(s, " (triggered, ");
5181         else
5182                 seq_printf(s, " (%s, ",
5183                            dmaengine_get_direction_text(uc->config.dir));
5184
5185         switch (uc->config.dir) {
5186         case DMA_MEM_TO_MEM:
5187                 if (uc->ud->match_data->type == DMA_TYPE_BCDMA) {
5188                         seq_printf(s, "bchan%d)\n", uc->bchan->id);
5189                         return;
5190                 }
5191
5192                 seq_printf(s, "chan%d pair [0x%04x -> 0x%04x], ", uc->tchan->id,
5193                            ucc->src_thread, ucc->dst_thread);
5194                 break;
5195         case DMA_DEV_TO_MEM:
5196                 seq_printf(s, "rchan%d [0x%04x -> 0x%04x], ", uc->rchan->id,
5197                            ucc->src_thread, ucc->dst_thread);
5198                 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA)
5199                         seq_printf(s, "rflow%d, ", uc->rflow->id);
5200                 break;
5201         case DMA_MEM_TO_DEV:
5202                 seq_printf(s, "tchan%d [0x%04x -> 0x%04x], ", uc->tchan->id,
5203                            ucc->src_thread, ucc->dst_thread);
5204                 if (uc->ud->match_data->type == DMA_TYPE_PKTDMA)
5205                         seq_printf(s, "tflow%d, ", uc->tchan->tflow_id);
5206                 break;
5207         default:
5208                 seq_printf(s, ")\n");
5209                 return;
5210         }
5211
5212         if (ucc->ep_type == PSIL_EP_NATIVE) {
5213                 seq_printf(s, "PSI-L Native");
5214                 if (ucc->metadata_size) {
5215                         seq_printf(s, "[%s", ucc->needs_epib ? " EPIB" : "");
5216                         if (ucc->psd_size)
5217                                 seq_printf(s, " PSDsize:%u", ucc->psd_size);
5218                         seq_printf(s, " ]");
5219                 }
5220         } else {
5221                 seq_printf(s, "PDMA");
5222                 if (ucc->enable_acc32 || ucc->enable_burst)
5223                         seq_printf(s, "[%s%s ]",
5224                                    ucc->enable_acc32 ? " ACC32" : "",
5225                                    ucc->enable_burst ? " BURST" : "");
5226         }
5227
5228         seq_printf(s, ", %s)\n", ucc->pkt_mode ? "Packet mode" : "TR mode");
5229 }
5230
5231 static void udma_dbg_summary_show(struct seq_file *s,
5232                                   struct dma_device *dma_dev)
5233 {
5234         struct dma_chan *chan;
5235
5236         list_for_each_entry(chan, &dma_dev->channels, device_node) {
5237                 if (chan->client_count)
5238                         udma_dbg_summary_show_chan(s, chan);
5239         }
5240 }
5241 #endif /* CONFIG_DEBUG_FS */
5242
5243 static enum dmaengine_alignment udma_get_copy_align(struct udma_dev *ud)
5244 {
5245         const struct udma_match_data *match_data = ud->match_data;
5246         u8 tpl;
5247
5248         if (!match_data->enable_memcpy_support)
5249                 return DMAENGINE_ALIGN_8_BYTES;
5250
5251         /* Get the highest TPL level the device supports for memcpy */
5252         if (ud->bchan_cnt)
5253                 tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, 0);
5254         else if (ud->tchan_cnt)
5255                 tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, 0);
5256         else
5257                 return DMAENGINE_ALIGN_8_BYTES;
5258
5259         switch (match_data->burst_size[tpl]) {
5260         case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES:
5261                 return DMAENGINE_ALIGN_256_BYTES;
5262         case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES:
5263                 return DMAENGINE_ALIGN_128_BYTES;
5264         case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES:
5265         fallthrough;
5266         default:
5267                 return DMAENGINE_ALIGN_64_BYTES;
5268         }
5269 }
5270
5271 #define TI_UDMAC_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
5272                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
5273                                  BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
5274                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
5275                                  BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
5276
5277 static int udma_probe(struct platform_device *pdev)
5278 {
5279         struct device_node *navss_node = pdev->dev.parent->of_node;
5280         const struct soc_device_attribute *soc;
5281         struct device *dev = &pdev->dev;
5282         struct udma_dev *ud;
5283         const struct of_device_id *match;
5284         int i, ret;
5285         int ch_count;
5286
5287         ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(48));
5288         if (ret)
5289                 dev_err(dev, "failed to set dma mask stuff\n");
5290
5291         ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL);
5292         if (!ud)
5293                 return -ENOMEM;
5294
5295         match = of_match_node(udma_of_match, dev->of_node);
5296         if (!match) {
5297                 dev_err(dev, "No compatible match found\n");
5298                 return -ENODEV;
5299         }
5300         ud->match_data = match->data;
5301
5302         ud->soc_data = ud->match_data->soc_data;
5303         if (!ud->soc_data) {
5304                 soc = soc_device_match(k3_soc_devices);
5305                 if (!soc) {
5306                         dev_err(dev, "No compatible SoC found\n");
5307                         return -ENODEV;
5308                 }
5309                 ud->soc_data = soc->data;
5310         }
5311
5312         ret = udma_get_mmrs(pdev, ud);
5313         if (ret)
5314                 return ret;
5315
5316         ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci");
5317         if (IS_ERR(ud->tisci_rm.tisci))
5318                 return PTR_ERR(ud->tisci_rm.tisci);
5319
5320         ret = of_property_read_u32(dev->of_node, "ti,sci-dev-id",
5321                                    &ud->tisci_rm.tisci_dev_id);
5322         if (ret) {
5323                 dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
5324                 return ret;
5325         }
5326         pdev->id = ud->tisci_rm.tisci_dev_id;
5327
5328         ret = of_property_read_u32(navss_node, "ti,sci-dev-id",
5329                                    &ud->tisci_rm.tisci_navss_dev_id);
5330         if (ret) {
5331                 dev_err(dev, "NAVSS ti,sci-dev-id read failure %d\n", ret);
5332                 return ret;
5333         }
5334
5335         if (ud->match_data->type == DMA_TYPE_UDMA) {
5336                 ret = of_property_read_u32(dev->of_node, "ti,udma-atype",
5337                                            &ud->atype);
5338                 if (!ret && ud->atype > 2) {
5339                         dev_err(dev, "Invalid atype: %u\n", ud->atype);
5340                         return -EINVAL;
5341                 }
5342         } else {
5343                 ret = of_property_read_u32(dev->of_node, "ti,asel",
5344                                            &ud->asel);
5345                 if (!ret && ud->asel > 15) {
5346                         dev_err(dev, "Invalid asel: %u\n", ud->asel);
5347                         return -EINVAL;
5348                 }
5349         }
5350
5351         ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops;
5352         ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops;
5353
5354         if (ud->match_data->type == DMA_TYPE_UDMA) {
5355                 ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc");
5356         } else {
5357                 struct k3_ringacc_init_data ring_init_data;
5358
5359                 ring_init_data.tisci = ud->tisci_rm.tisci;
5360                 ring_init_data.tisci_dev_id = ud->tisci_rm.tisci_dev_id;
5361                 if (ud->match_data->type == DMA_TYPE_BCDMA) {
5362                         ring_init_data.num_rings = ud->bchan_cnt +
5363                                                    ud->tchan_cnt +
5364                                                    ud->rchan_cnt;
5365                 } else {
5366                         ring_init_data.num_rings = ud->rflow_cnt +
5367                                                    ud->tflow_cnt;
5368                 }
5369
5370                 ud->ringacc = k3_ringacc_dmarings_init(pdev, &ring_init_data);
5371         }
5372
5373         if (IS_ERR(ud->ringacc))
5374                 return PTR_ERR(ud->ringacc);
5375
5376         dev->msi.domain = of_msi_get_domain(dev, dev->of_node,
5377                                             DOMAIN_BUS_TI_SCI_INTA_MSI);
5378         if (!dev->msi.domain) {
5379                 return -EPROBE_DEFER;
5380         }
5381
5382         dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask);
5383         /* cyclic operation is not supported via PKTDMA */
5384         if (ud->match_data->type != DMA_TYPE_PKTDMA) {
5385                 dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask);
5386                 ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic;
5387         }
5388
5389         ud->ddev.device_config = udma_slave_config;
5390         ud->ddev.device_prep_slave_sg = udma_prep_slave_sg;
5391         ud->ddev.device_issue_pending = udma_issue_pending;
5392         ud->ddev.device_tx_status = udma_tx_status;
5393         ud->ddev.device_pause = udma_pause;
5394         ud->ddev.device_resume = udma_resume;
5395         ud->ddev.device_terminate_all = udma_terminate_all;
5396         ud->ddev.device_synchronize = udma_synchronize;
5397 #ifdef CONFIG_DEBUG_FS
5398         ud->ddev.dbg_summary_show = udma_dbg_summary_show;
5399 #endif
5400
5401         switch (ud->match_data->type) {
5402         case DMA_TYPE_UDMA:
5403                 ud->ddev.device_alloc_chan_resources =
5404                                         udma_alloc_chan_resources;
5405                 break;
5406         case DMA_TYPE_BCDMA:
5407                 ud->ddev.device_alloc_chan_resources =
5408                                         bcdma_alloc_chan_resources;
5409                 ud->ddev.device_router_config = bcdma_router_config;
5410                 break;
5411         case DMA_TYPE_PKTDMA:
5412                 ud->ddev.device_alloc_chan_resources =
5413                                         pktdma_alloc_chan_resources;
5414                 break;
5415         default:
5416                 return -EINVAL;
5417         }
5418         ud->ddev.device_free_chan_resources = udma_free_chan_resources;
5419
5420         ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS;
5421         ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS;
5422         ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
5423         ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
5424         ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT |
5425                                        DESC_METADATA_ENGINE;
5426         if (ud->match_data->enable_memcpy_support &&
5427             !(ud->match_data->type == DMA_TYPE_BCDMA && ud->bchan_cnt == 0)) {
5428                 dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask);
5429                 ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy;
5430                 ud->ddev.directions |= BIT(DMA_MEM_TO_MEM);
5431         }
5432
5433         ud->ddev.dev = dev;
5434         ud->dev = dev;
5435         ud->psil_base = ud->match_data->psil_base;
5436
5437         INIT_LIST_HEAD(&ud->ddev.channels);
5438         INIT_LIST_HEAD(&ud->desc_to_purge);
5439
5440         ch_count = setup_resources(ud);
5441         if (ch_count <= 0)
5442                 return ch_count;
5443
5444         spin_lock_init(&ud->lock);
5445         INIT_WORK(&ud->purge_work, udma_purge_desc_work);
5446
5447         ud->desc_align = 64;
5448         if (ud->desc_align < dma_get_cache_alignment())
5449                 ud->desc_align = dma_get_cache_alignment();
5450
5451         ret = udma_setup_rx_flush(ud);
5452         if (ret)
5453                 return ret;
5454
5455         for (i = 0; i < ud->bchan_cnt; i++) {
5456                 struct udma_bchan *bchan = &ud->bchans[i];
5457
5458                 bchan->id = i;
5459                 bchan->reg_rt = ud->mmrs[MMR_BCHANRT] + i * 0x1000;
5460         }
5461
5462         for (i = 0; i < ud->tchan_cnt; i++) {
5463                 struct udma_tchan *tchan = &ud->tchans[i];
5464
5465                 tchan->id = i;
5466                 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000;
5467         }
5468
5469         for (i = 0; i < ud->rchan_cnt; i++) {
5470                 struct udma_rchan *rchan = &ud->rchans[i];
5471
5472                 rchan->id = i;
5473                 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000;
5474         }
5475
5476         for (i = 0; i < ud->rflow_cnt; i++) {
5477                 struct udma_rflow *rflow = &ud->rflows[i];
5478
5479                 rflow->id = i;
5480         }
5481
5482         for (i = 0; i < ch_count; i++) {
5483                 struct udma_chan *uc = &ud->channels[i];
5484
5485                 uc->ud = ud;
5486                 uc->vc.desc_free = udma_desc_free;
5487                 uc->id = i;
5488                 uc->bchan = NULL;
5489                 uc->tchan = NULL;
5490                 uc->rchan = NULL;
5491                 uc->config.remote_thread_id = -1;
5492                 uc->config.mapped_channel_id = -1;
5493                 uc->config.default_flow_id = -1;
5494                 uc->config.dir = DMA_MEM_TO_MEM;
5495                 uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d",
5496                                           dev_name(dev), i);
5497
5498                 vchan_init(&uc->vc, &ud->ddev);
5499                 /* Use custom vchan completion handling */
5500                 tasklet_setup(&uc->vc.task, udma_vchan_complete);
5501                 init_completion(&uc->teardown_completed);
5502                 INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion);
5503         }
5504
5505         /* Configure the copy_align to the maximum burst size the device supports */
5506         ud->ddev.copy_align = udma_get_copy_align(ud);
5507
5508         ret = dma_async_device_register(&ud->ddev);
5509         if (ret) {
5510                 dev_err(dev, "failed to register slave DMA engine: %d\n", ret);
5511                 return ret;
5512         }
5513
5514         platform_set_drvdata(pdev, ud);
5515
5516         ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud);
5517         if (ret) {
5518                 dev_err(dev, "failed to register of_dma controller\n");
5519                 dma_async_device_unregister(&ud->ddev);
5520         }
5521
5522         return ret;
5523 }
5524
5525 static struct platform_driver udma_driver = {
5526         .driver = {
5527                 .name   = "ti-udma",
5528                 .of_match_table = udma_of_match,
5529                 .suppress_bind_attrs = true,
5530         },
5531         .probe          = udma_probe,
5532 };
5533
5534 module_platform_driver(udma_driver);
5535 MODULE_LICENSE("GPL v2");
5536
5537 /* Private interfaces to UDMA */
5538 #include "k3-udma-private.c"