drivers/dma/k3dma.c

   1 /*
   2  * Copyright (c) 2013 - 2015 Linaro Ltd.
   3  * Copyright (c) 2013 Hisilicon Limited.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  */
   9 #include <linux/sched.h>
  10 #include <linux/device.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/dmapool.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/init.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/kernel.h>
  17 #include <linux/module.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/slab.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/of_device.h>
  22 #include <linux/of.h>
  23 #include <linux/clk.h>
  24 #include <linux/of_dma.h>
  25
  26 #include "virt-dma.h"
  27
  28 #define DRIVER_NAME             "k3-dma"
  29 #define DMA_MAX_SIZE            0x1ffc
  30 #define DMA_CYCLIC_MAX_PERIOD   0x1000
  31 #define LLI_BLOCK_SIZE          (4 * PAGE_SIZE)
  32
  33 #define INT_STAT                0x00
  34 #define INT_TC1                 0x04
  35 #define INT_TC2                 0x08
  36 #define INT_ERR1                0x0c
  37 #define INT_ERR2                0x10
  38 #define INT_TC1_MASK            0x18
  39 #define INT_TC2_MASK            0x1c
  40 #define INT_ERR1_MASK           0x20
  41 #define INT_ERR2_MASK           0x24
  42 #define INT_TC1_RAW             0x600
  43 #define INT_TC2_RAW             0x608
  44 #define INT_ERR1_RAW            0x610
  45 #define INT_ERR2_RAW            0x618
  46 #define CH_PRI                  0x688
  47 #define CH_STAT                 0x690
  48 #define CX_CUR_CNT              0x704
  49 #define CX_LLI                  0x800
  50 #define CX_CNT1                 0x80c
  51 #define CX_CNT0                 0x810
  52 #define CX_SRC                  0x814
  53 #define CX_DST                  0x818
  54 #define CX_CFG                  0x81c
  55
  56 #define CX_LLI_CHAIN_EN         0x2
  57 #define CX_CFG_EN               0x1
  58 #define CX_CFG_NODEIRQ          BIT(1)
  59 #define CX_CFG_MEM2PER          (0x1 << 2)
  60 #define CX_CFG_PER2MEM          (0x2 << 2)
  61 #define CX_CFG_SRCINCR          (0x1 << 31)
  62 #define CX_CFG_DSTINCR          (0x1 << 30)
  63
  64 struct k3_desc_hw {
  65         u32 lli;
  66         u32 reserved[3];
  67         u32 count;
  68         u32 saddr;
  69         u32 daddr;
  70         u32 config;
  71 } __aligned(32);
  72
  73 struct k3_dma_desc_sw {
  74         struct virt_dma_desc    vd;
  75         dma_addr_t              desc_hw_lli;
  76         size_t                  desc_num;
  77         size_t                  size;
  78         struct k3_desc_hw       *desc_hw;
  79 };
  80
  81 struct k3_dma_phy;
  82
  83 struct k3_dma_chan {
  84         u32                     ccfg;
  85         struct virt_dma_chan    vc;
  86         struct k3_dma_phy       *phy;
  87         struct list_head        node;
  88         dma_addr_t              dev_addr;
  89         enum dma_status         status;
  90         bool                    cyclic;
  91         struct dma_slave_config slave_config;
  92 };
  93
  94 struct k3_dma_phy {
  95         u32                     idx;
  96         void __iomem            *base;
  97         struct k3_dma_chan      *vchan;
  98         struct k3_dma_desc_sw   *ds_run;
  99         struct k3_dma_desc_sw   *ds_done;
 100 };
 101
 102 struct k3_dma_dev {
 103         struct dma_device       slave;
 104         void __iomem            *base;
 105         struct tasklet_struct   task;
 106         spinlock_t              lock;
 107         struct list_head        chan_pending;
 108         struct k3_dma_phy       *phy;
 109         struct k3_dma_chan      *chans;
 110         struct clk              *clk;
 111         struct dma_pool         *pool;
 112         u32                     dma_channels;
 113         u32                     dma_requests;
 114         u32                     dma_channel_mask;
 115         unsigned int            irq;
 116 };
 117
 118
 119 #define K3_FLAG_NOCLK   BIT(1)
 120
 121 struct k3dma_soc_data {
 122         unsigned long flags;
 123 };
 124
 125
 126 #define to_k3_dma(dmadev) container_of(dmadev, struct k3_dma_dev, slave)
 127
 128 static int k3_dma_config_write(struct dma_chan *chan,
 129                                enum dma_transfer_direction dir,
 130                                struct dma_slave_config *cfg);
 131
 132 static struct k3_dma_chan *to_k3_chan(struct dma_chan *chan)
 133 {
 134         return container_of(chan, struct k3_dma_chan, vc.chan);
 135 }
 136
 137 static void k3_dma_pause_dma(struct k3_dma_phy *phy, bool on)
 138 {
 139         u32 val = 0;
 140
 141         if (on) {
 142                 val = readl_relaxed(phy->base + CX_CFG);
 143                 val |= CX_CFG_EN;
 144                 writel_relaxed(val, phy->base + CX_CFG);
 145         } else {
 146                 val = readl_relaxed(phy->base + CX_CFG);
 147                 val &= ~CX_CFG_EN;
 148                 writel_relaxed(val, phy->base + CX_CFG);
 149         }
 150 }
 151
 152 static void k3_dma_terminate_chan(struct k3_dma_phy *phy, struct k3_dma_dev *d)
 153 {
 154         u32 val = 0;
 155
 156         k3_dma_pause_dma(phy, false);
 157
 158         val = 0x1 << phy->idx;
 159         writel_relaxed(val, d->base + INT_TC1_RAW);
 160         writel_relaxed(val, d->base + INT_TC2_RAW);
 161         writel_relaxed(val, d->base + INT_ERR1_RAW);
 162         writel_relaxed(val, d->base + INT_ERR2_RAW);
 163 }
 164
 165 static void k3_dma_set_desc(struct k3_dma_phy *phy, struct k3_desc_hw *hw)
 166 {
 167         writel_relaxed(hw->lli, phy->base + CX_LLI);
 168         writel_relaxed(hw->count, phy->base + CX_CNT0);
 169         writel_relaxed(hw->saddr, phy->base + CX_SRC);
 170         writel_relaxed(hw->daddr, phy->base + CX_DST);
 171         writel_relaxed(hw->config, phy->base + CX_CFG);
 172 }
 173
 174 static u32 k3_dma_get_curr_cnt(struct k3_dma_dev *d, struct k3_dma_phy *phy)
 175 {
 176         u32 cnt = 0;
 177
 178         cnt = readl_relaxed(d->base + CX_CUR_CNT + phy->idx * 0x10);
 179         cnt &= 0xffff;
 180         return cnt;
 181 }
 182
 183 static u32 k3_dma_get_curr_lli(struct k3_dma_phy *phy)
 184 {
 185         return readl_relaxed(phy->base + CX_LLI);
 186 }
 187
 188 static u32 k3_dma_get_chan_stat(struct k3_dma_dev *d)
 189 {
 190         return readl_relaxed(d->base + CH_STAT);
 191 }
 192
 193 static void k3_dma_enable_dma(struct k3_dma_dev *d, bool on)
 194 {
 195         if (on) {
 196                 /* set same priority */
 197                 writel_relaxed(0x0, d->base + CH_PRI);
 198
 199                 /* unmask irq */
 200                 writel_relaxed(0xffff, d->base + INT_TC1_MASK);
 201                 writel_relaxed(0xffff, d->base + INT_TC2_MASK);
 202                 writel_relaxed(0xffff, d->base + INT_ERR1_MASK);
 203                 writel_relaxed(0xffff, d->base + INT_ERR2_MASK);
 204         } else {
 205                 /* mask irq */
 206                 writel_relaxed(0x0, d->base + INT_TC1_MASK);
 207                 writel_relaxed(0x0, d->base + INT_TC2_MASK);
 208                 writel_relaxed(0x0, d->base + INT_ERR1_MASK);
 209                 writel_relaxed(0x0, d->base + INT_ERR2_MASK);
 210         }
 211 }
 212
 213 static irqreturn_t k3_dma_int_handler(int irq, void *dev_id)
 214 {
 215         struct k3_dma_dev *d = (struct k3_dma_dev *)dev_id;
 216         struct k3_dma_phy *p;
 217         struct k3_dma_chan *c;
 218         u32 stat = readl_relaxed(d->base + INT_STAT);
 219         u32 tc1  = readl_relaxed(d->base + INT_TC1);
 220         u32 tc2  = readl_relaxed(d->base + INT_TC2);
 221         u32 err1 = readl_relaxed(d->base + INT_ERR1);
 222         u32 err2 = readl_relaxed(d->base + INT_ERR2);
 223         u32 i, irq_chan = 0;
 224
 225         while (stat) {
 226                 i = __ffs(stat);
 227                 stat &= ~BIT(i);
 228                 if (likely(tc1 & BIT(i)) || (tc2 & BIT(i))) {
 229                         unsigned long flags;
 230
 231                         p = &d->phy[i];
 232                         c = p->vchan;
 233                         if (c && (tc1 & BIT(i))) {
 234                                 spin_lock_irqsave(&c->vc.lock, flags);
 235                                 vchan_cookie_complete(&p->ds_run->vd);
 236                                 p->ds_done = p->ds_run;
 237                                 p->ds_run = NULL;
 238                                 spin_unlock_irqrestore(&c->vc.lock, flags);
 239                         }
 240                         if (c && (tc2 & BIT(i))) {
 241                                 spin_lock_irqsave(&c->vc.lock, flags);
 242                                 if (p->ds_run != NULL)
 243                                         vchan_cyclic_callback(&p->ds_run->vd);
 244                                 spin_unlock_irqrestore(&c->vc.lock, flags);
 245                         }
 246                         irq_chan |= BIT(i);
 247                 }
 248                 if (unlikely((err1 & BIT(i)) || (err2 & BIT(i))))
 249                         dev_warn(d->slave.dev, "DMA ERR\n");
 250         }
 251
 252         writel_relaxed(irq_chan, d->base + INT_TC1_RAW);
 253         writel_relaxed(irq_chan, d->base + INT_TC2_RAW);
 254         writel_relaxed(err1, d->base + INT_ERR1_RAW);
 255         writel_relaxed(err2, d->base + INT_ERR2_RAW);
 256
 257         if (irq_chan)
 258                 tasklet_schedule(&d->task);
 259
 260         if (irq_chan || err1 || err2)
 261                 return IRQ_HANDLED;
 262
 263         return IRQ_NONE;
 264 }
 265
 266 static int k3_dma_start_txd(struct k3_dma_chan *c)
 267 {
 268         struct k3_dma_dev *d = to_k3_dma(c->vc.chan.device);
 269         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 270
 271         if (!c->phy)
 272                 return -EAGAIN;
 273
 274         if (BIT(c->phy->idx) & k3_dma_get_chan_stat(d))
 275                 return -EAGAIN;
 276
 277         if (vd) {
 278                 struct k3_dma_desc_sw *ds =
 279                         container_of(vd, struct k3_dma_desc_sw, vd);
 280                 /*
 281                  * fetch and remove request from vc->desc_issued
 282                  * so vc->desc_issued only contains desc pending
 283                  */
 284                 list_del(&ds->vd.node);
 285
 286                 c->phy->ds_run = ds;
 287                 c->phy->ds_done = NULL;
 288                 /* start dma */
 289                 k3_dma_set_desc(c->phy, &ds->desc_hw[0]);
 290                 return 0;
 291         }
 292         c->phy->ds_run = NULL;
 293         c->phy->ds_done = NULL;
 294         return -EAGAIN;
 295 }
 296
 297 static void k3_dma_tasklet(unsigned long arg)
 298 {
 299         struct k3_dma_dev *d = (struct k3_dma_dev *)arg;
 300         struct k3_dma_phy *p;
 301         struct k3_dma_chan *c, *cn;
 302         unsigned pch, pch_alloc = 0;
 303
 304         /* check new dma request of running channel in vc->desc_issued */
 305         list_for_each_entry_safe(c, cn, &d->slave.channels, vc.chan.device_node) {
 306                 spin_lock_irq(&c->vc.lock);
 307                 p = c->phy;
 308                 if (p && p->ds_done) {
 309                         if (k3_dma_start_txd(c)) {
 310                                 /* No current txd associated with this channel */
 311                                 dev_dbg(d->slave.dev, "pchan %u: free\n", p->idx);
 312                                 /* Mark this channel free */
 313                                 c->phy = NULL;
 314                                 p->vchan = NULL;
 315                         }
 316                 }
 317                 spin_unlock_irq(&c->vc.lock);
 318         }
 319
 320         /* check new channel request in d->chan_pending */
 321         spin_lock_irq(&d->lock);
 322         for (pch = 0; pch < d->dma_channels; pch++) {
 323                 if (!(d->dma_channel_mask & (1 << pch)))
 324                         continue;
 325
 326                 p = &d->phy[pch];
 327
 328                 if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
 329                         c = list_first_entry(&d->chan_pending,
 330                                 struct k3_dma_chan, node);
 331                         /* remove from d->chan_pending */
 332                         list_del_init(&c->node);
 333                         pch_alloc |= 1 << pch;
 334                         /* Mark this channel allocated */
 335                         p->vchan = c;
 336                         c->phy = p;
 337                         dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);
 338                 }
 339         }
 340         spin_unlock_irq(&d->lock);
 341
 342         for (pch = 0; pch < d->dma_channels; pch++) {
 343                 if (!(d->dma_channel_mask & (1 << pch)))
 344                         continue;
 345
 346                 if (pch_alloc & (1 << pch)) {
 347                         p = &d->phy[pch];
 348                         c = p->vchan;
 349                         if (c) {
 350                                 spin_lock_irq(&c->vc.lock);
 351                                 k3_dma_start_txd(c);
 352                                 spin_unlock_irq(&c->vc.lock);
 353                         }
 354                 }
 355         }
 356 }
 357
 358 static void k3_dma_free_chan_resources(struct dma_chan *chan)
 359 {
 360         struct k3_dma_chan *c = to_k3_chan(chan);
 361         struct k3_dma_dev *d = to_k3_dma(chan->device);
 362         unsigned long flags;
 363
 364         spin_lock_irqsave(&d->lock, flags);
 365         list_del_init(&c->node);
 366         spin_unlock_irqrestore(&d->lock, flags);
 367
 368         vchan_free_chan_resources(&c->vc);
 369         c->ccfg = 0;
 370 }
 371
 372 static enum dma_status k3_dma_tx_status(struct dma_chan *chan,
 373         dma_cookie_t cookie, struct dma_tx_state *state)
 374 {
 375         struct k3_dma_chan *c = to_k3_chan(chan);
 376         struct k3_dma_dev *d = to_k3_dma(chan->device);
 377         struct k3_dma_phy *p;
 378         struct virt_dma_desc *vd;
 379         unsigned long flags;
 380         enum dma_status ret;
 381         size_t bytes = 0;
 382
 383         ret = dma_cookie_status(&c->vc.chan, cookie, state);
 384         if (ret == DMA_COMPLETE)
 385                 return ret;
 386
 387         spin_lock_irqsave(&c->vc.lock, flags);
 388         p = c->phy;
 389         ret = c->status;
 390
 391         /*
 392          * If the cookie is on our issue queue, then the residue is
 393          * its total size.
 394          */
 395         vd = vchan_find_desc(&c->vc, cookie);
 396         if (vd && !c->cyclic) {
 397                 bytes = container_of(vd, struct k3_dma_desc_sw, vd)->size;
 398         } else if ((!p) || (!p->ds_run)) {
 399                 bytes = 0;
 400         } else {
 401                 struct k3_dma_desc_sw *ds = p->ds_run;
 402                 u32 clli = 0, index = 0;
 403
 404                 bytes = k3_dma_get_curr_cnt(d, p);
 405                 clli = k3_dma_get_curr_lli(p);
 406                 index = ((clli - ds->desc_hw_lli) /
 407                                 sizeof(struct k3_desc_hw)) + 1;
 408                 for (; index < ds->desc_num; index++) {
 409                         bytes += ds->desc_hw[index].count;
 410                         /* end of lli */
 411                         if (!ds->desc_hw[index].lli)
 412                                 break;
 413                 }
 414         }
 415         spin_unlock_irqrestore(&c->vc.lock, flags);
 416         dma_set_residue(state, bytes);
 417         return ret;
 418 }
 419
 420 static void k3_dma_issue_pending(struct dma_chan *chan)
 421 {
 422         struct k3_dma_chan *c = to_k3_chan(chan);
 423         struct k3_dma_dev *d = to_k3_dma(chan->device);
 424         unsigned long flags;
 425
 426         spin_lock_irqsave(&c->vc.lock, flags);
 427         /* add request to vc->desc_issued */
 428         if (vchan_issue_pending(&c->vc)) {
 429                 spin_lock(&d->lock);
 430                 if (!c->phy) {
 431                         if (list_empty(&c->node)) {
 432                                 /* if new channel, add chan_pending */
 433                                 list_add_tail(&c->node, &d->chan_pending);
 434                                 /* check in tasklet */
 435                                 tasklet_schedule(&d->task);
 436                                 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
 437                         }
 438                 }
 439                 spin_unlock(&d->lock);
 440         } else
 441                 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
 442         spin_unlock_irqrestore(&c->vc.lock, flags);
 443 }
 444
 445 static void k3_dma_fill_desc(struct k3_dma_desc_sw *ds, dma_addr_t dst,
 446                         dma_addr_t src, size_t len, u32 num, u32 ccfg)
 447 {
 448         if (num != ds->desc_num - 1)
 449                 ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) *
 450                         sizeof(struct k3_desc_hw);
 451
 452         ds->desc_hw[num].lli |= CX_LLI_CHAIN_EN;
 453         ds->desc_hw[num].count = len;
 454         ds->desc_hw[num].saddr = src;
 455         ds->desc_hw[num].daddr = dst;
 456         ds->desc_hw[num].config = ccfg;
 457 }
 458
 459 static struct k3_dma_desc_sw *k3_dma_alloc_desc_resource(int num,
 460                                                         struct dma_chan *chan)
 461 {
 462         struct k3_dma_chan *c = to_k3_chan(chan);
 463         struct k3_dma_desc_sw *ds;
 464         struct k3_dma_dev *d = to_k3_dma(chan->device);
 465         int lli_limit = LLI_BLOCK_SIZE / sizeof(struct k3_desc_hw);
 466
 467         if (num > lli_limit) {
 468                 dev_dbg(chan->device->dev, "vch %p: sg num %d exceed max %d\n",
 469                         &c->vc, num, lli_limit);
 470                 return NULL;
 471         }
 472
 473         ds = kzalloc(sizeof(*ds), GFP_NOWAIT);
 474         if (!ds)
 475                 return NULL;
 476
 477         ds->desc_hw = dma_pool_zalloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli);
 478         if (!ds->desc_hw) {
 479                 dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc);
 480                 kfree(ds);
 481                 return NULL;
 482         }
 483         ds->desc_num = num;
 484         return ds;
 485 }
 486
 487 static struct dma_async_tx_descriptor *k3_dma_prep_memcpy(
 488         struct dma_chan *chan,  dma_addr_t dst, dma_addr_t src,
 489         size_t len, unsigned long flags)
 490 {
 491         struct k3_dma_chan *c = to_k3_chan(chan);
 492         struct k3_dma_desc_sw *ds;
 493         size_t copy = 0;
 494         int num = 0;
 495
 496         if (!len)
 497                 return NULL;
 498
 499         num = DIV_ROUND_UP(len, DMA_MAX_SIZE);
 500
 501         ds = k3_dma_alloc_desc_resource(num, chan);
 502         if (!ds)
 503                 return NULL;
 504
 505         c->cyclic = 0;
 506         ds->size = len;
 507         num = 0;
 508
 509         if (!c->ccfg) {
 510                 /* default is memtomem, without calling device_config */
 511                 c->ccfg = CX_CFG_SRCINCR | CX_CFG_DSTINCR | CX_CFG_EN;
 512                 c->ccfg |= (0xf << 20) | (0xf << 24);   /* burst = 16 */
 513                 c->ccfg |= (0x3 << 12) | (0x3 << 16);   /* width = 64 bit */
 514         }
 515
 516         do {
 517                 copy = min_t(size_t, len, DMA_MAX_SIZE);
 518                 k3_dma_fill_desc(ds, dst, src, copy, num++, c->ccfg);
 519
 520                 src += copy;
 521                 dst += copy;
 522                 len -= copy;
 523         } while (len);
 524
 525         ds->desc_hw[num-1].lli = 0;     /* end of link */
 526         return vchan_tx_prep(&c->vc, &ds->vd, flags);
 527 }
 528
 529 static struct dma_async_tx_descriptor *k3_dma_prep_slave_sg(
 530         struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen,
 531         enum dma_transfer_direction dir, unsigned long flags, void *context)
 532 {
 533         struct k3_dma_chan *c = to_k3_chan(chan);
 534         struct k3_dma_desc_sw *ds;
 535         size_t len, avail, total = 0;
 536         struct scatterlist *sg;
 537         dma_addr_t addr, src = 0, dst = 0;
 538         int num = sglen, i;
 539
 540         if (sgl == NULL)
 541                 return NULL;
 542
 543         c->cyclic = 0;
 544
 545         for_each_sg(sgl, sg, sglen, i) {
 546                 avail = sg_dma_len(sg);
 547                 if (avail > DMA_MAX_SIZE)
 548                         num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1;
 549         }
 550
 551         ds = k3_dma_alloc_desc_resource(num, chan);
 552         if (!ds)
 553                 return NULL;
 554         num = 0;
 555         k3_dma_config_write(chan, dir, &c->slave_config);
 556
 557         for_each_sg(sgl, sg, sglen, i) {
 558                 addr = sg_dma_address(sg);
 559                 avail = sg_dma_len(sg);
 560                 total += avail;
 561
 562                 do {
 563                         len = min_t(size_t, avail, DMA_MAX_SIZE);
 564
 565                         if (dir == DMA_MEM_TO_DEV) {
 566                                 src = addr;
 567                                 dst = c->dev_addr;
 568                         } else if (dir == DMA_DEV_TO_MEM) {
 569                                 src = c->dev_addr;
 570                                 dst = addr;
 571                         }
 572
 573                         k3_dma_fill_desc(ds, dst, src, len, num++, c->ccfg);
 574
 575                         addr += len;
 576                         avail -= len;
 577                 } while (avail);
 578         }
 579
 580         ds->desc_hw[num-1].lli = 0;     /* end of link */
 581         ds->size = total;
 582         return vchan_tx_prep(&c->vc, &ds->vd, flags);
 583 }
 584
 585 static struct dma_async_tx_descriptor *
 586 k3_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
 587                        size_t buf_len, size_t period_len,
 588                        enum dma_transfer_direction dir,
 589                        unsigned long flags)
 590 {
 591         struct k3_dma_chan *c = to_k3_chan(chan);
 592         struct k3_dma_desc_sw *ds;
 593         size_t len, avail, total = 0;
 594         dma_addr_t addr, src = 0, dst = 0;
 595         int num = 1, since = 0;
 596         size_t modulo = DMA_CYCLIC_MAX_PERIOD;
 597         u32 en_tc2 = 0;
 598
 599         dev_dbg(chan->device->dev, "%s: buf %pad, dst %pad, buf len %zu, period_len = %zu, dir %d\n",
 600                __func__, &buf_addr, &to_k3_chan(chan)->dev_addr,
 601                buf_len, period_len, (int)dir);
 602
 603         avail = buf_len;
 604         if (avail > modulo)
 605                 num += DIV_ROUND_UP(avail, modulo) - 1;
 606
 607         ds = k3_dma_alloc_desc_resource(num, chan);
 608         if (!ds)
 609                 return NULL;
 610
 611         c->cyclic = 1;
 612         addr = buf_addr;
 613         avail = buf_len;
 614         total = avail;
 615         num = 0;
 616         k3_dma_config_write(chan, dir, &c->slave_config);
 617
 618         if (period_len < modulo)
 619                 modulo = period_len;
 620
 621         do {
 622                 len = min_t(size_t, avail, modulo);
 623
 624                 if (dir == DMA_MEM_TO_DEV) {
 625                         src = addr;
 626                         dst = c->dev_addr;
 627                 } else if (dir == DMA_DEV_TO_MEM) {
 628                         src = c->dev_addr;
 629                         dst = addr;
 630                 }
 631                 since += len;
 632                 if (since >= period_len) {
 633                         /* descriptor asks for TC2 interrupt on completion */
 634                         en_tc2 = CX_CFG_NODEIRQ;
 635                         since -= period_len;
 636                 } else
 637                         en_tc2 = 0;
 638
 639                 k3_dma_fill_desc(ds, dst, src, len, num++, c->ccfg | en_tc2);
 640
 641                 addr += len;
 642                 avail -= len;
 643         } while (avail);
 644
 645         /* "Cyclic" == end of link points back to start of link */
 646         ds->desc_hw[num - 1].lli |= ds->desc_hw_lli;
 647
 648         ds->size = total;
 649
 650         return vchan_tx_prep(&c->vc, &ds->vd, flags);
 651 }
 652
 653 static int k3_dma_config(struct dma_chan *chan,
 654                          struct dma_slave_config *cfg)
 655 {
 656         struct k3_dma_chan *c = to_k3_chan(chan);
 657
 658         memcpy(&c->slave_config, cfg, sizeof(*cfg));
 659
 660         return 0;
 661 }
 662
 663 static int k3_dma_config_write(struct dma_chan *chan,
 664                                enum dma_transfer_direction dir,
 665                                struct dma_slave_config *cfg)
 666 {
 667         struct k3_dma_chan *c = to_k3_chan(chan);
 668         u32 maxburst = 0, val = 0;
 669         enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED;
 670
 671         if (dir == DMA_DEV_TO_MEM) {
 672                 c->ccfg = CX_CFG_DSTINCR;
 673                 c->dev_addr = cfg->src_addr;
 674                 maxburst = cfg->src_maxburst;
 675                 width = cfg->src_addr_width;
 676         } else if (dir == DMA_MEM_TO_DEV) {
 677                 c->ccfg = CX_CFG_SRCINCR;
 678                 c->dev_addr = cfg->dst_addr;
 679                 maxburst = cfg->dst_maxburst;
 680                 width = cfg->dst_addr_width;
 681         }
 682         switch (width) {
 683         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 684         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 685         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 686         case DMA_SLAVE_BUSWIDTH_8_BYTES:
 687                 val =  __ffs(width);
 688                 break;
 689         default:
 690                 val = 3;
 691                 break;
 692         }
 693         c->ccfg |= (val << 12) | (val << 16);
 694
 695         if ((maxburst == 0) || (maxburst > 16))
 696                 val = 15;
 697         else
 698                 val = maxburst - 1;
 699         c->ccfg |= (val << 20) | (val << 24);
 700         c->ccfg |= CX_CFG_MEM2PER | CX_CFG_EN;
 701
 702         /* specific request line */
 703         c->ccfg |= c->vc.chan.chan_id << 4;
 704
 705         return 0;
 706 }
 707
 708 static void k3_dma_free_desc(struct virt_dma_desc *vd)
 709 {
 710         struct k3_dma_desc_sw *ds =
 711                 container_of(vd, struct k3_dma_desc_sw, vd);
 712         struct k3_dma_dev *d = to_k3_dma(vd->tx.chan->device);
 713
 714         dma_pool_free(d->pool, ds->desc_hw, ds->desc_hw_lli);
 715         kfree(ds);
 716 }
 717
 718 static int k3_dma_terminate_all(struct dma_chan *chan)
 719 {
 720         struct k3_dma_chan *c = to_k3_chan(chan);
 721         struct k3_dma_dev *d = to_k3_dma(chan->device);
 722         struct k3_dma_phy *p = c->phy;
 723         unsigned long flags;
 724         LIST_HEAD(head);
 725
 726         dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
 727
 728         /* Prevent this channel being scheduled */
 729         spin_lock(&d->lock);
 730         list_del_init(&c->node);
 731         spin_unlock(&d->lock);
 732
 733         /* Clear the tx descriptor lists */
 734         spin_lock_irqsave(&c->vc.lock, flags);
 735         vchan_get_all_descriptors(&c->vc, &head);
 736         if (p) {
 737                 /* vchan is assigned to a pchan - stop the channel */
 738                 k3_dma_terminate_chan(p, d);
 739                 c->phy = NULL;
 740                 p->vchan = NULL;
 741                 if (p->ds_run) {
 742                         vchan_terminate_vdesc(&p->ds_run->vd);
 743                         p->ds_run = NULL;
 744                 }
 745                 p->ds_done = NULL;
 746         }
 747         spin_unlock_irqrestore(&c->vc.lock, flags);
 748         vchan_dma_desc_free_list(&c->vc, &head);
 749
 750         return 0;
 751 }
 752
 753 static void k3_dma_synchronize(struct dma_chan *chan)
 754 {
 755         struct k3_dma_chan *c = to_k3_chan(chan);
 756
 757         vchan_synchronize(&c->vc);
 758 }
 759
 760 static int k3_dma_transfer_pause(struct dma_chan *chan)
 761 {
 762         struct k3_dma_chan *c = to_k3_chan(chan);
 763         struct k3_dma_dev *d = to_k3_dma(chan->device);
 764         struct k3_dma_phy *p = c->phy;
 765
 766         dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);
 767         if (c->status == DMA_IN_PROGRESS) {
 768                 c->status = DMA_PAUSED;
 769                 if (p) {
 770                         k3_dma_pause_dma(p, false);
 771                 } else {
 772                         spin_lock(&d->lock);
 773                         list_del_init(&c->node);
 774                         spin_unlock(&d->lock);
 775                 }
 776         }
 777
 778         return 0;
 779 }
 780
 781 static int k3_dma_transfer_resume(struct dma_chan *chan)
 782 {
 783         struct k3_dma_chan *c = to_k3_chan(chan);
 784         struct k3_dma_dev *d = to_k3_dma(chan->device);
 785         struct k3_dma_phy *p = c->phy;
 786         unsigned long flags;
 787
 788         dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);
 789         spin_lock_irqsave(&c->vc.lock, flags);
 790         if (c->status == DMA_PAUSED) {
 791                 c->status = DMA_IN_PROGRESS;
 792                 if (p) {
 793                         k3_dma_pause_dma(p, true);
 794                 } else if (!list_empty(&c->vc.desc_issued)) {
 795                         spin_lock(&d->lock);
 796                         list_add_tail(&c->node, &d->chan_pending);
 797                         spin_unlock(&d->lock);
 798                 }
 799         }
 800         spin_unlock_irqrestore(&c->vc.lock, flags);
 801
 802         return 0;
 803 }
 804
 805 static const struct k3dma_soc_data k3_v1_dma_data = {
 806         .flags = 0,
 807 };
 808
 809 static const struct k3dma_soc_data asp_v1_dma_data = {
 810         .flags = K3_FLAG_NOCLK,
 811 };
 812
 813 static const struct of_device_id k3_pdma_dt_ids[] = {
 814         { .compatible = "hisilicon,k3-dma-1.0",
 815           .data = &k3_v1_dma_data
 816         },
 817         { .compatible = "hisilicon,hisi-pcm-asp-dma-1.0",
 818           .data = &asp_v1_dma_data
 819         },
 820         {}
 821 };
 822 MODULE_DEVICE_TABLE(of, k3_pdma_dt_ids);
 823
 824 static struct dma_chan *k3_of_dma_simple_xlate(struct of_phandle_args *dma_spec,
 825                                                 struct of_dma *ofdma)
 826 {
 827         struct k3_dma_dev *d = ofdma->of_dma_data;
 828         unsigned int request = dma_spec->args[0];
 829
 830         if (request >= d->dma_requests)
 831                 return NULL;
 832
 833         return dma_get_slave_channel(&(d->chans[request].vc.chan));
 834 }
 835
 836 static int k3_dma_probe(struct platform_device *op)
 837 {
 838         const struct k3dma_soc_data *soc_data;
 839         struct k3_dma_dev *d;
 840         const struct of_device_id *of_id;
 841         struct resource *iores;
 842         int i, ret, irq = 0;
 843
 844         iores = platform_get_resource(op, IORESOURCE_MEM, 0);
 845         if (!iores)
 846                 return -EINVAL;
 847
 848         d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL);
 849         if (!d)
 850                 return -ENOMEM;
 851
 852         soc_data = device_get_match_data(&op->dev);
 853         if (!soc_data)
 854                 return -EINVAL;
 855
 856         d->base = devm_ioremap_resource(&op->dev, iores);
 857         if (IS_ERR(d->base))
 858                 return PTR_ERR(d->base);
 859
 860         of_id = of_match_device(k3_pdma_dt_ids, &op->dev);
 861         if (of_id) {
 862                 of_property_read_u32((&op->dev)->of_node,
 863                                 "dma-channels", &d->dma_channels);
 864                 of_property_read_u32((&op->dev)->of_node,
 865                                 "dma-requests", &d->dma_requests);
 866                 ret = of_property_read_u32((&op->dev)->of_node,
 867                                 "dma-channel-mask", &d->dma_channel_mask);
 868                 if (ret) {
 869                         dev_warn(&op->dev,
 870                                  "dma-channel-mask doesn't exist, considering all as available.\n");
 871                         d->dma_channel_mask = (u32)~0UL;
 872                 }
 873         }
 874
 875         if (!(soc_data->flags & K3_FLAG_NOCLK)) {
 876                 d->clk = devm_clk_get(&op->dev, NULL);
 877                 if (IS_ERR(d->clk)) {
 878                         dev_err(&op->dev, "no dma clk\n");
 879                         return PTR_ERR(d->clk);
 880                 }
 881         }
 882
 883         irq = platform_get_irq(op, 0);
 884         ret = devm_request_irq(&op->dev, irq,
 885                         k3_dma_int_handler, 0, DRIVER_NAME, d);
 886         if (ret)
 887                 return ret;
 888
 889         d->irq = irq;
 890
 891         /* A DMA memory pool for LLIs, align on 32-byte boundary */
 892         d->pool = dmam_pool_create(DRIVER_NAME, &op->dev,
 893                                         LLI_BLOCK_SIZE, 32, 0);
 894         if (!d->pool)
 895                 return -ENOMEM;
 896
 897         /* init phy channel */
 898         d->phy = devm_kcalloc(&op->dev,
 899                 d->dma_channels, sizeof(struct k3_dma_phy), GFP_KERNEL);
 900         if (d->phy == NULL)
 901                 return -ENOMEM;
 902
 903         for (i = 0; i < d->dma_channels; i++) {
 904                 struct k3_dma_phy *p;
 905
 906                 if (!(d->dma_channel_mask & BIT(i)))
 907                         continue;
 908
 909                 p = &d->phy[i];
 910                 p->idx = i;
 911                 p->base = d->base + i * 0x40;
 912         }
 913
 914         INIT_LIST_HEAD(&d->slave.channels);
 915         dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
 916         dma_cap_set(DMA_MEMCPY, d->slave.cap_mask);
 917         dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
 918         d->slave.dev = &op->dev;
 919         d->slave.device_free_chan_resources = k3_dma_free_chan_resources;
 920         d->slave.device_tx_status = k3_dma_tx_status;
 921         d->slave.device_prep_dma_memcpy = k3_dma_prep_memcpy;
 922         d->slave.device_prep_slave_sg = k3_dma_prep_slave_sg;
 923         d->slave.device_prep_dma_cyclic = k3_dma_prep_dma_cyclic;
 924         d->slave.device_issue_pending = k3_dma_issue_pending;
 925         d->slave.device_config = k3_dma_config;
 926         d->slave.device_pause = k3_dma_transfer_pause;
 927         d->slave.device_resume = k3_dma_transfer_resume;
 928         d->slave.device_terminate_all = k3_dma_terminate_all;
 929         d->slave.device_synchronize = k3_dma_synchronize;
 930         d->slave.copy_align = DMAENGINE_ALIGN_8_BYTES;
 931
 932         /* init virtual channel */
 933         d->chans = devm_kcalloc(&op->dev,
 934                 d->dma_requests, sizeof(struct k3_dma_chan), GFP_KERNEL);
 935         if (d->chans == NULL)
 936                 return -ENOMEM;
 937
 938         for (i = 0; i < d->dma_requests; i++) {
 939                 struct k3_dma_chan *c = &d->chans[i];
 940
 941                 c->status = DMA_IN_PROGRESS;
 942                 INIT_LIST_HEAD(&c->node);
 943                 c->vc.desc_free = k3_dma_free_desc;
 944                 vchan_init(&c->vc, &d->slave);
 945         }
 946
 947         /* Enable clock before accessing registers */
 948         ret = clk_prepare_enable(d->clk);
 949         if (ret < 0) {
 950                 dev_err(&op->dev, "clk_prepare_enable failed: %d\n", ret);
 951                 return ret;
 952         }
 953
 954         k3_dma_enable_dma(d, true);
 955
 956         ret = dma_async_device_register(&d->slave);
 957         if (ret)
 958                 goto dma_async_register_fail;
 959
 960         ret = of_dma_controller_register((&op->dev)->of_node,
 961                                         k3_of_dma_simple_xlate, d);
 962         if (ret)
 963                 goto of_dma_register_fail;
 964
 965         spin_lock_init(&d->lock);
 966         INIT_LIST_HEAD(&d->chan_pending);
 967         tasklet_init(&d->task, k3_dma_tasklet, (unsigned long)d);
 968         platform_set_drvdata(op, d);
 969         dev_info(&op->dev, "initialized\n");
 970
 971         return 0;
 972
 973 of_dma_register_fail:
 974         dma_async_device_unregister(&d->slave);
 975 dma_async_register_fail:
 976         clk_disable_unprepare(d->clk);
 977         return ret;
 978 }
 979
 980 static int k3_dma_remove(struct platform_device *op)
 981 {
 982         struct k3_dma_chan *c, *cn;
 983         struct k3_dma_dev *d = platform_get_drvdata(op);
 984
 985         dma_async_device_unregister(&d->slave);
 986         of_dma_controller_free((&op->dev)->of_node);
 987
 988         devm_free_irq(&op->dev, d->irq, d);
 989
 990         list_for_each_entry_safe(c, cn, &d->slave.channels, vc.chan.device_node) {
 991                 list_del(&c->vc.chan.device_node);
 992                 tasklet_kill(&c->vc.task);
 993         }
 994         tasklet_kill(&d->task);
 995         clk_disable_unprepare(d->clk);
 996         return 0;
 997 }
 998
 999 #ifdef CONFIG_PM_SLEEP
1000 static int k3_dma_suspend_dev(struct device *dev)
1001 {
1002         struct k3_dma_dev *d = dev_get_drvdata(dev);
1003         u32 stat = 0;
1004
1005         stat = k3_dma_get_chan_stat(d);
1006         if (stat) {
1007                 dev_warn(d->slave.dev,
1008                         "chan %d is running fail to suspend\n", stat);
1009                 return -1;
1010         }
1011         k3_dma_enable_dma(d, false);
1012         clk_disable_unprepare(d->clk);
1013         return 0;
1014 }
1015
1016 static int k3_dma_resume_dev(struct device *dev)
1017 {
1018         struct k3_dma_dev *d = dev_get_drvdata(dev);
1019         int ret = 0;
1020
1021         ret = clk_prepare_enable(d->clk);
1022         if (ret < 0) {
1023                 dev_err(d->slave.dev, "clk_prepare_enable failed: %d\n", ret);
1024                 return ret;
1025         }
1026         k3_dma_enable_dma(d, true);
1027         return 0;
1028 }
1029 #endif
1030
1031 static SIMPLE_DEV_PM_OPS(k3_dma_pmops, k3_dma_suspend_dev, k3_dma_resume_dev);
1032
1033 static struct platform_driver k3_pdma_driver = {
1034         .driver         = {
1035                 .name   = DRIVER_NAME,
1036                 .pm     = &k3_dma_pmops,
1037                 .of_match_table = k3_pdma_dt_ids,
1038         },
1039         .probe          = k3_dma_probe,
1040         .remove         = k3_dma_remove,
1041 };
1042
1043 module_platform_driver(k3_pdma_driver);
1044
1045 MODULE_DESCRIPTION("Hisilicon k3 DMA Driver");
1046 MODULE_ALIAS("platform:k3dma");
1047 MODULE_LICENSE("GPL v2");