drivers/dma/ti/omap-dma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * OMAP DMAengine support
   4  */
   5 #include <linux/delay.h>
   6 #include <linux/dmaengine.h>
   7 #include <linux/dma-mapping.h>
   8 #include <linux/dmapool.h>
   9 #include <linux/err.h>
  10 #include <linux/init.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/list.h>
  13 #include <linux/module.h>
  14 #include <linux/omap-dma.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/slab.h>
  17 #include <linux/spinlock.h>
  18 #include <linux/of_dma.h>
  19 #include <linux/of_device.h>
  20
  21 #include "../virt-dma.h"
  22
  23 #define OMAP_SDMA_REQUESTS      127
  24 #define OMAP_SDMA_CHANNELS      32
  25
  26 struct omap_dmadev {
  27         struct dma_device ddev;
  28         spinlock_t lock;
  29         void __iomem *base;
  30         const struct omap_dma_reg *reg_map;
  31         struct omap_system_dma_plat_info *plat;
  32         bool legacy;
  33         bool ll123_supported;
  34         struct dma_pool *desc_pool;
  35         unsigned dma_requests;
  36         spinlock_t irq_lock;
  37         uint32_t irq_enable_mask;
  38         struct omap_chan **lch_map;
  39 };
  40
  41 struct omap_chan {
  42         struct virt_dma_chan vc;
  43         void __iomem *channel_base;
  44         const struct omap_dma_reg *reg_map;
  45         uint32_t ccr;
  46
  47         struct dma_slave_config cfg;
  48         unsigned dma_sig;
  49         bool cyclic;
  50         bool paused;
  51         bool running;
  52
  53         int dma_ch;
  54         struct omap_desc *desc;
  55         unsigned sgidx;
  56 };
  57
  58 #define DESC_NXT_SV_REFRESH     (0x1 << 24)
  59 #define DESC_NXT_SV_REUSE       (0x2 << 24)
  60 #define DESC_NXT_DV_REFRESH     (0x1 << 26)
  61 #define DESC_NXT_DV_REUSE       (0x2 << 26)
  62 #define DESC_NTYPE_TYPE2        (0x2 << 29)
  63
  64 /* Type 2 descriptor with Source or Destination address update */
  65 struct omap_type2_desc {
  66         uint32_t next_desc;
  67         uint32_t en;
  68         uint32_t addr; /* src or dst */
  69         uint16_t fn;
  70         uint16_t cicr;
  71         int16_t cdei;
  72         int16_t csei;
  73         int32_t cdfi;
  74         int32_t csfi;
  75 } __packed;
  76
  77 struct omap_sg {
  78         dma_addr_t addr;
  79         uint32_t en;            /* number of elements (24-bit) */
  80         uint32_t fn;            /* number of frames (16-bit) */
  81         int32_t fi;             /* for double indexing */
  82         int16_t ei;             /* for double indexing */
  83
  84         /* Linked list */
  85         struct omap_type2_desc *t2_desc;
  86         dma_addr_t t2_desc_paddr;
  87 };
  88
  89 struct omap_desc {
  90         struct virt_dma_desc vd;
  91         bool using_ll;
  92         enum dma_transfer_direction dir;
  93         dma_addr_t dev_addr;
  94
  95         int32_t fi;             /* for OMAP_DMA_SYNC_PACKET / double indexing */
  96         int16_t ei;             /* for double indexing */
  97         uint8_t es;             /* CSDP_DATA_TYPE_xxx */
  98         uint32_t ccr;           /* CCR value */
  99         uint16_t clnk_ctrl;     /* CLNK_CTRL value */
 100         uint16_t cicr;          /* CICR value */
 101         uint32_t csdp;          /* CSDP value */
 102
 103         unsigned sglen;
 104         struct omap_sg sg[0];
 105 };
 106
 107 enum {
 108         CAPS_0_SUPPORT_LL123    = BIT(20),      /* Linked List type1/2/3 */
 109         CAPS_0_SUPPORT_LL4      = BIT(21),      /* Linked List type4 */
 110
 111         CCR_FS                  = BIT(5),
 112         CCR_READ_PRIORITY       = BIT(6),
 113         CCR_ENABLE              = BIT(7),
 114         CCR_AUTO_INIT           = BIT(8),       /* OMAP1 only */
 115         CCR_REPEAT              = BIT(9),       /* OMAP1 only */
 116         CCR_OMAP31_DISABLE      = BIT(10),      /* OMAP1 only */
 117         CCR_SUSPEND_SENSITIVE   = BIT(8),       /* OMAP2+ only */
 118         CCR_RD_ACTIVE           = BIT(9),       /* OMAP2+ only */
 119         CCR_WR_ACTIVE           = BIT(10),      /* OMAP2+ only */
 120         CCR_SRC_AMODE_CONSTANT  = 0 << 12,
 121         CCR_SRC_AMODE_POSTINC   = 1 << 12,
 122         CCR_SRC_AMODE_SGLIDX    = 2 << 12,
 123         CCR_SRC_AMODE_DBLIDX    = 3 << 12,
 124         CCR_DST_AMODE_CONSTANT  = 0 << 14,
 125         CCR_DST_AMODE_POSTINC   = 1 << 14,
 126         CCR_DST_AMODE_SGLIDX    = 2 << 14,
 127         CCR_DST_AMODE_DBLIDX    = 3 << 14,
 128         CCR_CONSTANT_FILL       = BIT(16),
 129         CCR_TRANSPARENT_COPY    = BIT(17),
 130         CCR_BS                  = BIT(18),
 131         CCR_SUPERVISOR          = BIT(22),
 132         CCR_PREFETCH            = BIT(23),
 133         CCR_TRIGGER_SRC         = BIT(24),
 134         CCR_BUFFERING_DISABLE   = BIT(25),
 135         CCR_WRITE_PRIORITY      = BIT(26),
 136         CCR_SYNC_ELEMENT        = 0,
 137         CCR_SYNC_FRAME          = CCR_FS,
 138         CCR_SYNC_BLOCK          = CCR_BS,
 139         CCR_SYNC_PACKET         = CCR_BS | CCR_FS,
 140
 141         CSDP_DATA_TYPE_8        = 0,
 142         CSDP_DATA_TYPE_16       = 1,
 143         CSDP_DATA_TYPE_32       = 2,
 144         CSDP_SRC_PORT_EMIFF     = 0 << 2, /* OMAP1 only */
 145         CSDP_SRC_PORT_EMIFS     = 1 << 2, /* OMAP1 only */
 146         CSDP_SRC_PORT_OCP_T1    = 2 << 2, /* OMAP1 only */
 147         CSDP_SRC_PORT_TIPB      = 3 << 2, /* OMAP1 only */
 148         CSDP_SRC_PORT_OCP_T2    = 4 << 2, /* OMAP1 only */
 149         CSDP_SRC_PORT_MPUI      = 5 << 2, /* OMAP1 only */
 150         CSDP_SRC_PACKED         = BIT(6),
 151         CSDP_SRC_BURST_1        = 0 << 7,
 152         CSDP_SRC_BURST_16       = 1 << 7,
 153         CSDP_SRC_BURST_32       = 2 << 7,
 154         CSDP_SRC_BURST_64       = 3 << 7,
 155         CSDP_DST_PORT_EMIFF     = 0 << 9, /* OMAP1 only */
 156         CSDP_DST_PORT_EMIFS     = 1 << 9, /* OMAP1 only */
 157         CSDP_DST_PORT_OCP_T1    = 2 << 9, /* OMAP1 only */
 158         CSDP_DST_PORT_TIPB      = 3 << 9, /* OMAP1 only */
 159         CSDP_DST_PORT_OCP_T2    = 4 << 9, /* OMAP1 only */
 160         CSDP_DST_PORT_MPUI      = 5 << 9, /* OMAP1 only */
 161         CSDP_DST_PACKED         = BIT(13),
 162         CSDP_DST_BURST_1        = 0 << 14,
 163         CSDP_DST_BURST_16       = 1 << 14,
 164         CSDP_DST_BURST_32       = 2 << 14,
 165         CSDP_DST_BURST_64       = 3 << 14,
 166         CSDP_WRITE_NON_POSTED   = 0 << 16,
 167         CSDP_WRITE_POSTED       = 1 << 16,
 168         CSDP_WRITE_LAST_NON_POSTED = 2 << 16,
 169
 170         CICR_TOUT_IE            = BIT(0),       /* OMAP1 only */
 171         CICR_DROP_IE            = BIT(1),
 172         CICR_HALF_IE            = BIT(2),
 173         CICR_FRAME_IE           = BIT(3),
 174         CICR_LAST_IE            = BIT(4),
 175         CICR_BLOCK_IE           = BIT(5),
 176         CICR_PKT_IE             = BIT(7),       /* OMAP2+ only */
 177         CICR_TRANS_ERR_IE       = BIT(8),       /* OMAP2+ only */
 178         CICR_SUPERVISOR_ERR_IE  = BIT(10),      /* OMAP2+ only */
 179         CICR_MISALIGNED_ERR_IE  = BIT(11),      /* OMAP2+ only */
 180         CICR_DRAIN_IE           = BIT(12),      /* OMAP2+ only */
 181         CICR_SUPER_BLOCK_IE     = BIT(14),      /* OMAP2+ only */
 182
 183         CLNK_CTRL_ENABLE_LNK    = BIT(15),
 184
 185         CDP_DST_VALID_INC       = 0 << 0,
 186         CDP_DST_VALID_RELOAD    = 1 << 0,
 187         CDP_DST_VALID_REUSE     = 2 << 0,
 188         CDP_SRC_VALID_INC       = 0 << 2,
 189         CDP_SRC_VALID_RELOAD    = 1 << 2,
 190         CDP_SRC_VALID_REUSE     = 2 << 2,
 191         CDP_NTYPE_TYPE1         = 1 << 4,
 192         CDP_NTYPE_TYPE2         = 2 << 4,
 193         CDP_NTYPE_TYPE3         = 3 << 4,
 194         CDP_TMODE_NORMAL        = 0 << 8,
 195         CDP_TMODE_LLIST         = 1 << 8,
 196         CDP_FAST                = BIT(10),
 197 };
 198
 199 static const unsigned es_bytes[] = {
 200         [CSDP_DATA_TYPE_8] = 1,
 201         [CSDP_DATA_TYPE_16] = 2,
 202         [CSDP_DATA_TYPE_32] = 4,
 203 };
 204
 205 static struct of_dma_filter_info omap_dma_info = {
 206         .filter_fn = omap_dma_filter_fn,
 207 };
 208
 209 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
 210 {
 211         return container_of(d, struct omap_dmadev, ddev);
 212 }
 213
 214 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
 215 {
 216         return container_of(c, struct omap_chan, vc.chan);
 217 }
 218
 219 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
 220 {
 221         return container_of(t, struct omap_desc, vd.tx);
 222 }
 223
 224 static void omap_dma_desc_free(struct virt_dma_desc *vd)
 225 {
 226         struct omap_desc *d = to_omap_dma_desc(&vd->tx);
 227
 228         if (d->using_ll) {
 229                 struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);
 230                 int i;
 231
 232                 for (i = 0; i < d->sglen; i++) {
 233                         if (d->sg[i].t2_desc)
 234                                 dma_pool_free(od->desc_pool, d->sg[i].t2_desc,
 235                                               d->sg[i].t2_desc_paddr);
 236                 }
 237         }
 238
 239         kfree(d);
 240 }
 241
 242 static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx,
 243                                      enum dma_transfer_direction dir, bool last)
 244 {
 245         struct omap_sg *sg = &d->sg[idx];
 246         struct omap_type2_desc *t2_desc = sg->t2_desc;
 247
 248         if (idx)
 249                 d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr;
 250         if (last)
 251                 t2_desc->next_desc = 0xfffffffc;
 252
 253         t2_desc->en = sg->en;
 254         t2_desc->addr = sg->addr;
 255         t2_desc->fn = sg->fn & 0xffff;
 256         t2_desc->cicr = d->cicr;
 257         if (!last)
 258                 t2_desc->cicr &= ~CICR_BLOCK_IE;
 259
 260         switch (dir) {
 261         case DMA_DEV_TO_MEM:
 262                 t2_desc->cdei = sg->ei;
 263                 t2_desc->csei = d->ei;
 264                 t2_desc->cdfi = sg->fi;
 265                 t2_desc->csfi = d->fi;
 266
 267                 t2_desc->en |= DESC_NXT_DV_REFRESH;
 268                 t2_desc->en |= DESC_NXT_SV_REUSE;
 269                 break;
 270         case DMA_MEM_TO_DEV:
 271                 t2_desc->cdei = d->ei;
 272                 t2_desc->csei = sg->ei;
 273                 t2_desc->cdfi = d->fi;
 274                 t2_desc->csfi = sg->fi;
 275
 276                 t2_desc->en |= DESC_NXT_SV_REFRESH;
 277                 t2_desc->en |= DESC_NXT_DV_REUSE;
 278                 break;
 279         default:
 280                 return;
 281         }
 282
 283         t2_desc->en |= DESC_NTYPE_TYPE2;
 284 }
 285
 286 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
 287 {
 288         switch (type) {
 289         case OMAP_DMA_REG_16BIT:
 290                 writew_relaxed(val, addr);
 291                 break;
 292         case OMAP_DMA_REG_2X16BIT:
 293                 writew_relaxed(val, addr);
 294                 writew_relaxed(val >> 16, addr + 2);
 295                 break;
 296         case OMAP_DMA_REG_32BIT:
 297                 writel_relaxed(val, addr);
 298                 break;
 299         default:
 300                 WARN_ON(1);
 301         }
 302 }
 303
 304 static unsigned omap_dma_read(unsigned type, void __iomem *addr)
 305 {
 306         unsigned val;
 307
 308         switch (type) {
 309         case OMAP_DMA_REG_16BIT:
 310                 val = readw_relaxed(addr);
 311                 break;
 312         case OMAP_DMA_REG_2X16BIT:
 313                 val = readw_relaxed(addr);
 314                 val |= readw_relaxed(addr + 2) << 16;
 315                 break;
 316         case OMAP_DMA_REG_32BIT:
 317                 val = readl_relaxed(addr);
 318                 break;
 319         default:
 320                 WARN_ON(1);
 321                 val = 0;
 322         }
 323
 324         return val;
 325 }
 326
 327 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
 328 {
 329         const struct omap_dma_reg *r = od->reg_map + reg;
 330
 331         WARN_ON(r->stride);
 332
 333         omap_dma_write(val, r->type, od->base + r->offset);
 334 }
 335
 336 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
 337 {
 338         const struct omap_dma_reg *r = od->reg_map + reg;
 339
 340         WARN_ON(r->stride);
 341
 342         return omap_dma_read(r->type, od->base + r->offset);
 343 }
 344
 345 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
 346 {
 347         const struct omap_dma_reg *r = c->reg_map + reg;
 348
 349         omap_dma_write(val, r->type, c->channel_base + r->offset);
 350 }
 351
 352 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
 353 {
 354         const struct omap_dma_reg *r = c->reg_map + reg;
 355
 356         return omap_dma_read(r->type, c->channel_base + r->offset);
 357 }
 358
 359 static void omap_dma_clear_csr(struct omap_chan *c)
 360 {
 361         if (dma_omap1())
 362                 omap_dma_chan_read(c, CSR);
 363         else
 364                 omap_dma_chan_write(c, CSR, ~0);
 365 }
 366
 367 static unsigned omap_dma_get_csr(struct omap_chan *c)
 368 {
 369         unsigned val = omap_dma_chan_read(c, CSR);
 370
 371         if (!dma_omap1())
 372                 omap_dma_chan_write(c, CSR, val);
 373
 374         return val;
 375 }
 376
 377 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
 378         unsigned lch)
 379 {
 380         c->channel_base = od->base + od->plat->channel_stride * lch;
 381
 382         od->lch_map[lch] = c;
 383 }
 384
 385 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
 386 {
 387         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 388         uint16_t cicr = d->cicr;
 389
 390         if (__dma_omap15xx(od->plat->dma_attr))
 391                 omap_dma_chan_write(c, CPC, 0);
 392         else
 393                 omap_dma_chan_write(c, CDAC, 0);
 394
 395         omap_dma_clear_csr(c);
 396
 397         if (d->using_ll) {
 398                 uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST;
 399
 400                 if (d->dir == DMA_DEV_TO_MEM)
 401                         cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE);
 402                 else
 403                         cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD);
 404                 omap_dma_chan_write(c, CDP, cdp);
 405
 406                 omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr);
 407                 omap_dma_chan_write(c, CCDN, 0);
 408                 omap_dma_chan_write(c, CCFN, 0xffff);
 409                 omap_dma_chan_write(c, CCEN, 0xffffff);
 410
 411                 cicr &= ~CICR_BLOCK_IE;
 412         } else if (od->ll123_supported) {
 413                 omap_dma_chan_write(c, CDP, 0);
 414         }
 415
 416         /* Enable interrupts */
 417         omap_dma_chan_write(c, CICR, cicr);
 418
 419         /* Enable channel */
 420         omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
 421
 422         c->running = true;
 423 }
 424
 425 static void omap_dma_drain_chan(struct omap_chan *c)
 426 {
 427         int i;
 428         u32 val;
 429
 430         /* Wait for sDMA FIFO to drain */
 431         for (i = 0; ; i++) {
 432                 val = omap_dma_chan_read(c, CCR);
 433                 if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
 434                         break;
 435
 436                 if (i > 100)
 437                         break;
 438
 439                 udelay(5);
 440         }
 441
 442         if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
 443                 dev_err(c->vc.chan.device->dev,
 444                         "DMA drain did not complete on lch %d\n",
 445                         c->dma_ch);
 446 }
 447
 448 static int omap_dma_stop(struct omap_chan *c)
 449 {
 450         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 451         uint32_t val;
 452
 453         /* disable irq */
 454         omap_dma_chan_write(c, CICR, 0);
 455
 456         omap_dma_clear_csr(c);
 457
 458         val = omap_dma_chan_read(c, CCR);
 459         if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
 460                 uint32_t sysconfig;
 461
 462                 sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
 463                 val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
 464                 val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
 465                 omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
 466
 467                 val = omap_dma_chan_read(c, CCR);
 468                 val &= ~CCR_ENABLE;
 469                 omap_dma_chan_write(c, CCR, val);
 470
 471                 if (!(c->ccr & CCR_BUFFERING_DISABLE))
 472                         omap_dma_drain_chan(c);
 473
 474                 omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
 475         } else {
 476                 if (!(val & CCR_ENABLE))
 477                         return -EINVAL;
 478
 479                 val &= ~CCR_ENABLE;
 480                 omap_dma_chan_write(c, CCR, val);
 481
 482                 if (!(c->ccr & CCR_BUFFERING_DISABLE))
 483                         omap_dma_drain_chan(c);
 484         }
 485
 486         mb();
 487
 488         if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
 489                 val = omap_dma_chan_read(c, CLNK_CTRL);
 490
 491                 if (dma_omap1())
 492                         val |= 1 << 14; /* set the STOP_LNK bit */
 493                 else
 494                         val &= ~CLNK_CTRL_ENABLE_LNK;
 495
 496                 omap_dma_chan_write(c, CLNK_CTRL, val);
 497         }
 498         c->running = false;
 499         return 0;
 500 }
 501
 502 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
 503 {
 504         struct omap_sg *sg = d->sg + c->sgidx;
 505         unsigned cxsa, cxei, cxfi;
 506
 507         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 508                 cxsa = CDSA;
 509                 cxei = CDEI;
 510                 cxfi = CDFI;
 511         } else {
 512                 cxsa = CSSA;
 513                 cxei = CSEI;
 514                 cxfi = CSFI;
 515         }
 516
 517         omap_dma_chan_write(c, cxsa, sg->addr);
 518         omap_dma_chan_write(c, cxei, sg->ei);
 519         omap_dma_chan_write(c, cxfi, sg->fi);
 520         omap_dma_chan_write(c, CEN, sg->en);
 521         omap_dma_chan_write(c, CFN, sg->fn);
 522
 523         omap_dma_start(c, d);
 524         c->sgidx++;
 525 }
 526
 527 static void omap_dma_start_desc(struct omap_chan *c)
 528 {
 529         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 530         struct omap_desc *d;
 531         unsigned cxsa, cxei, cxfi;
 532
 533         if (!vd) {
 534                 c->desc = NULL;
 535                 return;
 536         }
 537
 538         list_del(&vd->node);
 539
 540         c->desc = d = to_omap_dma_desc(&vd->tx);
 541         c->sgidx = 0;
 542
 543         /*
 544          * This provides the necessary barrier to ensure data held in
 545          * DMA coherent memory is visible to the DMA engine prior to
 546          * the transfer starting.
 547          */
 548         mb();
 549
 550         omap_dma_chan_write(c, CCR, d->ccr);
 551         if (dma_omap1())
 552                 omap_dma_chan_write(c, CCR2, d->ccr >> 16);
 553
 554         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 555                 cxsa = CSSA;
 556                 cxei = CSEI;
 557                 cxfi = CSFI;
 558         } else {
 559                 cxsa = CDSA;
 560                 cxei = CDEI;
 561                 cxfi = CDFI;
 562         }
 563
 564         omap_dma_chan_write(c, cxsa, d->dev_addr);
 565         omap_dma_chan_write(c, cxei, d->ei);
 566         omap_dma_chan_write(c, cxfi, d->fi);
 567         omap_dma_chan_write(c, CSDP, d->csdp);
 568         omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
 569
 570         omap_dma_start_sg(c, d);
 571 }
 572
 573 static void omap_dma_callback(int ch, u16 status, void *data)
 574 {
 575         struct omap_chan *c = data;
 576         struct omap_desc *d;
 577         unsigned long flags;
 578
 579         spin_lock_irqsave(&c->vc.lock, flags);
 580         d = c->desc;
 581         if (d) {
 582                 if (c->cyclic) {
 583                         vchan_cyclic_callback(&d->vd);
 584                 } else if (d->using_ll || c->sgidx == d->sglen) {
 585                         omap_dma_start_desc(c);
 586                         vchan_cookie_complete(&d->vd);
 587                 } else {
 588                         omap_dma_start_sg(c, d);
 589                 }
 590         }
 591         spin_unlock_irqrestore(&c->vc.lock, flags);
 592 }
 593
 594 static irqreturn_t omap_dma_irq(int irq, void *devid)
 595 {
 596         struct omap_dmadev *od = devid;
 597         unsigned status, channel;
 598
 599         spin_lock(&od->irq_lock);
 600
 601         status = omap_dma_glbl_read(od, IRQSTATUS_L1);
 602         status &= od->irq_enable_mask;
 603         if (status == 0) {
 604                 spin_unlock(&od->irq_lock);
 605                 return IRQ_NONE;
 606         }
 607
 608         while ((channel = ffs(status)) != 0) {
 609                 unsigned mask, csr;
 610                 struct omap_chan *c;
 611
 612                 channel -= 1;
 613                 mask = BIT(channel);
 614                 status &= ~mask;
 615
 616                 c = od->lch_map[channel];
 617                 if (c == NULL) {
 618                         /* This should never happen */
 619                         dev_err(od->ddev.dev, "invalid channel %u\n", channel);
 620                         continue;
 621                 }
 622
 623                 csr = omap_dma_get_csr(c);
 624                 omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
 625
 626                 omap_dma_callback(channel, csr, c);
 627         }
 628
 629         spin_unlock(&od->irq_lock);
 630
 631         return IRQ_HANDLED;
 632 }
 633
 634 static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
 635 {
 636         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 637         struct omap_chan *c = to_omap_dma_chan(chan);
 638         struct device *dev = od->ddev.dev;
 639         int ret;
 640
 641         if (od->legacy) {
 642                 ret = omap_request_dma(c->dma_sig, "DMA engine",
 643                                        omap_dma_callback, c, &c->dma_ch);
 644         } else {
 645                 ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL,
 646                                        &c->dma_ch);
 647         }
 648
 649         dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig);
 650
 651         if (ret >= 0) {
 652                 omap_dma_assign(od, c, c->dma_ch);
 653
 654                 if (!od->legacy) {
 655                         unsigned val;
 656
 657                         spin_lock_irq(&od->irq_lock);
 658                         val = BIT(c->dma_ch);
 659                         omap_dma_glbl_write(od, IRQSTATUS_L1, val);
 660                         od->irq_enable_mask |= val;
 661                         omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 662
 663                         val = omap_dma_glbl_read(od, IRQENABLE_L0);
 664                         val &= ~BIT(c->dma_ch);
 665                         omap_dma_glbl_write(od, IRQENABLE_L0, val);
 666                         spin_unlock_irq(&od->irq_lock);
 667                 }
 668         }
 669
 670         if (dma_omap1()) {
 671                 if (__dma_omap16xx(od->plat->dma_attr)) {
 672                         c->ccr = CCR_OMAP31_DISABLE;
 673                         /* Duplicate what plat-omap/dma.c does */
 674                         c->ccr |= c->dma_ch + 1;
 675                 } else {
 676                         c->ccr = c->dma_sig & 0x1f;
 677                 }
 678         } else {
 679                 c->ccr = c->dma_sig & 0x1f;
 680                 c->ccr |= (c->dma_sig & ~0x1f) << 14;
 681         }
 682         if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
 683                 c->ccr |= CCR_BUFFERING_DISABLE;
 684
 685         return ret;
 686 }
 687
 688 static void omap_dma_free_chan_resources(struct dma_chan *chan)
 689 {
 690         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 691         struct omap_chan *c = to_omap_dma_chan(chan);
 692
 693         if (!od->legacy) {
 694                 spin_lock_irq(&od->irq_lock);
 695                 od->irq_enable_mask &= ~BIT(c->dma_ch);
 696                 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 697                 spin_unlock_irq(&od->irq_lock);
 698         }
 699
 700         c->channel_base = NULL;
 701         od->lch_map[c->dma_ch] = NULL;
 702         vchan_free_chan_resources(&c->vc);
 703         omap_free_dma(c->dma_ch);
 704
 705         dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,
 706                 c->dma_sig);
 707         c->dma_sig = 0;
 708 }
 709
 710 static size_t omap_dma_sg_size(struct omap_sg *sg)
 711 {
 712         return sg->en * sg->fn;
 713 }
 714
 715 static size_t omap_dma_desc_size(struct omap_desc *d)
 716 {
 717         unsigned i;
 718         size_t size;
 719
 720         for (size = i = 0; i < d->sglen; i++)
 721                 size += omap_dma_sg_size(&d->sg[i]);
 722
 723         return size * es_bytes[d->es];
 724 }
 725
 726 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
 727 {
 728         unsigned i;
 729         size_t size, es_size = es_bytes[d->es];
 730
 731         for (size = i = 0; i < d->sglen; i++) {
 732                 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
 733
 734                 if (size)
 735                         size += this_size;
 736                 else if (addr >= d->sg[i].addr &&
 737                          addr < d->sg[i].addr + this_size)
 738                         size += d->sg[i].addr + this_size - addr;
 739         }
 740         return size;
 741 }
 742
 743 /*
 744  * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
 745  * read before the DMA controller finished disabling the channel.
 746  */
 747 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
 748 {
 749         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 750         uint32_t val;
 751
 752         val = omap_dma_chan_read(c, reg);
 753         if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
 754                 val = omap_dma_chan_read(c, reg);
 755
 756         return val;
 757 }
 758
 759 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
 760 {
 761         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 762         dma_addr_t addr, cdac;
 763
 764         if (__dma_omap15xx(od->plat->dma_attr)) {
 765                 addr = omap_dma_chan_read(c, CPC);
 766         } else {
 767                 addr = omap_dma_chan_read_3_3(c, CSAC);
 768                 cdac = omap_dma_chan_read_3_3(c, CDAC);
 769
 770                 /*
 771                  * CDAC == 0 indicates that the DMA transfer on the channel has
 772                  * not been started (no data has been transferred so far).
 773                  * Return the programmed source start address in this case.
 774                  */
 775                 if (cdac == 0)
 776                         addr = omap_dma_chan_read(c, CSSA);
 777         }
 778
 779         if (dma_omap1())
 780                 addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
 781
 782         return addr;
 783 }
 784
 785 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
 786 {
 787         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 788         dma_addr_t addr;
 789
 790         if (__dma_omap15xx(od->plat->dma_attr)) {
 791                 addr = omap_dma_chan_read(c, CPC);
 792         } else {
 793                 addr = omap_dma_chan_read_3_3(c, CDAC);
 794
 795                 /*
 796                  * CDAC == 0 indicates that the DMA transfer on the channel
 797                  * has not been started (no data has been transferred so
 798                  * far).  Return the programmed destination start address in
 799                  * this case.
 800                  */
 801                 if (addr == 0)
 802                         addr = omap_dma_chan_read(c, CDSA);
 803         }
 804
 805         if (dma_omap1())
 806                 addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
 807
 808         return addr;
 809 }
 810
 811 static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
 812         dma_cookie_t cookie, struct dma_tx_state *txstate)
 813 {
 814         struct omap_chan *c = to_omap_dma_chan(chan);
 815         struct virt_dma_desc *vd;
 816         enum dma_status ret;
 817         unsigned long flags;
 818
 819         ret = dma_cookie_status(chan, cookie, txstate);
 820
 821         if (!c->paused && c->running) {
 822                 uint32_t ccr = omap_dma_chan_read(c, CCR);
 823                 /*
 824                  * The channel is no longer active, set the return value
 825                  * accordingly
 826                  */
 827                 if (!(ccr & CCR_ENABLE))
 828                         ret = DMA_COMPLETE;
 829         }
 830
 831         if (ret == DMA_COMPLETE || !txstate)
 832                 return ret;
 833
 834         spin_lock_irqsave(&c->vc.lock, flags);
 835         vd = vchan_find_desc(&c->vc, cookie);
 836         if (vd) {
 837                 txstate->residue = omap_dma_desc_size(to_omap_dma_desc(&vd->tx));
 838         } else if (c->desc && c->desc->vd.tx.cookie == cookie) {
 839                 struct omap_desc *d = c->desc;
 840                 dma_addr_t pos;
 841
 842                 if (d->dir == DMA_MEM_TO_DEV)
 843                         pos = omap_dma_get_src_pos(c);
 844                 else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
 845                         pos = omap_dma_get_dst_pos(c);
 846                 else
 847                         pos = 0;
 848
 849                 txstate->residue = omap_dma_desc_size_pos(d, pos);
 850         } else {
 851                 txstate->residue = 0;
 852         }
 853         if (ret == DMA_IN_PROGRESS && c->paused)
 854                 ret = DMA_PAUSED;
 855         spin_unlock_irqrestore(&c->vc.lock, flags);
 856
 857         return ret;
 858 }
 859
 860 static void omap_dma_issue_pending(struct dma_chan *chan)
 861 {
 862         struct omap_chan *c = to_omap_dma_chan(chan);
 863         unsigned long flags;
 864
 865         spin_lock_irqsave(&c->vc.lock, flags);
 866         if (vchan_issue_pending(&c->vc) && !c->desc)
 867                 omap_dma_start_desc(c);
 868         spin_unlock_irqrestore(&c->vc.lock, flags);
 869 }
 870
 871 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 872         struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
 873         enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
 874 {
 875         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 876         struct omap_chan *c = to_omap_dma_chan(chan);
 877         enum dma_slave_buswidth dev_width;
 878         struct scatterlist *sgent;
 879         struct omap_desc *d;
 880         dma_addr_t dev_addr;
 881         unsigned i, es, en, frame_bytes;
 882         bool ll_failed = false;
 883         u32 burst;
 884         u32 port_window, port_window_bytes;
 885
 886         if (dir == DMA_DEV_TO_MEM) {
 887                 dev_addr = c->cfg.src_addr;
 888                 dev_width = c->cfg.src_addr_width;
 889                 burst = c->cfg.src_maxburst;
 890                 port_window = c->cfg.src_port_window_size;
 891         } else if (dir == DMA_MEM_TO_DEV) {
 892                 dev_addr = c->cfg.dst_addr;
 893                 dev_width = c->cfg.dst_addr_width;
 894                 burst = c->cfg.dst_maxburst;
 895                 port_window = c->cfg.dst_port_window_size;
 896         } else {
 897                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
 898                 return NULL;
 899         }
 900
 901         /* Bus width translates to the element size (ES) */
 902         switch (dev_width) {
 903         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 904                 es = CSDP_DATA_TYPE_8;
 905                 break;
 906         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 907                 es = CSDP_DATA_TYPE_16;
 908                 break;
 909         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 910                 es = CSDP_DATA_TYPE_32;
 911                 break;
 912         default: /* not reached */
 913                 return NULL;
 914         }
 915
 916         /* Now allocate and setup the descriptor. */
 917         d = kzalloc(struct_size(d, sg, sglen), GFP_ATOMIC);
 918         if (!d)
 919                 return NULL;
 920
 921         d->dir = dir;
 922         d->dev_addr = dev_addr;
 923         d->es = es;
 924
 925         /* When the port_window is used, one frame must cover the window */
 926         if (port_window) {
 927                 burst = port_window;
 928                 port_window_bytes = port_window * es_bytes[es];
 929
 930                 d->ei = 1;
 931                 /*
 932                  * One frame covers the port_window and by  configure
 933                  * the source frame index to be -1 * (port_window - 1)
 934                  * we instruct the sDMA that after a frame is processed
 935                  * it should move back to the start of the window.
 936                  */
 937                 d->fi = -(port_window_bytes - 1);
 938         }
 939
 940         d->ccr = c->ccr | CCR_SYNC_FRAME;
 941         if (dir == DMA_DEV_TO_MEM) {
 942                 d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
 943
 944                 d->ccr |= CCR_DST_AMODE_POSTINC;
 945                 if (port_window) {
 946                         d->ccr |= CCR_SRC_AMODE_DBLIDX;
 947
 948                         if (port_window_bytes >= 64)
 949                                 d->csdp |= CSDP_SRC_BURST_64;
 950                         else if (port_window_bytes >= 32)
 951                                 d->csdp |= CSDP_SRC_BURST_32;
 952                         else if (port_window_bytes >= 16)
 953                                 d->csdp |= CSDP_SRC_BURST_16;
 954
 955                 } else {
 956                         d->ccr |= CCR_SRC_AMODE_CONSTANT;
 957                 }
 958         } else {
 959                 d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
 960
 961                 d->ccr |= CCR_SRC_AMODE_POSTINC;
 962                 if (port_window) {
 963                         d->ccr |= CCR_DST_AMODE_DBLIDX;
 964
 965                         if (port_window_bytes >= 64)
 966                                 d->csdp |= CSDP_DST_BURST_64;
 967                         else if (port_window_bytes >= 32)
 968                                 d->csdp |= CSDP_DST_BURST_32;
 969                         else if (port_window_bytes >= 16)
 970                                 d->csdp |= CSDP_DST_BURST_16;
 971                 } else {
 972                         d->ccr |= CCR_DST_AMODE_CONSTANT;
 973                 }
 974         }
 975
 976         d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
 977         d->csdp |= es;
 978
 979         if (dma_omap1()) {
 980                 d->cicr |= CICR_TOUT_IE;
 981
 982                 if (dir == DMA_DEV_TO_MEM)
 983                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
 984                 else
 985                         d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
 986         } else {
 987                 if (dir == DMA_DEV_TO_MEM)
 988                         d->ccr |= CCR_TRIGGER_SRC;
 989
 990                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
 991
 992                 if (port_window)
 993                         d->csdp |= CSDP_WRITE_LAST_NON_POSTED;
 994         }
 995         if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
 996                 d->clnk_ctrl = c->dma_ch;
 997
 998         /*
 999          * Build our scatterlist entries: each contains the address,
1000          * the number of elements (EN) in each frame, and the number of
1001          * frames (FN).  Number of bytes for this entry = ES * EN * FN.
1002          *
1003          * Burst size translates to number of elements with frame sync.
1004          * Note: DMA engine defines burst to be the number of dev-width
1005          * transfers.
1006          */
1007         en = burst;
1008         frame_bytes = es_bytes[es] * en;
1009
1010         if (sglen >= 2)
1011                 d->using_ll = od->ll123_supported;
1012
1013         for_each_sg(sgl, sgent, sglen, i) {
1014                 struct omap_sg *osg = &d->sg[i];
1015
1016                 osg->addr = sg_dma_address(sgent);
1017                 osg->en = en;
1018                 osg->fn = sg_dma_len(sgent) / frame_bytes;
1019
1020                 if (d->using_ll) {
1021                         osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
1022                                                       &osg->t2_desc_paddr);
1023                         if (!osg->t2_desc) {
1024                                 dev_err(chan->device->dev,
1025                                         "t2_desc[%d] allocation failed\n", i);
1026                                 ll_failed = true;
1027                                 d->using_ll = false;
1028                                 continue;
1029                         }
1030
1031                         omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1));
1032                 }
1033         }
1034
1035         d->sglen = sglen;
1036
1037         /* Release the dma_pool entries if one allocation failed */
1038         if (ll_failed) {
1039                 for (i = 0; i < d->sglen; i++) {
1040                         struct omap_sg *osg = &d->sg[i];
1041
1042                         if (osg->t2_desc) {
1043                                 dma_pool_free(od->desc_pool, osg->t2_desc,
1044                                               osg->t2_desc_paddr);
1045                                 osg->t2_desc = NULL;
1046                         }
1047                 }
1048         }
1049
1050         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1051 }
1052
1053 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
1054         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1055         size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
1056 {
1057         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1058         struct omap_chan *c = to_omap_dma_chan(chan);
1059         enum dma_slave_buswidth dev_width;
1060         struct omap_desc *d;
1061         dma_addr_t dev_addr;
1062         unsigned es;
1063         u32 burst;
1064
1065         if (dir == DMA_DEV_TO_MEM) {
1066                 dev_addr = c->cfg.src_addr;
1067                 dev_width = c->cfg.src_addr_width;
1068                 burst = c->cfg.src_maxburst;
1069         } else if (dir == DMA_MEM_TO_DEV) {
1070                 dev_addr = c->cfg.dst_addr;
1071                 dev_width = c->cfg.dst_addr_width;
1072                 burst = c->cfg.dst_maxburst;
1073         } else {
1074                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
1075                 return NULL;
1076         }
1077
1078         /* Bus width translates to the element size (ES) */
1079         switch (dev_width) {
1080         case DMA_SLAVE_BUSWIDTH_1_BYTE:
1081                 es = CSDP_DATA_TYPE_8;
1082                 break;
1083         case DMA_SLAVE_BUSWIDTH_2_BYTES:
1084                 es = CSDP_DATA_TYPE_16;
1085                 break;
1086         case DMA_SLAVE_BUSWIDTH_4_BYTES:
1087                 es = CSDP_DATA_TYPE_32;
1088                 break;
1089         default: /* not reached */
1090                 return NULL;
1091         }
1092
1093         /* Now allocate and setup the descriptor. */
1094         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1095         if (!d)
1096                 return NULL;
1097
1098         d->dir = dir;
1099         d->dev_addr = dev_addr;
1100         d->fi = burst;
1101         d->es = es;
1102         d->sg[0].addr = buf_addr;
1103         d->sg[0].en = period_len / es_bytes[es];
1104         d->sg[0].fn = buf_len / period_len;
1105         d->sglen = 1;
1106
1107         d->ccr = c->ccr;
1108         if (dir == DMA_DEV_TO_MEM)
1109                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
1110         else
1111                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
1112
1113         d->cicr = CICR_DROP_IE;
1114         if (flags & DMA_PREP_INTERRUPT)
1115                 d->cicr |= CICR_FRAME_IE;
1116
1117         d->csdp = es;
1118
1119         if (dma_omap1()) {
1120                 d->cicr |= CICR_TOUT_IE;
1121
1122                 if (dir == DMA_DEV_TO_MEM)
1123                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
1124                 else
1125                         d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
1126         } else {
1127                 if (burst)
1128                         d->ccr |= CCR_SYNC_PACKET;
1129                 else
1130                         d->ccr |= CCR_SYNC_ELEMENT;
1131
1132                 if (dir == DMA_DEV_TO_MEM) {
1133                         d->ccr |= CCR_TRIGGER_SRC;
1134                         d->csdp |= CSDP_DST_PACKED;
1135                 } else {
1136                         d->csdp |= CSDP_SRC_PACKED;
1137                 }
1138
1139                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1140
1141                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1142         }
1143
1144         if (__dma_omap15xx(od->plat->dma_attr))
1145                 d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
1146         else
1147                 d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
1148
1149         c->cyclic = true;
1150
1151         return vchan_tx_prep(&c->vc, &d->vd, flags);
1152 }
1153
1154 static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
1155         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1156         size_t len, unsigned long tx_flags)
1157 {
1158         struct omap_chan *c = to_omap_dma_chan(chan);
1159         struct omap_desc *d;
1160         uint8_t data_type;
1161
1162         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1163         if (!d)
1164                 return NULL;
1165
1166         data_type = __ffs((src | dest | len));
1167         if (data_type > CSDP_DATA_TYPE_32)
1168                 data_type = CSDP_DATA_TYPE_32;
1169
1170         d->dir = DMA_MEM_TO_MEM;
1171         d->dev_addr = src;
1172         d->fi = 0;
1173         d->es = data_type;
1174         d->sg[0].en = len / BIT(data_type);
1175         d->sg[0].fn = 1;
1176         d->sg[0].addr = dest;
1177         d->sglen = 1;
1178         d->ccr = c->ccr;
1179         d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
1180
1181         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1182
1183         d->csdp = data_type;
1184
1185         if (dma_omap1()) {
1186                 d->cicr |= CICR_TOUT_IE;
1187                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1188         } else {
1189                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1190                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1191                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1192         }
1193
1194         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1195 }
1196
1197 static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
1198         struct dma_chan *chan, struct dma_interleaved_template *xt,
1199         unsigned long flags)
1200 {
1201         struct omap_chan *c = to_omap_dma_chan(chan);
1202         struct omap_desc *d;
1203         struct omap_sg *sg;
1204         uint8_t data_type;
1205         size_t src_icg, dst_icg;
1206
1207         /* Slave mode is not supported */
1208         if (is_slave_direction(xt->dir))
1209                 return NULL;
1210
1211         if (xt->frame_size != 1 || xt->numf == 0)
1212                 return NULL;
1213
1214         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1215         if (!d)
1216                 return NULL;
1217
1218         data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
1219         if (data_type > CSDP_DATA_TYPE_32)
1220                 data_type = CSDP_DATA_TYPE_32;
1221
1222         sg = &d->sg[0];
1223         d->dir = DMA_MEM_TO_MEM;
1224         d->dev_addr = xt->src_start;
1225         d->es = data_type;
1226         sg->en = xt->sgl[0].size / BIT(data_type);
1227         sg->fn = xt->numf;
1228         sg->addr = xt->dst_start;
1229         d->sglen = 1;
1230         d->ccr = c->ccr;
1231
1232         src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
1233         dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
1234         if (src_icg) {
1235                 d->ccr |= CCR_SRC_AMODE_DBLIDX;
1236                 d->ei = 1;
1237                 d->fi = src_icg + 1;
1238         } else if (xt->src_inc) {
1239                 d->ccr |= CCR_SRC_AMODE_POSTINC;
1240                 d->fi = 0;
1241         } else {
1242                 dev_err(chan->device->dev,
1243                         "%s: SRC constant addressing is not supported\n",
1244                         __func__);
1245                 kfree(d);
1246                 return NULL;
1247         }
1248
1249         if (dst_icg) {
1250                 d->ccr |= CCR_DST_AMODE_DBLIDX;
1251                 sg->ei = 1;
1252                 sg->fi = dst_icg + 1;
1253         } else if (xt->dst_inc) {
1254                 d->ccr |= CCR_DST_AMODE_POSTINC;
1255                 sg->fi = 0;
1256         } else {
1257                 dev_err(chan->device->dev,
1258                         "%s: DST constant addressing is not supported\n",
1259                         __func__);
1260                 kfree(d);
1261                 return NULL;
1262         }
1263
1264         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1265
1266         d->csdp = data_type;
1267
1268         if (dma_omap1()) {
1269                 d->cicr |= CICR_TOUT_IE;
1270                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1271         } else {
1272                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1273                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1274                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1275         }
1276
1277         return vchan_tx_prep(&c->vc, &d->vd, flags);
1278 }
1279
1280 static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
1281 {
1282         struct omap_chan *c = to_omap_dma_chan(chan);
1283
1284         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1285             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1286                 return -EINVAL;
1287
1288         if (cfg->src_maxburst > chan->device->max_burst ||
1289             cfg->dst_maxburst > chan->device->max_burst)
1290                 return -EINVAL;
1291
1292         memcpy(&c->cfg, cfg, sizeof(c->cfg));
1293
1294         return 0;
1295 }
1296
1297 static int omap_dma_terminate_all(struct dma_chan *chan)
1298 {
1299         struct omap_chan *c = to_omap_dma_chan(chan);
1300         unsigned long flags;
1301         LIST_HEAD(head);
1302
1303         spin_lock_irqsave(&c->vc.lock, flags);
1304
1305         /*
1306          * Stop DMA activity: we assume the callback will not be called
1307          * after omap_dma_stop() returns (even if it does, it will see
1308          * c->desc is NULL and exit.)
1309          */
1310         if (c->desc) {
1311                 vchan_terminate_vdesc(&c->desc->vd);
1312                 c->desc = NULL;
1313                 /* Avoid stopping the dma twice */
1314                 if (!c->paused)
1315                         omap_dma_stop(c);
1316         }
1317
1318         c->cyclic = false;
1319         c->paused = false;
1320
1321         vchan_get_all_descriptors(&c->vc, &head);
1322         spin_unlock_irqrestore(&c->vc.lock, flags);
1323         vchan_dma_desc_free_list(&c->vc, &head);
1324
1325         return 0;
1326 }
1327
1328 static void omap_dma_synchronize(struct dma_chan *chan)
1329 {
1330         struct omap_chan *c = to_omap_dma_chan(chan);
1331
1332         vchan_synchronize(&c->vc);
1333 }
1334
1335 static int omap_dma_pause(struct dma_chan *chan)
1336 {
1337         struct omap_chan *c = to_omap_dma_chan(chan);
1338         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1339         unsigned long flags;
1340         int ret = -EINVAL;
1341         bool can_pause = false;
1342
1343         spin_lock_irqsave(&od->irq_lock, flags);
1344
1345         if (!c->desc)
1346                 goto out;
1347
1348         if (c->cyclic)
1349                 can_pause = true;
1350
1351         /*
1352          * We do not allow DMA_MEM_TO_DEV transfers to be paused.
1353          * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer:
1354          * "When a channel is disabled during a transfer, the channel undergoes
1355          * an abort, unless it is hardware-source-synchronized …".
1356          * A source-synchronised channel is one where the fetching of data is
1357          * under control of the device. In other words, a device-to-memory
1358          * transfer. So, a destination-synchronised channel (which would be a
1359          * memory-to-device transfer) undergoes an abort if the the CCR_ENABLE
1360          * bit is cleared.
1361          * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel
1362          * aborts immediately after completion of current read/write
1363          * transactions and then the FIFO is cleaned up." The term "cleaned up"
1364          * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE
1365          * are both clear _before_ disabling the channel, otherwise data loss
1366          * will occur.
1367          * The problem is that if the channel is active, then device activity
1368          * can result in DMA activity starting between reading those as both
1369          * clear and the write to DMA_CCR to clear the enable bit hitting the
1370          * hardware. If the DMA hardware can't drain the data in its FIFO to the
1371          * destination, then data loss "might" occur (say if we write to an UART
1372          * and the UART is not accepting any further data).
1373          */
1374         else if (c->desc->dir == DMA_DEV_TO_MEM)
1375                 can_pause = true;
1376
1377         if (can_pause && !c->paused) {
1378                 ret = omap_dma_stop(c);
1379                 if (!ret)
1380                         c->paused = true;
1381         }
1382 out:
1383         spin_unlock_irqrestore(&od->irq_lock, flags);
1384
1385         return ret;
1386 }
1387
1388 static int omap_dma_resume(struct dma_chan *chan)
1389 {
1390         struct omap_chan *c = to_omap_dma_chan(chan);
1391         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1392         unsigned long flags;
1393         int ret = -EINVAL;
1394
1395         spin_lock_irqsave(&od->irq_lock, flags);
1396
1397         if (c->paused && c->desc) {
1398                 mb();
1399
1400                 /* Restore channel link register */
1401                 omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
1402
1403                 omap_dma_start(c, c->desc);
1404                 c->paused = false;
1405                 ret = 0;
1406         }
1407         spin_unlock_irqrestore(&od->irq_lock, flags);
1408
1409         return ret;
1410 }
1411
1412 static int omap_dma_chan_init(struct omap_dmadev *od)
1413 {
1414         struct omap_chan *c;
1415
1416         c = kzalloc(sizeof(*c), GFP_KERNEL);
1417         if (!c)
1418                 return -ENOMEM;
1419
1420         c->reg_map = od->reg_map;
1421         c->vc.desc_free = omap_dma_desc_free;
1422         vchan_init(&c->vc, &od->ddev);
1423
1424         return 0;
1425 }
1426
1427 static void omap_dma_free(struct omap_dmadev *od)
1428 {
1429         while (!list_empty(&od->ddev.channels)) {
1430                 struct omap_chan *c = list_first_entry(&od->ddev.channels,
1431                         struct omap_chan, vc.chan.device_node);
1432
1433                 list_del(&c->vc.chan.device_node);
1434                 tasklet_kill(&c->vc.task);
1435                 kfree(c);
1436         }
1437 }
1438
1439 #define OMAP_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1440                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1441                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1442
1443 static int omap_dma_probe(struct platform_device *pdev)
1444 {
1445         struct omap_dmadev *od;
1446         struct resource *res;
1447         int rc, i, irq;
1448         u32 lch_count;
1449
1450         od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1451         if (!od)
1452                 return -ENOMEM;
1453
1454         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1455         od->base = devm_ioremap_resource(&pdev->dev, res);
1456         if (IS_ERR(od->base))
1457                 return PTR_ERR(od->base);
1458
1459         od->plat = omap_get_plat_info();
1460         if (!od->plat)
1461                 return -EPROBE_DEFER;
1462
1463         od->reg_map = od->plat->reg_map;
1464
1465         dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
1466         dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
1467         dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
1468         dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
1469         od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
1470         od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
1471         od->ddev.device_tx_status = omap_dma_tx_status;
1472         od->ddev.device_issue_pending = omap_dma_issue_pending;
1473         od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
1474         od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
1475         od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
1476         od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
1477         od->ddev.device_config = omap_dma_slave_config;
1478         od->ddev.device_pause = omap_dma_pause;
1479         od->ddev.device_resume = omap_dma_resume;
1480         od->ddev.device_terminate_all = omap_dma_terminate_all;
1481         od->ddev.device_synchronize = omap_dma_synchronize;
1482         od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
1483         od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
1484         od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1485         if (__dma_omap15xx(od->plat->dma_attr))
1486                 od->ddev.residue_granularity =
1487                                 DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
1488         else
1489                 od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1490         od->ddev.max_burst = SZ_16M - 1; /* CCEN: 24bit unsigned */
1491         od->ddev.dev = &pdev->dev;
1492         INIT_LIST_HEAD(&od->ddev.channels);
1493         spin_lock_init(&od->lock);
1494         spin_lock_init(&od->irq_lock);
1495
1496         /* Number of DMA requests */
1497         od->dma_requests = OMAP_SDMA_REQUESTS;
1498         if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
1499                                                       "dma-requests",
1500                                                       &od->dma_requests)) {
1501                 dev_info(&pdev->dev,
1502                          "Missing dma-requests property, using %u.\n",
1503                          OMAP_SDMA_REQUESTS);
1504         }
1505
1506         /* Number of available logical channels */
1507         if (!pdev->dev.of_node) {
1508                 lch_count = od->plat->dma_attr->lch_count;
1509                 if (unlikely(!lch_count))
1510                         lch_count = OMAP_SDMA_CHANNELS;
1511         } else if (of_property_read_u32(pdev->dev.of_node, "dma-channels",
1512                                         &lch_count)) {
1513                 dev_info(&pdev->dev,
1514                          "Missing dma-channels property, using %u.\n",
1515                          OMAP_SDMA_CHANNELS);
1516                 lch_count = OMAP_SDMA_CHANNELS;
1517         }
1518
1519         od->lch_map = devm_kcalloc(&pdev->dev, lch_count, sizeof(*od->lch_map),
1520                                    GFP_KERNEL);
1521         if (!od->lch_map)
1522                 return -ENOMEM;
1523
1524         for (i = 0; i < od->dma_requests; i++) {
1525                 rc = omap_dma_chan_init(od);
1526                 if (rc) {
1527                         omap_dma_free(od);
1528                         return rc;
1529                 }
1530         }
1531
1532         irq = platform_get_irq(pdev, 1);
1533         if (irq <= 0) {
1534                 dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
1535                 od->legacy = true;
1536         } else {
1537                 /* Disable all interrupts */
1538                 od->irq_enable_mask = 0;
1539                 omap_dma_glbl_write(od, IRQENABLE_L1, 0);
1540
1541                 rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
1542                                       IRQF_SHARED, "omap-dma-engine", od);
1543                 if (rc)
1544                         return rc;
1545         }
1546
1547         if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
1548                 od->ll123_supported = true;
1549
1550         od->ddev.filter.map = od->plat->slave_map;
1551         od->ddev.filter.mapcnt = od->plat->slavecnt;
1552         od->ddev.filter.fn = omap_dma_filter_fn;
1553
1554         if (od->ll123_supported) {
1555                 od->desc_pool = dma_pool_create(dev_name(&pdev->dev),
1556                                                 &pdev->dev,
1557                                                 sizeof(struct omap_type2_desc),
1558                                                 4, 0);
1559                 if (!od->desc_pool) {
1560                         dev_err(&pdev->dev,
1561                                 "unable to allocate descriptor pool\n");
1562                         od->ll123_supported = false;
1563                 }
1564         }
1565
1566         rc = dma_async_device_register(&od->ddev);
1567         if (rc) {
1568                 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1569                         rc);
1570                 omap_dma_free(od);
1571                 return rc;
1572         }
1573
1574         platform_set_drvdata(pdev, od);
1575
1576         if (pdev->dev.of_node) {
1577                 omap_dma_info.dma_cap = od->ddev.cap_mask;
1578
1579                 /* Device-tree DMA controller registration */
1580                 rc = of_dma_controller_register(pdev->dev.of_node,
1581                                 of_dma_simple_xlate, &omap_dma_info);
1582                 if (rc) {
1583                         pr_warn("OMAP-DMA: failed to register DMA controller\n");
1584                         dma_async_device_unregister(&od->ddev);
1585                         omap_dma_free(od);
1586                 }
1587         }
1588
1589         dev_info(&pdev->dev, "OMAP DMA engine driver%s\n",
1590                  od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");
1591
1592         return rc;
1593 }
1594
1595 static int omap_dma_remove(struct platform_device *pdev)
1596 {
1597         struct omap_dmadev *od = platform_get_drvdata(pdev);
1598         int irq;
1599
1600         if (pdev->dev.of_node)
1601                 of_dma_controller_free(pdev->dev.of_node);
1602
1603         irq = platform_get_irq(pdev, 1);
1604         devm_free_irq(&pdev->dev, irq, od);
1605
1606         dma_async_device_unregister(&od->ddev);
1607
1608         if (!od->legacy) {
1609                 /* Disable all interrupts */
1610                 omap_dma_glbl_write(od, IRQENABLE_L0, 0);
1611         }
1612
1613         if (od->ll123_supported)
1614                 dma_pool_destroy(od->desc_pool);
1615
1616         omap_dma_free(od);
1617
1618         return 0;
1619 }
1620
1621 static const struct of_device_id omap_dma_match[] = {
1622         { .compatible = "ti,omap2420-sdma", },
1623         { .compatible = "ti,omap2430-sdma", },
1624         { .compatible = "ti,omap3430-sdma", },
1625         { .compatible = "ti,omap3630-sdma", },
1626         { .compatible = "ti,omap4430-sdma", },
1627         {},
1628 };
1629 MODULE_DEVICE_TABLE(of, omap_dma_match);
1630
1631 static struct platform_driver omap_dma_driver = {
1632         .probe  = omap_dma_probe,
1633         .remove = omap_dma_remove,
1634         .driver = {
1635                 .name = "omap-dma-engine",
1636                 .of_match_table = of_match_ptr(omap_dma_match),
1637         },
1638 };
1639
1640 bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
1641 {
1642         if (chan->device->dev->driver == &omap_dma_driver.driver) {
1643                 struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1644                 struct omap_chan *c = to_omap_dma_chan(chan);
1645                 unsigned req = *(unsigned *)param;
1646
1647                 if (req <= od->dma_requests) {
1648                         c->dma_sig = req;
1649                         return true;
1650                 }
1651         }
1652         return false;
1653 }
1654 EXPORT_SYMBOL_GPL(omap_dma_filter_fn);
1655
1656 static int omap_dma_init(void)
1657 {
1658         return platform_driver_register(&omap_dma_driver);
1659 }
1660 subsys_initcall(omap_dma_init);
1661
1662 static void __exit omap_dma_exit(void)
1663 {
1664         platform_driver_unregister(&omap_dma_driver);
1665 }
1666 module_exit(omap_dma_exit);
1667
1668 MODULE_AUTHOR("Russell King");
1669 MODULE_LICENSE("GPL");