drivers/dma/dw-edma/dw-edma-core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
   4  * Synopsys DesignWare eDMA core driver
   5  *
   6  * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/device.h>
  11 #include <linux/kernel.h>
  12 #include <linux/dmaengine.h>
  13 #include <linux/err.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/irq.h>
  16 #include <linux/dma/edma.h>
  17 #include <linux/dma-mapping.h>
  18
  19 #include "dw-edma-core.h"
  20 #include "dw-edma-v0-core.h"
  21 #include "dw-hdma-v0-core.h"
  22 #include "../dmaengine.h"
  23 #include "../virt-dma.h"
  24
  25 static inline
  26 struct device *dchan2dev(struct dma_chan *dchan)
  27 {
  28         return &dchan->dev->device;
  29 }
  30
  31 static inline
  32 struct device *chan2dev(struct dw_edma_chan *chan)
  33 {
  34         return &chan->vc.chan.dev->device;
  35 }
  36
  37 static inline
  38 struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
  39 {
  40         return container_of(vd, struct dw_edma_desc, vd);
  41 }
  42
  43 static inline
  44 u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr)
  45 {
  46         struct dw_edma_chip *chip = chan->dw->chip;
  47
  48         if (chip->ops->pci_address)
  49                 return chip->ops->pci_address(chip->dev, cpu_addr);
  50
  51         return cpu_addr;
  52 }
  53
  54 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
  55 {
  56         struct dw_edma_burst *burst;
  57
  58         burst = kzalloc(sizeof(*burst), GFP_NOWAIT);
  59         if (unlikely(!burst))
  60                 return NULL;
  61
  62         INIT_LIST_HEAD(&burst->list);
  63         if (chunk->burst) {
  64                 /* Create and add new element into the linked list */
  65                 chunk->bursts_alloc++;
  66                 list_add_tail(&burst->list, &chunk->burst->list);
  67         } else {
  68                 /* List head */
  69                 chunk->bursts_alloc = 0;
  70                 chunk->burst = burst;
  71         }
  72
  73         return burst;
  74 }
  75
  76 static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)
  77 {
  78         struct dw_edma_chip *chip = desc->chan->dw->chip;
  79         struct dw_edma_chan *chan = desc->chan;
  80         struct dw_edma_chunk *chunk;
  81
  82         chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT);
  83         if (unlikely(!chunk))
  84                 return NULL;
  85
  86         INIT_LIST_HEAD(&chunk->list);
  87         chunk->chan = chan;
  88         /* Toggling change bit (CB) in each chunk, this is a mechanism to
  89          * inform the eDMA HW block that this is a new linked list ready
  90          * to be consumed.
  91          *  - Odd chunks originate CB equal to 0
  92          *  - Even chunks originate CB equal to 1
  93          */
  94         chunk->cb = !(desc->chunks_alloc % 2);
  95         if (chan->dir == EDMA_DIR_WRITE) {
  96                 chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr;
  97                 chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr;
  98         } else {
  99                 chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr;
 100                 chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr;
 101         }
 102
 103         if (desc->chunk) {
 104                 /* Create and add new element into the linked list */
 105                 if (!dw_edma_alloc_burst(chunk)) {
 106                         kfree(chunk);
 107                         return NULL;
 108                 }
 109                 desc->chunks_alloc++;
 110                 list_add_tail(&chunk->list, &desc->chunk->list);
 111         } else {
 112                 /* List head */
 113                 chunk->burst = NULL;
 114                 desc->chunks_alloc = 0;
 115                 desc->chunk = chunk;
 116         }
 117
 118         return chunk;
 119 }
 120
 121 static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)
 122 {
 123         struct dw_edma_desc *desc;
 124
 125         desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
 126         if (unlikely(!desc))
 127                 return NULL;
 128
 129         desc->chan = chan;
 130         if (!dw_edma_alloc_chunk(desc)) {
 131                 kfree(desc);
 132                 return NULL;
 133         }
 134
 135         return desc;
 136 }
 137
 138 static void dw_edma_free_burst(struct dw_edma_chunk *chunk)
 139 {
 140         struct dw_edma_burst *child, *_next;
 141
 142         /* Remove all the list elements */
 143         list_for_each_entry_safe(child, _next, &chunk->burst->list, list) {
 144                 list_del(&child->list);
 145                 kfree(child);
 146                 chunk->bursts_alloc--;
 147         }
 148
 149         /* Remove the list head */
 150         kfree(child);
 151         chunk->burst = NULL;
 152 }
 153
 154 static void dw_edma_free_chunk(struct dw_edma_desc *desc)
 155 {
 156         struct dw_edma_chunk *child, *_next;
 157
 158         if (!desc->chunk)
 159                 return;
 160
 161         /* Remove all the list elements */
 162         list_for_each_entry_safe(child, _next, &desc->chunk->list, list) {
 163                 dw_edma_free_burst(child);
 164                 list_del(&child->list);
 165                 kfree(child);
 166                 desc->chunks_alloc--;
 167         }
 168
 169         /* Remove the list head */
 170         kfree(child);
 171         desc->chunk = NULL;
 172 }
 173
 174 static void dw_edma_free_desc(struct dw_edma_desc *desc)
 175 {
 176         dw_edma_free_chunk(desc);
 177         kfree(desc);
 178 }
 179
 180 static void vchan_free_desc(struct virt_dma_desc *vdesc)
 181 {
 182         dw_edma_free_desc(vd2dw_edma_desc(vdesc));
 183 }
 184
 185 static int dw_edma_start_transfer(struct dw_edma_chan *chan)
 186 {
 187         struct dw_edma *dw = chan->dw;
 188         struct dw_edma_chunk *child;
 189         struct dw_edma_desc *desc;
 190         struct virt_dma_desc *vd;
 191
 192         vd = vchan_next_desc(&chan->vc);
 193         if (!vd)
 194                 return 0;
 195
 196         desc = vd2dw_edma_desc(vd);
 197         if (!desc)
 198                 return 0;
 199
 200         child = list_first_entry_or_null(&desc->chunk->list,
 201                                          struct dw_edma_chunk, list);
 202         if (!child)
 203                 return 0;
 204
 205         dw_edma_core_start(dw, child, !desc->xfer_sz);
 206         desc->xfer_sz += child->ll_region.sz;
 207         dw_edma_free_burst(child);
 208         list_del(&child->list);
 209         kfree(child);
 210         desc->chunks_alloc--;
 211
 212         return 1;
 213 }
 214
 215 static void dw_edma_device_caps(struct dma_chan *dchan,
 216                                 struct dma_slave_caps *caps)
 217 {
 218         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 219
 220         if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
 221                 if (chan->dir == EDMA_DIR_READ)
 222                         caps->directions = BIT(DMA_DEV_TO_MEM);
 223                 else
 224                         caps->directions = BIT(DMA_MEM_TO_DEV);
 225         } else {
 226                 if (chan->dir == EDMA_DIR_WRITE)
 227                         caps->directions = BIT(DMA_DEV_TO_MEM);
 228                 else
 229                         caps->directions = BIT(DMA_MEM_TO_DEV);
 230         }
 231 }
 232
 233 static int dw_edma_device_config(struct dma_chan *dchan,
 234                                  struct dma_slave_config *config)
 235 {
 236         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 237
 238         memcpy(&chan->config, config, sizeof(*config));
 239         chan->configured = true;
 240
 241         return 0;
 242 }
 243
 244 static int dw_edma_device_pause(struct dma_chan *dchan)
 245 {
 246         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 247         int err = 0;
 248
 249         if (!chan->configured)
 250                 err = -EPERM;
 251         else if (chan->status != EDMA_ST_BUSY)
 252                 err = -EPERM;
 253         else if (chan->request != EDMA_REQ_NONE)
 254                 err = -EPERM;
 255         else
 256                 chan->request = EDMA_REQ_PAUSE;
 257
 258         return err;
 259 }
 260
 261 static int dw_edma_device_resume(struct dma_chan *dchan)
 262 {
 263         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 264         int err = 0;
 265
 266         if (!chan->configured) {
 267                 err = -EPERM;
 268         } else if (chan->status != EDMA_ST_PAUSE) {
 269                 err = -EPERM;
 270         } else if (chan->request != EDMA_REQ_NONE) {
 271                 err = -EPERM;
 272         } else {
 273                 chan->status = EDMA_ST_BUSY;
 274                 dw_edma_start_transfer(chan);
 275         }
 276
 277         return err;
 278 }
 279
 280 static int dw_edma_device_terminate_all(struct dma_chan *dchan)
 281 {
 282         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 283         int err = 0;
 284
 285         if (!chan->configured) {
 286                 /* Do nothing */
 287         } else if (chan->status == EDMA_ST_PAUSE) {
 288                 chan->status = EDMA_ST_IDLE;
 289                 chan->configured = false;
 290         } else if (chan->status == EDMA_ST_IDLE) {
 291                 chan->configured = false;
 292         } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) {
 293                 /*
 294                  * The channel is in a false BUSY state, probably didn't
 295                  * receive or lost an interrupt
 296                  */
 297                 chan->status = EDMA_ST_IDLE;
 298                 chan->configured = false;
 299         } else if (chan->request > EDMA_REQ_PAUSE) {
 300                 err = -EPERM;
 301         } else {
 302                 chan->request = EDMA_REQ_STOP;
 303         }
 304
 305         return err;
 306 }
 307
 308 static void dw_edma_device_issue_pending(struct dma_chan *dchan)
 309 {
 310         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 311         unsigned long flags;
 312
 313         if (!chan->configured)
 314                 return;
 315
 316         spin_lock_irqsave(&chan->vc.lock, flags);
 317         if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE &&
 318             chan->status == EDMA_ST_IDLE) {
 319                 chan->status = EDMA_ST_BUSY;
 320                 dw_edma_start_transfer(chan);
 321         }
 322         spin_unlock_irqrestore(&chan->vc.lock, flags);
 323 }
 324
 325 static enum dma_status
 326 dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
 327                          struct dma_tx_state *txstate)
 328 {
 329         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 330         struct dw_edma_desc *desc;
 331         struct virt_dma_desc *vd;
 332         unsigned long flags;
 333         enum dma_status ret;
 334         u32 residue = 0;
 335
 336         ret = dma_cookie_status(dchan, cookie, txstate);
 337         if (ret == DMA_COMPLETE)
 338                 return ret;
 339
 340         if (ret == DMA_IN_PROGRESS && chan->status == EDMA_ST_PAUSE)
 341                 ret = DMA_PAUSED;
 342
 343         if (!txstate)
 344                 goto ret_residue;
 345
 346         spin_lock_irqsave(&chan->vc.lock, flags);
 347         vd = vchan_find_desc(&chan->vc, cookie);
 348         if (vd) {
 349                 desc = vd2dw_edma_desc(vd);
 350                 if (desc)
 351                         residue = desc->alloc_sz - desc->xfer_sz;
 352         }
 353         spin_unlock_irqrestore(&chan->vc.lock, flags);
 354
 355 ret_residue:
 356         dma_set_residue(txstate, residue);
 357
 358         return ret;
 359 }
 360
 361 static struct dma_async_tx_descriptor *
 362 dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 363 {
 364         struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
 365         enum dma_transfer_direction dir = xfer->direction;
 366         struct scatterlist *sg = NULL;
 367         struct dw_edma_chunk *chunk;
 368         struct dw_edma_burst *burst;
 369         struct dw_edma_desc *desc;
 370         u64 src_addr, dst_addr;
 371         size_t fsz = 0;
 372         u32 cnt = 0;
 373         int i;
 374
 375         if (!chan->configured)
 376                 return NULL;
 377
 378         /*
 379          * Local Root Port/End-point              Remote End-point
 380          * +-----------------------+ PCIe bus +----------------------+
 381          * |                       |    +-+   |                      |
 382          * |    DEV_TO_MEM   Rx Ch <----+ +---+ Tx Ch  DEV_TO_MEM    |
 383          * |                       |    | |   |                      |
 384          * |    MEM_TO_DEV   Tx Ch +----+ +---> Rx Ch  MEM_TO_DEV    |
 385          * |                       |    +-+   |                      |
 386          * +-----------------------+          +----------------------+
 387          *
 388          * 1. Normal logic:
 389          * If eDMA is embedded into the DW PCIe RP/EP and controlled from the
 390          * CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used
 391          * for the device read operations (DEV_TO_MEM) and the Tx channel
 392          * (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV).
 393          *
 394          * 2. Inverted logic:
 395          * If eDMA is embedded into a Remote PCIe EP and is controlled by the
 396          * MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx
 397          * channel (EDMA_DIR_WRITE) will be used for the device read operations
 398          * (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write
 399          * operations (MEM_TO_DEV).
 400          *
 401          * It is the client driver responsibility to choose a proper channel
 402          * for the DMA transfers.
 403          */
 404         if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
 405                 if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) ||
 406                     (chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV))
 407                         return NULL;
 408         } else {
 409                 if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) ||
 410                     (chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV))
 411                         return NULL;
 412         }
 413
 414         if (xfer->type == EDMA_XFER_CYCLIC) {
 415                 if (!xfer->xfer.cyclic.len || !xfer->xfer.cyclic.cnt)
 416                         return NULL;
 417         } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
 418                 if (xfer->xfer.sg.len < 1)
 419                         return NULL;
 420         } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
 421                 if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1)
 422                         return NULL;
 423                 if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc)
 424                         return NULL;
 425         } else {
 426                 return NULL;
 427         }
 428
 429         desc = dw_edma_alloc_desc(chan);
 430         if (unlikely(!desc))
 431                 goto err_alloc;
 432
 433         chunk = dw_edma_alloc_chunk(desc);
 434         if (unlikely(!chunk))
 435                 goto err_alloc;
 436
 437         if (xfer->type == EDMA_XFER_INTERLEAVED) {
 438                 src_addr = xfer->xfer.il->src_start;
 439                 dst_addr = xfer->xfer.il->dst_start;
 440         } else {
 441                 src_addr = chan->config.src_addr;
 442                 dst_addr = chan->config.dst_addr;
 443         }
 444
 445         if (dir == DMA_DEV_TO_MEM)
 446                 src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr);
 447         else
 448                 dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr);
 449
 450         if (xfer->type == EDMA_XFER_CYCLIC) {
 451                 cnt = xfer->xfer.cyclic.cnt;
 452         } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
 453                 cnt = xfer->xfer.sg.len;
 454                 sg = xfer->xfer.sg.sgl;
 455         } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
 456                 cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size;
 457                 fsz = xfer->xfer.il->frame_size;
 458         }
 459
 460         for (i = 0; i < cnt; i++) {
 461                 if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
 462                         break;
 463
 464                 if (chunk->bursts_alloc == chan->ll_max) {
 465                         chunk = dw_edma_alloc_chunk(desc);
 466                         if (unlikely(!chunk))
 467                                 goto err_alloc;
 468                 }
 469
 470                 burst = dw_edma_alloc_burst(chunk);
 471                 if (unlikely(!burst))
 472                         goto err_alloc;
 473
 474                 if (xfer->type == EDMA_XFER_CYCLIC)
 475                         burst->sz = xfer->xfer.cyclic.len;
 476                 else if (xfer->type == EDMA_XFER_SCATTER_GATHER)
 477                         burst->sz = sg_dma_len(sg);
 478                 else if (xfer->type == EDMA_XFER_INTERLEAVED)
 479                         burst->sz = xfer->xfer.il->sgl[i % fsz].size;
 480
 481                 chunk->ll_region.sz += burst->sz;
 482                 desc->alloc_sz += burst->sz;
 483
 484                 if (dir == DMA_DEV_TO_MEM) {
 485                         burst->sar = src_addr;
 486                         if (xfer->type == EDMA_XFER_CYCLIC) {
 487                                 burst->dar = xfer->xfer.cyclic.paddr;
 488                         } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
 489                                 src_addr += sg_dma_len(sg);
 490                                 burst->dar = sg_dma_address(sg);
 491                                 /* Unlike the typical assumption by other
 492                                  * drivers/IPs the peripheral memory isn't
 493                                  * a FIFO memory, in this case, it's a
 494                                  * linear memory and that why the source
 495                                  * and destination addresses are increased
 496                                  * by the same portion (data length)
 497                                  */
 498                         } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
 499                                 burst->dar = dst_addr;
 500                         }
 501                 } else {
 502                         burst->dar = dst_addr;
 503                         if (xfer->type == EDMA_XFER_CYCLIC) {
 504                                 burst->sar = xfer->xfer.cyclic.paddr;
 505                         } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
 506                                 dst_addr += sg_dma_len(sg);
 507                                 burst->sar = sg_dma_address(sg);
 508                                 /* Unlike the typical assumption by other
 509                                  * drivers/IPs the peripheral memory isn't
 510                                  * a FIFO memory, in this case, it's a
 511                                  * linear memory and that why the source
 512                                  * and destination addresses are increased
 513                                  * by the same portion (data length)
 514                                  */
 515                         }  else if (xfer->type == EDMA_XFER_INTERLEAVED) {
 516                                 burst->sar = src_addr;
 517                         }
 518                 }
 519
 520                 if (xfer->type == EDMA_XFER_SCATTER_GATHER) {
 521                         sg = sg_next(sg);
 522                 } else if (xfer->type == EDMA_XFER_INTERLEAVED) {
 523                         struct dma_interleaved_template *il = xfer->xfer.il;
 524                         struct data_chunk *dc = &il->sgl[i % fsz];
 525
 526                         src_addr += burst->sz;
 527                         if (il->src_sgl)
 528                                 src_addr += dmaengine_get_src_icg(il, dc);
 529
 530                         dst_addr += burst->sz;
 531                         if (il->dst_sgl)
 532                                 dst_addr += dmaengine_get_dst_icg(il, dc);
 533                 }
 534         }
 535
 536         return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);
 537
 538 err_alloc:
 539         if (desc)
 540                 dw_edma_free_desc(desc);
 541
 542         return NULL;
 543 }
 544
 545 static struct dma_async_tx_descriptor *
 546 dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
 547                              unsigned int len,
 548                              enum dma_transfer_direction direction,
 549                              unsigned long flags, void *context)
 550 {
 551         struct dw_edma_transfer xfer;
 552
 553         xfer.dchan = dchan;
 554         xfer.direction = direction;
 555         xfer.xfer.sg.sgl = sgl;
 556         xfer.xfer.sg.len = len;
 557         xfer.flags = flags;
 558         xfer.type = EDMA_XFER_SCATTER_GATHER;
 559
 560         return dw_edma_device_transfer(&xfer);
 561 }
 562
 563 static struct dma_async_tx_descriptor *
 564 dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,
 565                                size_t len, size_t count,
 566                                enum dma_transfer_direction direction,
 567                                unsigned long flags)
 568 {
 569         struct dw_edma_transfer xfer;
 570
 571         xfer.dchan = dchan;
 572         xfer.direction = direction;
 573         xfer.xfer.cyclic.paddr = paddr;
 574         xfer.xfer.cyclic.len = len;
 575         xfer.xfer.cyclic.cnt = count;
 576         xfer.flags = flags;
 577         xfer.type = EDMA_XFER_CYCLIC;
 578
 579         return dw_edma_device_transfer(&xfer);
 580 }
 581
 582 static struct dma_async_tx_descriptor *
 583 dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
 584                                     struct dma_interleaved_template *ilt,
 585                                     unsigned long flags)
 586 {
 587         struct dw_edma_transfer xfer;
 588
 589         xfer.dchan = dchan;
 590         xfer.direction = ilt->dir;
 591         xfer.xfer.il = ilt;
 592         xfer.flags = flags;
 593         xfer.type = EDMA_XFER_INTERLEAVED;
 594
 595         return dw_edma_device_transfer(&xfer);
 596 }
 597
 598 static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
 599 {
 600         struct dw_edma_desc *desc;
 601         struct virt_dma_desc *vd;
 602         unsigned long flags;
 603
 604         spin_lock_irqsave(&chan->vc.lock, flags);
 605         vd = vchan_next_desc(&chan->vc);
 606         if (vd) {
 607                 switch (chan->request) {
 608                 case EDMA_REQ_NONE:
 609                         desc = vd2dw_edma_desc(vd);
 610                         if (!desc->chunks_alloc) {
 611                                 list_del(&vd->node);
 612                                 vchan_cookie_complete(vd);
 613                         }
 614
 615                         /* Continue transferring if there are remaining chunks or issued requests.
 616                          */
 617                         chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE;
 618                         break;
 619
 620                 case EDMA_REQ_STOP:
 621                         list_del(&vd->node);
 622                         vchan_cookie_complete(vd);
 623                         chan->request = EDMA_REQ_NONE;
 624                         chan->status = EDMA_ST_IDLE;
 625                         break;
 626
 627                 case EDMA_REQ_PAUSE:
 628                         chan->request = EDMA_REQ_NONE;
 629                         chan->status = EDMA_ST_PAUSE;
 630                         break;
 631
 632                 default:
 633                         break;
 634                 }
 635         }
 636         spin_unlock_irqrestore(&chan->vc.lock, flags);
 637 }
 638
 639 static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
 640 {
 641         struct virt_dma_desc *vd;
 642         unsigned long flags;
 643
 644         spin_lock_irqsave(&chan->vc.lock, flags);
 645         vd = vchan_next_desc(&chan->vc);
 646         if (vd) {
 647                 list_del(&vd->node);
 648                 vchan_cookie_complete(vd);
 649         }
 650         spin_unlock_irqrestore(&chan->vc.lock, flags);
 651         chan->request = EDMA_REQ_NONE;
 652         chan->status = EDMA_ST_IDLE;
 653 }
 654
 655 static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
 656 {
 657         struct dw_edma_irq *dw_irq = data;
 658
 659         return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE,
 660                                        dw_edma_done_interrupt,
 661                                        dw_edma_abort_interrupt);
 662 }
 663
 664 static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data)
 665 {
 666         struct dw_edma_irq *dw_irq = data;
 667
 668         return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ,
 669                                        dw_edma_done_interrupt,
 670                                        dw_edma_abort_interrupt);
 671 }
 672
 673 static irqreturn_t dw_edma_interrupt_common(int irq, void *data)
 674 {
 675         irqreturn_t ret = IRQ_NONE;
 676
 677         ret |= dw_edma_interrupt_write(irq, data);
 678         ret |= dw_edma_interrupt_read(irq, data);
 679
 680         return ret;
 681 }
 682
 683 static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)
 684 {
 685         struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
 686
 687         if (chan->status != EDMA_ST_IDLE)
 688                 return -EBUSY;
 689
 690         return 0;
 691 }
 692
 693 static void dw_edma_free_chan_resources(struct dma_chan *dchan)
 694 {
 695         unsigned long timeout = jiffies + msecs_to_jiffies(5000);
 696         int ret;
 697
 698         while (time_before(jiffies, timeout)) {
 699                 ret = dw_edma_device_terminate_all(dchan);
 700                 if (!ret)
 701                         break;
 702
 703                 if (time_after_eq(jiffies, timeout))
 704                         return;
 705
 706                 cpu_relax();
 707         }
 708 }
 709
 710 static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
 711 {
 712         struct dw_edma_chip *chip = dw->chip;
 713         struct device *dev = chip->dev;
 714         struct dw_edma_chan *chan;
 715         struct dw_edma_irq *irq;
 716         struct dma_device *dma;
 717         u32 i, ch_cnt;
 718         u32 pos;
 719
 720         ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
 721         dma = &dw->dma;
 722
 723         INIT_LIST_HEAD(&dma->channels);
 724
 725         for (i = 0; i < ch_cnt; i++) {
 726                 chan = &dw->chan[i];
 727
 728                 chan->dw = dw;
 729
 730                 if (i < dw->wr_ch_cnt) {
 731                         chan->id = i;
 732                         chan->dir = EDMA_DIR_WRITE;
 733                 } else {
 734                         chan->id = i - dw->wr_ch_cnt;
 735                         chan->dir = EDMA_DIR_READ;
 736                 }
 737
 738                 chan->configured = false;
 739                 chan->request = EDMA_REQ_NONE;
 740                 chan->status = EDMA_ST_IDLE;
 741
 742                 if (chan->dir == EDMA_DIR_WRITE)
 743                         chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ);
 744                 else
 745                         chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ);
 746                 chan->ll_max -= 1;
 747
 748                 dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
 749                          chan->dir == EDMA_DIR_WRITE ? "write" : "read",
 750                          chan->id, chan->ll_max);
 751
 752                 if (dw->nr_irqs == 1)
 753                         pos = 0;
 754                 else if (chan->dir == EDMA_DIR_WRITE)
 755                         pos = chan->id % wr_alloc;
 756                 else
 757                         pos = wr_alloc + chan->id % rd_alloc;
 758
 759                 irq = &dw->irq[pos];
 760
 761                 if (chan->dir == EDMA_DIR_WRITE)
 762                         irq->wr_mask |= BIT(chan->id);
 763                 else
 764                         irq->rd_mask |= BIT(chan->id);
 765
 766                 irq->dw = dw;
 767                 memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));
 768
 769                 dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n",
 770                          chan->dir == EDMA_DIR_WRITE  ? "write" : "read", chan->id,
 771                          chan->msi.address_hi, chan->msi.address_lo,
 772                          chan->msi.data);
 773
 774                 chan->vc.desc_free = vchan_free_desc;
 775                 chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ?
 776                                         &dw->chip->dt_region_wr[chan->id] :
 777                                         &dw->chip->dt_region_rd[chan->id];
 778
 779                 vchan_init(&chan->vc, dma);
 780
 781                 dw_edma_core_ch_config(chan);
 782         }
 783
 784         /* Set DMA channel capabilities */
 785         dma_cap_zero(dma->cap_mask);
 786         dma_cap_set(DMA_SLAVE, dma->cap_mask);
 787         dma_cap_set(DMA_CYCLIC, dma->cap_mask);
 788         dma_cap_set(DMA_PRIVATE, dma->cap_mask);
 789         dma_cap_set(DMA_INTERLEAVE, dma->cap_mask);
 790         dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 791         dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
 792         dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
 793         dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 794
 795         /* Set DMA channel callbacks */
 796         dma->dev = chip->dev;
 797         dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources;
 798         dma->device_free_chan_resources = dw_edma_free_chan_resources;
 799         dma->device_caps = dw_edma_device_caps;
 800         dma->device_config = dw_edma_device_config;
 801         dma->device_pause = dw_edma_device_pause;
 802         dma->device_resume = dw_edma_device_resume;
 803         dma->device_terminate_all = dw_edma_device_terminate_all;
 804         dma->device_issue_pending = dw_edma_device_issue_pending;
 805         dma->device_tx_status = dw_edma_device_tx_status;
 806         dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;
 807         dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
 808         dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma;
 809
 810         dma_set_max_seg_size(dma->dev, U32_MAX);
 811
 812         /* Register DMA device */
 813         return dma_async_device_register(dma);
 814 }
 815
 816 static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt)
 817 {
 818         if (*nr_irqs && *alloc < cnt) {
 819                 (*alloc)++;
 820                 (*nr_irqs)--;
 821         }
 822 }
 823
 824 static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt)
 825 {
 826         while (*mask * alloc < cnt)
 827                 (*mask)++;
 828 }
 829
 830 static int dw_edma_irq_request(struct dw_edma *dw,
 831                                u32 *wr_alloc, u32 *rd_alloc)
 832 {
 833         struct dw_edma_chip *chip = dw->chip;
 834         struct device *dev = dw->chip->dev;
 835         u32 wr_mask = 1;
 836         u32 rd_mask = 1;
 837         int i, err = 0;
 838         u32 ch_cnt;
 839         int irq;
 840
 841         ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt;
 842
 843         if (chip->nr_irqs < 1 || !chip->ops->irq_vector)
 844                 return -EINVAL;
 845
 846         dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL);
 847         if (!dw->irq)
 848                 return -ENOMEM;
 849
 850         if (chip->nr_irqs == 1) {
 851                 /* Common IRQ shared among all channels */
 852                 irq = chip->ops->irq_vector(dev, 0);
 853                 err = request_irq(irq, dw_edma_interrupt_common,
 854                                   IRQF_SHARED, dw->name, &dw->irq[0]);
 855                 if (err) {
 856                         dw->nr_irqs = 0;
 857                         return err;
 858                 }
 859
 860                 if (irq_get_msi_desc(irq))
 861                         get_cached_msi_msg(irq, &dw->irq[0].msi);
 862
 863                 dw->nr_irqs = 1;
 864         } else {
 865                 /* Distribute IRQs equally among all channels */
 866                 int tmp = chip->nr_irqs;
 867
 868                 while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) {
 869                         dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt);
 870                         dw_edma_dec_irq_alloc(&tmp, rd_alloc, dw->rd_ch_cnt);
 871                 }
 872
 873                 dw_edma_add_irq_mask(&wr_mask, *wr_alloc, dw->wr_ch_cnt);
 874                 dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt);
 875
 876                 for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {
 877                         irq = chip->ops->irq_vector(dev, i);
 878                         err = request_irq(irq,
 879                                           i < *wr_alloc ?
 880                                                 dw_edma_interrupt_write :
 881                                                 dw_edma_interrupt_read,
 882                                           IRQF_SHARED, dw->name,
 883                                           &dw->irq[i]);
 884                         if (err)
 885                                 goto err_irq_free;
 886
 887                         if (irq_get_msi_desc(irq))
 888                                 get_cached_msi_msg(irq, &dw->irq[i].msi);
 889                 }
 890
 891                 dw->nr_irqs = i;
 892         }
 893
 894         return 0;
 895
 896 err_irq_free:
 897         for  (i--; i >= 0; i--) {
 898                 irq = chip->ops->irq_vector(dev, i);
 899                 free_irq(irq, &dw->irq[i]);
 900         }
 901
 902         return err;
 903 }
 904
 905 int dw_edma_probe(struct dw_edma_chip *chip)
 906 {
 907         struct device *dev;
 908         struct dw_edma *dw;
 909         u32 wr_alloc = 0;
 910         u32 rd_alloc = 0;
 911         int i, err;
 912
 913         if (!chip)
 914                 return -EINVAL;
 915
 916         dev = chip->dev;
 917         if (!dev || !chip->ops)
 918                 return -EINVAL;
 919
 920         dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL);
 921         if (!dw)
 922                 return -ENOMEM;
 923
 924         dw->chip = chip;
 925
 926         if (dw->chip->mf == EDMA_MF_HDMA_NATIVE)
 927                 dw_hdma_v0_core_register(dw);
 928         else
 929                 dw_edma_v0_core_register(dw);
 930
 931         raw_spin_lock_init(&dw->lock);
 932
 933         dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt,
 934                               dw_edma_core_ch_count(dw, EDMA_DIR_WRITE));
 935         dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH);
 936
 937         dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt,
 938                               dw_edma_core_ch_count(dw, EDMA_DIR_READ));
 939         dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH);
 940
 941         if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)
 942                 return -EINVAL;
 943
 944         dev_vdbg(dev, "Channels:\twrite=%d, read=%d\n",
 945                  dw->wr_ch_cnt, dw->rd_ch_cnt);
 946
 947         /* Allocate channels */
 948         dw->chan = devm_kcalloc(dev, dw->wr_ch_cnt + dw->rd_ch_cnt,
 949                                 sizeof(*dw->chan), GFP_KERNEL);
 950         if (!dw->chan)
 951                 return -ENOMEM;
 952
 953         snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s",
 954                  dev_name(chip->dev));
 955
 956         /* Disable eDMA, only to establish the ideal initial conditions */
 957         dw_edma_core_off(dw);
 958
 959         /* Request IRQs */
 960         err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc);
 961         if (err)
 962                 return err;
 963
 964         /* Setup write/read channels */
 965         err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc);
 966         if (err)
 967                 goto err_irq_free;
 968
 969         /* Turn debugfs on */
 970         dw_edma_core_debugfs_on(dw);
 971
 972         chip->dw = dw;
 973
 974         return 0;
 975
 976 err_irq_free:
 977         for (i = (dw->nr_irqs - 1); i >= 0; i--)
 978                 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
 979
 980         return err;
 981 }
 982 EXPORT_SYMBOL_GPL(dw_edma_probe);
 983
 984 int dw_edma_remove(struct dw_edma_chip *chip)
 985 {
 986         struct dw_edma_chan *chan, *_chan;
 987         struct device *dev = chip->dev;
 988         struct dw_edma *dw = chip->dw;
 989         int i;
 990
 991         /* Skip removal if no private data found */
 992         if (!dw)
 993                 return -ENODEV;
 994
 995         /* Disable eDMA */
 996         dw_edma_core_off(dw);
 997
 998         /* Free irqs */
 999         for (i = (dw->nr_irqs - 1); i >= 0; i--)
1000                 free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
1001
1002         /* Deregister eDMA device */
1003         dma_async_device_unregister(&dw->dma);
1004         list_for_each_entry_safe(chan, _chan, &dw->dma.channels,
1005                                  vc.chan.device_node) {
1006                 tasklet_kill(&chan->vc.task);
1007                 list_del(&chan->vc.chan.device_node);
1008         }
1009
1010         return 0;
1011 }
1012 EXPORT_SYMBOL_GPL(dw_edma_remove);
1013
1014 MODULE_LICENSE("GPL v2");
1015 MODULE_DESCRIPTION("Synopsys DesignWare eDMA controller core driver");
1016 MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>");