1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates.
4 * Synopsys DesignWare eDMA v0 core
6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
9 #include <linux/bitfield.h>
10 #include <linux/irqreturn.h>
11 #include <linux/io-64-nonatomic-lo-hi.h>
13 #include "dw-edma-core.h"
14 #include "dw-edma-v0-core.h"
15 #include "dw-edma-v0-regs.h"
16 #include "dw-edma-v0-debugfs.h"
18 enum dw_edma_control {
19 DW_EDMA_V0_CB = BIT(0),
20 DW_EDMA_V0_TCB = BIT(1),
21 DW_EDMA_V0_LLP = BIT(2),
22 DW_EDMA_V0_LIE = BIT(3),
23 DW_EDMA_V0_RIE = BIT(4),
24 DW_EDMA_V0_CCS = BIT(8),
25 DW_EDMA_V0_LLE = BIT(9),
28 static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)
30 return dw->chip->reg_base;
33 #define SET_32(dw, name, value) \
34 writel(value, &(__dw_regs(dw)->name))
36 #define GET_32(dw, name) \
37 readl(&(__dw_regs(dw)->name))
39 #define SET_RW_32(dw, dir, name, value) \
41 if ((dir) == EDMA_DIR_WRITE) \
42 SET_32(dw, wr_##name, value); \
44 SET_32(dw, rd_##name, value); \
47 #define GET_RW_32(dw, dir, name) \
48 ((dir) == EDMA_DIR_WRITE \
49 ? GET_32(dw, wr_##name) \
50 : GET_32(dw, rd_##name))
52 #define SET_BOTH_32(dw, name, value) \
54 SET_32(dw, wr_##name, value); \
55 SET_32(dw, rd_##name, value); \
58 #define SET_64(dw, name, value) \
59 writeq(value, &(__dw_regs(dw)->name))
61 #define GET_64(dw, name) \
62 readq(&(__dw_regs(dw)->name))
64 #define SET_RW_64(dw, dir, name, value) \
66 if ((dir) == EDMA_DIR_WRITE) \
67 SET_64(dw, wr_##name, value); \
69 SET_64(dw, rd_##name, value); \
72 #define GET_RW_64(dw, dir, name) \
73 ((dir) == EDMA_DIR_WRITE \
74 ? GET_64(dw, wr_##name) \
75 : GET_64(dw, rd_##name))
77 #define SET_BOTH_64(dw, name, value) \
79 SET_64(dw, wr_##name, value); \
80 SET_64(dw, rd_##name, value); \
83 #define SET_COMPAT(dw, name, value) \
84 writel(value, &(__dw_regs(dw)->type.unroll.name))
86 #define SET_RW_COMPAT(dw, dir, name, value) \
88 if ((dir) == EDMA_DIR_WRITE) \
89 SET_COMPAT(dw, wr_##name, value); \
91 SET_COMPAT(dw, rd_##name, value); \
94 static inline struct dw_edma_v0_ch_regs __iomem *
95 __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
97 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY)
98 return &(__dw_regs(dw)->type.legacy.ch);
100 if (dir == EDMA_DIR_WRITE)
101 return &__dw_regs(dw)->type.unroll.ch[ch].wr;
103 return &__dw_regs(dw)->type.unroll.ch[ch].rd;
106 static inline void writel_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
107 u32 value, void __iomem *addr)
109 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) {
113 raw_spin_lock_irqsave(&dw->lock, flags);
115 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
116 if (dir == EDMA_DIR_READ)
117 viewport_sel |= BIT(31);
120 &(__dw_regs(dw)->type.legacy.viewport_sel));
123 raw_spin_unlock_irqrestore(&dw->lock, flags);
129 static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
130 const void __iomem *addr)
134 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) {
138 raw_spin_lock_irqsave(&dw->lock, flags);
140 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch);
141 if (dir == EDMA_DIR_READ)
142 viewport_sel |= BIT(31);
145 &(__dw_regs(dw)->type.legacy.viewport_sel));
148 raw_spin_unlock_irqrestore(&dw->lock, flags);
156 #define SET_CH_32(dw, dir, ch, name, value) \
157 writel_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name))
159 #define GET_CH_32(dw, dir, ch, name) \
160 readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))
162 /* eDMA management callbacks */
163 static void dw_edma_v0_core_off(struct dw_edma *dw)
165 SET_BOTH_32(dw, int_mask,
166 EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
167 SET_BOTH_32(dw, int_clear,
168 EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
169 SET_BOTH_32(dw, engine_en, 0);
172 static u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
176 if (dir == EDMA_DIR_WRITE)
177 num_ch = FIELD_GET(EDMA_V0_WRITE_CH_COUNT_MASK,
180 num_ch = FIELD_GET(EDMA_V0_READ_CH_COUNT_MASK,
183 if (num_ch > EDMA_V0_MAX_NR_CH)
184 num_ch = EDMA_V0_MAX_NR_CH;
189 static enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan)
191 struct dw_edma *dw = chan->dw;
194 tmp = FIELD_GET(EDMA_V0_CH_STATUS_MASK,
195 GET_CH_32(dw, chan->dir, chan->id, ch_control1));
198 return DMA_IN_PROGRESS;
205 static void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan)
207 struct dw_edma *dw = chan->dw;
209 SET_RW_32(dw, chan->dir, int_clear,
210 FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)));
213 static void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
215 struct dw_edma *dw = chan->dw;
217 SET_RW_32(dw, chan->dir, int_clear,
218 FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)));
221 static u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir)
223 return FIELD_GET(EDMA_V0_DONE_INT_MASK,
224 GET_RW_32(dw, dir, int_status));
227 static u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir)
229 return FIELD_GET(EDMA_V0_ABORT_INT_MASK,
230 GET_RW_32(dw, dir, int_status));
234 dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
235 dw_edma_handler_t done, dw_edma_handler_t abort)
237 struct dw_edma *dw = dw_irq->dw;
238 unsigned long total, pos, val;
239 irqreturn_t ret = IRQ_NONE;
240 struct dw_edma_chan *chan;
244 if (dir == EDMA_DIR_WRITE) {
245 total = dw->wr_ch_cnt;
247 mask = dw_irq->wr_mask;
249 total = dw->rd_ch_cnt;
251 mask = dw_irq->rd_mask;
254 val = dw_edma_v0_core_status_done_int(dw, dir);
256 for_each_set_bit(pos, &val, total) {
257 chan = &dw->chan[pos + off];
259 dw_edma_v0_core_clear_done_int(chan);
265 val = dw_edma_v0_core_status_abort_int(dw, dir);
267 for_each_set_bit(pos, &val, total) {
268 chan = &dw->chan[pos + off];
270 dw_edma_v0_core_clear_abort_int(chan);
279 static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,
280 u32 control, u32 size, u64 sar, u64 dar)
282 ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);
284 if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
285 struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;
287 lli->control = control;
288 lli->transfer_size = size;
292 struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;
294 writel(control, &lli->control);
295 writel(size, &lli->transfer_size);
296 writeq(sar, &lli->sar.reg);
297 writeq(dar, &lli->dar.reg);
301 static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk,
302 int i, u32 control, u64 pointer)
304 ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);
306 if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
307 struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;
309 llp->control = control;
310 llp->llp.reg = pointer;
312 struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;
314 writel(control, &llp->control);
315 writeq(pointer, &llp->llp.reg);
319 static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
321 struct dw_edma_burst *child;
322 struct dw_edma_chan *chan = chunk->chan;
323 u32 control = 0, i = 0;
327 control = DW_EDMA_V0_CB;
329 j = chunk->bursts_alloc;
330 list_for_each_entry(child, &chunk->burst->list, list) {
333 control |= DW_EDMA_V0_LIE;
334 if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
335 control |= DW_EDMA_V0_RIE;
338 dw_edma_v0_write_ll_data(chunk, i++, control, child->sz,
339 child->sar, child->dar);
342 control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB;
344 control |= DW_EDMA_V0_CB;
346 dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);
349 static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
352 * In case of remote eDMA engine setup, the DW PCIe RP/EP internal
353 * configuration registers and application memory are normally accessed
354 * over different buses. Ensure LL-data reaches the memory before the
355 * doorbell register is toggled by issuing the dummy-read from the remote
356 * LL memory in a hope that the MRd TLP will return only after the
357 * last MWr TLP is completed
359 if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
360 readl(chunk->ll_region.vaddr.io);
363 static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
365 struct dw_edma_chan *chan = chunk->chan;
366 struct dw_edma *dw = chan->dw;
369 dw_edma_v0_core_write_chunk(chunk);
373 SET_RW_32(dw, chan->dir, engine_en, BIT(0));
374 if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) {
377 SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en,
381 SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en,
385 SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en,
389 SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en,
393 SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en,
397 SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en,
401 SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en,
405 SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en,
410 /* Interrupt unmask - done, abort */
411 tmp = GET_RW_32(dw, chan->dir, int_mask);
412 tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));
413 tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));
414 SET_RW_32(dw, chan->dir, int_mask, tmp);
415 /* Linked list error */
416 tmp = GET_RW_32(dw, chan->dir, linked_list_err_en);
417 tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id));
418 SET_RW_32(dw, chan->dir, linked_list_err_en, tmp);
419 /* Channel control */
420 SET_CH_32(dw, chan->dir, chan->id, ch_control1,
421 (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));
423 /* llp is not aligned on 64bit -> keep 32bit accesses */
424 SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
425 lower_32_bits(chunk->ll_region.paddr));
426 SET_CH_32(dw, chan->dir, chan->id, llp.msb,
427 upper_32_bits(chunk->ll_region.paddr));
430 dw_edma_v0_sync_ll_data(chunk);
433 SET_RW_32(dw, chan->dir, doorbell,
434 FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
437 static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan)
439 struct dw_edma *dw = chan->dw;
442 /* MSI done addr - low, high */
443 SET_RW_32(dw, chan->dir, done_imwr.lsb, chan->msi.address_lo);
444 SET_RW_32(dw, chan->dir, done_imwr.msb, chan->msi.address_hi);
445 /* MSI abort addr - low, high */
446 SET_RW_32(dw, chan->dir, abort_imwr.lsb, chan->msi.address_lo);
447 SET_RW_32(dw, chan->dir, abort_imwr.msb, chan->msi.address_hi);
448 /* MSI data - low, high */
452 tmp = GET_RW_32(dw, chan->dir, ch01_imwr_data);
457 tmp = GET_RW_32(dw, chan->dir, ch23_imwr_data);
462 tmp = GET_RW_32(dw, chan->dir, ch45_imwr_data);
467 tmp = GET_RW_32(dw, chan->dir, ch67_imwr_data);
471 if (chan->id & BIT(0)) {
472 /* Channel odd {1, 3, 5, 7} */
473 tmp &= EDMA_V0_CH_EVEN_MSI_DATA_MASK;
474 tmp |= FIELD_PREP(EDMA_V0_CH_ODD_MSI_DATA_MASK,
477 /* Channel even {0, 2, 4, 6} */
478 tmp &= EDMA_V0_CH_ODD_MSI_DATA_MASK;
479 tmp |= FIELD_PREP(EDMA_V0_CH_EVEN_MSI_DATA_MASK,
486 SET_RW_32(dw, chan->dir, ch01_imwr_data, tmp);
491 SET_RW_32(dw, chan->dir, ch23_imwr_data, tmp);
496 SET_RW_32(dw, chan->dir, ch45_imwr_data, tmp);
501 SET_RW_32(dw, chan->dir, ch67_imwr_data, tmp);
506 /* eDMA debugfs callbacks */
507 static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw)
509 dw_edma_v0_debugfs_on(dw);
512 static const struct dw_edma_core_ops dw_edma_v0_core = {
513 .off = dw_edma_v0_core_off,
514 .ch_count = dw_edma_v0_core_ch_count,
515 .ch_status = dw_edma_v0_core_ch_status,
516 .handle_int = dw_edma_v0_core_handle_int,
517 .start = dw_edma_v0_core_start,
518 .ch_config = dw_edma_v0_core_ch_config,
519 .debugfs_on = dw_edma_v0_core_debugfs_on,
522 void dw_edma_v0_core_register(struct dw_edma *dw)
524 dw->core = &dw_edma_v0_core;