1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * Copyright(c) 2006, Intel Corporation.
7 #include <linux/types.h>
9 #include <mach/hardware.h>
10 #include <asm/hardware/iop_adma.h>
12 #define ADMA_ACCR(chan) (chan->mmr_base + 0x0)
13 #define ADMA_ACSR(chan) (chan->mmr_base + 0x4)
14 #define ADMA_ADAR(chan) (chan->mmr_base + 0x8)
15 #define ADMA_IIPCR(chan) (chan->mmr_base + 0x18)
16 #define ADMA_IIPAR(chan) (chan->mmr_base + 0x1c)
17 #define ADMA_IIPUAR(chan) (chan->mmr_base + 0x20)
18 #define ADMA_ANDAR(chan) (chan->mmr_base + 0x24)
19 #define ADMA_ADCR(chan) (chan->mmr_base + 0x28)
20 #define ADMA_CARMD(chan) (chan->mmr_base + 0x2c)
21 #define ADMA_ABCR(chan) (chan->mmr_base + 0x30)
22 #define ADMA_DLADR(chan) (chan->mmr_base + 0x34)
23 #define ADMA_DUADR(chan) (chan->mmr_base + 0x38)
24 #define ADMA_SLAR(src, chan) (chan->mmr_base + (0x3c + (src << 3)))
25 #define ADMA_SUAR(src, chan) (chan->mmr_base + (0x40 + (src << 3)))
27 struct iop13xx_adma_src {
32 unsigned int pq_upper_src_addr:24;
33 unsigned int pq_dmlt:8;
38 struct iop13xx_adma_desc_ctrl {
39 unsigned int int_en:1;
40 unsigned int xfer_dir:2;
41 unsigned int src_select:4;
42 unsigned int zero_result:1;
43 unsigned int block_fill_en:1;
44 unsigned int crc_gen_en:1;
45 unsigned int crc_xfer_dis:1;
46 unsigned int crc_seed_fetch_dis:1;
47 unsigned int status_write_back_en:1;
48 unsigned int endian_swap_en:1;
49 unsigned int reserved0:2;
50 unsigned int pq_update_xfer_en:1;
51 unsigned int dual_xor_en:1;
52 unsigned int pq_xfer_en:1;
53 unsigned int p_xfer_dis:1;
54 unsigned int reserved1:10;
55 unsigned int relax_order_en:1;
56 unsigned int no_snoop_en:1;
59 struct iop13xx_adma_byte_count {
60 unsigned int byte_count:24;
61 unsigned int host_if:3;
62 unsigned int reserved:2;
63 unsigned int zero_result_err_q:1;
64 unsigned int zero_result_err:1;
65 unsigned int tx_complete:1;
68 struct iop13xx_adma_desc_hw {
72 struct iop13xx_adma_desc_ctrl desc_ctrl_field;
81 struct iop13xx_adma_byte_count byte_count_field;
89 u32 pq_upper_dest_addr;
91 struct iop13xx_adma_src src[1];
94 struct iop13xx_adma_desc_dual_xor {
100 u32 h_upper_dest_addr;
106 u32 h_upper_src_addr;
108 u32 d_upper_src_addr;
110 u32 d_upper_dest_addr;
113 struct iop13xx_adma_desc_pq_update {
119 u32 p_upper_dest_addr;
125 u32 p_upper_src_addr;
128 unsigned int q_upper_src_addr:24;
129 unsigned int q_dmlt:8;
132 u32 q_upper_dest_addr;
135 static inline int iop_adma_get_max_xor(void)
140 #define iop_adma_get_max_pq iop_adma_get_max_xor
142 static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan)
144 return __raw_readl(ADMA_ADAR(chan));
147 static inline void iop_chan_set_next_descriptor(struct iop_adma_chan *chan,
150 __raw_writel(next_desc_addr, ADMA_ANDAR(chan));
153 #define ADMA_STATUS_BUSY (1 << 13)
155 static inline char iop_chan_is_busy(struct iop_adma_chan *chan)
157 if (__raw_readl(ADMA_ACSR(chan)) &
165 iop_chan_get_desc_align(struct iop_adma_chan *chan, int num_slots)
169 #define iop_desc_is_aligned(x, y) 1
172 iop_chan_memcpy_slot_count(size_t len, int *slots_per_op)
178 #define iop_chan_interrupt_slot_count(s, c) iop_chan_memcpy_slot_count(0, s)
181 iop_chan_memset_slot_count(size_t len, int *slots_per_op)
188 iop_chan_xor_slot_count(size_t len, int src_cnt, int *slots_per_op)
190 static const char slot_count_table[] = { 1, 2, 2, 2,
195 *slots_per_op = slot_count_table[src_cnt - 1];
196 return *slots_per_op;
199 #define ADMA_MAX_BYTE_COUNT (16 * 1024 * 1024)
200 #define IOP_ADMA_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
201 #define IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
202 #define IOP_ADMA_XOR_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
203 #define IOP_ADMA_PQ_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
204 #define iop_chan_zero_sum_slot_count(l, s, o) iop_chan_xor_slot_count(l, s, o)
205 #define iop_chan_pq_slot_count iop_chan_xor_slot_count
206 #define iop_chan_pq_zero_sum_slot_count iop_chan_xor_slot_count
208 static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc,
209 struct iop_adma_chan *chan)
211 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
212 return hw_desc->byte_count_field.byte_count;
215 static inline u32 iop_desc_get_src_addr(struct iop_adma_desc_slot *desc,
216 struct iop_adma_chan *chan,
219 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
220 return hw_desc->src[src_idx].src_addr;
223 static inline u32 iop_desc_get_src_count(struct iop_adma_desc_slot *desc,
224 struct iop_adma_chan *chan)
226 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
227 return hw_desc->desc_ctrl_field.src_select + 1;
231 iop_desc_init_memcpy(struct iop_adma_desc_slot *desc, unsigned long flags)
233 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
236 struct iop13xx_adma_desc_ctrl field;
239 u_desc_ctrl.value = 0;
240 u_desc_ctrl.field.xfer_dir = 3; /* local to internal bus */
241 u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
242 hw_desc->desc_ctrl = u_desc_ctrl.value;
243 hw_desc->crc_addr = 0;
247 iop_desc_init_memset(struct iop_adma_desc_slot *desc, unsigned long flags)
249 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
252 struct iop13xx_adma_desc_ctrl field;
255 u_desc_ctrl.value = 0;
256 u_desc_ctrl.field.xfer_dir = 3; /* local to internal bus */
257 u_desc_ctrl.field.block_fill_en = 1;
258 u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
259 hw_desc->desc_ctrl = u_desc_ctrl.value;
260 hw_desc->crc_addr = 0;
263 /* to do: support buffers larger than ADMA_MAX_BYTE_COUNT */
265 iop_desc_init_xor(struct iop_adma_desc_slot *desc, int src_cnt,
268 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
271 struct iop13xx_adma_desc_ctrl field;
274 u_desc_ctrl.value = 0;
275 u_desc_ctrl.field.src_select = src_cnt - 1;
276 u_desc_ctrl.field.xfer_dir = 3; /* local to internal bus */
277 u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
278 hw_desc->desc_ctrl = u_desc_ctrl.value;
279 hw_desc->crc_addr = 0;
282 #define iop_desc_init_null_xor(d, s, i) iop_desc_init_xor(d, s, i)
284 /* to do: support buffers larger than ADMA_MAX_BYTE_COUNT */
286 iop_desc_init_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
289 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
292 struct iop13xx_adma_desc_ctrl field;
295 u_desc_ctrl.value = 0;
296 u_desc_ctrl.field.src_select = src_cnt - 1;
297 u_desc_ctrl.field.xfer_dir = 3; /* local to internal bus */
298 u_desc_ctrl.field.zero_result = 1;
299 u_desc_ctrl.field.status_write_back_en = 1;
300 u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
301 hw_desc->desc_ctrl = u_desc_ctrl.value;
302 hw_desc->crc_addr = 0;
308 iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt,
311 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
314 struct iop13xx_adma_desc_ctrl field;
317 u_desc_ctrl.value = 0;
318 u_desc_ctrl.field.src_select = src_cnt - 1;
319 u_desc_ctrl.field.xfer_dir = 3; /* local to internal bus */
320 u_desc_ctrl.field.pq_xfer_en = 1;
321 u_desc_ctrl.field.p_xfer_dis = !!(flags & DMA_PREP_PQ_DISABLE_P);
322 u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
323 hw_desc->desc_ctrl = u_desc_ctrl.value;
327 iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
330 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
333 struct iop13xx_adma_desc_ctrl field;
336 u_desc_ctrl.value = 0;
337 u_desc_ctrl.field.src_select = src_cnt - 1;
338 u_desc_ctrl.field.xfer_dir = 3; /* local to internal bus */
339 u_desc_ctrl.field.zero_result = 1;
340 u_desc_ctrl.field.status_write_back_en = 1;
341 u_desc_ctrl.field.pq_xfer_en = 1;
342 u_desc_ctrl.field.p_xfer_dis = !!(flags & DMA_PREP_PQ_DISABLE_P);
343 u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
344 hw_desc->desc_ctrl = u_desc_ctrl.value;
347 static inline void iop_desc_set_byte_count(struct iop_adma_desc_slot *desc,
348 struct iop_adma_chan *chan,
351 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
352 hw_desc->byte_count = byte_count;
356 iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
358 int slots_per_op = desc->slots_per_op;
359 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc, *iter;
362 if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
363 hw_desc->byte_count = len;
366 iter = iop_hw_desc_slot_idx(hw_desc, i);
367 iter->byte_count = IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
368 len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
370 } while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT);
373 iter = iop_hw_desc_slot_idx(hw_desc, i);
374 iter->byte_count = len;
379 #define iop_desc_set_pq_zero_sum_byte_count iop_desc_set_zero_sum_byte_count
381 static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc,
382 struct iop_adma_chan *chan,
385 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
386 hw_desc->dest_addr = addr;
387 hw_desc->upper_dest_addr = 0;
391 iop_desc_set_pq_addr(struct iop_adma_desc_slot *desc, dma_addr_t *addr)
393 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
395 hw_desc->dest_addr = addr[0];
396 hw_desc->q_dest_addr = addr[1];
397 hw_desc->upper_dest_addr = 0;
400 static inline void iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot *desc,
403 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
404 hw_desc->src[0].src_addr = addr;
405 hw_desc->src[0].upper_src_addr = 0;
408 static inline void iop_desc_set_xor_src_addr(struct iop_adma_desc_slot *desc,
409 int src_idx, dma_addr_t addr)
411 int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
412 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc, *iter;
416 iter = iop_hw_desc_slot_idx(hw_desc, i);
417 iter->src[src_idx].src_addr = addr;
418 iter->src[src_idx].upper_src_addr = 0;
419 slot_cnt -= slots_per_op;
422 addr += IOP_ADMA_XOR_MAX_BYTE_COUNT;
428 iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
429 dma_addr_t addr, unsigned char coef)
431 int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
432 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc, *iter;
433 struct iop13xx_adma_src *src;
437 iter = iop_hw_desc_slot_idx(hw_desc, i);
438 src = &iter->src[src_idx];
439 src->src_addr = addr;
440 src->pq_upper_src_addr = 0;
442 slot_cnt -= slots_per_op;
445 addr += IOP_ADMA_PQ_MAX_BYTE_COUNT;
451 iop_desc_init_interrupt(struct iop_adma_desc_slot *desc,
452 struct iop_adma_chan *chan)
454 iop_desc_init_memcpy(desc, 1);
455 iop_desc_set_byte_count(desc, chan, 0);
456 iop_desc_set_dest_addr(desc, chan, 0);
457 iop_desc_set_memcpy_src_addr(desc, 0);
460 #define iop_desc_set_zero_sum_src_addr iop_desc_set_xor_src_addr
461 #define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr
464 iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx,
467 iop_desc_set_xor_src_addr(desc, pq_idx, src[pq_idx]);
468 iop_desc_set_xor_src_addr(desc, pq_idx+1, src[pq_idx+1]);
471 static inline void iop_desc_set_next_desc(struct iop_adma_desc_slot *desc,
474 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
476 iop_paranoia(hw_desc->next_desc);
477 hw_desc->next_desc = next_desc_addr;
480 static inline u32 iop_desc_get_next_desc(struct iop_adma_desc_slot *desc)
482 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
483 return hw_desc->next_desc;
486 static inline void iop_desc_clear_next_desc(struct iop_adma_desc_slot *desc)
488 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
489 hw_desc->next_desc = 0;
492 static inline void iop_desc_set_block_fill_val(struct iop_adma_desc_slot *desc,
495 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
496 hw_desc->block_fill_data = val;
499 static inline enum sum_check_flags
500 iop_desc_get_zero_result(struct iop_adma_desc_slot *desc)
502 struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
503 struct iop13xx_adma_desc_ctrl desc_ctrl = hw_desc->desc_ctrl_field;
504 struct iop13xx_adma_byte_count byte_count = hw_desc->byte_count_field;
505 enum sum_check_flags flags;
507 BUG_ON(!(byte_count.tx_complete && desc_ctrl.zero_result));
509 flags = byte_count.zero_result_err_q << SUM_CHECK_Q;
510 flags |= byte_count.zero_result_err << SUM_CHECK_P;
515 static inline void iop_chan_append(struct iop_adma_chan *chan)
519 adma_accr = __raw_readl(ADMA_ACCR(chan));
521 __raw_writel(adma_accr, ADMA_ACCR(chan));
524 static inline u32 iop_chan_get_status(struct iop_adma_chan *chan)
526 return __raw_readl(ADMA_ACSR(chan));
529 static inline void iop_chan_disable(struct iop_adma_chan *chan)
531 u32 adma_chan_ctrl = __raw_readl(ADMA_ACCR(chan));
532 adma_chan_ctrl &= ~0x1;
533 __raw_writel(adma_chan_ctrl, ADMA_ACCR(chan));
536 static inline void iop_chan_enable(struct iop_adma_chan *chan)
540 adma_chan_ctrl = __raw_readl(ADMA_ACCR(chan));
541 adma_chan_ctrl |= 0x1;
542 __raw_writel(adma_chan_ctrl, ADMA_ACCR(chan));
545 static inline void iop_adma_device_clear_eot_status(struct iop_adma_chan *chan)
547 u32 status = __raw_readl(ADMA_ACSR(chan));
549 __raw_writel(status, ADMA_ACSR(chan));
552 static inline void iop_adma_device_clear_eoc_status(struct iop_adma_chan *chan)
554 u32 status = __raw_readl(ADMA_ACSR(chan));
556 __raw_writel(status, ADMA_ACSR(chan));
559 static inline void iop_adma_device_clear_err_status(struct iop_adma_chan *chan)
561 u32 status = __raw_readl(ADMA_ACSR(chan));
562 status &= (1 << 9) | (1 << 5) | (1 << 4) | (1 << 3);
563 __raw_writel(status, ADMA_ACSR(chan));
567 iop_is_err_int_parity(unsigned long status, struct iop_adma_chan *chan)
569 return test_bit(9, &status);
573 iop_is_err_mcu_abort(unsigned long status, struct iop_adma_chan *chan)
575 return test_bit(5, &status);
579 iop_is_err_int_tabort(unsigned long status, struct iop_adma_chan *chan)
581 return test_bit(4, &status);
585 iop_is_err_int_mabort(unsigned long status, struct iop_adma_chan *chan)
587 return test_bit(3, &status);
591 iop_is_err_pci_tabort(unsigned long status, struct iop_adma_chan *chan)
597 iop_is_err_pci_mabort(unsigned long status, struct iop_adma_chan *chan)
603 iop_is_err_split_tx(unsigned long status, struct iop_adma_chan *chan)