arch/arm/include/asm/hardware/iop3xx-adma.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright © 2006, Intel Corporation.
   4  */
   5 #ifndef _ADMA_H
   6 #define _ADMA_H
   7 #include <linux/types.h>
   8 #include <linux/io.h>
   9 #include <mach/hardware.h>
  10 #include <asm/hardware/iop_adma.h>
  11
  12 /* Memory copy units */
  13 #define DMA_CCR(chan)           (chan->mmr_base + 0x0)
  14 #define DMA_CSR(chan)           (chan->mmr_base + 0x4)
  15 #define DMA_DAR(chan)           (chan->mmr_base + 0xc)
  16 #define DMA_NDAR(chan)          (chan->mmr_base + 0x10)
  17 #define DMA_PADR(chan)          (chan->mmr_base + 0x14)
  18 #define DMA_PUADR(chan) (chan->mmr_base + 0x18)
  19 #define DMA_LADR(chan)          (chan->mmr_base + 0x1c)
  20 #define DMA_BCR(chan)           (chan->mmr_base + 0x20)
  21 #define DMA_DCR(chan)           (chan->mmr_base + 0x24)
  22
  23 /* Application accelerator unit  */
  24 #define AAU_ACR(chan)           (chan->mmr_base + 0x0)
  25 #define AAU_ASR(chan)           (chan->mmr_base + 0x4)
  26 #define AAU_ADAR(chan)          (chan->mmr_base + 0x8)
  27 #define AAU_ANDAR(chan) (chan->mmr_base + 0xc)
  28 #define AAU_SAR(src, chan)      (chan->mmr_base + (0x10 + ((src) << 2)))
  29 #define AAU_DAR(chan)           (chan->mmr_base + 0x20)
  30 #define AAU_ABCR(chan)          (chan->mmr_base + 0x24)
  31 #define AAU_ADCR(chan)          (chan->mmr_base + 0x28)
  32 #define AAU_SAR_EDCR(src_edc)   (chan->mmr_base + (0x02c + ((src_edc-4) << 2)))
  33 #define AAU_EDCR0_IDX   8
  34 #define AAU_EDCR1_IDX   17
  35 #define AAU_EDCR2_IDX   26
  36
  37 #define DMA0_ID 0
  38 #define DMA1_ID 1
  39 #define AAU_ID 2
  40
  41 struct iop3xx_aau_desc_ctrl {
  42         unsigned int int_en:1;
  43         unsigned int blk1_cmd_ctrl:3;
  44         unsigned int blk2_cmd_ctrl:3;
  45         unsigned int blk3_cmd_ctrl:3;
  46         unsigned int blk4_cmd_ctrl:3;
  47         unsigned int blk5_cmd_ctrl:3;
  48         unsigned int blk6_cmd_ctrl:3;
  49         unsigned int blk7_cmd_ctrl:3;
  50         unsigned int blk8_cmd_ctrl:3;
  51         unsigned int blk_ctrl:2;
  52         unsigned int dual_xor_en:1;
  53         unsigned int tx_complete:1;
  54         unsigned int zero_result_err:1;
  55         unsigned int zero_result_en:1;
  56         unsigned int dest_write_en:1;
  57 };
  58
  59 struct iop3xx_aau_e_desc_ctrl {
  60         unsigned int reserved:1;
  61         unsigned int blk1_cmd_ctrl:3;
  62         unsigned int blk2_cmd_ctrl:3;
  63         unsigned int blk3_cmd_ctrl:3;
  64         unsigned int blk4_cmd_ctrl:3;
  65         unsigned int blk5_cmd_ctrl:3;
  66         unsigned int blk6_cmd_ctrl:3;
  67         unsigned int blk7_cmd_ctrl:3;
  68         unsigned int blk8_cmd_ctrl:3;
  69         unsigned int reserved2:7;
  70 };
  71
  72 struct iop3xx_dma_desc_ctrl {
  73         unsigned int pci_transaction:4;
  74         unsigned int int_en:1;
  75         unsigned int dac_cycle_en:1;
  76         unsigned int mem_to_mem_en:1;
  77         unsigned int crc_data_tx_en:1;
  78         unsigned int crc_gen_en:1;
  79         unsigned int crc_seed_dis:1;
  80         unsigned int reserved:21;
  81         unsigned int crc_tx_complete:1;
  82 };
  83
  84 struct iop3xx_desc_dma {
  85         u32 next_desc;
  86         union {
  87                 u32 pci_src_addr;
  88                 u32 pci_dest_addr;
  89                 u32 src_addr;
  90         };
  91         union {
  92                 u32 upper_pci_src_addr;
  93                 u32 upper_pci_dest_addr;
  94         };
  95         union {
  96                 u32 local_pci_src_addr;
  97                 u32 local_pci_dest_addr;
  98                 u32 dest_addr;
  99         };
 100         u32 byte_count;
 101         union {
 102                 u32 desc_ctrl;
 103                 struct iop3xx_dma_desc_ctrl desc_ctrl_field;
 104         };
 105         u32 crc_addr;
 106 };
 107
 108 struct iop3xx_desc_aau {
 109         u32 next_desc;
 110         u32 src[4];
 111         u32 dest_addr;
 112         u32 byte_count;
 113         union {
 114                 u32 desc_ctrl;
 115                 struct iop3xx_aau_desc_ctrl desc_ctrl_field;
 116         };
 117         union {
 118                 u32 src_addr;
 119                 u32 e_desc_ctrl;
 120                 struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field;
 121         } src_edc[31];
 122 };
 123
 124 struct iop3xx_aau_gfmr {
 125         unsigned int gfmr1:8;
 126         unsigned int gfmr2:8;
 127         unsigned int gfmr3:8;
 128         unsigned int gfmr4:8;
 129 };
 130
 131 struct iop3xx_desc_pq_xor {
 132         u32 next_desc;
 133         u32 src[3];
 134         union {
 135                 u32 data_mult1;
 136                 struct iop3xx_aau_gfmr data_mult1_field;
 137         };
 138         u32 dest_addr;
 139         u32 byte_count;
 140         union {
 141                 u32 desc_ctrl;
 142                 struct iop3xx_aau_desc_ctrl desc_ctrl_field;
 143         };
 144         union {
 145                 u32 src_addr;
 146                 u32 e_desc_ctrl;
 147                 struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field;
 148                 u32 data_multiplier;
 149                 struct iop3xx_aau_gfmr data_mult_field;
 150                 u32 reserved;
 151         } src_edc_gfmr[19];
 152 };
 153
 154 struct iop3xx_desc_dual_xor {
 155         u32 next_desc;
 156         u32 src0_addr;
 157         u32 src1_addr;
 158         u32 h_src_addr;
 159         u32 d_src_addr;
 160         u32 h_dest_addr;
 161         u32 byte_count;
 162         union {
 163                 u32 desc_ctrl;
 164                 struct iop3xx_aau_desc_ctrl desc_ctrl_field;
 165         };
 166         u32 d_dest_addr;
 167 };
 168
 169 union iop3xx_desc {
 170         struct iop3xx_desc_aau *aau;
 171         struct iop3xx_desc_dma *dma;
 172         struct iop3xx_desc_pq_xor *pq_xor;
 173         struct iop3xx_desc_dual_xor *dual_xor;
 174         void *ptr;
 175 };
 176
 177 /* No support for p+q operations */
 178 static inline int
 179 iop_chan_pq_slot_count(size_t len, int src_cnt, int *slots_per_op)
 180 {
 181         BUG();
 182         return 0;
 183 }
 184
 185 static inline void
 186 iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt,
 187                   unsigned long flags)
 188 {
 189         BUG();
 190 }
 191
 192 static inline void
 193 iop_desc_set_pq_addr(struct iop_adma_desc_slot *desc, dma_addr_t *addr)
 194 {
 195         BUG();
 196 }
 197
 198 static inline void
 199 iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
 200                          dma_addr_t addr, unsigned char coef)
 201 {
 202         BUG();
 203 }
 204
 205 static inline int
 206 iop_chan_pq_zero_sum_slot_count(size_t len, int src_cnt, int *slots_per_op)
 207 {
 208         BUG();
 209         return 0;
 210 }
 211
 212 static inline void
 213 iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
 214                           unsigned long flags)
 215 {
 216         BUG();
 217 }
 218
 219 static inline void
 220 iop_desc_set_pq_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
 221 {
 222         BUG();
 223 }
 224
 225 #define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr
 226
 227 static inline void
 228 iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx,
 229                               dma_addr_t *src)
 230 {
 231         BUG();
 232 }
 233
 234 static inline int iop_adma_get_max_xor(void)
 235 {
 236         return 32;
 237 }
 238
 239 static inline int iop_adma_get_max_pq(void)
 240 {
 241         BUG();
 242         return 0;
 243 }
 244
 245 static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan)
 246 {
 247         int id = chan->device->id;
 248
 249         switch (id) {
 250         case DMA0_ID:
 251         case DMA1_ID:
 252                 return __raw_readl(DMA_DAR(chan));
 253         case AAU_ID:
 254                 return __raw_readl(AAU_ADAR(chan));
 255         default:
 256                 BUG();
 257         }
 258         return 0;
 259 }
 260
 261 static inline void iop_chan_set_next_descriptor(struct iop_adma_chan *chan,
 262                                                 u32 next_desc_addr)
 263 {
 264         int id = chan->device->id;
 265
 266         switch (id) {
 267         case DMA0_ID:
 268         case DMA1_ID:
 269                 __raw_writel(next_desc_addr, DMA_NDAR(chan));
 270                 break;
 271         case AAU_ID:
 272                 __raw_writel(next_desc_addr, AAU_ANDAR(chan));
 273                 break;
 274         }
 275
 276 }
 277
 278 #define IOP_ADMA_STATUS_BUSY (1 << 10)
 279 #define IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT (1024)
 280 #define IOP_ADMA_XOR_MAX_BYTE_COUNT (16 * 1024 * 1024)
 281 #define IOP_ADMA_MAX_BYTE_COUNT (16 * 1024 * 1024)
 282
 283 static inline int iop_chan_is_busy(struct iop_adma_chan *chan)
 284 {
 285         u32 status = __raw_readl(DMA_CSR(chan));
 286         return (status & IOP_ADMA_STATUS_BUSY) ? 1 : 0;
 287 }
 288
 289 static inline int iop_desc_is_aligned(struct iop_adma_desc_slot *desc,
 290                                         int num_slots)
 291 {
 292         /* num_slots will only ever be 1, 2, 4, or 8 */
 293         return (desc->idx & (num_slots - 1)) ? 0 : 1;
 294 }
 295
 296 /* to do: support large (i.e. > hw max) buffer sizes */
 297 static inline int iop_chan_memcpy_slot_count(size_t len, int *slots_per_op)
 298 {
 299         *slots_per_op = 1;
 300         return 1;
 301 }
 302
 303 /* to do: support large (i.e. > hw max) buffer sizes */
 304 static inline int iop_chan_memset_slot_count(size_t len, int *slots_per_op)
 305 {
 306         *slots_per_op = 1;
 307         return 1;
 308 }
 309
 310 static inline int iop3xx_aau_xor_slot_count(size_t len, int src_cnt,
 311                                         int *slots_per_op)
 312 {
 313         static const char slot_count_table[] = {
 314                                                 1, 1, 1, 1, /* 01 - 04 */
 315                                                 2, 2, 2, 2, /* 05 - 08 */
 316                                                 4, 4, 4, 4, /* 09 - 12 */
 317                                                 4, 4, 4, 4, /* 13 - 16 */
 318                                                 8, 8, 8, 8, /* 17 - 20 */
 319                                                 8, 8, 8, 8, /* 21 - 24 */
 320                                                 8, 8, 8, 8, /* 25 - 28 */
 321                                                 8, 8, 8, 8, /* 29 - 32 */
 322                                               };
 323         *slots_per_op = slot_count_table[src_cnt - 1];
 324         return *slots_per_op;
 325 }
 326
 327 static inline int
 328 iop_chan_interrupt_slot_count(int *slots_per_op, struct iop_adma_chan *chan)
 329 {
 330         switch (chan->device->id) {
 331         case DMA0_ID:
 332         case DMA1_ID:
 333                 return iop_chan_memcpy_slot_count(0, slots_per_op);
 334         case AAU_ID:
 335                 return iop3xx_aau_xor_slot_count(0, 2, slots_per_op);
 336         default:
 337                 BUG();
 338         }
 339         return 0;
 340 }
 341
 342 static inline int iop_chan_xor_slot_count(size_t len, int src_cnt,
 343                                                 int *slots_per_op)
 344 {
 345         int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op);
 346
 347         if (len <= IOP_ADMA_XOR_MAX_BYTE_COUNT)
 348                 return slot_cnt;
 349
 350         len -= IOP_ADMA_XOR_MAX_BYTE_COUNT;
 351         while (len > IOP_ADMA_XOR_MAX_BYTE_COUNT) {
 352                 len -= IOP_ADMA_XOR_MAX_BYTE_COUNT;
 353                 slot_cnt += *slots_per_op;
 354         }
 355
 356         slot_cnt += *slots_per_op;
 357
 358         return slot_cnt;
 359 }
 360
 361 /* zero sum on iop3xx is limited to 1k at a time so it requires multiple
 362  * descriptors
 363  */
 364 static inline int iop_chan_zero_sum_slot_count(size_t len, int src_cnt,
 365                                                 int *slots_per_op)
 366 {
 367         int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op);
 368
 369         if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT)
 370                 return slot_cnt;
 371
 372         len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
 373         while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
 374                 len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
 375                 slot_cnt += *slots_per_op;
 376         }
 377
 378         slot_cnt += *slots_per_op;
 379
 380         return slot_cnt;
 381 }
 382
 383 static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc,
 384                                         struct iop_adma_chan *chan)
 385 {
 386         union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
 387
 388         switch (chan->device->id) {
 389         case DMA0_ID:
 390         case DMA1_ID:
 391                 return hw_desc.dma->byte_count;
 392         case AAU_ID:
 393                 return hw_desc.aau->byte_count;
 394         default:
 395                 BUG();
 396         }
 397         return 0;
 398 }
 399
 400 /* translate the src_idx to a descriptor word index */
 401 static inline int __desc_idx(int src_idx)
 402 {
 403         static const int desc_idx_table[] = { 0, 0, 0, 0,
 404                                               0, 1, 2, 3,
 405                                               5, 6, 7, 8,
 406                                               9, 10, 11, 12,
 407                                               14, 15, 16, 17,
 408                                               18, 19, 20, 21,
 409                                               23, 24, 25, 26,
 410                                               27, 28, 29, 30,
 411                                             };
 412
 413         return desc_idx_table[src_idx];
 414 }
 415
 416 static inline u32 iop_desc_get_src_addr(struct iop_adma_desc_slot *desc,
 417                                         struct iop_adma_chan *chan,
 418                                         int src_idx)
 419 {
 420         union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
 421
 422         switch (chan->device->id) {
 423         case DMA0_ID:
 424         case DMA1_ID:
 425                 return hw_desc.dma->src_addr;
 426         case AAU_ID:
 427                 break;
 428         default:
 429                 BUG();
 430         }
 431
 432         if (src_idx < 4)
 433                 return hw_desc.aau->src[src_idx];
 434         else
 435                 return hw_desc.aau->src_edc[__desc_idx(src_idx)].src_addr;
 436 }
 437
 438 static inline void iop3xx_aau_desc_set_src_addr(struct iop3xx_desc_aau *hw_desc,
 439                                         int src_idx, dma_addr_t addr)
 440 {
 441         if (src_idx < 4)
 442                 hw_desc->src[src_idx] = addr;
 443         else
 444                 hw_desc->src_edc[__desc_idx(src_idx)].src_addr = addr;
 445 }
 446
 447 static inline void
 448 iop_desc_init_memcpy(struct iop_adma_desc_slot *desc, unsigned long flags)
 449 {
 450         struct iop3xx_desc_dma *hw_desc = desc->hw_desc;
 451         union {
 452                 u32 value;
 453                 struct iop3xx_dma_desc_ctrl field;
 454         } u_desc_ctrl;
 455
 456         u_desc_ctrl.value = 0;
 457         u_desc_ctrl.field.mem_to_mem_en = 1;
 458         u_desc_ctrl.field.pci_transaction = 0xe; /* memory read block */
 459         u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
 460         hw_desc->desc_ctrl = u_desc_ctrl.value;
 461         hw_desc->upper_pci_src_addr = 0;
 462         hw_desc->crc_addr = 0;
 463 }
 464
 465 static inline void
 466 iop_desc_init_memset(struct iop_adma_desc_slot *desc, unsigned long flags)
 467 {
 468         struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
 469         union {
 470                 u32 value;
 471                 struct iop3xx_aau_desc_ctrl field;
 472         } u_desc_ctrl;
 473
 474         u_desc_ctrl.value = 0;
 475         u_desc_ctrl.field.blk1_cmd_ctrl = 0x2; /* memory block fill */
 476         u_desc_ctrl.field.dest_write_en = 1;
 477         u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
 478         hw_desc->desc_ctrl = u_desc_ctrl.value;
 479 }
 480
 481 static inline u32
 482 iop3xx_desc_init_xor(struct iop3xx_desc_aau *hw_desc, int src_cnt,
 483                      unsigned long flags)
 484 {
 485         int i, shift;
 486         u32 edcr;
 487         union {
 488                 u32 value;
 489                 struct iop3xx_aau_desc_ctrl field;
 490         } u_desc_ctrl;
 491
 492         u_desc_ctrl.value = 0;
 493         switch (src_cnt) {
 494         case 25 ... 32:
 495                 u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
 496                 edcr = 0;
 497                 shift = 1;
 498                 for (i = 24; i < src_cnt; i++) {
 499                         edcr |= (1 << shift);
 500                         shift += 3;
 501                 }
 502                 hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = edcr;
 503                 src_cnt = 24;
 504                 /* fall through */
 505         case 17 ... 24:
 506                 if (!u_desc_ctrl.field.blk_ctrl) {
 507                         hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0;
 508                         u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
 509                 }
 510                 edcr = 0;
 511                 shift = 1;
 512                 for (i = 16; i < src_cnt; i++) {
 513                         edcr |= (1 << shift);
 514                         shift += 3;
 515                 }
 516                 hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = edcr;
 517                 src_cnt = 16;
 518                 /* fall through */
 519         case 9 ... 16:
 520                 if (!u_desc_ctrl.field.blk_ctrl)
 521                         u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */
 522                 edcr = 0;
 523                 shift = 1;
 524                 for (i = 8; i < src_cnt; i++) {
 525                         edcr |= (1 << shift);
 526                         shift += 3;
 527                 }
 528                 hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = edcr;
 529                 src_cnt = 8;
 530                 /* fall through */
 531         case 2 ... 8:
 532                 shift = 1;
 533                 for (i = 0; i < src_cnt; i++) {
 534                         u_desc_ctrl.value |= (1 << shift);
 535                         shift += 3;
 536                 }
 537
 538                 if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4)
 539                         u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */
 540         }
 541
 542         u_desc_ctrl.field.dest_write_en = 1;
 543         u_desc_ctrl.field.blk1_cmd_ctrl = 0x7; /* direct fill */
 544         u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
 545         hw_desc->desc_ctrl = u_desc_ctrl.value;
 546
 547         return u_desc_ctrl.value;
 548 }
 549
 550 static inline void
 551 iop_desc_init_xor(struct iop_adma_desc_slot *desc, int src_cnt,
 552                   unsigned long flags)
 553 {
 554         iop3xx_desc_init_xor(desc->hw_desc, src_cnt, flags);
 555 }
 556
 557 /* return the number of operations */
 558 static inline int
 559 iop_desc_init_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
 560                        unsigned long flags)
 561 {
 562         int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
 563         struct iop3xx_desc_aau *hw_desc, *prev_hw_desc, *iter;
 564         union {
 565                 u32 value;
 566                 struct iop3xx_aau_desc_ctrl field;
 567         } u_desc_ctrl;
 568         int i, j;
 569
 570         hw_desc = desc->hw_desc;
 571
 572         for (i = 0, j = 0; (slot_cnt -= slots_per_op) >= 0;
 573                 i += slots_per_op, j++) {
 574                 iter = iop_hw_desc_slot_idx(hw_desc, i);
 575                 u_desc_ctrl.value = iop3xx_desc_init_xor(iter, src_cnt, flags);
 576                 u_desc_ctrl.field.dest_write_en = 0;
 577                 u_desc_ctrl.field.zero_result_en = 1;
 578                 u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
 579                 iter->desc_ctrl = u_desc_ctrl.value;
 580
 581                 /* for the subsequent descriptors preserve the store queue
 582                  * and chain them together
 583                  */
 584                 if (i) {
 585                         prev_hw_desc =
 586                                 iop_hw_desc_slot_idx(hw_desc, i - slots_per_op);
 587                         prev_hw_desc->next_desc =
 588                                 (u32) (desc->async_tx.phys + (i << 5));
 589                 }
 590         }
 591
 592         return j;
 593 }
 594
 595 static inline void
 596 iop_desc_init_null_xor(struct iop_adma_desc_slot *desc, int src_cnt,
 597                        unsigned long flags)
 598 {
 599         struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
 600         union {
 601                 u32 value;
 602                 struct iop3xx_aau_desc_ctrl field;
 603         } u_desc_ctrl;
 604
 605         u_desc_ctrl.value = 0;
 606         switch (src_cnt) {
 607         case 25 ... 32:
 608                 u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
 609                 hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0;
 610                 /* fall through */
 611         case 17 ... 24:
 612                 if (!u_desc_ctrl.field.blk_ctrl) {
 613                         hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0;
 614                         u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */
 615                 }
 616                 hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = 0;
 617                 /* fall through */
 618         case 9 ... 16:
 619                 if (!u_desc_ctrl.field.blk_ctrl)
 620                         u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */
 621                 hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = 0;
 622                 /* fall through */
 623         case 1 ... 8:
 624                 if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4)
 625                         u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */
 626         }
 627
 628         u_desc_ctrl.field.dest_write_en = 0;
 629         u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
 630         hw_desc->desc_ctrl = u_desc_ctrl.value;
 631 }
 632
 633 static inline void iop_desc_set_byte_count(struct iop_adma_desc_slot *desc,
 634                                         struct iop_adma_chan *chan,
 635                                         u32 byte_count)
 636 {
 637         union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
 638
 639         switch (chan->device->id) {
 640         case DMA0_ID:
 641         case DMA1_ID:
 642                 hw_desc.dma->byte_count = byte_count;
 643                 break;
 644         case AAU_ID:
 645                 hw_desc.aau->byte_count = byte_count;
 646                 break;
 647         default:
 648                 BUG();
 649         }
 650 }
 651
 652 static inline void
 653 iop_desc_init_interrupt(struct iop_adma_desc_slot *desc,
 654                         struct iop_adma_chan *chan)
 655 {
 656         union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
 657
 658         switch (chan->device->id) {
 659         case DMA0_ID:
 660         case DMA1_ID:
 661                 iop_desc_init_memcpy(desc, 1);
 662                 hw_desc.dma->byte_count = 0;
 663                 hw_desc.dma->dest_addr = 0;
 664                 hw_desc.dma->src_addr = 0;
 665                 break;
 666         case AAU_ID:
 667                 iop_desc_init_null_xor(desc, 2, 1);
 668                 hw_desc.aau->byte_count = 0;
 669                 hw_desc.aau->dest_addr = 0;
 670                 hw_desc.aau->src[0] = 0;
 671                 hw_desc.aau->src[1] = 0;
 672                 break;
 673         default:
 674                 BUG();
 675         }
 676 }
 677
 678 static inline void
 679 iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
 680 {
 681         int slots_per_op = desc->slots_per_op;
 682         struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter;
 683         int i = 0;
 684
 685         if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
 686                 hw_desc->byte_count = len;
 687         } else {
 688                 do {
 689                         iter = iop_hw_desc_slot_idx(hw_desc, i);
 690                         iter->byte_count = IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
 691                         len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT;
 692                         i += slots_per_op;
 693                 } while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT);
 694
 695                 iter = iop_hw_desc_slot_idx(hw_desc, i);
 696                 iter->byte_count = len;
 697         }
 698 }
 699
 700 static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc,
 701                                         struct iop_adma_chan *chan,
 702                                         dma_addr_t addr)
 703 {
 704         union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
 705
 706         switch (chan->device->id) {
 707         case DMA0_ID:
 708         case DMA1_ID:
 709                 hw_desc.dma->dest_addr = addr;
 710                 break;
 711         case AAU_ID:
 712                 hw_desc.aau->dest_addr = addr;
 713                 break;
 714         default:
 715                 BUG();
 716         }
 717 }
 718
 719 static inline void iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot *desc,
 720                                         dma_addr_t addr)
 721 {
 722         struct iop3xx_desc_dma *hw_desc = desc->hw_desc;
 723         hw_desc->src_addr = addr;
 724 }
 725
 726 static inline void
 727 iop_desc_set_zero_sum_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
 728                                 dma_addr_t addr)
 729 {
 730
 731         struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter;
 732         int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
 733         int i;
 734
 735         for (i = 0; (slot_cnt -= slots_per_op) >= 0;
 736                 i += slots_per_op, addr += IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) {
 737                 iter = iop_hw_desc_slot_idx(hw_desc, i);
 738                 iop3xx_aau_desc_set_src_addr(iter, src_idx, addr);
 739         }
 740 }
 741
 742 static inline void iop_desc_set_xor_src_addr(struct iop_adma_desc_slot *desc,
 743                                         int src_idx, dma_addr_t addr)
 744 {
 745
 746         struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter;
 747         int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
 748         int i;
 749
 750         for (i = 0; (slot_cnt -= slots_per_op) >= 0;
 751                 i += slots_per_op, addr += IOP_ADMA_XOR_MAX_BYTE_COUNT) {
 752                 iter = iop_hw_desc_slot_idx(hw_desc, i);
 753                 iop3xx_aau_desc_set_src_addr(iter, src_idx, addr);
 754         }
 755 }
 756
 757 static inline void iop_desc_set_next_desc(struct iop_adma_desc_slot *desc,
 758                                         u32 next_desc_addr)
 759 {
 760         /* hw_desc->next_desc is the same location for all channels */
 761         union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
 762
 763         iop_paranoia(hw_desc.dma->next_desc);
 764         hw_desc.dma->next_desc = next_desc_addr;
 765 }
 766
 767 static inline u32 iop_desc_get_next_desc(struct iop_adma_desc_slot *desc)
 768 {
 769         /* hw_desc->next_desc is the same location for all channels */
 770         union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
 771         return hw_desc.dma->next_desc;
 772 }
 773
 774 static inline void iop_desc_clear_next_desc(struct iop_adma_desc_slot *desc)
 775 {
 776         /* hw_desc->next_desc is the same location for all channels */
 777         union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, };
 778         hw_desc.dma->next_desc = 0;
 779 }
 780
 781 static inline void iop_desc_set_block_fill_val(struct iop_adma_desc_slot *desc,
 782                                                 u32 val)
 783 {
 784         struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
 785         hw_desc->src[0] = val;
 786 }
 787
 788 static inline enum sum_check_flags
 789 iop_desc_get_zero_result(struct iop_adma_desc_slot *desc)
 790 {
 791         struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
 792         struct iop3xx_aau_desc_ctrl desc_ctrl = hw_desc->desc_ctrl_field;
 793
 794         iop_paranoia(!(desc_ctrl.tx_complete && desc_ctrl.zero_result_en));
 795         return desc_ctrl.zero_result_err << SUM_CHECK_P;
 796 }
 797
 798 static inline void iop_chan_append(struct iop_adma_chan *chan)
 799 {
 800         u32 dma_chan_ctrl;
 801
 802         dma_chan_ctrl = __raw_readl(DMA_CCR(chan));
 803         dma_chan_ctrl |= 0x2;
 804         __raw_writel(dma_chan_ctrl, DMA_CCR(chan));
 805 }
 806
 807 static inline u32 iop_chan_get_status(struct iop_adma_chan *chan)
 808 {
 809         return __raw_readl(DMA_CSR(chan));
 810 }
 811
 812 static inline void iop_chan_disable(struct iop_adma_chan *chan)
 813 {
 814         u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan));
 815         dma_chan_ctrl &= ~1;
 816         __raw_writel(dma_chan_ctrl, DMA_CCR(chan));
 817 }
 818
 819 static inline void iop_chan_enable(struct iop_adma_chan *chan)
 820 {
 821         u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan));
 822
 823         dma_chan_ctrl |= 1;
 824         __raw_writel(dma_chan_ctrl, DMA_CCR(chan));
 825 }
 826
 827 static inline void iop_adma_device_clear_eot_status(struct iop_adma_chan *chan)
 828 {
 829         u32 status = __raw_readl(DMA_CSR(chan));
 830         status &= (1 << 9);
 831         __raw_writel(status, DMA_CSR(chan));
 832 }
 833
 834 static inline void iop_adma_device_clear_eoc_status(struct iop_adma_chan *chan)
 835 {
 836         u32 status = __raw_readl(DMA_CSR(chan));
 837         status &= (1 << 8);
 838         __raw_writel(status, DMA_CSR(chan));
 839 }
 840
 841 static inline void iop_adma_device_clear_err_status(struct iop_adma_chan *chan)
 842 {
 843         u32 status = __raw_readl(DMA_CSR(chan));
 844
 845         switch (chan->device->id) {
 846         case DMA0_ID:
 847         case DMA1_ID:
 848                 status &= (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1);
 849                 break;
 850         case AAU_ID:
 851                 status &= (1 << 5);
 852                 break;
 853         default:
 854                 BUG();
 855         }
 856
 857         __raw_writel(status, DMA_CSR(chan));
 858 }
 859
 860 static inline int
 861 iop_is_err_int_parity(unsigned long status, struct iop_adma_chan *chan)
 862 {
 863         return 0;
 864 }
 865
 866 static inline int
 867 iop_is_err_mcu_abort(unsigned long status, struct iop_adma_chan *chan)
 868 {
 869         return 0;
 870 }
 871
 872 static inline int
 873 iop_is_err_int_tabort(unsigned long status, struct iop_adma_chan *chan)
 874 {
 875         return 0;
 876 }
 877
 878 static inline int
 879 iop_is_err_int_mabort(unsigned long status, struct iop_adma_chan *chan)
 880 {
 881         return test_bit(5, &status);
 882 }
 883
 884 static inline int
 885 iop_is_err_pci_tabort(unsigned long status, struct iop_adma_chan *chan)
 886 {
 887         switch (chan->device->id) {
 888         case DMA0_ID:
 889         case DMA1_ID:
 890                 return test_bit(2, &status);
 891         default:
 892                 return 0;
 893         }
 894 }
 895
 896 static inline int
 897 iop_is_err_pci_mabort(unsigned long status, struct iop_adma_chan *chan)
 898 {
 899         switch (chan->device->id) {
 900         case DMA0_ID:
 901         case DMA1_ID:
 902                 return test_bit(3, &status);
 903         default:
 904                 return 0;
 905         }
 906 }
 907
 908 static inline int
 909 iop_is_err_split_tx(unsigned long status, struct iop_adma_chan *chan)
 910 {
 911         switch (chan->device->id) {
 912         case DMA0_ID:
 913         case DMA1_ID:
 914                 return test_bit(1, &status);
 915         default:
 916                 return 0;
 917         }
 918 }
 919 #endif /* _ADMA_H */