drivers/mmc/host/bcm2835.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * bcm2835 sdhost driver.
   4  *
   5  * The 2835 has two SD controllers: The Arasan sdhci controller
   6  * (supported by the iproc driver) and a custom sdhost controller
   7  * (supported by this driver).
   8  *
   9  * The sdhci controller supports both sdcard and sdio.  The sdhost
  10  * controller supports the sdcard only, but has better performance.
  11  * Also note that the rpi3 has sdio wifi, so driving the sdcard with
  12  * the sdhost controller allows to use the sdhci controller for wifi
  13  * support.
  14  *
  15  * The configuration is done by devicetree via pin muxing.  Both
  16  * SD controller are available on the same pins (2 pin groups = pin 22
  17  * to 27 + pin 48 to 53).  So it's possible to use both SD controllers
  18  * at the same time with different pin groups.
  19  *
  20  * Author:      Phil Elwell <phil@raspberrypi.org>
  21  *              Copyright (C) 2015-2016 Raspberry Pi (Trading) Ltd.
  22  *
  23  * Based on
  24  *  mmc-bcm2835.c by Gellert Weisz
  25  * which is, in turn, based on
  26  *  sdhci-bcm2708.c by Broadcom
  27  *  sdhci-bcm2835.c by Stephen Warren and Oleksandr Tymoshenko
  28  *  sdhci.c and sdhci-pci.c by Pierre Ossman
  29  */
  30 #include <linux/clk.h>
  31 #include <linux/delay.h>
  32 #include <linux/device.h>
  33 #include <linux/dmaengine.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/err.h>
  36 #include <linux/highmem.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/io.h>
  39 #include <linux/iopoll.h>
  40 #include <linux/module.h>
  41 #include <linux/of_address.h>
  42 #include <linux/of_irq.h>
  43 #include <linux/platform_device.h>
  44 #include <linux/scatterlist.h>
  45 #include <linux/time.h>
  46 #include <linux/workqueue.h>
  47
  48 #include <linux/mmc/host.h>
  49 #include <linux/mmc/mmc.h>
  50 #include <linux/mmc/sd.h>
  51
  52 #define SDCMD  0x00 /* Command to SD card              - 16 R/W */
  53 #define SDARG  0x04 /* Argument to SD card             - 32 R/W */
  54 #define SDTOUT 0x08 /* Start value for timeout counter - 32 R/W */
  55 #define SDCDIV 0x0c /* Start value for clock divider   - 11 R/W */
  56 #define SDRSP0 0x10 /* SD card response (31:0)         - 32 R   */
  57 #define SDRSP1 0x14 /* SD card response (63:32)        - 32 R   */
  58 #define SDRSP2 0x18 /* SD card response (95:64)        - 32 R   */
  59 #define SDRSP3 0x1c /* SD card response (127:96)       - 32 R   */
  60 #define SDHSTS 0x20 /* SD host status                  - 11 R/W */
  61 #define SDVDD  0x30 /* SD card power control           -  1 R/W */
  62 #define SDEDM  0x34 /* Emergency Debug Mode            - 13 R/W */
  63 #define SDHCFG 0x38 /* Host configuration              -  2 R/W */
  64 #define SDHBCT 0x3c /* Host byte count (debug)         - 32 R/W */
  65 #define SDDATA 0x40 /* Data to/from SD card            - 32 R/W */
  66 #define SDHBLC 0x50 /* Host block count (SDIO/SDHC)    -  9 R/W */
  67
  68 #define SDCMD_NEW_FLAG                  0x8000
  69 #define SDCMD_FAIL_FLAG                 0x4000
  70 #define SDCMD_BUSYWAIT                  0x800
  71 #define SDCMD_NO_RESPONSE               0x400
  72 #define SDCMD_LONG_RESPONSE             0x200
  73 #define SDCMD_WRITE_CMD                 0x80
  74 #define SDCMD_READ_CMD                  0x40
  75 #define SDCMD_CMD_MASK                  0x3f
  76
  77 #define SDCDIV_MAX_CDIV                 0x7ff
  78
  79 #define SDHSTS_BUSY_IRPT                0x400
  80 #define SDHSTS_BLOCK_IRPT               0x200
  81 #define SDHSTS_SDIO_IRPT                0x100
  82 #define SDHSTS_REW_TIME_OUT             0x80
  83 #define SDHSTS_CMD_TIME_OUT             0x40
  84 #define SDHSTS_CRC16_ERROR              0x20
  85 #define SDHSTS_CRC7_ERROR               0x10
  86 #define SDHSTS_FIFO_ERROR               0x08
  87 /* Reserved */
  88 /* Reserved */
  89 #define SDHSTS_DATA_FLAG                0x01
  90
  91 #define SDHSTS_TRANSFER_ERROR_MASK      (SDHSTS_CRC7_ERROR | \
  92                                          SDHSTS_CRC16_ERROR | \
  93                                          SDHSTS_REW_TIME_OUT | \
  94                                          SDHSTS_FIFO_ERROR)
  95
  96 #define SDHSTS_ERROR_MASK               (SDHSTS_CMD_TIME_OUT | \
  97                                          SDHSTS_TRANSFER_ERROR_MASK)
  98
  99 #define SDHCFG_BUSY_IRPT_EN     BIT(10)
 100 #define SDHCFG_BLOCK_IRPT_EN    BIT(8)
 101 #define SDHCFG_SDIO_IRPT_EN     BIT(5)
 102 #define SDHCFG_DATA_IRPT_EN     BIT(4)
 103 #define SDHCFG_SLOW_CARD        BIT(3)
 104 #define SDHCFG_WIDE_EXT_BUS     BIT(2)
 105 #define SDHCFG_WIDE_INT_BUS     BIT(1)
 106 #define SDHCFG_REL_CMD_LINE     BIT(0)
 107
 108 #define SDVDD_POWER_OFF         0
 109 #define SDVDD_POWER_ON          1
 110
 111 #define SDEDM_FORCE_DATA_MODE   BIT(19)
 112 #define SDEDM_CLOCK_PULSE       BIT(20)
 113 #define SDEDM_BYPASS            BIT(21)
 114
 115 #define SDEDM_WRITE_THRESHOLD_SHIFT     9
 116 #define SDEDM_READ_THRESHOLD_SHIFT      14
 117 #define SDEDM_THRESHOLD_MASK            0x1f
 118
 119 #define SDEDM_FSM_MASK          0xf
 120 #define SDEDM_FSM_IDENTMODE     0x0
 121 #define SDEDM_FSM_DATAMODE      0x1
 122 #define SDEDM_FSM_READDATA      0x2
 123 #define SDEDM_FSM_WRITEDATA     0x3
 124 #define SDEDM_FSM_READWAIT      0x4
 125 #define SDEDM_FSM_READCRC       0x5
 126 #define SDEDM_FSM_WRITECRC      0x6
 127 #define SDEDM_FSM_WRITEWAIT1    0x7
 128 #define SDEDM_FSM_POWERDOWN     0x8
 129 #define SDEDM_FSM_POWERUP       0x9
 130 #define SDEDM_FSM_WRITESTART1   0xa
 131 #define SDEDM_FSM_WRITESTART2   0xb
 132 #define SDEDM_FSM_GENPULSES     0xc
 133 #define SDEDM_FSM_WRITEWAIT2    0xd
 134 #define SDEDM_FSM_STARTPOWDOWN  0xf
 135
 136 #define SDDATA_FIFO_WORDS       16
 137
 138 #define FIFO_READ_THRESHOLD     4
 139 #define FIFO_WRITE_THRESHOLD    4
 140 #define SDDATA_FIFO_PIO_BURST   8
 141
 142 #define PIO_THRESHOLD   1  /* Maximum block count for PIO (0 = always DMA) */
 143
 144 struct bcm2835_host {
 145         spinlock_t              lock;
 146         struct mutex            mutex;
 147
 148         void __iomem            *ioaddr;
 149         u32                     phys_addr;
 150
 151         struct mmc_host         *mmc;
 152         struct platform_device  *pdev;
 153
 154         int                     clock;          /* Current clock speed */
 155         unsigned int            max_clk;        /* Max possible freq */
 156         struct work_struct      dma_work;
 157         struct delayed_work     timeout_work;   /* Timer for timeouts */
 158         struct sg_mapping_iter  sg_miter;       /* SG state for PIO */
 159         unsigned int            blocks;         /* remaining PIO blocks */
 160         int                     irq;            /* Device IRQ */
 161
 162         u32                     ns_per_fifo_word;
 163
 164         /* cached registers */
 165         u32                     hcfg;
 166         u32                     cdiv;
 167
 168         struct mmc_request      *mrq;           /* Current request */
 169         struct mmc_command      *cmd;           /* Current command */
 170         struct mmc_data         *data;          /* Current data request */
 171         bool                    data_complete:1;/* Data finished before cmd */
 172         bool                    use_busy:1;     /* Wait for busy interrupt */
 173         bool                    use_sbc:1;      /* Send CMD23 */
 174
 175         /* for threaded irq handler */
 176         bool                    irq_block;
 177         bool                    irq_busy;
 178         bool                    irq_data;
 179
 180         /* DMA part */
 181         struct dma_chan         *dma_chan_rxtx;
 182         struct dma_chan         *dma_chan;
 183         struct dma_slave_config dma_cfg_rx;
 184         struct dma_slave_config dma_cfg_tx;
 185         struct dma_async_tx_descriptor  *dma_desc;
 186         u32                     dma_dir;
 187         u32                     drain_words;
 188         struct page             *drain_page;
 189         u32                     drain_offset;
 190         bool                    use_dma;
 191 };
 192
 193 static void bcm2835_dumpcmd(struct bcm2835_host *host, struct mmc_command *cmd,
 194                             const char *label)
 195 {
 196         struct device *dev = &host->pdev->dev;
 197
 198         if (!cmd)
 199                 return;
 200
 201         dev_dbg(dev, "%c%s op %d arg 0x%x flags 0x%x - resp %08x %08x %08x %08x, err %d\n",
 202                 (cmd == host->cmd) ? '>' : ' ',
 203                 label, cmd->opcode, cmd->arg, cmd->flags,
 204                 cmd->resp[0], cmd->resp[1], cmd->resp[2], cmd->resp[3],
 205                 cmd->error);
 206 }
 207
 208 static void bcm2835_dumpregs(struct bcm2835_host *host)
 209 {
 210         struct mmc_request *mrq = host->mrq;
 211         struct device *dev = &host->pdev->dev;
 212
 213         if (mrq) {
 214                 bcm2835_dumpcmd(host, mrq->sbc, "sbc");
 215                 bcm2835_dumpcmd(host, mrq->cmd, "cmd");
 216                 if (mrq->data) {
 217                         dev_dbg(dev, "data blocks %x blksz %x - err %d\n",
 218                                 mrq->data->blocks,
 219                                 mrq->data->blksz,
 220                                 mrq->data->error);
 221                 }
 222                 bcm2835_dumpcmd(host, mrq->stop, "stop");
 223         }
 224
 225         dev_dbg(dev, "=========== REGISTER DUMP ===========\n");
 226         dev_dbg(dev, "SDCMD  0x%08x\n", readl(host->ioaddr + SDCMD));
 227         dev_dbg(dev, "SDARG  0x%08x\n", readl(host->ioaddr + SDARG));
 228         dev_dbg(dev, "SDTOUT 0x%08x\n", readl(host->ioaddr + SDTOUT));
 229         dev_dbg(dev, "SDCDIV 0x%08x\n", readl(host->ioaddr + SDCDIV));
 230         dev_dbg(dev, "SDRSP0 0x%08x\n", readl(host->ioaddr + SDRSP0));
 231         dev_dbg(dev, "SDRSP1 0x%08x\n", readl(host->ioaddr + SDRSP1));
 232         dev_dbg(dev, "SDRSP2 0x%08x\n", readl(host->ioaddr + SDRSP2));
 233         dev_dbg(dev, "SDRSP3 0x%08x\n", readl(host->ioaddr + SDRSP3));
 234         dev_dbg(dev, "SDHSTS 0x%08x\n", readl(host->ioaddr + SDHSTS));
 235         dev_dbg(dev, "SDVDD  0x%08x\n", readl(host->ioaddr + SDVDD));
 236         dev_dbg(dev, "SDEDM  0x%08x\n", readl(host->ioaddr + SDEDM));
 237         dev_dbg(dev, "SDHCFG 0x%08x\n", readl(host->ioaddr + SDHCFG));
 238         dev_dbg(dev, "SDHBCT 0x%08x\n", readl(host->ioaddr + SDHBCT));
 239         dev_dbg(dev, "SDHBLC 0x%08x\n", readl(host->ioaddr + SDHBLC));
 240         dev_dbg(dev, "===========================================\n");
 241 }
 242
 243 static void bcm2835_reset_internal(struct bcm2835_host *host)
 244 {
 245         u32 temp;
 246
 247         writel(SDVDD_POWER_OFF, host->ioaddr + SDVDD);
 248         writel(0, host->ioaddr + SDCMD);
 249         writel(0, host->ioaddr + SDARG);
 250         writel(0xf00000, host->ioaddr + SDTOUT);
 251         writel(0, host->ioaddr + SDCDIV);
 252         writel(0x7f8, host->ioaddr + SDHSTS); /* Write 1s to clear */
 253         writel(0, host->ioaddr + SDHCFG);
 254         writel(0, host->ioaddr + SDHBCT);
 255         writel(0, host->ioaddr + SDHBLC);
 256
 257         /* Limit fifo usage due to silicon bug */
 258         temp = readl(host->ioaddr + SDEDM);
 259         temp &= ~((SDEDM_THRESHOLD_MASK << SDEDM_READ_THRESHOLD_SHIFT) |
 260                   (SDEDM_THRESHOLD_MASK << SDEDM_WRITE_THRESHOLD_SHIFT));
 261         temp |= (FIFO_READ_THRESHOLD << SDEDM_READ_THRESHOLD_SHIFT) |
 262                 (FIFO_WRITE_THRESHOLD << SDEDM_WRITE_THRESHOLD_SHIFT);
 263         writel(temp, host->ioaddr + SDEDM);
 264         msleep(20);
 265         writel(SDVDD_POWER_ON, host->ioaddr + SDVDD);
 266         msleep(20);
 267         host->clock = 0;
 268         writel(host->hcfg, host->ioaddr + SDHCFG);
 269         writel(host->cdiv, host->ioaddr + SDCDIV);
 270 }
 271
 272 static void bcm2835_reset(struct mmc_host *mmc)
 273 {
 274         struct bcm2835_host *host = mmc_priv(mmc);
 275
 276         if (host->dma_chan)
 277                 dmaengine_terminate_sync(host->dma_chan);
 278         host->dma_chan = NULL;
 279         bcm2835_reset_internal(host);
 280 }
 281
 282 static void bcm2835_finish_command(struct bcm2835_host *host);
 283
 284 static void bcm2835_wait_transfer_complete(struct bcm2835_host *host)
 285 {
 286         int timediff;
 287         u32 alternate_idle;
 288
 289         alternate_idle = (host->mrq->data->flags & MMC_DATA_READ) ?
 290                 SDEDM_FSM_READWAIT : SDEDM_FSM_WRITESTART1;
 291
 292         timediff = 0;
 293
 294         while (1) {
 295                 u32 edm, fsm;
 296
 297                 edm = readl(host->ioaddr + SDEDM);
 298                 fsm = edm & SDEDM_FSM_MASK;
 299
 300                 if ((fsm == SDEDM_FSM_IDENTMODE) ||
 301                     (fsm == SDEDM_FSM_DATAMODE))
 302                         break;
 303                 if (fsm == alternate_idle) {
 304                         writel(edm | SDEDM_FORCE_DATA_MODE,
 305                                host->ioaddr + SDEDM);
 306                         break;
 307                 }
 308
 309                 timediff++;
 310                 if (timediff == 100000) {
 311                         dev_err(&host->pdev->dev,
 312                                 "wait_transfer_complete - still waiting after %d retries\n",
 313                                 timediff);
 314                         bcm2835_dumpregs(host);
 315                         host->mrq->data->error = -ETIMEDOUT;
 316                         return;
 317                 }
 318                 cpu_relax();
 319         }
 320 }
 321
 322 static void bcm2835_dma_complete(void *param)
 323 {
 324         struct bcm2835_host *host = param;
 325
 326         schedule_work(&host->dma_work);
 327 }
 328
 329 static void bcm2835_transfer_block_pio(struct bcm2835_host *host, bool is_read)
 330 {
 331         unsigned long flags;
 332         size_t blksize;
 333         unsigned long wait_max;
 334
 335         blksize = host->data->blksz;
 336
 337         wait_max = jiffies + msecs_to_jiffies(500);
 338
 339         local_irq_save(flags);
 340
 341         while (blksize) {
 342                 int copy_words;
 343                 u32 hsts = 0;
 344                 size_t len;
 345                 u32 *buf;
 346
 347                 if (!sg_miter_next(&host->sg_miter)) {
 348                         host->data->error = -EINVAL;
 349                         break;
 350                 }
 351
 352                 len = min(host->sg_miter.length, blksize);
 353                 if (len % 4) {
 354                         host->data->error = -EINVAL;
 355                         break;
 356                 }
 357
 358                 blksize -= len;
 359                 host->sg_miter.consumed = len;
 360
 361                 buf = (u32 *)host->sg_miter.addr;
 362
 363                 copy_words = len / 4;
 364
 365                 while (copy_words) {
 366                         int burst_words, words;
 367                         u32 edm;
 368
 369                         burst_words = min(SDDATA_FIFO_PIO_BURST, copy_words);
 370                         edm = readl(host->ioaddr + SDEDM);
 371                         if (is_read)
 372                                 words = ((edm >> 4) & 0x1f);
 373                         else
 374                                 words = SDDATA_FIFO_WORDS - ((edm >> 4) & 0x1f);
 375
 376                         if (words < burst_words) {
 377                                 int fsm_state = (edm & SDEDM_FSM_MASK);
 378                                 struct device *dev = &host->pdev->dev;
 379
 380                                 if ((is_read &&
 381                                      (fsm_state != SDEDM_FSM_READDATA &&
 382                                       fsm_state != SDEDM_FSM_READWAIT &&
 383                                       fsm_state != SDEDM_FSM_READCRC)) ||
 384                                     (!is_read &&
 385                                      (fsm_state != SDEDM_FSM_WRITEDATA &&
 386                                       fsm_state != SDEDM_FSM_WRITESTART1 &&
 387                                       fsm_state != SDEDM_FSM_WRITESTART2))) {
 388                                         hsts = readl(host->ioaddr + SDHSTS);
 389                                         dev_err(dev, "fsm %x, hsts %08x\n",
 390                                                 fsm_state, hsts);
 391                                         if (hsts & SDHSTS_ERROR_MASK)
 392                                                 break;
 393                                 }
 394
 395                                 if (time_after(jiffies, wait_max)) {
 396                                         dev_err(dev, "PIO %s timeout - EDM %08x\n",
 397                                                 is_read ? "read" : "write",
 398                                                 edm);
 399                                         hsts = SDHSTS_REW_TIME_OUT;
 400                                         break;
 401                                 }
 402                                 ndelay((burst_words - words) *
 403                                        host->ns_per_fifo_word);
 404                                 continue;
 405                         } else if (words > copy_words) {
 406                                 words = copy_words;
 407                         }
 408
 409                         copy_words -= words;
 410
 411                         while (words) {
 412                                 if (is_read)
 413                                         *(buf++) = readl(host->ioaddr + SDDATA);
 414                                 else
 415                                         writel(*(buf++), host->ioaddr + SDDATA);
 416                                 words--;
 417                         }
 418                 }
 419
 420                 if (hsts & SDHSTS_ERROR_MASK)
 421                         break;
 422         }
 423
 424         sg_miter_stop(&host->sg_miter);
 425
 426         local_irq_restore(flags);
 427 }
 428
 429 static void bcm2835_transfer_pio(struct bcm2835_host *host)
 430 {
 431         struct device *dev = &host->pdev->dev;
 432         u32 sdhsts;
 433         bool is_read;
 434
 435         is_read = (host->data->flags & MMC_DATA_READ) != 0;
 436         bcm2835_transfer_block_pio(host, is_read);
 437
 438         sdhsts = readl(host->ioaddr + SDHSTS);
 439         if (sdhsts & (SDHSTS_CRC16_ERROR |
 440                       SDHSTS_CRC7_ERROR |
 441                       SDHSTS_FIFO_ERROR)) {
 442                 dev_err(dev, "%s transfer error - HSTS %08x\n",
 443                         is_read ? "read" : "write", sdhsts);
 444                 host->data->error = -EILSEQ;
 445         } else if ((sdhsts & (SDHSTS_CMD_TIME_OUT |
 446                               SDHSTS_REW_TIME_OUT))) {
 447                 dev_err(dev, "%s timeout error - HSTS %08x\n",
 448                         is_read ? "read" : "write", sdhsts);
 449                 host->data->error = -ETIMEDOUT;
 450         }
 451 }
 452
 453 static
 454 void bcm2835_prepare_dma(struct bcm2835_host *host, struct mmc_data *data)
 455 {
 456         int sg_len, dir_data, dir_slave;
 457         struct dma_async_tx_descriptor *desc = NULL;
 458         struct dma_chan *dma_chan;
 459
 460         dma_chan = host->dma_chan_rxtx;
 461         if (data->flags & MMC_DATA_READ) {
 462                 dir_data = DMA_FROM_DEVICE;
 463                 dir_slave = DMA_DEV_TO_MEM;
 464         } else {
 465                 dir_data = DMA_TO_DEVICE;
 466                 dir_slave = DMA_MEM_TO_DEV;
 467         }
 468
 469         /* The block doesn't manage the FIFO DREQs properly for
 470          * multi-block transfers, so don't attempt to DMA the final
 471          * few words.  Unfortunately this requires the final sg entry
 472          * to be trimmed.  N.B. This code demands that the overspill
 473          * is contained in a single sg entry.
 474          */
 475
 476         host->drain_words = 0;
 477         if ((data->blocks > 1) && (dir_data == DMA_FROM_DEVICE)) {
 478                 struct scatterlist *sg;
 479                 u32 len;
 480                 int i;
 481
 482                 len = min((u32)(FIFO_READ_THRESHOLD - 1) * 4,
 483                           (u32)data->blocks * data->blksz);
 484
 485                 for_each_sg(data->sg, sg, data->sg_len, i) {
 486                         if (sg_is_last(sg)) {
 487                                 WARN_ON(sg->length < len);
 488                                 sg->length -= len;
 489                                 host->drain_page = sg_page(sg);
 490                                 host->drain_offset = sg->offset + sg->length;
 491                         }
 492                 }
 493                 host->drain_words = len / 4;
 494         }
 495
 496         /* The parameters have already been validated, so this will not fail */
 497         (void)dmaengine_slave_config(dma_chan,
 498                                      (dir_data == DMA_FROM_DEVICE) ?
 499                                      &host->dma_cfg_rx :
 500                                      &host->dma_cfg_tx);
 501
 502         sg_len = dma_map_sg(dma_chan->device->dev, data->sg, data->sg_len,
 503                             dir_data);
 504         if (!sg_len)
 505                 return;
 506
 507         desc = dmaengine_prep_slave_sg(dma_chan, data->sg, sg_len, dir_slave,
 508                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 509
 510         if (!desc) {
 511                 dma_unmap_sg(dma_chan->device->dev, data->sg, sg_len, dir_data);
 512                 return;
 513         }
 514
 515         desc->callback = bcm2835_dma_complete;
 516         desc->callback_param = host;
 517         host->dma_desc = desc;
 518         host->dma_chan = dma_chan;
 519         host->dma_dir = dir_data;
 520 }
 521
 522 static void bcm2835_start_dma(struct bcm2835_host *host)
 523 {
 524         dmaengine_submit(host->dma_desc);
 525         dma_async_issue_pending(host->dma_chan);
 526 }
 527
 528 static void bcm2835_set_transfer_irqs(struct bcm2835_host *host)
 529 {
 530         u32 all_irqs = SDHCFG_DATA_IRPT_EN | SDHCFG_BLOCK_IRPT_EN |
 531                 SDHCFG_BUSY_IRPT_EN;
 532
 533         if (host->dma_desc) {
 534                 host->hcfg = (host->hcfg & ~all_irqs) |
 535                         SDHCFG_BUSY_IRPT_EN;
 536         } else {
 537                 host->hcfg = (host->hcfg & ~all_irqs) |
 538                         SDHCFG_DATA_IRPT_EN |
 539                         SDHCFG_BUSY_IRPT_EN;
 540         }
 541
 542         writel(host->hcfg, host->ioaddr + SDHCFG);
 543 }
 544
 545 static
 546 void bcm2835_prepare_data(struct bcm2835_host *host, struct mmc_command *cmd)
 547 {
 548         struct mmc_data *data = cmd->data;
 549
 550         WARN_ON(host->data);
 551
 552         host->data = data;
 553         if (!data)
 554                 return;
 555
 556         host->data_complete = false;
 557         host->data->bytes_xfered = 0;
 558
 559         if (!host->dma_desc) {
 560                 /* Use PIO */
 561                 int flags = SG_MITER_ATOMIC;
 562
 563                 if (data->flags & MMC_DATA_READ)
 564                         flags |= SG_MITER_TO_SG;
 565                 else
 566                         flags |= SG_MITER_FROM_SG;
 567                 sg_miter_start(&host->sg_miter, data->sg, data->sg_len, flags);
 568                 host->blocks = data->blocks;
 569         }
 570
 571         bcm2835_set_transfer_irqs(host);
 572
 573         writel(data->blksz, host->ioaddr + SDHBCT);
 574         writel(data->blocks, host->ioaddr + SDHBLC);
 575 }
 576
 577 static u32 bcm2835_read_wait_sdcmd(struct bcm2835_host *host, u32 max_ms)
 578 {
 579         struct device *dev = &host->pdev->dev;
 580         u32 value;
 581         int ret;
 582
 583         ret = readl_poll_timeout(host->ioaddr + SDCMD, value,
 584                                  !(value & SDCMD_NEW_FLAG), 1, 10);
 585         if (ret == -ETIMEDOUT)
 586                 /* if it takes a while make poll interval bigger */
 587                 ret = readl_poll_timeout(host->ioaddr + SDCMD, value,
 588                                          !(value & SDCMD_NEW_FLAG),
 589                                          10, max_ms * 1000);
 590         if (ret == -ETIMEDOUT)
 591                 dev_err(dev, "%s: timeout (%d ms)\n", __func__, max_ms);
 592
 593         return value;
 594 }
 595
 596 static void bcm2835_finish_request(struct bcm2835_host *host)
 597 {
 598         struct dma_chan *terminate_chan = NULL;
 599         struct mmc_request *mrq;
 600
 601         cancel_delayed_work_sync(&host->timeout_work);
 602
 603         mrq = host->mrq;
 604
 605         host->mrq = NULL;
 606         host->cmd = NULL;
 607         host->data = NULL;
 608
 609         host->dma_desc = NULL;
 610         terminate_chan = host->dma_chan;
 611         host->dma_chan = NULL;
 612
 613         if (terminate_chan) {
 614                 int err = dmaengine_terminate_all(terminate_chan);
 615
 616                 if (err)
 617                         dev_err(&host->pdev->dev,
 618                                 "failed to terminate DMA (%d)\n", err);
 619         }
 620
 621         mmc_request_done(host->mmc, mrq);
 622 }
 623
 624 static
 625 bool bcm2835_send_command(struct bcm2835_host *host, struct mmc_command *cmd)
 626 {
 627         struct device *dev = &host->pdev->dev;
 628         u32 sdcmd, sdhsts;
 629         unsigned long timeout;
 630
 631         WARN_ON(host->cmd);
 632
 633         sdcmd = bcm2835_read_wait_sdcmd(host, 100);
 634         if (sdcmd & SDCMD_NEW_FLAG) {
 635                 dev_err(dev, "previous command never completed.\n");
 636                 bcm2835_dumpregs(host);
 637                 cmd->error = -EILSEQ;
 638                 bcm2835_finish_request(host);
 639                 return false;
 640         }
 641
 642         if (!cmd->data && cmd->busy_timeout > 9000)
 643                 timeout = DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ;
 644         else
 645                 timeout = 10 * HZ;
 646         schedule_delayed_work(&host->timeout_work, timeout);
 647
 648         host->cmd = cmd;
 649
 650         /* Clear any error flags */
 651         sdhsts = readl(host->ioaddr + SDHSTS);
 652         if (sdhsts & SDHSTS_ERROR_MASK)
 653                 writel(sdhsts, host->ioaddr + SDHSTS);
 654
 655         if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) {
 656                 dev_err(dev, "unsupported response type!\n");
 657                 cmd->error = -EINVAL;
 658                 bcm2835_finish_request(host);
 659                 return false;
 660         }
 661
 662         bcm2835_prepare_data(host, cmd);
 663
 664         writel(cmd->arg, host->ioaddr + SDARG);
 665
 666         sdcmd = cmd->opcode & SDCMD_CMD_MASK;
 667
 668         host->use_busy = false;
 669         if (!(cmd->flags & MMC_RSP_PRESENT)) {
 670                 sdcmd |= SDCMD_NO_RESPONSE;
 671         } else {
 672                 if (cmd->flags & MMC_RSP_136)
 673                         sdcmd |= SDCMD_LONG_RESPONSE;
 674                 if (cmd->flags & MMC_RSP_BUSY) {
 675                         sdcmd |= SDCMD_BUSYWAIT;
 676                         host->use_busy = true;
 677                 }
 678         }
 679
 680         if (cmd->data) {
 681                 if (cmd->data->flags & MMC_DATA_WRITE)
 682                         sdcmd |= SDCMD_WRITE_CMD;
 683                 if (cmd->data->flags & MMC_DATA_READ)
 684                         sdcmd |= SDCMD_READ_CMD;
 685         }
 686
 687         writel(sdcmd | SDCMD_NEW_FLAG, host->ioaddr + SDCMD);
 688
 689         return true;
 690 }
 691
 692 static void bcm2835_transfer_complete(struct bcm2835_host *host)
 693 {
 694         struct mmc_data *data;
 695
 696         WARN_ON(!host->data_complete);
 697
 698         data = host->data;
 699         host->data = NULL;
 700
 701         /* Need to send CMD12 if -
 702          * a) open-ended multiblock transfer (no CMD23)
 703          * b) error in multiblock transfer
 704          */
 705         if (host->mrq->stop && (data->error || !host->use_sbc)) {
 706                 if (bcm2835_send_command(host, host->mrq->stop)) {
 707                         /* No busy, so poll for completion */
 708                         if (!host->use_busy)
 709                                 bcm2835_finish_command(host);
 710                 }
 711         } else {
 712                 bcm2835_wait_transfer_complete(host);
 713                 bcm2835_finish_request(host);
 714         }
 715 }
 716
 717 static void bcm2835_finish_data(struct bcm2835_host *host)
 718 {
 719         struct device *dev = &host->pdev->dev;
 720         struct mmc_data *data;
 721
 722         data = host->data;
 723
 724         host->hcfg &= ~(SDHCFG_DATA_IRPT_EN | SDHCFG_BLOCK_IRPT_EN);
 725         writel(host->hcfg, host->ioaddr + SDHCFG);
 726
 727         data->bytes_xfered = data->error ? 0 : (data->blksz * data->blocks);
 728
 729         host->data_complete = true;
 730
 731         if (host->cmd) {
 732                 /* Data managed to finish before the
 733                  * command completed. Make sure we do
 734                  * things in the proper order.
 735                  */
 736                 dev_dbg(dev, "Finished early - HSTS %08x\n",
 737                         readl(host->ioaddr + SDHSTS));
 738         } else {
 739                 bcm2835_transfer_complete(host);
 740         }
 741 }
 742
 743 static void bcm2835_finish_command(struct bcm2835_host *host)
 744 {
 745         struct device *dev = &host->pdev->dev;
 746         struct mmc_command *cmd = host->cmd;
 747         u32 sdcmd;
 748
 749         sdcmd = bcm2835_read_wait_sdcmd(host, 100);
 750
 751         /* Check for errors */
 752         if (sdcmd & SDCMD_NEW_FLAG) {
 753                 dev_err(dev, "command never completed.\n");
 754                 bcm2835_dumpregs(host);
 755                 host->cmd->error = -EIO;
 756                 bcm2835_finish_request(host);
 757                 return;
 758         } else if (sdcmd & SDCMD_FAIL_FLAG) {
 759                 u32 sdhsts = readl(host->ioaddr + SDHSTS);
 760
 761                 /* Clear the errors */
 762                 writel(SDHSTS_ERROR_MASK, host->ioaddr + SDHSTS);
 763
 764                 if (!(sdhsts & SDHSTS_CRC7_ERROR) ||
 765                     (host->cmd->opcode != MMC_SEND_OP_COND)) {
 766                         u32 edm, fsm;
 767
 768                         if (sdhsts & SDHSTS_CMD_TIME_OUT) {
 769                                 host->cmd->error = -ETIMEDOUT;
 770                         } else {
 771                                 dev_err(dev, "unexpected command %d error\n",
 772                                         host->cmd->opcode);
 773                                 bcm2835_dumpregs(host);
 774                                 host->cmd->error = -EILSEQ;
 775                         }
 776                         edm = readl(host->ioaddr + SDEDM);
 777                         fsm = edm & SDEDM_FSM_MASK;
 778                         if (fsm == SDEDM_FSM_READWAIT ||
 779                             fsm == SDEDM_FSM_WRITESTART1)
 780                                 /* Kick the FSM out of its wait */
 781                                 writel(edm | SDEDM_FORCE_DATA_MODE,
 782                                        host->ioaddr + SDEDM);
 783                         bcm2835_finish_request(host);
 784                         return;
 785                 }
 786         }
 787
 788         if (cmd->flags & MMC_RSP_PRESENT) {
 789                 if (cmd->flags & MMC_RSP_136) {
 790                         int i;
 791
 792                         for (i = 0; i < 4; i++) {
 793                                 cmd->resp[3 - i] =
 794                                         readl(host->ioaddr + SDRSP0 + i * 4);
 795                         }
 796                 } else {
 797                         cmd->resp[0] = readl(host->ioaddr + SDRSP0);
 798                 }
 799         }
 800
 801         if (cmd == host->mrq->sbc) {
 802                 /* Finished CMD23, now send actual command. */
 803                 host->cmd = NULL;
 804                 if (bcm2835_send_command(host, host->mrq->cmd)) {
 805                         if (host->data && host->dma_desc)
 806                                 /* DMA transfer starts now, PIO starts
 807                                  * after irq
 808                                  */
 809                                 bcm2835_start_dma(host);
 810
 811                         if (!host->use_busy)
 812                                 bcm2835_finish_command(host);
 813                 }
 814         } else if (cmd == host->mrq->stop) {
 815                 /* Finished CMD12 */
 816                 bcm2835_finish_request(host);
 817         } else {
 818                 /* Processed actual command. */
 819                 host->cmd = NULL;
 820                 if (!host->data)
 821                         bcm2835_finish_request(host);
 822                 else if (host->data_complete)
 823                         bcm2835_transfer_complete(host);
 824         }
 825 }
 826
 827 static void bcm2835_timeout(struct work_struct *work)
 828 {
 829         struct delayed_work *d = to_delayed_work(work);
 830         struct bcm2835_host *host =
 831                 container_of(d, struct bcm2835_host, timeout_work);
 832         struct device *dev = &host->pdev->dev;
 833
 834         mutex_lock(&host->mutex);
 835
 836         if (host->mrq) {
 837                 dev_err(dev, "timeout waiting for hardware interrupt.\n");
 838                 bcm2835_dumpregs(host);
 839
 840                 bcm2835_reset(host->mmc);
 841
 842                 if (host->data) {
 843                         host->data->error = -ETIMEDOUT;
 844                         bcm2835_finish_data(host);
 845                 } else {
 846                         if (host->cmd)
 847                                 host->cmd->error = -ETIMEDOUT;
 848                         else
 849                                 host->mrq->cmd->error = -ETIMEDOUT;
 850
 851                         bcm2835_finish_request(host);
 852                 }
 853         }
 854
 855         mutex_unlock(&host->mutex);
 856 }
 857
 858 static bool bcm2835_check_cmd_error(struct bcm2835_host *host, u32 intmask)
 859 {
 860         struct device *dev = &host->pdev->dev;
 861
 862         if (!(intmask & SDHSTS_ERROR_MASK))
 863                 return false;
 864
 865         if (!host->cmd)
 866                 return true;
 867
 868         dev_err(dev, "sdhost_busy_irq: intmask %08x\n", intmask);
 869         if (intmask & SDHSTS_CRC7_ERROR) {
 870                 host->cmd->error = -EILSEQ;
 871         } else if (intmask & (SDHSTS_CRC16_ERROR |
 872                               SDHSTS_FIFO_ERROR)) {
 873                 if (host->mrq->data)
 874                         host->mrq->data->error = -EILSEQ;
 875                 else
 876                         host->cmd->error = -EILSEQ;
 877         } else if (intmask & SDHSTS_REW_TIME_OUT) {
 878                 if (host->mrq->data)
 879                         host->mrq->data->error = -ETIMEDOUT;
 880                 else
 881                         host->cmd->error = -ETIMEDOUT;
 882         } else if (intmask & SDHSTS_CMD_TIME_OUT) {
 883                 host->cmd->error = -ETIMEDOUT;
 884         }
 885         bcm2835_dumpregs(host);
 886         return true;
 887 }
 888
 889 static void bcm2835_check_data_error(struct bcm2835_host *host, u32 intmask)
 890 {
 891         if (!host->data)
 892                 return;
 893         if (intmask & (SDHSTS_CRC16_ERROR | SDHSTS_FIFO_ERROR))
 894                 host->data->error = -EILSEQ;
 895         if (intmask & SDHSTS_REW_TIME_OUT)
 896                 host->data->error = -ETIMEDOUT;
 897 }
 898
 899 static void bcm2835_busy_irq(struct bcm2835_host *host)
 900 {
 901         if (WARN_ON(!host->cmd)) {
 902                 bcm2835_dumpregs(host);
 903                 return;
 904         }
 905
 906         if (WARN_ON(!host->use_busy)) {
 907                 bcm2835_dumpregs(host);
 908                 return;
 909         }
 910         host->use_busy = false;
 911
 912         bcm2835_finish_command(host);
 913 }
 914
 915 static void bcm2835_data_irq(struct bcm2835_host *host, u32 intmask)
 916 {
 917         /* There are no dedicated data/space available interrupt
 918          * status bits, so it is necessary to use the single shared
 919          * data/space available FIFO status bits. It is therefore not
 920          * an error to get here when there is no data transfer in
 921          * progress.
 922          */
 923         if (!host->data)
 924                 return;
 925
 926         bcm2835_check_data_error(host, intmask);
 927         if (host->data->error)
 928                 goto finished;
 929
 930         if (host->data->flags & MMC_DATA_WRITE) {
 931                 /* Use the block interrupt for writes after the first block */
 932                 host->hcfg &= ~(SDHCFG_DATA_IRPT_EN);
 933                 host->hcfg |= SDHCFG_BLOCK_IRPT_EN;
 934                 writel(host->hcfg, host->ioaddr + SDHCFG);
 935                 bcm2835_transfer_pio(host);
 936         } else {
 937                 bcm2835_transfer_pio(host);
 938                 host->blocks--;
 939                 if ((host->blocks == 0) || host->data->error)
 940                         goto finished;
 941         }
 942         return;
 943
 944 finished:
 945         host->hcfg &= ~(SDHCFG_DATA_IRPT_EN | SDHCFG_BLOCK_IRPT_EN);
 946         writel(host->hcfg, host->ioaddr + SDHCFG);
 947 }
 948
 949 static void bcm2835_data_threaded_irq(struct bcm2835_host *host)
 950 {
 951         if (!host->data)
 952                 return;
 953         if ((host->blocks == 0) || host->data->error)
 954                 bcm2835_finish_data(host);
 955 }
 956
 957 static void bcm2835_block_irq(struct bcm2835_host *host)
 958 {
 959         if (WARN_ON(!host->data)) {
 960                 bcm2835_dumpregs(host);
 961                 return;
 962         }
 963
 964         if (!host->dma_desc) {
 965                 WARN_ON(!host->blocks);
 966                 if (host->data->error || (--host->blocks == 0))
 967                         bcm2835_finish_data(host);
 968                 else
 969                         bcm2835_transfer_pio(host);
 970         } else if (host->data->flags & MMC_DATA_WRITE) {
 971                 bcm2835_finish_data(host);
 972         }
 973 }
 974
 975 static irqreturn_t bcm2835_irq(int irq, void *dev_id)
 976 {
 977         irqreturn_t result = IRQ_NONE;
 978         struct bcm2835_host *host = dev_id;
 979         u32 intmask;
 980
 981         spin_lock(&host->lock);
 982
 983         intmask = readl(host->ioaddr + SDHSTS);
 984
 985         writel(SDHSTS_BUSY_IRPT |
 986                SDHSTS_BLOCK_IRPT |
 987                SDHSTS_SDIO_IRPT |
 988                SDHSTS_DATA_FLAG,
 989                host->ioaddr + SDHSTS);
 990
 991         if (intmask & SDHSTS_BLOCK_IRPT) {
 992                 bcm2835_check_data_error(host, intmask);
 993                 host->irq_block = true;
 994                 result = IRQ_WAKE_THREAD;
 995         }
 996
 997         if (intmask & SDHSTS_BUSY_IRPT) {
 998                 if (!bcm2835_check_cmd_error(host, intmask)) {
 999                         host->irq_busy = true;
1000                         result = IRQ_WAKE_THREAD;
1001                 } else {
1002                         result = IRQ_HANDLED;
1003                 }
1004         }
1005
1006         /* There is no true data interrupt status bit, so it is
1007          * necessary to qualify the data flag with the interrupt
1008          * enable bit.
1009          */
1010         if ((intmask & SDHSTS_DATA_FLAG) &&
1011             (host->hcfg & SDHCFG_DATA_IRPT_EN)) {
1012                 bcm2835_data_irq(host, intmask);
1013                 host->irq_data = true;
1014                 result = IRQ_WAKE_THREAD;
1015         }
1016
1017         spin_unlock(&host->lock);
1018
1019         return result;
1020 }
1021
1022 static irqreturn_t bcm2835_threaded_irq(int irq, void *dev_id)
1023 {
1024         struct bcm2835_host *host = dev_id;
1025         unsigned long flags;
1026         bool block, busy, data;
1027
1028         spin_lock_irqsave(&host->lock, flags);
1029
1030         block = host->irq_block;
1031         busy  = host->irq_busy;
1032         data  = host->irq_data;
1033         host->irq_block = false;
1034         host->irq_busy  = false;
1035         host->irq_data  = false;
1036
1037         spin_unlock_irqrestore(&host->lock, flags);
1038
1039         mutex_lock(&host->mutex);
1040
1041         if (block)
1042                 bcm2835_block_irq(host);
1043         if (busy)
1044                 bcm2835_busy_irq(host);
1045         if (data)
1046                 bcm2835_data_threaded_irq(host);
1047
1048         mutex_unlock(&host->mutex);
1049
1050         return IRQ_HANDLED;
1051 }
1052
1053 static void bcm2835_dma_complete_work(struct work_struct *work)
1054 {
1055         struct bcm2835_host *host =
1056                 container_of(work, struct bcm2835_host, dma_work);
1057         struct mmc_data *data;
1058
1059         mutex_lock(&host->mutex);
1060
1061         data = host->data;
1062
1063         if (host->dma_chan) {
1064                 dma_unmap_sg(host->dma_chan->device->dev,
1065                              data->sg, data->sg_len,
1066                              host->dma_dir);
1067
1068                 host->dma_chan = NULL;
1069         }
1070
1071         if (host->drain_words) {
1072                 unsigned long flags;
1073                 void *page;
1074                 u32 *buf;
1075
1076                 if (host->drain_offset & PAGE_MASK) {
1077                         host->drain_page += host->drain_offset >> PAGE_SHIFT;
1078                         host->drain_offset &= ~PAGE_MASK;
1079                 }
1080                 local_irq_save(flags);
1081                 page = kmap_atomic(host->drain_page);
1082                 buf = page + host->drain_offset;
1083
1084                 while (host->drain_words) {
1085                         u32 edm = readl(host->ioaddr + SDEDM);
1086
1087                         if ((edm >> 4) & 0x1f)
1088                                 *(buf++) = readl(host->ioaddr + SDDATA);
1089                         host->drain_words--;
1090                 }
1091
1092                 kunmap_atomic(page);
1093                 local_irq_restore(flags);
1094         }
1095
1096         bcm2835_finish_data(host);
1097
1098         mutex_unlock(&host->mutex);
1099 }
1100
1101 static void bcm2835_set_clock(struct bcm2835_host *host, unsigned int clock)
1102 {
1103         int div;
1104
1105         /* The SDCDIV register has 11 bits, and holds (div - 2).  But
1106          * in data mode the max is 50MHz wihout a minimum, and only
1107          * the bottom 3 bits are used. Since the switch over is
1108          * automatic (unless we have marked the card as slow...),
1109          * chosen values have to make sense in both modes.  Ident mode
1110          * must be 100-400KHz, so can range check the requested
1111          * clock. CMD15 must be used to return to data mode, so this
1112          * can be monitored.
1113          *
1114          * clock 250MHz -> 0->125MHz, 1->83.3MHz, 2->62.5MHz, 3->50.0MHz
1115          *                 4->41.7MHz, 5->35.7MHz, 6->31.3MHz, 7->27.8MHz
1116          *
1117          *               623->400KHz/27.8MHz
1118          *               reset value (507)->491159/50MHz
1119          *
1120          * BUT, the 3-bit clock divisor in data mode is too small if
1121          * the core clock is higher than 250MHz, so instead use the
1122          * SLOW_CARD configuration bit to force the use of the ident
1123          * clock divisor at all times.
1124          */
1125
1126         if (clock < 100000) {
1127                 /* Can't stop the clock, but make it as slow as possible
1128                  * to show willing
1129                  */
1130                 host->cdiv = SDCDIV_MAX_CDIV;
1131                 writel(host->cdiv, host->ioaddr + SDCDIV);
1132                 return;
1133         }
1134
1135         div = host->max_clk / clock;
1136         if (div < 2)
1137                 div = 2;
1138         if ((host->max_clk / div) > clock)
1139                 div++;
1140         div -= 2;
1141
1142         if (div > SDCDIV_MAX_CDIV)
1143                 div = SDCDIV_MAX_CDIV;
1144
1145         clock = host->max_clk / (div + 2);
1146         host->mmc->actual_clock = clock;
1147
1148         /* Calibrate some delays */
1149
1150         host->ns_per_fifo_word = (1000000000 / clock) *
1151                 ((host->mmc->caps & MMC_CAP_4_BIT_DATA) ? 8 : 32);
1152
1153         host->cdiv = div;
1154         writel(host->cdiv, host->ioaddr + SDCDIV);
1155
1156         /* Set the timeout to 500ms */
1157         writel(host->mmc->actual_clock / 2, host->ioaddr + SDTOUT);
1158 }
1159
1160 static void bcm2835_request(struct mmc_host *mmc, struct mmc_request *mrq)
1161 {
1162         struct bcm2835_host *host = mmc_priv(mmc);
1163         struct device *dev = &host->pdev->dev;
1164         u32 edm, fsm;
1165
1166         /* Reset the error statuses in case this is a retry */
1167         if (mrq->sbc)
1168                 mrq->sbc->error = 0;
1169         if (mrq->cmd)
1170                 mrq->cmd->error = 0;
1171         if (mrq->data)
1172                 mrq->data->error = 0;
1173         if (mrq->stop)
1174                 mrq->stop->error = 0;
1175
1176         if (mrq->data && !is_power_of_2(mrq->data->blksz)) {
1177                 dev_err(dev, "unsupported block size (%d bytes)\n",
1178                         mrq->data->blksz);
1179
1180                 if (mrq->cmd)
1181                         mrq->cmd->error = -EINVAL;
1182
1183                 mmc_request_done(mmc, mrq);
1184                 return;
1185         }
1186
1187         mutex_lock(&host->mutex);
1188
1189         WARN_ON(host->mrq);
1190         host->mrq = mrq;
1191
1192         edm = readl(host->ioaddr + SDEDM);
1193         fsm = edm & SDEDM_FSM_MASK;
1194
1195         if ((fsm != SDEDM_FSM_IDENTMODE) &&
1196             (fsm != SDEDM_FSM_DATAMODE)) {
1197                 dev_err(dev, "previous command (%d) not complete (EDM %08x)\n",
1198                         readl(host->ioaddr + SDCMD) & SDCMD_CMD_MASK,
1199                         edm);
1200                 bcm2835_dumpregs(host);
1201
1202                 if (mrq->cmd)
1203                         mrq->cmd->error = -EILSEQ;
1204
1205                 bcm2835_finish_request(host);
1206                 mutex_unlock(&host->mutex);
1207                 return;
1208         }
1209
1210         if (host->use_dma && mrq->data && (mrq->data->blocks > PIO_THRESHOLD))
1211                 bcm2835_prepare_dma(host, mrq->data);
1212
1213         host->use_sbc = !!mrq->sbc && host->mrq->data &&
1214                         (host->mrq->data->flags & MMC_DATA_READ);
1215         if (host->use_sbc) {
1216                 if (bcm2835_send_command(host, mrq->sbc)) {
1217                         if (!host->use_busy)
1218                                 bcm2835_finish_command(host);
1219                 }
1220         } else if (mrq->cmd && bcm2835_send_command(host, mrq->cmd)) {
1221                 if (host->data && host->dma_desc) {
1222                         /* DMA transfer starts now, PIO starts after irq */
1223                         bcm2835_start_dma(host);
1224                 }
1225
1226                 if (!host->use_busy)
1227                         bcm2835_finish_command(host);
1228         }
1229
1230         mutex_unlock(&host->mutex);
1231 }
1232
1233 static void bcm2835_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1234 {
1235         struct bcm2835_host *host = mmc_priv(mmc);
1236
1237         mutex_lock(&host->mutex);
1238
1239         if (!ios->clock || ios->clock != host->clock) {
1240                 bcm2835_set_clock(host, ios->clock);
1241                 host->clock = ios->clock;
1242         }
1243
1244         /* set bus width */
1245         host->hcfg &= ~SDHCFG_WIDE_EXT_BUS;
1246         if (ios->bus_width == MMC_BUS_WIDTH_4)
1247                 host->hcfg |= SDHCFG_WIDE_EXT_BUS;
1248
1249         host->hcfg |= SDHCFG_WIDE_INT_BUS;
1250
1251         /* Disable clever clock switching, to cope with fast core clocks */
1252         host->hcfg |= SDHCFG_SLOW_CARD;
1253
1254         writel(host->hcfg, host->ioaddr + SDHCFG);
1255
1256         mutex_unlock(&host->mutex);
1257 }
1258
1259 static const struct mmc_host_ops bcm2835_ops = {
1260         .request = bcm2835_request,
1261         .set_ios = bcm2835_set_ios,
1262         .hw_reset = bcm2835_reset,
1263 };
1264
1265 static int bcm2835_add_host(struct bcm2835_host *host)
1266 {
1267         struct mmc_host *mmc = host->mmc;
1268         struct device *dev = &host->pdev->dev;
1269         char pio_limit_string[20];
1270         int ret;
1271
1272         if (!mmc->f_max || mmc->f_max > host->max_clk)
1273                 mmc->f_max = host->max_clk;
1274         mmc->f_min = host->max_clk / SDCDIV_MAX_CDIV;
1275
1276         mmc->max_busy_timeout = ~0 / (mmc->f_max / 1000);
1277
1278         dev_dbg(dev, "f_max %d, f_min %d, max_busy_timeout %d\n",
1279                 mmc->f_max, mmc->f_min, mmc->max_busy_timeout);
1280
1281         /* host controller capabilities */
1282         mmc->caps |= MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED |
1283                      MMC_CAP_NEEDS_POLL | MMC_CAP_HW_RESET | MMC_CAP_ERASE |
1284                      MMC_CAP_CMD23;
1285
1286         spin_lock_init(&host->lock);
1287         mutex_init(&host->mutex);
1288
1289         if (IS_ERR_OR_NULL(host->dma_chan_rxtx)) {
1290                 dev_warn(dev, "unable to initialise DMA channel. Falling back to PIO\n");
1291                 host->use_dma = false;
1292         } else {
1293                 host->use_dma = true;
1294
1295                 host->dma_cfg_tx.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1296                 host->dma_cfg_tx.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1297                 host->dma_cfg_tx.slave_id = 13;         /* DREQ channel */
1298                 host->dma_cfg_tx.direction = DMA_MEM_TO_DEV;
1299                 host->dma_cfg_tx.src_addr = 0;
1300                 host->dma_cfg_tx.dst_addr = host->phys_addr + SDDATA;
1301
1302                 host->dma_cfg_rx.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1303                 host->dma_cfg_rx.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
1304                 host->dma_cfg_rx.slave_id = 13;         /* DREQ channel */
1305                 host->dma_cfg_rx.direction = DMA_DEV_TO_MEM;
1306                 host->dma_cfg_rx.src_addr = host->phys_addr + SDDATA;
1307                 host->dma_cfg_rx.dst_addr = 0;
1308
1309                 if (dmaengine_slave_config(host->dma_chan_rxtx,
1310                                            &host->dma_cfg_tx) != 0 ||
1311                     dmaengine_slave_config(host->dma_chan_rxtx,
1312                                            &host->dma_cfg_rx) != 0)
1313                         host->use_dma = false;
1314         }
1315
1316         mmc->max_segs = 128;
1317         mmc->max_req_size = 524288;
1318         mmc->max_seg_size = mmc->max_req_size;
1319         mmc->max_blk_size = 1024;
1320         mmc->max_blk_count =  65535;
1321
1322         /* report supported voltage ranges */
1323         mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34;
1324
1325         INIT_WORK(&host->dma_work, bcm2835_dma_complete_work);
1326         INIT_DELAYED_WORK(&host->timeout_work, bcm2835_timeout);
1327
1328         /* Set interrupt enables */
1329         host->hcfg = SDHCFG_BUSY_IRPT_EN;
1330
1331         bcm2835_reset_internal(host);
1332
1333         ret = request_threaded_irq(host->irq, bcm2835_irq,
1334                                    bcm2835_threaded_irq,
1335                                    0, mmc_hostname(mmc), host);
1336         if (ret) {
1337                 dev_err(dev, "failed to request IRQ %d: %d\n", host->irq, ret);
1338                 return ret;
1339         }
1340
1341         ret = mmc_add_host(mmc);
1342         if (ret) {
1343                 free_irq(host->irq, host);
1344                 return ret;
1345         }
1346
1347         pio_limit_string[0] = '\0';
1348         if (host->use_dma && (PIO_THRESHOLD > 0))
1349                 sprintf(pio_limit_string, " (>%d)", PIO_THRESHOLD);
1350         dev_info(dev, "loaded - DMA %s%s\n",
1351                  host->use_dma ? "enabled" : "disabled", pio_limit_string);
1352
1353         return 0;
1354 }
1355
1356 static int bcm2835_probe(struct platform_device *pdev)
1357 {
1358         struct device *dev = &pdev->dev;
1359         struct clk *clk;
1360         struct resource *iomem;
1361         struct bcm2835_host *host;
1362         struct mmc_host *mmc;
1363         const __be32 *regaddr_p;
1364         int ret;
1365
1366         dev_dbg(dev, "%s\n", __func__);
1367         mmc = mmc_alloc_host(sizeof(*host), dev);
1368         if (!mmc)
1369                 return -ENOMEM;
1370
1371         mmc->ops = &bcm2835_ops;
1372         host = mmc_priv(mmc);
1373         host->mmc = mmc;
1374         host->pdev = pdev;
1375         spin_lock_init(&host->lock);
1376
1377         iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1378         host->ioaddr = devm_ioremap_resource(dev, iomem);
1379         if (IS_ERR(host->ioaddr)) {
1380                 ret = PTR_ERR(host->ioaddr);
1381                 goto err;
1382         }
1383
1384         /* Parse OF address directly to get the physical address for
1385          * DMA to our registers.
1386          */
1387         regaddr_p = of_get_address(pdev->dev.of_node, 0, NULL, NULL);
1388         if (!regaddr_p) {
1389                 dev_err(dev, "Can't get phys address\n");
1390                 ret = -EINVAL;
1391                 goto err;
1392         }
1393
1394         host->phys_addr = be32_to_cpup(regaddr_p);
1395
1396         host->dma_chan = NULL;
1397         host->dma_desc = NULL;
1398
1399         host->dma_chan_rxtx = dma_request_slave_channel(dev, "rx-tx");
1400
1401         clk = devm_clk_get(dev, NULL);
1402         if (IS_ERR(clk)) {
1403                 ret = PTR_ERR(clk);
1404                 if (ret != -EPROBE_DEFER)
1405                         dev_err(dev, "could not get clk: %d\n", ret);
1406                 goto err;
1407         }
1408
1409         host->max_clk = clk_get_rate(clk);
1410
1411         host->irq = platform_get_irq(pdev, 0);
1412         if (host->irq <= 0) {
1413                 dev_err(dev, "get IRQ failed\n");
1414                 ret = -EINVAL;
1415                 goto err;
1416         }
1417
1418         ret = mmc_of_parse(mmc);
1419         if (ret)
1420                 goto err;
1421
1422         ret = bcm2835_add_host(host);
1423         if (ret)
1424                 goto err;
1425
1426         platform_set_drvdata(pdev, host);
1427
1428         dev_dbg(dev, "%s -> OK\n", __func__);
1429
1430         return 0;
1431
1432 err:
1433         dev_dbg(dev, "%s -> err %d\n", __func__, ret);
1434         if (host->dma_chan_rxtx)
1435                 dma_release_channel(host->dma_chan_rxtx);
1436         mmc_free_host(mmc);
1437
1438         return ret;
1439 }
1440
1441 static int bcm2835_remove(struct platform_device *pdev)
1442 {
1443         struct bcm2835_host *host = platform_get_drvdata(pdev);
1444
1445         mmc_remove_host(host->mmc);
1446
1447         writel(SDVDD_POWER_OFF, host->ioaddr + SDVDD);
1448
1449         free_irq(host->irq, host);
1450
1451         cancel_work_sync(&host->dma_work);
1452         cancel_delayed_work_sync(&host->timeout_work);
1453
1454         if (host->dma_chan_rxtx)
1455                 dma_release_channel(host->dma_chan_rxtx);
1456
1457         mmc_free_host(host->mmc);
1458         platform_set_drvdata(pdev, NULL);
1459
1460         return 0;
1461 }
1462
1463 static const struct of_device_id bcm2835_match[] = {
1464         { .compatible = "brcm,bcm2835-sdhost" },
1465         { }
1466 };
1467 MODULE_DEVICE_TABLE(of, bcm2835_match);
1468
1469 static struct platform_driver bcm2835_driver = {
1470         .probe      = bcm2835_probe,
1471         .remove     = bcm2835_remove,
1472         .driver     = {
1473                 .name           = "sdhost-bcm2835",
1474                 .of_match_table = bcm2835_match,
1475         },
1476 };
1477 module_platform_driver(bcm2835_driver);
1478
1479 MODULE_ALIAS("platform:sdhost-bcm2835");
1480 MODULE_DESCRIPTION("BCM2835 SDHost driver");
1481 MODULE_LICENSE("GPL v2");
1482 MODULE_AUTHOR("Phil Elwell");